From nobody Mon Nov 25 10:58:36 2024 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-1.web.codeaurora.org [10.30.226.201]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 7001D2022F0; Mon, 28 Oct 2024 21:48:25 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=10.30.226.201 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1730152105; cv=none; b=TbOZvp7yVZUbylOhhaKVFv2h5TOfjU469mPBrww2xLvWsVKHk+oQLsUGP/DJCKzDDNnouyl0iVfIdLlSQ4qOnLQA/kK7bVlpxS6oBc540jdGBqwzhA+cwz3tVctPnPZpxxlfb0cN8LDAfRMQXJ6GkB56Fiv8TYCn9hL+B5RiRPg= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1730152105; c=relaxed/simple; bh=BicEo2O0Udl64XLuU3OuWtNR3+Ob8ZlZ0QEDE8lLAZE=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=hRt1ApzeibNmc1mUQMeo3iGeN5cUWFAvYD/Z2gc77NxSCogvzwH5bNL9QYyAPW+Ssnh37oMnfBJExogGehfwfs/vrdJUH8oR026qL4imN1veF4l/6zS5In1ak9wd+0ocbJUWMkN89IIAQ6mH4fDbod5OMWYPgI8z+jryoWpdSZk= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=GpKgQcny; arc=none smtp.client-ip=10.30.226.201 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="GpKgQcny" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 26EFAC4CEE9; Mon, 28 Oct 2024 21:48:24 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1730152105; bh=BicEo2O0Udl64XLuU3OuWtNR3+Ob8ZlZ0QEDE8lLAZE=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=GpKgQcnyUPTPNvBhvfPV3yf0r/NW8TngUvZBraTuvGpEWeoPgOnJGm+OYXP3YLq83 Ui6KbmlyIi4jKHVWc+/Fnyr+yZQ5nVDlh5hHpQdTII61dp5KOHxaZNd6rvz1EbxBis ZebOnGZImA6T9uCLf+B5GumktZ8B2UXB3Fb1yetRlvpoff8fOKHw8WBivmG75HWzSY HvIKyYA/JACFdVhb6ThHn8gXYutF0UtKq4c751ZXsOTwIfWc4VIlH2gzTR01s7Nlvn 8rq9tfzkhnIclt2JWNj1bi6UC1IzmPAJUvKH+YLBaupmmoMgNT9xhUFnjrD4/oZmNE nlnBZ4gNHRfNw== From: Josh Poimboeuf To: x86@kernel.org Cc: Peter Zijlstra , Steven Rostedt , Ingo Molnar , Arnaldo Carvalho de Melo , linux-kernel@vger.kernel.org, Indu Bhagat , Mark Rutland , Alexander Shishkin , Jiri Olsa , Namhyung Kim , Ian Rogers , Adrian Hunter , linux-perf-users@vger.kernel.org, Mark Brown , linux-toolchains@vger.kernel.org, Jordan Rome , Sam James , linux-trace-kernel@vger.kerne.org, Andrii Nakryiko , Jens Remus , Mathieu Desnoyers , Florian Weimer , Andy Lutomirski Subject: [PATCH v3 15/19] perf: Add deferred user callchains Date: Mon, 28 Oct 2024 14:47:42 -0700 Message-ID: <1ce857387c781afa66efaa61eb88ff596b352500.1730150953.git.jpoimboe@kernel.org> X-Mailer: git-send-email 2.47.0 In-Reply-To: References: Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Instead of attempting to unwind user space from the NMI handler, defer it to run in task context by sending a self-IPI and then scheduling the unwind to run in the IRQ's exit task work before returning to user space. This allows the user stack page to be paged in if needed, avoids duplicate unwinds for kernel-bound workloads, and prepares for SFrame unwinding (so .sframe sections can be paged in on demand). Suggested-by: Steven Rostedt Suggested-by: Peter Zijlstra Signed-off-by: Josh Poimboeuf --- arch/Kconfig | 3 ++ include/linux/perf_event.h | 10 ++++- include/uapi/linux/perf_event.h | 22 +++++++++- kernel/bpf/stackmap.c | 6 +-- kernel/events/callchain.c | 11 ++++- kernel/events/core.c | 63 ++++++++++++++++++++++++++- tools/include/uapi/linux/perf_event.h | 22 +++++++++- 7 files changed, 129 insertions(+), 8 deletions(-) diff --git a/arch/Kconfig b/arch/Kconfig index e769c39dd221..33449485eafd 100644 --- a/arch/Kconfig +++ b/arch/Kconfig @@ -446,6 +446,9 @@ config HAVE_UNWIND_USER_SFRAME bool select UNWIND_USER =20 +config HAVE_PERF_CALLCHAIN_DEFERRED + bool + config HAVE_PERF_REGS bool help diff --git a/include/linux/perf_event.h b/include/linux/perf_event.h index 788f6971d32d..2193b3d16820 100644 --- a/include/linux/perf_event.h +++ b/include/linux/perf_event.h @@ -808,9 +808,11 @@ struct perf_event { unsigned long pending_addr; /* SIGTRAP */ struct irq_work pending_irq; struct irq_work pending_disable_irq; + struct irq_work pending_unwind_irq; struct callback_head pending_task; unsigned int pending_work; struct rcuwait pending_work_wait; + unsigned int pending_unwind; =20 atomic_t event_limit; =20 @@ -1569,12 +1571,18 @@ extern void perf_callchain_user(struct perf_callcha= in_entry_ctx *entry, struct p extern void perf_callchain_kernel(struct perf_callchain_entry_ctx *entry, = struct pt_regs *regs); extern struct perf_callchain_entry * get_perf_callchain(struct pt_regs *regs, bool kernel, bool user, - u32 max_stack, bool add_mark); + u32 max_stack, bool add_mark, bool defer_user); extern int get_callchain_buffers(int max_stack); extern void put_callchain_buffers(void); extern struct perf_callchain_entry *get_callchain_entry(int *rctx); extern void put_callchain_entry(int rctx); =20 +#ifdef CONFIG_HAVE_PERF_CALLCHAIN_DEFERRED +extern void perf_callchain_user_deferred(struct perf_callchain_entry_ctx *= entry, struct pt_regs *regs); +#else +static inline void perf_callchain_user_deferred(struct perf_callchain_entr= y_ctx *entry, struct pt_regs *regs) {} +#endif + extern int sysctl_perf_event_max_stack; extern int sysctl_perf_event_max_contexts_per_stack; =20 diff --git a/include/uapi/linux/perf_event.h b/include/uapi/linux/perf_even= t.h index 4842c36fdf80..6d0524b7d082 100644 --- a/include/uapi/linux/perf_event.h +++ b/include/uapi/linux/perf_event.h @@ -460,7 +460,8 @@ struct perf_event_attr { inherit_thread : 1, /* children only inherit if cloned with CLONE_THR= EAD */ remove_on_exec : 1, /* event is removed from task on exec */ sigtrap : 1, /* send synchronous SIGTRAP on event */ - __reserved_1 : 26; + defer_callchain: 1, /* generate PERF_RECORD_CALLCHAIN_DEFERRED record= s */ + __reserved_1 : 25; =20 union { __u32 wakeup_events; /* wakeup every n events */ @@ -1217,6 +1218,24 @@ enum perf_event_type { */ PERF_RECORD_AUX_OUTPUT_HW_ID =3D 21, =20 + /* + * This user callchain capture was deferred until shortly before + * returning to user space. Previous samples would have kernel + * callchains only and they need to be stitched with this to make full + * callchains. + * + * TODO: do PERF_SAMPLE_{REGS,STACK}_USER also need deferral? + * + * struct { + * struct perf_event_header header; + * u64 ctx_cookie; + * u64 nr; + * u64 ips[nr]; + * struct sample_id sample_id; + * }; + */ + PERF_RECORD_CALLCHAIN_DEFERRED =3D 22, + PERF_RECORD_MAX, /* non-ABI */ }; =20 @@ -1247,6 +1266,7 @@ enum perf_callchain_context { PERF_CONTEXT_HV =3D (__u64)-32, PERF_CONTEXT_KERNEL =3D (__u64)-128, PERF_CONTEXT_USER =3D (__u64)-512, + PERF_CONTEXT_USER_DEFERRED =3D (__u64)-640, =20 PERF_CONTEXT_GUEST =3D (__u64)-2048, PERF_CONTEXT_GUEST_KERNEL =3D (__u64)-2176, diff --git a/kernel/bpf/stackmap.c b/kernel/bpf/stackmap.c index ee9701337912..f073ebaf9c30 100644 --- a/kernel/bpf/stackmap.c +++ b/kernel/bpf/stackmap.c @@ -314,8 +314,7 @@ BPF_CALL_3(bpf_get_stackid, struct pt_regs *, regs, str= uct bpf_map *, map, if (max_depth > sysctl_perf_event_max_stack) max_depth =3D sysctl_perf_event_max_stack; =20 - trace =3D get_perf_callchain(regs, kernel, user, max_depth, false); - + trace =3D get_perf_callchain(regs, kernel, user, max_depth, false, false); if (unlikely(!trace)) /* couldn't fetch the stack trace */ return -EFAULT; @@ -448,7 +447,8 @@ static long __bpf_get_stack(struct pt_regs *regs, struc= t task_struct *task, else if (kernel && task) trace =3D get_callchain_entry_for_task(task, max_depth); else - trace =3D get_perf_callchain(regs, kernel, user, max_depth,false); + trace =3D get_perf_callchain(regs, kernel, user, max_depth, + false, false); =20 if (unlikely(!trace) || trace->nr < skip) { if (may_fault) diff --git a/kernel/events/callchain.c b/kernel/events/callchain.c index 2278402b7ac9..eeb15ba0137f 100644 --- a/kernel/events/callchain.c +++ b/kernel/events/callchain.c @@ -217,7 +217,7 @@ static void fixup_uretprobe_trampoline_entries(struct p= erf_callchain_entry *entr =20 struct perf_callchain_entry * get_perf_callchain(struct pt_regs *regs, bool kernel, bool user, - u32 max_stack, bool add_mark) + u32 max_stack, bool add_mark, bool defer_user) { struct perf_callchain_entry *entry; struct perf_callchain_entry_ctx ctx; @@ -246,6 +246,15 @@ get_perf_callchain(struct pt_regs *regs, bool kernel, = bool user, regs =3D task_pt_regs(current); } =20 + if (defer_user) { + /* + * Foretell the coming of PERF_RECORD_CALLCHAIN_DEFERRED + * which can be stitched to this one. + */ + perf_callchain_store_context(&ctx, PERF_CONTEXT_USER_DEFERRED); + goto exit_put; + } + if (add_mark) perf_callchain_store_context(&ctx, PERF_CONTEXT_USER); =20 diff --git a/kernel/events/core.c b/kernel/events/core.c index ebf143aa427b..bf97b2fa8a9c 100644 --- a/kernel/events/core.c +++ b/kernel/events/core.c @@ -55,11 +55,14 @@ #include #include #include +#include =20 #include "internal.h" =20 #include =20 +static struct unwind_callback perf_unwind_callback_cb; + typedef int (*remote_function_f)(void *); =20 struct remote_function_call { @@ -6955,6 +6958,53 @@ static void perf_pending_irq(struct irq_work *entry) perf_swevent_put_recursion_context(rctx); } =20 +static void perf_pending_unwind_irq(struct irq_work *entry) +{ + struct perf_event *event =3D container_of(entry, struct perf_event, pendi= ng_unwind_irq); + + if (event->pending_unwind) { + unwind_user_deferred(&perf_unwind_callback_cb, NULL, event); + event->pending_unwind =3D 0; + } +} + +struct perf_callchain_deferred_event { + struct perf_event_header header; + u64 ctx_cookie; + u64 nr; + u64 ips[]; +}; + +static void perf_event_callchain_deferred(struct unwind_stacktrace *trace, + u64 ctx_cookie, void *_data) +{ + struct perf_callchain_deferred_event deferred_event; + u64 callchain_context =3D PERF_CONTEXT_USER; + struct perf_output_handle handle; + struct perf_event *event =3D _data; + struct perf_sample_data data; + u64 nr =3D trace->nr + 1 /* callchain_context */; + + deferred_event.header.type =3D PERF_RECORD_CALLCHAIN_DEFERRED; + deferred_event.header.misc =3D PERF_RECORD_MISC_USER; + deferred_event.header.size =3D sizeof(deferred_event) + (nr * sizeof(u64)= ); + + deferred_event.ctx_cookie =3D ctx_cookie; + deferred_event.nr =3D nr; + + perf_event_header__init_id(&deferred_event.header, &data, event); + + if (perf_output_begin(&handle, &data, event, deferred_event.header.size)) + return; + + perf_output_put(&handle, deferred_event); + perf_output_put(&handle, callchain_context); + perf_output_copy(&handle, trace->entries, trace->nr * sizeof(u64)); + perf_event__output_id_sample(event, &handle, &data); + + perf_output_end(&handle); +} + static void perf_pending_task(struct callback_head *head) { struct perf_event *event =3D container_of(head, struct perf_event, pendin= g_task); @@ -7794,6 +7844,8 @@ perf_callchain(struct perf_event *event, struct pt_re= gs *regs) bool user =3D !event->attr.exclude_callchain_user; const u32 max_stack =3D event->attr.sample_max_stack; struct perf_callchain_entry *callchain; + bool defer_user =3D IS_ENABLED(CONFIG_UNWIND_USER) && + event->attr.defer_callchain; =20 if (!kernel && !user) return &__empty_callchain; @@ -7802,7 +7854,14 @@ perf_callchain(struct perf_event *event, struct pt_r= egs *regs) if (event->ctx->task && event->ctx->task !=3D current) return &__empty_callchain; =20 - callchain =3D get_perf_callchain(regs, kernel, user, max_stack, true); + callchain =3D get_perf_callchain(regs, kernel, user, max_stack, true, + defer_user); + + if (user && defer_user && !event->pending_unwind) { + event->pending_unwind =3D 1; + irq_work_queue(&event->pending_unwind_irq); + } + return callchain ?: &__empty_callchain; } =20 @@ -12171,6 +12230,7 @@ perf_event_alloc(struct perf_event_attr *attr, int = cpu, =20 init_waitqueue_head(&event->waitq); init_irq_work(&event->pending_irq, perf_pending_irq); + event->pending_unwind_irq =3D IRQ_WORK_INIT_HARD(perf_pending_unwind_irq); event->pending_disable_irq =3D IRQ_WORK_INIT_HARD(perf_pending_disable); init_task_work(&event->pending_task, perf_pending_task); rcuwait_init(&event->pending_work_wait); @@ -14093,6 +14153,7 @@ void __init perf_event_init(void) perf_tp_register(); perf_event_init_cpu(smp_processor_id()); register_reboot_notifier(&perf_reboot_notifier); + unwind_user_register(&perf_unwind_callback_cb, perf_event_callchain_defer= red); =20 ret =3D init_hw_breakpoint(); WARN(ret, "hw_breakpoint initialization failed with: %d", ret); diff --git a/tools/include/uapi/linux/perf_event.h b/tools/include/uapi/lin= ux/perf_event.h index 4842c36fdf80..6d0524b7d082 100644 --- a/tools/include/uapi/linux/perf_event.h +++ b/tools/include/uapi/linux/perf_event.h @@ -460,7 +460,8 @@ struct perf_event_attr { inherit_thread : 1, /* children only inherit if cloned with CLONE_THR= EAD */ remove_on_exec : 1, /* event is removed from task on exec */ sigtrap : 1, /* send synchronous SIGTRAP on event */ - __reserved_1 : 26; + defer_callchain: 1, /* generate PERF_RECORD_CALLCHAIN_DEFERRED record= s */ + __reserved_1 : 25; =20 union { __u32 wakeup_events; /* wakeup every n events */ @@ -1217,6 +1218,24 @@ enum perf_event_type { */ PERF_RECORD_AUX_OUTPUT_HW_ID =3D 21, =20 + /* + * This user callchain capture was deferred until shortly before + * returning to user space. Previous samples would have kernel + * callchains only and they need to be stitched with this to make full + * callchains. + * + * TODO: do PERF_SAMPLE_{REGS,STACK}_USER also need deferral? + * + * struct { + * struct perf_event_header header; + * u64 ctx_cookie; + * u64 nr; + * u64 ips[nr]; + * struct sample_id sample_id; + * }; + */ + PERF_RECORD_CALLCHAIN_DEFERRED =3D 22, + PERF_RECORD_MAX, /* non-ABI */ }; =20 @@ -1247,6 +1266,7 @@ enum perf_callchain_context { PERF_CONTEXT_HV =3D (__u64)-32, PERF_CONTEXT_KERNEL =3D (__u64)-128, PERF_CONTEXT_USER =3D (__u64)-512, + PERF_CONTEXT_USER_DEFERRED =3D (__u64)-640, =20 PERF_CONTEXT_GUEST =3D (__u64)-2048, PERF_CONTEXT_GUEST_KERNEL =3D (__u64)-2176, --=20 2.47.0