From nobody Tue Dec 2 02:04:05 2025 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-1.web.codeaurora.org [10.30.226.201]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id E16963242A1; Thu, 20 Nov 2025 21:25:36 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=10.30.226.201 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1763673937; cv=none; b=KvN1gsYXjuR3m3ZM6iorStMgmrrYz5gACJRnc1SUwjJVc1o/J39XGl0N0YfJ2Gp5l6ObTRyE2FncLQuJsyJ1onQdwRPsVPUmcRWSvDmMw+2p04jyx4ptgE8WVWnld7zH2jbqZcwrg07h/TlDtQpb1Jet543l9Vcu+sw53d5XaUc= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1763673937; c=relaxed/simple; bh=tWsIX28eUU09Y+4XW8T9/1bbDivP+CjAgctxdhSf0nM=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=J4NgM5pkF+jWujHc9ATYpQcHDNrolWwu3/IAvOHPSrWmPLRJV4yyOX4XGdFUH6qbmHymBQ3/GvTU3NQjPM5d07YalMmwnfh2Uje56PZdKnY65582mSsP6Qfx62rqhHuROYr3E8tO2XX/AGAj/Te33hdq/vXQGkrdOSeQKhjWHLw= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=G5bdWYXb; arc=none smtp.client-ip=10.30.226.201 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="G5bdWYXb" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 940F6C4CEF1; Thu, 20 Nov 2025 21:25:33 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1763673936; bh=tWsIX28eUU09Y+4XW8T9/1bbDivP+CjAgctxdhSf0nM=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=G5bdWYXbUX7EtiNCcmeVlhB9VsPW+pdJ9kuu2dZxAM15q+VCh9jqvzzI3YvzYjLV7 WyY5d3z1KexwMY14vA19/UaPPX15FsSDX6FpNXEsDEYCyG/sQDY3zicRRnWS1pLzhO aa2Prrs92VBCtpZ5ZYadyKGTB1kn2Kdq5FDNueFlsU3swjzCOtWaAMhcorraHx1kUa xqZnOLkg3uejk5h7l1ZbwWY7UqtphvcIemA1lUQRCHV7vCxoLeBbJpI5ROaqleSApi fr0/L4gigkYqmCUnAguD0B8uXObigT8WS7R/VGoIxChs3B0sKAU+g4gqY0YVpIl8y1 DJ9a8t03RKUGA== From: Jiri Olsa To: Steven Rostedt , Florent Revest , Mark Rutland Cc: bpf@vger.kernel.org, linux-kernel@vger.kernel.org, linux-trace-kernel@vger.kernel.org, linux-arm-kernel@lists.infradead.org, Alexei Starovoitov , Daniel Borkmann , Andrii Nakryiko , Menglong Dong , Song Liu Subject: [PATCHv3 bpf-next 8/8] bpf, x86: Use single ftrace_ops for direct calls Date: Thu, 20 Nov 2025 22:24:02 +0100 Message-ID: <20251120212402.466524-9-jolsa@kernel.org> X-Mailer: git-send-email 2.51.1 In-Reply-To: <20251120212402.466524-1-jolsa@kernel.org> References: <20251120212402.466524-1-jolsa@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Using single ftrace_ops for direct calls update instead of allocating ftrace_ops object for each trampoline. With single ftrace_ops object we can use update_ftrace_direct_* api that allows multiple ip sites updates on single ftrace_ops object. Adding HAVE_SINGLE_FTRACE_DIRECT_OPS config option to be enabled on each arch that supports this. At the moment we can enable this only on x86 arch, because arm relies on ftrace_ops object representing just single trampoline image (stored in ftrace_ops::direct_call). Ach that do not support this will continue to use *_ftrace_direct api. Signed-off-by: Jiri Olsa --- arch/x86/Kconfig | 1 + kernel/bpf/trampoline.c | 172 +++++++++++++++++++++++++++++++++++----- kernel/trace/Kconfig | 3 + kernel/trace/ftrace.c | 7 +- 4 files changed, 164 insertions(+), 19 deletions(-) diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index fa3b616af03a..65a2fc279b46 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -332,6 +332,7 @@ config X86 select SCHED_SMT if SMP select ARCH_SUPPORTS_SCHED_CLUSTER if SMP select ARCH_SUPPORTS_SCHED_MC if SMP + select HAVE_SINGLE_FTRACE_DIRECT_OPS if X86_64 && DYNAMIC_FTRACE_WITH_DIR= ECT_CALLS =20 config INSTRUCTION_DECODER def_bool y diff --git a/kernel/bpf/trampoline.c b/kernel/bpf/trampoline.c index d8762f7990ed..9cc4de215df7 100644 --- a/kernel/bpf/trampoline.c +++ b/kernel/bpf/trampoline.c @@ -33,12 +33,40 @@ static DEFINE_MUTEX(trampoline_mutex); #ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS static int bpf_trampoline_update(struct bpf_trampoline *tr, bool lock_dire= ct_mutex); =20 +#ifdef CONFIG_HAVE_SINGLE_FTRACE_DIRECT_OPS +static struct bpf_trampoline *direct_ops_ip_lookup(struct ftrace_ops *ops,= unsigned long ip) +{ + struct hlist_head *head_ip; + struct bpf_trampoline *tr; + + mutex_lock(&trampoline_mutex); + head_ip =3D &trampoline_ip_table[hash_64(ip, TRAMPOLINE_HASH_BITS)]; + hlist_for_each_entry(tr, head_ip, hlist_ip) { + if (tr->ip =3D=3D ip) + goto out; + } + tr =3D NULL; +out: + mutex_unlock(&trampoline_mutex); + return tr; +} +#else +static struct bpf_trampoline *direct_ops_ip_lookup(struct ftrace_ops *ops,= unsigned long ip) +{ + return ops->private; +} +#endif /* CONFIG_HAVE_SINGLE_FTRACE_DIRECT_OPS */ + static int bpf_tramp_ftrace_ops_func(struct ftrace_ops *ops, unsigned long= ip, enum ftrace_ops_cmd cmd) { - struct bpf_trampoline *tr =3D ops->private; + struct bpf_trampoline *tr; int ret =3D 0; =20 + tr =3D direct_ops_ip_lookup(ops, ip); + if (!tr) + return -EINVAL; + if (cmd =3D=3D FTRACE_OPS_CMD_ENABLE_SHARE_IPMODIFY_SELF) { /* This is called inside register_ftrace_direct_multi(), so * tr->mutex is already locked. @@ -137,6 +165,126 @@ void bpf_image_ksym_del(struct bpf_ksym *ksym) PAGE_SIZE, true, ksym->name); } =20 +#ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS +#ifdef CONFIG_HAVE_SINGLE_FTRACE_DIRECT_OPS +/* + * We have only single direct_ops which contains all the direct call + * sites and is the only global ftrace_ops for all trampolines. + * + * We use 'update_ftrace_direct_*' api for attachment. + */ +struct ftrace_ops direct_ops =3D { + .ops_func =3D bpf_tramp_ftrace_ops_func, +}; + +static int direct_ops_alloc(struct bpf_trampoline *tr) +{ + tr->fops =3D &direct_ops; + return 0; +} + +static void direct_ops_free(struct bpf_trampoline *tr) { } + +static struct ftrace_hash *hash_from(unsigned long ip, void *addr) +{ + struct ftrace_hash *hash; + + ip =3D ftrace_location(ip); + if (!ip) + return NULL; + hash =3D alloc_ftrace_hash(FTRACE_HASH_DEFAULT_BITS); + if (!hash) + return NULL; + if (!add_hash_entry_direct(hash, ip, (unsigned long) addr)) { + free_ftrace_hash(hash); + return NULL; + } + return hash; +} + +static int direct_ops_add(struct ftrace_ops *ops, unsigned long ip, void *= addr) +{ + struct ftrace_hash *hash =3D hash_from(ip, addr); + int err =3D -ENOMEM; + + if (hash) + err =3D update_ftrace_direct_add(ops, hash); + free_ftrace_hash(hash); + return err; +} + +static int direct_ops_del(struct ftrace_ops *ops, unsigned long ip, void *= addr) +{ + struct ftrace_hash *hash =3D hash_from(ip, addr); + int err =3D -ENOMEM; + + if (hash) + err =3D update_ftrace_direct_del(ops, hash); + free_ftrace_hash(hash); + return err; +} + +static int direct_ops_mod(struct ftrace_ops *ops, unsigned long ip, void *= addr, bool lock_direct_mutex) +{ + struct ftrace_hash *hash =3D hash_from(ip, addr); + int err =3D -ENOMEM; + + if (hash) + err =3D update_ftrace_direct_mod(ops, hash, lock_direct_mutex); + free_ftrace_hash(hash); + return err; +} +#else +/* + * We allocate ftrace_ops object for each trampoline and it contains + * call site specific for that trampoline. + * + * We use *_ftrace_direct api for attachment. + */ +static int direct_ops_alloc(struct bpf_trampoline *tr) +{ + tr->fops =3D kzalloc(sizeof(struct ftrace_ops), GFP_KERNEL); + if (!tr->fops) + return -ENOMEM; + tr->fops->private =3D tr; + tr->fops->ops_func =3D bpf_tramp_ftrace_ops_func; + return 0; +} + +static void direct_ops_free(struct bpf_trampoline *tr) +{ + if (tr->fops) { + ftrace_free_filter(tr->fops); + kfree(tr->fops); + } +} + +static int direct_ops_add(struct ftrace_ops *ops, unsigned long ip, void *= addr) +{ + int ret; + + ret =3D ftrace_set_filter_ip(ops, (unsigned long)ip, 0, 1); + if (ret) + return ret; + return register_ftrace_direct(ops, (long)addr); +} + +static int direct_ops_del(struct ftrace_ops *ops, unsigned long ip, void *= addr) +{ + return unregister_ftrace_direct(ops, (long)addr, false); +} + +static int direct_ops_mod(struct ftrace_ops *ops, unsigned long ip, void *= addr, bool lock_direct_mutex) +{ + if (lock_direct_mutex) + return modify_ftrace_direct(ops, (long)addr); + return modify_ftrace_direct_nolock(ops, (long)addr); +} +#endif /* CONFIG_HAVE_SINGLE_FTRACE_DIRECT_OPS */ +#else +static void direct_ops_free(struct bpf_trampoline *tr) { } +#endif /* CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS */ + static struct bpf_trampoline *bpf_trampoline_lookup(u64 key, unsigned long= ip) { struct bpf_trampoline *tr; @@ -155,14 +303,11 @@ static struct bpf_trampoline *bpf_trampoline_lookup(u= 64 key, unsigned long ip) if (!tr) goto out; #ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS - tr->fops =3D kzalloc(sizeof(struct ftrace_ops), GFP_KERNEL); - if (!tr->fops) { + if (direct_ops_alloc(tr)) { kfree(tr); tr =3D NULL; goto out; } - tr->fops->private =3D tr; - tr->fops->ops_func =3D bpf_tramp_ftrace_ops_func; #endif =20 tr->key =3D key; @@ -187,7 +332,7 @@ static int unregister_fentry(struct bpf_trampoline *tr,= void *old_addr) int ret; =20 if (tr->func.ftrace_managed) - ret =3D unregister_ftrace_direct(tr->fops, (long)old_addr, false); + ret =3D direct_ops_del(tr->fops, tr->ip, old_addr); else ret =3D bpf_arch_text_poke(ip, BPF_MOD_CALL, old_addr, NULL); =20 @@ -201,10 +346,7 @@ static int modify_fentry(struct bpf_trampoline *tr, vo= id *old_addr, void *new_ad int ret; =20 if (tr->func.ftrace_managed) { - if (lock_direct_mutex) - ret =3D modify_ftrace_direct(tr->fops, (long)new_addr); - else - ret =3D modify_ftrace_direct_nolock(tr->fops, (long)new_addr); + ret =3D direct_ops_mod(tr->fops, tr->ip, new_addr, lock_direct_mutex); } else { ret =3D bpf_arch_text_poke(ip, BPF_MOD_CALL, old_addr, new_addr); } @@ -226,10 +368,7 @@ static int register_fentry(struct bpf_trampoline *tr, = void *new_addr) } =20 if (tr->func.ftrace_managed) { - ret =3D ftrace_set_filter_ip(tr->fops, (unsigned long)ip, 0, 1); - if (ret) - return ret; - ret =3D register_ftrace_direct(tr->fops, (long)new_addr); + ret =3D direct_ops_add(tr->fops, tr->ip, new_addr); } else { ret =3D bpf_arch_text_poke(ip, BPF_MOD_CALL, NULL, new_addr); } @@ -865,10 +1004,7 @@ void bpf_trampoline_put(struct bpf_trampoline *tr) */ hlist_del(&tr->hlist_key); hlist_del(&tr->hlist_ip); - if (tr->fops) { - ftrace_free_filter(tr->fops); - kfree(tr->fops); - } + direct_ops_free(tr); kfree(tr); out: mutex_unlock(&trampoline_mutex); diff --git a/kernel/trace/Kconfig b/kernel/trace/Kconfig index d2c79da81e4f..4bf5beb04a5b 100644 --- a/kernel/trace/Kconfig +++ b/kernel/trace/Kconfig @@ -50,6 +50,9 @@ config HAVE_DYNAMIC_FTRACE_WITH_REGS config HAVE_DYNAMIC_FTRACE_WITH_DIRECT_CALLS bool =20 +config HAVE_SINGLE_FTRACE_DIRECT_OPS + bool + config HAVE_DYNAMIC_FTRACE_WITH_CALL_OPS bool =20 diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c index b0e4d7a5d47c..4380839d2422 100644 --- a/kernel/trace/ftrace.c +++ b/kernel/trace/ftrace.c @@ -2605,8 +2605,13 @@ unsigned long ftrace_find_rec_direct(unsigned long i= p) static void call_direct_funcs(unsigned long ip, unsigned long pip, struct ftrace_ops *ops, struct ftrace_regs *fregs) { - unsigned long addr =3D READ_ONCE(ops->direct_call); + unsigned long addr; =20 +#ifdef CONFIG_HAVE_SINGLE_FTRACE_DIRECT_OPS + addr =3D ftrace_find_rec_direct(ip); +#else + addr =3D READ_ONCE(ops->direct_call); +#endif if (!addr) return; =20 --=20 2.51.1