From nobody Thu Oct 2 01:57:53 2025 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-1.web.codeaurora.org [10.30.226.201]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id CF29B30E0FB; Tue, 23 Sep 2025 21:52:09 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=10.30.226.201 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1758664329; cv=none; b=jNt3V2eXZPOn3OKZNXIMg36nywFFgH8EJhQVM3edkI2RdZYbMLz3cwb7rwsUUe1WKoMldSVkcI5QZw1x9xttNZtGGKMXk7+/Za0Xs12Wz58RHo4/0FaU9U0/PoOWIJPSlLzTN5HPeOZB2RW/ufGw2VXHQc0QUnY9dkVkp5q0PZQ= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1758664329; c=relaxed/simple; bh=0OT+Cifnt0VO7z3Jrp77kFKvZcc+x/w351c++TwCC8k=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=foSp9j2usGzVS/aEsZfJ4uFWmLa66k/mDcDD20Zc9U8V/GokjzXO1fI2pI19albquRJITZan9jmqOyqnDRVjJh3S6jSLqt5HwJTX+3HMPE6k91j//HXj1LVHqL0RxE6lN/kjRRuxqWbehqiUJ+j3bBAd5GFm3VjAFqsE15t7wXA= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=lFQrqZvy; arc=none smtp.client-ip=10.30.226.201 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="lFQrqZvy" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 2FC33C4CEF5; Tue, 23 Sep 2025 21:52:05 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1758664329; bh=0OT+Cifnt0VO7z3Jrp77kFKvZcc+x/w351c++TwCC8k=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=lFQrqZvyVzpwGobW1Sq6klf1KfypkcKGBdHHJ8VMP9SnEbZViyQP4J7B5JuLsvUaC yBATX7bUKRqJ5wRjrnH1jm5X8vV16rqXhk8kCHQ9BRGQVsNsqK+5CufVCl3dFjUYts Fv/qTGYF7wy0FkOlJMSpnVXjchYpYWvyBQ189bvTa0go0zCsyZE+X+ax5RUjLOWRI7 zw+m+G1qf/QMP0Ge7KXiikMSO9ROZIYu71LzbJzZ+HRSUMuwqZ584aQoTV8pQoBfPI BXVvjV608ylTuVn2HZm9R2k+i2bptD5vaeA8B2vsVY5cb2l7k7nPUASXMEE0rDBDHE jQuYWJgg8TchA== From: Jiri Olsa To: Steven Rostedt , Florent Revest , Mark Rutland Cc: bpf@vger.kernel.org, linux-kernel@vger.kernel.org, linux-trace-kernel@vger.kernel.org, linux-arm-kernel@lists.infradead.org, Alexei Starovoitov , Daniel Borkmann , Andrii Nakryiko , Menglong Dong Subject: [PATCH 1/9] ftrace: Make alloc_and_copy_ftrace_hash direct friendly Date: Tue, 23 Sep 2025 23:51:39 +0200 Message-ID: <20250923215147.1571952-2-jolsa@kernel.org> X-Mailer: git-send-email 2.51.0 In-Reply-To: <20250923215147.1571952-1-jolsa@kernel.org> References: <20250923215147.1571952-1-jolsa@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Make alloc_and_copy_ftrace_hash to copy also direct address for each hash entry. Signed-off-by: Jiri Olsa --- kernel/trace/ftrace.c | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c index a69067367c29..a45556257963 100644 --- a/kernel/trace/ftrace.c +++ b/kernel/trace/ftrace.c @@ -1186,7 +1186,7 @@ static void __add_hash_entry(struct ftrace_hash *hash, } =20 static struct ftrace_func_entry * -add_hash_entry(struct ftrace_hash *hash, unsigned long ip) +add_hash_entry_direct(struct ftrace_hash *hash, unsigned long ip, unsigned= long direct) { struct ftrace_func_entry *entry; =20 @@ -1195,11 +1195,18 @@ add_hash_entry(struct ftrace_hash *hash, unsigned l= ong ip) return NULL; =20 entry->ip =3D ip; + entry->direct =3D direct; __add_hash_entry(hash, entry); =20 return entry; } =20 +static struct ftrace_func_entry * +add_hash_entry(struct ftrace_hash *hash, unsigned long ip) +{ + return add_hash_entry_direct(hash, ip, 0); +} + static void free_hash_entry(struct ftrace_hash *hash, struct ftrace_func_entry *entry) @@ -1372,7 +1379,7 @@ alloc_and_copy_ftrace_hash(int size_bits, struct ftra= ce_hash *hash) size =3D 1 << hash->size_bits; for (i =3D 0; i < size; i++) { hlist_for_each_entry(entry, &hash->buckets[i], hlist) { - if (add_hash_entry(new_hash, entry->ip) =3D=3D NULL) + if (add_hash_entry_direct(new_hash, entry->ip, entry->direct) =3D=3D NU= LL) goto free_hash; } } --=20 2.51.0 From nobody Thu Oct 2 01:57:53 2025 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-1.web.codeaurora.org [10.30.226.201]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id A8AC030F556; Tue, 23 Sep 2025 21:52:20 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=10.30.226.201 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1758664340; cv=none; b=KLDVKIE2Dpc6d7kNzpMIFurUxiZMscQCG1G9Ys0jsLcIOMzpoYrgYVaCdgiCf8SobAerytZhW1lQ6251aIU4azeymAxu6OZsqFQ9fC7oGik8YS+OFziIVlzI8UQYz6CtL82RR306bek+5XvJz440BFVNrbyyNY10L8/RmfALkXc= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1758664340; c=relaxed/simple; bh=owMdIa2olCHJ3eYUyQpQ21kOHUHt5sxjjmLt+nAB5Gc=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=YMmB4F1mQ+uecememrMw8nIWIGn1dja9l2jHOM6gTk9Up9Kvn9nFneKebKt3Q+qordD9stUO2073YxMs4FAbhyd06D7zcmj7sTvOXjCuGXzYQZDv1f7rMtSfr64O4A2oYC0KwuvSJLUqFAI7yas3a8Z/1mTqsWOhgyea3XETKHQ= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=G/RqXC3K; arc=none smtp.client-ip=10.30.226.201 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="G/RqXC3K" Received: by smtp.kernel.org (Postfix) with ESMTPSA id EE1A9C4CEF5; Tue, 23 Sep 2025 21:52:16 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1758664340; bh=owMdIa2olCHJ3eYUyQpQ21kOHUHt5sxjjmLt+nAB5Gc=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=G/RqXC3KmUV/CsJGvJtZGwzPF2/7gsfLJPdooZB5WN/gLN1DiiKGgZjP4UllLkQy7 tRj1xNyIWXuw6H/dNVx6X7zF64rjQYHboNCdMyOkiixCFiFBpMQcLG65Ffii91LuXD 81Y2gWy20aCGQfNCg8AOz9VvC89cxDkWm+0y+RFuA5/DJnzKsgPTgQp0qFSB0HxOXL Q58zDHoFmrrD7A77l7gQZWGsTV+oPZQEPcMm0GX+tZ0PJGnr42my2EogblAnLk50ei litgKOD8b9mI2oso6hzd2EsX3parhlBtgRGWKnipP45cFN5H2S66gcWvYLs9EISmr3 4yC1bb25iTEDA== From: Jiri Olsa To: Steven Rostedt , Florent Revest , Mark Rutland Cc: bpf@vger.kernel.org, linux-kernel@vger.kernel.org, linux-trace-kernel@vger.kernel.org, linux-arm-kernel@lists.infradead.org, Alexei Starovoitov , Daniel Borkmann , Andrii Nakryiko , Menglong Dong Subject: [PATCH 2/9] ftrace: Add register_ftrace_direct_hash function Date: Tue, 23 Sep 2025 23:51:40 +0200 Message-ID: <20250923215147.1571952-3-jolsa@kernel.org> X-Mailer: git-send-email 2.51.0 In-Reply-To: <20250923215147.1571952-1-jolsa@kernel.org> References: <20250923215147.1571952-1-jolsa@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Adding register_ftrace_direct_hash function that registers all entries (ip -> direct) provided in hash argument. The difference to current register_ftrace_direct is - hash argument that allows to register multiple ip -> direct entries at once - we can call register_ftrace_direct_hash multiple times on the same ftrace_ops object, becase after first registration with register_ftrace_function_nolock, it uses ftrace_update_ops to update the ftrace_ops object This change will allow us to have simple ftrace_ops for all bpf direct interface users in following changes. Signed-off-by: Jiri Olsa --- include/linux/ftrace.h | 7 +++ kernel/trace/ftrace.c | 110 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 117 insertions(+) diff --git a/include/linux/ftrace.h b/include/linux/ftrace.h index 7ded7df6e9b5..2705c292341a 100644 --- a/include/linux/ftrace.h +++ b/include/linux/ftrace.h @@ -526,6 +526,8 @@ int unregister_ftrace_direct(struct ftrace_ops *ops, un= signed long addr, int modify_ftrace_direct(struct ftrace_ops *ops, unsigned long addr); int modify_ftrace_direct_nolock(struct ftrace_ops *ops, unsigned long addr= ); =20 +int register_ftrace_direct_hash(struct ftrace_ops *ops, struct ftrace_hash= *hash); + void ftrace_stub_direct_tramp(void); =20 #else @@ -552,6 +554,11 @@ static inline int modify_ftrace_direct_nolock(struct f= trace_ops *ops, unsigned l return -ENODEV; } =20 +int register_ftrace_direct_hash(struct ftrace_ops *ops, struct ftrace_hash= *hash) +{ + return -ENODEV; +} + /* * This must be implemented by the architecture. * It is the way the ftrace direct_ops helper, when called diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c index a45556257963..06528af2281f 100644 --- a/kernel/trace/ftrace.c +++ b/kernel/trace/ftrace.c @@ -6219,6 +6219,116 @@ int modify_ftrace_direct(struct ftrace_ops *ops, un= signed long addr) return err; } EXPORT_SYMBOL_GPL(modify_ftrace_direct); + +static unsigned long hash_count(struct ftrace_hash *hash) +{ + return hash ? hash->count : 0; +} + +/** + * hash_add - adds two struct ftrace_hash and returns the result + * @a: struct ftrace_hash object + * @b: struct ftrace_hash object + * + * Returns struct ftrace_hash object on success, NULL on error. + */ +static struct ftrace_hash *hash_add(struct ftrace_hash *a, struct ftrace_h= ash *b) +{ + struct ftrace_func_entry *entry; + struct ftrace_hash *add; + int size, i; + + size =3D hash_count(a) + hash_count(b); + if (size > 32) + size =3D 32; + + add =3D alloc_and_copy_ftrace_hash(fls(size), a); + if (!add) + goto error; + + size =3D 1 << b->size_bits; + for (i =3D 0; i < size; i++) { + hlist_for_each_entry(entry, &b->buckets[i], hlist) { + if (add_hash_entry_direct(add, entry->ip, entry->direct) =3D=3D NULL) + goto error; + } + } + return add; + + error: + free_ftrace_hash(add); + return NULL; +} + +int register_ftrace_direct_hash(struct ftrace_ops *ops, struct ftrace_hash= *hash) +{ + struct ftrace_hash *filter_hash =3D NULL, *new_hash =3D NULL, *free_hash = =3D NULL; + struct ftrace_func_entry *entry; + int i, size, err =3D -EINVAL; + bool reg; + + if (!hash_count(hash)) + return 0; + + mutex_lock(&direct_mutex); + + /* Make sure requested entries are not already registered. */ + size =3D 1 << hash->size_bits; + for (i =3D 0; i < size; i++) { + hlist_for_each_entry(entry, &hash->buckets[i], hlist) { + if (__ftrace_lookup_ip(direct_functions, entry->ip)) + goto out_unlock; + } + } + + filter_hash =3D ops->func_hash ? ops->func_hash->filter_hash : NULL; + + /* If there's nothing in filter_hash we need to register the ops. */ + reg =3D hash_count(filter_hash) =3D=3D 0; + if (reg) { + if (ops->func || ops->trampoline) + goto out_unlock; + if (ops->flags & FTRACE_OPS_FL_ENABLED) + goto out_unlock; + } + + err =3D -ENOMEM; + filter_hash =3D hash_add(filter_hash, hash); + if (!filter_hash) + goto out_unlock; + + new_hash =3D hash_add(direct_functions, hash); + if (!new_hash) + goto out_unlock; + + free_hash =3D direct_functions; + rcu_assign_pointer(direct_functions, new_hash); + + if (reg) { + ops->func =3D call_direct_funcs; + ops->flags =3D MULTI_FLAGS; + ops->trampoline =3D FTRACE_REGS_ADDR; + ops->local_hash.filter_hash =3D filter_hash; + + err =3D register_ftrace_function_nolock(ops); + if (!err) + filter_hash =3D NULL; + } else { + err =3D ftrace_update_ops(ops, filter_hash, EMPTY_HASH); + } + + out_unlock: + mutex_unlock(&direct_mutex); + + if (free_hash && free_hash !=3D EMPTY_HASH) + call_rcu_tasks(&free_hash->rcu, register_ftrace_direct_cb); + if (filter_hash) + free_ftrace_hash(filter_hash); + + return err; +} +EXPORT_SYMBOL_GPL(register_ftrace_direct_hash); + #endif /* CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS */ =20 /** --=20 2.51.0 From nobody Thu Oct 2 01:57:53 2025 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-1.web.codeaurora.org [10.30.226.201]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 8FB4130F552; Tue, 23 Sep 2025 21:52:31 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=10.30.226.201 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1758664351; cv=none; b=YCjgZrYYZLfwXOcV/4jwxVTxb4Y0wDejqFjA/KoSt7WmUxwEQ5kn1NmeDiBcIXzVIycN+WLXVw5q4ctAgO3M5DUUSze/bf5VKr6r4l5flwue7CYFXYsmo+V+X4exgJZWQE19KRXggQm90y0LKiL9nuPLDIvJHSFux5bOCB1kJxc= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1758664351; c=relaxed/simple; bh=BULakDdKw0A4NZGkv919IsMp+rW5CmZOYDx2mdUjOFA=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=YTlyli4McVqF7SsRxaGxbTQylgHMT1IkZ6/wijW1WfQQoPW4m38+flninm8wOZ/C/Qv9DOLDZtTZX+4PohJHI6wgbwMTK0JsXZGp6q69kohVTDufdwLgcGBdF1EyV/gH198/hZTHjGDJ2Ct5MLCOVsMxR0LYe74bDSFsC9JnIkM= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=bbPDkg8x; arc=none smtp.client-ip=10.30.226.201 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="bbPDkg8x" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 2E5A7C4CEF5; Tue, 23 Sep 2025 21:52:27 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1758664351; bh=BULakDdKw0A4NZGkv919IsMp+rW5CmZOYDx2mdUjOFA=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=bbPDkg8x5vH27VfUoF+koTvyt23ssm+a7u9uXZl1YlS2HNFgfS/7XXfJJcS36jBK+ jbI0CCQwFbtQmwdOyJUvQVRVRLOnaomAwjKVnHzJlEMNZ7MsqpA/Hi9F/qxuFUNWVx iZqOyoV41gMhmdN2/QBQgUO8pTSfw2YthTD/82vNjswNC9ftWo3DzQDeriMn8V51WJ 4QUpXkEpfQZ9mRg7szIgWPrpYfg3LfGY59rOM9URqQDQTBOYCpp4RwL10WqZlRfHgb 0PPOBNd2hpo/yAWgbRVCF8pm1SDCpjwqIQqAJxTL2hpp7trEedJx/XLhPz0U5mZ02H Q1QkPGTs/N0Bw== From: Jiri Olsa To: Steven Rostedt , Florent Revest , Mark Rutland Cc: bpf@vger.kernel.org, linux-kernel@vger.kernel.org, linux-trace-kernel@vger.kernel.org, linux-arm-kernel@lists.infradead.org, Alexei Starovoitov , Daniel Borkmann , Andrii Nakryiko , Menglong Dong Subject: [PATCH 3/9] ftrace: Add unregister_ftrace_direct_hash function Date: Tue, 23 Sep 2025 23:51:41 +0200 Message-ID: <20250923215147.1571952-4-jolsa@kernel.org> X-Mailer: git-send-email 2.51.0 In-Reply-To: <20250923215147.1571952-1-jolsa@kernel.org> References: <20250923215147.1571952-1-jolsa@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Adding unregister_ftrace_direct_hash function that unregisters all entries (ip -> direct) provided in hash argument. The difference to current unregister_ftrace_direct is - hash argument that allows to unregister multiple ip -> direct entries at once - we can call unregister_ftrace_direct_hash multiple times on the same ftrace_ops object, becase we do not need to unregister all entries at once, we can do it gradualy with the help of ftrace_update_ops function This change will allow us to have simple ftrace_ops for all bpf direct interface users in following changes. Signed-off-by: Jiri Olsa --- include/linux/ftrace.h | 6 +++ kernel/trace/ftrace.c | 98 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 104 insertions(+) diff --git a/include/linux/ftrace.h b/include/linux/ftrace.h index 2705c292341a..55f5ead5d4ff 100644 --- a/include/linux/ftrace.h +++ b/include/linux/ftrace.h @@ -527,6 +527,7 @@ int modify_ftrace_direct(struct ftrace_ops *ops, unsign= ed long addr); int modify_ftrace_direct_nolock(struct ftrace_ops *ops, unsigned long addr= ); =20 int register_ftrace_direct_hash(struct ftrace_ops *ops, struct ftrace_hash= *hash); +int unregister_ftrace_direct_hash(struct ftrace_ops *ops, struct ftrace_ha= sh *hash); =20 void ftrace_stub_direct_tramp(void); =20 @@ -559,6 +560,11 @@ int register_ftrace_direct_hash(struct ftrace_ops *ops= , struct ftrace_hash *hash return -ENODEV; } =20 +int unregister_ftrace_direct_hash(struct ftrace_ops *ops, struct ftrace_ha= sh *hash) +{ + return -ENODEV; +} + /* * This must be implemented by the architecture. * It is the way the ftrace direct_ops helper, when called diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c index 06528af2281f..ab5739f72933 100644 --- a/kernel/trace/ftrace.c +++ b/kernel/trace/ftrace.c @@ -6329,6 +6329,104 @@ int register_ftrace_direct_hash(struct ftrace_ops *= ops, struct ftrace_hash *hash } EXPORT_SYMBOL_GPL(register_ftrace_direct_hash); =20 +/** + * hash_sub - substracts @b from @a and returns the result + * @a: struct ftrace_hash object + * @b: struct ftrace_hash object + * + * Returns struct ftrace_hash object on success, NULL on error. + */ +static struct ftrace_hash *hash_sub(struct ftrace_hash *a, struct ftrace_h= ash *b) +{ + struct ftrace_func_entry *entry, *del; + struct ftrace_hash *sub; + int size, i; + + sub =3D alloc_and_copy_ftrace_hash(a->size_bits, a); + if (!sub) + goto error; + + size =3D 1 << b->size_bits; + for (i =3D 0; i < size; i++) { + hlist_for_each_entry(entry, &b->buckets[i], hlist) { + del =3D __ftrace_lookup_ip(sub, entry->ip); + if (WARN_ON_ONCE(!del)) + goto error; + remove_hash_entry(sub, del); + kfree(del); + } + } + return sub; + + error: + free_ftrace_hash(sub); + return NULL; +} + +int unregister_ftrace_direct_hash(struct ftrace_ops *ops, struct ftrace_ha= sh *hash) +{ + struct ftrace_hash *new_hash =3D NULL, *filter_hash =3D NULL, *free_hash = =3D NULL; + struct ftrace_func_entry *del, *entry; + unsigned long size, i; + int err =3D -EINVAL; + + if (!hash_count(hash)) + return 0; + if (check_direct_multi(ops)) + return -EINVAL; + if (!(ops->flags & FTRACE_OPS_FL_ENABLED)) + return -EINVAL; + if (direct_functions =3D=3D EMPTY_HASH) + return -EINVAL; + + mutex_lock(&direct_mutex); + + /* Make sure requested entries are already registered. */ + size =3D 1 << hash->size_bits; + for (i =3D 0; i < size; i++) { + hlist_for_each_entry(entry, &hash->buckets[i], hlist) { + del =3D __ftrace_lookup_ip(direct_functions, entry->ip); + if (!del || del->direct !=3D entry->direct) + goto out_unlock; + } + } + + err =3D -ENOMEM; + filter_hash =3D hash_sub(ops->func_hash->filter_hash, hash); + if (!filter_hash) + goto out_unlock; + + new_hash =3D hash_sub(direct_functions, hash); + if (!new_hash) + goto out_unlock; + + /* If there's nothing left, we need to unregister the ops. */ + if (ftrace_hash_empty(filter_hash)) { + err =3D unregister_ftrace_function(ops); + /* cleanup for possible another register call */ + ops->func =3D NULL; + ops->trampoline =3D 0; + ftrace_free_filter(ops); + ops->func_hash->filter_hash =3D NULL; + } else { + err =3D ftrace_update_ops(ops, filter_hash, EMPTY_HASH); + } + + free_hash =3D direct_functions; + rcu_assign_pointer(direct_functions, new_hash); + + out_unlock: + mutex_unlock(&direct_mutex); + + if (free_hash && free_hash !=3D EMPTY_HASH) + call_rcu_tasks(&free_hash->rcu, register_ftrace_direct_cb); + if (filter_hash) + free_ftrace_hash(filter_hash); + + return err; +} +EXPORT_SYMBOL_GPL(unregister_ftrace_direct_hash); + #endif /* CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS */ =20 /** --=20 2.51.0 From nobody Thu Oct 2 01:57:53 2025 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-1.web.codeaurora.org [10.30.226.201]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 0FCF630F95C; Tue, 23 Sep 2025 21:52:41 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=10.30.226.201 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1758664362; cv=none; b=pTH5A4kJgcLeLMVnOuQXAp/WQWbgFrJQ/3LMkXBaebcQd9SpYrc9IajR6hajJEi7S6ibBtzlpcfANawCNXojhBBlAsHpdtWri1XIARnPTBU08T5K364EFhRtQ8RyRg2oYGBV5MKDDHpKXdpHV8dB1nZU5FbxfImEuvPh23LcURM= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1758664362; c=relaxed/simple; bh=pmgvSXZ8M3DblJ5mReSvIJZRwy/X4JecZikM42Xh1AE=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=BYVOIKatCp/uFTgsshaLuRrrM373NJlMtIJ/P+p+Wr6xO1cxDEn92i3h+yFkrXAn8DEoTOBc464YWzmQejJZAKXIrR4K4hsbCCZdrhKb2vx+JJsCCwJrd1oXKRIzBwZB8Joiqe3AW7bDwmbDLRDe/fwTVZa+ceCwtxb8pnpDcrA= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=bWaFgWqA; arc=none smtp.client-ip=10.30.226.201 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="bWaFgWqA" Received: by smtp.kernel.org (Postfix) with ESMTPSA id A2F2BC4CEF5; Tue, 23 Sep 2025 21:52:38 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1758664361; bh=pmgvSXZ8M3DblJ5mReSvIJZRwy/X4JecZikM42Xh1AE=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=bWaFgWqA0PP9vYEvVPItBeOobenZz3qoM+P2SNYzAZs15aInDppLM96GRRkH+GaAr 2ErT/bv9j+Tay0R7AfvXMWXQbds/ipp02jc51MKsd+U6FSdeb8UDwWZJ1B7OyO4tpd kvY1LAknkNHDrCMSWzTGcJvTOUpP7u+mZ353V7y/qut2do2VpvOmGz8btOj03VgA7X 1mgrQLbs7OX7QiHzZTAiRJGevmLXX9r6kPEywUJ2P5L9Rnnz6TRMicB1ABf1zjVijp TzniDmpUdBRpNclKKUpFaF8Rk/hIDSY97XXPYHZMMbiVM29BxgE2MVUuoMg+UPK7kg ycdOUo9yrhCoQ== From: Jiri Olsa To: Steven Rostedt , Florent Revest , Mark Rutland Cc: bpf@vger.kernel.org, linux-kernel@vger.kernel.org, linux-trace-kernel@vger.kernel.org, linux-arm-kernel@lists.infradead.org, Alexei Starovoitov , Daniel Borkmann , Andrii Nakryiko , Menglong Dong Subject: [PATCH 4/9] ftrace: Add modify_ftrace_direct_hash function Date: Tue, 23 Sep 2025 23:51:42 +0200 Message-ID: <20250923215147.1571952-5-jolsa@kernel.org> X-Mailer: git-send-email 2.51.0 In-Reply-To: <20250923215147.1571952-1-jolsa@kernel.org> References: <20250923215147.1571952-1-jolsa@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Adding modify_ftrace_direct_hash function that modifies all entries (ip -> direct) provided in hash argument. The difference to current unregister_ftrace_direct is - hash argument that allows to modify multiple ip -> direct entries at once This change will allow us to have simple ftrace_ops for all bpf direct interface users in following changes. Signed-off-by: Jiri Olsa --- include/linux/ftrace.h | 6 +++++ kernel/trace/ftrace.c | 58 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+) diff --git a/include/linux/ftrace.h b/include/linux/ftrace.h index 55f5ead5d4ff..1cf6d0bb9a81 100644 --- a/include/linux/ftrace.h +++ b/include/linux/ftrace.h @@ -528,6 +528,7 @@ int modify_ftrace_direct_nolock(struct ftrace_ops *ops,= unsigned long addr); =20 int register_ftrace_direct_hash(struct ftrace_ops *ops, struct ftrace_hash= *hash); int unregister_ftrace_direct_hash(struct ftrace_ops *ops, struct ftrace_ha= sh *hash); +int modify_ftrace_direct_hash(struct ftrace_ops *ops, struct ftrace_hash *= hash, bool do_direct_lock); =20 void ftrace_stub_direct_tramp(void); =20 @@ -565,6 +566,11 @@ int unregister_ftrace_direct_hash(struct ftrace_ops *o= ps, struct ftrace_hash *ha return -ENODEV; } =20 +int modify_ftrace_direct_hash(struct ftrace_ops *ops, struct ftrace_hash *= hash, bool do_direct_lock) +{ + return -ENODEV; +} + /* * This must be implemented by the architecture. * It is the way the ftrace direct_ops helper, when called diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c index ab5739f72933..4eb08817e4ee 100644 --- a/kernel/trace/ftrace.c +++ b/kernel/trace/ftrace.c @@ -6427,6 +6427,64 @@ int unregister_ftrace_direct_hash(struct ftrace_ops = *ops, struct ftrace_hash *ha } EXPORT_SYMBOL_GPL(unregister_ftrace_direct_hash); =20 +int modify_ftrace_direct_hash(struct ftrace_ops *ops, struct ftrace_hash *= hash, bool do_direct_lock) +{ + struct ftrace_func_entry *entry, *tmp; + static struct ftrace_ops tmp_ops =3D { + .func =3D ftrace_stub, + .flags =3D FTRACE_OPS_FL_STUB, + }; + unsigned long size, i; + int err; + + if (!hash_count(hash)) + return 0; + if (check_direct_multi(ops)) + return -EINVAL; + if (!(ops->flags & FTRACE_OPS_FL_ENABLED)) + return -EINVAL; + if (direct_functions =3D=3D EMPTY_HASH) + return -EINVAL; + + if (do_direct_lock) + mutex_lock(&direct_mutex); + + /* Enable the tmp_ops to have the same functions as the direct ops */ + ftrace_ops_init(&tmp_ops); + tmp_ops.func_hash =3D ops->func_hash; + + err =3D register_ftrace_function_nolock(&tmp_ops); + if (err) + goto unlock; + + /* + * Now the ftrace_ops_list_func() is called to do the direct callers. + * We can safely change the direct functions attached to each entry. + */ + mutex_lock(&ftrace_lock); + + size =3D 1 << hash->size_bits; + for (i =3D 0; i < size; i++) { + hlist_for_each_entry(entry, &hash->buckets[i], hlist) { + tmp =3D __ftrace_lookup_ip(direct_functions, entry->ip); + if (!tmp) + continue; + tmp->direct =3D entry->direct; + } + } + + mutex_unlock(&ftrace_lock); + + /* Removing the tmp_ops will add the updated direct callers to the functi= ons */ + unregister_ftrace_function(&tmp_ops); + +unlock: + if (do_direct_lock) + mutex_unlock(&direct_mutex); + return err; +} +EXPORT_SYMBOL_GPL(modify_ftrace_direct_hash); + #endif /* CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS */ =20 /** --=20 2.51.0 From nobody Thu Oct 2 01:57:53 2025 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-1.web.codeaurora.org [10.30.226.201]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 10AB130FC36; Tue, 23 Sep 2025 21:52:52 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=10.30.226.201 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1758664373; cv=none; b=c1PIFsm0kz/9MYGzjBbBQOsmhXhewQabHfSm+hO039kEgdh/RzT3PLfXXylNatasBidQNVY5KOyCNd51hyeOEEUR1JlLzi+hhaEWbPE8OsHZVfUjSEsYIzS/PcoDr0H89EPBFRB179c4yNw8MqNEVVVrq4ZF2hiY2+u7skS8m6M= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1758664373; c=relaxed/simple; bh=/3umJShdbnhAxPz6ow/VdTepJxIa2+Qo1lw/fdFc2zQ=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=uvhog3Cn92QT/w7m1TKMwm5nGt+7SMJu9L3LaC80kicTYHtBV2U0WB2G8kzOvTvO9V0AUrL/I6TS2L873179OnQ+zE94qmIiUqoP84ECasPQCWDw/jyC8o9w9CZ2U+Y7p7KXH8d1RH8DIYD05cafUPy4MspGl84lBQCGguGSHBE= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=ENSO36TM; arc=none smtp.client-ip=10.30.226.201 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="ENSO36TM" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 4C3D2C4CEF5; Tue, 23 Sep 2025 21:52:49 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1758664372; bh=/3umJShdbnhAxPz6ow/VdTepJxIa2+Qo1lw/fdFc2zQ=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=ENSO36TMfNdTqWne6KZSr2xujAgaqO/1HFu/e7x2DWmnSVQVRcSiuLT6WDKaLGUv4 s295SRO1S6IxfDWxqOotwhjpm9AEMgMk+IthEpy/2iXCyDIFcM+9DjUZxEqEQZ/yX/ CSfFHxlUoOrHSnQpUNcJ5cTh5BGsoGAMJkNJ4konyK1EYFWnc0zkzpv43SmeQJA7e7 YcgKOCSIA3AKXLWMoeOrE5pq20hBeg7t6SzA73RsFaFaTywUsKWsATqsEOdwUg7Qou 2MM5SXH93urieyBXMcTAQ3aIII7Z5K1KdnYgHSXARDe5D6RIVKR5ROlrmcA/4Zzrix LGoF+Nin2S8tg== From: Jiri Olsa To: Steven Rostedt , Florent Revest , Mark Rutland Cc: bpf@vger.kernel.org, linux-kernel@vger.kernel.org, linux-trace-kernel@vger.kernel.org, linux-arm-kernel@lists.infradead.org, Alexei Starovoitov , Daniel Borkmann , Andrii Nakryiko , Menglong Dong Subject: [PATCH 5/9] ftrace: Export some of hash related functions Date: Tue, 23 Sep 2025 23:51:43 +0200 Message-ID: <20250923215147.1571952-6-jolsa@kernel.org> X-Mailer: git-send-email 2.51.0 In-Reply-To: <20250923215147.1571952-1-jolsa@kernel.org> References: <20250923215147.1571952-1-jolsa@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" We are going to use these functions in following changes. Signed-off-by: Jiri Olsa --- include/linux/ftrace.h | 16 ++++++++++++++++ kernel/trace/ftrace.c | 7 +++---- kernel/trace/trace.h | 8 -------- 3 files changed, 19 insertions(+), 12 deletions(-) diff --git a/include/linux/ftrace.h b/include/linux/ftrace.h index 1cf6d0bb9a81..a8e351df3a52 100644 --- a/include/linux/ftrace.h +++ b/include/linux/ftrace.h @@ -397,6 +397,22 @@ enum ftrace_ops_cmd { typedef int (*ftrace_ops_func_t)(struct ftrace_ops *op, enum ftrace_ops_cm= d cmd); =20 #ifdef CONFIG_DYNAMIC_FTRACE + +#define FTRACE_HASH_DEFAULT_BITS 10 + +struct ftrace_hash { + unsigned long size_bits; + struct hlist_head *buckets; + unsigned long count; + unsigned long flags; + struct rcu_head rcu; +}; + +struct ftrace_hash *alloc_ftrace_hash(int size_bits); +void free_ftrace_hash(struct ftrace_hash *hash); +struct ftrace_func_entry *add_hash_entry_direct(struct ftrace_hash *hash, + unsigned long ip, unsigned long direct); + /* The hash used to know what functions callbacks trace */ struct ftrace_ops_hash { struct ftrace_hash __rcu *notrace_hash; diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c index 4eb08817e4ee..75bea44e4f8e 100644 --- a/kernel/trace/ftrace.c +++ b/kernel/trace/ftrace.c @@ -68,7 +68,6 @@ }) =20 /* hash bits for specific function selection */ -#define FTRACE_HASH_DEFAULT_BITS 10 #define FTRACE_HASH_MAX_BITS 12 =20 #ifdef CONFIG_DYNAMIC_FTRACE @@ -1185,7 +1184,7 @@ static void __add_hash_entry(struct ftrace_hash *hash, hash->count++; } =20 -static struct ftrace_func_entry * +struct ftrace_func_entry * add_hash_entry_direct(struct ftrace_hash *hash, unsigned long ip, unsigned= long direct) { struct ftrace_func_entry *entry; @@ -1265,7 +1264,7 @@ static void clear_ftrace_mod_list(struct list_head *h= ead) mutex_unlock(&ftrace_lock); } =20 -static void free_ftrace_hash(struct ftrace_hash *hash) +void free_ftrace_hash(struct ftrace_hash *hash) { if (!hash || hash =3D=3D EMPTY_HASH) return; @@ -1305,7 +1304,7 @@ void ftrace_free_filter(struct ftrace_ops *ops) } EXPORT_SYMBOL_GPL(ftrace_free_filter); =20 -static struct ftrace_hash *alloc_ftrace_hash(int size_bits) +struct ftrace_hash *alloc_ftrace_hash(int size_bits) { struct ftrace_hash *hash; int size; diff --git a/kernel/trace/trace.h b/kernel/trace/trace.h index 5f4bed5842f9..5070aafa590c 100644 --- a/kernel/trace/trace.h +++ b/kernel/trace/trace.h @@ -899,14 +899,6 @@ enum { FTRACE_HASH_FL_MOD =3D (1 << 0), }; =20 -struct ftrace_hash { - unsigned long size_bits; - struct hlist_head *buckets; - unsigned long count; - unsigned long flags; - struct rcu_head rcu; -}; - struct ftrace_func_entry * ftrace_lookup_ip(struct ftrace_hash *hash, unsigned long ip); =20 --=20 2.51.0 From nobody Thu Oct 2 01:57:53 2025 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-1.web.codeaurora.org [10.30.226.201]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 0E4C030CB20; Tue, 23 Sep 2025 21:53:03 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=10.30.226.201 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1758664384; cv=none; b=N0pUEbk2qGqmjw5+rKDeuFktk6PQ4lcaVHiar4VDBz6YXyKfM6wcAbqqZkSX+yOi030inJL2d2CuEoJehOoENwkNO0XMxyUjsqgLdkyv4+z3e6BvgxH/fwhJGIPpLpNr+LUVqnrsJJs+mE0Gqk+T+Yz93U2r4mv7jnS/R6+OJDo= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1758664384; c=relaxed/simple; bh=Od/250IBHM1cPiNEf2fed2b1+BCC097uzTTqHeA3ouc=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=NUu7heHfhbT0JHNoR3QWgJ431CpEa+Peg39/g/p5bnv4Iu1/L8UedzDFKevQZ5QhesGdnWlvYODq3KJKLyCpKtz/FBu/JZ5BOGPv2PLwdkP3ar+tPDrDrZm2syWmMYBh1c5+6C1re6036VhO5NAg7UNzbl6IV8eaqJx1ONOWBKQ= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=NWfMHYYa; arc=none smtp.client-ip=10.30.226.201 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="NWfMHYYa" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 51D0CC4CEF5; Tue, 23 Sep 2025 21:53:00 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1758664383; bh=Od/250IBHM1cPiNEf2fed2b1+BCC097uzTTqHeA3ouc=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=NWfMHYYaiTs9lsBWqQ1w5EDeBCgIGBGRjR/kNBQ9pXZv4RrSzuuJH/McPFOfUj8JD hq1mThRS5+tXnRscMOTNFLhLmIe/W2NzUHjhcgrGMvH8cm6dUByLE0O7N+POIikAu9 0vIeYkEtEquGPv4LQhyzQZE+8bUBVkR6l4rxZeqZvPyONErfFmE5ksxStBeWI34W2T I7/aZSqO7I7qvaDARLNFgFuzYTUPWIYv2F+Snvg/JgobHoJ57BkUxh6s8btsPVkPv0 HSLD/ASrxlI2QZUgZfbKI9xZtyB0P5SoRERsxfe0E3BMrO2nJcoxx4WWYySBAXcmhj ER1k13RjplTqA== From: Jiri Olsa To: Steven Rostedt , Florent Revest , Mark Rutland Cc: bpf@vger.kernel.org, linux-kernel@vger.kernel.org, linux-trace-kernel@vger.kernel.org, linux-arm-kernel@lists.infradead.org, Alexei Starovoitov , Daniel Borkmann , Andrii Nakryiko , Menglong Dong Subject: [PATCH 6/9] ftrace: Use direct hash interface in direct functions Date: Tue, 23 Sep 2025 23:51:44 +0200 Message-ID: <20250923215147.1571952-7-jolsa@kernel.org> X-Mailer: git-send-email 2.51.0 In-Reply-To: <20250923215147.1571952-1-jolsa@kernel.org> References: <20250923215147.1571952-1-jolsa@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Implement current *_ftrace_direct function with their *_hash function counterparts. Signed-off-by: Jiri Olsa --- include/linux/ftrace.h | 17 +-- kernel/bpf/trampoline.c | 10 +- kernel/trace/ftrace.c | 232 +++++----------------------------- kernel/trace/trace_selftest.c | 5 +- 4 files changed, 45 insertions(+), 219 deletions(-) diff --git a/include/linux/ftrace.h b/include/linux/ftrace.h index a8e351df3a52..42da4dd08759 100644 --- a/include/linux/ftrace.h +++ b/include/linux/ftrace.h @@ -536,11 +536,10 @@ struct ftrace_func_entry { =20 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS unsigned long ftrace_find_rec_direct(unsigned long ip); -int register_ftrace_direct(struct ftrace_ops *ops, unsigned long addr); -int unregister_ftrace_direct(struct ftrace_ops *ops, unsigned long addr, +int register_ftrace_direct(struct ftrace_ops *ops, unsigned long ip, unsig= ned long addr); +int unregister_ftrace_direct(struct ftrace_ops *ops, unsigned long ip, uns= igned long addr, bool free_filters); -int modify_ftrace_direct(struct ftrace_ops *ops, unsigned long addr); -int modify_ftrace_direct_nolock(struct ftrace_ops *ops, unsigned long addr= ); +int modify_ftrace_direct(struct ftrace_ops *ops, unsigned long ip, unsigne= d long addr, bool lock_direct_mutex); =20 int register_ftrace_direct_hash(struct ftrace_ops *ops, struct ftrace_hash= *hash); int unregister_ftrace_direct_hash(struct ftrace_ops *ops, struct ftrace_ha= sh *hash); @@ -554,20 +553,16 @@ static inline unsigned long ftrace_find_rec_direct(un= signed long ip) { return 0; } -static inline int register_ftrace_direct(struct ftrace_ops *ops, unsigned = long addr) +static inline int register_ftrace_direct(struct ftrace_ops *ops, unsigned = long ip, unsigned long addr) { return -ENODEV; } -static inline int unregister_ftrace_direct(struct ftrace_ops *ops, unsigne= d long addr, +static inline int unregister_ftrace_direct(struct ftrace_ops *ops, unsigne= d long ip, unsigned long addr, bool free_filters) { return -ENODEV; } -static inline int modify_ftrace_direct(struct ftrace_ops *ops, unsigned lo= ng addr) -{ - return -ENODEV; -} -static inline int modify_ftrace_direct_nolock(struct ftrace_ops *ops, unsi= gned long addr) +static inline int modify_ftrace_direct(struct ftrace_ops *ops, unsigned lo= ng ip, unsigned long addr, bool lock_direct_mutex) { return -ENODEV; } diff --git a/kernel/bpf/trampoline.c b/kernel/bpf/trampoline.c index 5949095e51c3..97df6e482bf4 100644 --- a/kernel/bpf/trampoline.c +++ b/kernel/bpf/trampoline.c @@ -181,7 +181,7 @@ static int unregister_fentry(struct bpf_trampoline *tr,= void *old_addr) int ret; =20 if (tr->func.ftrace_managed) - ret =3D unregister_ftrace_direct(tr->fops, (long)old_addr, false); + ret =3D unregister_ftrace_direct(tr->fops, (unsigned long) ip, (long)old= _addr, false); else ret =3D bpf_arch_text_poke(ip, BPF_MOD_CALL, old_addr, NULL); =20 @@ -195,10 +195,7 @@ static int modify_fentry(struct bpf_trampoline *tr, vo= id *old_addr, void *new_ad int ret; =20 if (tr->func.ftrace_managed) { - if (lock_direct_mutex) - ret =3D modify_ftrace_direct(tr->fops, (long)new_addr); - else - ret =3D modify_ftrace_direct_nolock(tr->fops, (long)new_addr); + ret =3D modify_ftrace_direct(tr->fops, (unsigned long) ip, (long)new_add= r, lock_direct_mutex); } else { ret =3D bpf_arch_text_poke(ip, BPF_MOD_CALL, old_addr, new_addr); } @@ -220,8 +217,7 @@ static int register_fentry(struct bpf_trampoline *tr, v= oid *new_addr) } =20 if (tr->func.ftrace_managed) { - ftrace_set_filter_ip(tr->fops, (unsigned long)ip, 0, 1); - ret =3D register_ftrace_direct(tr->fops, (long)new_addr); + ret =3D register_ftrace_direct(tr->fops, (unsigned long)ip, (long)new_ad= dr); } else { ret =3D bpf_arch_text_poke(ip, BPF_MOD_CALL, NULL, new_addr); } diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c index 75bea44e4f8e..6e9d5975477d 100644 --- a/kernel/trace/ftrace.c +++ b/kernel/trace/ftrace.c @@ -5935,28 +5935,24 @@ static int check_direct_multi(struct ftrace_ops *op= s) return 0; } =20 -static void remove_direct_functions_hash(struct ftrace_hash *hash, unsigne= d long addr) +static void register_ftrace_direct_cb(struct rcu_head *rhp) { - struct ftrace_func_entry *entry, *del; - int size, i; + struct ftrace_hash *fhp =3D container_of(rhp, struct ftrace_hash, rcu); =20 - size =3D 1 << hash->size_bits; - for (i =3D 0; i < size; i++) { - hlist_for_each_entry(entry, &hash->buckets[i], hlist) { - del =3D __ftrace_lookup_ip(direct_functions, entry->ip); - if (del && del->direct =3D=3D addr) { - remove_hash_entry(direct_functions, del); - kfree(del); - } - } - } + free_ftrace_hash(fhp); } =20 -static void register_ftrace_direct_cb(struct rcu_head *rhp) +static struct ftrace_hash *hash_from_ip(unsigned long ip, unsigned long ad= dr) { - struct ftrace_hash *fhp =3D container_of(rhp, struct ftrace_hash, rcu); + struct ftrace_hash *hash; =20 - free_ftrace_hash(fhp); + ip =3D ftrace_location(ip); + if (!ip) + return NULL; + hash =3D alloc_ftrace_hash(FTRACE_HASH_DEFAULT_BITS); + if (!hash || !add_hash_entry_direct(hash, ip, addr)) + return NULL; + return hash; } =20 /** @@ -5981,89 +5977,17 @@ static void register_ftrace_direct_cb(struct rcu_he= ad *rhp) * -ENODEV - @ip does not point to a ftrace nop location (or not support= ed) * -ENOMEM - There was an allocation failure. */ -int register_ftrace_direct(struct ftrace_ops *ops, unsigned long addr) +int register_ftrace_direct(struct ftrace_ops *ops, unsigned long ip, unsig= ned long addr) { - struct ftrace_hash *hash, *new_hash =3D NULL, *free_hash =3D NULL; - struct ftrace_func_entry *entry, *new; - int err =3D -EBUSY, size, i; - - if (ops->func || ops->trampoline) - return -EINVAL; - if (!(ops->flags & FTRACE_OPS_FL_INITIALIZED)) - return -EINVAL; - if (ops->flags & FTRACE_OPS_FL_ENABLED) - return -EINVAL; - - hash =3D ops->func_hash->filter_hash; - if (ftrace_hash_empty(hash)) - return -EINVAL; - - mutex_lock(&direct_mutex); - - /* Make sure requested entries are not already registered.. */ - size =3D 1 << hash->size_bits; - for (i =3D 0; i < size; i++) { - hlist_for_each_entry(entry, &hash->buckets[i], hlist) { - if (ftrace_find_rec_direct(entry->ip)) - goto out_unlock; - } - } - - err =3D -ENOMEM; - - /* Make a copy hash to place the new and the old entries in */ - size =3D hash->count + direct_functions->count; - size =3D fls(size); - if (size > FTRACE_HASH_MAX_BITS) - size =3D FTRACE_HASH_MAX_BITS; - new_hash =3D alloc_ftrace_hash(size); - if (!new_hash) - goto out_unlock; - - /* Now copy over the existing direct entries */ - size =3D 1 << direct_functions->size_bits; - for (i =3D 0; i < size; i++) { - hlist_for_each_entry(entry, &direct_functions->buckets[i], hlist) { - new =3D add_hash_entry(new_hash, entry->ip); - if (!new) - goto out_unlock; - new->direct =3D entry->direct; - } - } - - /* ... and add the new entries */ - size =3D 1 << hash->size_bits; - for (i =3D 0; i < size; i++) { - hlist_for_each_entry(entry, &hash->buckets[i], hlist) { - new =3D add_hash_entry(new_hash, entry->ip); - if (!new) - goto out_unlock; - /* Update both the copy and the hash entry */ - new->direct =3D addr; - entry->direct =3D addr; - } - } - - free_hash =3D direct_functions; - rcu_assign_pointer(direct_functions, new_hash); - new_hash =3D NULL; - - ops->func =3D call_direct_funcs; - ops->flags =3D MULTI_FLAGS; - ops->trampoline =3D FTRACE_REGS_ADDR; - ops->direct_call =3D addr; - - err =3D register_ftrace_function_nolock(ops); - - out_unlock: - mutex_unlock(&direct_mutex); - - if (free_hash && free_hash !=3D EMPTY_HASH) - call_rcu_tasks(&free_hash->rcu, register_ftrace_direct_cb); + struct ftrace_hash *hash; + int err; =20 - if (new_hash) - free_ftrace_hash(new_hash); + hash =3D hash_from_ip(ip, addr); + if (!hash) + return -ENOMEM; =20 + err =3D register_ftrace_direct_hash(ops, hash); + free_ftrace_hash(hash); return err; } EXPORT_SYMBOL_GPL(register_ftrace_direct); @@ -6083,111 +6007,24 @@ EXPORT_SYMBOL_GPL(register_ftrace_direct); * 0 on success * -EINVAL - The @ops object was not properly registered. */ -int unregister_ftrace_direct(struct ftrace_ops *ops, unsigned long addr, +int unregister_ftrace_direct(struct ftrace_ops *ops, unsigned long ip, uns= igned long addr, bool free_filters) { - struct ftrace_hash *hash =3D ops->func_hash->filter_hash; + struct ftrace_hash *hash; int err; =20 - if (check_direct_multi(ops)) - return -EINVAL; - if (!(ops->flags & FTRACE_OPS_FL_ENABLED)) - return -EINVAL; - - mutex_lock(&direct_mutex); - err =3D unregister_ftrace_function(ops); - remove_direct_functions_hash(hash, addr); - mutex_unlock(&direct_mutex); - - /* cleanup for possible another register call */ - ops->func =3D NULL; - ops->trampoline =3D 0; + hash =3D hash_from_ip(ip, addr); + if (!hash) + return -ENOMEM; =20 + err =3D unregister_ftrace_direct_hash(ops, hash); + free_ftrace_hash(hash); if (free_filters) ftrace_free_filter(ops); return err; } EXPORT_SYMBOL_GPL(unregister_ftrace_direct); =20 -static int -__modify_ftrace_direct(struct ftrace_ops *ops, unsigned long addr) -{ - struct ftrace_hash *hash; - struct ftrace_func_entry *entry, *iter; - static struct ftrace_ops tmp_ops =3D { - .func =3D ftrace_stub, - .flags =3D FTRACE_OPS_FL_STUB, - }; - int i, size; - int err; - - lockdep_assert_held_once(&direct_mutex); - - /* Enable the tmp_ops to have the same functions as the direct ops */ - ftrace_ops_init(&tmp_ops); - tmp_ops.func_hash =3D ops->func_hash; - tmp_ops.direct_call =3D addr; - - err =3D register_ftrace_function_nolock(&tmp_ops); - if (err) - return err; - - /* - * Now the ftrace_ops_list_func() is called to do the direct callers. - * We can safely change the direct functions attached to each entry. - */ - mutex_lock(&ftrace_lock); - - hash =3D ops->func_hash->filter_hash; - size =3D 1 << hash->size_bits; - for (i =3D 0; i < size; i++) { - hlist_for_each_entry(iter, &hash->buckets[i], hlist) { - entry =3D __ftrace_lookup_ip(direct_functions, iter->ip); - if (!entry) - continue; - entry->direct =3D addr; - } - } - /* Prevent store tearing if a trampoline concurrently accesses the value = */ - WRITE_ONCE(ops->direct_call, addr); - - mutex_unlock(&ftrace_lock); - - /* Removing the tmp_ops will add the updated direct callers to the functi= ons */ - unregister_ftrace_function(&tmp_ops); - - return err; -} - -/** - * modify_ftrace_direct_nolock - Modify an existing direct 'multi' call - * to call something else - * @ops: The address of the struct ftrace_ops object - * @addr: The address of the new trampoline to call at @ops functions - * - * This is used to unregister currently registered direct caller and - * register new one @addr on functions registered in @ops object. - * - * Note there's window between ftrace_shutdown and ftrace_startup calls - * where there will be no callbacks called. - * - * Caller should already have direct_mutex locked, so we don't lock - * direct_mutex here. - * - * Returns: zero on success. Non zero on error, which includes: - * -EINVAL - The @ops object was not properly registered. - */ -int modify_ftrace_direct_nolock(struct ftrace_ops *ops, unsigned long addr) -{ - if (check_direct_multi(ops)) - return -EINVAL; - if (!(ops->flags & FTRACE_OPS_FL_ENABLED)) - return -EINVAL; - - return __modify_ftrace_direct(ops, addr); -} -EXPORT_SYMBOL_GPL(modify_ftrace_direct_nolock); - /** * modify_ftrace_direct - Modify an existing direct 'multi' call * to call something else @@ -6203,18 +6040,17 @@ EXPORT_SYMBOL_GPL(modify_ftrace_direct_nolock); * Returns: zero on success. Non zero on error, which includes: * -EINVAL - The @ops object was not properly registered. */ -int modify_ftrace_direct(struct ftrace_ops *ops, unsigned long addr) +int modify_ftrace_direct(struct ftrace_ops *ops, unsigned long ip, unsigne= d long addr, bool lock_direct_mutex) { + struct ftrace_hash *hash; int err; =20 - if (check_direct_multi(ops)) - return -EINVAL; - if (!(ops->flags & FTRACE_OPS_FL_ENABLED)) - return -EINVAL; + hash =3D hash_from_ip(ip, addr); + if (!hash) + return -ENOMEM; =20 - mutex_lock(&direct_mutex); - err =3D __modify_ftrace_direct(ops, addr); - mutex_unlock(&direct_mutex); + err =3D modify_ftrace_direct_hash(ops, hash, lock_direct_mutex); + free_ftrace_hash(hash); return err; } EXPORT_SYMBOL_GPL(modify_ftrace_direct); diff --git a/kernel/trace/trace_selftest.c b/kernel/trace/trace_selftest.c index d88c44f1dfa5..37f5eb1f252b 100644 --- a/kernel/trace/trace_selftest.c +++ b/kernel/trace/trace_selftest.c @@ -1135,8 +1135,7 @@ trace_selftest_startup_function_graph(struct tracer *= trace, * Register direct function together with graph tracer * and make sure we get graph trace. */ - ftrace_set_filter_ip(&direct, (unsigned long)DYN_FTRACE_TEST_NAME, 0, 0); - ret =3D register_ftrace_direct(&direct, + ret =3D register_ftrace_direct(&direct, (unsigned long)DYN_FTRACE_TEST_NA= ME, (unsigned long)ftrace_stub_direct_tramp); if (ret) goto out; @@ -1159,7 +1158,7 @@ trace_selftest_startup_function_graph(struct tracer *= trace, =20 unregister_ftrace_graph(&fgraph_ops); =20 - ret =3D unregister_ftrace_direct(&direct, + ret =3D unregister_ftrace_direct(&direct, (unsigned long)DYN_FTRACE_TEST_= NAME, (unsigned long)ftrace_stub_direct_tramp, true); if (ret) --=20 2.51.0 From nobody Thu Oct 2 01:57:53 2025 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-1.web.codeaurora.org [10.30.226.201]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 7E9AF30E82D; Tue, 23 Sep 2025 21:53:14 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=10.30.226.201 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1758664394; cv=none; b=kRhE0x9v4tEceS1VyGRbLQC8rojVGjP3IpSqN907tkqNTvgFAFoO6E8sKK6iKOLJFSVKwA5k7WOtYbvjGuSXPYjplC6a2kA+r2WrvReRecMbrhkJJv6VNSQzU9fxFN18XILTNOyvw/CNG3a9if9aVmqsi0ZPO5ye5B2iV6grqdc= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1758664394; c=relaxed/simple; bh=Uwsp9gplgTwpqAp66Yn02rMHn1hEFutbsYgZHk2p0JI=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=n0qUCz+QCmOP3MyTw5RyoF2CHODwcy1i2rBHZUdj/bPeiDBaq7ZJ/rvZrslojgY+304OFClQrlD5zsAUUAnwzUePww1PIo6WLgRtOcxfuhBhXWa+caZtgzT3cLRRezbGHP/D0xg2n9+oF4vJxycZWah9OwRp7iBBizVhIgc6+ws= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=T0us82hA; arc=none smtp.client-ip=10.30.226.201 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="T0us82hA" Received: by smtp.kernel.org (Postfix) with ESMTPSA id F2B7AC4CEF5; Tue, 23 Sep 2025 21:53:10 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1758664394; bh=Uwsp9gplgTwpqAp66Yn02rMHn1hEFutbsYgZHk2p0JI=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=T0us82hAVX0ap4TCL9y8VXnNkg3BBkCEkOQkqxp31s/Pwt//JL+mQ5hV1mi5bghks IkumBZXCGU+NK3pTDJ4+8DATUQCmSJVXtV/Rp29s8Zu2C+qL3CToFvYHJ5ue91hOL0 PPXOWXk5eTWCVt5rniQ1Xf+0irjmBjyM9DNymy5FoPmH5n1/jZROqCUG8kMzTUtMV0 ETZ9TXLYRshVwuDv5nH2O26fVCuQsoCiVhXWJCLcgTogRn0z/CAAww/U6jS4eCMYQo HHHNUcZpAOmN+jG6Do7eOTeNDaqjksdJfK2OUhFjqRstf4Agh37WV5DuK3PKSon74O vT0ZRk0n8/eoA== From: Jiri Olsa To: Steven Rostedt , Florent Revest , Mark Rutland Cc: bpf@vger.kernel.org, linux-kernel@vger.kernel.org, linux-trace-kernel@vger.kernel.org, linux-arm-kernel@lists.infradead.org, Alexei Starovoitov , Daniel Borkmann , Andrii Nakryiko , Menglong Dong Subject: [PATCH 7/9] bpf: Add trampoline ip hash table Date: Tue, 23 Sep 2025 23:51:45 +0200 Message-ID: <20250923215147.1571952-8-jolsa@kernel.org> X-Mailer: git-send-email 2.51.0 In-Reply-To: <20250923215147.1571952-1-jolsa@kernel.org> References: <20250923215147.1571952-1-jolsa@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Following changes need to lookup trampoline based on its ip address, adding hash table for that. Signed-off-by: Jiri Olsa --- include/linux/bpf.h | 7 +++++-- kernel/bpf/trampoline.c | 30 +++++++++++++++++++----------- 2 files changed, 24 insertions(+), 13 deletions(-) diff --git a/include/linux/bpf.h b/include/linux/bpf.h index dfc1a27b56d5..9d97c64cbf47 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -1271,14 +1271,17 @@ struct bpf_tramp_image { }; =20 struct bpf_trampoline { - /* hlist for trampoline_table */ - struct hlist_node hlist; + /* hlist for trampoline_key_table */ + struct hlist_node hlist_key; + /* hlist for trampoline_ip_table */ + struct hlist_node hlist_ip; struct ftrace_ops *fops; /* serializes access to fields of this trampoline */ struct mutex mutex; refcount_t refcnt; u32 flags; u64 key; + unsigned long ip; struct { struct btf_func_model model; void *addr; diff --git a/kernel/bpf/trampoline.c b/kernel/bpf/trampoline.c index 97df6e482bf4..bdebaa94ca37 100644 --- a/kernel/bpf/trampoline.c +++ b/kernel/bpf/trampoline.c @@ -24,9 +24,10 @@ const struct bpf_prog_ops bpf_extension_prog_ops =3D { #define TRAMPOLINE_HASH_BITS 10 #define TRAMPOLINE_TABLE_SIZE (1 << TRAMPOLINE_HASH_BITS) =20 -static struct hlist_head trampoline_table[TRAMPOLINE_TABLE_SIZE]; +static struct hlist_head trampoline_key_table[TRAMPOLINE_TABLE_SIZE]; +static struct hlist_head trampoline_ip_table[TRAMPOLINE_TABLE_SIZE]; =20 -/* serializes access to trampoline_table */ +/* serializes access to trampoline tables */ static DEFINE_MUTEX(trampoline_mutex); =20 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS @@ -135,15 +136,15 @@ void bpf_image_ksym_del(struct bpf_ksym *ksym) PAGE_SIZE, true, ksym->name); } =20 -static struct bpf_trampoline *bpf_trampoline_lookup(u64 key) +static struct bpf_trampoline *bpf_trampoline_lookup(u64 key, unsigned long= ip) { struct bpf_trampoline *tr; struct hlist_head *head; int i; =20 mutex_lock(&trampoline_mutex); - head =3D &trampoline_table[hash_64(key, TRAMPOLINE_HASH_BITS)]; - hlist_for_each_entry(tr, head, hlist) { + head =3D &trampoline_key_table[hash_64(key, TRAMPOLINE_HASH_BITS)]; + hlist_for_each_entry(tr, head, hlist_key) { if (tr->key =3D=3D key) { refcount_inc(&tr->refcnt); goto out; @@ -164,8 +165,12 @@ static struct bpf_trampoline *bpf_trampoline_lookup(u6= 4 key) #endif =20 tr->key =3D key; - INIT_HLIST_NODE(&tr->hlist); - hlist_add_head(&tr->hlist, head); + tr->ip =3D ip; + INIT_HLIST_NODE(&tr->hlist_key); + INIT_HLIST_NODE(&tr->hlist_ip); + hlist_add_head(&tr->hlist_key, head); + head =3D &trampoline_ip_table[hash_64(ip, TRAMPOLINE_HASH_BITS)]; + hlist_add_head(&tr->hlist_ip, head); refcount_set(&tr->refcnt, 1); mutex_init(&tr->mutex); for (i =3D 0; i < BPF_TRAMP_MAX; i++) @@ -800,7 +805,7 @@ void bpf_trampoline_unlink_cgroup_shim(struct bpf_prog = *prog) prog->aux->attach_btf_id); =20 bpf_lsm_find_cgroup_shim(prog, &bpf_func); - tr =3D bpf_trampoline_lookup(key); + tr =3D bpf_trampoline_lookup(key, 0); if (WARN_ON_ONCE(!tr)) return; =20 @@ -820,7 +825,7 @@ struct bpf_trampoline *bpf_trampoline_get(u64 key, { struct bpf_trampoline *tr; =20 - tr =3D bpf_trampoline_lookup(key); + tr =3D bpf_trampoline_lookup(key, tgt_info->tgt_addr); if (!tr) return NULL; =20 @@ -856,7 +861,8 @@ void bpf_trampoline_put(struct bpf_trampoline *tr) * fexit progs. The fentry-only trampoline will be freed via * multiple rcu callbacks. */ - hlist_del(&tr->hlist); + hlist_del(&tr->hlist_key); + hlist_del(&tr->hlist_ip); if (tr->fops) { ftrace_free_filter(tr->fops); kfree(tr->fops); @@ -1129,7 +1135,9 @@ static int __init init_trampolines(void) int i; =20 for (i =3D 0; i < TRAMPOLINE_TABLE_SIZE; i++) - INIT_HLIST_HEAD(&trampoline_table[i]); + INIT_HLIST_HEAD(&trampoline_key_table[i]); + for (i =3D 0; i < TRAMPOLINE_TABLE_SIZE; i++) + INIT_HLIST_HEAD(&trampoline_ip_table[i]); return 0; } late_initcall(init_trampolines); --=20 2.51.0 From nobody Thu Oct 2 01:57:53 2025 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-1.web.codeaurora.org [10.30.226.201]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id D9CE430E82D; Tue, 23 Sep 2025 21:53:24 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=10.30.226.201 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1758664405; cv=none; b=XnfgGHHqcsWgcT7ZArZswbdEcWxkLD++zhGTUabFlk0q4mKV6IIDe/qPwDN9p0k6Lx9FlxHL9RNsaQRnBZZKVFZmTOhkOg6yQrM0mgTkJehmIf2WAZPb/0E2DsHPnXKKNVusLmwHWum4Ub2S1/+f7xoMsxGmI+P3fsTELm0gzDo= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1758664405; c=relaxed/simple; bh=OpzLg/8ZZk8IB4Rmz2mXM8LnzYXPoiDOAj+Xzo3GFkg=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=mHyT8GGQq964UGV+I9zP4c+2KZ5cktJkiupvUEKHweR5UpNsXo3vKvigOeenv2CtVTuRlyeIzSWcs30/WV7Q+r/wFLMQPL0NFqhjGJ9zzYXg1UiFuiNm0dI1rEz2Ja9hSt0m1401WKGZ5CXkyOPyHrihuwjbI7NnWoc/GzRavFg= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=Mh7VPZ0p; arc=none smtp.client-ip=10.30.226.201 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="Mh7VPZ0p" Received: by smtp.kernel.org (Postfix) with ESMTPSA id CD203C4CEF5; Tue, 23 Sep 2025 21:53:21 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1758664404; bh=OpzLg/8ZZk8IB4Rmz2mXM8LnzYXPoiDOAj+Xzo3GFkg=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=Mh7VPZ0ppE54wcUXXtdWf/h4beC0/2JSrie7Qwn/1/Hsvl159FNkdrv6cncih/LT8 TGW0xy+bLrW0b9Q+K8G5+KMpM4lfVH4zTaho2hSERtcoXUm6FKGE4mUJdR9ISk7+Yg kRb2Fx+aukn8sfS4VBpkrJ2tRf5k1TycG7Fg8gyhznkXiinKdsCoi2tZPcIVOmCnop Hwki4MyaA2oSMH0b3VNLUHm/u9zTcAoSz/wO99NPeNKq61oEsh+NkPQWdKIBXwDxjp fo8iWIQpnsGPOzHpT66QneK3cdvN0vgVsu5IVAe8740bH+9WXe6gww7WvFQFap4Qla VIULln/yaqa7Q== From: Jiri Olsa To: Steven Rostedt , Florent Revest , Mark Rutland Cc: bpf@vger.kernel.org, linux-kernel@vger.kernel.org, linux-trace-kernel@vger.kernel.org, linux-arm-kernel@lists.infradead.org, Alexei Starovoitov , Daniel Borkmann , Andrii Nakryiko , Menglong Dong Subject: [PATCH 8/9] ftrace: Factor ftrace_ops ops_func interface Date: Tue, 23 Sep 2025 23:51:46 +0200 Message-ID: <20250923215147.1571952-9-jolsa@kernel.org> X-Mailer: git-send-email 2.51.0 In-Reply-To: <20250923215147.1571952-1-jolsa@kernel.org> References: <20250923215147.1571952-1-jolsa@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" We are going to remove "ftrace_ops->private =3D=3D bpf_trampoline" setup in following changes. Adding ip argument to ftrace_ops_func_t callback function, so we can use it to look up the trampoline. Signed-off-by: Jiri Olsa --- include/linux/ftrace.h | 2 +- kernel/bpf/trampoline.c | 3 ++- kernel/trace/ftrace.c | 6 +++--- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/include/linux/ftrace.h b/include/linux/ftrace.h index 42da4dd08759..98e983d3d6b3 100644 --- a/include/linux/ftrace.h +++ b/include/linux/ftrace.h @@ -394,7 +394,7 @@ enum ftrace_ops_cmd { * Negative on failure. The return value is dependent on the * callback. */ -typedef int (*ftrace_ops_func_t)(struct ftrace_ops *op, enum ftrace_ops_cm= d cmd); +typedef int (*ftrace_ops_func_t)(struct ftrace_ops *op, unsigned long ip, = enum ftrace_ops_cmd cmd); =20 #ifdef CONFIG_DYNAMIC_FTRACE =20 diff --git a/kernel/bpf/trampoline.c b/kernel/bpf/trampoline.c index bdebaa94ca37..3464859189a2 100644 --- a/kernel/bpf/trampoline.c +++ b/kernel/bpf/trampoline.c @@ -33,7 +33,8 @@ static DEFINE_MUTEX(trampoline_mutex); #ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS static int bpf_trampoline_update(struct bpf_trampoline *tr, bool lock_dire= ct_mutex); =20 -static int bpf_tramp_ftrace_ops_func(struct ftrace_ops *ops, enum ftrace_o= ps_cmd cmd) +static int bpf_tramp_ftrace_ops_func(struct ftrace_ops *ops, unsigned long= ip, + enum ftrace_ops_cmd cmd) { struct bpf_trampoline *tr =3D ops->private; int ret =3D 0; diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c index 6e9d5975477d..2af1304d1a83 100644 --- a/kernel/trace/ftrace.c +++ b/kernel/trace/ftrace.c @@ -2036,7 +2036,7 @@ static int __ftrace_hash_update_ipmodify(struct ftrac= e_ops *ops, */ if (!ops->ops_func) return -EBUSY; - ret =3D ops->ops_func(ops, FTRACE_OPS_CMD_ENABLE_SHARE_IPMODIFY_SELF); + ret =3D ops->ops_func(ops, rec->ip, FTRACE_OPS_CMD_ENABLE_SHARE_IPMODI= FY_SELF); if (ret) return ret; } else if (is_ipmodify) { @@ -8740,7 +8740,7 @@ static int prepare_direct_functions_for_ipmodify(stru= ct ftrace_ops *ops) if (!op->ops_func) return -EBUSY; =20 - ret =3D op->ops_func(op, FTRACE_OPS_CMD_ENABLE_SHARE_IPMODIFY_PEER); + ret =3D op->ops_func(op, ip, FTRACE_OPS_CMD_ENABLE_SHARE_IPMODIFY_PEER= ); if (ret) return ret; } @@ -8787,7 +8787,7 @@ static void cleanup_direct_functions_after_ipmodify(s= truct ftrace_ops *ops) =20 /* The cleanup is optional, ignore any errors */ if (found_op && op->ops_func) - op->ops_func(op, FTRACE_OPS_CMD_DISABLE_SHARE_IPMODIFY_PEER); + op->ops_func(op, ip, FTRACE_OPS_CMD_DISABLE_SHARE_IPMODIFY_PEER); } } mutex_unlock(&direct_mutex); --=20 2.51.0 From nobody Thu Oct 2 01:57:53 2025 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-1.web.codeaurora.org [10.30.226.201]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 5B16730E82D; Tue, 23 Sep 2025 21:53:35 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=10.30.226.201 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1758664415; cv=none; b=lZuGaznuT9v70/McuBHi0NymrMz+0XtQK+ajskw4KHvhSWHqM6DwZ2/EJm3NrI2/qgg6fhNruopIYlgDSbA+vI7hOle8YpYxDiLXJjzPdtQc3ecwUuY6bCkYO9Ks+jyY/2g8pV3RZvRgg9xTmdmGGpoB4rluNkzEGdROOVNF32Q= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1758664415; c=relaxed/simple; bh=43T9HLopp5TSSfaS/PpEBs04jnF5DnlAwLGcj6tvQ8M=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=bw5+c6M+t4iZhi6yWxSPk3N2z0WQrWxZpSDK9RTxAT/Uwcr4njGdqmOtpZBOUO1C7US5SNvjpybfacuxHbrygVoYVu3L+9cavlZFdFeW1NunwfS6sX036IBtB/G60SkG/UzDw+eRtG2n7xkONMKnQxlNCsn1AgTPM7HB6oIdHYY= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=QFtdiphn; arc=none smtp.client-ip=10.30.226.201 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="QFtdiphn" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 2D072C4CEF5; Tue, 23 Sep 2025 21:53:31 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1758664415; bh=43T9HLopp5TSSfaS/PpEBs04jnF5DnlAwLGcj6tvQ8M=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=QFtdiphnN9U6PqfI4yB+FMCirtdRv+x4abZHcBByKq62QTC45K2gDopib2ulQNxDb w+HUWaDby+yb+VNyzwN+0gQrESBnT43VpPMATSYgc1DbjLs7r+YgDnxeIg0K8xfDdv pE5yM0R/CrUBYjye8s/9oNBEKYKWFdpsVUbaLuyWDfI3Tlq7DFt8sEn1o3fjG0psp3 gflVn+TR38Tu5VPU5U+AwKXvQgdmNeUpOpFhr6Sc+Rz8+LNVSSSN/pI76pqzXROjDb YnuZUIIpmZMN4HAXlMmYj5gleREbbO7ujyhVDrMCjLTHukL9FRbMD9Y4w+gEWW5a/q kqq2AXN8Lgi5w== From: Jiri Olsa To: Steven Rostedt , Florent Revest , Mark Rutland Cc: bpf@vger.kernel.org, linux-kernel@vger.kernel.org, linux-trace-kernel@vger.kernel.org, linux-arm-kernel@lists.infradead.org, Alexei Starovoitov , Daniel Borkmann , Andrii Nakryiko , Menglong Dong Subject: [PATCH 9/9] bpf, x86: Use single ftrace_ops for direct calls Date: Tue, 23 Sep 2025 23:51:47 +0200 Message-ID: <20250923215147.1571952-10-jolsa@kernel.org> X-Mailer: git-send-email 2.51.0 In-Reply-To: <20250923215147.1571952-1-jolsa@kernel.org> References: <20250923215147.1571952-1-jolsa@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Using single ftrace_ops for direct calls update instead of allocating ftrace_ops obejct for each trampoline. At the moment we can enable this only on x86 arch, because arm relies on ftrace_ops object representing just single trampoline image (stored in ftrace_ops::direct_call). Adding HAVE_SINGLE_FTRACE_DIRECT_OPS config option to be enabled on each arch that supports this. Signed-off-by: Jiri Olsa --- arch/x86/Kconfig | 1 + kernel/bpf/trampoline.c | 85 +++++++++++++++++++++++++++++++++++------ kernel/trace/Kconfig | 3 ++ kernel/trace/ftrace.c | 15 +++++++- 4 files changed, 92 insertions(+), 12 deletions(-) diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index 52c8910ba2ef..bfc9115baa6f 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -330,6 +330,7 @@ config X86 imply IMA_SECURE_AND_OR_TRUSTED_BOOT if EFI select HAVE_DYNAMIC_FTRACE_NO_PATCHABLE select ARCH_SUPPORTS_PT_RECLAIM if X86_64 + select HAVE_SINGLE_FTRACE_DIRECT_OPS if X86_64 && DYNAMIC_FTRACE_WITH_DIR= ECT_CALLS =20 config INSTRUCTION_DECODER def_bool y diff --git a/kernel/bpf/trampoline.c b/kernel/bpf/trampoline.c index 3464859189a2..e3b721773dfb 100644 --- a/kernel/bpf/trampoline.c +++ b/kernel/bpf/trampoline.c @@ -33,12 +33,40 @@ static DEFINE_MUTEX(trampoline_mutex); #ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS static int bpf_trampoline_update(struct bpf_trampoline *tr, bool lock_dire= ct_mutex); =20 +#ifdef CONFIG_HAVE_SINGLE_FTRACE_DIRECT_OPS +static struct bpf_trampoline *direct_ops_ip_lookup(struct ftrace_ops *ops,= unsigned long ip) +{ + struct hlist_head *head_ip; + struct bpf_trampoline *tr; + + mutex_lock(&trampoline_mutex); + head_ip =3D &trampoline_ip_table[hash_64(ip, TRAMPOLINE_HASH_BITS)]; + hlist_for_each_entry(tr, head_ip, hlist_ip) { + if (tr->func.addr =3D=3D (void *) ip) + goto out; + } + tr =3D NULL; +out: + mutex_unlock(&trampoline_mutex); + return tr; +} +#else +static struct bpf_trampoline *direct_ops_ip_lookup(struct ftrace_ops *ops,= unsigned long ip) +{ + return ops->private; +} +#endif /* CONFIG_HAVE_SINGLE_FTRACE_DIRECT_OPS */ + static int bpf_tramp_ftrace_ops_func(struct ftrace_ops *ops, unsigned long= ip, enum ftrace_ops_cmd cmd) { - struct bpf_trampoline *tr =3D ops->private; + struct bpf_trampoline *tr; int ret =3D 0; =20 + tr =3D direct_ops_ip_lookup(ops, ip); + if (!tr) + return -EINVAL; + if (cmd =3D=3D FTRACE_OPS_CMD_ENABLE_SHARE_IPMODIFY_SELF) { /* This is called inside register_ftrace_direct_multi(), so * tr->mutex is already locked. @@ -137,6 +165,48 @@ void bpf_image_ksym_del(struct bpf_ksym *ksym) PAGE_SIZE, true, ksym->name); } =20 +#ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS +#ifdef CONFIG_HAVE_SINGLE_FTRACE_DIRECT_OPS +struct ftrace_ops direct_ops =3D { + .ops_func =3D bpf_tramp_ftrace_ops_func, +}; + +static int direct_ops_get(struct bpf_trampoline *tr) +{ + tr->fops =3D &direct_ops; + return 0; +} +static void direct_ops_clear(struct bpf_trampoline *tr) { } +static void direct_ops_free(struct bpf_trampoline *tr) { } +#else +static int direct_ops_get(struct bpf_trampoline *tr) +{ + tr->fops =3D kzalloc(sizeof(struct ftrace_ops), GFP_KERNEL); + if (!tr->fops) + return -1; + tr->fops->private =3D tr; + tr->fops->ops_func =3D bpf_tramp_ftrace_ops_func; + return 0; +} + +static void direct_ops_clear(struct bpf_trampoline *tr) +{ + tr->fops->func =3D NULL; + tr->fops->trampoline =3D 0; +} + +static void direct_ops_free(struct bpf_trampoline *tr) +{ + if (tr->fops) { + ftrace_free_filter(tr->fops); + kfree(tr->fops); + } +} +#endif /* CONFIG_HAVE_SINGLE_FTRACE_DIRECT_OPS */ +#else +static void direct_ops_free(struct bpf_trampoline *tr) { } +#endif /* CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS */ + static struct bpf_trampoline *bpf_trampoline_lookup(u64 key, unsigned long= ip) { struct bpf_trampoline *tr; @@ -155,14 +225,11 @@ static struct bpf_trampoline *bpf_trampoline_lookup(u= 64 key, unsigned long ip) if (!tr) goto out; #ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS - tr->fops =3D kzalloc(sizeof(struct ftrace_ops), GFP_KERNEL); - if (!tr->fops) { + if (direct_ops_get(tr)) { kfree(tr); tr =3D NULL; goto out; } - tr->fops->private =3D tr; - tr->fops->ops_func =3D bpf_tramp_ftrace_ops_func; #endif =20 tr->key =3D key; @@ -482,8 +549,7 @@ static int bpf_trampoline_update(struct bpf_trampoline = *tr, bool lock_direct_mut * trampoline again, and retry register. */ /* reset fops->func and fops->trampoline for re-register */ - tr->fops->func =3D NULL; - tr->fops->trampoline =3D 0; + direct_ops_clear(tr); =20 /* free im memory and reallocate later */ bpf_tramp_image_free(im); @@ -864,10 +930,7 @@ void bpf_trampoline_put(struct bpf_trampoline *tr) */ hlist_del(&tr->hlist_key); hlist_del(&tr->hlist_ip); - if (tr->fops) { - ftrace_free_filter(tr->fops); - kfree(tr->fops); - } + direct_ops_free(tr); kfree(tr); out: mutex_unlock(&trampoline_mutex); diff --git a/kernel/trace/Kconfig b/kernel/trace/Kconfig index d2c79da81e4f..4bf5beb04a5b 100644 --- a/kernel/trace/Kconfig +++ b/kernel/trace/Kconfig @@ -50,6 +50,9 @@ config HAVE_DYNAMIC_FTRACE_WITH_REGS config HAVE_DYNAMIC_FTRACE_WITH_DIRECT_CALLS bool =20 +config HAVE_SINGLE_FTRACE_DIRECT_OPS + bool + config HAVE_DYNAMIC_FTRACE_WITH_CALL_OPS bool =20 diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c index 2af1304d1a83..c4b969fb1010 100644 --- a/kernel/trace/ftrace.c +++ b/kernel/trace/ftrace.c @@ -2592,8 +2592,13 @@ unsigned long ftrace_find_rec_direct(unsigned long i= p) static void call_direct_funcs(unsigned long ip, unsigned long pip, struct ftrace_ops *ops, struct ftrace_regs *fregs) { - unsigned long addr =3D READ_ONCE(ops->direct_call); + unsigned long addr; =20 +#ifdef CONFIG_HAVE_SINGLE_FTRACE_DIRECT_OPS + addr =3D ftrace_find_rec_direct(ip); +#else + addr =3D READ_ONCE(ops->direct_call); +#endif if (!addr) return; =20 @@ -5986,6 +5991,10 @@ int register_ftrace_direct(struct ftrace_ops *ops, u= nsigned long ip, unsigned lo if (!hash) return -ENOMEM; =20 +#ifndef CONFIG_HAVE_SINGLE_FTRACE_DIRECT_OPS + ops->direct_call =3D addr; +#endif + err =3D register_ftrace_direct_hash(ops, hash); free_ftrace_hash(hash); return err; @@ -6050,6 +6059,10 @@ int modify_ftrace_direct(struct ftrace_ops *ops, uns= igned long ip, unsigned long return -ENOMEM; =20 err =3D modify_ftrace_direct_hash(ops, hash, lock_direct_mutex); +#ifndef CONFIG_HAVE_SINGLE_FTRACE_DIRECT_OPS + if (!err) + ops->direct_call =3D addr; +#endif free_ftrace_hash(hash); return err; } --=20 2.51.0