From nobody Mon Jun 15 22:05:37 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-1.web.codeaurora.org [10.30.226.201]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id D53BC3B961A; Tue, 14 Apr 2026 09:14:51 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=10.30.226.201 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1776158091; cv=none; b=ip538h6qpGH+6sFInpCKSMVEb1ip0QidjCWOBQCdC5wPJ0yb8/rZY4uOWQVRax19wvXmYsaDz/S+ush6A2jQL8JCsBERJTQSJQhv+MWDJeCup4mBIsCjy779ZrwfQV78ZB1WqkmkCJyUlXrZ1Va2zXlpo8/ZEIwndA4/hGJScPU= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1776158091; c=relaxed/simple; bh=rqs+M1PyAlmYt6BIh4c8u/Ci2zxPhYMYAfWmhLkdOIg=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version:Content-Type; b=GAEU6EYSVE6GmfrYYfhgplqL1VvkP4X4Y34bVj2b6ifdPhw8U6J0VH2Ohu8OqhjehWwCMQvsp0ICqCKjf3CwkjQW7MT3I34nUs70wu8tlp0pCYCDsijBhFefvXzShVvcuEa+EuONyY9ArIuoT6RCN8UxVsI/BHosDYXWSVB5s/k= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=ti7Elztc; arc=none smtp.client-ip=10.30.226.201 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="ti7Elztc" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 87C0DC19425; Tue, 14 Apr 2026 09:14:50 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1776158091; bh=rqs+M1PyAlmYt6BIh4c8u/Ci2zxPhYMYAfWmhLkdOIg=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=ti7ElztcLtiIZag17zTXfv7rVzVDq+35nVrgRJcWdGweR9XSNUNhvZLthDHB57azv fO5emaQ5afHS8nPcPnGBvpIuLRmUr+oCBz/9ROaJ0BnjdqRKCn0Wy1FMqyZjX2R+Fq gwlgMHyBGKeCbeaSplbOUXrwU3uiPnvS5PNdEoFS+EA7+r708awvW1tC0iaWnjVkf+ WPqWUtX6ogAFw74VXYCO3OBxoQ6WrL2yiOf+kZUKVC2BYrKkyNxyJBGTcaxlFZHIjd b+ZEZ++6UP9KcgFwm84/8Iq6ruQw+lWL5OVrSJta9lzeacfAN7AgPFvle8/bQZTvDv vgJo5uY+uoHKg== From: "Masami Hiramatsu (Google)" To: Steven Rostedt , Masami Hiramatsu Cc: Menglong Dong , Mathieu Desnoyers , jiang.biao@linux.dev, linux-kernel@vger.kernel.org, linux-trace-kernel@vger.kernel.org Subject: [PATCH v6 1/5] tracing/fprobe: Reject registration of a registered fprobe before init Date: Tue, 14 Apr 2026 18:14:49 +0900 Message-ID: <177615808886.1165997.1302205504890949839.stgit@mhiramat.tok.corp.google.com> X-Mailer: git-send-email 2.54.0.rc0.605.g598a273b03-goog In-Reply-To: <177615807787.1165997.921227352050738693.stgit@mhiramat.tok.corp.google.com> References: <177615807787.1165997.921227352050738693.stgit@mhiramat.tok.corp.google.com> User-Agent: StGit/0.19 Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Type: text/plain; charset="utf-8" Content-Transfer-Encoding: quoted-printable From: Masami Hiramatsu (Google) Reject registration of a registered fprobe which is on the fprobe hash table before initializing fprobe. The add_fprobe_hash() checks this re-register fprobe, but since fprobe_init() clears hlist_array field, it is too late to check it. It has to check the re-registration before touncing fprobe. Fixes: 4346ba160409 ("fprobe: Rewrite fprobe on function-graph tracer") Cc: stable@vger.kernel.org Signed-off-by: Masami Hiramatsu (Google) --- Changes in v6: - Newly added. --- kernel/trace/fprobe.c | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/kernel/trace/fprobe.c b/kernel/trace/fprobe.c index dcadf1d23b8a..fc7018b28fdd 100644 --- a/kernel/trace/fprobe.c +++ b/kernel/trace/fprobe.c @@ -4,6 +4,7 @@ */ #define pr_fmt(fmt) "fprobe: " fmt =20 +#include #include #include #include @@ -107,7 +108,7 @@ static bool delete_fprobe_node(struct fprobe_hlist_node= *node) } =20 /* Check existence of the fprobe */ -static bool is_fprobe_still_exist(struct fprobe *fp) +static bool fprobe_registered(struct fprobe *fp) { struct hlist_head *head; struct fprobe_hlist *fph; @@ -120,7 +121,7 @@ static bool is_fprobe_still_exist(struct fprobe *fp) } return false; } -NOKPROBE_SYMBOL(is_fprobe_still_exist); +NOKPROBE_SYMBOL(fprobe_registered); =20 static int add_fprobe_hash(struct fprobe *fp) { @@ -132,9 +133,6 @@ static int add_fprobe_hash(struct fprobe *fp) if (WARN_ON_ONCE(!fph)) return -EINVAL; =20 - if (is_fprobe_still_exist(fp)) - return -EEXIST; - head =3D &fprobe_table[hash_ptr(fp, FPROBE_HASH_BITS)]; hlist_add_head_rcu(&fp->hlist_array->hlist, head); return 0; @@ -149,7 +147,7 @@ static int del_fprobe_hash(struct fprobe *fp) if (WARN_ON_ONCE(!fph)) return -EINVAL; =20 - if (!is_fprobe_still_exist(fp)) + if (!fprobe_registered(fp)) return -ENOENT; =20 fph->fp =3D NULL; @@ -482,7 +480,7 @@ static void fprobe_return(struct ftrace_graph_ret *trac= e, if (!fp) break; curr +=3D FPROBE_HEADER_SIZE_IN_LONG; - if (is_fprobe_still_exist(fp) && !fprobe_disabled(fp)) { + if (fprobe_registered(fp) && !fprobe_disabled(fp)) { if (WARN_ON_ONCE(curr + size > size_words)) break; fp->exit_handler(fp, trace->func, ret_ip, fregs, @@ -841,12 +839,14 @@ int register_fprobe_ips(struct fprobe *fp, unsigned l= ong *addrs, int num) struct fprobe_hlist *hlist_array; int ret, i; =20 + guard(mutex)(&fprobe_mutex); + if (fprobe_registered(fp)) + return -EEXIST; + ret =3D fprobe_init(fp, addrs, num); if (ret) return ret; =20 - mutex_lock(&fprobe_mutex); - hlist_array =3D fp->hlist_array; if (fprobe_is_ftrace(fp)) ret =3D fprobe_ftrace_add_ips(addrs, num); @@ -866,7 +866,6 @@ int register_fprobe_ips(struct fprobe *fp, unsigned lon= g *addrs, int num) delete_fprobe_node(&hlist_array->array[i]); } } - mutex_unlock(&fprobe_mutex); =20 if (ret) fprobe_fail_cleanup(fp); @@ -928,7 +927,7 @@ int unregister_fprobe(struct fprobe *fp) int ret =3D 0, i, count; =20 mutex_lock(&fprobe_mutex); - if (!fp || !is_fprobe_still_exist(fp)) { + if (!fp || !fprobe_registered(fp)) { ret =3D -EINVAL; goto out; } From nobody Mon Jun 15 22:05:37 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-1.web.codeaurora.org [10.30.226.201]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id B2C223B9DAC; Tue, 14 Apr 2026 09:14:59 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=10.30.226.201 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1776158099; cv=none; b=JjL2HXmK+VQfWAAXcmqHB1zJup+i0YFR3djdTOA3YdFMY8fMzvrrR4x3Ta15qpWsL1XhV1vA0XRArt9HkU3nxaTUXtTVICjMyOKK0G4WXtDik349vjQ5ja5VIpWB4ihp0hDk2V5Bru8ai0MQWArOARV3bKxqiSMw45r+pORrXCM= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1776158099; c=relaxed/simple; bh=Aa8wrn1cZ8tiqKsLwbrqbJX7XrGkRumRdPDSZdDUhtU=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version:Content-Type; b=AHraRXPb8KR3LSyWloZMvolr2iADI/aA7zaOLlEFmVD8y8/8GgBTZW6rXg9oGoxhhzo2xxxf36hJOnSA3Y8dr9Ixbw5tk5+IrKyEXQoMqZJKvcySapg56TiudVRyp7QQDHB8XUl3jEZiAcXk0x49FAo08miVz4mR31YB6TtySmk= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=kM9Iypje; arc=none smtp.client-ip=10.30.226.201 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="kM9Iypje" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 6E74BC19425; Tue, 14 Apr 2026 09:14:58 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1776158099; bh=Aa8wrn1cZ8tiqKsLwbrqbJX7XrGkRumRdPDSZdDUhtU=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=kM9IypjeV6WC4K4flk+aQQgVFbS2bCRHQeJLGHn+DES6duURcW1bWfcsZ6p5GZVFp 4dBLChKwBJvTzdKmU5id2LuRgmTS2Lv1+h6M1Ip9pu7YyMmeUOxmasIerhcifuiU17 KjP4l3dfAS+r2DgYe+4/1kLGnFVNVk9qv63EmMzwOLPpDOya75oQhHUo8OT27x9rAU s/M82GtpX6H3o84/6zIUranjocgEc2NvbomdKnHWX1E5wWoqDKIQ9Y183HNd2aF6PI QMz9C/4MyEjMsF+3f3g73lpD8WaIjQT3gvVoYbGpIiqncmpP9vgoNOWmnRUNjFOQR2 uk85MJIc2zjCA== From: "Masami Hiramatsu (Google)" To: Steven Rostedt , Masami Hiramatsu Cc: Menglong Dong , Mathieu Desnoyers , jiang.biao@linux.dev, linux-kernel@vger.kernel.org, linux-trace-kernel@vger.kernel.org Subject: [PATCH v6 2/5] tracing/fprobe: Remove fprobe from hash in failure path Date: Tue, 14 Apr 2026 18:14:57 +0900 Message-ID: <177615809677.1165997.619922394559783590.stgit@mhiramat.tok.corp.google.com> X-Mailer: git-send-email 2.54.0.rc0.605.g598a273b03-goog In-Reply-To: <177615807787.1165997.921227352050738693.stgit@mhiramat.tok.corp.google.com> References: <177615807787.1165997.921227352050738693.stgit@mhiramat.tok.corp.google.com> User-Agent: StGit/0.19 Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Type: text/plain; charset="utf-8" Content-Transfer-Encoding: quoted-printable From: Masami Hiramatsu (Google) When register_fprobe_ips() fails, it tries to remove a list of fprobe_hash_node from fprobe_ip_table, but it missed to remove fprobe itself from fprobe_table. Moreover, when removing the fprobe_hash_node which is added to rhltable once, it must use kfree_rcu() after removing from rhltable. To fix these issues, this reuses unregister_fprobe() internal code to rollback the half-way registered fprobe. Fixes: 4346ba160409 ("fprobe: Rewrite fprobe on function-graph tracer") Cc: stable@vger.kernel.org Signed-off-by: Masami Hiramatsu (Google) --- Changes in v6: - Wait for an RCU grace period before returning error in unregister_fprobe_nolock(). Changes in v5: - When rolling back an fprobe that failed to register, the fprobe_hash_node are forcibly removed and warn if failure. Changes in v4: - Remove short-cut case because we always need to upadte ftrace_ops. - Use guard(mutex) in register_fprobe_ips() to unlock it correctly. - Remove redundant !ret check in register_fprobe_ips(). - Do not set hlist_array->size in failure case, instead, hlist_array->array[i].fp is set only when insertion is succeeded. Changes in v3: - Newly added. --- kernel/trace/fprobe.c | 109 ++++++++++++++++++++++++++++-----------------= ---- 1 file changed, 63 insertions(+), 46 deletions(-) diff --git a/kernel/trace/fprobe.c b/kernel/trace/fprobe.c index fc7018b28fdd..6a23bb787295 100644 --- a/kernel/trace/fprobe.c +++ b/kernel/trace/fprobe.c @@ -79,20 +79,27 @@ static const struct rhashtable_params fprobe_rht_params= =3D { }; =20 /* Node insertion and deletion requires the fprobe_mutex */ -static int insert_fprobe_node(struct fprobe_hlist_node *node) +static int insert_fprobe_node(struct fprobe_hlist_node *node, struct fprob= e *fp) { + int ret; + lockdep_assert_held(&fprobe_mutex); =20 - return rhltable_insert(&fprobe_ip_table, &node->hlist, fprobe_rht_params); + ret =3D rhltable_insert(&fprobe_ip_table, &node->hlist, fprobe_rht_params= ); + /* Set the fprobe pointer if insertion was successful. */ + if (!ret) + WRITE_ONCE(node->fp, fp); + return ret; } =20 /* Return true if there are synonims */ static bool delete_fprobe_node(struct fprobe_hlist_node *node) { - lockdep_assert_held(&fprobe_mutex); bool ret; =20 - /* Avoid double deleting */ + lockdep_assert_held(&fprobe_mutex); + + /* Avoid double deleting and non-inserted nodes */ if (READ_ONCE(node->fp) !=3D NULL) { WRITE_ONCE(node->fp, NULL); rhltable_remove(&fprobe_ip_table, &node->hlist, @@ -757,7 +764,6 @@ static int fprobe_init(struct fprobe *fp, unsigned long= *addrs, int num) fp->hlist_array =3D hlist_array; hlist_array->fp =3D fp; for (i =3D 0; i < num; i++) { - hlist_array->array[i].fp =3D fp; addr =3D ftrace_location(addrs[i]); if (!addr) { fprobe_fail_cleanup(fp); @@ -821,6 +827,8 @@ int register_fprobe(struct fprobe *fp, const char *filt= er, const char *notfilter } EXPORT_SYMBOL_GPL(register_fprobe); =20 +static int unregister_fprobe_nolock(struct fprobe *fp, bool force); + /** * register_fprobe_ips() - Register fprobe to ftrace by address. * @fp: A fprobe data structure to be registered. @@ -847,29 +855,26 @@ int register_fprobe_ips(struct fprobe *fp, unsigned l= ong *addrs, int num) if (ret) return ret; =20 - hlist_array =3D fp->hlist_array; if (fprobe_is_ftrace(fp)) ret =3D fprobe_ftrace_add_ips(addrs, num); else ret =3D fprobe_graph_add_ips(addrs, num); + if (ret) { + fprobe_fail_cleanup(fp); + return ret; + } =20 - if (!ret) { - add_fprobe_hash(fp); - for (i =3D 0; i < hlist_array->size; i++) { - ret =3D insert_fprobe_node(&hlist_array->array[i]); - if (ret) - break; - } - /* fallback on insert error */ + hlist_array =3D fp->hlist_array; + add_fprobe_hash(fp); + for (i =3D 0; i < hlist_array->size; i++) { + ret =3D insert_fprobe_node(&hlist_array->array[i], fp); if (ret) { - for (i--; i >=3D 0; i--) - delete_fprobe_node(&hlist_array->array[i]); + if (unregister_fprobe_nolock(fp, true)) + pr_warn("Failed to cleanup fprobe after insertion failure.\n"); + break; } } =20 - if (ret) - fprobe_fail_cleanup(fp); - return ret; } EXPORT_SYMBOL_GPL(register_fprobe_ips); @@ -912,37 +917,29 @@ bool fprobe_is_registered(struct fprobe *fp) return true; } =20 -/** - * unregister_fprobe() - Unregister fprobe. - * @fp: A fprobe data structure to be unregistered. - * - * Unregister fprobe (and remove ftrace hooks from the function entries). - * - * Return 0 if @fp is unregistered successfully, -errno if not. - */ -int unregister_fprobe(struct fprobe *fp) +static int unregister_fprobe_nolock(struct fprobe *fp, bool force) { - struct fprobe_hlist *hlist_array; + struct fprobe_hlist *hlist_array =3D fp->hlist_array; unsigned long *addrs =3D NULL; - int ret =3D 0, i, count; + int i, count; =20 - mutex_lock(&fprobe_mutex); - if (!fp || !fprobe_registered(fp)) { - ret =3D -EINVAL; - goto out; - } - - hlist_array =3D fp->hlist_array; addrs =3D kcalloc(hlist_array->size, sizeof(unsigned long), GFP_KERNEL); - if (!addrs) { - ret =3D -ENOMEM; /* TODO: Fallback to one-by-one loop */ - goto out; - } + if (!addrs && !force) + return -ENOMEM; + /* + * If @force is set, this function will remove fprobe_hash_node + * from the hash table even if memory allocation fails. However, + * ftrace_ops will not be updated. Anyway, when the last fprobe + * is unregistered, ftrace_ops is also unregistered. + */ =20 /* Remove non-synonim ips from table and hash */ count =3D 0; for (i =3D 0; i < hlist_array->size; i++) { - if (!delete_fprobe_node(&hlist_array->array[i])) + if (delete_fprobe_node(&hlist_array->array[i])) + continue; + + if (addrs) addrs[count++] =3D hlist_array->array[i].addr; } del_fprobe_hash(fp); @@ -951,15 +948,35 @@ int unregister_fprobe(struct fprobe *fp) fprobe_ftrace_remove_ips(addrs, count); else fprobe_graph_remove_ips(addrs, count); + /* + * If count =3D=3D 0, instead of calling ftrace_set_filter_ips(), + * we must wait for RCU grace period to finish del_fprobe_hash(). + */ + if (!count) + synchronize_rcu(); =20 kfree_rcu(hlist_array, rcu); fp->hlist_array =3D NULL; + kfree(addrs); =20 -out: - mutex_unlock(&fprobe_mutex); + return !addrs ? -ENOMEM : 0; +} =20 - kfree(addrs); - return ret; +/** + * unregister_fprobe() - Unregister fprobe. + * @fp: A fprobe data structure to be unregistered. + * + * Unregister fprobe (and remove ftrace hooks from the function entries). + * + * Return 0 if @fp is unregistered successfully, -errno if not. + */ +int unregister_fprobe(struct fprobe *fp) +{ + guard(mutex)(&fprobe_mutex); + if (!fp || !fprobe_registered(fp)) + return -EINVAL; + + return unregister_fprobe_nolock(fp, false); } EXPORT_SYMBOL_GPL(unregister_fprobe); From nobody Mon Jun 15 22:05:37 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-1.web.codeaurora.org [10.30.226.201]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 9B8AE3B9DB3; Tue, 14 Apr 2026 09:15:07 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=10.30.226.201 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1776158107; cv=none; b=irAT2TG/285zImqD2MyXo1t9ZlGOmVzP8HT0MOFwdHi2/LlThDySc39REsSBCPGt8PbCk/S3XOwe6HefBhBJHRL/24DAHNbTpkKHjC0AvwHb0jW5kOlF1jWCF2pK9Gc6NjjL/QnGD4Hc6LKdXtWLwftLkDlhtsc/ZQbprBy+xZ0= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1776158107; c=relaxed/simple; bh=FWrOCS/pOl3Q1NUJiO1do/wRFbutj/NaXgGyX3ousy8=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version:Content-Type; b=eb3HcJtG23AFPMNSNcIL2EGXDlySvvrBBrE1TJCXpEH10XViyxR2/KYsTsnhQAtwEujAAnC2P+KDiEOuLUFA09uIBcvgLvIEi3cgouqunbqrIGL16ZeeWeky7VwHXx8aZQXOMqNsi/+uoJ0TucrftzvTMRTHaKKKbu6H1BWQvvU= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=iwpDO5f7; arc=none smtp.client-ip=10.30.226.201 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="iwpDO5f7" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 51023C19425; Tue, 14 Apr 2026 09:15:06 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1776158107; bh=FWrOCS/pOl3Q1NUJiO1do/wRFbutj/NaXgGyX3ousy8=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=iwpDO5f7jKOZW2x7txY+xgL6TV9q9jyeD67OgPSZmskoMathoE3KgHu+kOAYSnva2 Yq8wZZKx1TY4cZ+GHJfnjDdjCIpkM0tOxti3FcejSXN4q+qohm6gw1IYGpfz9YL5UX qN0sLWiZFuCuzliSAoDEyjELc3TkPsUFnkjSQBIL+8lPgQK1rEu2KxzUL6FqCZOB36 LOucBs12dBoR/NFQrfasC0VKYeLXLUWq64+N9DDjOam8CmnDmJfjb6t6QMgX+/l0bf cIgYklUbRnW+Efn4ChgrHRyPUU1TBYDf39Au6I2L6zPboRc6dZ/p26MayUa5ZPKRcM rPIL3EZutJcvg== From: "Masami Hiramatsu (Google)" To: Steven Rostedt , Masami Hiramatsu Cc: Menglong Dong , Mathieu Desnoyers , jiang.biao@linux.dev, linux-kernel@vger.kernel.org, linux-trace-kernel@vger.kernel.org Subject: [PATCH v6 3/5] tracing/fprobe: Avoid kcalloc() in rcu_read_lock section Date: Tue, 14 Apr 2026 18:15:04 +0900 Message-ID: <177615810462.1165997.15742135204310342097.stgit@mhiramat.tok.corp.google.com> X-Mailer: git-send-email 2.54.0.rc0.605.g598a273b03-goog In-Reply-To: <177615807787.1165997.921227352050738693.stgit@mhiramat.tok.corp.google.com> References: <177615807787.1165997.921227352050738693.stgit@mhiramat.tok.corp.google.com> User-Agent: StGit/0.19 Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Type: text/plain; charset="utf-8" Content-Transfer-Encoding: quoted-printable From: Masami Hiramatsu (Google) fprobe_remove_node_in_module() is called under RCU read locked, but this invokes kcalloc() if there are more than 8 fprobes installed on the module. Sashiko warns it because kcalloc() can sleep [1]. [1] https://sashiko.dev/#/patchset/177552432201.853249.5125045538812833325= .stgit%40mhiramat.tok.corp.google.com To fix this issue, expand the batch size to 128 and do not expand the fprobe_addr_list, but just cancel walking on fprobe_ip_table, update fgraph/ftrace_ops and retry the loop again. Fixes: 0de4c70d04a4 ("tracing: fprobe: use rhltable for fprobe_ip_table") Cc: stable@vger.kernel.org Signed-off-by: Masami Hiramatsu (Google) --- Changes in v6: - Retry outside rhltable_walk_enter/exit() again. Changes in v5: - Skip updating ftrace_ops when fails to allocate memory in module unloading. Changes in v4: - fix a build error typo in case of CONFIG_DYNAMIC_FTRACE=3Dn. Changes in v3: - Retry inside rhltable_walk_enter/exit(). - Rename fprobe_set_ips() to fprobe_remove_ips(). - Rename 'retry' label to 'again'. --- kernel/trace/fprobe.c | 92 ++++++++++++++++++++++++---------------------= ---- 1 file changed, 45 insertions(+), 47 deletions(-) diff --git a/kernel/trace/fprobe.c b/kernel/trace/fprobe.c index 6a23bb787295..2059d8d83b4c 100644 --- a/kernel/trace/fprobe.c +++ b/kernel/trace/fprobe.c @@ -343,11 +343,10 @@ static bool fprobe_is_ftrace(struct fprobe *fp) } =20 #ifdef CONFIG_MODULES -static void fprobe_set_ips(unsigned long *ips, unsigned int cnt, int remov= e, - int reset) +static void fprobe_remove_ips(unsigned long *ips, unsigned int cnt) { - ftrace_set_filter_ips(&fprobe_graph_ops.ops, ips, cnt, remove, reset); - ftrace_set_filter_ips(&fprobe_ftrace_ops, ips, cnt, remove, reset); + ftrace_set_filter_ips(&fprobe_graph_ops.ops, ips, cnt, 1, 0); + ftrace_set_filter_ips(&fprobe_ftrace_ops, ips, cnt, 1, 0); } #endif #else @@ -366,10 +365,9 @@ static bool fprobe_is_ftrace(struct fprobe *fp) } =20 #ifdef CONFIG_MODULES -static void fprobe_set_ips(unsigned long *ips, unsigned int cnt, int remov= e, - int reset) +static void fprobe_remove_ips(unsigned long *ips, unsigned int cnt) { - ftrace_set_filter_ips(&fprobe_graph_ops.ops, ips, cnt, remove, reset); + ftrace_set_filter_ips(&fprobe_graph_ops.ops, ips, cnt, 1, 0); } #endif #endif /* !CONFIG_DYNAMIC_FTRACE_WITH_ARGS && !CONFIG_DYNAMIC_FTRACE_WITH_= REGS */ @@ -543,7 +541,7 @@ static void fprobe_graph_remove_ips(unsigned long *addr= s, int num) =20 #ifdef CONFIG_MODULES =20 -#define FPROBE_IPS_BATCH_INIT 8 +#define FPROBE_IPS_BATCH_INIT 128 /* instruction pointer address list */ struct fprobe_addr_list { int index; @@ -551,45 +549,24 @@ struct fprobe_addr_list { unsigned long *addrs; }; =20 -static int fprobe_addr_list_add(struct fprobe_addr_list *alist, unsigned l= ong addr) +static int fprobe_remove_node_in_module(struct module *mod, struct fprobe_= hlist_node *node, + struct fprobe_addr_list *alist) { - unsigned long *addrs; - - /* Previously we failed to expand the list. */ - if (alist->index =3D=3D alist->size) - return -ENOSPC; - - alist->addrs[alist->index++] =3D addr; - if (alist->index < alist->size) + if (!within_module(node->addr, mod)) return 0; =20 - /* Expand the address list */ - addrs =3D kcalloc(alist->size * 2, sizeof(*addrs), GFP_KERNEL); - if (!addrs) - return -ENOMEM; - - memcpy(addrs, alist->addrs, alist->size * sizeof(*addrs)); - alist->size *=3D 2; - kfree(alist->addrs); - alist->addrs =3D addrs; + if (delete_fprobe_node(node)) + return 0; + /* If no address list is available, we can't track this address. */ + if (!alist->addrs) + return 0; =20 + alist->addrs[alist->index++] =3D node->addr; + if (alist->index =3D=3D alist->size) + return -ENOSPC; return 0; } =20 -static void fprobe_remove_node_in_module(struct module *mod, struct fprobe= _hlist_node *node, - struct fprobe_addr_list *alist) -{ - if (!within_module(node->addr, mod)) - return; - if (delete_fprobe_node(node)) - return; - /* - * If failed to update alist, just continue to update hlist. - * Therefore, at list user handler will not hit anymore. - */ - fprobe_addr_list_add(alist, node->addr); -} - /* Handle module unloading to manage fprobe_ip_table. */ static int fprobe_module_callback(struct notifier_block *nb, unsigned long val, void *data) @@ -598,29 +575,50 @@ static int fprobe_module_callback(struct notifier_blo= ck *nb, struct fprobe_hlist_node *node; struct rhashtable_iter iter; struct module *mod =3D data; + bool retry; =20 if (val !=3D MODULE_STATE_GOING) return NOTIFY_DONE; =20 alist.addrs =3D kcalloc(alist.size, sizeof(*alist.addrs), GFP_KERNEL); - /* If failed to alloc memory, we can not remove ips from hash. */ - if (!alist.addrs) - return NOTIFY_DONE; + /* + * If failed to alloc memory, ftrace_ops will not be able to remove ips f= rom + * hash, but we can still remove nodes from fprobe_ip_table, so we can av= oid + * the potential wrong callback. So just print a warning here and try to + * continue without address list. + */ + WARN_ONCE(!alist.addrs, + "Failed to allocate memory for fprobe_addr_list, ftrace_ops will not be = updated"); =20 mutex_lock(&fprobe_mutex); +again: + retry =3D false; + alist.index =3D 0; rhltable_walk_enter(&fprobe_ip_table, &iter); do { rhashtable_walk_start(&iter); =20 while ((node =3D rhashtable_walk_next(&iter)) && !IS_ERR(node)) - fprobe_remove_node_in_module(mod, node, &alist); + if (fprobe_remove_node_in_module(mod, node, &alist) < 0) { + retry =3D true; + break; + } =20 rhashtable_walk_stop(&iter); - } while (node =3D=3D ERR_PTR(-EAGAIN)); + } while (node =3D=3D ERR_PTR(-EAGAIN) && !retry); rhashtable_walk_exit(&iter); + /* Remove any ips from hash table(s) */ + if (alist.index > 0) { + fprobe_remove_ips(alist.addrs, alist.index); + /* + * If we break rhashtable walk loop except for -EAGAIN, we need + * to restart looping from start for safety. Anyway, this is + * not a hotpath. + */ + if (retry) + goto again; + } =20 - if (alist.index > 0) - fprobe_set_ips(alist.addrs, alist.index, 1, 0); mutex_unlock(&fprobe_mutex); =20 kfree(alist.addrs); From nobody Mon Jun 15 22:05:37 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-1.web.codeaurora.org [10.30.226.201]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 2B5822C08DC; Tue, 14 Apr 2026 09:15:15 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=10.30.226.201 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1776158116; cv=none; b=ojGrgoIrmphOtNtVY5e96hqaCCCK9O4xbIx6baDIZOm2qARt+h8bHypnfyzproRKTp6M8uQFhm4g3KbnxGgGeBTOJodrMGAcP0N6pbQSNZWOfN9v2DXorXqs2rJLXZI22KBe7B6JO/UJdcgaG9MOKb1bDfx08WM1mEEjjbYpc98= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1776158116; c=relaxed/simple; bh=w9wNvh3LBzJer6yALKf8V8zxQN7awdvGVKG9Nl+47tg=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version:Content-Type; b=IWE4O7vgOXo5iAgRhqIscKsOi+qNu4WGi3/WmSTNMIeU05OldKQcUGAygYfQm0QPn3eYSUp9tHNpfTfYRi4JqwZGJMXnIG+SFMyW0rorzQfqNZYP8/UDG9A4ir0MxqPsDDxwC1gGLFJ/1BL364Cmr23hw/GmeCYwNuYPLSZGfuI= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=HeQ7qDO/; arc=none smtp.client-ip=10.30.226.201 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="HeQ7qDO/" Received: by smtp.kernel.org (Postfix) with ESMTPSA id DA479C2BCB7; Tue, 14 Apr 2026 09:15:14 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1776158115; bh=w9wNvh3LBzJer6yALKf8V8zxQN7awdvGVKG9Nl+47tg=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=HeQ7qDO/kTmIjZ02HxZ8uvMGnx0DxQvGHhq2d7m7qNnQ9XK0ETuIiuylvp39kk8eX RxTAa4rRof6q2V6sU9TplvU1SDSf/FvhPPgqq3Mi9YHPWtmOSxKB6x/YUab1W8ZQrF SR8tUEwydHBLz7ZQ2pFvbKXB5pbLW+ffsedLmqvYYJeoQLmO5cWBA4g8Riyc9MP6et FTm2fHEtIXO7aju70lK/CdcmA1IKc6vpH7W1pdEWklmKnx3V9xG6W3tnDsGl51Wa5o pzCK+HkASnfB0uj4iqbbpe0pYc+CRDNpdVfmOjFhXHqwUrboASE9umYDARTGXLVpNe 9LC1+jbJLpRZA== From: "Masami Hiramatsu (Google)" To: Steven Rostedt , Masami Hiramatsu Cc: Menglong Dong , Mathieu Desnoyers , jiang.biao@linux.dev, linux-kernel@vger.kernel.org, linux-trace-kernel@vger.kernel.org Subject: [PATCH v6 4/5] tracing/fprobe: Check the same type fprobe on table as the unregistered one Date: Tue, 14 Apr 2026 18:15:12 +0900 Message-ID: <177615811251.1165997.1309350521488300755.stgit@mhiramat.tok.corp.google.com> X-Mailer: git-send-email 2.54.0.rc0.605.g598a273b03-goog In-Reply-To: <177615807787.1165997.921227352050738693.stgit@mhiramat.tok.corp.google.com> References: <177615807787.1165997.921227352050738693.stgit@mhiramat.tok.corp.google.com> User-Agent: StGit/0.19 Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Type: text/plain; charset="utf-8" Content-Transfer-Encoding: quoted-printable From: Masami Hiramatsu (Google) Commit 2c67dc457bc6 ("tracing: fprobe: optimization for entry only case") introduced a different ftrace_ops for entry-only fprobes. However, when unregistering an fprobe, the kernel only checks if another fprobe exists at the same address, without checking which type of fprobe it is. If different fprobes are registered at the same address, the same address will be registered in both fgraph_ops and ftrace_ops, but only one of them will be deleted when unregistering. (the one removed first will not be deleted from the ops). This results in junk entries remaining in either fgraph_ops or ftrace_ops. For example: =3D=3D=3D=3D=3D=3D=3D cd /sys/kernel/tracing # 'Add entry and exit events on the same place' echo 'f:event1 vfs_read' >> dynamic_events echo 'f:event2 vfs_read%return' >> dynamic_events # 'Enable both of them' echo 1 > events/fprobes/enable cat enabled_functions vfs_read (2) ->arch_ftrace_ops_list_func+0x0/0x210 # 'Disable and remove exit event' echo 0 > events/fprobes/event2/enable echo -:event2 >> dynamic_events # 'Disable and remove all events' echo 0 > events/fprobes/enable echo > dynamic_events # 'Add another event' echo 'f:event3 vfs_open%return' > dynamic_events cat dynamic_events f:fprobes/event3 vfs_open%return echo 1 > events/fprobes/enable cat enabled_functions vfs_open (1) tramp: 0xffffffffa0001000 (ftrace_graph_func+0x0/0x= 60) ->ftrace_graph_func+0x0/0x60 subops: {ent:fprobe_fgraph_entry+0x0/0x= 620 ret:fprobe_return+0x0/0x150} vfs_read (1) tramp: 0xffffffffa0001000 (ftrace_graph_func+0x0/0x= 60) ->ftrace_graph_func+0x0/0x60 subops: {ent:fprobe_fgraph_entry+0x0/0x= 620 ret:fprobe_return+0x0/0x150} =3D=3D=3D=3D=3D=3D=3D As you can see, an entry for the vfs_read remains. To fix this issue, when unregistering, the kernel should also check if there is the same type of fprobes still exist at the same address, and if not, delete its entry from either fgraph_ops or ftrace_ops. Fixes: 2c67dc457bc6 ("tracing: fprobe: optimization for entry only case") Cc: stable@vger.kernel.org Signed-off-by: Masami Hiramatsu (Google) --- kernel/trace/fprobe.c | 85 +++++++++++++++++++++++++++++++++++++--------= ---- 1 file changed, 65 insertions(+), 20 deletions(-) diff --git a/kernel/trace/fprobe.c b/kernel/trace/fprobe.c index 2059d8d83b4c..1767c2b0884c 100644 --- a/kernel/trace/fprobe.c +++ b/kernel/trace/fprobe.c @@ -92,11 +92,8 @@ static int insert_fprobe_node(struct fprobe_hlist_node *= node, struct fprobe *fp) return ret; } =20 -/* Return true if there are synonims */ -static bool delete_fprobe_node(struct fprobe_hlist_node *node) +static void delete_fprobe_node(struct fprobe_hlist_node *node) { - bool ret; - lockdep_assert_held(&fprobe_mutex); =20 /* Avoid double deleting and non-inserted nodes */ @@ -105,13 +102,6 @@ static bool delete_fprobe_node(struct fprobe_hlist_nod= e *node) rhltable_remove(&fprobe_ip_table, &node->hlist, fprobe_rht_params); } - - rcu_read_lock(); - ret =3D !!rhltable_lookup(&fprobe_ip_table, &node->addr, - fprobe_rht_params); - rcu_read_unlock(); - - return ret; } =20 /* Check existence of the fprobe */ @@ -342,6 +332,32 @@ static bool fprobe_is_ftrace(struct fprobe *fp) return !fp->exit_handler; } =20 +static bool fprobe_exists_on_hash(unsigned long ip, bool ftrace) +{ + struct rhlist_head *head, *pos; + struct fprobe_hlist_node *node; + struct fprobe *fp; + + guard(rcu)(); + head =3D rhltable_lookup(&fprobe_ip_table, &ip, + fprobe_rht_params); + if (!head) + return false; + /* We have to check the same type on the list. */ + rhl_for_each_entry_rcu(node, pos, head, hlist) { + if (node->addr !=3D ip) + break; + fp =3D READ_ONCE(node->fp); + if (likely(fp)) { + if ((!ftrace && fp->exit_handler) || + (ftrace && !fp->exit_handler)) + return true; + } + } + + return false; +} + #ifdef CONFIG_MODULES static void fprobe_remove_ips(unsigned long *ips, unsigned int cnt) { @@ -364,6 +380,29 @@ static bool fprobe_is_ftrace(struct fprobe *fp) return false; } =20 +static bool fprobe_exists_on_hash(unsigned long ip, bool ftrace __maybe_un= used) +{ + struct rhlist_head *head, *pos; + struct fprobe_hlist_node *node; + struct fprobe *fp; + + guard(rcu)(); + head =3D rhltable_lookup(&fprobe_ip_table, &ip, + fprobe_rht_params); + if (!head) + return false; + /* We only need to check fp is there. */ + rhl_for_each_entry_rcu(node, pos, head, hlist) { + if (node->addr !=3D ip) + break; + fp =3D READ_ONCE(node->fp); + if (likely(fp)) + return true; + } + + return false; +} + #ifdef CONFIG_MODULES static void fprobe_remove_ips(unsigned long *ips, unsigned int cnt) { @@ -552,18 +591,25 @@ struct fprobe_addr_list { static int fprobe_remove_node_in_module(struct module *mod, struct fprobe_= hlist_node *node, struct fprobe_addr_list *alist) { + lockdep_assert_in_rcu_read_lock(); + if (!within_module(node->addr, mod)) return 0; =20 - if (delete_fprobe_node(node)) - return 0; + delete_fprobe_node(node); /* If no address list is available, we can't track this address. */ if (!alist->addrs) return 0; + /* + * Don't care the type here, because all fprobes on the same + * address must be removed eventually. + */ + if (!rhltable_lookup(&fprobe_ip_table, &node->addr, fprobe_rht_params)) { + alist->addrs[alist->index++] =3D node->addr; + if (alist->index =3D=3D alist->size) + return -ENOSPC; + } =20 - alist->addrs[alist->index++] =3D node->addr; - if (alist->index =3D=3D alist->size) - return -ENOSPC; return 0; } =20 @@ -934,10 +980,9 @@ static int unregister_fprobe_nolock(struct fprobe *fp,= bool force) /* Remove non-synonim ips from table and hash */ count =3D 0; for (i =3D 0; i < hlist_array->size; i++) { - if (delete_fprobe_node(&hlist_array->array[i])) - continue; - - if (addrs) + delete_fprobe_node(&hlist_array->array[i]); + if (addrs && !fprobe_exists_on_hash(hlist_array->array[i].addr, + fprobe_is_ftrace(fp))) addrs[count++] =3D hlist_array->array[i].addr; } del_fprobe_hash(fp); From nobody Mon Jun 15 22:05:37 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-1.web.codeaurora.org [10.30.226.201]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id C46D42C08DC; Tue, 14 Apr 2026 09:15:23 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=10.30.226.201 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1776158123; cv=none; b=QI6sumWrYNQHUqUhpxHAdTMgwESRYJuLDPDCKpqfry4M1MB0IMvl+fDT9bJAc7admder7VVdMLDw6ucbjOEVzKFULIPtEf7uilkXCjwJpDplpp9Ng5FHDfGKNiPT3lMyAgVaetbGeHGXGBN9s1Teukvwt6BJom38u2DppUUYWdI= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1776158123; c=relaxed/simple; bh=fxND62kk/BNWeW2usnWLW8esxN1tMBZh3OkSlWRq17Q=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version:Content-Type; b=MGgoou3ElrkWdAkVSt2VfWrS48elSGgnV81MtU5nEMgHNb8uchRqnTnn0dGSqKSQchNzU3b768NM6sojK6Nu7zl5PIiFhsVHEJ/9r/bSdaUs30vP2G1ad/5M7whzvYfUzKXk+v07FqNb5/F/EQ4h6W39KikHeyUnReLtoeg4rdM= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=qW8uH5g3; arc=none smtp.client-ip=10.30.226.201 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="qW8uH5g3" Received: by smtp.kernel.org (Postfix) with ESMTPSA id B4932C19425; Tue, 14 Apr 2026 09:15:22 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1776158123; bh=fxND62kk/BNWeW2usnWLW8esxN1tMBZh3OkSlWRq17Q=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=qW8uH5g3X2vVv/qRPrqVpLzNecIzMjlPaFXRRux4aeLxisOGTUDdh8M2tJGCmpFt3 BBm1r/l6+o9oyyJHaMD3bDfDAFpVjidn6ruYMLS8s3YHhrhCzPqHOi0wvoWE2viSAp 2A41/dIn+t0Irnoqb2nuA9egOG4rOS1EjiZwG35btbaNOI9VjylRW9nYPsR4x22kA4 K+jfpqQIaARyxKwrMJJHDEMfo0YOr8p2V53zb9+z7T8MrKu37L1VXzlIw89MlVXwnX 1ErCmR0G5tlGEeoEQRBc8bGnHKhrp0iwoSTKRKyhS76JGN2zX1v9WwOwGsANBu/F1n smIDwswtdBkdQ== From: "Masami Hiramatsu (Google)" To: Steven Rostedt , Masami Hiramatsu Cc: Menglong Dong , Mathieu Desnoyers , jiang.biao@linux.dev, linux-kernel@vger.kernel.org, linux-trace-kernel@vger.kernel.org Subject: [PATCH v6 5/5] tracing/fprobe: Fix to unregister ftrace_ops if it is empty on module unloading Date: Tue, 14 Apr 2026 18:15:21 +0900 Message-ID: <177615812107.1165997.3971187297247500979.stgit@mhiramat.tok.corp.google.com> X-Mailer: git-send-email 2.54.0.rc0.605.g598a273b03-goog In-Reply-To: <177615807787.1165997.921227352050738693.stgit@mhiramat.tok.corp.google.com> References: <177615807787.1165997.921227352050738693.stgit@mhiramat.tok.corp.google.com> User-Agent: StGit/0.19 Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Type: text/plain; charset="utf-8" Content-Transfer-Encoding: quoted-printable From: Masami Hiramatsu (Google) Fix fprobe to unregister ftrace_ops if corresponding type of fprobe does not exist on the fprobe_ip_table and it is expected to be empty when unloading modules. Since ftrace thinks that the empty hash means everything to be traced, if we set fprobes only on the unloaded module, all functions are traced unexpectedly after unloading module. e.g. # modprobe xt_LOG.ko # echo 'f:test log_tg*' > dynamic_events # echo 1 > events/fprobes/test/enable # cat enabled_functions log_tg [xt_LOG] (1) tramp: 0xffffffffa0004000 (fprobe_ftrace_en= try+0x0/0x490) ->fprobe_ftrace_entry+0x0/0x490 log_tg_check [xt_LOG] (1) tramp: 0xffffffffa0004000 (fprobe_f= trace_entry+0x0/0x490) ->fprobe_ftrace_entry+0x0/0x490 log_tg_destroy [xt_LOG] (1) tramp: 0xffffffffa0004000 (fprobe_f= trace_entry+0x0/0x490) ->fprobe_ftrace_entry+0x0/0x490 # rmmod xt_LOG # wc -l enabled_functions 34085 enabled_functions Signed-off-by: Masami Hiramatsu (Google) --- Changes in v6: - Newly added. --- kernel/trace/fprobe.c | 191 +++++++++++++++++++++++++++++++++++++--------= ---- 1 file changed, 143 insertions(+), 48 deletions(-) diff --git a/kernel/trace/fprobe.c b/kernel/trace/fprobe.c index 1767c2b0884c..5cbe7deb855a 100644 --- a/kernel/trace/fprobe.c +++ b/kernel/trace/fprobe.c @@ -79,7 +79,7 @@ static const struct rhashtable_params fprobe_rht_params = =3D { }; =20 /* Node insertion and deletion requires the fprobe_mutex */ -static int insert_fprobe_node(struct fprobe_hlist_node *node, struct fprob= e *fp) +static int __insert_fprobe_node(struct fprobe_hlist_node *node, struct fpr= obe *fp) { int ret; =20 @@ -92,7 +92,7 @@ static int insert_fprobe_node(struct fprobe_hlist_node *n= ode, struct fprobe *fp) return ret; } =20 -static void delete_fprobe_node(struct fprobe_hlist_node *node) +static void __delete_fprobe_node(struct fprobe_hlist_node *node) { lockdep_assert_held(&fprobe_mutex); =20 @@ -250,7 +250,63 @@ static inline int __fprobe_kprobe_handler(unsigned lon= g ip, unsigned long parent return ret; } =20 +static int fprobe_fgraph_entry(struct ftrace_graph_ent *trace, struct fgra= ph_ops *gops, + struct ftrace_regs *fregs); +static void fprobe_return(struct ftrace_graph_ret *trace, + struct fgraph_ops *gops, + struct ftrace_regs *fregs); + +static struct fgraph_ops fprobe_graph_ops =3D { + .entryfunc =3D fprobe_fgraph_entry, + .retfunc =3D fprobe_return, +}; +static int fprobe_graph_active; +/* Number of fgraph fprobes */ +static int nr_fgraph_fprobes; + +/* Add @addrs to the ftrace filter and register fgraph if needed. */ +static int fprobe_graph_add_ips(unsigned long *addrs, int num) +{ + int ret; + + lockdep_assert_held(&fprobe_mutex); + + ret =3D ftrace_set_filter_ips(&fprobe_graph_ops.ops, addrs, num, 0, 0); + if (ret) + return ret; + + if (!fprobe_graph_active) { + ret =3D register_ftrace_graph(&fprobe_graph_ops); + if (WARN_ON_ONCE(ret)) { + ftrace_free_filter(&fprobe_graph_ops.ops); + return ret; + } + } + fprobe_graph_active++; + return 0; +} + +/* Remove @addrs from the ftrace filter and unregister fgraph if possible.= */ +static void fprobe_graph_remove_ips(unsigned long *addrs, int num) +{ + lockdep_assert_held(&fprobe_mutex); + + if (!fprobe_graph_active) + return; + fprobe_graph_active--; + if (!fprobe_graph_active) { + unregister_ftrace_graph(&fprobe_graph_ops); + ftrace_free_filter(&fprobe_graph_ops.ops); + } + + if (num) + ftrace_set_filter_ips(&fprobe_graph_ops.ops, addrs, num, 1, 0); +} + #if defined(CONFIG_DYNAMIC_FTRACE_WITH_ARGS) || defined(CONFIG_DYNAMIC_FTR= ACE_WITH_REGS) +/* Number of ftrace fprobes */ +static int nr_ftrace_fprobes; + /* ftrace_ops callback, this processes fprobes which have only entry_handl= er. */ static void fprobe_ftrace_entry(unsigned long ip, unsigned long parent_ip, struct ftrace_ops *ops, struct ftrace_regs *fregs) @@ -320,9 +376,14 @@ static void fprobe_ftrace_remove_ips(unsigned long *ad= drs, int num) { lockdep_assert_held(&fprobe_mutex); =20 - fprobe_ftrace_active--; if (!fprobe_ftrace_active) + return; + + fprobe_ftrace_active--; + if (!fprobe_ftrace_active) { unregister_ftrace_function(&fprobe_ftrace_ops); + ftrace_free_filter(&fprobe_ftrace_ops); + } if (num) ftrace_set_filter_ips(&fprobe_ftrace_ops, addrs, num, 1, 0); } @@ -332,6 +393,40 @@ static bool fprobe_is_ftrace(struct fprobe *fp) return !fp->exit_handler; } =20 +/* Node insertion and deletion requires the fprobe_mutex */ +static int insert_fprobe_node(struct fprobe_hlist_node *node, struct fprob= e *fp) +{ + int ret; + + lockdep_assert_held(&fprobe_mutex); + + ret =3D __insert_fprobe_node(node, fp); + if (!ret) { + if (fprobe_is_ftrace(fp)) + nr_ftrace_fprobes++; + else + nr_fgraph_fprobes++; + } + + return ret; +} + +static void delete_fprobe_node(struct fprobe_hlist_node *node) +{ + struct fprobe *fp; + + lockdep_assert_held(&fprobe_mutex); + + fp =3D READ_ONCE(node->fp); + if (fp) { + if (fprobe_is_ftrace(fp)) + nr_ftrace_fprobes--; + else + nr_fgraph_fprobes--; + } + __delete_fprobe_node(node); +} + static bool fprobe_exists_on_hash(unsigned long ip, bool ftrace) { struct rhlist_head *head, *pos; @@ -361,8 +456,19 @@ static bool fprobe_exists_on_hash(unsigned long ip, bo= ol ftrace) #ifdef CONFIG_MODULES static void fprobe_remove_ips(unsigned long *ips, unsigned int cnt) { - ftrace_set_filter_ips(&fprobe_graph_ops.ops, ips, cnt, 1, 0); - ftrace_set_filter_ips(&fprobe_ftrace_ops, ips, cnt, 1, 0); + if (!nr_fgraph_fprobes && fprobe_graph_active) { + unregister_ftrace_graph(&fprobe_graph_ops); + ftrace_free_filter(&fprobe_graph_ops.ops); + fprobe_graph_active =3D 0; + } else + ftrace_set_filter_ips(&fprobe_graph_ops.ops, ips, cnt, 1, 0); + + if (!nr_ftrace_fprobes && fprobe_ftrace_active) { + unregister_ftrace_function(&fprobe_ftrace_ops); + ftrace_free_filter(&fprobe_ftrace_ops); + fprobe_ftrace_active =3D 0; + } else + ftrace_set_filter_ips(&fprobe_ftrace_ops, ips, cnt, 1, 0); } #endif #else @@ -380,6 +486,32 @@ static bool fprobe_is_ftrace(struct fprobe *fp) return false; } =20 +/* Node insertion and deletion requires the fprobe_mutex */ +static int insert_fprobe_node(struct fprobe_hlist_node *node, struct fprob= e *fp) +{ + int ret; + + lockdep_assert_held(&fprobe_mutex); + + ret =3D __insert_fprobe_node(node, fp); + if (!ret) + nr_fgraph_fprobes++; + + return ret; +} + +static void delete_fprobe_node(struct fprobe_hlist_node *node) +{ + struct fprobe *fp; + + lockdep_assert_held(&fprobe_mutex); + + fp =3D READ_ONCE(node->fp); + if (fp) + nr_fgraph_fprobes--; + __delete_fprobe_node(node); +} + static bool fprobe_exists_on_hash(unsigned long ip, bool ftrace __maybe_un= used) { struct rhlist_head *head, *pos; @@ -406,7 +538,12 @@ static bool fprobe_exists_on_hash(unsigned long ip, bo= ol ftrace __maybe_unused) #ifdef CONFIG_MODULES static void fprobe_remove_ips(unsigned long *ips, unsigned int cnt) { - ftrace_set_filter_ips(&fprobe_graph_ops.ops, ips, cnt, 1, 0); + if (!nr_fgraph_fprobes && fprobe_graph_active) { + unregister_ftrace_graph(&fprobe_graph_ops); + ftrace_free_filter(&fprobe_graph_ops.ops); + fprobe_graph_active =3D 0; + } else + ftrace_set_filter_ips(&fprobe_graph_ops.ops, ips, cnt, 1, 0); } #endif #endif /* !CONFIG_DYNAMIC_FTRACE_WITH_ARGS && !CONFIG_DYNAMIC_FTRACE_WITH_= REGS */ @@ -536,48 +673,6 @@ static void fprobe_return(struct ftrace_graph_ret *tra= ce, } NOKPROBE_SYMBOL(fprobe_return); =20 -static struct fgraph_ops fprobe_graph_ops =3D { - .entryfunc =3D fprobe_fgraph_entry, - .retfunc =3D fprobe_return, -}; -static int fprobe_graph_active; - -/* Add @addrs to the ftrace filter and register fgraph if needed. */ -static int fprobe_graph_add_ips(unsigned long *addrs, int num) -{ - int ret; - - lockdep_assert_held(&fprobe_mutex); - - ret =3D ftrace_set_filter_ips(&fprobe_graph_ops.ops, addrs, num, 0, 0); - if (ret) - return ret; - - if (!fprobe_graph_active) { - ret =3D register_ftrace_graph(&fprobe_graph_ops); - if (WARN_ON_ONCE(ret)) { - ftrace_free_filter(&fprobe_graph_ops.ops); - return ret; - } - } - fprobe_graph_active++; - return 0; -} - -/* Remove @addrs from the ftrace filter and unregister fgraph if possible.= */ -static void fprobe_graph_remove_ips(unsigned long *addrs, int num) -{ - lockdep_assert_held(&fprobe_mutex); - - fprobe_graph_active--; - /* Q: should we unregister it ? */ - if (!fprobe_graph_active) - unregister_ftrace_graph(&fprobe_graph_ops); - - if (num) - ftrace_set_filter_ips(&fprobe_graph_ops.ops, addrs, num, 1, 0); -} - #ifdef CONFIG_MODULES =20 #define FPROBE_IPS_BATCH_INIT 128