1. Patchew
  2. linux
  3. [PATCH bpf-next v4 0/2] bpf, x86: inline bpf_get_current_task() for x86_64
  4. View patch

[PATCH bpf-next v4 1/2] bpf, x86: inline bpf_get_current_task() for x86_64

Menglong Dong posted 2 patches 3 weeks, 6 days ago
There is a newer version of this series
[PATCH bpf-next v4 1/2] bpf, x86: inline bpf_get_current_task() for x86_64
Posted by Menglong Dong 3 weeks, 6 days ago 
Inline bpf_get_current_task() and bpf_get_current_task_btf() for x86_64
to obtain better performance.

In !CONFIG_SMP case, the percpu variable is just a normal variable, and
we can read the current_task directly.

Signed-off-by: Menglong Dong <dongml2@chinatelecom.cn>
---
v4:
- handle the !CONFIG_SMP case

v3:
- implement it in the verifier with BPF_MOV64_PERCPU_REG() instead of in
  x86_64 JIT.
---
 kernel/bpf/verifier.c | 29 +++++++++++++++++++++++++++++
 1 file changed, 29 insertions(+)

diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index 3d44c5d06623..12e99171afd8 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -17688,6 +17688,8 @@ static bool verifier_inlines_helper_call(struct bpf_verifier_env *env, s32 imm)
 	switch (imm) {
 #ifdef CONFIG_X86_64
 	case BPF_FUNC_get_smp_processor_id:
+	case BPF_FUNC_get_current_task_btf:
+	case BPF_FUNC_get_current_task:
 		return env->prog->jit_requested && bpf_jit_supports_percpu_insn();
 #endif
 	default:
@@ -23273,6 +23275,33 @@ static int do_misc_fixups(struct bpf_verifier_env *env)
 			insn      = new_prog->insnsi + i + delta;
 			goto next_insn;
 		}
+
+		/* Implement bpf_get_current_task() and bpf_get_current_task_btf() inline. */
+		if ((insn->imm == BPF_FUNC_get_current_task || insn->imm == BPF_FUNC_get_current_task_btf) &&
+		    verifier_inlines_helper_call(env, insn->imm)) {
+#ifdef CONFIG_SMP
+			insn_buf[0] = BPF_MOV64_IMM(BPF_REG_0, (u32)(unsigned long)&current_task);
+			insn_buf[1] = BPF_MOV64_PERCPU_REG(BPF_REG_0, BPF_REG_0);
+			insn_buf[2] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_0, 0);
+#else
+			struct bpf_insn ld_current_addr[2] = {
+				BPF_LD_IMM64(BPF_REG_0, (unsigned long)&current_task)
+			};
+			insn_buf[0] = ld_current_addr[0];
+			insn_buf[1] = ld_current_addr[1];
+			insn_buf[2] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_0, 0);
+#endif
+			cnt = 3;
+
+			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
+			if (!new_prog)
+				return -ENOMEM;
+
+			delta    += cnt - 1;
+			env->prog = prog = new_prog;
+			insn      = new_prog->insnsi + i + delta;
+			goto next_insn;
+		}
 #endif
 		/* Implement bpf_get_func_arg inline. */
 		if (prog_type == BPF_PROG_TYPE_TRACING &&
-- 
2.52.0
Re: [PATCH bpf-next v4 1/2] bpf, x86: inline bpf_get_current_task() for x86_64
Posted by Alexei Starovoitov 3 weeks, 5 days ago 
On Mon, Jan 12, 2026 at 2:45 AM Menglong Dong <menglong8.dong@gmail.com> wrote:
>
> Inline bpf_get_current_task() and bpf_get_current_task_btf() for x86_64
> to obtain better performance.
>
> In !CONFIG_SMP case, the percpu variable is just a normal variable, and
> we can read the current_task directly.
>
> Signed-off-by: Menglong Dong <dongml2@chinatelecom.cn>
> ---
> v4:
> - handle the !CONFIG_SMP case
>
> v3:
> - implement it in the verifier with BPF_MOV64_PERCPU_REG() instead of in
>   x86_64 JIT.
> ---
>  kernel/bpf/verifier.c | 29 +++++++++++++++++++++++++++++
>  1 file changed, 29 insertions(+)
>
> diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
> index 3d44c5d06623..12e99171afd8 100644
> --- a/kernel/bpf/verifier.c
> +++ b/kernel/bpf/verifier.c
> @@ -17688,6 +17688,8 @@ static bool verifier_inlines_helper_call(struct bpf_verifier_env *env, s32 imm)
>         switch (imm) {
>  #ifdef CONFIG_X86_64
>         case BPF_FUNC_get_smp_processor_id:
> +       case BPF_FUNC_get_current_task_btf:
> +       case BPF_FUNC_get_current_task:
>                 return env->prog->jit_requested && bpf_jit_supports_percpu_insn();
>  #endif
>         default:
> @@ -23273,6 +23275,33 @@ static int do_misc_fixups(struct bpf_verifier_env *env)
>                         insn      = new_prog->insnsi + i + delta;
>                         goto next_insn;
>                 }
> +
> +               /* Implement bpf_get_current_task() and bpf_get_current_task_btf() inline. */
> +               if ((insn->imm == BPF_FUNC_get_current_task || insn->imm == BPF_FUNC_get_current_task_btf) &&
> +                   verifier_inlines_helper_call(env, insn->imm)) {

Though verifier_inlines_helper_call() gates this with CONFIG_X86_64,
I think we still need explicit:
#if defined(CONFIG_X86_64) && !defined(CONFIG_UML)

just like we did for BPF_FUNC_get_smp_processor_id.
Please check. I suspect UML will break without it.

> +#ifdef CONFIG_SMP
> +                       insn_buf[0] = BPF_MOV64_IMM(BPF_REG_0, (u32)(unsigned long)&current_task);
> +                       insn_buf[1] = BPF_MOV64_PERCPU_REG(BPF_REG_0, BPF_REG_0);
> +                       insn_buf[2] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_0, 0);
> +#else
> +                       struct bpf_insn ld_current_addr[2] = {
> +                               BPF_LD_IMM64(BPF_REG_0, (unsigned long)&current_task)
> +                       };
> +                       insn_buf[0] = ld_current_addr[0];
> +                       insn_buf[1] = ld_current_addr[1];
> +                       insn_buf[2] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_0, 0);
> +#endif

I wouldn't bother with !SMP.
If we need to add


On Mon, Jan 12, 2026 at 2:45 AM Menglong Dong <menglong8.dong@gmail.com> wrote:
>
> Inline bpf_get_current_task() and bpf_get_current_task_btf() for x86_64
> to obtain better performance.
>
> In !CONFIG_SMP case, the percpu variable is just a normal variable, and
> we can read the current_task directly.
>
> Signed-off-by: Menglong Dong <dongml2@chinatelecom.cn>
> ---
> v4:
> - handle the !CONFIG_SMP case
>
> v3:
> - implement it in the verifier with BPF_MOV64_PERCPU_REG() instead of in
>   x86_64 JIT.
> ---
>  kernel/bpf/verifier.c | 29 +++++++++++++++++++++++++++++
>  1 file changed, 29 insertions(+)
>
> diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
> index 3d44c5d06623..12e99171afd8 100644
> --- a/kernel/bpf/verifier.c
> +++ b/kernel/bpf/verifier.c
> @@ -17688,6 +17688,8 @@ static bool verifier_inlines_helper_call(struct bpf_verifier_env *env, s32 imm)
>         switch (imm) {
>  #ifdef CONFIG_X86_64
>         case BPF_FUNC_get_smp_processor_id:
> +       case BPF_FUNC_get_current_task_btf:
> +       case BPF_FUNC_get_current_task:
>                 return env->prog->jit_requested && bpf_jit_supports_percpu_insn();
>  #endif
>         default:
> @@ -23273,6 +23275,33 @@ static int do_misc_fixups(struct bpf_verifier_env *env)
>                         insn      = new_prog->insnsi + i + delta;
>                         goto next_insn;
>                 }
> +
> +               /* Implement bpf_get_current_task() and bpf_get_current_task_btf() inline. */
> +               if ((insn->imm == BPF_FUNC_get_current_task || insn->imm == BPF_FUNC_get_current_task_btf) &&
> +                   verifier_inlines_helper_call(env, insn->imm)) {

Though verifier_inlines_helper_call() gates this with CONFIG_X86_64,
I think we still need explicit:
#if defined(CONFIG_X86_64) && !defined(CONFIG_UML)

just like we did for BPF_FUNC_get_smp_processor_id.
Please check. I suspect UML will break without it.

> +#ifdef CONFIG_SMP
> +                       insn_buf[0] = BPF_MOV64_IMM(BPF_REG_0, (u32)(unsigned long)&current_task);
> +                       insn_buf[1] = BPF_MOV64_PERCPU_REG(BPF_REG_0, BPF_REG_0);
> +                       insn_buf[2] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_0, 0);
> +#else
> +                       struct bpf_insn ld_current_addr[2] = {
> +                               BPF_LD_IMM64(BPF_REG_0, (unsigned long)&current_task)
> +                       };
> +                       insn_buf[0] = ld_current_addr[0];
> +                       insn_buf[1] = ld_current_addr[1];
> +                       insn_buf[2] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_0, 0);
> +#endif

I wouldn't bother with !SMP.
If we need to add


On Mon, Jan 12, 2026 at 2:45 AM Menglong Dong <menglong8.dong@gmail.com> wrote:
>
> Inline bpf_get_current_task() and bpf_get_current_task_btf() for x86_64
> to obtain better performance.
>
> In !CONFIG_SMP case, the percpu variable is just a normal variable, and
> we can read the current_task directly.
>
> Signed-off-by: Menglong Dong <dongml2@chinatelecom.cn>
> ---
> v4:
> - handle the !CONFIG_SMP case
>
> v3:
> - implement it in the verifier with BPF_MOV64_PERCPU_REG() instead of in
>   x86_64 JIT.
> ---
>  kernel/bpf/verifier.c | 29 +++++++++++++++++++++++++++++
>  1 file changed, 29 insertions(+)
>
> diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
> index 3d44c5d06623..12e99171afd8 100644
> --- a/kernel/bpf/verifier.c
> +++ b/kernel/bpf/verifier.c
> @@ -17688,6 +17688,8 @@ static bool verifier_inlines_helper_call(struct bpf_verifier_env *env, s32 imm)
>         switch (imm) {
>  #ifdef CONFIG_X86_64
>         case BPF_FUNC_get_smp_processor_id:
> +       case BPF_FUNC_get_current_task_btf:
> +       case BPF_FUNC_get_current_task:
>                 return env->prog->jit_requested && bpf_jit_supports_percpu_insn();
>  #endif
>         default:
> @@ -23273,6 +23275,33 @@ static int do_misc_fixups(struct bpf_verifier_env *env)
>                         insn      = new_prog->insnsi + i + delta;
>                         goto next_insn;
>                 }
> +
> +               /* Implement bpf_get_current_task() and bpf_get_current_task_btf() inline. */
> +               if ((insn->imm == BPF_FUNC_get_current_task || insn->imm == BPF_FUNC_get_current_task_btf) &&
> +                   verifier_inlines_helper_call(env, insn->imm)) {

Though verifier_inlines_helper_call() gates this with CONFIG_X86_64,
I think we still need explicit:
#if defined(CONFIG_X86_64) && !defined(CONFIG_UML)

just like we did for BPF_FUNC_get_smp_processor_id.
Please check. I suspect UML will break without it.

> +#ifdef CONFIG_SMP
> +                       insn_buf[0] = BPF_MOV64_IMM(BPF_REG_0, (u32)(unsigned long)&current_task);
> +                       insn_buf[1] = BPF_MOV64_PERCPU_REG(BPF_REG_0, BPF_REG_0);
> +                       insn_buf[2] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_0, 0);
> +#else
> +                       struct bpf_insn ld_current_addr[2] = {
> +                               BPF_LD_IMM64(BPF_REG_0, (unsigned long)&current_task)
> +                       };
> +                       insn_buf[0] = ld_current_addr[0];
> +                       insn_buf[1] = ld_current_addr[1];
> +                       insn_buf[2] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_0, 0);
> +#endif

I wouldn't bother with !SMP.
If we need to add defined(CONFIG_X86_64) && !defined(CONFIG_UML)
I would add && defined(CONFIG_SMP) to it.

pw-bot: cr

Patchew 2.1-devel-b67e4c2

© 2016 - 2026 Red Hat, Inc.