From: Xu Kuohai <xukuohai@huawei.com>
Pass bpf_verifier_env to bpf_int_jit_compile(). The follow-up patch will
use env->insn_aux_data in the JIT stage to detect indirect jump targets.
Since bpf_prog_select_runtime() can be called by cbpf and lib/test_bpf.c
code without verifier, introduce helper __bpf_prog_select_runtime()
to accept the env parameter.
Remove the call to bpf_prog_select_runtime() in bpf_prog_load(), and
switch to call __bpf_prog_select_runtime() in the verifier, with env
variable passed. The original bpf_prog_select_runtime() is preserved for
cbpf and lib/test_bpf.c, where env is NULL.
Now all constants blinding calls are moved into the verifier, except
the cbpf and lib/test_bpf.c cases. The instructions arrays are adjusted
by bpf_patch_insn_data() function for normal cases, so there is no need
to call adjust_insn_arrays() in bpf_jit_blind_constants(). Remove it.
Reviewed-by: Anton Protopopov <a.s.protopopov@gmail.com>
Signed-off-by: Xu Kuohai <xukuohai@huawei.com>
---
arch/arc/net/bpf_jit_core.c | 2 +-
arch/arm/net/bpf_jit_32.c | 2 +-
arch/arm64/net/bpf_jit_comp.c | 2 +-
arch/loongarch/net/bpf_jit.c | 2 +-
arch/mips/net/bpf_jit_comp.c | 2 +-
arch/parisc/net/bpf_jit_core.c | 2 +-
arch/powerpc/net/bpf_jit_comp.c | 2 +-
arch/riscv/net/bpf_jit_core.c | 2 +-
arch/s390/net/bpf_jit_comp.c | 2 +-
arch/sparc/net/bpf_jit_comp_64.c | 2 +-
arch/x86/net/bpf_jit_comp.c | 2 +-
arch/x86/net/bpf_jit_comp32.c | 2 +-
include/linux/filter.h | 17 +++++-
kernel/bpf/core.c | 93 +++++++++++++++++---------------
kernel/bpf/syscall.c | 4 --
kernel/bpf/verifier.c | 36 +++++++------
16 files changed, 98 insertions(+), 76 deletions(-)
diff --git a/arch/arc/net/bpf_jit_core.c b/arch/arc/net/bpf_jit_core.c
index 973ceae48675..639a2736f029 100644
--- a/arch/arc/net/bpf_jit_core.c
+++ b/arch/arc/net/bpf_jit_core.c
@@ -1400,7 +1400,7 @@ static struct bpf_prog *do_extra_pass(struct bpf_prog *prog)
* (re)locations involved that their addresses are not known
* during the first run.
*/
-struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
+struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog)
{
vm_dump(prog);
diff --git a/arch/arm/net/bpf_jit_32.c b/arch/arm/net/bpf_jit_32.c
index e6b1bb2de627..1628b6fc70a4 100644
--- a/arch/arm/net/bpf_jit_32.c
+++ b/arch/arm/net/bpf_jit_32.c
@@ -2142,7 +2142,7 @@ bool bpf_jit_needs_zext(void)
return true;
}
-struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
+struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog)
{
struct bpf_binary_header *header;
struct jit_ctx ctx;
diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c
index cd5a72fff500..7212ec89dfe3 100644
--- a/arch/arm64/net/bpf_jit_comp.c
+++ b/arch/arm64/net/bpf_jit_comp.c
@@ -2006,7 +2006,7 @@ struct arm64_jit_data {
struct jit_ctx ctx;
};
-struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
+struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog)
{
int image_size, prog_size, extable_size, extable_align, extable_offset;
struct bpf_binary_header *header;
diff --git a/arch/loongarch/net/bpf_jit.c b/arch/loongarch/net/bpf_jit.c
index fcc8c0c29fb0..5149ce4cef7e 100644
--- a/arch/loongarch/net/bpf_jit.c
+++ b/arch/loongarch/net/bpf_jit.c
@@ -1920,7 +1920,7 @@ int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags,
return ret < 0 ? ret : ret * LOONGARCH_INSN_SIZE;
}
-struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
+struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog)
{
bool extra_pass = false;
u8 *image_ptr, *ro_image_ptr;
diff --git a/arch/mips/net/bpf_jit_comp.c b/arch/mips/net/bpf_jit_comp.c
index d2b6c955f18e..6ee4abe6a1f7 100644
--- a/arch/mips/net/bpf_jit_comp.c
+++ b/arch/mips/net/bpf_jit_comp.c
@@ -909,7 +909,7 @@ bool bpf_jit_needs_zext(void)
return true;
}
-struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
+struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog)
{
struct bpf_binary_header *header = NULL;
struct jit_context ctx;
diff --git a/arch/parisc/net/bpf_jit_core.c b/arch/parisc/net/bpf_jit_core.c
index 35dca372b5df..172770132440 100644
--- a/arch/parisc/net/bpf_jit_core.c
+++ b/arch/parisc/net/bpf_jit_core.c
@@ -41,7 +41,7 @@ bool bpf_jit_needs_zext(void)
return true;
}
-struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
+struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog)
{
unsigned int prog_size = 0, extable_size = 0;
bool extra_pass = false;
diff --git a/arch/powerpc/net/bpf_jit_comp.c b/arch/powerpc/net/bpf_jit_comp.c
index 711028bebea3..27fecb4cc063 100644
--- a/arch/powerpc/net/bpf_jit_comp.c
+++ b/arch/powerpc/net/bpf_jit_comp.c
@@ -129,7 +129,7 @@ bool bpf_jit_needs_zext(void)
return true;
}
-struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp)
+struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *fp)
{
u32 proglen;
u32 alloclen;
diff --git a/arch/riscv/net/bpf_jit_core.c b/arch/riscv/net/bpf_jit_core.c
index 527baa50dc68..768ac686b359 100644
--- a/arch/riscv/net/bpf_jit_core.c
+++ b/arch/riscv/net/bpf_jit_core.c
@@ -41,7 +41,7 @@ bool bpf_jit_needs_zext(void)
return true;
}
-struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
+struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog)
{
unsigned int prog_size = 0, extable_size = 0;
bool extra_pass = false;
diff --git a/arch/s390/net/bpf_jit_comp.c b/arch/s390/net/bpf_jit_comp.c
index 2dfc279b1be2..94128fe6be23 100644
--- a/arch/s390/net/bpf_jit_comp.c
+++ b/arch/s390/net/bpf_jit_comp.c
@@ -2312,7 +2312,7 @@ static struct bpf_binary_header *bpf_jit_alloc(struct bpf_jit *jit,
/*
* Compile eBPF program "fp"
*/
-struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp)
+struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *fp)
{
struct bpf_binary_header *header;
struct s390_jit_data *jit_data;
diff --git a/arch/sparc/net/bpf_jit_comp_64.c b/arch/sparc/net/bpf_jit_comp_64.c
index e83e29137566..2fa0e9375127 100644
--- a/arch/sparc/net/bpf_jit_comp_64.c
+++ b/arch/sparc/net/bpf_jit_comp_64.c
@@ -1477,7 +1477,7 @@ struct sparc64_jit_data {
struct jit_ctx ctx;
};
-struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
+struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog)
{
struct sparc64_jit_data *jit_data;
struct bpf_binary_header *header;
diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c
index 77d00a8dec87..72d9a5faa230 100644
--- a/arch/x86/net/bpf_jit_comp.c
+++ b/arch/x86/net/bpf_jit_comp.c
@@ -3713,7 +3713,7 @@ struct x64_jit_data {
#define MAX_PASSES 20
#define PADDING_PASSES (MAX_PASSES - 5)
-struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
+struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog)
{
struct bpf_binary_header *rw_header = NULL;
struct bpf_binary_header *header = NULL;
diff --git a/arch/x86/net/bpf_jit_comp32.c b/arch/x86/net/bpf_jit_comp32.c
index 5f259577614a..852baf2e4db4 100644
--- a/arch/x86/net/bpf_jit_comp32.c
+++ b/arch/x86/net/bpf_jit_comp32.c
@@ -2518,7 +2518,7 @@ bool bpf_jit_needs_zext(void)
return true;
}
-struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
+struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog)
{
struct bpf_binary_header *header = NULL;
int proglen, oldproglen = 0;
diff --git a/include/linux/filter.h b/include/linux/filter.h
index d396e55c9a1d..83f37d38c5c1 100644
--- a/include/linux/filter.h
+++ b/include/linux/filter.h
@@ -1107,6 +1107,8 @@ static inline int sk_filter_reason(struct sock *sk, struct sk_buff *skb,
return sk_filter_trim_cap(sk, skb, 1, reason);
}
+struct bpf_prog *__bpf_prog_select_runtime(struct bpf_verifier_env *env, struct bpf_prog *fp,
+ int *err);
struct bpf_prog *bpf_prog_select_runtime(struct bpf_prog *fp, int *err);
void bpf_prog_free(struct bpf_prog *fp);
@@ -1152,7 +1154,7 @@ u64 __bpf_call_base(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5);
((u64 (*)(u64, u64, u64, u64, u64, const struct bpf_insn *)) \
(void *)__bpf_call_base)
-struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog);
+struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog);
void bpf_jit_compile(struct bpf_prog *prog);
bool bpf_jit_needs_zext(void);
bool bpf_jit_inlines_helper_call(s32 imm);
@@ -1187,12 +1189,25 @@ struct bpf_prog *bpf_patch_insn_single(struct bpf_prog *prog, u32 off,
#ifdef CONFIG_BPF_SYSCALL
struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off,
const struct bpf_insn *patch, u32 len);
+struct bpf_insn_aux_data *bpf_dup_insn_aux_data(struct bpf_verifier_env *env);
+void bpf_restore_insn_aux_data(struct bpf_verifier_env *env,
+ struct bpf_insn_aux_data *orig_insn_aux);
#else
static inline struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off,
const struct bpf_insn *patch, u32 len)
{
return ERR_PTR(-ENOTSUPP);
}
+
+static inline struct bpf_insn_aux_data *bpf_dup_insn_aux_data(struct bpf_verifier_env *env)
+{
+ return NULL;
+}
+
+static inline void bpf_restore_insn_aux_data(struct bpf_verifier_env *env,
+ struct bpf_insn_aux_data *orig_insn_aux)
+{
+}
#endif /* CONFIG_BPF_SYSCALL */
int bpf_remove_insns(struct bpf_prog *prog, u32 off, u32 cnt);
diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c
index cc61fe57b98d..093ab0f68c81 100644
--- a/kernel/bpf/core.c
+++ b/kernel/bpf/core.c
@@ -1489,23 +1489,6 @@ void bpf_jit_prog_release_other(struct bpf_prog *fp, struct bpf_prog *fp_other)
bpf_prog_clone_free(fp_other);
}
-static void adjust_insn_arrays(struct bpf_prog *prog, u32 off, u32 len)
-{
-#ifdef CONFIG_BPF_SYSCALL
- struct bpf_map *map;
- int i;
-
- if (len <= 1)
- return;
-
- for (i = 0; i < prog->aux->used_map_cnt; i++) {
- map = prog->aux->used_maps[i];
- if (map->map_type == BPF_MAP_TYPE_INSN_ARRAY)
- bpf_insn_array_adjust(map, off, len);
- }
-#endif
-}
-
/* Now this function is used only to blind the main prog and must be invoked only when
* bpf_prog_need_blind() returns true.
*/
@@ -1577,12 +1560,6 @@ struct bpf_prog *bpf_jit_blind_constants(struct bpf_verifier_env *env, struct bp
if (env)
env->prog = clone;
- else
- /* Instructions arrays must be updated using absolute xlated offsets.
- * The arrays have already been adjusted by bpf_patch_insn_data() when
- * env is not NULL.
- */
- adjust_insn_arrays(clone, i, rewritten);
/* Walk new program and skip insns we just inserted. */
insn = clone->insnsi + i + insn_delta;
@@ -2551,47 +2528,63 @@ static bool bpf_prog_select_interpreter(struct bpf_prog *fp)
return select_interpreter;
}
-static struct bpf_prog *bpf_prog_jit_compile(struct bpf_prog *prog)
+static struct bpf_prog *bpf_prog_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog)
{
#ifdef CONFIG_BPF_JIT
bool blinded = false;
struct bpf_prog *orig_prog = prog;
+ struct bpf_insn_aux_data *orig_insn_aux;
if (bpf_prog_need_blind(orig_prog)) {
- prog = bpf_jit_blind_constants(NULL, orig_prog);
+ if (env) {
+ /* If env is not NULL, we are called from the end of bpf_check(), at this
+ * point, only insn_aux_data is used after failure, so we only restore it
+ * here.
+ */
+ orig_insn_aux = bpf_dup_insn_aux_data(env);
+ if (!orig_insn_aux)
+ return orig_prog;
+ }
+ prog = bpf_jit_blind_constants(env, orig_prog);
/* If blinding was requested and we failed during blinding, we must fall
* back to the interpreter.
*/
- if (IS_ERR(prog))
- return orig_prog;
+ if (IS_ERR(prog)) {
+ prog = orig_prog;
+ if (env)
+ goto out_restore;
+ else
+ return prog;
+ }
blinded = true;
}
- prog = bpf_int_jit_compile(prog);
+ prog = bpf_int_jit_compile(env, prog);
if (blinded) {
if (!prog->jited) {
bpf_jit_prog_release_other(orig_prog, prog);
prog = orig_prog;
+ if (env)
+ goto out_restore;
} else {
bpf_jit_prog_release_other(prog, orig_prog);
+ if (env)
+ goto out_free;
}
}
+
+ return prog;
+
+out_restore:
+ bpf_restore_insn_aux_data(env, orig_insn_aux);
+out_free:
+ kvfree(orig_insn_aux);
#endif
return prog;
}
-/**
- * bpf_prog_select_runtime - select exec runtime for BPF program
- * @fp: bpf_prog populated with BPF program
- * @err: pointer to error variable
- *
- * Try to JIT eBPF program, if JIT is not available, use interpreter.
- * The BPF program will be executed via bpf_prog_run() function.
- *
- * Return: the &fp argument along with &err set to 0 for success or
- * a negative errno code on failure
- */
-struct bpf_prog *bpf_prog_select_runtime(struct bpf_prog *fp, int *err)
+struct bpf_prog *__bpf_prog_select_runtime(struct bpf_verifier_env *env, struct bpf_prog *fp,
+ int *err)
{
/* In case of BPF to BPF calls, verifier did all the prep
* work with regards to JITing, etc.
@@ -2619,7 +2612,7 @@ struct bpf_prog *bpf_prog_select_runtime(struct bpf_prog *fp, int *err)
if (*err)
return fp;
- fp = bpf_prog_jit_compile(fp);
+ fp = bpf_prog_jit_compile(env, fp);
bpf_prog_jit_attempt_done(fp);
if (!fp->jited && jit_needed) {
*err = -ENOTSUPP;
@@ -2645,6 +2638,22 @@ struct bpf_prog *bpf_prog_select_runtime(struct bpf_prog *fp, int *err)
return fp;
}
+
+/**
+ * bpf_prog_select_runtime - select exec runtime for BPF program
+ * @fp: bpf_prog populated with BPF program
+ * @err: pointer to error variable
+ *
+ * Try to JIT eBPF program, if JIT is not available, use interpreter.
+ * The BPF program will be executed via bpf_prog_run() function.
+ *
+ * Return: the &fp argument along with &err set to 0 for success or
+ * a negative errno code on failure
+ */
+struct bpf_prog *bpf_prog_select_runtime(struct bpf_prog *fp, int *err)
+{
+ return __bpf_prog_select_runtime(NULL, fp, err);
+}
EXPORT_SYMBOL_GPL(bpf_prog_select_runtime);
static unsigned int __bpf_prog_ret1(const void *ctx,
@@ -3132,7 +3141,7 @@ const struct bpf_func_proto bpf_tail_call_proto = {
* It is encouraged to implement bpf_int_jit_compile() instead, so that
* eBPF and implicitly also cBPF can get JITed!
*/
-struct bpf_prog * __weak bpf_int_jit_compile(struct bpf_prog *prog)
+struct bpf_prog * __weak bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog)
{
return prog;
}
diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c
index e1505c9cd09e..553dca175640 100644
--- a/kernel/bpf/syscall.c
+++ b/kernel/bpf/syscall.c
@@ -3090,10 +3090,6 @@ static int bpf_prog_load(union bpf_attr *attr, bpfptr_t uattr, u32 uattr_size)
if (err < 0)
goto free_used_maps;
- prog = bpf_prog_select_runtime(prog, &err);
- if (err < 0)
- goto free_used_maps;
-
err = bpf_prog_mark_insn_arrays_ready(prog);
if (err < 0)
goto free_used_maps;
diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index 66cef3744fde..5084a754a748 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -22983,7 +22983,7 @@ static int convert_ctx_accesses(struct bpf_verifier_env *env)
return 0;
}
-static u32 *dup_subprog_starts(struct bpf_verifier_env *env)
+static u32 *bpf_dup_subprog_starts(struct bpf_verifier_env *env)
{
u32 *starts = NULL;
@@ -22995,13 +22995,13 @@ static u32 *dup_subprog_starts(struct bpf_verifier_env *env)
return starts;
}
-static void restore_subprog_starts(struct bpf_verifier_env *env, u32 *orig_starts)
+static void bpf_restore_subprog_starts(struct bpf_verifier_env *env, u32 *orig_starts)
{
for (int i = 0; i < env->subprog_cnt; i++)
env->subprog_info[i].start = orig_starts[i];
}
-static struct bpf_insn_aux_data *dup_insn_aux_data(struct bpf_verifier_env *env)
+struct bpf_insn_aux_data *bpf_dup_insn_aux_data(struct bpf_verifier_env *env)
{
size_t size;
@@ -23009,8 +23009,8 @@ static struct bpf_insn_aux_data *dup_insn_aux_data(struct bpf_verifier_env *env)
return kvmemdup(env->insn_aux_data, size, GFP_KERNEL_ACCOUNT);
}
-static void restore_insn_aux_data(struct bpf_verifier_env *env,
- struct bpf_insn_aux_data *orig_insn_aux)
+void bpf_restore_insn_aux_data(struct bpf_verifier_env *env,
+ struct bpf_insn_aux_data *orig_insn_aux)
{
/* the expanded elements are zero-filled, so no special handling is required */
vfree(env->insn_aux_data);
@@ -23153,7 +23153,7 @@ static int __jit_subprogs(struct bpf_verifier_env *env)
func[i]->aux->might_sleep = env->subprog_info[i].might_sleep;
if (!i)
func[i]->aux->exception_boundary = env->seen_exception;
- func[i] = bpf_int_jit_compile(func[i]);
+ func[i] = bpf_int_jit_compile(env, func[i]);
if (!func[i]->jited) {
err = -ENOTSUPP;
goto out_free;
@@ -23197,7 +23197,7 @@ static int __jit_subprogs(struct bpf_verifier_env *env)
}
for (i = 0; i < env->subprog_cnt; i++) {
old_bpf_func = func[i]->bpf_func;
- tmp = bpf_int_jit_compile(func[i]);
+ tmp = bpf_int_jit_compile(env, func[i]);
if (tmp != func[i] || func[i]->bpf_func != old_bpf_func) {
verbose(env, "JIT doesn't support bpf-to-bpf calls\n");
err = -ENOTSUPP;
@@ -23297,12 +23297,12 @@ static int jit_subprogs(struct bpf_verifier_env *env)
prog = orig_prog = env->prog;
if (bpf_prog_need_blind(orig_prog)) {
- orig_insn_aux = dup_insn_aux_data(env);
+ orig_insn_aux = bpf_dup_insn_aux_data(env);
if (!orig_insn_aux) {
err = -ENOMEM;
goto out_cleanup;
}
- orig_subprog_starts = dup_subprog_starts(env);
+ orig_subprog_starts = bpf_dup_subprog_starts(env);
if (!orig_subprog_starts) {
err = -ENOMEM;
goto out_free_aux;
@@ -23347,8 +23347,8 @@ static int jit_subprogs(struct bpf_verifier_env *env)
return 0;
out_restore:
- restore_subprog_starts(env, orig_subprog_starts);
- restore_insn_aux_data(env, orig_insn_aux);
+ bpf_restore_subprog_starts(env, orig_subprog_starts);
+ bpf_restore_insn_aux_data(env, orig_insn_aux);
kvfree(orig_subprog_starts);
out_free_aux:
kvfree(orig_insn_aux);
@@ -26523,6 +26523,14 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, __u3
adjust_btf_func(env);
+ /* extension progs temporarily inherit the attach_type of their targets
+ for verification purposes, so set it back to zero before returning
+ */
+ if (env->prog->type == BPF_PROG_TYPE_EXT)
+ env->prog->expected_attach_type = 0;
+
+ env->prog = __bpf_prog_select_runtime(env, env->prog, &ret);
+
err_release_maps:
if (ret)
release_insn_arrays(env);
@@ -26534,12 +26542,6 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, __u3
if (!env->prog->aux->used_btfs)
release_btfs(env);
- /* extension progs temporarily inherit the attach_type of their targets
- for verification purposes, so set it back to zero before returning
- */
- if (env->prog->type == BPF_PROG_TYPE_EXT)
- env->prog->expected_attach_type = 0;
-
*prog = env->prog;
module_put(env->attach_btf_mod);
--
2.43.0On Fri Apr 3, 2026 at 9:28 AM EDT, Xu Kuohai wrote:
> From: Xu Kuohai <xukuohai@huawei.com>
>
> Pass bpf_verifier_env to bpf_int_jit_compile(). The follow-up patch will
> use env->insn_aux_data in the JIT stage to detect indirect jump targets.
>
> Since bpf_prog_select_runtime() can be called by cbpf and lib/test_bpf.c
> code without verifier, introduce helper __bpf_prog_select_runtime()
> to accept the env parameter.
>
> Remove the call to bpf_prog_select_runtime() in bpf_prog_load(), and
> switch to call __bpf_prog_select_runtime() in the verifier, with env
> variable passed. The original bpf_prog_select_runtime() is preserved for
> cbpf and lib/test_bpf.c, where env is NULL.
>
> Now all constants blinding calls are moved into the verifier, except
> the cbpf and lib/test_bpf.c cases. The instructions arrays are adjusted
> by bpf_patch_insn_data() function for normal cases, so there is no need
> to call adjust_insn_arrays() in bpf_jit_blind_constants(). Remove it.
>
Provided the bot comments are fixed (also check the comment Sashiko raises about a
kvmalloc()-to-vfree mismatch):
Reviewed-by: Emil Tsalapatis <emil@etsalapatis.com>
> Reviewed-by: Anton Protopopov <a.s.protopopov@gmail.com>
> Signed-off-by: Xu Kuohai <xukuohai@huawei.com>
> ---
> arch/arc/net/bpf_jit_core.c | 2 +-
> arch/arm/net/bpf_jit_32.c | 2 +-
> arch/arm64/net/bpf_jit_comp.c | 2 +-
> arch/loongarch/net/bpf_jit.c | 2 +-
> arch/mips/net/bpf_jit_comp.c | 2 +-
> arch/parisc/net/bpf_jit_core.c | 2 +-
> arch/powerpc/net/bpf_jit_comp.c | 2 +-
> arch/riscv/net/bpf_jit_core.c | 2 +-
> arch/s390/net/bpf_jit_comp.c | 2 +-
> arch/sparc/net/bpf_jit_comp_64.c | 2 +-
> arch/x86/net/bpf_jit_comp.c | 2 +-
> arch/x86/net/bpf_jit_comp32.c | 2 +-
> include/linux/filter.h | 17 +++++-
> kernel/bpf/core.c | 93 +++++++++++++++++---------------
> kernel/bpf/syscall.c | 4 --
> kernel/bpf/verifier.c | 36 +++++++------
> 16 files changed, 98 insertions(+), 76 deletions(-)
>
> diff --git a/arch/arc/net/bpf_jit_core.c b/arch/arc/net/bpf_jit_core.c
> index 973ceae48675..639a2736f029 100644
> --- a/arch/arc/net/bpf_jit_core.c
> +++ b/arch/arc/net/bpf_jit_core.c
> @@ -1400,7 +1400,7 @@ static struct bpf_prog *do_extra_pass(struct bpf_prog *prog)
> * (re)locations involved that their addresses are not known
> * during the first run.
> */
> -struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
> +struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog)
> {
> vm_dump(prog);
>
> diff --git a/arch/arm/net/bpf_jit_32.c b/arch/arm/net/bpf_jit_32.c
> index e6b1bb2de627..1628b6fc70a4 100644
> --- a/arch/arm/net/bpf_jit_32.c
> +++ b/arch/arm/net/bpf_jit_32.c
> @@ -2142,7 +2142,7 @@ bool bpf_jit_needs_zext(void)
> return true;
> }
>
> -struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
> +struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog)
> {
> struct bpf_binary_header *header;
> struct jit_ctx ctx;
> diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c
> index cd5a72fff500..7212ec89dfe3 100644
> --- a/arch/arm64/net/bpf_jit_comp.c
> +++ b/arch/arm64/net/bpf_jit_comp.c
> @@ -2006,7 +2006,7 @@ struct arm64_jit_data {
> struct jit_ctx ctx;
> };
>
> -struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
> +struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog)
> {
> int image_size, prog_size, extable_size, extable_align, extable_offset;
> struct bpf_binary_header *header;
> diff --git a/arch/loongarch/net/bpf_jit.c b/arch/loongarch/net/bpf_jit.c
> index fcc8c0c29fb0..5149ce4cef7e 100644
> --- a/arch/loongarch/net/bpf_jit.c
> +++ b/arch/loongarch/net/bpf_jit.c
> @@ -1920,7 +1920,7 @@ int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags,
> return ret < 0 ? ret : ret * LOONGARCH_INSN_SIZE;
> }
>
> -struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
> +struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog)
> {
> bool extra_pass = false;
> u8 *image_ptr, *ro_image_ptr;
> diff --git a/arch/mips/net/bpf_jit_comp.c b/arch/mips/net/bpf_jit_comp.c
> index d2b6c955f18e..6ee4abe6a1f7 100644
> --- a/arch/mips/net/bpf_jit_comp.c
> +++ b/arch/mips/net/bpf_jit_comp.c
> @@ -909,7 +909,7 @@ bool bpf_jit_needs_zext(void)
> return true;
> }
>
> -struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
> +struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog)
> {
> struct bpf_binary_header *header = NULL;
> struct jit_context ctx;
> diff --git a/arch/parisc/net/bpf_jit_core.c b/arch/parisc/net/bpf_jit_core.c
> index 35dca372b5df..172770132440 100644
> --- a/arch/parisc/net/bpf_jit_core.c
> +++ b/arch/parisc/net/bpf_jit_core.c
> @@ -41,7 +41,7 @@ bool bpf_jit_needs_zext(void)
> return true;
> }
>
> -struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
> +struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog)
> {
> unsigned int prog_size = 0, extable_size = 0;
> bool extra_pass = false;
> diff --git a/arch/powerpc/net/bpf_jit_comp.c b/arch/powerpc/net/bpf_jit_comp.c
> index 711028bebea3..27fecb4cc063 100644
> --- a/arch/powerpc/net/bpf_jit_comp.c
> +++ b/arch/powerpc/net/bpf_jit_comp.c
> @@ -129,7 +129,7 @@ bool bpf_jit_needs_zext(void)
> return true;
> }
>
> -struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp)
> +struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *fp)
> {
> u32 proglen;
> u32 alloclen;
> diff --git a/arch/riscv/net/bpf_jit_core.c b/arch/riscv/net/bpf_jit_core.c
> index 527baa50dc68..768ac686b359 100644
> --- a/arch/riscv/net/bpf_jit_core.c
> +++ b/arch/riscv/net/bpf_jit_core.c
> @@ -41,7 +41,7 @@ bool bpf_jit_needs_zext(void)
> return true;
> }
>
> -struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
> +struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog)
> {
> unsigned int prog_size = 0, extable_size = 0;
> bool extra_pass = false;
> diff --git a/arch/s390/net/bpf_jit_comp.c b/arch/s390/net/bpf_jit_comp.c
> index 2dfc279b1be2..94128fe6be23 100644
> --- a/arch/s390/net/bpf_jit_comp.c
> +++ b/arch/s390/net/bpf_jit_comp.c
> @@ -2312,7 +2312,7 @@ static struct bpf_binary_header *bpf_jit_alloc(struct bpf_jit *jit,
> /*
> * Compile eBPF program "fp"
> */
> -struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp)
> +struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *fp)
> {
> struct bpf_binary_header *header;
> struct s390_jit_data *jit_data;
> diff --git a/arch/sparc/net/bpf_jit_comp_64.c b/arch/sparc/net/bpf_jit_comp_64.c
> index e83e29137566..2fa0e9375127 100644
> --- a/arch/sparc/net/bpf_jit_comp_64.c
> +++ b/arch/sparc/net/bpf_jit_comp_64.c
> @@ -1477,7 +1477,7 @@ struct sparc64_jit_data {
> struct jit_ctx ctx;
> };
>
> -struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
> +struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog)
> {
> struct sparc64_jit_data *jit_data;
> struct bpf_binary_header *header;
> diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c
> index 77d00a8dec87..72d9a5faa230 100644
> --- a/arch/x86/net/bpf_jit_comp.c
> +++ b/arch/x86/net/bpf_jit_comp.c
> @@ -3713,7 +3713,7 @@ struct x64_jit_data {
> #define MAX_PASSES 20
> #define PADDING_PASSES (MAX_PASSES - 5)
>
> -struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
> +struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog)
> {
> struct bpf_binary_header *rw_header = NULL;
> struct bpf_binary_header *header = NULL;
> diff --git a/arch/x86/net/bpf_jit_comp32.c b/arch/x86/net/bpf_jit_comp32.c
> index 5f259577614a..852baf2e4db4 100644
> --- a/arch/x86/net/bpf_jit_comp32.c
> +++ b/arch/x86/net/bpf_jit_comp32.c
> @@ -2518,7 +2518,7 @@ bool bpf_jit_needs_zext(void)
> return true;
> }
>
> -struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
> +struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog)
> {
> struct bpf_binary_header *header = NULL;
> int proglen, oldproglen = 0;
> diff --git a/include/linux/filter.h b/include/linux/filter.h
> index d396e55c9a1d..83f37d38c5c1 100644
> --- a/include/linux/filter.h
> +++ b/include/linux/filter.h
> @@ -1107,6 +1107,8 @@ static inline int sk_filter_reason(struct sock *sk, struct sk_buff *skb,
> return sk_filter_trim_cap(sk, skb, 1, reason);
> }
>
> +struct bpf_prog *__bpf_prog_select_runtime(struct bpf_verifier_env *env, struct bpf_prog *fp,
> + int *err);
> struct bpf_prog *bpf_prog_select_runtime(struct bpf_prog *fp, int *err);
> void bpf_prog_free(struct bpf_prog *fp);
>
> @@ -1152,7 +1154,7 @@ u64 __bpf_call_base(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5);
> ((u64 (*)(u64, u64, u64, u64, u64, const struct bpf_insn *)) \
> (void *)__bpf_call_base)
>
> -struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog);
> +struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog);
> void bpf_jit_compile(struct bpf_prog *prog);
> bool bpf_jit_needs_zext(void);
> bool bpf_jit_inlines_helper_call(s32 imm);
> @@ -1187,12 +1189,25 @@ struct bpf_prog *bpf_patch_insn_single(struct bpf_prog *prog, u32 off,
> #ifdef CONFIG_BPF_SYSCALL
> struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off,
> const struct bpf_insn *patch, u32 len);
> +struct bpf_insn_aux_data *bpf_dup_insn_aux_data(struct bpf_verifier_env *env);
> +void bpf_restore_insn_aux_data(struct bpf_verifier_env *env,
> + struct bpf_insn_aux_data *orig_insn_aux);
> #else
> static inline struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off,
> const struct bpf_insn *patch, u32 len)
> {
> return ERR_PTR(-ENOTSUPP);
> }
> +
> +static inline struct bpf_insn_aux_data *bpf_dup_insn_aux_data(struct bpf_verifier_env *env)
> +{
> + return NULL;
> +}
> +
> +static inline void bpf_restore_insn_aux_data(struct bpf_verifier_env *env,
> + struct bpf_insn_aux_data *orig_insn_aux)
> +{
> +}
> #endif /* CONFIG_BPF_SYSCALL */
>
> int bpf_remove_insns(struct bpf_prog *prog, u32 off, u32 cnt);
> diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c
> index cc61fe57b98d..093ab0f68c81 100644
> --- a/kernel/bpf/core.c
> +++ b/kernel/bpf/core.c
> @@ -1489,23 +1489,6 @@ void bpf_jit_prog_release_other(struct bpf_prog *fp, struct bpf_prog *fp_other)
> bpf_prog_clone_free(fp_other);
> }
>
> -static void adjust_insn_arrays(struct bpf_prog *prog, u32 off, u32 len)
> -{
> -#ifdef CONFIG_BPF_SYSCALL
> - struct bpf_map *map;
> - int i;
> -
> - if (len <= 1)
> - return;
> -
> - for (i = 0; i < prog->aux->used_map_cnt; i++) {
> - map = prog->aux->used_maps[i];
> - if (map->map_type == BPF_MAP_TYPE_INSN_ARRAY)
> - bpf_insn_array_adjust(map, off, len);
> - }
> -#endif
> -}
> -
> /* Now this function is used only to blind the main prog and must be invoked only when
> * bpf_prog_need_blind() returns true.
> */
> @@ -1577,12 +1560,6 @@ struct bpf_prog *bpf_jit_blind_constants(struct bpf_verifier_env *env, struct bp
>
> if (env)
> env->prog = clone;
> - else
> - /* Instructions arrays must be updated using absolute xlated offsets.
> - * The arrays have already been adjusted by bpf_patch_insn_data() when
> - * env is not NULL.
> - */
> - adjust_insn_arrays(clone, i, rewritten);
>
> /* Walk new program and skip insns we just inserted. */
> insn = clone->insnsi + i + insn_delta;
> @@ -2551,47 +2528,63 @@ static bool bpf_prog_select_interpreter(struct bpf_prog *fp)
> return select_interpreter;
> }
>
> -static struct bpf_prog *bpf_prog_jit_compile(struct bpf_prog *prog)
> +static struct bpf_prog *bpf_prog_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog)
> {
> #ifdef CONFIG_BPF_JIT
> bool blinded = false;
> struct bpf_prog *orig_prog = prog;
> + struct bpf_insn_aux_data *orig_insn_aux;
>
> if (bpf_prog_need_blind(orig_prog)) {
> - prog = bpf_jit_blind_constants(NULL, orig_prog);
> + if (env) {
> + /* If env is not NULL, we are called from the end of bpf_check(), at this
> + * point, only insn_aux_data is used after failure, so we only restore it
> + * here.
> + */
> + orig_insn_aux = bpf_dup_insn_aux_data(env);
> + if (!orig_insn_aux)
> + return orig_prog;
> + }
> + prog = bpf_jit_blind_constants(env, orig_prog);
> /* If blinding was requested and we failed during blinding, we must fall
> * back to the interpreter.
> */
> - if (IS_ERR(prog))
> - return orig_prog;
> + if (IS_ERR(prog)) {
> + prog = orig_prog;
> + if (env)
> + goto out_restore;
> + else
> + return prog;
> + }
> blinded = true;
> }
>
> - prog = bpf_int_jit_compile(prog);
> + prog = bpf_int_jit_compile(env, prog);
> if (blinded) {
> if (!prog->jited) {
> bpf_jit_prog_release_other(orig_prog, prog);
> prog = orig_prog;
> + if (env)
> + goto out_restore;
> } else {
> bpf_jit_prog_release_other(prog, orig_prog);
> + if (env)
> + goto out_free;
> }
> }
> +
> + return prog;
> +
> +out_restore:
> + bpf_restore_insn_aux_data(env, orig_insn_aux);
> +out_free:
> + kvfree(orig_insn_aux);
> #endif
> return prog;
> }
>
> -/**
> - * bpf_prog_select_runtime - select exec runtime for BPF program
> - * @fp: bpf_prog populated with BPF program
> - * @err: pointer to error variable
> - *
> - * Try to JIT eBPF program, if JIT is not available, use interpreter.
> - * The BPF program will be executed via bpf_prog_run() function.
> - *
> - * Return: the &fp argument along with &err set to 0 for success or
> - * a negative errno code on failure
> - */
> -struct bpf_prog *bpf_prog_select_runtime(struct bpf_prog *fp, int *err)
> +struct bpf_prog *__bpf_prog_select_runtime(struct bpf_verifier_env *env, struct bpf_prog *fp,
> + int *err)
> {
> /* In case of BPF to BPF calls, verifier did all the prep
> * work with regards to JITing, etc.
> @@ -2619,7 +2612,7 @@ struct bpf_prog *bpf_prog_select_runtime(struct bpf_prog *fp, int *err)
> if (*err)
> return fp;
>
> - fp = bpf_prog_jit_compile(fp);
> + fp = bpf_prog_jit_compile(env, fp);
> bpf_prog_jit_attempt_done(fp);
> if (!fp->jited && jit_needed) {
> *err = -ENOTSUPP;
> @@ -2645,6 +2638,22 @@ struct bpf_prog *bpf_prog_select_runtime(struct bpf_prog *fp, int *err)
>
> return fp;
> }
> +
> +/**
> + * bpf_prog_select_runtime - select exec runtime for BPF program
> + * @fp: bpf_prog populated with BPF program
> + * @err: pointer to error variable
> + *
> + * Try to JIT eBPF program, if JIT is not available, use interpreter.
> + * The BPF program will be executed via bpf_prog_run() function.
> + *
> + * Return: the &fp argument along with &err set to 0 for success or
> + * a negative errno code on failure
> + */
> +struct bpf_prog *bpf_prog_select_runtime(struct bpf_prog *fp, int *err)
> +{
> + return __bpf_prog_select_runtime(NULL, fp, err);
> +}
> EXPORT_SYMBOL_GPL(bpf_prog_select_runtime);
>
> static unsigned int __bpf_prog_ret1(const void *ctx,
> @@ -3132,7 +3141,7 @@ const struct bpf_func_proto bpf_tail_call_proto = {
> * It is encouraged to implement bpf_int_jit_compile() instead, so that
> * eBPF and implicitly also cBPF can get JITed!
> */
> -struct bpf_prog * __weak bpf_int_jit_compile(struct bpf_prog *prog)
> +struct bpf_prog * __weak bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog)
> {
> return prog;
> }
> diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c
> index e1505c9cd09e..553dca175640 100644
> --- a/kernel/bpf/syscall.c
> +++ b/kernel/bpf/syscall.c
> @@ -3090,10 +3090,6 @@ static int bpf_prog_load(union bpf_attr *attr, bpfptr_t uattr, u32 uattr_size)
> if (err < 0)
> goto free_used_maps;
>
> - prog = bpf_prog_select_runtime(prog, &err);
> - if (err < 0)
> - goto free_used_maps;
> -
> err = bpf_prog_mark_insn_arrays_ready(prog);
> if (err < 0)
> goto free_used_maps;
> diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
> index 66cef3744fde..5084a754a748 100644
> --- a/kernel/bpf/verifier.c
> +++ b/kernel/bpf/verifier.c
> @@ -22983,7 +22983,7 @@ static int convert_ctx_accesses(struct bpf_verifier_env *env)
> return 0;
> }
>
> -static u32 *dup_subprog_starts(struct bpf_verifier_env *env)
> +static u32 *bpf_dup_subprog_starts(struct bpf_verifier_env *env)
> {
> u32 *starts = NULL;
>
> @@ -22995,13 +22995,13 @@ static u32 *dup_subprog_starts(struct bpf_verifier_env *env)
> return starts;
> }
>
> -static void restore_subprog_starts(struct bpf_verifier_env *env, u32 *orig_starts)
> +static void bpf_restore_subprog_starts(struct bpf_verifier_env *env, u32 *orig_starts)
> {
> for (int i = 0; i < env->subprog_cnt; i++)
> env->subprog_info[i].start = orig_starts[i];
> }
>
> -static struct bpf_insn_aux_data *dup_insn_aux_data(struct bpf_verifier_env *env)
> +struct bpf_insn_aux_data *bpf_dup_insn_aux_data(struct bpf_verifier_env *env)
> {
> size_t size;
>
> @@ -23009,8 +23009,8 @@ static struct bpf_insn_aux_data *dup_insn_aux_data(struct bpf_verifier_env *env)
> return kvmemdup(env->insn_aux_data, size, GFP_KERNEL_ACCOUNT);
> }
>
> -static void restore_insn_aux_data(struct bpf_verifier_env *env,
> - struct bpf_insn_aux_data *orig_insn_aux)
> +void bpf_restore_insn_aux_data(struct bpf_verifier_env *env,
> + struct bpf_insn_aux_data *orig_insn_aux)
> {
> /* the expanded elements are zero-filled, so no special handling is required */
> vfree(env->insn_aux_data);
> @@ -23153,7 +23153,7 @@ static int __jit_subprogs(struct bpf_verifier_env *env)
> func[i]->aux->might_sleep = env->subprog_info[i].might_sleep;
> if (!i)
> func[i]->aux->exception_boundary = env->seen_exception;
> - func[i] = bpf_int_jit_compile(func[i]);
> + func[i] = bpf_int_jit_compile(env, func[i]);
> if (!func[i]->jited) {
> err = -ENOTSUPP;
> goto out_free;
> @@ -23197,7 +23197,7 @@ static int __jit_subprogs(struct bpf_verifier_env *env)
> }
> for (i = 0; i < env->subprog_cnt; i++) {
> old_bpf_func = func[i]->bpf_func;
> - tmp = bpf_int_jit_compile(func[i]);
> + tmp = bpf_int_jit_compile(env, func[i]);
> if (tmp != func[i] || func[i]->bpf_func != old_bpf_func) {
> verbose(env, "JIT doesn't support bpf-to-bpf calls\n");
> err = -ENOTSUPP;
> @@ -23297,12 +23297,12 @@ static int jit_subprogs(struct bpf_verifier_env *env)
>
> prog = orig_prog = env->prog;
> if (bpf_prog_need_blind(orig_prog)) {
> - orig_insn_aux = dup_insn_aux_data(env);
> + orig_insn_aux = bpf_dup_insn_aux_data(env);
> if (!orig_insn_aux) {
> err = -ENOMEM;
> goto out_cleanup;
> }
> - orig_subprog_starts = dup_subprog_starts(env);
> + orig_subprog_starts = bpf_dup_subprog_starts(env);
> if (!orig_subprog_starts) {
> err = -ENOMEM;
> goto out_free_aux;
> @@ -23347,8 +23347,8 @@ static int jit_subprogs(struct bpf_verifier_env *env)
> return 0;
>
> out_restore:
> - restore_subprog_starts(env, orig_subprog_starts);
> - restore_insn_aux_data(env, orig_insn_aux);
> + bpf_restore_subprog_starts(env, orig_subprog_starts);
> + bpf_restore_insn_aux_data(env, orig_insn_aux);
> kvfree(orig_subprog_starts);
> out_free_aux:
> kvfree(orig_insn_aux);
> @@ -26523,6 +26523,14 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, __u3
>
> adjust_btf_func(env);
>
> + /* extension progs temporarily inherit the attach_type of their targets
> + for verification purposes, so set it back to zero before returning
> + */
> + if (env->prog->type == BPF_PROG_TYPE_EXT)
> + env->prog->expected_attach_type = 0;
> +
> + env->prog = __bpf_prog_select_runtime(env, env->prog, &ret);
> +
> err_release_maps:
> if (ret)
> release_insn_arrays(env);
> @@ -26534,12 +26542,6 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, __u3
> if (!env->prog->aux->used_btfs)
> release_btfs(env);
>
> - /* extension progs temporarily inherit the attach_type of their targets
> - for verification purposes, so set it back to zero before returning
> - */
> - if (env->prog->type == BPF_PROG_TYPE_EXT)
> - env->prog->expected_attach_type = 0;
> -
> *prog = env->prog;
>
> module_put(env->attach_btf_mod);
> diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c
> index cc61fe57b98d..093ab0f68c81 100644
> --- a/kernel/bpf/core.c
> +++ b/kernel/bpf/core.c
[ ... ]
> @@ -2551,47 +2528,63 @@ static bool bpf_prog_select_interpreter(struct bpf_prog *fp)
> return select_interpreter;
> }
>
> -static struct bpf_prog *bpf_prog_jit_compile(struct bpf_prog *prog)
> +static struct bpf_prog *bpf_prog_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog)
> {
> #ifdef CONFIG_BPF_JIT
> bool blinded = false;
> struct bpf_prog *orig_prog = prog;
> + struct bpf_insn_aux_data *orig_insn_aux;
>
> if (bpf_prog_need_blind(orig_prog)) {
> + if (env) {
> + /* If env is not NULL, we are called from the end of bpf_check(), at this
> + * point, only insn_aux_data is used after failure, so we only restore it
> + * here.
> + */
> + orig_insn_aux = bpf_dup_insn_aux_data(env);
> + if (!orig_insn_aux)
> + return orig_prog;
> + }
> + prog = bpf_jit_blind_constants(env, orig_prog);
[ ... ]
> + return prog;
> +
> +out_restore:
> + bpf_restore_insn_aux_data(env, orig_insn_aux);
> +out_free:
> + kvfree(orig_insn_aux);
> #endif
> return prog;
> }
When the out_restore label is reached (blinding or JIT failure with
env != NULL), bpf_restore_insn_aux_data() sets env->insn_aux_data to
point to orig_insn_aux:
kernel/bpf/verifier.c:bpf_restore_insn_aux_data() {
vfree(env->insn_aux_data);
env->insn_aux_data = orig_insn_aux;
}
The code then falls through to the out_free label, which calls
kvfree(orig_insn_aux). This frees the memory that env->insn_aux_data
now points to, leaving it as a dangling pointer.
Since this function is called from inside bpf_check() via
__bpf_prog_select_runtime(), the subsequent cleanup in bpf_check()
accesses the freed memory:
kernel/bpf/verifier.c:bpf_check() {
...
clear_insn_aux_data(env, 0, env->prog->len);
vfree(env->insn_aux_data);
}
Could the out_restore path return directly instead of falling through
to out_free, since after the restore env now owns orig_insn_aux and
bpf_check() will free it during normal cleanup?
---
AI reviewed your patch. Please fix the bug or email reply why it's not a bug.
See: https://github.com/kernel-patches/vmtest/blob/master/ci/claude/README.md
CI run summary: https://github.com/kernel-patches/bpf/actions/runs/23948799625
© 2016 - 2026 Red Hat, Inc.