From nobody Sun Oct 5 20:10:33 2025 Received: from mailgw.kylinos.cn (mailgw.kylinos.cn [124.126.103.232]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 2539F293C6C; Wed, 30 Jul 2025 13:13:31 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=124.126.103.232 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1753881214; cv=none; b=BR4qG56GcKao9t9txXQkWMRQWhNmhbafbx7/lmjzoCK7faEK5W/uo7C9HMQc00lKfkYcbl2ep3uUHf6Zr9XgCGHii0BioQdYwcxe+LVFqI/sIUtuh3U+KrpS4AXdSGzoxlxqn0WDWpZErOTn5dOWwY+HPJdmJrRiB8ls0gUA+f0= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1753881214; c=relaxed/simple; bh=LUMOlBxRpVSF/DaA7+L6uY0i/NbsiEchRZqGxHIdjHg=; h=From:To:Cc:Subject:Date:Message-Id:In-Reply-To:References: MIME-Version; b=pBG24456jzTBldR2G+iO1FUus3na875xGsw+4WbPwlkqxnQgl/9P/8h1mxoo+deyzvD4crH7hY1W8PJ6GUsjW1xYXB6KDrQdKXSBIMpLcmR8J0rzEnT9cmYrvouVMsrzmjL21e9l8TrFoQBIvt7wZTCXHcCtB3Sy1qHf3gCW9VQ= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=none (p=none dis=none) header.from=kylinos.cn; spf=pass smtp.mailfrom=kylinos.cn; arc=none smtp.client-ip=124.126.103.232 Authentication-Results: smtp.subspace.kernel.org; dmarc=none (p=none dis=none) header.from=kylinos.cn Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=kylinos.cn X-UUID: f70674b06d4611f0b29709d653e92f7d-20250730 X-CTIC-Tags: HR_CC_COUNT, HR_CC_DOMAIN_COUNT, HR_CC_NAME, HR_CC_NO_NAME, HR_CTE_8B HR_CTT_MISS, HR_DATE_H, HR_DATE_WKD, HR_DATE_ZONE, HR_FROM_NAME HR_SJ_DIGIT_LEN, HR_SJ_LANG, HR_SJ_LEN, HR_SJ_LETTER, HR_SJ_NOR_SYM HR_SJ_PHRASE, HR_SJ_PHRASE_LEN, HR_SJ_WS, HR_TO_COUNT, HR_TO_DOMAIN_COUNT HR_TO_NO_NAME, IP_TRUSTED, SRC_TRUSTED, DN_TRUSTED, SA_EXISTED SN_EXISTED, SPF_NOPASS, DKIM_NOPASS, DMARC_NOPASS, CIE_BAD CIE_GOOD_SPF, GTI_FG_BS, GTI_RG_INFO, GTI_C_BU, AMN_T1 AMN_GOOD, AMN_C_TI, AMN_C_BU, ABX_MISS_RDNS X-CID-P-RULE: Release_Ham X-CID-O-INFO: VERSION:1.1.45,REQID:6acf9f90-2011-4819-9da4-6cd2ce9c47e8,IP:15, URL:0,TC:0,Content:0,EDM:0,RT:0,SF:-15,FILE:0,BULK:0,RULE:Release_Ham,ACTI ON:release,TS:0 X-CID-INFO: VERSION:1.1.45,REQID:6acf9f90-2011-4819-9da4-6cd2ce9c47e8,IP:15,UR L:0,TC:0,Content:0,EDM:0,RT:0,SF:-15,FILE:0,BULK:0,RULE:Release_Ham,ACTION :release,TS:0 X-CID-META: VersionHash:6493067,CLOUDID:780eeee6487f9fb36ecea225df88561c,BulkI D:2507302113222L4W0OVG,BulkQuantity:0,Recheck:0,SF:17|19|24|38|44|66|78|81 |82|102,TC:nil,Content:0|50,EDM:-3,IP:-2,URL:0,File:nil,RT:nil,Bulk:nil,QS :nil,BEC:nil,COL:0,OSI:0,OSA:0,AV:0,LES:1,SPR:NO,DKR:0,DKP:0,BRR:0,BRE:0,A RC:0 X-CID-BVR: 0 X-CID-BAS: 0,_,0,_ X-CID-FACTOR: TF_CID_SPAM_FAS,TF_CID_SPAM_FSD,TF_CID_SPAM_FSI,TF_CID_SPAM_SNR X-UUID: f70674b06d4611f0b29709d653e92f7d-20250730 X-User: duanchenghao@kylinos.cn Received: from localhost.localdomain [(116.128.244.171)] by mailgw.kylinos.cn (envelope-from ) (Generic MTA) with ESMTP id 1963060957; Wed, 30 Jul 2025 21:13:20 +0800 From: Chenghao Duan To: ast@kernel.org, daniel@iogearbox.net, andrii@kernel.org, yangtiezhu@loongson.cn, hengqi.chen@gmail.com, chenhuacai@kernel.org Cc: martin.lau@linux.dev, eddyz87@gmail.com, song@kernel.org, yonghong.song@linux.dev, john.fastabend@gmail.com, kpsingh@kernel.org, sdf@fomichev.me, haoluo@google.com, jolsa@kernel.org, kernel@xen0n.name, linux-kernel@vger.kernel.org, loongarch@lists.linux.dev, bpf@vger.kernel.org, guodongtai@kylinos.cn, duanchenghao@kylinos.cn, youling.tang@linux.dev, jianghaoran@kylinos.cn, vincent.mc.li@gmail.com, geliang@kernel.org, Youling Tang Subject: [PATCH v5 1/5] LoongArch: Add larch_insn_gen_{beq,bne} helpers Date: Wed, 30 Jul 2025 21:12:53 +0800 Message-Id: <20250730131257.124153-2-duanchenghao@kylinos.cn> X-Mailer: git-send-email 2.25.1 In-Reply-To: <20250730131257.124153-1-duanchenghao@kylinos.cn> References: <20250730131257.124153-1-duanchenghao@kylinos.cn> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Add larch_insn_gen_beq() and larch_insn_gen_bne() helpers which will be used in BPF trampoline implementation. Co-developed-by: George Guo Signed-off-by: George Guo Co-developed-by: Youling Tang Signed-off-by: Youling Tang Signed-off-by: Chenghao Duan Reviewed-by: Hengqi Chen --- arch/loongarch/include/asm/inst.h | 2 ++ arch/loongarch/kernel/inst.c | 28 ++++++++++++++++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/arch/loongarch/include/asm/inst.h b/arch/loongarch/include/asm= /inst.h index 3089785ca..2ae96a35d 100644 --- a/arch/loongarch/include/asm/inst.h +++ b/arch/loongarch/include/asm/inst.h @@ -511,6 +511,8 @@ u32 larch_insn_gen_lu12iw(enum loongarch_gpr rd, int im= m); u32 larch_insn_gen_lu32id(enum loongarch_gpr rd, int imm); u32 larch_insn_gen_lu52id(enum loongarch_gpr rd, enum loongarch_gpr rj, in= t imm); u32 larch_insn_gen_jirl(enum loongarch_gpr rd, enum loongarch_gpr rj, int = imm); +u32 larch_insn_gen_beq(enum loongarch_gpr rd, enum loongarch_gpr rj, int i= mm); +u32 larch_insn_gen_bne(enum loongarch_gpr rd, enum loongarch_gpr rj, int i= mm); =20 static inline bool signed_imm_check(long val, unsigned int bit) { diff --git a/arch/loongarch/kernel/inst.c b/arch/loongarch/kernel/inst.c index 14d7d700b..674e3b322 100644 --- a/arch/loongarch/kernel/inst.c +++ b/arch/loongarch/kernel/inst.c @@ -336,3 +336,31 @@ u32 larch_insn_gen_jirl(enum loongarch_gpr rd, enum lo= ongarch_gpr rj, int imm) =20 return insn.word; } + +u32 larch_insn_gen_beq(enum loongarch_gpr rd, enum loongarch_gpr rj, int i= mm) +{ + union loongarch_instruction insn; + + if ((imm & 3) || imm < -SZ_128K || imm >=3D SZ_128K) { + pr_warn("The generated beq instruction is out of range.\n"); + return INSN_BREAK; + } + + emit_beq(&insn, rj, rd, imm >> 2); + + return insn.word; +} + +u32 larch_insn_gen_bne(enum loongarch_gpr rd, enum loongarch_gpr rj, int i= mm) +{ + union loongarch_instruction insn; + + if ((imm & 3) || imm < -SZ_128K || imm >=3D SZ_128K) { + pr_warn("The generated bne instruction is out of range.\n"); + return INSN_BREAK; + } + + emit_bne(&insn, rj, rd, imm >> 2); + + return insn.word; +} --=20 2.25.1 From nobody Sun Oct 5 20:10:33 2025 Received: from mailgw.kylinos.cn (mailgw.kylinos.cn [124.126.103.232]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id C93E686338; Wed, 30 Jul 2025 13:13:29 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=124.126.103.232 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1753881213; cv=none; b=GIOvDkpLJHzFX5OrwXNAHN6jczYe/DSSiSRN/amKSy7D7hnbJkTt6PL1Cgj0haEuF7Oe9yKnm9QOnE6bPNcL9Rk1akNx7NFAOACkEf0uTySWqx4KU62T4N4yhE7pav9PMEd1Vl1IHnVh7RsNESDIhhT/TjvgDV5vrMeJfOM/KGI= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1753881213; c=relaxed/simple; bh=8gsKNoaRf2wBX1FEn+/nFc6lS1oUbhLbg6mIU1IDR4k=; h=From:To:Cc:Subject:Date:Message-Id:In-Reply-To:References: MIME-Version; b=XQtqxgNaf4iDN8cXyftP8T8chCFzHvXKwuznfKBQ0Ep6XUOYevNG62w19Y6uEva744UsRhbrqoSnpvQau3WGp/LljALULXh+2ykCGmqhhmSVDTkRBsvwFltgoVtVjrS1l3Q9KEY/jIXUXCf/ne3goJ5gu6BxxcPd0UyHlqLuTns= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=none (p=none dis=none) header.from=kylinos.cn; spf=pass smtp.mailfrom=kylinos.cn; arc=none smtp.client-ip=124.126.103.232 Authentication-Results: smtp.subspace.kernel.org; dmarc=none (p=none dis=none) header.from=kylinos.cn Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=kylinos.cn X-UUID: f91aa8ca6d4611f0b29709d653e92f7d-20250730 X-CTIC-Tags: HR_CC_COUNT, HR_CC_DOMAIN_COUNT, HR_CC_NO_NAME, HR_CTE_8B, HR_CTT_MISS HR_DATE_H, HR_DATE_WKD, HR_DATE_ZONE, HR_FROM_NAME, HR_SJ_DIGIT_LEN HR_SJ_LANG, HR_SJ_LEN, HR_SJ_LETTER, HR_SJ_NOR_SYM, HR_SJ_PHRASE HR_SJ_PHRASE_LEN, HR_SJ_WS, HR_TO_COUNT, HR_TO_DOMAIN_COUNT, HR_TO_NO_NAME IP_TRUSTED, SRC_TRUSTED, DN_TRUSTED, SA_EXISTED, SN_EXISTED SPF_NOPASS, DKIM_NOPASS, DMARC_NOPASS, CIE_BAD, CIE_GOOD_SPF GTI_FG_BS, GTI_RG_INFO, GTI_C_BU, AMN_T1, AMN_GOOD AMN_C_TI, AMN_C_BU, ABX_MISS_RDNS X-CID-P-RULE: Release_Ham X-CID-O-INFO: VERSION:1.1.45,REQID:f2f1176d-f414-49da-a45a-d9022fafc130,IP:15, URL:0,TC:0,Content:0,EDM:0,RT:0,SF:-15,FILE:0,BULK:0,RULE:Release_Ham,ACTI ON:release,TS:0 X-CID-INFO: VERSION:1.1.45,REQID:f2f1176d-f414-49da-a45a-d9022fafc130,IP:15,UR L:0,TC:0,Content:0,EDM:0,RT:0,SF:-15,FILE:0,BULK:0,RULE:Release_Ham,ACTION :release,TS:0 X-CID-META: VersionHash:6493067,CLOUDID:66a4d24c57cf3838fd5d3428b1aa4dd7,BulkI D:2507302113258GOLYIXK,BulkQuantity:0,Recheck:0,SF:17|19|24|38|44|66|78|81 |82|102,TC:nil,Content:0|50,EDM:-3,IP:-2,URL:0,File:nil,RT:nil,Bulk:nil,QS :nil,BEC:nil,COL:0,OSI:0,OSA:0,AV:0,LES:1,SPR:NO,DKR:0,DKP:0,BRR:0,BRE:0,A RC:0 X-CID-BVR: 0,NGT X-CID-BAS: 0,NGT,0,_ X-CID-FACTOR: TF_CID_SPAM_SNR,TF_CID_SPAM_FAS,TF_CID_SPAM_FSD,TF_CID_SPAM_FSI X-UUID: f91aa8ca6d4611f0b29709d653e92f7d-20250730 X-User: duanchenghao@kylinos.cn Received: from localhost.localdomain [(116.128.244.171)] by mailgw.kylinos.cn (envelope-from ) (Generic MTA) with ESMTP id 1056094278; Wed, 30 Jul 2025 21:13:24 +0800 From: Chenghao Duan To: ast@kernel.org, daniel@iogearbox.net, andrii@kernel.org, yangtiezhu@loongson.cn, hengqi.chen@gmail.com, chenhuacai@kernel.org Cc: martin.lau@linux.dev, eddyz87@gmail.com, song@kernel.org, yonghong.song@linux.dev, john.fastabend@gmail.com, kpsingh@kernel.org, sdf@fomichev.me, haoluo@google.com, jolsa@kernel.org, kernel@xen0n.name, linux-kernel@vger.kernel.org, loongarch@lists.linux.dev, bpf@vger.kernel.org, guodongtai@kylinos.cn, duanchenghao@kylinos.cn, youling.tang@linux.dev, jianghaoran@kylinos.cn, vincent.mc.li@gmail.com, geliang@kernel.org Subject: [PATCH v5 2/5] LoongArch: BPF: Update the code to rename validate_code to validate_ctx Date: Wed, 30 Jul 2025 21:12:54 +0800 Message-Id: <20250730131257.124153-3-duanchenghao@kylinos.cn> X-Mailer: git-send-email 2.25.1 In-Reply-To: <20250730131257.124153-1-duanchenghao@kylinos.cn> References: <20250730131257.124153-1-duanchenghao@kylinos.cn> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Rename the existing validate_code() to validate_ctx() Factor out the code validation handling into a new helper validate_code() * validate_code is used to check the validity of code. * validate_ctx is used to check both code validity and table entry correctness. The new validate_code() will be used in subsequent changes. Co-developed-by: George Guo Signed-off-by: George Guo Signed-off-by: Chenghao Duan Reviewed-by: Hengqi Chen --- arch/loongarch/net/bpf_jit.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/arch/loongarch/net/bpf_jit.c b/arch/loongarch/net/bpf_jit.c index fa1500d4a..7032f11d3 100644 --- a/arch/loongarch/net/bpf_jit.c +++ b/arch/loongarch/net/bpf_jit.c @@ -1180,6 +1180,14 @@ static int validate_code(struct jit_ctx *ctx) return -1; } =20 + return 0; +} + +static int validate_ctx(struct jit_ctx *ctx) +{ + if (validate_code(ctx)) + return -1; + if (WARN_ON_ONCE(ctx->num_exentries !=3D ctx->prog->aux->num_exentries)) return -1; =20 @@ -1288,7 +1296,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog = *prog) build_epilogue(&ctx); =20 /* 3. Extra pass to validate JITed code */ - if (validate_code(&ctx)) { + if (validate_ctx(&ctx)) { bpf_jit_binary_free(header); prog =3D orig_prog; goto out_offset; --=20 2.25.1 From nobody Sun Oct 5 20:10:33 2025 Received: from mailgw.kylinos.cn (mailgw.kylinos.cn [124.126.103.232]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 6D4C4A921; Wed, 30 Jul 2025 13:13:32 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=124.126.103.232 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1753881214; cv=none; b=tRdjEO8u7DKZu+e/yFAgS/gP+D13aM2xwUaMd7jgIAIyv0tmQrzcUu6it55mnnt38yzcw54dz9dw2gbR++CzEc1jfGend2yF9Ds5K5+JNvsO+ryGHRdnGK2i/bidGd1Ysil/pPBbQGfanjcWp08OzJ8VjuD+R0FaGdo7xcWnA/o= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1753881214; c=relaxed/simple; bh=I1dfZ625WX06wEo2cHQO5cIfwJ8aaH9vBk58GBB15o8=; h=From:To:Cc:Subject:Date:Message-Id:In-Reply-To:References: MIME-Version; b=EBmxxxmmw5fe8etsNgyWvyroAhdTZ92KuhbG5y7VguH47nXmItyDK9QAMOQipwiGAnTbFCg8Ioz6VvNRYBytyZhLYFT2N4fKs/FtijjL5stMLTo7n56l9hsqypSGSGo8PRLvoBtr3SLjQ/yS04fd+TpUtfDBaWaGRd/JTe8o3KI= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=none (p=none dis=none) header.from=kylinos.cn; spf=pass smtp.mailfrom=kylinos.cn; arc=none smtp.client-ip=124.126.103.232 Authentication-Results: smtp.subspace.kernel.org; dmarc=none (p=none dis=none) header.from=kylinos.cn Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=kylinos.cn X-UUID: fa73457e6d4611f0b29709d653e92f7d-20250730 X-CTIC-Tags: HR_CC_COUNT, HR_CC_DOMAIN_COUNT, HR_CC_NO_NAME, HR_CTE_8B, HR_CTT_MISS HR_DATE_H, HR_DATE_WKD, HR_DATE_ZONE, HR_FROM_NAME, HR_SJ_DIGIT_LEN HR_SJ_LANG, HR_SJ_LEN, HR_SJ_LETTER, HR_SJ_NOR_SYM, HR_SJ_PHRASE HR_SJ_PHRASE_LEN, HR_SJ_WS, HR_TO_COUNT, HR_TO_DOMAIN_COUNT, HR_TO_NO_NAME IP_TRUSTED, SRC_TRUSTED, DN_TRUSTED, SA_EXISTED, SN_EXISTED SPF_NOPASS, DKIM_NOPASS, DMARC_NOPASS, CIE_BAD, CIE_GOOD_SPF GTI_FG_BS, GTI_RG_INFO, GTI_C_BU, AMN_T1, AMN_GOOD AMN_C_TI, AMN_C_BU, ABX_MISS_RDNS X-CID-P-RULE: Release_Ham X-CID-O-INFO: VERSION:1.1.45,REQID:750fafda-84ec-4d36-b4ba-944dfeca72ca,IP:15, URL:0,TC:0,Content:-25,EDM:0,RT:0,SF:-15,FILE:0,BULK:0,RULE:Release_Ham,AC TION:release,TS:-25 X-CID-INFO: VERSION:1.1.45,REQID:750fafda-84ec-4d36-b4ba-944dfeca72ca,IP:15,UR L:0,TC:0,Content:-25,EDM:0,RT:0,SF:-15,FILE:0,BULK:0,RULE:Release_Ham,ACTI ON:release,TS:-25 X-CID-META: VersionHash:6493067,CLOUDID:f38d398521fd7c9f829081806d7a0498,BulkI D:2507302113286PVWODW6,BulkQuantity:0,Recheck:0,SF:17|19|24|38|44|66|78|81 |82|102,TC:nil,Content:0|50,EDM:-3,IP:-2,URL:0,File:nil,RT:nil,Bulk:nil,QS :nil,BEC:nil,COL:0,OSI:0,OSA:0,AV:0,LES:1,SPR:NO,DKR:0,DKP:0,BRR:0,BRE:0,A RC:0 X-CID-BVR: 0 X-CID-BAS: 0,_,0,_ X-CID-FACTOR: TF_CID_SPAM_SNR,TF_CID_SPAM_FAS,TF_CID_SPAM_FSD,TF_CID_SPAM_FSI X-UUID: fa73457e6d4611f0b29709d653e92f7d-20250730 X-User: duanchenghao@kylinos.cn Received: from localhost.localdomain [(116.128.244.171)] by mailgw.kylinos.cn (envelope-from ) (Generic MTA) with ESMTP id 1862959900; Wed, 30 Jul 2025 21:13:26 +0800 From: Chenghao Duan To: ast@kernel.org, daniel@iogearbox.net, andrii@kernel.org, yangtiezhu@loongson.cn, hengqi.chen@gmail.com, chenhuacai@kernel.org Cc: martin.lau@linux.dev, eddyz87@gmail.com, song@kernel.org, yonghong.song@linux.dev, john.fastabend@gmail.com, kpsingh@kernel.org, sdf@fomichev.me, haoluo@google.com, jolsa@kernel.org, kernel@xen0n.name, linux-kernel@vger.kernel.org, loongarch@lists.linux.dev, bpf@vger.kernel.org, guodongtai@kylinos.cn, duanchenghao@kylinos.cn, youling.tang@linux.dev, jianghaoran@kylinos.cn, vincent.mc.li@gmail.com, geliang@kernel.org Subject: [PATCH v5 3/5] LoongArch: BPF: Implement dynamic code modification support Date: Wed, 30 Jul 2025 21:12:55 +0800 Message-Id: <20250730131257.124153-4-duanchenghao@kylinos.cn> X-Mailer: git-send-email 2.25.1 In-Reply-To: <20250730131257.124153-1-duanchenghao@kylinos.cn> References: <20250730131257.124153-1-duanchenghao@kylinos.cn> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" This commit adds support for BPF dynamic code modification on the LoongArch architecture.: 1. Implement bpf_arch_text_poke() for runtime instruction patching. 2. Add bpf_arch_text_copy() for instruction block copying. 3. Create bpf_arch_text_invalidate() for code invalidation. On LoongArch, since symbol addresses in the direct mapping region cannot be reached via relative jump instructions from the paged mapping region, we use the move_imm+jirl instruction pair as absolute jump instructions. These require 2-5 instructions, so we reserve 5 NOP instructions in the program as placeholders for function jumps. larch_insn_text_copy is solely used for BPF. The use of larch_insn_text_copy() requires page_size alignment. Currently, only the size of the trampoline is page-aligned. Co-developed-by: George Guo Signed-off-by: George Guo Signed-off-by: Chenghao Duan Tested-by: Vincent Li --- arch/loongarch/include/asm/inst.h | 1 + arch/loongarch/kernel/inst.c | 27 ++++++++ arch/loongarch/net/bpf_jit.c | 104 ++++++++++++++++++++++++++++++ 3 files changed, 132 insertions(+) diff --git a/arch/loongarch/include/asm/inst.h b/arch/loongarch/include/asm= /inst.h index 2ae96a35d..88bb73e46 100644 --- a/arch/loongarch/include/asm/inst.h +++ b/arch/loongarch/include/asm/inst.h @@ -497,6 +497,7 @@ void arch_simulate_insn(union loongarch_instruction ins= n, struct pt_regs *regs); int larch_insn_read(void *addr, u32 *insnp); int larch_insn_write(void *addr, u32 insn); int larch_insn_patch_text(void *addr, u32 insn); +int larch_insn_text_copy(void *dst, void *src, size_t len); =20 u32 larch_insn_gen_nop(void); u32 larch_insn_gen_b(unsigned long pc, unsigned long dest); diff --git a/arch/loongarch/kernel/inst.c b/arch/loongarch/kernel/inst.c index 674e3b322..7df63a950 100644 --- a/arch/loongarch/kernel/inst.c +++ b/arch/loongarch/kernel/inst.c @@ -4,6 +4,7 @@ */ #include #include +#include =20 #include #include @@ -218,6 +219,32 @@ int larch_insn_patch_text(void *addr, u32 insn) return ret; } =20 +int larch_insn_text_copy(void *dst, void *src, size_t len) +{ + int ret; + unsigned long flags; + unsigned long dst_start, dst_end, dst_len; + + dst_start =3D round_down((unsigned long)dst, PAGE_SIZE); + dst_end =3D round_up((unsigned long)dst + len, PAGE_SIZE); + dst_len =3D dst_end - dst_start; + + set_memory_rw(dst_start, dst_len / PAGE_SIZE); + raw_spin_lock_irqsave(&patch_lock, flags); + + ret =3D copy_to_kernel_nofault(dst, src, len); + if (ret) + pr_err("%s: operation failed\n", __func__); + + raw_spin_unlock_irqrestore(&patch_lock, flags); + set_memory_rox(dst_start, dst_len / PAGE_SIZE); + + if (!ret) + flush_icache_range((unsigned long)dst, (unsigned long)dst + len); + + return ret; +} + u32 larch_insn_gen_nop(void) { return INSN_NOP; diff --git a/arch/loongarch/net/bpf_jit.c b/arch/loongarch/net/bpf_jit.c index 7032f11d3..5e6ae7e0e 100644 --- a/arch/loongarch/net/bpf_jit.c +++ b/arch/loongarch/net/bpf_jit.c @@ -4,8 +4,12 @@ * * Copyright (C) 2022 Loongson Technology Corporation Limited */ +#include #include "bpf_jit.h" =20 +#define LOONGARCH_LONG_JUMP_NINSNS 5 +#define LOONGARCH_LONG_JUMP_NBYTES (LOONGARCH_LONG_JUMP_NINSNS * 4) + #define REG_TCC LOONGARCH_GPR_A6 #define TCC_SAVED LOONGARCH_GPR_S5 =20 @@ -88,6 +92,7 @@ static u8 tail_call_reg(struct jit_ctx *ctx) */ static void build_prologue(struct jit_ctx *ctx) { + int i; int stack_adjust =3D 0, store_offset, bpf_stack_adjust; =20 bpf_stack_adjust =3D round_up(ctx->prog->aux->stack_depth, 16); @@ -98,6 +103,10 @@ static void build_prologue(struct jit_ctx *ctx) stack_adjust =3D round_up(stack_adjust, 16); stack_adjust +=3D bpf_stack_adjust; =20 + /* Reserve space for the move_imm + jirl instruction */ + for (i =3D 0; i < LOONGARCH_LONG_JUMP_NINSNS; i++) + emit_insn(ctx, nop); + /* * First instruction initializes the tail call count (TCC). * On tail call we skip this instruction, and the TCC is @@ -1367,3 +1376,98 @@ bool bpf_jit_supports_subprog_tailcalls(void) { return true; } + +static int emit_jump_and_link(struct jit_ctx *ctx, u8 rd, u64 target) +{ + if (!target) { + pr_err("bpf_jit: jump target address is error\n"); + return -EFAULT; + } + + move_imm(ctx, LOONGARCH_GPR_T1, target, false); + emit_insn(ctx, jirl, rd, LOONGARCH_GPR_T1, 0); + + return 0; +} + +static int gen_jump_or_nops(void *target, void *ip, u32 *insns, bool is_ca= ll) +{ + struct jit_ctx ctx; + + ctx.idx =3D 0; + ctx.image =3D (union loongarch_instruction *)insns; + + if (!target) { + emit_insn((&ctx), nop); + emit_insn((&ctx), nop); + return 0; + } + + return emit_jump_and_link(&ctx, is_call ? LOONGARCH_GPR_T0 : LOONGARCH_GP= R_ZERO, + (unsigned long)target); +} + +int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type poke_type, + void *old_addr, void *new_addr) +{ + u32 old_insns[LOONGARCH_LONG_JUMP_NINSNS] =3D {[0 ... 4] =3D INSN_NOP}; + u32 new_insns[LOONGARCH_LONG_JUMP_NINSNS] =3D {[0 ... 4] =3D INSN_NOP}; + bool is_call =3D poke_type =3D=3D BPF_MOD_CALL; + int ret; + + if (!is_kernel_text((unsigned long)ip) && + !is_bpf_text_address((unsigned long)ip)) + return -ENOTSUPP; + + ret =3D gen_jump_or_nops(old_addr, ip, old_insns, is_call); + if (ret) + return ret; + + if (memcmp(ip, old_insns, LOONGARCH_LONG_JUMP_NBYTES)) + return -EFAULT; + + ret =3D gen_jump_or_nops(new_addr, ip, new_insns, is_call); + if (ret) + return ret; + + mutex_lock(&text_mutex); + if (memcmp(ip, new_insns, LOONGARCH_LONG_JUMP_NBYTES)) + ret =3D larch_insn_text_copy(ip, new_insns, LOONGARCH_LONG_JUMP_NBYTES); + mutex_unlock(&text_mutex); + return ret; +} + +int bpf_arch_text_invalidate(void *dst, size_t len) +{ + int i; + int ret =3D 0; + u32 *inst; + + inst =3D kvmalloc(len, GFP_KERNEL); + if (!inst) + return -ENOMEM; + + for (i =3D 0; i < (len/sizeof(u32)); i++) + inst[i] =3D INSN_BREAK; + + mutex_lock(&text_mutex); + if (larch_insn_text_copy(dst, inst, len)) + ret =3D -EINVAL; + mutex_unlock(&text_mutex); + + kvfree(inst); + return ret; +} + +void *bpf_arch_text_copy(void *dst, void *src, size_t len) +{ + int ret; + + mutex_lock(&text_mutex); + ret =3D larch_insn_text_copy(dst, src, len); + mutex_unlock(&text_mutex); + if (ret) + return ERR_PTR(-EINVAL); + + return dst; +} --=20 2.25.1 From nobody Sun Oct 5 20:10:33 2025 Received: from mailgw.kylinos.cn (mailgw.kylinos.cn [124.126.103.232]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id CEB2929C32C; Wed, 30 Jul 2025 13:13:35 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=124.126.103.232 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1753881218; cv=none; b=VO/4gI9jFZCk0prVCa7dRq28y8e/oqGLySZcmLAgwAfvttL246pX0sEcQpevbtg9dLSz6p9NzqEA6L4P0SnF/fobYyfD2zd8KPNgqatH6QshChmLs+ui/86cH9sr4W7pYA7YUKKFxJl1rw/jX0vdL4E7lisLkIsa6OlqJLKyxZ0= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1753881218; c=relaxed/simple; bh=G2bSj6HAOlvZ3U8pYY5wothca7WAcupr/sg7e+23gA8=; h=From:To:Cc:Subject:Date:Message-Id:In-Reply-To:References: MIME-Version; b=MUNLw9F7RJMJndZ5ITMfHoN92zCL1FTD/6kbdMeyn4/FJau2TliO3rcVctzpMWdXZp6ez5EUQSdpnyuSjp2nX/LBeZPY71sznERMey2cINe3eUjGn1T6ilfe9RcobrX6sI7xSgjTH+slfvkRe4XFMKPz6NYsTffnoQrp48uUOoQ= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=none (p=none dis=none) header.from=kylinos.cn; spf=pass smtp.mailfrom=kylinos.cn; arc=none smtp.client-ip=124.126.103.232 Authentication-Results: smtp.subspace.kernel.org; dmarc=none (p=none dis=none) header.from=kylinos.cn Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=kylinos.cn X-UUID: fb9f09386d4611f0b29709d653e92f7d-20250730 X-CTIC-Tags: HR_CC_COUNT, HR_CC_DOMAIN_COUNT, HR_CC_NAME, HR_CC_NO_NAME, HR_CTE_8B HR_CTT_MISS, HR_DATE_H, HR_DATE_WKD, HR_DATE_ZONE, HR_FROM_NAME HR_SJ_DIGIT_LEN, HR_SJ_LANG, HR_SJ_LEN, HR_SJ_LETTER, HR_SJ_NOR_SYM HR_SJ_PHRASE, HR_SJ_PHRASE_LEN, HR_SJ_WS, HR_TO_COUNT, HR_TO_DOMAIN_COUNT HR_TO_NO_NAME, IP_TRUSTED, SRC_TRUSTED, DN_TRUSTED, SA_EXISTED SN_EXISTED, SPF_NOPASS, DKIM_NOPASS, DMARC_NOPASS, CIE_BAD CIE_GOOD_SPF, GTI_FG_BS, GTI_C_CI, GTI_FG_IT, GTI_RG_INFO GTI_C_BU, AMN_T1, AMN_GOOD, AMN_C_TI, AMN_C_BU ABX_MISS_RDNS X-CID-P-RULE: Release_Ham X-CID-O-INFO: VERSION:1.1.45,REQID:9eef090d-57ec-4a9f-b2b2-ca13e4b88b9f,IP:15, URL:0,TC:0,Content:0,EDM:0,RT:0,SF:-9,FILE:0,BULK:0,RULE:Release_Ham,ACTIO N:release,TS:6 X-CID-INFO: VERSION:1.1.45,REQID:9eef090d-57ec-4a9f-b2b2-ca13e4b88b9f,IP:15,UR L:0,TC:0,Content:0,EDM:0,RT:0,SF:-9,FILE:0,BULK:0,RULE:Release_Ham,ACTION: release,TS:6 X-CID-META: VersionHash:6493067,CLOUDID:5e9e005e2faceebe73ba698bb736d13c,BulkI D:250730211331AJFTUS93,BulkQuantity:0,Recheck:0,SF:17|19|24|38|43|66|74|78 |81|82|102,TC:nil,Content:0|50,EDM:-3,IP:-2,URL:99|1,File:nil,RT:nil,Bulk: nil,QS:nil,BEC:nil,COL:0,OSI:0,OSA:0,AV:0,LES:1,SPR:NO,DKR:0,DKP:0,BRR:0,B RE:0,ARC:0 X-CID-BVR: 0 X-CID-BAS: 0,_,0,_ X-CID-FACTOR: TF_CID_SPAM_SNR,TF_CID_SPAM_FAS,TF_CID_SPAM_FSD,TF_CID_SPAM_FSI, TF_CID_SPAM_ULS X-UUID: fb9f09386d4611f0b29709d653e92f7d-20250730 X-User: duanchenghao@kylinos.cn Received: from localhost.localdomain [(116.128.244.171)] by mailgw.kylinos.cn (envelope-from ) (Generic MTA) with ESMTP id 2100349911; Wed, 30 Jul 2025 21:13:28 +0800 From: Chenghao Duan To: ast@kernel.org, daniel@iogearbox.net, andrii@kernel.org, yangtiezhu@loongson.cn, hengqi.chen@gmail.com, chenhuacai@kernel.org Cc: martin.lau@linux.dev, eddyz87@gmail.com, song@kernel.org, yonghong.song@linux.dev, john.fastabend@gmail.com, kpsingh@kernel.org, sdf@fomichev.me, haoluo@google.com, jolsa@kernel.org, kernel@xen0n.name, linux-kernel@vger.kernel.org, loongarch@lists.linux.dev, bpf@vger.kernel.org, guodongtai@kylinos.cn, duanchenghao@kylinos.cn, youling.tang@linux.dev, jianghaoran@kylinos.cn, vincent.mc.li@gmail.com, geliang@kernel.org, kernel test robot Subject: [PATCH v5 4/5] LoongArch: BPF: Add bpf trampoline support for Loongarch Date: Wed, 30 Jul 2025 21:12:56 +0800 Message-Id: <20250730131257.124153-5-duanchenghao@kylinos.cn> X-Mailer: git-send-email 2.25.1 In-Reply-To: <20250730131257.124153-1-duanchenghao@kylinos.cn> References: <20250730131257.124153-1-duanchenghao@kylinos.cn> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" BPF trampoline is the critical infrastructure of the BPF subsystem, acting as a mediator between kernel functions and BPF programs. Numerous important features, such as using BPF program for zero overhead kernel introspection, rely on this key component. The related tests have passed, Including the following technical points: 1. fentry 2. fmod_ret 3. fexit The following related testcases passed on LoongArch: sudo ./test_progs -a fentry_test/fentry sudo ./test_progs -a fexit_test/fexit sudo ./test_progs -a fentry_fexit sudo ./test_progs -a modify_return sudo ./test_progs -a fexit_sleep sudo ./test_progs -a test_overhead sudo ./test_progs -a trampoline_count Reported-by: kernel test robot Closes: https://lore.kernel.org/oe-kbuild-all/202507100034.wXofj6VX-lkp@int= el.com/ Reported-by: Geliang Tang Co-developed-by: George Guo Signed-off-by: George Guo Signed-off-by: Chenghao Duan Tested-by: Tiezhu Yang Tested-by: Vincent Li --- arch/loongarch/net/bpf_jit.c | 390 +++++++++++++++++++++++++++++++++++ arch/loongarch/net/bpf_jit.h | 6 + 2 files changed, 396 insertions(+) diff --git a/arch/loongarch/net/bpf_jit.c b/arch/loongarch/net/bpf_jit.c index 5e6ae7e0e..eddf582e4 100644 --- a/arch/loongarch/net/bpf_jit.c +++ b/arch/loongarch/net/bpf_jit.c @@ -7,9 +7,15 @@ #include #include "bpf_jit.h" =20 +#define LOONGARCH_MAX_REG_ARGS 8 + #define LOONGARCH_LONG_JUMP_NINSNS 5 #define LOONGARCH_LONG_JUMP_NBYTES (LOONGARCH_LONG_JUMP_NINSNS * 4) =20 +#define LOONGARCH_FENTRY_NINSNS 2 +#define LOONGARCH_FENTRY_NBYTES (LOONGARCH_FENTRY_NINSNS * 4) +#define LOONGARCH_BPF_FENTRY_NBYTES (LOONGARCH_LONG_JUMP_NINSNS * 4) + #define REG_TCC LOONGARCH_GPR_A6 #define TCC_SAVED LOONGARCH_GPR_S5 =20 @@ -1407,6 +1413,11 @@ static int gen_jump_or_nops(void *target, void *ip, = u32 *insns, bool is_call) (unsigned long)target); } =20 +static int emit_call(struct jit_ctx *ctx, u64 addr) +{ + return emit_jump_and_link(ctx, LOONGARCH_GPR_RA, addr); +} + int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type poke_type, void *old_addr, void *new_addr) { @@ -1471,3 +1482,382 @@ void *bpf_arch_text_copy(void *dst, void *src, size= _t len) =20 return dst; } + +static void store_args(struct jit_ctx *ctx, int nargs, int args_off) +{ + int i; + + for (i =3D 0; i < nargs; i++) { + emit_insn(ctx, std, LOONGARCH_GPR_A0 + i, LOONGARCH_GPR_FP, -args_off); + args_off -=3D 8; + } +} + +static void restore_args(struct jit_ctx *ctx, int nargs, int args_off) +{ + int i; + + for (i =3D 0; i < nargs; i++) { + emit_insn(ctx, ldd, LOONGARCH_GPR_A0 + i, LOONGARCH_GPR_FP, -args_off); + args_off -=3D 8; + } +} + +static int invoke_bpf_prog(struct jit_ctx *ctx, struct bpf_tramp_link *l, + int args_off, int retval_off, + int run_ctx_off, bool save_ret) +{ + int ret; + u32 *branch; + struct bpf_prog *p =3D l->link.prog; + int cookie_off =3D offsetof(struct bpf_tramp_run_ctx, bpf_cookie); + + if (l->cookie) { + move_imm(ctx, LOONGARCH_GPR_T1, l->cookie, false); + emit_insn(ctx, std, LOONGARCH_GPR_T1, LOONGARCH_GPR_FP, -run_ctx_off + c= ookie_off); + } else { + emit_insn(ctx, std, LOONGARCH_GPR_ZERO, LOONGARCH_GPR_FP, + -run_ctx_off + cookie_off); + } + + /* arg1: prog */ + move_imm(ctx, LOONGARCH_GPR_A0, (const s64)p, false); + /* arg2: &run_ctx */ + emit_insn(ctx, addid, LOONGARCH_GPR_A1, LOONGARCH_GPR_FP, -run_ctx_off); + ret =3D emit_call(ctx, (const u64)bpf_trampoline_enter(p)); + if (ret) + return ret; + + /* store prog start time */ + move_reg(ctx, LOONGARCH_GPR_S1, LOONGARCH_GPR_A0); + + /* if (__bpf_prog_enter(prog) =3D=3D 0) + * goto skip_exec_of_prog; + * + */ + branch =3D (u32 *)ctx->image + ctx->idx; + /* nop reserved for conditional jump */ + emit_insn(ctx, nop); + + /* arg1: &args_off */ + emit_insn(ctx, addid, LOONGARCH_GPR_A0, LOONGARCH_GPR_FP, -args_off); + if (!p->jited) + move_imm(ctx, LOONGARCH_GPR_A1, (const s64)p->insnsi, false); + ret =3D emit_call(ctx, (const u64)p->bpf_func); + if (ret) + return ret; + + if (save_ret) { + emit_insn(ctx, std, LOONGARCH_GPR_A0, LOONGARCH_GPR_FP, -retval_off); + emit_insn(ctx, std, regmap[BPF_REG_0], LOONGARCH_GPR_FP, -(retval_off - = 8)); + } + + /* update branch with beqz */ + if (ctx->image) { + int offset =3D (void *)(&ctx->image[ctx->idx]) - (void *)branch; + *branch =3D larch_insn_gen_beq(LOONGARCH_GPR_A0, LOONGARCH_GPR_ZERO, off= set); + } + + /* arg1: prog */ + move_imm(ctx, LOONGARCH_GPR_A0, (const s64)p, false); + /* arg2: prog start time */ + move_reg(ctx, LOONGARCH_GPR_A1, LOONGARCH_GPR_S1); + /* arg3: &run_ctx */ + emit_insn(ctx, addid, LOONGARCH_GPR_A2, LOONGARCH_GPR_FP, -run_ctx_off); + ret =3D emit_call(ctx, (const u64)bpf_trampoline_exit(p)); + + return ret; +} + +static void invoke_bpf_mod_ret(struct jit_ctx *ctx, struct bpf_tramp_links= *tl, + int args_off, int retval_off, int run_ctx_off, u32 **branches) +{ + int i; + + emit_insn(ctx, std, LOONGARCH_GPR_ZERO, LOONGARCH_GPR_FP, -retval_off); + for (i =3D 0; i < tl->nr_links; i++) { + invoke_bpf_prog(ctx, tl->links[i], args_off, retval_off, + run_ctx_off, true); + emit_insn(ctx, ldd, LOONGARCH_GPR_T1, LOONGARCH_GPR_FP, -retval_off); + branches[i] =3D (u32 *)ctx->image + ctx->idx; + emit_insn(ctx, nop); + } +} + +u64 bpf_jit_alloc_exec_limit(void) +{ + return VMALLOC_END - VMALLOC_START; +} + +void *arch_alloc_bpf_trampoline(unsigned int size) +{ + return bpf_prog_pack_alloc(size, jit_fill_hole); +} + +void arch_free_bpf_trampoline(void *image, unsigned int size) +{ + bpf_prog_pack_free(image, size); +} + +static int __arch_prepare_bpf_trampoline(struct jit_ctx *ctx, struct bpf_t= ramp_image *im, + const struct btf_func_model *m, + struct bpf_tramp_links *tlinks, + void *func_addr, u32 flags) +{ + int i; + int stack_size =3D 0, nargs =3D 0; + int retval_off, args_off, nargs_off, ip_off, run_ctx_off, sreg_off; + struct bpf_tramp_links *fentry =3D &tlinks[BPF_TRAMP_FENTRY]; + struct bpf_tramp_links *fexit =3D &tlinks[BPF_TRAMP_FEXIT]; + struct bpf_tramp_links *fmod_ret =3D &tlinks[BPF_TRAMP_MODIFY_RETURN]; + int ret, save_ret; + void *orig_call =3D func_addr; + u32 **branches =3D NULL; + + if (flags & (BPF_TRAMP_F_ORIG_STACK | BPF_TRAMP_F_SHARE_IPMODIFY)) + return -ENOTSUPP; + + /* + * FP + 8 [ RA to parent func ] return address to parent + * function + * FP + 0 [ FP of parent func ] frame pointer of parent + * function + * FP - 8 [ T0 to traced func ] return address of traced + * function + * FP - 16 [ FP of traced func ] frame pointer of traced + * function + * + * FP - retval_off [ return value ] BPF_TRAMP_F_CALL_ORIG or + * BPF_TRAMP_F_RET_FENTRY_RET + * [ argN ] + * [ ... ] + * FP - args_off [ arg1 ] + * + * FP - nargs_off [ regs count ] + * + * FP - ip_off [ traced func ] BPF_TRAMP_F_IP_ARG + * + * FP - run_ctx_off [ bpf_tramp_run_ctx ] + * + * FP - sreg_off [ callee saved reg ] + * + */ + + if (m->nr_args > LOONGARCH_MAX_REG_ARGS) + return -ENOTSUPP; + + if (flags & (BPF_TRAMP_F_ORIG_STACK | BPF_TRAMP_F_SHARE_IPMODIFY)) + return -ENOTSUPP; + + stack_size =3D 0; + + /* room of trampoline frame to store return address and frame pointer */ + stack_size +=3D 16; + + save_ret =3D flags & (BPF_TRAMP_F_CALL_ORIG | BPF_TRAMP_F_RET_FENTRY_RET); + if (save_ret) { + /* Save BPF R0 and A0 */ + stack_size +=3D 16; + retval_off =3D stack_size; + } + + /* room of trampoline frame to store args */ + nargs =3D m->nr_args; + stack_size +=3D nargs * 8; + args_off =3D stack_size; + + /* room of trampoline frame to store args number */ + stack_size +=3D 8; + nargs_off =3D stack_size; + + /* room of trampoline frame to store ip address */ + if (flags & BPF_TRAMP_F_IP_ARG) { + stack_size +=3D 8; + ip_off =3D stack_size; + } + + /* room of trampoline frame to store struct bpf_tramp_run_ctx */ + stack_size +=3D round_up(sizeof(struct bpf_tramp_run_ctx), 8); + run_ctx_off =3D stack_size; + + stack_size +=3D 8; + sreg_off =3D stack_size; + + stack_size =3D round_up(stack_size, 16); + + /* For the trampoline called from function entry */ + /* RA and FP for parent function*/ + emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, -16); + emit_insn(ctx, std, LOONGARCH_GPR_RA, LOONGARCH_GPR_SP, 8); + emit_insn(ctx, std, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, 0); + emit_insn(ctx, addid, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, 16); + + /* RA and FP for traced function*/ + emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, -stack_size); + emit_insn(ctx, std, LOONGARCH_GPR_T0, LOONGARCH_GPR_SP, stack_size - 8); + emit_insn(ctx, std, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, stack_size - 16); + emit_insn(ctx, addid, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, stack_size); + + /* callee saved register S1 to pass start time */ + emit_insn(ctx, std, LOONGARCH_GPR_S1, LOONGARCH_GPR_FP, -sreg_off); + + /* store ip address of the traced function */ + if (flags & BPF_TRAMP_F_IP_ARG) { + move_imm(ctx, LOONGARCH_GPR_T1, (const s64)func_addr, false); + emit_insn(ctx, std, LOONGARCH_GPR_T1, LOONGARCH_GPR_FP, -ip_off); + } + + /* store nargs number*/ + move_imm(ctx, LOONGARCH_GPR_T1, nargs, false); + emit_insn(ctx, std, LOONGARCH_GPR_T1, LOONGARCH_GPR_FP, -nargs_off); + + store_args(ctx, nargs, args_off); + + /* To traced function */ + /* Ftrace jump skips 2 NOP instructions */ + if (is_kernel_text((unsigned long)orig_call)) + orig_call +=3D LOONGARCH_FENTRY_NBYTES; + /* Direct jump skips 5 NOP instructions */ + else if (is_bpf_text_address((unsigned long)orig_call)) + orig_call +=3D LOONGARCH_BPF_FENTRY_NBYTES; + + if (flags & BPF_TRAMP_F_CALL_ORIG) { + move_imm(ctx, LOONGARCH_GPR_A0, (const s64)im, false); + ret =3D emit_call(ctx, (const u64)__bpf_tramp_enter); + if (ret) + return ret; + } + + for (i =3D 0; i < fentry->nr_links; i++) { + ret =3D invoke_bpf_prog(ctx, fentry->links[i], args_off, retval_off, + run_ctx_off, flags & BPF_TRAMP_F_RET_FENTRY_RET); + if (ret) + return ret; + } + if (fmod_ret->nr_links) { + branches =3D kcalloc(fmod_ret->nr_links, sizeof(u32 *), GFP_KERNEL); + if (!branches) + return -ENOMEM; + + invoke_bpf_mod_ret(ctx, fmod_ret, args_off, retval_off, + run_ctx_off, branches); + } + + if (flags & BPF_TRAMP_F_CALL_ORIG) { + restore_args(ctx, m->nr_args, args_off); + ret =3D emit_call(ctx, (const u64)orig_call); + if (ret) + goto out; + emit_insn(ctx, std, LOONGARCH_GPR_A0, LOONGARCH_GPR_FP, -retval_off); + emit_insn(ctx, std, regmap[BPF_REG_0], LOONGARCH_GPR_FP, -(retval_off - = 8)); + im->ip_after_call =3D ctx->ro_image + ctx->idx; + /* Reserve space for the move_imm + jirl instruction */ + for (i =3D 0; i < LOONGARCH_LONG_JUMP_NINSNS; i++) + emit_insn(ctx, nop); + } + + for (i =3D 0; ctx->image && i < fmod_ret->nr_links; i++) { + int offset =3D (void *)(&ctx->image[ctx->idx]) - (void *)branches[i]; + *branches[i] =3D larch_insn_gen_bne(LOONGARCH_GPR_T1, LOONGARCH_GPR_ZERO= , offset); + } + + for (i =3D 0; i < fexit->nr_links; i++) { + ret =3D invoke_bpf_prog(ctx, fexit->links[i], args_off, retval_off, + run_ctx_off, false); + if (ret) + goto out; + } + + if (flags & BPF_TRAMP_F_CALL_ORIG) { + im->ip_epilogue =3D ctx->ro_image + ctx->idx; + move_imm(ctx, LOONGARCH_GPR_A0, (const s64)im, false); + ret =3D emit_call(ctx, (const u64)__bpf_tramp_exit); + if (ret) + goto out; + } + + if (flags & BPF_TRAMP_F_RESTORE_REGS) + restore_args(ctx, m->nr_args, args_off); + + if (save_ret) { + emit_insn(ctx, ldd, LOONGARCH_GPR_A0, LOONGARCH_GPR_FP, -retval_off); + emit_insn(ctx, ldd, regmap[BPF_REG_0], LOONGARCH_GPR_FP, -(retval_off - = 8)); + } + + emit_insn(ctx, ldd, LOONGARCH_GPR_S1, LOONGARCH_GPR_FP, -sreg_off); + + /* trampoline called from function entry */ + emit_insn(ctx, ldd, LOONGARCH_GPR_T0, LOONGARCH_GPR_SP, stack_size - 8); + emit_insn(ctx, ldd, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, stack_size - 16); + emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, stack_size); + + emit_insn(ctx, ldd, LOONGARCH_GPR_RA, LOONGARCH_GPR_SP, 8); + emit_insn(ctx, ldd, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, 0); + emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, 16); + + if (flags & BPF_TRAMP_F_SKIP_FRAME) + /* return to parent function */ + emit_insn(ctx, jirl, LOONGARCH_GPR_ZERO, LOONGARCH_GPR_RA, 0); + else + /* return to traced function */ + emit_insn(ctx, jirl, LOONGARCH_GPR_ZERO, LOONGARCH_GPR_T0, 0); + + ret =3D ctx->idx; +out: + kfree(branches); + + return ret; +} + +int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *ro_image, + void *ro_image_end, const struct btf_func_model *m, + u32 flags, struct bpf_tramp_links *tlinks, + void *func_addr) +{ + int ret; + void *image, *tmp; + struct jit_ctx ctx; + u32 size =3D ro_image_end - ro_image; + + image =3D kvmalloc(size, GFP_KERNEL); + if (!image) + return -ENOMEM; + + ctx.image =3D (union loongarch_instruction *)image; + ctx.ro_image =3D (union loongarch_instruction *)ro_image; + ctx.idx =3D 0; + + jit_fill_hole(image, (unsigned int)(ro_image_end - ro_image)); + ret =3D __arch_prepare_bpf_trampoline(&ctx, im, m, tlinks, func_addr, fla= gs); + if (ret > 0 && validate_code(&ctx) < 0) { + ret =3D -EINVAL; + goto out; + } + + tmp =3D bpf_arch_text_copy(ro_image, image, size); + if (IS_ERR(tmp)) { + ret =3D PTR_ERR(tmp); + goto out; + } + + bpf_flush_icache(ro_image, ro_image_end); +out: + kvfree(image); + return ret < 0 ? ret : size; +} + +int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags, + struct bpf_tramp_links *tlinks, void *func_addr) +{ + struct bpf_tramp_image im; + struct jit_ctx ctx; + int ret; + + ctx.image =3D NULL; + ctx.idx =3D 0; + + ret =3D __arch_prepare_bpf_trampoline(&ctx, &im, m, tlinks, func_addr, fl= ags); + + /* Page align */ + return ret < 0 ? ret : round_up(ret * LOONGARCH_INSN_SIZE, PAGE_SIZE); +} diff --git a/arch/loongarch/net/bpf_jit.h b/arch/loongarch/net/bpf_jit.h index f9c569f53..5697158fd 100644 --- a/arch/loongarch/net/bpf_jit.h +++ b/arch/loongarch/net/bpf_jit.h @@ -18,6 +18,7 @@ struct jit_ctx { u32 *offset; int num_exentries; union loongarch_instruction *image; + union loongarch_instruction *ro_image; u32 stack_size; }; =20 @@ -308,3 +309,8 @@ static inline int emit_tailcall_jmp(struct jit_ctx *ctx= , u8 cond, enum loongarch =20 return -EINVAL; } + +static inline void bpf_flush_icache(void *start, void *end) +{ + flush_icache_range((unsigned long)start, (unsigned long)end); +} --=20 2.25.1 From nobody Sun Oct 5 20:10:33 2025 Received: from mailgw.kylinos.cn (mailgw.kylinos.cn [124.126.103.232]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id D36D429A300; Wed, 30 Jul 2025 13:13:39 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=124.126.103.232 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1753881222; cv=none; b=VqKqfLqi53cGYhWu/ZpCFuMfV/DKrsOLSuxqHOCZIx3TBNdZz8s2KAbB7R8RHJ0Ysz1RatZiTGu8J1W7XCabGyrnqjvknx3YlQr2k+LqsC8JgNF1VqEliG/G93gjdbDgpxfbOstZZSVU52iiFXfK7leDGRZCLmV8JfqWAjQbeVY= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1753881222; c=relaxed/simple; bh=s2SFJKdX9ENPtxVodS2TvRTGzN4XABBgJcW6wnXoGtQ=; h=From:To:Cc:Subject:Date:Message-Id:In-Reply-To:References: MIME-Version; b=Nz0uL7Xgmykwk0Gnx3w4vJXwUCN9dwjDAkCHzZ/rRtiM4YCIzp7XUmwLTtdh+mxd9bTEKRaJV4czHGSr5rodFq8gnOnL9m/DfcA8jgh6atI3mi9iPa1FEBS5DDO1nFnOiKQQOc8ZZQpdYeOXekp1mSOQCC7m3bU/aZkPZavzLLc= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=none (p=none dis=none) header.from=kylinos.cn; spf=pass smtp.mailfrom=kylinos.cn; arc=none smtp.client-ip=124.126.103.232 Authentication-Results: smtp.subspace.kernel.org; dmarc=none (p=none dis=none) header.from=kylinos.cn Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=kylinos.cn X-UUID: fd59ff1c6d4611f0b29709d653e92f7d-20250730 X-CTIC-Tags: HR_CC_COUNT, HR_CC_DOMAIN_COUNT, HR_CC_NO_NAME, HR_CTE_8B, HR_CTT_MISS HR_DATE_H, HR_DATE_WKD, HR_DATE_ZONE, HR_FROM_NAME, HR_SJ_DIGIT_LEN HR_SJ_LANG, HR_SJ_LEN, HR_SJ_LETTER, HR_SJ_NOR_SYM, HR_SJ_PHRASE HR_SJ_PHRASE_LEN, HR_SJ_WS, HR_TO_COUNT, HR_TO_DOMAIN_COUNT, HR_TO_NO_NAME IP_TRUSTED, SRC_TRUSTED, DN_TRUSTED, SA_EXISTED, SN_EXISTED SPF_NOPASS, DKIM_NOPASS, DMARC_NOPASS, CIE_BAD, CIE_GOOD_SPF GTI_FG_BS, GTI_RG_INFO, GTI_C_BU, AMN_T1, AMN_GOOD AMN_C_TI, AMN_C_BU, ABX_MISS_RDNS X-CID-P-RULE: Release_Ham X-CID-O-INFO: VERSION:1.1.45,REQID:f50ad0ef-3db2-421d-bf79-3f952d5904d6,IP:15, URL:0,TC:0,Content:0,EDM:0,RT:0,SF:-15,FILE:0,BULK:0,RULE:Release_Ham,ACTI ON:release,TS:0 X-CID-INFO: VERSION:1.1.45,REQID:f50ad0ef-3db2-421d-bf79-3f952d5904d6,IP:15,UR L:0,TC:0,Content:0,EDM:0,RT:0,SF:-15,FILE:0,BULK:0,RULE:Release_Ham,ACTION :release,TS:0 X-CID-META: VersionHash:6493067,CLOUDID:f6e644fb00daaf355476098d584512f5,BulkI D:250730211334BS297NHW,BulkQuantity:0,Recheck:0,SF:17|19|24|38|44|66|78|81 |82|102,TC:nil,Content:0|50,EDM:-3,IP:-2,URL:0,File:nil,RT:nil,Bulk:nil,QS :nil,BEC:nil,COL:0,OSI:0,OSA:0,AV:0,LES:1,SPR:NO,DKR:0,DKP:0,BRR:0,BRE:0,A RC:0 X-CID-BVR: 0 X-CID-BAS: 0,_,0,_ X-CID-FACTOR: TF_CID_SPAM_FSI,TF_CID_SPAM_SNR,TF_CID_SPAM_FAS,TF_CID_SPAM_FSD X-UUID: fd59ff1c6d4611f0b29709d653e92f7d-20250730 X-User: duanchenghao@kylinos.cn Received: from localhost.localdomain [(116.128.244.171)] by mailgw.kylinos.cn (envelope-from ) (Generic MTA) with ESMTP id 2002870157; Wed, 30 Jul 2025 21:13:31 +0800 From: Chenghao Duan To: ast@kernel.org, daniel@iogearbox.net, andrii@kernel.org, yangtiezhu@loongson.cn, hengqi.chen@gmail.com, chenhuacai@kernel.org Cc: martin.lau@linux.dev, eddyz87@gmail.com, song@kernel.org, yonghong.song@linux.dev, john.fastabend@gmail.com, kpsingh@kernel.org, sdf@fomichev.me, haoluo@google.com, jolsa@kernel.org, kernel@xen0n.name, linux-kernel@vger.kernel.org, loongarch@lists.linux.dev, bpf@vger.kernel.org, guodongtai@kylinos.cn, duanchenghao@kylinos.cn, youling.tang@linux.dev, jianghaoran@kylinos.cn, vincent.mc.li@gmail.com, geliang@kernel.org Subject: [PATCH v5 5/5] LoongArch: BPF: Add struct ops support for trampoline Date: Wed, 30 Jul 2025 21:12:57 +0800 Message-Id: <20250730131257.124153-6-duanchenghao@kylinos.cn> X-Mailer: git-send-email 2.25.1 In-Reply-To: <20250730131257.124153-1-duanchenghao@kylinos.cn> References: <20250730131257.124153-1-duanchenghao@kylinos.cn> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" From: Tiezhu Yang Use BPF_TRAMP_F_INDIRECT flag to detect struct ops and emit proper prologue and epilogue for this case. With this patch, all of the struct_ops related testcases (except struct_ops_multi_pages) passed on LoongArch. The testcase struct_ops_multi_pages failed is because the actual image_pages_cnt is 40 which is bigger than MAX_TRAMP_IMAGE_PAGES. Before: $ sudo ./test_progs -t struct_ops -d struct_ops_multi_pages ... WATCHDOG: test case struct_ops_module/struct_ops_load executes for 10 sec= onds... After: $ sudo ./test_progs -t struct_ops -d struct_ops_multi_pages ... #15 bad_struct_ops:OK ... #399 struct_ops_autocreate:OK ... #400 struct_ops_kptr_return:OK ... #401 struct_ops_maybe_null:OK ... #402 struct_ops_module:OK ... #404 struct_ops_no_cfi:OK ... #405 struct_ops_private_stack:SKIP ... #406 struct_ops_refcounted:OK Summary: 8/25 PASSED, 3 SKIPPED, 0 FAILED Signed-off-by: Tiezhu Yang --- arch/loongarch/net/bpf_jit.c | 71 ++++++++++++++++++++++++------------ 1 file changed, 47 insertions(+), 24 deletions(-) diff --git a/arch/loongarch/net/bpf_jit.c b/arch/loongarch/net/bpf_jit.c index eddf582e4..725c2d5ee 100644 --- a/arch/loongarch/net/bpf_jit.c +++ b/arch/loongarch/net/bpf_jit.c @@ -1610,6 +1610,7 @@ static int __arch_prepare_bpf_trampoline(struct jit_c= tx *ctx, struct bpf_tramp_i struct bpf_tramp_links *fentry =3D &tlinks[BPF_TRAMP_FENTRY]; struct bpf_tramp_links *fexit =3D &tlinks[BPF_TRAMP_FEXIT]; struct bpf_tramp_links *fmod_ret =3D &tlinks[BPF_TRAMP_MODIFY_RETURN]; + bool is_struct_ops =3D flags & BPF_TRAMP_F_INDIRECT; int ret, save_ret; void *orig_call =3D func_addr; u32 **branches =3D NULL; @@ -1685,18 +1686,31 @@ static int __arch_prepare_bpf_trampoline(struct jit= _ctx *ctx, struct bpf_tramp_i =20 stack_size =3D round_up(stack_size, 16); =20 - /* For the trampoline called from function entry */ - /* RA and FP for parent function*/ - emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, -16); - emit_insn(ctx, std, LOONGARCH_GPR_RA, LOONGARCH_GPR_SP, 8); - emit_insn(ctx, std, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, 0); - emit_insn(ctx, addid, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, 16); - - /* RA and FP for traced function*/ - emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, -stack_size); - emit_insn(ctx, std, LOONGARCH_GPR_T0, LOONGARCH_GPR_SP, stack_size - 8); - emit_insn(ctx, std, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, stack_size - 16); - emit_insn(ctx, addid, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, stack_size); + if (!is_struct_ops) { + /* + * For the trampoline called from function entry, + * the frame of traced function and the frame of + * trampoline need to be considered. + */ + emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, -16); + emit_insn(ctx, std, LOONGARCH_GPR_RA, LOONGARCH_GPR_SP, 8); + emit_insn(ctx, std, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, 0); + emit_insn(ctx, addid, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, 16); + + emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, -stack_size); + emit_insn(ctx, std, LOONGARCH_GPR_T0, LOONGARCH_GPR_SP, stack_size - 8); + emit_insn(ctx, std, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, stack_size - 16); + emit_insn(ctx, addid, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, stack_size); + } else { + /* + * For the trampoline called directly, just handle + * the frame of trampoline. + */ + emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, -stack_size); + emit_insn(ctx, std, LOONGARCH_GPR_RA, LOONGARCH_GPR_SP, stack_size - 8); + emit_insn(ctx, std, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, stack_size - 16); + emit_insn(ctx, addid, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, stack_size); + } =20 /* callee saved register S1 to pass start time */ emit_insn(ctx, std, LOONGARCH_GPR_S1, LOONGARCH_GPR_FP, -sreg_off); @@ -1786,21 +1800,30 @@ static int __arch_prepare_bpf_trampoline(struct jit= _ctx *ctx, struct bpf_tramp_i =20 emit_insn(ctx, ldd, LOONGARCH_GPR_S1, LOONGARCH_GPR_FP, -sreg_off); =20 - /* trampoline called from function entry */ - emit_insn(ctx, ldd, LOONGARCH_GPR_T0, LOONGARCH_GPR_SP, stack_size - 8); - emit_insn(ctx, ldd, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, stack_size - 16); - emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, stack_size); + if (!is_struct_ops) { + /* trampoline called from function entry */ + emit_insn(ctx, ldd, LOONGARCH_GPR_T0, LOONGARCH_GPR_SP, stack_size - 8); + emit_insn(ctx, ldd, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, stack_size - 16); + emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, stack_size); + + emit_insn(ctx, ldd, LOONGARCH_GPR_RA, LOONGARCH_GPR_SP, 8); + emit_insn(ctx, ldd, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, 0); + emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, 16); =20 - emit_insn(ctx, ldd, LOONGARCH_GPR_RA, LOONGARCH_GPR_SP, 8); - emit_insn(ctx, ldd, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, 0); - emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, 16); + if (flags & BPF_TRAMP_F_SKIP_FRAME) + /* return to parent function */ + emit_insn(ctx, jirl, LOONGARCH_GPR_ZERO, LOONGARCH_GPR_RA, 0); + else + /* return to traced function */ + emit_insn(ctx, jirl, LOONGARCH_GPR_ZERO, LOONGARCH_GPR_T0, 0); + } else { + /* trampoline called directly */ + emit_insn(ctx, ldd, LOONGARCH_GPR_RA, LOONGARCH_GPR_SP, stack_size - 8); + emit_insn(ctx, ldd, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, stack_size - 16); + emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, stack_size); =20 - if (flags & BPF_TRAMP_F_SKIP_FRAME) - /* return to parent function */ emit_insn(ctx, jirl, LOONGARCH_GPR_ZERO, LOONGARCH_GPR_RA, 0); - else - /* return to traced function */ - emit_insn(ctx, jirl, LOONGARCH_GPR_ZERO, LOONGARCH_GPR_T0, 0); + } =20 ret =3D ctx->idx; out: --=20 2.25.1