From nobody Thu Nov 13 16:18:20 2025 Delivered-To: importer@patchew.org Received-SPF: pass (zohomail.com: domain of gnu.org designates 209.51.188.17 as permitted sender) client-ip=209.51.188.17; envelope-from=qemu-devel-bounces+importer=patchew.org@nongnu.org; helo=lists.gnu.org; Authentication-Results: mx.zohomail.com; spf=pass (zohomail.com: domain of gnu.org designates 209.51.188.17 as permitted sender) smtp.mailfrom=qemu-devel-bounces+importer=patchew.org@nongnu.org Return-Path: Received: from lists.gnu.org (lists.gnu.org [209.51.188.17]) by mx.zohomail.com with SMTPS id 1581320673639434.6315367477639; Sun, 9 Feb 2020 23:44:33 -0800 (PST) Received: from localhost ([::1]:57896 helo=lists1p.gnu.org) by lists.gnu.org with esmtp (Exim 4.90_1) (envelope-from ) id 1j13k7-0004UN-Ky for importer@patchew.org; Mon, 10 Feb 2020 02:44:31 -0500 Received: from eggs.gnu.org ([2001:470:142:3::10]:33894) by lists.gnu.org with esmtp (Exim 4.90_1) (envelope-from ) id 1j13iv-0002qt-TB for qemu-devel@nongnu.org; Mon, 10 Feb 2020 02:43:19 -0500 Received: from Debian-exim by eggs.gnu.org with spam-scanned (Exim 4.71) (envelope-from ) id 1j13it-00054t-Nq for qemu-devel@nongnu.org; Mon, 10 Feb 2020 02:43:17 -0500 Received: from smtp2200-217.mail.aliyun.com ([121.197.200.217]:55417) by eggs.gnu.org with esmtps (TLS1.0:DHE_RSA_AES_256_CBC_SHA1:32) (Exim 4.71) (envelope-from ) id 1j13is-0004bm-U5; Mon, 10 Feb 2020 02:43:15 -0500 Received: from L-PF1D6DP4-1208.hz.ali.com(mailfrom:zhiwei_liu@c-sky.com fp:SMTPD_---.GmNZEYU_1581320582) by smtp.aliyun-inc.com(10.147.41.158); Mon, 10 Feb 2020 15:43:05 +0800 X-Alimail-AntiSpam: AC=CONTINUE; BC=0.07436282|-1; CH=green; DM=CONTINUE|CONTINUE|true|0.65445-0.0401631-0.305387; DS=CONTINUE|ham_system_inform|0.760458-0.000229679-0.239313; FP=0|0|0|0|0|-1|-1|-1; HT=e01l07426; MF=zhiwei_liu@c-sky.com; NM=1; PH=DS; RN=9; RT=9; SR=0; TI=SMTPD_---.GmNZEYU_1581320582; From: LIU Zhiwei To: richard.henderson@linaro.org, alistair23@gmail.com, chihmin.chao@sifive.com, palmer@dabbelt.com Subject: [PATCH v3 4/5] target/riscv: add fault-only-first unit stride load Date: Mon, 10 Feb 2020 15:42:55 +0800 Message-Id: <20200210074256.11412-5-zhiwei_liu@c-sky.com> X-Mailer: git-send-email 2.23.0 In-Reply-To: <20200210074256.11412-1-zhiwei_liu@c-sky.com> References: <20200210074256.11412-1-zhiwei_liu@c-sky.com> MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable X-detected-operating-system: by eggs.gnu.org: GNU/Linux 3.x [generic] [fuzzy] X-Received-From: 121.197.200.217 X-BeenThere: qemu-devel@nongnu.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Cc: wenmeng_zhang@c-sky.com, qemu-riscv@nongnu.org, qemu-devel@nongnu.org, wxy194768@alibaba-inc.com, LIU Zhiwei Errors-To: qemu-devel-bounces+importer=patchew.org@nongnu.org Sender: "Qemu-devel" Content-Type: text/plain; charset="utf-8" The unit-stride fault-only-fault load instructions are used to vectorize loops with data-dependent exit conditions(while loops). These instructions execute as a regular load except that they will only take a trap on element 0. Signed-off-by: LIU Zhiwei --- target/riscv/helper.h | 22 ++++ target/riscv/insn32.decode | 7 ++ target/riscv/insn_trans/trans_rvv.inc.c | 88 +++++++++++++++ target/riscv/vector_helper.c | 138 ++++++++++++++++++++++++ 4 files changed, 255 insertions(+) diff --git a/target/riscv/helper.h b/target/riscv/helper.h index 5ebd3d6ccd..893dfc0fb8 100644 --- a/target/riscv/helper.h +++ b/target/riscv/helper.h @@ -218,3 +218,25 @@ DEF_HELPER_6(vsxe_v_b_mask, void, ptr, tl, ptr, ptr, e= nv, i32) DEF_HELPER_6(vsxe_v_h_mask, void, ptr, tl, ptr, ptr, env, i32) DEF_HELPER_6(vsxe_v_w_mask, void, ptr, tl, ptr, ptr, env, i32) DEF_HELPER_6(vsxe_v_d_mask, void, ptr, tl, ptr, ptr, env, i32) +DEF_HELPER_5(vlbff_v_b_mask, void, ptr, tl, ptr, env, i32) +DEF_HELPER_5(vlbff_v_h_mask, void, ptr, tl, ptr, env, i32) +DEF_HELPER_5(vlbff_v_w_mask, void, ptr, tl, ptr, env, i32) +DEF_HELPER_5(vlbff_v_d_mask, void, ptr, tl, ptr, env, i32) +DEF_HELPER_5(vlhff_v_h_mask, void, ptr, tl, ptr, env, i32) +DEF_HELPER_5(vlhff_v_w_mask, void, ptr, tl, ptr, env, i32) +DEF_HELPER_5(vlhff_v_d_mask, void, ptr, tl, ptr, env, i32) +DEF_HELPER_5(vlwff_v_w_mask, void, ptr, tl, ptr, env, i32) +DEF_HELPER_5(vlwff_v_d_mask, void, ptr, tl, ptr, env, i32) +DEF_HELPER_5(vleff_v_b_mask, void, ptr, tl, ptr, env, i32) +DEF_HELPER_5(vleff_v_h_mask, void, ptr, tl, ptr, env, i32) +DEF_HELPER_5(vleff_v_w_mask, void, ptr, tl, ptr, env, i32) +DEF_HELPER_5(vleff_v_d_mask, void, ptr, tl, ptr, env, i32) +DEF_HELPER_5(vlbuff_v_b_mask, void, ptr, tl, ptr, env, i32) +DEF_HELPER_5(vlbuff_v_h_mask, void, ptr, tl, ptr, env, i32) +DEF_HELPER_5(vlbuff_v_w_mask, void, ptr, tl, ptr, env, i32) +DEF_HELPER_5(vlbuff_v_d_mask, void, ptr, tl, ptr, env, i32) +DEF_HELPER_5(vlhuff_v_h_mask, void, ptr, tl, ptr, env, i32) +DEF_HELPER_5(vlhuff_v_w_mask, void, ptr, tl, ptr, env, i32) +DEF_HELPER_5(vlhuff_v_d_mask, void, ptr, tl, ptr, env, i32) +DEF_HELPER_5(vlwuff_v_w_mask, void, ptr, tl, ptr, env, i32) +DEF_HELPER_5(vlwuff_v_d_mask, void, ptr, tl, ptr, env, i32) diff --git a/target/riscv/insn32.decode b/target/riscv/insn32.decode index 6a363a6b7e..973ac63fda 100644 --- a/target/riscv/insn32.decode +++ b/target/riscv/insn32.decode @@ -219,6 +219,13 @@ vle_v ... 000 . 00000 ..... 111 ..... 0000111 @r2= _nfvm vlbu_v ... 000 . 00000 ..... 000 ..... 0000111 @r2_nfvm vlhu_v ... 000 . 00000 ..... 101 ..... 0000111 @r2_nfvm vlwu_v ... 000 . 00000 ..... 110 ..... 0000111 @r2_nfvm +vlbff_v ... 100 . 10000 ..... 000 ..... 0000111 @r2_nfvm +vlhff_v ... 100 . 10000 ..... 101 ..... 0000111 @r2_nfvm +vlwff_v ... 100 . 10000 ..... 110 ..... 0000111 @r2_nfvm +vleff_v ... 000 . 10000 ..... 111 ..... 0000111 @r2_nfvm +vlbuff_v ... 000 . 10000 ..... 000 ..... 0000111 @r2_nfvm +vlhuff_v ... 000 . 10000 ..... 101 ..... 0000111 @r2_nfvm +vlwuff_v ... 000 . 10000 ..... 110 ..... 0000111 @r2_nfvm vsb_v ... 000 . 00000 ..... 000 ..... 0100111 @r2_nfvm vsh_v ... 000 . 00000 ..... 101 ..... 0100111 @r2_nfvm vsw_v ... 000 . 00000 ..... 110 ..... 0100111 @r2_nfvm diff --git a/target/riscv/insn_trans/trans_rvv.inc.c b/target/riscv/insn_tr= ans/trans_rvv.inc.c index 13033b3906..66caa16d18 100644 --- a/target/riscv/insn_trans/trans_rvv.inc.c +++ b/target/riscv/insn_trans/trans_rvv.inc.c @@ -663,3 +663,91 @@ static bool trans_vsuxe_v(DisasContext *s, arg_rnfvm* = a) { return trans_vsxe_v(s, a); } + +/* unit stride fault-only-first load */ +typedef void gen_helper_vext_ldff(TCGv_ptr, TCGv, TCGv_ptr, + TCGv_env, TCGv_i32); + +static bool do_vext_ldff_trans(uint32_t vd, uint32_t rs1, uint32_t data, + gen_helper_vext_ldff *fn, DisasContext *s) +{ + TCGv_ptr dest, mask; + TCGv base; + TCGv_i32 desc; + + dest =3D tcg_temp_new_ptr(); + mask =3D tcg_temp_new_ptr(); + base =3D tcg_temp_new(); + desc =3D tcg_const_i32(simd_desc(0, maxsz_part1(s->maxsz), data)); + + gen_get_gpr(base, rs1); + tcg_gen_addi_ptr(dest, cpu_env, vreg_ofs(s, vd)); + tcg_gen_addi_ptr(mask, cpu_env, vreg_ofs(s, 0)); + + fn(dest, base, mask, cpu_env, desc); + + tcg_temp_free_ptr(dest); + tcg_temp_free_ptr(mask); + tcg_temp_free(base); + tcg_temp_free_i32(desc); + return true; +} + +static bool vext_ldff_trans(DisasContext *s, arg_r2nfvm *a, uint8_t seq) +{ + uint8_t nf =3D a->nf + 1; + uint32_t data =3D s->mlen | (a->vm << 8) | (maxsz_part2(s->maxsz) << 9) + | (nf << 12); + gen_helper_vext_ldff *fn; + static gen_helper_vext_ldff * const fns[7][4] =3D { + /* masked unit stride fault-only-first load */ + { gen_helper_vlbff_v_b_mask, gen_helper_vlbff_v_h_mask, + gen_helper_vlbff_v_w_mask, gen_helper_vlbff_v_d_mask }, + { NULL, gen_helper_vlhff_v_h_mask, + gen_helper_vlhff_v_w_mask, gen_helper_vlhff_v_d_mask }, + { NULL, NULL, + gen_helper_vlwff_v_w_mask, gen_helper_vlwff_v_d_mask }, + { gen_helper_vleff_v_b_mask, gen_helper_vleff_v_h_mask, + gen_helper_vleff_v_w_mask, gen_helper_vleff_v_d_mask }, + { gen_helper_vlbuff_v_b_mask, gen_helper_vlbuff_v_h_mask, + gen_helper_vlbuff_v_w_mask, gen_helper_vlbuff_v_d_mask }, + { NULL, gen_helper_vlhuff_v_h_mask, + gen_helper_vlhuff_v_w_mask, gen_helper_vlhuff_v_d_mask }, + { NULL, NULL, + gen_helper_vlwuff_v_w_mask, gen_helper_vlwuff_v_d_mask } + }; + + fn =3D fns[seq][s->sew]; + if (fn =3D=3D NULL) { + return false; + } + + return do_vext_ldff_trans(a->rd, a->rs1, data, fn, s); +} + +#define GEN_VEXT_LDFF_TRANS(NAME, DO_OP, SEQ) \ +static bool trans_##NAME(DisasContext *s, arg_r2nfvm* a) \ +{ \ + vchkctx.check_misa =3D RVV; \ + vchkctx.check_overlap_mask.need_check =3D true; \ + vchkctx.check_overlap_mask.reg =3D a->rd; \ + vchkctx.check_overlap_mask.vm =3D a->vm; \ + vchkctx.check_reg[0].need_check =3D true; \ + vchkctx.check_reg[0].reg =3D a->rd; \ + vchkctx.check_reg[0].widen =3D false; \ + vchkctx.check_nf.need_check =3D true; \ + vchkctx.check_nf.nf =3D a->nf; \ + \ + if (!vext_check(s)) { \ + return false; \ + } \ + return DO_OP(s, a, SEQ); \ +} + +GEN_VEXT_LDFF_TRANS(vlbff_v, vext_ldff_trans, 0) +GEN_VEXT_LDFF_TRANS(vlhff_v, vext_ldff_trans, 1) +GEN_VEXT_LDFF_TRANS(vlwff_v, vext_ldff_trans, 2) +GEN_VEXT_LDFF_TRANS(vleff_v, vext_ldff_trans, 3) +GEN_VEXT_LDFF_TRANS(vlbuff_v, vext_ldff_trans, 4) +GEN_VEXT_LDFF_TRANS(vlhuff_v, vext_ldff_trans, 5) +GEN_VEXT_LDFF_TRANS(vlwuff_v, vext_ldff_trans, 6) diff --git a/target/riscv/vector_helper.c b/target/riscv/vector_helper.c index 0404394588..941851ab28 100644 --- a/target/riscv/vector_helper.c +++ b/target/riscv/vector_helper.c @@ -301,6 +301,28 @@ GEN_VEXT_LD_ELEM(vlxhu_v_w, uint16_t, uint32_t, H4, ld= uw) GEN_VEXT_LD_ELEM(vlxhu_v_d, uint16_t, uint64_t, H8, lduw) GEN_VEXT_LD_ELEM(vlxwu_v_w, uint32_t, uint32_t, H4, ldl) GEN_VEXT_LD_ELEM(vlxwu_v_d, uint32_t, uint64_t, H8, ldl) +GEN_VEXT_LD_ELEM(vlbff_v_b, int8_t, int8_t, H1, ldsb) +GEN_VEXT_LD_ELEM(vlbff_v_h, int8_t, int16_t, H2, ldsb) +GEN_VEXT_LD_ELEM(vlbff_v_w, int8_t, int32_t, H4, ldsb) +GEN_VEXT_LD_ELEM(vlbff_v_d, int8_t, int64_t, H8, ldsb) +GEN_VEXT_LD_ELEM(vlhff_v_h, int16_t, int16_t, H2, ldsw) +GEN_VEXT_LD_ELEM(vlhff_v_w, int16_t, int32_t, H4, ldsw) +GEN_VEXT_LD_ELEM(vlhff_v_d, int16_t, int64_t, H8, ldsw) +GEN_VEXT_LD_ELEM(vlwff_v_w, int32_t, int32_t, H4, ldl) +GEN_VEXT_LD_ELEM(vlwff_v_d, int32_t, int64_t, H8, ldl) +GEN_VEXT_LD_ELEM(vleff_v_b, int8_t, int8_t, H1, ldsb) +GEN_VEXT_LD_ELEM(vleff_v_h, int16_t, int16_t, H2, ldsw) +GEN_VEXT_LD_ELEM(vleff_v_w, int32_t, int32_t, H4, ldl) +GEN_VEXT_LD_ELEM(vleff_v_d, int64_t, int64_t, H8, ldq) +GEN_VEXT_LD_ELEM(vlbuff_v_b, uint8_t, uint8_t, H1, ldub) +GEN_VEXT_LD_ELEM(vlbuff_v_h, uint8_t, uint16_t, H2, ldub) +GEN_VEXT_LD_ELEM(vlbuff_v_w, uint8_t, uint32_t, H4, ldub) +GEN_VEXT_LD_ELEM(vlbuff_v_d, uint8_t, uint64_t, H8, ldub) +GEN_VEXT_LD_ELEM(vlhuff_v_h, uint16_t, uint16_t, H2, lduw) +GEN_VEXT_LD_ELEM(vlhuff_v_w, uint16_t, uint32_t, H4, lduw) +GEN_VEXT_LD_ELEM(vlhuff_v_d, uint16_t, uint64_t, H8, lduw) +GEN_VEXT_LD_ELEM(vlwuff_v_w, uint32_t, uint32_t, H4, ldl) +GEN_VEXT_LD_ELEM(vlwuff_v_d, uint32_t, uint64_t, H8, ldl) =20 #define GEN_VEXT_ST_ELEM(NAME, ETYPE, H, STSUF) \ static void vext_##NAME##_st_elem(CPURISCVState *env, abi_ptr addr, \ @@ -868,3 +890,119 @@ GEN_VEXT_ST_INDEX(vsxe_v_b, int8_t, int8_t) GEN_VEXT_ST_INDEX(vsxe_v_h, int16_t, int16_t) GEN_VEXT_ST_INDEX(vsxe_v_w, int32_t, int32_t) GEN_VEXT_ST_INDEX(vsxe_v_d, int64_t, int64_t) + +/* unit-stride fault-only-fisrt load instructions */ +static void vext_ldff_mask(void *vd, void *v0, CPURISCVState *env, + struct vext_ldst_ctx *ctx, uintptr_t ra) +{ + void *host; + uint32_t i, k, vl =3D 0; + target_ulong addr, offset, remain; + struct vext_common_ctx *s =3D &ctx->vcc; + + if (s->vl =3D=3D 0) { + return; + } + /* probe every access*/ + for (i =3D 0; i < s->vl; i++) { + if (!s->vm && !vext_elem_mask(v0, s->mlen, i)) { + continue; + } + addr =3D ctx->base + ctx->nf * i * s->msz; + if (i =3D=3D 0) { + probe_read_access(env, addr, ctx->nf * s->msz, ra); + } else { + /* if it triggles an exception, no need to check watchpoint */ + offset =3D -(addr | TARGET_PAGE_MASK); + remain =3D ctx->nf * s->msz; + while (remain > 0) { + host =3D tlb_vaddr_to_host(env, addr, MMU_DATA_LOAD, + ctx->mmuidx); + if (host) { +#ifdef CONFIG_USER_ONLY + if (page_check_range(addr, ctx->nf * s->msz, + PAGE_READ) < 0) { + vl =3D i; + goto ProbeSuccess; + } +#else + probe_read_access(env, addr, ctx->nf * s->msz, ra); +#endif + } else { + vl =3D i; + goto ProbeSuccess; + } + if (remain <=3D offset) { + break; + } + remain -=3D offset; + addr +=3D offset; + offset =3D -(addr | TARGET_PAGE_MASK); + } + } + } +ProbeSuccess: + /* load bytes from guest memory */ + if (vl !=3D 0) { + s->vl =3D vl; + } + for (i =3D 0; i < s->vl; i++) { + k =3D 0; + if (!s->vm && !vext_elem_mask(v0, s->mlen, i)) { + continue; + } + while (k < ctx->nf) { + target_ulong addr =3D ctx->base + (i * ctx->nf + k) * s->msz; + ctx->ld_elem(env, addr, i + k * s->vlmax, vd, ra); + k++; + } + } + /* clear tail elements */ + if (vl !=3D 0) { + env->vext.vl =3D vl; + return; + } + for (k =3D 0; k < ctx->nf; k++) { + ctx->clear_elem(vd, s->vl + k * s->vlmax, s->vl * s->esz, + s->vlmax * s->esz); + } +} + +#define GEN_VEXT_LDFF(NAME, MTYPE, ETYPE, MMUIDX) \ +void HELPER(NAME##_mask)(void *vd, target_ulong base, void *v0, \ + CPURISCVState *env, uint32_t desc) \ +{ \ + static struct vext_ldst_ctx ctx; \ + vext_common_ctx_init(&ctx.vcc, sizeof(ETYPE), \ + sizeof(MTYPE), env->vext.vl, desc); \ + ctx.nf =3D vext_nf(desc); \ + ctx.base =3D base; \ + ctx.mmuidx =3D MMUIDX; \ + ctx.ld_elem =3D vext_##NAME##_ld_elem; \ + ctx.clear_elem =3D vext_##NAME##_clear_elem; \ + \ + vext_ldff_mask(vd, v0, env, &ctx, GETPC()); \ +} + +GEN_VEXT_LDFF(vlbff_v_b, int8_t, int8_t, MO_SB) +GEN_VEXT_LDFF(vlbff_v_h, int8_t, int16_t, MO_SB) +GEN_VEXT_LDFF(vlbff_v_w, int8_t, int32_t, MO_SB) +GEN_VEXT_LDFF(vlbff_v_d, int8_t, int64_t, MO_SB) +GEN_VEXT_LDFF(vlhff_v_h, int16_t, int16_t, MO_LESW) +GEN_VEXT_LDFF(vlhff_v_w, int16_t, int32_t, MO_LESW) +GEN_VEXT_LDFF(vlhff_v_d, int16_t, int64_t, MO_LESW) +GEN_VEXT_LDFF(vlwff_v_w, int32_t, int32_t, MO_LESL) +GEN_VEXT_LDFF(vlwff_v_d, int32_t, int64_t, MO_LESL) +GEN_VEXT_LDFF(vleff_v_b, int8_t, int8_t, MO_SB) +GEN_VEXT_LDFF(vleff_v_h, int16_t, int16_t, MO_LESW) +GEN_VEXT_LDFF(vleff_v_w, int32_t, int32_t, MO_LESL) +GEN_VEXT_LDFF(vleff_v_d, int64_t, int64_t, MO_LEQ) +GEN_VEXT_LDFF(vlbuff_v_b, uint8_t, uint8_t, MO_UB) +GEN_VEXT_LDFF(vlbuff_v_h, uint8_t, uint16_t, MO_UB) +GEN_VEXT_LDFF(vlbuff_v_w, uint8_t, uint32_t, MO_UB) +GEN_VEXT_LDFF(vlbuff_v_d, uint8_t, uint64_t, MO_UB) +GEN_VEXT_LDFF(vlhuff_v_h, uint16_t, uint16_t, MO_LEUW) +GEN_VEXT_LDFF(vlhuff_v_w, uint16_t, uint32_t, MO_LEUW) +GEN_VEXT_LDFF(vlhuff_v_d, uint16_t, uint64_t, MO_LEUW) +GEN_VEXT_LDFF(vlwuff_v_w, uint32_t, uint32_t, MO_LEUL) +GEN_VEXT_LDFF(vlwuff_v_d, uint32_t, uint64_t, MO_LEUL) --=20 2.23.0