This is BFDOT for both AArch64 AdvSIMD and SVE,
and VDOT.BF16 for AArch32 NEON.
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
target/arm/helper.h | 2 ++
target/arm/neon-shared.decode | 2 ++
target/arm/sve.decode | 3 +++
target/arm/translate-a64.c | 41 +++++++++++++++++++++++++--------
target/arm/translate-sve.c | 12 ++++++++++
target/arm/vec_helper.c | 20 ++++++++++++++++
target/arm/translate-neon.c.inc | 9 ++++++++
7 files changed, 80 insertions(+), 9 deletions(-)
diff --git a/target/arm/helper.h b/target/arm/helper.h
index eb4cb2b65b..af0ee8f693 100644
--- a/target/arm/helper.h
+++ b/target/arm/helper.h
@@ -1005,6 +1005,8 @@ DEF_HELPER_FLAGS_5(gvec_usmmla_b, TCG_CALL_NO_RWG,
DEF_HELPER_FLAGS_5(gvec_bfdot, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(gvec_bfdot_idx, TCG_CALL_NO_RWG,
+ void, ptr, ptr, ptr, ptr, i32)
#ifdef TARGET_AARCH64
#include "helper-a64.h"
diff --git a/target/arm/neon-shared.decode b/target/arm/neon-shared.decode
index 31a0839bbb..fa3cf14e3a 100644
--- a/target/arm/neon-shared.decode
+++ b/target/arm/neon-shared.decode
@@ -81,6 +81,8 @@ VUSDOT_scalar 1111 1110 1 . 00 .... .... 1101 . q:1 index:1 0 vm:4 \
vn=%vn_dp vd=%vd_dp
VSUDOT_scalar 1111 1110 1 . 00 .... .... 1101 . q:1 index:1 1 vm:4 \
vn=%vn_dp vd=%vd_dp
+VDOT_b16_scal 1111 1110 0 . 00 .... .... 1101 . q:1 index:1 0 vm:4 \
+ vn=%vn_dp vd=%vd_dp
%vfml_scalar_q0_rm 0:3 5:1
%vfml_scalar_q1_index 5:1 3:1
diff --git a/target/arm/sve.decode b/target/arm/sve.decode
index 523140ca56..d5e1e5d400 100644
--- a/target/arm/sve.decode
+++ b/target/arm/sve.decode
@@ -1584,3 +1584,6 @@ FMLALB_zzxw 01100100 10 1 ..... 0100.0 ..... ..... @rrxr_3a esz=2
FMLALT_zzxw 01100100 10 1 ..... 0100.1 ..... ..... @rrxr_3a esz=2
FMLSLB_zzxw 01100100 10 1 ..... 0110.0 ..... ..... @rrxr_3a esz=2
FMLSLT_zzxw 01100100 10 1 ..... 0110.1 ..... ..... @rrxr_3a esz=2
+
+### SVE2 floating-point bfloat16 dot-product (indexed)
+BFDOT_zzxz 01100100 01 1 ..... 010000 ..... ..... @rrxr_2 esz=2
diff --git a/target/arm/translate-a64.c b/target/arm/translate-a64.c
index fc16e0a126..f60afbbd06 100644
--- a/target/arm/translate-a64.c
+++ b/target/arm/translate-a64.c
@@ -13457,8 +13457,22 @@ static void disas_simd_indexed(DisasContext *s, uint32_t insn)
return;
}
break;
- case 0x0f: /* SUDOT, USDOT */
- if (is_scalar || (size & 1) || !dc_isar_feature(aa64_i8mm, s)) {
+ case 0x0f:
+ switch (size) {
+ case 0: /* SUDOT */
+ case 2: /* USDOT */
+ if (is_scalar || !dc_isar_feature(aa64_i8mm, s)) {
+ unallocated_encoding(s);
+ return;
+ }
+ break;
+ case 1: /* BFDOT */
+ if (is_scalar || !dc_isar_feature(aa64_bf16, s)) {
+ unallocated_encoding(s);
+ return;
+ }
+ break;
+ default:
unallocated_encoding(s);
return;
}
@@ -13578,13 +13592,22 @@ static void disas_simd_indexed(DisasContext *s, uint32_t insn)
u ? gen_helper_gvec_udot_idx_b
: gen_helper_gvec_sdot_idx_b);
return;
- case 0x0f: /* SUDOT, USDOT */
- gen_gvec_op4_ool(s, is_q, rd, rn, rm, rd, index,
- extract32(insn, 23, 1)
- ? gen_helper_gvec_usdot_idx_b
- : gen_helper_gvec_sudot_idx_b);
- return;
-
+ case 0x0f:
+ switch (extract32(insn, 22, 2)) {
+ case 0: /* SUDOT */
+ gen_gvec_op4_ool(s, is_q, rd, rn, rm, rd, index,
+ gen_helper_gvec_sudot_idx_b);
+ return;
+ case 1: /* BFDOT */
+ gen_gvec_op4_ool(s, is_q, rd, rn, rm, rd, index,
+ gen_helper_gvec_bfdot_idx);
+ return;
+ case 2: /* USDOT */
+ gen_gvec_op4_ool(s, is_q, rd, rn, rm, rd, index,
+ gen_helper_gvec_usdot_idx_b);
+ return;
+ }
+ g_assert_not_reached();
case 0x11: /* FCMLA #0 */
case 0x13: /* FCMLA #90 */
case 0x15: /* FCMLA #180 */
diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c
index 3527430c1a..ef6828c632 100644
--- a/target/arm/translate-sve.c
+++ b/target/arm/translate-sve.c
@@ -8598,3 +8598,15 @@ static bool trans_BFDOT_zzzz(DisasContext *s, arg_rrrr_esz *a)
}
return true;
}
+
+static bool trans_BFDOT_zzxz(DisasContext *s, arg_rrxr_esz *a)
+{
+ if (!dc_isar_feature(aa64_sve_bf16, s)) {
+ return false;
+ }
+ if (sve_access_check(s)) {
+ gen_gvec_ool_zzzz(s, gen_helper_gvec_bfdot_idx,
+ a->rd, a->rn, a->rm, a->ra, a->index);
+ }
+ return true;
+}
diff --git a/target/arm/vec_helper.c b/target/arm/vec_helper.c
index e227ba6590..3e26fb0e5f 100644
--- a/target/arm/vec_helper.c
+++ b/target/arm/vec_helper.c
@@ -2695,3 +2695,23 @@ void HELPER(gvec_bfdot)(void *vd, void *vn, void *vm, void *va, uint32_t desc)
}
clear_tail(d, opr_sz, simd_maxsz(desc));
}
+
+void HELPER(gvec_bfdot_idx)(void *vd, void *vn, void *vm,
+ void *va, uint32_t desc)
+{
+ intptr_t i, j, opr_sz = simd_oprsz(desc);
+ intptr_t index = simd_data(desc);
+ intptr_t elements = opr_sz / 4;
+ intptr_t eltspersegment = MIN(16 / 4, elements);
+ float32 *d = vd, *a = va;
+ uint32_t *n = vn, *m = vm;
+
+ for (i = 0; i < elements; i += eltspersegment) {
+ uint32_t m_idx = m[i + H4(index)];
+
+ for (j = i; j < i + eltspersegment; j++) {
+ d[j] = bfdotadd(a[j], n[j], m_idx);
+ }
+ }
+ clear_tail(d, opr_sz, simd_maxsz(desc));
+}
diff --git a/target/arm/translate-neon.c.inc b/target/arm/translate-neon.c.inc
index aed8a565e0..bb0adf4756 100644
--- a/target/arm/translate-neon.c.inc
+++ b/target/arm/translate-neon.c.inc
@@ -381,6 +381,15 @@ static bool trans_VSUDOT_scalar(DisasContext *s, arg_VSUDOT_scalar *a)
gen_helper_gvec_sudot_idx_b);
}
+static bool trans_VDOT_b16_scal(DisasContext *s, arg_VDOT_b16_scal *a)
+{
+ if (!dc_isar_feature(aa32_bf16, s)) {
+ return false;
+ }
+ return do_neon_ddda(s, a->q * 6, a->vd, a->vn, a->vm, a->index,
+ gen_helper_gvec_bfdot_idx);
+}
+
static bool trans_VFML_scalar(DisasContext *s, arg_VFML_scalar *a)
{
int opr_sz;
--
2.25.1
On Sat, 17 Apr 2021 at 01:06, Richard Henderson
<richard.henderson@linaro.org> wrote:
>
> This is BFDOT for both AArch64 AdvSIMD and SVE,
> and VDOT.BF16 for AArch32 NEON.
>
> Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
> ---
> target/arm/helper.h | 2 ++
> target/arm/neon-shared.decode | 2 ++
> target/arm/sve.decode | 3 +++
> target/arm/translate-a64.c | 41 +++++++++++++++++++++++++--------
> target/arm/translate-sve.c | 12 ++++++++++
> target/arm/vec_helper.c | 20 ++++++++++++++++
> target/arm/translate-neon.c.inc | 9 ++++++++
> 7 files changed, 80 insertions(+), 9 deletions(-)
>
> @@ -13578,13 +13592,22 @@ static void disas_simd_indexed(DisasContext *s, uint32_t insn)
> u ? gen_helper_gvec_udot_idx_b
> : gen_helper_gvec_sdot_idx_b);
> return;
> - case 0x0f: /* SUDOT, USDOT */
> - gen_gvec_op4_ool(s, is_q, rd, rn, rm, rd, index,
> - extract32(insn, 23, 1)
> - ? gen_helper_gvec_usdot_idx_b
> - : gen_helper_gvec_sudot_idx_b);
> - return;
> -
> + case 0x0f:
> + switch (extract32(insn, 22, 2)) {
You already have bits [23:22] in 'size' at this point, I think.
> + case 0: /* SUDOT */
> + gen_gvec_op4_ool(s, is_q, rd, rn, rm, rd, index,
> + gen_helper_gvec_sudot_idx_b);
> + return;
> + case 1: /* BFDOT */
> + gen_gvec_op4_ool(s, is_q, rd, rn, rm, rd, index,
> + gen_helper_gvec_bfdot_idx);
> + return;
> + case 2: /* USDOT */
> + gen_gvec_op4_ool(s, is_q, rd, rn, rm, rd, index,
> + gen_helper_gvec_usdot_idx_b);
> + return;
> + }
> + g_assert_not_reached();
otherwise
Reviewed-by: Peter Maydell <peter.maydell@linaro.org>
thanks
-- PMM
On 5/18/21 7:24 AM, Peter Maydell wrote:
>> - case 0x0f: /* SUDOT, USDOT */
>> - gen_gvec_op4_ool(s, is_q, rd, rn, rm, rd, index,
>> - extract32(insn, 23, 1)
>> - ? gen_helper_gvec_usdot_idx_b
>> - : gen_helper_gvec_sudot_idx_b);
>> - return;
>> -
>> + case 0x0f:
>> + switch (extract32(insn, 22, 2)) {
>
> You already have bits [23:22] in 'size' at this point, I think.
Irritatingly not. For 0xf [23:22] is opcode, and size has been squashed to
MO_32 in order to get the indexing correct in the middle of this function.
It's just outside the patch context here. Later it will be moved inside a
switch for BFMLAL.
One of the many ways in which decodetree would be better here.
r~
© 2016 - 2025 Red Hat, Inc.