target/arm/tcg/sme_helper.c | 77 ++++++++++++++++++------------- tests/tcg/aarch64/sme-smopa-1.c | 47 +++++++++++++++++++ tests/tcg/aarch64/sme-smopa-2.c | 54 ++++++++++++++++++++++ tests/tcg/aarch64/Makefile.target | 2 +- 4 files changed, 147 insertions(+), 33 deletions(-) create mode 100644 tests/tcg/aarch64/sme-smopa-1.c create mode 100644 tests/tcg/aarch64/sme-smopa-2.c
The while the 8-bit input elements are sequential in the input vector,
the 32-bit output elements are not sequential in the output matrix.
Do not attempt to compute 2 32-bit outputs at the same time.
Cc: qemu-stable@nongnu.org
Fixes: 23a5e3859f5 ("target/arm: Implement SME integer outer product")
Resolves: https://gitlab.com/qemu-project/qemu/-/issues/2083
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
target/arm/tcg/sme_helper.c | 77 ++++++++++++++++++-------------
tests/tcg/aarch64/sme-smopa-1.c | 47 +++++++++++++++++++
tests/tcg/aarch64/sme-smopa-2.c | 54 ++++++++++++++++++++++
tests/tcg/aarch64/Makefile.target | 2 +-
4 files changed, 147 insertions(+), 33 deletions(-)
create mode 100644 tests/tcg/aarch64/sme-smopa-1.c
create mode 100644 tests/tcg/aarch64/sme-smopa-2.c
diff --git a/target/arm/tcg/sme_helper.c b/target/arm/tcg/sme_helper.c
index 904bfdac43..ef39eee48d 100644
--- a/target/arm/tcg/sme_helper.c
+++ b/target/arm/tcg/sme_helper.c
@@ -1083,11 +1083,32 @@ void HELPER(sme_bfmopa)(void *vza, void *vzn, void *vzm, void *vpn,
}
}
-typedef uint64_t IMOPFn(uint64_t, uint64_t, uint64_t, uint8_t, bool);
+typedef uint32_t IMOPFn32(uint32_t, uint32_t, uint32_t, uint8_t, bool);
+static inline void do_imopa_s(uint32_t *za, uint32_t *zn, uint32_t *zm,
+ uint8_t *pn, uint8_t *pm,
+ uint32_t desc, IMOPFn32 *fn)
+{
+ intptr_t row, col, oprsz = simd_oprsz(desc) / 4;
+ bool neg = simd_data(desc);
-static inline void do_imopa(uint64_t *za, uint64_t *zn, uint64_t *zm,
- uint8_t *pn, uint8_t *pm,
- uint32_t desc, IMOPFn *fn)
+ for (row = 0; row < oprsz; ++row) {
+ uint8_t pa = pn[H1(row >> 1)] >> ((row & 1) * 4);
+ uint32_t *za_row = &za[H4(tile_vslice_index(row))];
+ uint32_t n = zn[H4(row)];
+
+ for (col = 0; col < oprsz; ++col) {
+ uint8_t pb = pm[H1(col >> 1)] >> ((col & 1) * 4);
+ uint32_t *a = &za_row[col];
+
+ *a = fn(n, zm[H4(col)], *a, pa & pb & 0xf, neg);
+ }
+ }
+}
+
+typedef uint64_t IMOPFn64(uint64_t, uint64_t, uint64_t, uint8_t, bool);
+static inline void do_imopa_d(uint64_t *za, uint64_t *zn, uint64_t *zm,
+ uint8_t *pn, uint8_t *pm,
+ uint32_t desc, IMOPFn64 *fn)
{
intptr_t row, col, oprsz = simd_oprsz(desc) / 8;
bool neg = simd_data(desc);
@@ -1107,25 +1128,16 @@ static inline void do_imopa(uint64_t *za, uint64_t *zn, uint64_t *zm,
}
#define DEF_IMOP_32(NAME, NTYPE, MTYPE) \
-static uint64_t NAME(uint64_t n, uint64_t m, uint64_t a, uint8_t p, bool neg) \
+static uint32_t NAME(uint32_t n, uint32_t m, uint32_t a, uint8_t p, bool neg) \
{ \
- uint32_t sum0 = 0, sum1 = 0; \
+ uint32_t sum = 0; \
/* Apply P to N as a mask, making the inactive elements 0. */ \
n &= expand_pred_b(p); \
- sum0 += (NTYPE)(n >> 0) * (MTYPE)(m >> 0); \
- sum0 += (NTYPE)(n >> 8) * (MTYPE)(m >> 8); \
- sum0 += (NTYPE)(n >> 16) * (MTYPE)(m >> 16); \
- sum0 += (NTYPE)(n >> 24) * (MTYPE)(m >> 24); \
- sum1 += (NTYPE)(n >> 32) * (MTYPE)(m >> 32); \
- sum1 += (NTYPE)(n >> 40) * (MTYPE)(m >> 40); \
- sum1 += (NTYPE)(n >> 48) * (MTYPE)(m >> 48); \
- sum1 += (NTYPE)(n >> 56) * (MTYPE)(m >> 56); \
- if (neg) { \
- sum0 = (uint32_t)a - sum0, sum1 = (uint32_t)(a >> 32) - sum1; \
- } else { \
- sum0 = (uint32_t)a + sum0, sum1 = (uint32_t)(a >> 32) + sum1; \
- } \
- return ((uint64_t)sum1 << 32) | sum0; \
+ sum += (NTYPE)(n >> 0) * (MTYPE)(m >> 0); \
+ sum += (NTYPE)(n >> 8) * (MTYPE)(m >> 8); \
+ sum += (NTYPE)(n >> 16) * (MTYPE)(m >> 16); \
+ sum += (NTYPE)(n >> 24) * (MTYPE)(m >> 24); \
+ return neg ? a - sum : a + sum; \
}
#define DEF_IMOP_64(NAME, NTYPE, MTYPE) \
@@ -1151,16 +1163,17 @@ DEF_IMOP_64(umopa_d, uint16_t, uint16_t)
DEF_IMOP_64(sumopa_d, int16_t, uint16_t)
DEF_IMOP_64(usmopa_d, uint16_t, int16_t)
-#define DEF_IMOPH(NAME) \
- void HELPER(sme_##NAME)(void *vza, void *vzn, void *vzm, void *vpn, \
- void *vpm, uint32_t desc) \
- { do_imopa(vza, vzn, vzm, vpn, vpm, desc, NAME); }
+#define DEF_IMOPH(NAME, S) \
+ void HELPER(sme_##NAME##_##S)(void *vza, void *vzn, void *vzm, \
+ void *vpn, void *vpm, uint32_t desc) \
+ { do_imopa_##S(vza, vzn, vzm, vpn, vpm, desc, NAME##_##S); }
-DEF_IMOPH(smopa_s)
-DEF_IMOPH(umopa_s)
-DEF_IMOPH(sumopa_s)
-DEF_IMOPH(usmopa_s)
-DEF_IMOPH(smopa_d)
-DEF_IMOPH(umopa_d)
-DEF_IMOPH(sumopa_d)
-DEF_IMOPH(usmopa_d)
+DEF_IMOPH(smopa, s)
+DEF_IMOPH(umopa, s)
+DEF_IMOPH(sumopa, s)
+DEF_IMOPH(usmopa, s)
+
+DEF_IMOPH(smopa, d)
+DEF_IMOPH(umopa, d)
+DEF_IMOPH(sumopa, d)
+DEF_IMOPH(usmopa, d)
diff --git a/tests/tcg/aarch64/sme-smopa-1.c b/tests/tcg/aarch64/sme-smopa-1.c
new file mode 100644
index 0000000000..c62d5e0007
--- /dev/null
+++ b/tests/tcg/aarch64/sme-smopa-1.c
@@ -0,0 +1,47 @@
+#include <stdio.h>
+#include <string.h>
+
+int main()
+{
+ static const int cmp[4][4] = {
+ { 110, 134, 158, 182 },
+ { 390, 478, 566, 654 },
+ { 670, 822, 974, 1126 },
+ { 950, 1166, 1382, 1598 }
+ };
+ int dst[4][4];
+ int *tmp = &dst[0][0];
+
+ asm volatile(
+ ".arch armv8-r+sme\n\t"
+ "smstart\n\t"
+ "index z0.b, #0, #1\n\t"
+ "movprfx z1, z0\n\t"
+ "add z1.b, z1.b, #16\n\t"
+ "ptrue p0.b\n\t"
+ "smopa za0.s, p0/m, p0/m, z0.b, z1.b\n\t"
+ "ptrue p0.s, vl4\n\t"
+ "mov w12, #0\n\t"
+ "st1w { za0h.s[w12, #0] }, p0, [%0]\n\t"
+ "add %0, %0, #16\n\t"
+ "st1w { za0h.s[w12, #1] }, p0, [%0]\n\t"
+ "add %0, %0, #16\n\t"
+ "st1w { za0h.s[w12, #2] }, p0, [%0]\n\t"
+ "add %0, %0, #16\n\t"
+ "st1w { za0h.s[w12, #3] }, p0, [%0]\n\t"
+ "smstop"
+ : "+r"(tmp) : : "memory");
+
+ if (memcmp(cmp, dst, sizeof(dst)) == 0) {
+ return 0;
+ }
+
+ /* See above for correct results. */
+ for (int i = 0; i < 4; ++i) {
+ for (int j = 0; j < 4; ++j) {
+ printf("%6d", dst[i][j]);
+ }
+ printf("\n");
+ }
+ return 1;
+}
diff --git a/tests/tcg/aarch64/sme-smopa-2.c b/tests/tcg/aarch64/sme-smopa-2.c
new file mode 100644
index 0000000000..c9f48c3bfc
--- /dev/null
+++ b/tests/tcg/aarch64/sme-smopa-2.c
@@ -0,0 +1,54 @@
+#include <stdio.h>
+#include <string.h>
+
+int main()
+{
+ static const long cmp[4][4] = {
+ { 110, 134, 158, 182 },
+ { 390, 478, 566, 654 },
+ { 670, 822, 974, 1126 },
+ { 950, 1166, 1382, 1598 }
+ };
+ long dst[4][4];
+ long *tmp = &dst[0][0];
+ long svl;
+
+ /* Validate that we have a wide enough vector for 4 elements. */
+ asm(".arch armv8-r+sme-i64\n\trdsvl %0, #1" : "=r"(svl));
+ if (svl < 32) {
+ return 0;
+ }
+
+ asm volatile(
+ "smstart\n\t"
+ "index z0.h, #0, #1\n\t"
+ "movprfx z1, z0\n\t"
+ "add z1.h, z1.h, #16\n\t"
+ "ptrue p0.b\n\t"
+ "smopa za0.d, p0/m, p0/m, z0.h, z1.h\n\t"
+ "ptrue p0.d, vl4\n\t"
+ "mov w12, #0\n\t"
+ "st1d { za0h.d[w12, #0] }, p0, [%0]\n\t"
+ "add %0, %0, #32\n\t"
+ "st1d { za0h.d[w12, #1] }, p0, [%0]\n\t"
+ "mov w12, #2\n\t"
+ "add %0, %0, #32\n\t"
+ "st1d { za0h.d[w12, #0] }, p0, [%0]\n\t"
+ "add %0, %0, #32\n\t"
+ "st1d { za0h.d[w12, #1] }, p0, [%0]\n\t"
+ "smstop"
+ : "+r"(tmp) : : "memory");
+
+ if (memcmp(cmp, dst, sizeof(dst)) == 0) {
+ return 0;
+ }
+
+ /* See above for correct results. */
+ for (int i = 0; i < 4; ++i) {
+ for (int j = 0; j < 4; ++j) {
+ printf("%6ld", dst[i][j]);
+ }
+ printf("\n");
+ }
+ return 1;
+}
diff --git a/tests/tcg/aarch64/Makefile.target b/tests/tcg/aarch64/Makefile.target
index cded1d01fc..ea3e232e65 100644
--- a/tests/tcg/aarch64/Makefile.target
+++ b/tests/tcg/aarch64/Makefile.target
@@ -67,7 +67,7 @@ endif
# SME Tests
ifneq ($(CROSS_AS_HAS_ARMV9_SME),)
-AARCH64_TESTS += sme-outprod1
+AARCH64_TESTS += sme-outprod1 sme-smopa-1 sme-smopa-2
endif
# System Registers Tests
--
2.34.105.03.2024 05:31, Richard Henderson : > The while the 8-bit input elements are sequential in the input vector, Nitpick: "The while the" /mjt
Hi Richard,
On 5/3/24 03:31, Richard Henderson wrote:
> The while the 8-bit input elements are sequential in the input vector,
> the 32-bit output elements are not sequential in the output matrix.
> Do not attempt to compute 2 32-bit outputs at the same time.
>
> Cc: qemu-stable@nongnu.org
> Fixes: 23a5e3859f5 ("target/arm: Implement SME integer outer product")
> Resolves: https://gitlab.com/qemu-project/qemu/-/issues/2083
> Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
> ---
> target/arm/tcg/sme_helper.c | 77 ++++++++++++++++++-------------
> tests/tcg/aarch64/sme-smopa-1.c | 47 +++++++++++++++++++
> tests/tcg/aarch64/sme-smopa-2.c | 54 ++++++++++++++++++++++
> tests/tcg/aarch64/Makefile.target | 2 +-
> 4 files changed, 147 insertions(+), 33 deletions(-)
> create mode 100644 tests/tcg/aarch64/sme-smopa-1.c
> create mode 100644 tests/tcg/aarch64/sme-smopa-2.c
>
> diff --git a/target/arm/tcg/sme_helper.c b/target/arm/tcg/sme_helper.c
> index 904bfdac43..ef39eee48d 100644
> --- a/target/arm/tcg/sme_helper.c
> +++ b/target/arm/tcg/sme_helper.c
> @@ -1083,11 +1083,32 @@ void HELPER(sme_bfmopa)(void *vza, void *vzn, void *vzm, void *vpn,
> }
> }
>
> -typedef uint64_t IMOPFn(uint64_t, uint64_t, uint64_t, uint8_t, bool);
> +typedef uint32_t IMOPFn32(uint32_t, uint32_t, uint32_t, uint8_t, bool);
> +static inline void do_imopa_s(uint32_t *za, uint32_t *zn, uint32_t *zm,
> + uint8_t *pn, uint8_t *pm,
> + uint32_t desc, IMOPFn32 *fn)
> +{
> + intptr_t row, col, oprsz = simd_oprsz(desc) / 4;
> + bool neg = simd_data(desc);
>
> -static inline void do_imopa(uint64_t *za, uint64_t *zn, uint64_t *zm,
> - uint8_t *pn, uint8_t *pm,
> - uint32_t desc, IMOPFn *fn)
> + for (row = 0; row < oprsz; ++row) {
> + uint8_t pa = pn[H1(row >> 1)] >> ((row & 1) * 4);
> + uint32_t *za_row = &za[H4(tile_vslice_index(row))];
> + uint32_t n = zn[H4(row)];
> +
> + for (col = 0; col < oprsz; ++col) {
> + uint8_t pb = pm[H1(col >> 1)] >> ((col & 1) * 4);
> + uint32_t *a = &za_row[col];
Shouldn't this be:
uint32_t *a = &za_row[H4(col)];
to work on big endian hosts?
> +
> + *a = fn(n, zm[H4(col)], *a, pa & pb & 0xf, neg);
> + }
> + }
> +}
> +
> +typedef uint64_t IMOPFn64(uint64_t, uint64_t, uint64_t, uint8_t, bool);
> +static inline void do_imopa_d(uint64_t *za, uint64_t *zn, uint64_t *zm,
> + uint8_t *pn, uint8_t *pm,
> + uint32_t desc, IMOPFn64 *fn)
> {
> intptr_t row, col, oprsz = simd_oprsz(desc) / 8;
> bool neg = simd_data(desc);
> @@ -1107,25 +1128,16 @@ static inline void do_imopa(uint64_t *za, uint64_t *zn, uint64_t *zm,
> }
>
> #define DEF_IMOP_32(NAME, NTYPE, MTYPE) \
> -static uint64_t NAME(uint64_t n, uint64_t m, uint64_t a, uint8_t p, bool neg) \
> +static uint32_t NAME(uint32_t n, uint32_t m, uint32_t a, uint8_t p, bool neg) \
> { \
> - uint32_t sum0 = 0, sum1 = 0; \
> + uint32_t sum = 0; \
> /* Apply P to N as a mask, making the inactive elements 0. */ \
> n &= expand_pred_b(p); \
> - sum0 += (NTYPE)(n >> 0) * (MTYPE)(m >> 0); \
> - sum0 += (NTYPE)(n >> 8) * (MTYPE)(m >> 8); \
> - sum0 += (NTYPE)(n >> 16) * (MTYPE)(m >> 16); \
> - sum0 += (NTYPE)(n >> 24) * (MTYPE)(m >> 24); \
> - sum1 += (NTYPE)(n >> 32) * (MTYPE)(m >> 32); \
> - sum1 += (NTYPE)(n >> 40) * (MTYPE)(m >> 40); \
> - sum1 += (NTYPE)(n >> 48) * (MTYPE)(m >> 48); \
> - sum1 += (NTYPE)(n >> 56) * (MTYPE)(m >> 56); \
> - if (neg) { \
> - sum0 = (uint32_t)a - sum0, sum1 = (uint32_t)(a >> 32) - sum1; \
> - } else { \
> - sum0 = (uint32_t)a + sum0, sum1 = (uint32_t)(a >> 32) + sum1; \
> - } \
> - return ((uint64_t)sum1 << 32) | sum0; \
> + sum += (NTYPE)(n >> 0) * (MTYPE)(m >> 0); \
> + sum += (NTYPE)(n >> 8) * (MTYPE)(m >> 8); \
> + sum += (NTYPE)(n >> 16) * (MTYPE)(m >> 16); \
> + sum += (NTYPE)(n >> 24) * (MTYPE)(m >> 24); \
> + return neg ? a - sum : a + sum; \
> }
On 3/4/24 23:19, Philippe Mathieu-Daudé wrote:
>> + uint32_t *za_row = &za[H4(tile_vslice_index(row))];
>> + uint32_t n = zn[H4(row)];
>> +
>> + for (col = 0; col < oprsz; ++col) {
>> + uint8_t pb = pm[H1(col >> 1)] >> ((col & 1) * 4);
>> + uint32_t *a = &za_row[col];
>
> Shouldn't this be:
>
> uint32_t *a = &za_row[H4(col)];
>
> to work on big endian hosts?
Oof. Yes, the H4 adjustment belongs here, not on za_row above.
r~
© 2016 - 2024 Red Hat, Inc.