The following changes since commit ad1b4ec39caa5b3f17cbd8160283a03a3dcfe2ae: Merge remote-tracking branch 'remotes/kraxel/tags/input-20180515-pull-request' into staging (2018-05-15 12:50:06 +0100) are available in the Git repository at: git://git.linaro.org/people/pmaydell/qemu-arm.git tags/pull-target-arm-20180515 for you to fetch changes up to ae7651804748c6b479d5ae09aeac4edb9c44f76e: tcg: Optionally log FPU state in TCG -d cpu logging (2018-05-15 14:58:44 +0100) ---------------------------------------------------------------- target-arm queue: * Fix coverity nit in int_to_float code * Don't set Invalid for float-to-int(MAXINT) * Fix fp_status_f16 tininess before rounding * Add various missing insns from the v8.2-FP16 extension * Fix sqrt_f16 exception raising * sdcard: Correct CRC16 offset in sd_function_switch() * tcg: Optionally log FPU state in TCG -d cpu logging ---------------------------------------------------------------- Alex Bennée (5): fpu/softfloat: int_to_float ensure r fully initialised target/arm: Implement FCMP for fp16 target/arm: Implement FCSEL for fp16 target/arm: Implement FMOV (immediate) for fp16 target/arm: Fix sqrt_f16 exception raising Peter Maydell (3): fpu/softfloat: Don't set Invalid for float-to-int(MAXINT) target/arm: Fix fp_status_f16 tininess before rounding tcg: Optionally log FPU state in TCG -d cpu logging Philippe Mathieu-Daudé (1): sdcard: Correct CRC16 offset in sd_function_switch() Richard Henderson (7): target/arm: Implement FMOV (general) for fp16 target/arm: Early exit after unallocated_encoding in disas_fp_int_conv target/arm: Implement FCVT (scalar, integer) for fp16 target/arm: Implement FCVT (scalar, fixed-point) for fp16 target/arm: Introduce and use read_fp_hreg target/arm: Implement FP data-processing (2 source) for fp16 target/arm: Implement FP data-processing (3 source) for fp16 include/qemu/log.h | 1 + target/arm/helper-a64.h | 2 + target/arm/helper.h | 6 + accel/tcg/cpu-exec.c | 9 +- fpu/softfloat.c | 6 +- hw/sd/sd.c | 2 +- target/arm/cpu.c | 2 + target/arm/helper-a64.c | 10 ++ target/arm/helper.c | 38 +++- target/arm/translate-a64.c | 421 ++++++++++++++++++++++++++++++++++++++------- util/log.c | 2 + 11 files changed, 428 insertions(+), 71 deletions(-) From: Alex Bennée <alex.bennee@linaro.org> Reported by Coverity (CID1390635). We ensure this for uint_to_float later on so we might as well mirror that. Signed-off-by: Alex Bennée <alex.bennee@linaro.org> Reviewed-by: Richard Henderson <richard.henderson@linaro.org> Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- fpu/softfloat.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fpu/softfloat.c b/fpu/softfloat.c index XXXXXXX..XXXXXXX 100644 --- a/fpu/softfloat.c +++ b/fpu/softfloat.c @@ -XXX,XX +XXX,XX @@ FLOAT_TO_UINT(64, 64) static FloatParts int_to_float(int64_t a, float_status *status) { - FloatParts r; + FloatParts r = {}; if (a == 0) { r.cls = float_class_zero; r.sign = false; -- 2.17.0 In float-to-integer conversion, if the floating point input converts exactly to the largest or smallest integer that fits in to the result type, this is not an overflow. In this situation we were producing the correct result value, but were incorrectly setting the Invalid flag. For example for Arm A64, "FCVTAS w0, d0" on an input of 0x41dfffffffc00000 should produce 0x7fffffff and set no flags. Fix the boundary case to take the right half of the if() statements. This fixes a regression from 2.11 introduced by the softfloat refactoring. Cc: qemu-stable@nongnu.org Fixes: ab52f973a50 Signed-off-by: Peter Maydell <peter.maydell@linaro.org> Reviewed-by: Richard Henderson <richard.henderson@linaro.org> Message-id: 20180510140141.12120-1-peter.maydell@linaro.org --- fpu/softfloat.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fpu/softfloat.c b/fpu/softfloat.c index XXXXXXX..XXXXXXX 100644 --- a/fpu/softfloat.c +++ b/fpu/softfloat.c @@ -XXX,XX +XXX,XX @@ static int64_t round_to_int_and_pack(FloatParts in, int rmode, r = UINT64_MAX; } if (p.sign) { - if (r < -(uint64_t) min) { + if (r <= -(uint64_t) min) { return -r; } else { s->float_exception_flags = orig_flags | float_flag_invalid; return min; } } else { - if (r < max) { + if (r <= max) { return r; } else { s->float_exception_flags = orig_flags | float_flag_invalid; -- 2.17.0 In commit d81ce0ef2c4f105 we added an extra float_status field fp_status_fp16 for Arm, but forgot to initialize it correctly by setting it to float_tininess_before_rounding. This currently will only cause problems for the new V8_FP16 feature, since the float-to-float conversion code doesn't use it yet. The effect would be that we failed to set the Underflow IEEE exception flag in all the cases where we should. Add the missing initialization. Fixes: d81ce0ef2c4f105 Cc: qemu-stable@nongnu.org Reviewed-by: Richard Henderson <richard.henderson@linaro.org> Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Signed-off-by: Peter Maydell <peter.maydell@linaro.org> Message-id: 20180512004311.9299-16-richard.henderson@linaro.org --- target/arm/cpu.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/target/arm/cpu.c b/target/arm/cpu.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/cpu.c +++ b/target/arm/cpu.c @@ -XXX,XX +XXX,XX @@ static void arm_cpu_reset(CPUState *s) &env->vfp.fp_status); set_float_detect_tininess(float_tininess_before_rounding, &env->vfp.standard_fp_status); + set_float_detect_tininess(float_tininess_before_rounding, + &env->vfp.fp_status_f16); #ifndef CONFIG_USER_ONLY if (kvm_enabled()) { kvm_arm_reset_vcpu(cpu); -- 2.17.0 From: Richard Henderson <richard.henderson@linaro.org> Adding the fp16 moves to/from general registers. Cc: qemu-stable@nongnu.org Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Tested-by: Alex Bennée <alex.bennee@linaro.org> Message-id: 20180512003217.9105-2-richard.henderson@linaro.org Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- target/arm/translate-a64.c | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/target/arm/translate-a64.c b/target/arm/translate-a64.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/translate-a64.c +++ b/target/arm/translate-a64.c @@ -XXX,XX +XXX,XX @@ static void handle_fmov(DisasContext *s, int rd, int rn, int type, bool itof) tcg_gen_st_i64(tcg_rn, cpu_env, fp_reg_hi_offset(s, rd)); clear_vec_high(s, true, rd); break; + case 3: + /* 16 bit */ + tmp = tcg_temp_new_i64(); + tcg_gen_ext16u_i64(tmp, tcg_rn); + write_fp_dreg(s, rd, tmp); + tcg_temp_free_i64(tmp); + break; + default: + g_assert_not_reached(); } } else { TCGv_i64 tcg_rd = cpu_reg(s, rd); @@ -XXX,XX +XXX,XX @@ static void handle_fmov(DisasContext *s, int rd, int rn, int type, bool itof) /* 64 bits from top half */ tcg_gen_ld_i64(tcg_rd, cpu_env, fp_reg_hi_offset(s, rn)); break; + case 3: + /* 16 bit */ + tcg_gen_ld16u_i64(tcg_rd, cpu_env, fp_reg_offset(s, rn, MO_16)); + break; + default: + g_assert_not_reached(); } } } @@ -XXX,XX +XXX,XX @@ static void disas_fp_int_conv(DisasContext *s, uint32_t insn) case 0xa: /* 64 bit */ case 0xd: /* 64 bit to top half of quad */ break; + case 0x6: /* 16-bit float, 32-bit int */ + case 0xe: /* 16-bit float, 64-bit int */ + if (arm_dc_feature(s, ARM_FEATURE_V8_FP16)) { + break; + } + /* fallthru */ default: /* all other sf/type/rmode combinations are invalid */ unallocated_encoding(s); -- 2.17.0 From: Richard Henderson <richard.henderson@linaro.org> No sense in emitting code after the exception. Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Tested-by: Alex Bennée <alex.bennee@linaro.org> Message-id: 20180512003217.9105-3-richard.henderson@linaro.org Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- target/arm/translate-a64.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/target/arm/translate-a64.c b/target/arm/translate-a64.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/translate-a64.c +++ b/target/arm/translate-a64.c @@ -XXX,XX +XXX,XX @@ static void disas_fp_int_conv(DisasContext *s, uint32_t insn) default: /* all other sf/type/rmode combinations are invalid */ unallocated_encoding(s); - break; + return; } if (!fp_access_check(s)) { -- 2.17.0 From: Richard Henderson <richard.henderson@linaro.org> Cc: qemu-stable@nongnu.org Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Tested-by: Alex Bennée <alex.bennee@linaro.org> Message-id: 20180512003217.9105-4-richard.henderson@linaro.org Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- target/arm/helper.h | 6 +++ target/arm/helper.c | 38 ++++++++++++++- target/arm/translate-a64.c | 96 +++++++++++++++++++++++++++++++------- 3 files changed, 122 insertions(+), 18 deletions(-) diff --git a/target/arm/helper.h b/target/arm/helper.h index XXXXXXX..XXXXXXX 100644 --- a/target/arm/helper.h +++ b/target/arm/helper.h @@ -XXX,XX +XXX,XX @@ DEF_HELPER_3(vfp_touhd_round_to_zero, i64, f64, i32, ptr) DEF_HELPER_3(vfp_tould_round_to_zero, i64, f64, i32, ptr) DEF_HELPER_3(vfp_touhh, i32, f16, i32, ptr) DEF_HELPER_3(vfp_toshh, i32, f16, i32, ptr) +DEF_HELPER_3(vfp_toulh, i32, f16, i32, ptr) +DEF_HELPER_3(vfp_toslh, i32, f16, i32, ptr) +DEF_HELPER_3(vfp_touqh, i64, f16, i32, ptr) +DEF_HELPER_3(vfp_tosqh, i64, f16, i32, ptr) DEF_HELPER_3(vfp_toshs, i32, f32, i32, ptr) DEF_HELPER_3(vfp_tosls, i32, f32, i32, ptr) DEF_HELPER_3(vfp_tosqs, i64, f32, i32, ptr) @@ -XXX,XX +XXX,XX @@ DEF_HELPER_3(vfp_ultod, f64, i64, i32, ptr) DEF_HELPER_3(vfp_uqtod, f64, i64, i32, ptr) DEF_HELPER_3(vfp_sltoh, f16, i32, i32, ptr) DEF_HELPER_3(vfp_ultoh, f16, i32, i32, ptr) +DEF_HELPER_3(vfp_sqtoh, f16, i64, i32, ptr) +DEF_HELPER_3(vfp_uqtoh, f16, i64, i32, ptr) DEF_HELPER_FLAGS_2(set_rmode, TCG_CALL_NO_RWG, i32, i32, ptr) DEF_HELPER_FLAGS_2(set_neon_rmode, TCG_CALL_NO_RWG, i32, i32, env) diff --git a/target/arm/helper.c b/target/arm/helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/helper.c +++ b/target/arm/helper.c @@ -XXX,XX +XXX,XX @@ VFP_CONV_FIX_A64(uq, s, 32, 64, uint64) #undef VFP_CONV_FIX_A64 /* Conversion to/from f16 can overflow to infinity before/after scaling. - * Therefore we convert to f64 (which does not round), scale, - * and then convert f64 to f16 (which may round). + * Therefore we convert to f64, scale, and then convert f64 to f16; or + * vice versa for conversion to integer. + * + * For 16- and 32-bit integers, the conversion to f64 never rounds. + * For 64-bit integers, any integer that would cause rounding will also + * overflow to f16 infinity, so there is no double rounding problem. */ static float16 do_postscale_fp16(float64 f, int shift, float_status *fpst) @@ -XXX,XX +XXX,XX @@ float16 HELPER(vfp_ultoh)(uint32_t x, uint32_t shift, void *fpst) return do_postscale_fp16(uint32_to_float64(x, fpst), shift, fpst); } +float16 HELPER(vfp_sqtoh)(uint64_t x, uint32_t shift, void *fpst) +{ + return do_postscale_fp16(int64_to_float64(x, fpst), shift, fpst); +} + +float16 HELPER(vfp_uqtoh)(uint64_t x, uint32_t shift, void *fpst) +{ + return do_postscale_fp16(uint64_to_float64(x, fpst), shift, fpst); +} + static float64 do_prescale_fp16(float16 f, int shift, float_status *fpst) { if (unlikely(float16_is_any_nan(f))) { @@ -XXX,XX +XXX,XX @@ uint32_t HELPER(vfp_touhh)(float16 x, uint32_t shift, void *fpst) return float64_to_uint16(do_prescale_fp16(x, shift, fpst), fpst); } +uint32_t HELPER(vfp_toslh)(float16 x, uint32_t shift, void *fpst) +{ + return float64_to_int32(do_prescale_fp16(x, shift, fpst), fpst); +} + +uint32_t HELPER(vfp_toulh)(float16 x, uint32_t shift, void *fpst) +{ + return float64_to_uint32(do_prescale_fp16(x, shift, fpst), fpst); +} + +uint64_t HELPER(vfp_tosqh)(float16 x, uint32_t shift, void *fpst) +{ + return float64_to_int64(do_prescale_fp16(x, shift, fpst), fpst); +} + +uint64_t HELPER(vfp_touqh)(float16 x, uint32_t shift, void *fpst) +{ + return float64_to_uint64(do_prescale_fp16(x, shift, fpst), fpst); +} + /* Set the current fp rounding mode and return the old one. * The argument is a softfloat float_round_ value. */ diff --git a/target/arm/translate-a64.c b/target/arm/translate-a64.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/translate-a64.c +++ b/target/arm/translate-a64.c @@ -XXX,XX +XXX,XX @@ static void handle_fpfpcvt(DisasContext *s, int rd, int rn, int opcode, bool itof, int rmode, int scale, int sf, int type) { bool is_signed = !(opcode & 1); - bool is_double = type; TCGv_ptr tcg_fpstatus; - TCGv_i32 tcg_shift; + TCGv_i32 tcg_shift, tcg_single; + TCGv_i64 tcg_double; - tcg_fpstatus = get_fpstatus_ptr(false); + tcg_fpstatus = get_fpstatus_ptr(type == 3); tcg_shift = tcg_const_i32(64 - scale); @@ -XXX,XX +XXX,XX @@ static void handle_fpfpcvt(DisasContext *s, int rd, int rn, int opcode, tcg_int = tcg_extend; } - if (is_double) { - TCGv_i64 tcg_double = tcg_temp_new_i64(); + switch (type) { + case 1: /* float64 */ + tcg_double = tcg_temp_new_i64(); if (is_signed) { gen_helper_vfp_sqtod(tcg_double, tcg_int, tcg_shift, tcg_fpstatus); @@ -XXX,XX +XXX,XX @@ static void handle_fpfpcvt(DisasContext *s, int rd, int rn, int opcode, } write_fp_dreg(s, rd, tcg_double); tcg_temp_free_i64(tcg_double); - } else { - TCGv_i32 tcg_single = tcg_temp_new_i32(); + break; + + case 0: /* float32 */ + tcg_single = tcg_temp_new_i32(); if (is_signed) { gen_helper_vfp_sqtos(tcg_single, tcg_int, tcg_shift, tcg_fpstatus); @@ -XXX,XX +XXX,XX @@ static void handle_fpfpcvt(DisasContext *s, int rd, int rn, int opcode, } write_fp_sreg(s, rd, tcg_single); tcg_temp_free_i32(tcg_single); + break; + + case 3: /* float16 */ + tcg_single = tcg_temp_new_i32(); + if (is_signed) { + gen_helper_vfp_sqtoh(tcg_single, tcg_int, + tcg_shift, tcg_fpstatus); + } else { + gen_helper_vfp_uqtoh(tcg_single, tcg_int, + tcg_shift, tcg_fpstatus); + } + write_fp_sreg(s, rd, tcg_single); + tcg_temp_free_i32(tcg_single); + break; + + default: + g_assert_not_reached(); } } else { TCGv_i64 tcg_int = cpu_reg(s, rd); @@ -XXX,XX +XXX,XX @@ static void handle_fpfpcvt(DisasContext *s, int rd, int rn, int opcode, gen_helper_set_rmode(tcg_rmode, tcg_rmode, tcg_fpstatus); - if (is_double) { - TCGv_i64 tcg_double = read_fp_dreg(s, rn); + switch (type) { + case 1: /* float64 */ + tcg_double = read_fp_dreg(s, rn); if (is_signed) { if (!sf) { gen_helper_vfp_tosld(tcg_int, tcg_double, @@ -XXX,XX +XXX,XX @@ static void handle_fpfpcvt(DisasContext *s, int rd, int rn, int opcode, tcg_shift, tcg_fpstatus); } } + if (!sf) { + tcg_gen_ext32u_i64(tcg_int, tcg_int); + } tcg_temp_free_i64(tcg_double); - } else { - TCGv_i32 tcg_single = read_fp_sreg(s, rn); + break; + + case 0: /* float32 */ + tcg_single = read_fp_sreg(s, rn); if (sf) { if (is_signed) { gen_helper_vfp_tosqs(tcg_int, tcg_single, @@ -XXX,XX +XXX,XX @@ static void handle_fpfpcvt(DisasContext *s, int rd, int rn, int opcode, tcg_temp_free_i32(tcg_dest); } tcg_temp_free_i32(tcg_single); + break; + + case 3: /* float16 */ + tcg_single = read_fp_sreg(s, rn); + if (sf) { + if (is_signed) { + gen_helper_vfp_tosqh(tcg_int, tcg_single, + tcg_shift, tcg_fpstatus); + } else { + gen_helper_vfp_touqh(tcg_int, tcg_single, + tcg_shift, tcg_fpstatus); + } + } else { + TCGv_i32 tcg_dest = tcg_temp_new_i32(); + if (is_signed) { + gen_helper_vfp_toslh(tcg_dest, tcg_single, + tcg_shift, tcg_fpstatus); + } else { + gen_helper_vfp_toulh(tcg_dest, tcg_single, + tcg_shift, tcg_fpstatus); + } + tcg_gen_extu_i32_i64(tcg_int, tcg_dest); + tcg_temp_free_i32(tcg_dest); + } + tcg_temp_free_i32(tcg_single); + break; + + default: + g_assert_not_reached(); } gen_helper_set_rmode(tcg_rmode, tcg_rmode, tcg_fpstatus); tcg_temp_free_i32(tcg_rmode); - - if (!sf) { - tcg_gen_ext32u_i64(tcg_int, tcg_int); - } } tcg_temp_free_ptr(tcg_fpstatus); @@ -XXX,XX +XXX,XX @@ static void disas_fp_int_conv(DisasContext *s, uint32_t insn) /* actual FP conversions */ bool itof = extract32(opcode, 1, 1); - if (type > 1 || (rmode != 0 && opcode > 1)) { + if (rmode != 0 && opcode > 1) { + unallocated_encoding(s); + return; + } + switch (type) { + case 0: /* float32 */ + case 1: /* float64 */ + break; + case 3: /* float16 */ + if (arm_dc_feature(s, ARM_FEATURE_V8_FP16)) { + break; + } + /* fallthru */ + default: unallocated_encoding(s); return; } -- 2.17.0 From: Richard Henderson <richard.henderson@linaro.org> Cc: qemu-stable@nongnu.org Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Tested-by: Alex Bennée <alex.bennee@linaro.org> Message-id: 20180512003217.9105-5-richard.henderson@linaro.org Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- target/arm/translate-a64.c | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/target/arm/translate-a64.c b/target/arm/translate-a64.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/translate-a64.c +++ b/target/arm/translate-a64.c @@ -XXX,XX +XXX,XX @@ static void disas_fp_fixed_conv(DisasContext *s, uint32_t insn) bool sf = extract32(insn, 31, 1); bool itof; - if (sbit || (type > 1) - || (!sf && scale < 32)) { + if (sbit || (!sf && scale < 32)) { + unallocated_encoding(s); + return; + } + + switch (type) { + case 0: /* float32 */ + case 1: /* float64 */ + break; + case 3: /* float16 */ + if (arm_dc_feature(s, ARM_FEATURE_V8_FP16)) { + break; + } + /* fallthru */ + default: unallocated_encoding(s); return; } -- 2.17.0 From: Richard Henderson <richard.henderson@linaro.org> Cc: qemu-stable@nongnu.org Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Tested-by: Alex Bennée <alex.bennee@linaro.org> Message-id: 20180512003217.9105-6-richard.henderson@linaro.org Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- target/arm/translate-a64.c | 30 ++++++++++++++---------------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/target/arm/translate-a64.c b/target/arm/translate-a64.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/translate-a64.c +++ b/target/arm/translate-a64.c @@ -XXX,XX +XXX,XX @@ static TCGv_i32 read_fp_sreg(DisasContext *s, int reg) return v; } +static TCGv_i32 read_fp_hreg(DisasContext *s, int reg) +{ + TCGv_i32 v = tcg_temp_new_i32(); + + tcg_gen_ld16u_i32(v, cpu_env, fp_reg_offset(s, reg, MO_16)); + return v; +} + /* Clear the bits above an N-bit vector, for N = (is_q ? 128 : 64). * If SVE is not enabled, then there are only 128 bits in the vector. */ @@ -XXX,XX +XXX,XX @@ static void disas_fp_csel(DisasContext *s, uint32_t insn) static void handle_fp_1src_half(DisasContext *s, int opcode, int rd, int rn) { TCGv_ptr fpst = NULL; - TCGv_i32 tcg_op = tcg_temp_new_i32(); + TCGv_i32 tcg_op = read_fp_hreg(s, rn); TCGv_i32 tcg_res = tcg_temp_new_i32(); - read_vec_element_i32(s, tcg_op, rn, 0, MO_16); - switch (opcode) { case 0x0: /* FMOV */ tcg_gen_mov_i32(tcg_res, tcg_op); @@ -XXX,XX +XXX,XX @@ static void disas_simd_scalar_three_reg_diff(DisasContext *s, uint32_t insn) tcg_temp_free_i64(tcg_op2); tcg_temp_free_i64(tcg_res); } else { - TCGv_i32 tcg_op1 = tcg_temp_new_i32(); - TCGv_i32 tcg_op2 = tcg_temp_new_i32(); + TCGv_i32 tcg_op1 = read_fp_hreg(s, rn); + TCGv_i32 tcg_op2 = read_fp_hreg(s, rm); TCGv_i64 tcg_res = tcg_temp_new_i64(); - read_vec_element_i32(s, tcg_op1, rn, 0, MO_16); - read_vec_element_i32(s, tcg_op2, rm, 0, MO_16); - gen_helper_neon_mull_s16(tcg_res, tcg_op1, tcg_op2); gen_helper_neon_addl_saturate_s32(tcg_res, cpu_env, tcg_res, tcg_res); @@ -XXX,XX +XXX,XX @@ static void disas_simd_scalar_three_reg_same_fp16(DisasContext *s, fpst = get_fpstatus_ptr(true); - tcg_op1 = tcg_temp_new_i32(); - tcg_op2 = tcg_temp_new_i32(); + tcg_op1 = read_fp_hreg(s, rn); + tcg_op2 = read_fp_hreg(s, rm); tcg_res = tcg_temp_new_i32(); - read_vec_element_i32(s, tcg_op1, rn, 0, MO_16); - read_vec_element_i32(s, tcg_op2, rm, 0, MO_16); - switch (fpopcode) { case 0x03: /* FMULX */ gen_helper_advsimd_mulxh(tcg_res, tcg_op1, tcg_op2, fpst); @@ -XXX,XX +XXX,XX @@ static void disas_simd_two_reg_misc_fp16(DisasContext *s, uint32_t insn) } if (is_scalar) { - TCGv_i32 tcg_op = tcg_temp_new_i32(); + TCGv_i32 tcg_op = read_fp_hreg(s, rn); TCGv_i32 tcg_res = tcg_temp_new_i32(); - read_vec_element_i32(s, tcg_op, rn, 0, MO_16); - switch (fpop) { case 0x1a: /* FCVTNS */ case 0x1b: /* FCVTMS */ -- 2.17.0 From: Richard Henderson <richard.henderson@linaro.org> We missed all of the scalar fp16 binary operations. Cc: qemu-stable@nongnu.org Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Tested-by: Alex Bennée <alex.bennee@linaro.org> Message-id: 20180512003217.9105-7-richard.henderson@linaro.org Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- target/arm/translate-a64.c | 65 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) diff --git a/target/arm/translate-a64.c b/target/arm/translate-a64.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/translate-a64.c +++ b/target/arm/translate-a64.c @@ -XXX,XX +XXX,XX @@ static void handle_fp_2src_double(DisasContext *s, int opcode, tcg_temp_free_i64(tcg_res); } +/* Floating-point data-processing (2 source) - half precision */ +static void handle_fp_2src_half(DisasContext *s, int opcode, + int rd, int rn, int rm) +{ + TCGv_i32 tcg_op1; + TCGv_i32 tcg_op2; + TCGv_i32 tcg_res; + TCGv_ptr fpst; + + tcg_res = tcg_temp_new_i32(); + fpst = get_fpstatus_ptr(true); + tcg_op1 = read_fp_hreg(s, rn); + tcg_op2 = read_fp_hreg(s, rm); + + switch (opcode) { + case 0x0: /* FMUL */ + gen_helper_advsimd_mulh(tcg_res, tcg_op1, tcg_op2, fpst); + break; + case 0x1: /* FDIV */ + gen_helper_advsimd_divh(tcg_res, tcg_op1, tcg_op2, fpst); + break; + case 0x2: /* FADD */ + gen_helper_advsimd_addh(tcg_res, tcg_op1, tcg_op2, fpst); + break; + case 0x3: /* FSUB */ + gen_helper_advsimd_subh(tcg_res, tcg_op1, tcg_op2, fpst); + break; + case 0x4: /* FMAX */ + gen_helper_advsimd_maxh(tcg_res, tcg_op1, tcg_op2, fpst); + break; + case 0x5: /* FMIN */ + gen_helper_advsimd_minh(tcg_res, tcg_op1, tcg_op2, fpst); + break; + case 0x6: /* FMAXNM */ + gen_helper_advsimd_maxnumh(tcg_res, tcg_op1, tcg_op2, fpst); + break; + case 0x7: /* FMINNM */ + gen_helper_advsimd_minnumh(tcg_res, tcg_op1, tcg_op2, fpst); + break; + case 0x8: /* FNMUL */ + gen_helper_advsimd_mulh(tcg_res, tcg_op1, tcg_op2, fpst); + tcg_gen_xori_i32(tcg_res, tcg_res, 0x8000); + break; + default: + g_assert_not_reached(); + } + + write_fp_sreg(s, rd, tcg_res); + + tcg_temp_free_ptr(fpst); + tcg_temp_free_i32(tcg_op1); + tcg_temp_free_i32(tcg_op2); + tcg_temp_free_i32(tcg_res); +} + /* Floating point data-processing (2 source) * 31 30 29 28 24 23 22 21 20 16 15 12 11 10 9 5 4 0 * +---+---+---+-----------+------+---+------+--------+-----+------+------+ @@ -XXX,XX +XXX,XX @@ static void disas_fp_2src(DisasContext *s, uint32_t insn) } handle_fp_2src_double(s, opcode, rd, rn, rm); break; + case 3: + if (!arm_dc_feature(s, ARM_FEATURE_V8_FP16)) { + unallocated_encoding(s); + return; + } + if (!fp_access_check(s)) { + return; + } + handle_fp_2src_half(s, opcode, rd, rn, rm); + break; default: unallocated_encoding(s); } -- 2.17.0 From: Richard Henderson <richard.henderson@linaro.org> We missed all of the scalar fp16 fma operations. Cc: qemu-stable@nongnu.org Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Tested-by: Alex Bennée <alex.bennee@linaro.org> Message-id: 20180512003217.9105-8-richard.henderson@linaro.org Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- target/arm/translate-a64.c | 48 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/target/arm/translate-a64.c b/target/arm/translate-a64.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/translate-a64.c +++ b/target/arm/translate-a64.c @@ -XXX,XX +XXX,XX @@ static void handle_fp_3src_double(DisasContext *s, bool o0, bool o1, tcg_temp_free_i64(tcg_res); } +/* Floating-point data-processing (3 source) - half precision */ +static void handle_fp_3src_half(DisasContext *s, bool o0, bool o1, + int rd, int rn, int rm, int ra) +{ + TCGv_i32 tcg_op1, tcg_op2, tcg_op3; + TCGv_i32 tcg_res = tcg_temp_new_i32(); + TCGv_ptr fpst = get_fpstatus_ptr(true); + + tcg_op1 = read_fp_hreg(s, rn); + tcg_op2 = read_fp_hreg(s, rm); + tcg_op3 = read_fp_hreg(s, ra); + + /* These are fused multiply-add, and must be done as one + * floating point operation with no rounding between the + * multiplication and addition steps. + * NB that doing the negations here as separate steps is + * correct : an input NaN should come out with its sign bit + * flipped if it is a negated-input. + */ + if (o1 == true) { + tcg_gen_xori_i32(tcg_op3, tcg_op3, 0x8000); + } + + if (o0 != o1) { + tcg_gen_xori_i32(tcg_op1, tcg_op1, 0x8000); + } + + gen_helper_advsimd_muladdh(tcg_res, tcg_op1, tcg_op2, tcg_op3, fpst); + + write_fp_sreg(s, rd, tcg_res); + + tcg_temp_free_ptr(fpst); + tcg_temp_free_i32(tcg_op1); + tcg_temp_free_i32(tcg_op2); + tcg_temp_free_i32(tcg_op3); + tcg_temp_free_i32(tcg_res); +} + /* Floating point data-processing (3 source) * 31 30 29 28 24 23 22 21 20 16 15 14 10 9 5 4 0 * +---+---+---+-----------+------+----+------+----+------+------+------+ @@ -XXX,XX +XXX,XX @@ static void disas_fp_3src(DisasContext *s, uint32_t insn) } handle_fp_3src_double(s, o0, o1, rd, rn, rm, ra); break; + case 3: + if (!arm_dc_feature(s, ARM_FEATURE_V8_FP16)) { + unallocated_encoding(s); + return; + } + if (!fp_access_check(s)) { + return; + } + handle_fp_3src_half(s, o0, o1, rd, rn, rm, ra); + break; default: unallocated_encoding(s); } -- 2.17.0 From: Alex Bennée <alex.bennee@linaro.org> These where missed out from the rest of the half-precision work. Cc: qemu-stable@nongnu.org Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Signed-off-by: Alex Bennée <alex.bennee@linaro.org> Tested-by: Alex Bennée <alex.bennee@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Message-id: 20180512003217.9105-9-richard.henderson@linaro.org [rth: Diagnose lack of FP16 before fp_access_check] Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- target/arm/helper-a64.h | 2 + target/arm/helper-a64.c | 10 +++++ target/arm/translate-a64.c | 88 ++++++++++++++++++++++++++++++-------- 3 files changed, 83 insertions(+), 17 deletions(-) diff --git a/target/arm/helper-a64.h b/target/arm/helper-a64.h index XXXXXXX..XXXXXXX 100644 --- a/target/arm/helper-a64.h +++ b/target/arm/helper-a64.h @@ -XXX,XX +XXX,XX @@ DEF_HELPER_FLAGS_2(udiv64, TCG_CALL_NO_RWG_SE, i64, i64, i64) DEF_HELPER_FLAGS_2(sdiv64, TCG_CALL_NO_RWG_SE, s64, s64, s64) DEF_HELPER_FLAGS_1(rbit64, TCG_CALL_NO_RWG_SE, i64, i64) +DEF_HELPER_3(vfp_cmph_a64, i64, f16, f16, ptr) +DEF_HELPER_3(vfp_cmpeh_a64, i64, f16, f16, ptr) DEF_HELPER_3(vfp_cmps_a64, i64, f32, f32, ptr) DEF_HELPER_3(vfp_cmpes_a64, i64, f32, f32, ptr) DEF_HELPER_3(vfp_cmpd_a64, i64, f64, f64, ptr) diff --git a/target/arm/helper-a64.c b/target/arm/helper-a64.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/helper-a64.c +++ b/target/arm/helper-a64.c @@ -XXX,XX +XXX,XX @@ static inline uint32_t float_rel_to_flags(int res) return flags; } +uint64_t HELPER(vfp_cmph_a64)(float16 x, float16 y, void *fp_status) +{ + return float_rel_to_flags(float16_compare_quiet(x, y, fp_status)); +} + +uint64_t HELPER(vfp_cmpeh_a64)(float16 x, float16 y, void *fp_status) +{ + return float_rel_to_flags(float16_compare(x, y, fp_status)); +} + uint64_t HELPER(vfp_cmps_a64)(float32 x, float32 y, void *fp_status) { return float_rel_to_flags(float32_compare_quiet(x, y, fp_status)); diff --git a/target/arm/translate-a64.c b/target/arm/translate-a64.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/translate-a64.c +++ b/target/arm/translate-a64.c @@ -XXX,XX +XXX,XX @@ static void disas_data_proc_reg(DisasContext *s, uint32_t insn) } } -static void handle_fp_compare(DisasContext *s, bool is_double, +static void handle_fp_compare(DisasContext *s, int size, unsigned int rn, unsigned int rm, bool cmp_with_zero, bool signal_all_nans) { TCGv_i64 tcg_flags = tcg_temp_new_i64(); - TCGv_ptr fpst = get_fpstatus_ptr(false); + TCGv_ptr fpst = get_fpstatus_ptr(size == MO_16); - if (is_double) { + if (size == MO_64) { TCGv_i64 tcg_vn, tcg_vm; tcg_vn = read_fp_dreg(s, rn); @@ -XXX,XX +XXX,XX @@ static void handle_fp_compare(DisasContext *s, bool is_double, tcg_temp_free_i64(tcg_vn); tcg_temp_free_i64(tcg_vm); } else { - TCGv_i32 tcg_vn, tcg_vm; + TCGv_i32 tcg_vn = tcg_temp_new_i32(); + TCGv_i32 tcg_vm = tcg_temp_new_i32(); - tcg_vn = read_fp_sreg(s, rn); + read_vec_element_i32(s, tcg_vn, rn, 0, size); if (cmp_with_zero) { - tcg_vm = tcg_const_i32(0); + tcg_gen_movi_i32(tcg_vm, 0); } else { - tcg_vm = read_fp_sreg(s, rm); + read_vec_element_i32(s, tcg_vm, rm, 0, size); } - if (signal_all_nans) { - gen_helper_vfp_cmpes_a64(tcg_flags, tcg_vn, tcg_vm, fpst); - } else { - gen_helper_vfp_cmps_a64(tcg_flags, tcg_vn, tcg_vm, fpst); + + switch (size) { + case MO_32: + if (signal_all_nans) { + gen_helper_vfp_cmpes_a64(tcg_flags, tcg_vn, tcg_vm, fpst); + } else { + gen_helper_vfp_cmps_a64(tcg_flags, tcg_vn, tcg_vm, fpst); + } + break; + case MO_16: + if (signal_all_nans) { + gen_helper_vfp_cmpeh_a64(tcg_flags, tcg_vn, tcg_vm, fpst); + } else { + gen_helper_vfp_cmph_a64(tcg_flags, tcg_vn, tcg_vm, fpst); + } + break; + default: + g_assert_not_reached(); } + tcg_temp_free_i32(tcg_vn); tcg_temp_free_i32(tcg_vm); } @@ -XXX,XX +XXX,XX @@ static void handle_fp_compare(DisasContext *s, bool is_double, static void disas_fp_compare(DisasContext *s, uint32_t insn) { unsigned int mos, type, rm, op, rn, opc, op2r; + int size; mos = extract32(insn, 29, 3); - type = extract32(insn, 22, 2); /* 0 = single, 1 = double */ + type = extract32(insn, 22, 2); rm = extract32(insn, 16, 5); op = extract32(insn, 14, 2); rn = extract32(insn, 5, 5); opc = extract32(insn, 3, 2); op2r = extract32(insn, 0, 3); - if (mos || op || op2r || type > 1) { + if (mos || op || op2r) { + unallocated_encoding(s); + return; + } + + switch (type) { + case 0: + size = MO_32; + break; + case 1: + size = MO_64; + break; + case 3: + size = MO_16; + if (arm_dc_feature(s, ARM_FEATURE_V8_FP16)) { + break; + } + /* fallthru */ + default: unallocated_encoding(s); return; } @@ -XXX,XX +XXX,XX @@ static void disas_fp_compare(DisasContext *s, uint32_t insn) return; } - handle_fp_compare(s, type, rn, rm, opc & 1, opc & 2); + handle_fp_compare(s, size, rn, rm, opc & 1, opc & 2); } /* Floating point conditional compare @@ -XXX,XX +XXX,XX @@ static void disas_fp_ccomp(DisasContext *s, uint32_t insn) unsigned int mos, type, rm, cond, rn, op, nzcv; TCGv_i64 tcg_flags; TCGLabel *label_continue = NULL; + int size; mos = extract32(insn, 29, 3); - type = extract32(insn, 22, 2); /* 0 = single, 1 = double */ + type = extract32(insn, 22, 2); rm = extract32(insn, 16, 5); cond = extract32(insn, 12, 4); rn = extract32(insn, 5, 5); op = extract32(insn, 4, 1); nzcv = extract32(insn, 0, 4); - if (mos || type > 1) { + if (mos) { + unallocated_encoding(s); + return; + } + + switch (type) { + case 0: + size = MO_32; + break; + case 1: + size = MO_64; + break; + case 3: + size = MO_16; + if (arm_dc_feature(s, ARM_FEATURE_V8_FP16)) { + break; + } + /* fallthru */ + default: unallocated_encoding(s); return; } @@ -XXX,XX +XXX,XX @@ static void disas_fp_ccomp(DisasContext *s, uint32_t insn) gen_set_label(label_match); } - handle_fp_compare(s, type, rn, rm, false, op); + handle_fp_compare(s, size, rn, rm, false, op); if (cond < 0x0e) { gen_set_label(label_continue); -- 2.17.0 From: Alex Bennée <alex.bennee@linaro.org> These were missed out from the rest of the half-precision work. Cc: qemu-stable@nongnu.org Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Signed-off-by: Alex Bennée <alex.bennee@linaro.org> Tested-by: Alex Bennée <alex.bennee@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Message-id: 20180512003217.9105-10-richard.henderson@linaro.org [rth: Fix erroneous check vs type] Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- target/arm/translate-a64.c | 31 +++++++++++++++++++++++++------ 1 file changed, 25 insertions(+), 6 deletions(-) diff --git a/target/arm/translate-a64.c b/target/arm/translate-a64.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/translate-a64.c +++ b/target/arm/translate-a64.c @@ -XXX,XX +XXX,XX @@ static void disas_fp_csel(DisasContext *s, uint32_t insn) unsigned int mos, type, rm, cond, rn, rd; TCGv_i64 t_true, t_false, t_zero; DisasCompare64 c; + TCGMemOp sz; mos = extract32(insn, 29, 3); - type = extract32(insn, 22, 2); /* 0 = single, 1 = double */ + type = extract32(insn, 22, 2); rm = extract32(insn, 16, 5); cond = extract32(insn, 12, 4); rn = extract32(insn, 5, 5); rd = extract32(insn, 0, 5); - if (mos || type > 1) { + if (mos) { + unallocated_encoding(s); + return; + } + + switch (type) { + case 0: + sz = MO_32; + break; + case 1: + sz = MO_64; + break; + case 3: + sz = MO_16; + if (arm_dc_feature(s, ARM_FEATURE_V8_FP16)) { + break; + } + /* fallthru */ + default: unallocated_encoding(s); return; } @@ -XXX,XX +XXX,XX @@ static void disas_fp_csel(DisasContext *s, uint32_t insn) return; } - /* Zero extend sreg inputs to 64 bits now. */ + /* Zero extend sreg & hreg inputs to 64 bits now. */ t_true = tcg_temp_new_i64(); t_false = tcg_temp_new_i64(); - read_vec_element(s, t_true, rn, 0, type ? MO_64 : MO_32); - read_vec_element(s, t_false, rm, 0, type ? MO_64 : MO_32); + read_vec_element(s, t_true, rn, 0, sz); + read_vec_element(s, t_false, rm, 0, sz); a64_test_cc(&c, cond); t_zero = tcg_const_i64(0); @@ -XXX,XX +XXX,XX @@ static void disas_fp_csel(DisasContext *s, uint32_t insn) tcg_temp_free_i64(t_false); a64_free_cc(&c); - /* Note that sregs write back zeros to the high bits, + /* Note that sregs & hregs write back zeros to the high bits, and we've already done the zero-extension. */ write_fp_dreg(s, rd, t_true); tcg_temp_free_i64(t_true); -- 2.17.0 From: Alex Bennée <alex.bennee@linaro.org> All the hard work is already done by vfp_expand_imm, we just need to make sure we pick up the correct size. Cc: qemu-stable@nongnu.org Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Signed-off-by: Alex Bennée <alex.bennee@linaro.org> Tested-by: Alex Bennée <alex.bennee@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Message-id: 20180512003217.9105-11-richard.henderson@linaro.org [rth: Merge unallocated_encoding check with TCGMemOp conversion.] Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- target/arm/translate-a64.c | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/target/arm/translate-a64.c b/target/arm/translate-a64.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/translate-a64.c +++ b/target/arm/translate-a64.c @@ -XXX,XX +XXX,XX @@ static void disas_fp_imm(DisasContext *s, uint32_t insn) { int rd = extract32(insn, 0, 5); int imm8 = extract32(insn, 13, 8); - int is_double = extract32(insn, 22, 2); + int type = extract32(insn, 22, 2); uint64_t imm; TCGv_i64 tcg_res; + TCGMemOp sz; - if (is_double > 1) { + switch (type) { + case 0: + sz = MO_32; + break; + case 1: + sz = MO_64; + break; + case 3: + sz = MO_16; + if (arm_dc_feature(s, ARM_FEATURE_V8_FP16)) { + break; + } + /* fallthru */ + default: unallocated_encoding(s); return; } @@ -XXX,XX +XXX,XX @@ static void disas_fp_imm(DisasContext *s, uint32_t insn) return; } - imm = vfp_expand_imm(MO_32 + is_double, imm8); + imm = vfp_expand_imm(sz, imm8); tcg_res = tcg_const_i64(imm); write_fp_dreg(s, rd, tcg_res); -- 2.17.0 From: Alex Bennée <alex.bennee@linaro.org> We are meant to explicitly pass fpst, not cpu_env. Cc: qemu-stable@nongnu.org Signed-off-by: Alex Bennée <alex.bennee@linaro.org> Reviewed-by: Richard Henderson <richard.henderson@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Tested-by: Alex Bennée <alex.bennee@linaro.org> Message-id: 20180512003217.9105-12-richard.henderson@linaro.org Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- target/arm/translate-a64.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/target/arm/translate-a64.c b/target/arm/translate-a64.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/translate-a64.c +++ b/target/arm/translate-a64.c @@ -XXX,XX +XXX,XX @@ static void handle_fp_1src_half(DisasContext *s, int opcode, int rd, int rn) tcg_gen_xori_i32(tcg_res, tcg_op, 0x8000); break; case 0x3: /* FSQRT */ - gen_helper_sqrt_f16(tcg_res, tcg_op, cpu_env); + fpst = get_fpstatus_ptr(true); + gen_helper_sqrt_f16(tcg_res, tcg_op, fpst); break; case 0x8: /* FRINTN */ case 0x9: /* FRINTP */ -- 2.17.0 From: Philippe Mathieu-Daudé <f4bug@amsat.org> Per the Physical Layer Simplified Spec. "4.3.10.4 Switch Function Status": The block length is predefined to 512 bits and "4.10.2 SD Status": The SD Status contains status bits that are related to the SD Memory Card proprietary features and may be used for future application-specific usage. The size of the SD Status is one data block of 512 bit. The content of this register is transmitted to the Host over the DAT bus along with a 16-bit CRC. Thus the 16-bit CRC goes at offset 64. Signed-off-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Message-id: 20180509060104.4458-3-f4bug@amsat.org Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- hw/sd/sd.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hw/sd/sd.c b/hw/sd/sd.c index XXXXXXX..XXXXXXX 100644 --- a/hw/sd/sd.c +++ b/hw/sd/sd.c @@ -XXX,XX +XXX,XX @@ static void sd_function_switch(SDState *sd, uint32_t arg) sd->data[14 + (i >> 1)] = new_func << ((i * 4) & 4); } memset(&sd->data[17], 0, 47); - stw_be_p(sd->data + 65, sd_crc16(sd->data, 64)); + stw_be_p(sd->data + 64, sd_crc16(sd->data, 64)); } static inline bool sd_wp_addr(SDState *sd, uint64_t addr) -- 2.17.0 Usually the logging of the CPU state produced by -d cpu is sufficient to diagnose problems, but sometimes you want to see the state of the floating point registers as well. We don't want to enable that by default as it adds a lot of extra data to the log; instead, allow it to be optionally enabled via -d fpu. Signed-off-by: Peter Maydell <peter.maydell@linaro.org> Reviewed-by: Richard Henderson <richard.henderson@linaro.org> Message-id: 20180510130024.31678-1-peter.maydell@linaro.org --- include/qemu/log.h | 1 + accel/tcg/cpu-exec.c | 9 ++++++--- util/log.c | 2 ++ 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/include/qemu/log.h b/include/qemu/log.h index XXXXXXX..XXXXXXX 100644 --- a/include/qemu/log.h +++ b/include/qemu/log.h @@ -XXX,XX +XXX,XX @@ static inline bool qemu_log_separate(void) #define CPU_LOG_PAGE (1 << 14) /* LOG_TRACE (1 << 15) is defined in log-for-trace.h */ #define CPU_LOG_TB_OP_IND (1 << 16) +#define CPU_LOG_TB_FPU (1 << 17) /* Lock output for a series of related logs. Since this is not needed * for a single qemu_log / qemu_log_mask / qemu_log_mask_and_addr, we diff --git a/accel/tcg/cpu-exec.c b/accel/tcg/cpu-exec.c index XXXXXXX..XXXXXXX 100644 --- a/accel/tcg/cpu-exec.c +++ b/accel/tcg/cpu-exec.c @@ -XXX,XX +XXX,XX @@ static inline tcg_target_ulong cpu_tb_exec(CPUState *cpu, TranslationBlock *itb) if (qemu_loglevel_mask(CPU_LOG_TB_CPU) && qemu_log_in_addr_range(itb->pc)) { qemu_log_lock(); + int flags = 0; + if (qemu_loglevel_mask(CPU_LOG_TB_FPU)) { + flags |= CPU_DUMP_FPU; + } #if defined(TARGET_I386) - log_cpu_state(cpu, CPU_DUMP_CCOP); -#else - log_cpu_state(cpu, 0); + flags |= CPU_DUMP_CCOP; #endif + log_cpu_state(cpu, flags); qemu_log_unlock(); } #endif /* DEBUG_DISAS */ diff --git a/util/log.c b/util/log.c index XXXXXXX..XXXXXXX 100644 --- a/util/log.c +++ b/util/log.c @@ -XXX,XX +XXX,XX @@ const QEMULogItem qemu_log_items[] = { "show trace before each executed TB (lots of logs)" }, { CPU_LOG_TB_CPU, "cpu", "show CPU registers before entering a TB (lots of logs)" }, + { CPU_LOG_TB_FPU, "fpu", + "include FPU registers in the 'cpu' logging" }, { CPU_LOG_MMU, "mmu", "log MMU-related activities" }, { CPU_LOG_PCALL, "pcall", -- 2.17.0 Nothing earth-shaking in here, just a lot of refactoring and cleanup and a few bugfixes. I suspect I'll have another pullreq to come in the early part of next week... The following changes since commit 19591e9e0938ea5066984553c256a043bd5d822f: Merge remote-tracking branch 'remotes/mst/tags/for_upstream' into staging (2020-08-27 16:59:02 +0100) are available in the Git repository at: https://git.linaro.org/people/pmaydell/qemu-arm.git tags/pull-target-arm-20200828 for you to fetch changes up to ed78849d9711805bda37ee026018d6ee7a606d0e: target/arm: Convert sq{, r}dmulh to gvec for aa64 advsimd (2020-08-28 10:02:50 +0100) ---------------------------------------------------------------- target-arm queue: * target/arm: Cleanup and refactoring preparatory to SVE2 * armsse: Define ARMSSEClass correctly * hw/misc/unimp: Improve information provided in log messages * hw/qdev-clock: Avoid calling qdev_connect_clock_in after DeviceRealize * hw/arm/xilinx_zynq: Call qdev_connect_clock_in() before DeviceRealize * hw/net/allwinner-sun8i-emac: Use AddressSpace for DMA transfers * hw/sd/allwinner-sdhost: Use AddressSpace for DMA transfers * target/arm: Fill in the WnR syndrome bit in mte_check_fail * target/arm: Clarify HCR_EL2 ARMCPRegInfo type * hw/arm/musicpal: Use AddressSpace for DMA transfers * hw/clock: Minor cleanups * hw/arm/sbsa-ref: fix typo breaking PCIe IRQs ---------------------------------------------------------------- Eduardo Habkost (1): armsse: Define ARMSSEClass correctly Graeme Gregory (1): hw/arm/sbsa-ref: fix typo breaking PCIe IRQs Philippe Mathieu-Daudé (14): hw/clock: Remove unused clock_init*() functions hw/clock: Let clock_set() return boolean value hw/clock: Only propagate clock changes if the clock is changed hw/arm/musicpal: Use AddressSpace for DMA transfers target/arm: Clarify HCR_EL2 ARMCPRegInfo type hw/sd/allwinner-sdhost: Use AddressSpace for DMA transfers hw/net/allwinner-sun8i-emac: Use AddressSpace for DMA transfers hw/arm/xilinx_zynq: Uninline cadence_uart_create() hw/arm/xilinx_zynq: Call qdev_connect_clock_in() before DeviceRealize hw/qdev-clock: Uninline qdev_connect_clock_in() hw/qdev-clock: Avoid calling qdev_connect_clock_in after DeviceRealize hw/misc/unimp: Display value after offset hw/misc/unimp: Display the value with width of the access size hw/misc/unimp: Display the offset with width of the region size Richard Henderson (19): target/arm: Pass the entire mte descriptor to mte_check_fail target/arm: Fill in the WnR syndrome bit in mte_check_fail qemu/int128: Add int128_lshift target/arm: Split out gen_gvec_fn_zz target/arm: Split out gen_gvec_fn_zzz, do_zzz_fn target/arm: Rearrange {sve,fp}_check_access assert target/arm: Merge do_vector2_p into do_mov_p target/arm: Clean up 4-operand predicate expansion target/arm: Use tcg_gen_gvec_bitsel for trans_SEL_pppp target/arm: Split out gen_gvec_ool_zzzp target/arm: Merge helper_sve_clr_* and helper_sve_movz_* target/arm: Split out gen_gvec_ool_zzp target/arm: Split out gen_gvec_ool_zzz target/arm: Split out gen_gvec_ool_zz target/arm: Tidy SVE tszimm shift formats target/arm: Generalize inl_qrdmlah_* helper functions target/arm: Convert integer multiply (indexed) to gvec for aa64 advsimd target/arm: Convert integer multiply-add (indexed) to gvec for aa64 advsimd target/arm: Convert sq{, r}dmulh to gvec for aa64 advsimd include/hw/arm/armsse.h | 2 +- include/hw/char/cadence_uart.h | 17 -- include/hw/clock.h | 30 +-- include/hw/misc/unimp.h | 1 + include/hw/net/allwinner-sun8i-emac.h | 6 + include/hw/qdev-clock.h | 8 +- include/hw/sd/allwinner-sdhost.h | 6 + include/qemu/int128.h | 16 ++ target/arm/helper-sve.h | 5 - target/arm/helper.h | 28 +++ target/arm/translate.h | 1 + target/arm/sve.decode | 35 ++- hw/arm/allwinner-a10.c | 2 + hw/arm/allwinner-h3.c | 4 + hw/arm/armsse.c | 1 + hw/arm/musicpal.c | 45 ++-- hw/arm/sbsa-ref.c | 2 +- hw/arm/xilinx_zynq.c | 24 +- hw/core/clock.c | 7 +- hw/core/qdev-clock.c | 6 + hw/misc/unimp.c | 14 +- hw/net/allwinner-sun8i-emac.c | 46 ++-- hw/sd/allwinner-sdhost.c | 37 +++- target/arm/helper.c | 1 - target/arm/mte_helper.c | 19 +- target/arm/sve_helper.c | 70 ++---- target/arm/translate-a64.c | 110 ++++++++-- target/arm/translate-sve.c | 399 ++++++++++++++-------------------- target/arm/vec_helper.c | 182 +++++++++++----- 29 files changed, 629 insertions(+), 495 deletions(-) From: Graeme Gregory <graeme@nuviainc.com> Fixing a typo in a previous patch that translated an "i" to a 1 and therefore breaking the allocation of PCIe interrupts. This was discovered when virtio-net-pci devices ceased to function correctly. Cc: qemu-stable@nongnu.org Fixes: 48ba18e6d3f3 ("hw/arm/sbsa-ref: Simplify by moving the gic in the machine state") Signed-off-by: Graeme Gregory <graeme@nuviainc.com> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Message-id: 20200821083853.356490-1-graeme@nuviainc.com Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- hw/arm/sbsa-ref.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hw/arm/sbsa-ref.c b/hw/arm/sbsa-ref.c index XXXXXXX..XXXXXXX 100644 --- a/hw/arm/sbsa-ref.c +++ b/hw/arm/sbsa-ref.c @@ -XXX,XX +XXX,XX @@ static void create_pcie(SBSAMachineState *sms) for (i = 0; i < GPEX_NUM_IRQS; i++) { sysbus_connect_irq(SYS_BUS_DEVICE(dev), i, - qdev_get_gpio_in(sms->gic, irq + 1)); + qdev_get_gpio_in(sms->gic, irq + i)); gpex_set_irq_num(GPEX_HOST(dev), i, irq + i); } -- 2.20.1 From: Philippe Mathieu-Daudé <f4bug@amsat.org> clock_init*() inlined funtions are simple wrappers around clock_set*() and are not used. Remove them in favor of clock_set*(). Signed-off-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Reviewed-by: Richard Henderson <richard.henderson@linaro.org> Message-id: 20200806123858.30058-2-f4bug@amsat.org Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- include/hw/clock.h | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/include/hw/clock.h b/include/hw/clock.h index XXXXXXX..XXXXXXX 100644 --- a/include/hw/clock.h +++ b/include/hw/clock.h @@ -XXX,XX +XXX,XX @@ static inline bool clock_is_enabled(const Clock *clk) return clock_get(clk) != 0; } -static inline void clock_init(Clock *clk, uint64_t value) -{ - clock_set(clk, value); -} -static inline void clock_init_hz(Clock *clk, uint64_t value) -{ - clock_set_hz(clk, value); -} -static inline void clock_init_ns(Clock *clk, uint64_t value) -{ - clock_set_ns(clk, value); -} - #endif /* QEMU_HW_CLOCK_H */ -- 2.20.1 From: Philippe Mathieu-Daudé <f4bug@amsat.org> Let clock_set() return a boolean value whether the clock has been updated or not. Signed-off-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Reviewed-by: Richard Henderson <richard.henderson@linaro.org> Message-id: 20200806123858.30058-3-f4bug@amsat.org Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- include/hw/clock.h | 12 +++++++----- hw/core/clock.c | 7 ++++++- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/include/hw/clock.h b/include/hw/clock.h index XXXXXXX..XXXXXXX 100644 --- a/include/hw/clock.h +++ b/include/hw/clock.h @@ -XXX,XX +XXX,XX @@ void clock_set_source(Clock *clk, Clock *src); * @value: the clock's value, 0 means unclocked * * Set the local cached period value of @clk to @value. + * + * @return: true if the clock is changed. */ -void clock_set(Clock *clk, uint64_t value); +bool clock_set(Clock *clk, uint64_t value); -static inline void clock_set_hz(Clock *clk, unsigned hz) +static inline bool clock_set_hz(Clock *clk, unsigned hz) { - clock_set(clk, CLOCK_PERIOD_FROM_HZ(hz)); + return clock_set(clk, CLOCK_PERIOD_FROM_HZ(hz)); } -static inline void clock_set_ns(Clock *clk, unsigned ns) +static inline bool clock_set_ns(Clock *clk, unsigned ns) { - clock_set(clk, CLOCK_PERIOD_FROM_NS(ns)); + return clock_set(clk, CLOCK_PERIOD_FROM_NS(ns)); } /** diff --git a/hw/core/clock.c b/hw/core/clock.c index XXXXXXX..XXXXXXX 100644 --- a/hw/core/clock.c +++ b/hw/core/clock.c @@ -XXX,XX +XXX,XX @@ void clock_clear_callback(Clock *clk) clock_set_callback(clk, NULL, NULL); } -void clock_set(Clock *clk, uint64_t period) +bool clock_set(Clock *clk, uint64_t period) { + if (clk->period == period) { + return false; + } trace_clock_set(CLOCK_PATH(clk), CLOCK_PERIOD_TO_NS(clk->period), CLOCK_PERIOD_TO_NS(period)); clk->period = period; + + return true; } static void clock_propagate_period(Clock *clk, bool call_callbacks) -- 2.20.1 From: Philippe Mathieu-Daudé <f4bug@amsat.org> Avoid propagating the clock change when the clock does not change. Signed-off-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Reviewed-by: Richard Henderson <richard.henderson@linaro.org> Message-id: 20200806123858.30058-4-f4bug@amsat.org Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- include/hw/clock.h | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/include/hw/clock.h b/include/hw/clock.h index XXXXXXX..XXXXXXX 100644 --- a/include/hw/clock.h +++ b/include/hw/clock.h @@ -XXX,XX +XXX,XX @@ void clock_propagate(Clock *clk); */ static inline void clock_update(Clock *clk, uint64_t value) { - clock_set(clk, value); - clock_propagate(clk); + if (clock_set(clk, value)) { + clock_propagate(clk); + } } static inline void clock_update_hz(Clock *clk, unsigned hz) -- 2.20.1 From: Philippe Mathieu-Daudé <f4bug@amsat.org> Allow the device to execute the DMA transfers in a different AddressSpace. We keep using the system_memory address space, but via the proper dma_memory_access() API. Signed-off-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Reviewed-by: Richard Henderson <richard.henderson@linaro.org> Message-id: 20200814125533.4047-1-f4bug@amsat.org Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- hw/arm/musicpal.c | 45 +++++++++++++++++++++++++++++++-------------- 1 file changed, 31 insertions(+), 14 deletions(-) diff --git a/hw/arm/musicpal.c b/hw/arm/musicpal.c index XXXXXXX..XXXXXXX 100644 --- a/hw/arm/musicpal.c +++ b/hw/arm/musicpal.c @@ -XXX,XX +XXX,XX @@ #include "hw/audio/wm8750.h" #include "sysemu/block-backend.h" #include "sysemu/runstate.h" +#include "sysemu/dma.h" #include "exec/address-spaces.h" #include "ui/pixel_ops.h" #include "qemu/cutils.h" @@ -XXX,XX +XXX,XX @@ typedef struct mv88w8618_eth_state { MemoryRegion iomem; qemu_irq irq; + MemoryRegion *dma_mr; + AddressSpace dma_as; uint32_t smir; uint32_t icr; uint32_t imr; @@ -XXX,XX +XXX,XX @@ typedef struct mv88w8618_eth_state { NICConf conf; } mv88w8618_eth_state; -static void eth_rx_desc_put(uint32_t addr, mv88w8618_rx_desc *desc) +static void eth_rx_desc_put(AddressSpace *dma_as, uint32_t addr, + mv88w8618_rx_desc *desc) { cpu_to_le32s(&desc->cmdstat); cpu_to_le16s(&desc->bytes); cpu_to_le16s(&desc->buffer_size); cpu_to_le32s(&desc->buffer); cpu_to_le32s(&desc->next); - cpu_physical_memory_write(addr, desc, sizeof(*desc)); + dma_memory_write(dma_as, addr, desc, sizeof(*desc)); } -static void eth_rx_desc_get(uint32_t addr, mv88w8618_rx_desc *desc) +static void eth_rx_desc_get(AddressSpace *dma_as, uint32_t addr, + mv88w8618_rx_desc *desc) { - cpu_physical_memory_read(addr, desc, sizeof(*desc)); + dma_memory_read(dma_as, addr, desc, sizeof(*desc)); le32_to_cpus(&desc->cmdstat); le16_to_cpus(&desc->bytes); le16_to_cpus(&desc->buffer_size); @@ -XXX,XX +XXX,XX @@ static ssize_t eth_receive(NetClientState *nc, const uint8_t *buf, size_t size) continue; } do { - eth_rx_desc_get(desc_addr, &desc); + eth_rx_desc_get(&s->dma_as, desc_addr, &desc); if ((desc.cmdstat & MP_ETH_RX_OWN) && desc.buffer_size >= size) { - cpu_physical_memory_write(desc.buffer + s->vlan_header, + dma_memory_write(&s->dma_as, desc.buffer + s->vlan_header, buf, size); desc.bytes = size + s->vlan_header; desc.cmdstat &= ~MP_ETH_RX_OWN; @@ -XXX,XX +XXX,XX @@ static ssize_t eth_receive(NetClientState *nc, const uint8_t *buf, size_t size) if (s->icr & s->imr) { qemu_irq_raise(s->irq); } - eth_rx_desc_put(desc_addr, &desc); + eth_rx_desc_put(&s->dma_as, desc_addr, &desc); return size; } desc_addr = desc.next; @@ -XXX,XX +XXX,XX @@ static ssize_t eth_receive(NetClientState *nc, const uint8_t *buf, size_t size) return size; } -static void eth_tx_desc_put(uint32_t addr, mv88w8618_tx_desc *desc) +static void eth_tx_desc_put(AddressSpace *dma_as, uint32_t addr, + mv88w8618_tx_desc *desc) { cpu_to_le32s(&desc->cmdstat); cpu_to_le16s(&desc->res); cpu_to_le16s(&desc->bytes); cpu_to_le32s(&desc->buffer); cpu_to_le32s(&desc->next); - cpu_physical_memory_write(addr, desc, sizeof(*desc)); + dma_memory_write(dma_as, addr, desc, sizeof(*desc)); } -static void eth_tx_desc_get(uint32_t addr, mv88w8618_tx_desc *desc) +static void eth_tx_desc_get(AddressSpace *dma_as, uint32_t addr, + mv88w8618_tx_desc *desc) { - cpu_physical_memory_read(addr, desc, sizeof(*desc)); + dma_memory_read(dma_as, addr, desc, sizeof(*desc)); le32_to_cpus(&desc->cmdstat); le16_to_cpus(&desc->res); le16_to_cpus(&desc->bytes); @@ -XXX,XX +XXX,XX @@ static void eth_send(mv88w8618_eth_state *s, int queue_index) int len; do { - eth_tx_desc_get(desc_addr, &desc); + eth_tx_desc_get(&s->dma_as, desc_addr, &desc); next_desc = desc.next; if (desc.cmdstat & MP_ETH_TX_OWN) { len = desc.bytes; if (len < 2048) { - cpu_physical_memory_read(desc.buffer, buf, len); + dma_memory_read(&s->dma_as, desc.buffer, buf, len); qemu_send_packet(qemu_get_queue(s->nic), buf, len); } desc.cmdstat &= ~MP_ETH_TX_OWN; s->icr |= 1 << (MP_ETH_IRQ_TXLO_BIT - queue_index); - eth_tx_desc_put(desc_addr, &desc); + eth_tx_desc_put(&s->dma_as, desc_addr, &desc); } desc_addr = next_desc; } while (desc_addr != s->tx_queue[queue_index]); @@ -XXX,XX +XXX,XX @@ static void mv88w8618_eth_realize(DeviceState *dev, Error **errp) { mv88w8618_eth_state *s = MV88W8618_ETH(dev); + if (!s->dma_mr) { + error_setg(errp, TYPE_MV88W8618_ETH " 'dma-memory' link not set"); + return; + } + + address_space_init(&s->dma_as, s->dma_mr, "emac-dma"); s->nic = qemu_new_nic(&net_mv88w8618_info, &s->conf, object_get_typename(OBJECT(dev)), dev->id, s); } @@ -XXX,XX +XXX,XX @@ static const VMStateDescription mv88w8618_eth_vmsd = { static Property mv88w8618_eth_properties[] = { DEFINE_NIC_PROPERTIES(mv88w8618_eth_state, conf), + DEFINE_PROP_LINK("dma-memory", mv88w8618_eth_state, dma_mr, + TYPE_MEMORY_REGION, MemoryRegion *), DEFINE_PROP_END_OF_LIST(), }; @@ -XXX,XX +XXX,XX @@ static void musicpal_init(MachineState *machine) qemu_check_nic_model(&nd_table[0], "mv88w8618"); dev = qdev_new(TYPE_MV88W8618_ETH); qdev_set_nic_properties(dev, &nd_table[0]); + object_property_set_link(OBJECT(dev), "dma-memory", + OBJECT(get_system_memory()), &error_fatal); sysbus_realize_and_unref(SYS_BUS_DEVICE(dev), &error_fatal); sysbus_mmio_map(SYS_BUS_DEVICE(dev), 0, MP_ETH_BASE); sysbus_connect_irq(SYS_BUS_DEVICE(dev), 0, pic[MP_ETH_IRQ]); -- 2.20.1 From: Philippe Mathieu-Daudé <f4bug@amsat.org> In commit ce4afed839 ("target/arm: Implement AArch32 HCR and HCR2") the HCR_EL2 register has been changed from type NO_RAW (no underlying state and does not support raw access for state saving/loading) to type CONST (TCG can assume the value to be constant), removing the read/write accessors. We forgot to remove the previous type ARM_CP_NO_RAW. This is not really a problem since the field is overwritten. However it makes code review confuse, so remove it. Signed-off-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Reviewed-by: Edgar E. Iglesias <edgar.iglesias@xilinx.com> Reviewed-by: Richard Henderson <richard.henderson@linaro.org> Message-id: 20200812111223.7787-1-f4bug@amsat.org Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- target/arm/helper.c | 1 - 1 file changed, 1 deletion(-) diff --git a/target/arm/helper.c b/target/arm/helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/helper.c +++ b/target/arm/helper.c @@ -XXX,XX +XXX,XX @@ static const ARMCPRegInfo el3_no_el2_cp_reginfo[] = { .access = PL2_RW, .readfn = arm_cp_read_zero, .writefn = arm_cp_write_ignore }, { .name = "HCR_EL2", .state = ARM_CP_STATE_BOTH, - .type = ARM_CP_NO_RAW, .opc0 = 3, .opc1 = 4, .crn = 1, .crm = 1, .opc2 = 0, .access = PL2_RW, .type = ARM_CP_CONST, .resetvalue = 0 }, -- 2.20.1 From: Richard Henderson <richard.henderson@linaro.org> We need more information than just the mmu_idx in order to create the proper exception syndrome. Only change the function signature so far. Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Message-id: 20200813200816.3037186-2-richard.henderson@linaro.org Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- target/arm/mte_helper.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/target/arm/mte_helper.c b/target/arm/mte_helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/mte_helper.c +++ b/target/arm/mte_helper.c @@ -XXX,XX +XXX,XX @@ void HELPER(stzgm_tags)(CPUARMState *env, uint64_t ptr, uint64_t val) } /* Record a tag check failure. */ -static void mte_check_fail(CPUARMState *env, int mmu_idx, +static void mte_check_fail(CPUARMState *env, uint32_t desc, uint64_t dirty_ptr, uintptr_t ra) { + int mmu_idx = FIELD_EX32(desc, MTEDESC, MIDX); ARMMMUIdx arm_mmu_idx = core_to_aa64_mmu_idx(mmu_idx); int el, reg_el, tcf, select; uint64_t sctlr; @@ -XXX,XX +XXX,XX @@ uint64_t mte_check1(CPUARMState *env, uint32_t desc, } if (unlikely(!mte_probe1_int(env, desc, ptr, ra, bit55))) { - int mmu_idx = FIELD_EX32(desc, MTEDESC, MIDX); - mte_check_fail(env, mmu_idx, ptr, ra); + mte_check_fail(env, desc, ptr, ra); } return useronly_clean_ptr(ptr); @@ -XXX,XX +XXX,XX @@ uint64_t mte_checkN(CPUARMState *env, uint32_t desc, fail_ofs = tag_first + n * TAG_GRANULE - ptr; fail_ofs = ROUND_UP(fail_ofs, esize); - mte_check_fail(env, mmu_idx, ptr + fail_ofs, ra); + mte_check_fail(env, desc, ptr + fail_ofs, ra); } done: @@ -XXX,XX +XXX,XX @@ uint64_t HELPER(mte_check_zva)(CPUARMState *env, uint32_t desc, uint64_t ptr) fail: /* Locate the first nibble that differs. */ i = ctz64(mem_tag ^ ptr_tag) >> 4; - mte_check_fail(env, mmu_idx, align_ptr + i * TAG_GRANULE, ra); + mte_check_fail(env, desc, align_ptr + i * TAG_GRANULE, ra); done: return useronly_clean_ptr(ptr); -- 2.20.1 From: Richard Henderson <richard.henderson@linaro.org> According to AArch64.TagCheckFault, none of the other ISS values are provided, so we do not need to go so far as merge_syn_data_abort. But we were missing the WnR bit. Tested-by: Andrey Konovalov <andreyknvl@google.com> Reported-by: Andrey Konovalov <andreyknvl@google.com> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Message-id: 20200813200816.3037186-3-richard.henderson@linaro.org Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- target/arm/mte_helper.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/target/arm/mte_helper.c b/target/arm/mte_helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/mte_helper.c +++ b/target/arm/mte_helper.c @@ -XXX,XX +XXX,XX @@ static void mte_check_fail(CPUARMState *env, uint32_t desc, { int mmu_idx = FIELD_EX32(desc, MTEDESC, MIDX); ARMMMUIdx arm_mmu_idx = core_to_aa64_mmu_idx(mmu_idx); - int el, reg_el, tcf, select; + int el, reg_el, tcf, select, is_write, syn; uint64_t sctlr; reg_el = regime_el(env, arm_mmu_idx); @@ -XXX,XX +XXX,XX @@ static void mte_check_fail(CPUARMState *env, uint32_t desc, */ cpu_restore_state(env_cpu(env), ra, true); env->exception.vaddress = dirty_ptr; - raise_exception(env, EXCP_DATA_ABORT, - syn_data_abort_no_iss(el != 0, 0, 0, 0, 0, 0, 0x11), - exception_target_el(env)); + + is_write = FIELD_EX32(desc, MTEDESC, WRITE); + syn = syn_data_abort_no_iss(el != 0, 0, 0, 0, 0, is_write, 0x11); + raise_exception(env, EXCP_DATA_ABORT, syn, exception_target_el(env)); /* noreturn, but fall through to the assert anyway */ case 0: -- 2.20.1 From: Philippe Mathieu-Daudé <f4bug@amsat.org> Allow the device to execute the DMA transfers in a different AddressSpace. The A10 and H3 SoC keep using the system_memory address space, but via the proper dma_memory_access() API. Signed-off-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Tested-by: Niek Linnenbank <nieklinnenbank@gmail.com> Reviewed-by: Niek Linnenbank <nieklinnenbank@gmail.com> Message-id: 20200814110057.307-1-f4bug@amsat.org Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- include/hw/sd/allwinner-sdhost.h | 6 ++++++ hw/arm/allwinner-a10.c | 2 ++ hw/arm/allwinner-h3.c | 2 ++ hw/sd/allwinner-sdhost.c | 37 ++++++++++++++++++++++++++------ 4 files changed, 41 insertions(+), 6 deletions(-) diff --git a/include/hw/sd/allwinner-sdhost.h b/include/hw/sd/allwinner-sdhost.h index XXXXXXX..XXXXXXX 100644 --- a/include/hw/sd/allwinner-sdhost.h +++ b/include/hw/sd/allwinner-sdhost.h @@ -XXX,XX +XXX,XX @@ typedef struct AwSdHostState { /** Interrupt output signal to notify CPU */ qemu_irq irq; + /** Memory region where DMA transfers are done */ + MemoryRegion *dma_mr; + + /** Address space used internally for DMA transfers */ + AddressSpace dma_as; + /** Number of bytes left in current DMA transfer */ uint32_t transfer_cnt; diff --git a/hw/arm/allwinner-a10.c b/hw/arm/allwinner-a10.c index XXXXXXX..XXXXXXX 100644 --- a/hw/arm/allwinner-a10.c +++ b/hw/arm/allwinner-a10.c @@ -XXX,XX +XXX,XX @@ static void aw_a10_realize(DeviceState *dev, Error **errp) } /* SD/MMC */ + object_property_set_link(OBJECT(&s->mmc0), "dma-memory", + OBJECT(get_system_memory()), &error_fatal); sysbus_realize(SYS_BUS_DEVICE(&s->mmc0), &error_fatal); sysbus_mmio_map(SYS_BUS_DEVICE(&s->mmc0), 0, AW_A10_MMC0_BASE); sysbus_connect_irq(SYS_BUS_DEVICE(&s->mmc0), 0, qdev_get_gpio_in(dev, 32)); diff --git a/hw/arm/allwinner-h3.c b/hw/arm/allwinner-h3.c index XXXXXXX..XXXXXXX 100644 --- a/hw/arm/allwinner-h3.c +++ b/hw/arm/allwinner-h3.c @@ -XXX,XX +XXX,XX @@ static void allwinner_h3_realize(DeviceState *dev, Error **errp) sysbus_mmio_map(SYS_BUS_DEVICE(&s->sid), 0, s->memmap[AW_H3_SID]); /* SD/MMC */ + object_property_set_link(OBJECT(&s->mmc0), "dma-memory", + OBJECT(get_system_memory()), &error_fatal); sysbus_realize(SYS_BUS_DEVICE(&s->mmc0), &error_fatal); sysbus_mmio_map(SYS_BUS_DEVICE(&s->mmc0), 0, s->memmap[AW_H3_MMC0]); sysbus_connect_irq(SYS_BUS_DEVICE(&s->mmc0), 0, diff --git a/hw/sd/allwinner-sdhost.c b/hw/sd/allwinner-sdhost.c index XXXXXXX..XXXXXXX 100644 --- a/hw/sd/allwinner-sdhost.c +++ b/hw/sd/allwinner-sdhost.c @@ -XXX,XX +XXX,XX @@ #include "qemu/log.h" #include "qemu/module.h" #include "qemu/units.h" +#include "qapi/error.h" #include "sysemu/blockdev.h" +#include "sysemu/dma.h" +#include "hw/qdev-properties.h" #include "hw/irq.h" #include "hw/sd/allwinner-sdhost.h" #include "migration/vmstate.h" @@ -XXX,XX +XXX,XX @@ static uint32_t allwinner_sdhost_process_desc(AwSdHostState *s, uint8_t buf[1024]; /* Read descriptor */ - cpu_physical_memory_read(desc_addr, desc, sizeof(*desc)); + dma_memory_read(&s->dma_as, desc_addr, desc, sizeof(*desc)); if (desc->size == 0) { desc->size = klass->max_desc_size; } else if (desc->size > klass->max_desc_size) { @@ -XXX,XX +XXX,XX @@ static uint32_t allwinner_sdhost_process_desc(AwSdHostState *s, /* Write to SD bus */ if (is_write) { - cpu_physical_memory_read((desc->addr & DESC_SIZE_MASK) + num_done, - buf, buf_bytes); + dma_memory_read(&s->dma_as, + (desc->addr & DESC_SIZE_MASK) + num_done, + buf, buf_bytes); sdbus_write_data(&s->sdbus, buf, buf_bytes); /* Read from SD bus */ } else { sdbus_read_data(&s->sdbus, buf, buf_bytes); - cpu_physical_memory_write((desc->addr & DESC_SIZE_MASK) + num_done, - buf, buf_bytes); + dma_memory_write(&s->dma_as, + (desc->addr & DESC_SIZE_MASK) + num_done, + buf, buf_bytes); } num_done += buf_bytes; } /* Clear hold flag and flush descriptor */ desc->status &= ~DESC_STATUS_HOLD; - cpu_physical_memory_write(desc_addr, desc, sizeof(*desc)); + dma_memory_write(&s->dma_as, desc_addr, desc, sizeof(*desc)); return num_done; } @@ -XXX,XX +XXX,XX @@ static const VMStateDescription vmstate_allwinner_sdhost = { } }; +static Property allwinner_sdhost_properties[] = { + DEFINE_PROP_LINK("dma-memory", AwSdHostState, dma_mr, + TYPE_MEMORY_REGION, MemoryRegion *), + DEFINE_PROP_END_OF_LIST(), +}; + static void allwinner_sdhost_init(Object *obj) { AwSdHostState *s = AW_SDHOST(obj); @@ -XXX,XX +XXX,XX @@ static void allwinner_sdhost_init(Object *obj) sysbus_init_irq(SYS_BUS_DEVICE(s), &s->irq); } +static void allwinner_sdhost_realize(DeviceState *dev, Error **errp) +{ + AwSdHostState *s = AW_SDHOST(dev); + + if (!s->dma_mr) { + error_setg(errp, TYPE_AW_SDHOST " 'dma-memory' link not set"); + return; + } + + address_space_init(&s->dma_as, s->dma_mr, "sdhost-dma"); +} + static void allwinner_sdhost_reset(DeviceState *dev) { AwSdHostState *s = AW_SDHOST(dev); @@ -XXX,XX +XXX,XX @@ static void allwinner_sdhost_class_init(ObjectClass *klass, void *data) dc->reset = allwinner_sdhost_reset; dc->vmsd = &vmstate_allwinner_sdhost; + dc->realize = allwinner_sdhost_realize; + device_class_set_props(dc, allwinner_sdhost_properties); } static void allwinner_sdhost_sun4i_class_init(ObjectClass *klass, void *data) -- 2.20.1 From: Philippe Mathieu-Daudé <f4bug@amsat.org> Allow the device to execute the DMA transfers in a different AddressSpace. The H3 SoC keeps using the system_memory address space, but via the proper dma_memory_access() API. Signed-off-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Reviewed-by: Richard Henderson <richard.henderson@linaro.org> Reviewed-by: Niek Linnenbank <nieklinnenbank@gmail.com> Tested-by: Niek Linnenbank <nieklinnenbank@gmail.com> Message-id: 20200814122907.27732-1-f4bug@amsat.org Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- include/hw/net/allwinner-sun8i-emac.h | 6 ++++ hw/arm/allwinner-h3.c | 2 ++ hw/net/allwinner-sun8i-emac.c | 46 +++++++++++++++++---------- 3 files changed, 38 insertions(+), 16 deletions(-) diff --git a/include/hw/net/allwinner-sun8i-emac.h b/include/hw/net/allwinner-sun8i-emac.h index XXXXXXX..XXXXXXX 100644 --- a/include/hw/net/allwinner-sun8i-emac.h +++ b/include/hw/net/allwinner-sun8i-emac.h @@ -XXX,XX +XXX,XX @@ typedef struct AwSun8iEmacState { /** Interrupt output signal to notify CPU */ qemu_irq irq; + /** Memory region where DMA transfers are done */ + MemoryRegion *dma_mr; + + /** Address space used internally for DMA transfers */ + AddressSpace dma_as; + /** Generic Network Interface Controller (NIC) for networking API */ NICState *nic; diff --git a/hw/arm/allwinner-h3.c b/hw/arm/allwinner-h3.c index XXXXXXX..XXXXXXX 100644 --- a/hw/arm/allwinner-h3.c +++ b/hw/arm/allwinner-h3.c @@ -XXX,XX +XXX,XX @@ static void allwinner_h3_realize(DeviceState *dev, Error **errp) qemu_check_nic_model(&nd_table[0], TYPE_AW_SUN8I_EMAC); qdev_set_nic_properties(DEVICE(&s->emac), &nd_table[0]); } + object_property_set_link(OBJECT(&s->emac), "dma-memory", + OBJECT(get_system_memory()), &error_fatal); sysbus_realize(SYS_BUS_DEVICE(&s->emac), &error_fatal); sysbus_mmio_map(SYS_BUS_DEVICE(&s->emac), 0, s->memmap[AW_H3_EMAC]); sysbus_connect_irq(SYS_BUS_DEVICE(&s->emac), 0, diff --git a/hw/net/allwinner-sun8i-emac.c b/hw/net/allwinner-sun8i-emac.c index XXXXXXX..XXXXXXX 100644 --- a/hw/net/allwinner-sun8i-emac.c +++ b/hw/net/allwinner-sun8i-emac.c @@ -XXX,XX +XXX,XX @@ #include "qemu/osdep.h" #include "qemu/units.h" +#include "qapi/error.h" #include "hw/sysbus.h" #include "migration/vmstate.h" #include "net/net.h" @@ -XXX,XX +XXX,XX @@ #include "net/checksum.h" #include "qemu/module.h" #include "exec/cpu-common.h" +#include "sysemu/dma.h" #include "hw/net/allwinner-sun8i-emac.h" /* EMAC register offsets */ @@ -XXX,XX +XXX,XX @@ static void allwinner_sun8i_emac_update_irq(AwSun8iEmacState *s) qemu_set_irq(s->irq, (s->int_sta & s->int_en) != 0); } -static uint32_t allwinner_sun8i_emac_next_desc(FrameDescriptor *desc, +static uint32_t allwinner_sun8i_emac_next_desc(AwSun8iEmacState *s, + FrameDescriptor *desc, size_t min_size) { uint32_t paddr = desc->next; - cpu_physical_memory_read(paddr, desc, sizeof(*desc)); + dma_memory_read(&s->dma_as, paddr, desc, sizeof(*desc)); if ((desc->status & DESC_STATUS_CTL) && (desc->status2 & DESC_STATUS2_BUF_SIZE_MASK) >= min_size) { @@ -XXX,XX +XXX,XX @@ static uint32_t allwinner_sun8i_emac_next_desc(FrameDescriptor *desc, } } -static uint32_t allwinner_sun8i_emac_get_desc(FrameDescriptor *desc, +static uint32_t allwinner_sun8i_emac_get_desc(AwSun8iEmacState *s, + FrameDescriptor *desc, uint32_t start_addr, size_t min_size) { @@ -XXX,XX +XXX,XX @@ static uint32_t allwinner_sun8i_emac_get_desc(FrameDescriptor *desc, /* Note that the list is a cycle. Last entry points back to the head. */ while (desc_addr != 0) { - cpu_physical_memory_read(desc_addr, desc, sizeof(*desc)); + dma_memory_read(&s->dma_as, desc_addr, desc, sizeof(*desc)); if ((desc->status & DESC_STATUS_CTL) && (desc->status2 & DESC_STATUS2_BUF_SIZE_MASK) >= min_size) { @@ -XXX,XX +XXX,XX @@ static uint32_t allwinner_sun8i_emac_rx_desc(AwSun8iEmacState *s, FrameDescriptor *desc, size_t min_size) { - return allwinner_sun8i_emac_get_desc(desc, s->rx_desc_curr, min_size); + return allwinner_sun8i_emac_get_desc(s, desc, s->rx_desc_curr, min_size); } static uint32_t allwinner_sun8i_emac_tx_desc(AwSun8iEmacState *s, FrameDescriptor *desc, size_t min_size) { - return allwinner_sun8i_emac_get_desc(desc, s->tx_desc_head, min_size); + return allwinner_sun8i_emac_get_desc(s, desc, s->tx_desc_head, min_size); } -static void allwinner_sun8i_emac_flush_desc(FrameDescriptor *desc, +static void allwinner_sun8i_emac_flush_desc(AwSun8iEmacState *s, + FrameDescriptor *desc, uint32_t phys_addr) { - cpu_physical_memory_write(phys_addr, desc, sizeof(*desc)); + dma_memory_write(&s->dma_as, phys_addr, desc, sizeof(*desc)); } static bool allwinner_sun8i_emac_can_receive(NetClientState *nc) @@ -XXX,XX +XXX,XX @@ static ssize_t allwinner_sun8i_emac_receive(NetClientState *nc, << RX_DESC_STATUS_FRM_LEN_SHIFT; } - cpu_physical_memory_write(desc.addr, buf, desc_bytes); - allwinner_sun8i_emac_flush_desc(&desc, s->rx_desc_curr); + dma_memory_write(&s->dma_as, desc.addr, buf, desc_bytes); + allwinner_sun8i_emac_flush_desc(s, &desc, s->rx_desc_curr); trace_allwinner_sun8i_emac_receive(s->rx_desc_curr, desc.addr, desc_bytes); @@ -XXX,XX +XXX,XX @@ static ssize_t allwinner_sun8i_emac_receive(NetClientState *nc, bytes_left -= desc_bytes; /* Move to the next descriptor */ - s->rx_desc_curr = allwinner_sun8i_emac_next_desc(&desc, 64); + s->rx_desc_curr = allwinner_sun8i_emac_next_desc(s, &desc, 64); if (!s->rx_desc_curr) { /* Not enough buffer space available */ s->int_sta |= INT_STA_RX_BUF_UA; @@ -XXX,XX +XXX,XX @@ static void allwinner_sun8i_emac_transmit(AwSun8iEmacState *s) desc.status |= TX_DESC_STATUS_LENGTH_ERR; break; } - cpu_physical_memory_read(desc.addr, packet_buf + packet_bytes, bytes); + dma_memory_read(&s->dma_as, desc.addr, packet_buf + packet_bytes, bytes); packet_bytes += bytes; desc.status &= ~DESC_STATUS_CTL; - allwinner_sun8i_emac_flush_desc(&desc, s->tx_desc_curr); + allwinner_sun8i_emac_flush_desc(s, &desc, s->tx_desc_curr); /* After the last descriptor, send the packet */ if (desc.status2 & TX_DESC_STATUS2_LAST_DESC) { @@ -XXX,XX +XXX,XX @@ static void allwinner_sun8i_emac_transmit(AwSun8iEmacState *s) packet_bytes = 0; transmitted++; } - s->tx_desc_curr = allwinner_sun8i_emac_next_desc(&desc, 0); + s->tx_desc_curr = allwinner_sun8i_emac_next_desc(s, &desc, 0); } /* Raise transmit completed interrupt */ @@ -XXX,XX +XXX,XX @@ static uint64_t allwinner_sun8i_emac_read(void *opaque, hwaddr offset, break; case REG_TX_CUR_BUF: /* Transmit Current Buffer */ if (s->tx_desc_curr != 0) { - cpu_physical_memory_read(s->tx_desc_curr, &desc, sizeof(desc)); + dma_memory_read(&s->dma_as, s->tx_desc_curr, &desc, sizeof(desc)); value = desc.addr; } else { value = 0; @@ -XXX,XX +XXX,XX @@ static uint64_t allwinner_sun8i_emac_read(void *opaque, hwaddr offset, break; case REG_RX_CUR_BUF: /* Receive Current Buffer */ if (s->rx_desc_curr != 0) { - cpu_physical_memory_read(s->rx_desc_curr, &desc, sizeof(desc)); + dma_memory_read(&s->dma_as, s->rx_desc_curr, &desc, sizeof(desc)); value = desc.addr; } else { value = 0; @@ -XXX,XX +XXX,XX @@ static void allwinner_sun8i_emac_realize(DeviceState *dev, Error **errp) { AwSun8iEmacState *s = AW_SUN8I_EMAC(dev); + if (!s->dma_mr) { + error_setg(errp, TYPE_AW_SUN8I_EMAC " 'dma-memory' link not set"); + return; + } + + address_space_init(&s->dma_as, s->dma_mr, "emac-dma"); + qemu_macaddr_default_if_unset(&s->conf.macaddr); s->nic = qemu_new_nic(&net_allwinner_sun8i_emac_info, &s->conf, object_get_typename(OBJECT(dev)), dev->id, s); @@ -XXX,XX +XXX,XX @@ static void allwinner_sun8i_emac_realize(DeviceState *dev, Error **errp) static Property allwinner_sun8i_emac_properties[] = { DEFINE_NIC_PROPERTIES(AwSun8iEmacState, conf), DEFINE_PROP_UINT8("phy-addr", AwSun8iEmacState, mii_phy_addr, 0), + DEFINE_PROP_LINK("dma-memory", AwSun8iEmacState, dma_mr, + TYPE_MEMORY_REGION, MemoryRegion *), DEFINE_PROP_END_OF_LIST(), }; -- 2.20.1 From: Philippe Mathieu-Daudé <f4bug@amsat.org> As we want to call qdev_connect_clock_in() before the device is realized, we need to uninline cadence_uart_create() first. Signed-off-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Reviewed-by: Alistair Francis <alistair.francis@wdc.com> Message-id: 20200803105647.22223-2-f4bug@amsat.org Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- include/hw/char/cadence_uart.h | 17 ----------------- hw/arm/xilinx_zynq.c | 14 ++++++++++++-- 2 files changed, 12 insertions(+), 19 deletions(-) diff --git a/include/hw/char/cadence_uart.h b/include/hw/char/cadence_uart.h index XXXXXXX..XXXXXXX 100644 --- a/include/hw/char/cadence_uart.h +++ b/include/hw/char/cadence_uart.h @@ -XXX,XX +XXX,XX @@ typedef struct { Clock *refclk; } CadenceUARTState; -static inline DeviceState *cadence_uart_create(hwaddr addr, - qemu_irq irq, - Chardev *chr) -{ - DeviceState *dev; - SysBusDevice *s; - - dev = qdev_new(TYPE_CADENCE_UART); - s = SYS_BUS_DEVICE(dev); - qdev_prop_set_chr(dev, "chardev", chr); - sysbus_realize_and_unref(s, &error_fatal); - sysbus_mmio_map(s, 0, addr); - sysbus_connect_irq(s, 0, irq); - - return dev; -} - #endif diff --git a/hw/arm/xilinx_zynq.c b/hw/arm/xilinx_zynq.c index XXXXXXX..XXXXXXX 100644 --- a/hw/arm/xilinx_zynq.c +++ b/hw/arm/xilinx_zynq.c @@ -XXX,XX +XXX,XX @@ static void zynq_init(MachineState *machine) sysbus_create_simple(TYPE_CHIPIDEA, 0xE0002000, pic[53 - IRQ_OFFSET]); sysbus_create_simple(TYPE_CHIPIDEA, 0xE0003000, pic[76 - IRQ_OFFSET]); - dev = cadence_uart_create(0xE0000000, pic[59 - IRQ_OFFSET], serial_hd(0)); + dev = qdev_new(TYPE_CADENCE_UART); + busdev = SYS_BUS_DEVICE(dev); + qdev_prop_set_chr(dev, "chardev", serial_hd(0)); + sysbus_realize_and_unref(busdev, &error_fatal); + sysbus_mmio_map(busdev, 0, 0xE0000000); + sysbus_connect_irq(busdev, 0, pic[59 - IRQ_OFFSET]); qdev_connect_clock_in(dev, "refclk", qdev_get_clock_out(slcr, "uart0_ref_clk")); - dev = cadence_uart_create(0xE0001000, pic[82 - IRQ_OFFSET], serial_hd(1)); + dev = qdev_new(TYPE_CADENCE_UART); + busdev = SYS_BUS_DEVICE(dev); + qdev_prop_set_chr(dev, "chardev", serial_hd(1)); + sysbus_realize_and_unref(busdev, &error_fatal); + sysbus_mmio_map(busdev, 0, 0xE0001000); + sysbus_connect_irq(busdev, 0, pic[82 - IRQ_OFFSET]); qdev_connect_clock_in(dev, "refclk", qdev_get_clock_out(slcr, "uart1_ref_clk")); -- 2.20.1 From: Philippe Mathieu-Daudé <f4bug@amsat.org> Clock canonical name is set in device_set_realized (see the block added to hw/core/qdev.c in commit 0e6934f264). If we connect a clock after the device is realized, this code is not executed. This is currently not a problem as this name is only used for trace events, however this disrupt tracing. Fix by calling qdev_connect_clock_in() before realizing. Signed-off-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Reviewed-by: Alistair Francis <alistair.francis@wdc.com> Message-id: 20200803105647.22223-3-f4bug@amsat.org Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- hw/arm/xilinx_zynq.c | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/hw/arm/xilinx_zynq.c b/hw/arm/xilinx_zynq.c index XXXXXXX..XXXXXXX 100644 --- a/hw/arm/xilinx_zynq.c +++ b/hw/arm/xilinx_zynq.c @@ -XXX,XX +XXX,XX @@ static void zynq_init(MachineState *machine) 1, 0x0066, 0x0022, 0x0000, 0x0000, 0x0555, 0x2aa, 0); - /* Create slcr, keep a pointer to connect clocks */ - slcr = qdev_new("xilinx,zynq_slcr"); - sysbus_realize_and_unref(SYS_BUS_DEVICE(slcr), &error_fatal); - sysbus_mmio_map(SYS_BUS_DEVICE(slcr), 0, 0xF8000000); - /* Create the main clock source, and feed slcr with it */ zynq_machine->ps_clk = CLOCK(object_new(TYPE_CLOCK)); object_property_add_child(OBJECT(zynq_machine), "ps_clk", OBJECT(zynq_machine->ps_clk)); object_unref(OBJECT(zynq_machine->ps_clk)); clock_set_hz(zynq_machine->ps_clk, PS_CLK_FREQUENCY); + + /* Create slcr, keep a pointer to connect clocks */ + slcr = qdev_new("xilinx,zynq_slcr"); qdev_connect_clock_in(slcr, "ps_clk", zynq_machine->ps_clk); + sysbus_realize_and_unref(SYS_BUS_DEVICE(slcr), &error_fatal); + sysbus_mmio_map(SYS_BUS_DEVICE(slcr), 0, 0xF8000000); dev = qdev_new(TYPE_A9MPCORE_PRIV); qdev_prop_set_uint32(dev, "num-cpu", 1); @@ -XXX,XX +XXX,XX @@ static void zynq_init(MachineState *machine) dev = qdev_new(TYPE_CADENCE_UART); busdev = SYS_BUS_DEVICE(dev); qdev_prop_set_chr(dev, "chardev", serial_hd(0)); + qdev_connect_clock_in(dev, "refclk", + qdev_get_clock_out(slcr, "uart0_ref_clk")); sysbus_realize_and_unref(busdev, &error_fatal); sysbus_mmio_map(busdev, 0, 0xE0000000); sysbus_connect_irq(busdev, 0, pic[59 - IRQ_OFFSET]); - qdev_connect_clock_in(dev, "refclk", - qdev_get_clock_out(slcr, "uart0_ref_clk")); dev = qdev_new(TYPE_CADENCE_UART); busdev = SYS_BUS_DEVICE(dev); qdev_prop_set_chr(dev, "chardev", serial_hd(1)); + qdev_connect_clock_in(dev, "refclk", + qdev_get_clock_out(slcr, "uart1_ref_clk")); sysbus_realize_and_unref(busdev, &error_fatal); sysbus_mmio_map(busdev, 0, 0xE0001000); sysbus_connect_irq(busdev, 0, pic[82 - IRQ_OFFSET]); - qdev_connect_clock_in(dev, "refclk", - qdev_get_clock_out(slcr, "uart1_ref_clk")); sysbus_create_varargs("cadence_ttc", 0xF8001000, pic[42-IRQ_OFFSET], pic[43-IRQ_OFFSET], pic[44-IRQ_OFFSET], NULL); -- 2.20.1 From: Philippe Mathieu-Daudé <f4bug@amsat.org> We want to assert the device is not realized. To avoid overloading this header including "hw/qdev-core.h", uninline the function first. Signed-off-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Reviewed-by: Alistair Francis <alistair.francis@wdc.com> Message-id: 20200803105647.22223-4-f4bug@amsat.org Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- include/hw/qdev-clock.h | 6 +----- hw/core/qdev-clock.c | 5 +++++ 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/include/hw/qdev-clock.h b/include/hw/qdev-clock.h index XXXXXXX..XXXXXXX 100644 --- a/include/hw/qdev-clock.h +++ b/include/hw/qdev-clock.h @@ -XXX,XX +XXX,XX @@ Clock *qdev_get_clock_out(DeviceState *dev, const char *name); * Set the source clock of input clock @name of device @dev to @source. * @source period update will be propagated to @name clock. */ -static inline void qdev_connect_clock_in(DeviceState *dev, const char *name, - Clock *source) -{ - clock_set_source(qdev_get_clock_in(dev, name), source); -} +void qdev_connect_clock_in(DeviceState *dev, const char *name, Clock *source); /** * qdev_alias_clock: diff --git a/hw/core/qdev-clock.c b/hw/core/qdev-clock.c index XXXXXXX..XXXXXXX 100644 --- a/hw/core/qdev-clock.c +++ b/hw/core/qdev-clock.c @@ -XXX,XX +XXX,XX @@ Clock *qdev_alias_clock(DeviceState *dev, const char *name, return ncl->clock; } + +void qdev_connect_clock_in(DeviceState *dev, const char *name, Clock *source) +{ + clock_set_source(qdev_get_clock_in(dev, name), source); +} -- 2.20.1 From: Philippe Mathieu-Daudé <f4bug@amsat.org> Clock canonical name is set in device_set_realized (see the block added to hw/core/qdev.c in commit 0e6934f264). If we connect a clock after the device is realized, this code is not executed. This is currently not a problem as this name is only used for trace events, however this disrupt tracing. Add a comment to document qdev_connect_clock_in() must be called before the device is realized, and assert this condition. Signed-off-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Reviewed-by: Alistair Francis <alistair.francis@wdc.com> Message-id: 20200803105647.22223-5-f4bug@amsat.org Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- include/hw/qdev-clock.h | 2 ++ hw/core/qdev-clock.c | 1 + 2 files changed, 3 insertions(+) diff --git a/include/hw/qdev-clock.h b/include/hw/qdev-clock.h index XXXXXXX..XXXXXXX 100644 --- a/include/hw/qdev-clock.h +++ b/include/hw/qdev-clock.h @@ -XXX,XX +XXX,XX @@ Clock *qdev_get_clock_out(DeviceState *dev, const char *name); * * Set the source clock of input clock @name of device @dev to @source. * @source period update will be propagated to @name clock. + * + * Must be called before @dev is realized. */ void qdev_connect_clock_in(DeviceState *dev, const char *name, Clock *source); diff --git a/hw/core/qdev-clock.c b/hw/core/qdev-clock.c index XXXXXXX..XXXXXXX 100644 --- a/hw/core/qdev-clock.c +++ b/hw/core/qdev-clock.c @@ -XXX,XX +XXX,XX @@ Clock *qdev_alias_clock(DeviceState *dev, const char *name, void qdev_connect_clock_in(DeviceState *dev, const char *name, Clock *source) { + assert(!dev->realized); clock_set_source(qdev_get_clock_in(dev, name), source); } -- 2.20.1 From: Philippe Mathieu-Daudé <f4bug@amsat.org> To better align the read/write accesses, display the value after the offset (read accesses only display the offset). Reviewed-by: Richard Henderson <richard.henderson@linaro.org> Signed-off-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Message-id: 20200812190206.31595-2-f4bug@amsat.org Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- hw/misc/unimp.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/hw/misc/unimp.c b/hw/misc/unimp.c index XXXXXXX..XXXXXXX 100644 --- a/hw/misc/unimp.c +++ b/hw/misc/unimp.c @@ -XXX,XX +XXX,XX @@ static uint64_t unimp_read(void *opaque, hwaddr offset, unsigned size) { UnimplementedDeviceState *s = UNIMPLEMENTED_DEVICE(opaque); - qemu_log_mask(LOG_UNIMP, "%s: unimplemented device read " + qemu_log_mask(LOG_UNIMP, "%s: unimplemented device read " "(size %d, offset 0x%" HWADDR_PRIx ")\n", s->name, size, offset); return 0; @@ -XXX,XX +XXX,XX @@ static void unimp_write(void *opaque, hwaddr offset, UnimplementedDeviceState *s = UNIMPLEMENTED_DEVICE(opaque); qemu_log_mask(LOG_UNIMP, "%s: unimplemented device write " - "(size %d, value 0x%" PRIx64 - ", offset 0x%" HWADDR_PRIx ")\n", - s->name, size, value, offset); + "(size %d, offset 0x%" HWADDR_PRIx + ", value 0x%" PRIx64 ")\n", + s->name, size, offset, value); } static const MemoryRegionOps unimp_ops = { -- 2.20.1 From: Philippe Mathieu-Daudé <f4bug@amsat.org> To quickly notice the access size, display the value with the width of the access (i.e. 16-bit access is displayed 0x0000, while 8-bit access 0x00). Reviewed-by: Richard Henderson <richard.henderson@linaro.org> Signed-off-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Message-id: 20200812190206.31595-3-f4bug@amsat.org Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- hw/misc/unimp.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hw/misc/unimp.c b/hw/misc/unimp.c index XXXXXXX..XXXXXXX 100644 --- a/hw/misc/unimp.c +++ b/hw/misc/unimp.c @@ -XXX,XX +XXX,XX @@ static void unimp_write(void *opaque, hwaddr offset, qemu_log_mask(LOG_UNIMP, "%s: unimplemented device write " "(size %d, offset 0x%" HWADDR_PRIx - ", value 0x%" PRIx64 ")\n", - s->name, size, offset, value); + ", value 0x%0*" PRIx64 ")\n", + s->name, size, offset, size << 1, value); } static const MemoryRegionOps unimp_ops = { -- 2.20.1 From: Philippe Mathieu-Daudé <f4bug@amsat.org> To have a better idea of how big is the region where the offset belongs, display the value with the width of the region size (i.e. a region of 0x1000 bytes uses 0x000 format). Reviewed-by: Richard Henderson <richard.henderson@linaro.org> Signed-off-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Message-id: 20200812190206.31595-4-f4bug@amsat.org Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- include/hw/misc/unimp.h | 1 + hw/misc/unimp.c | 10 ++++++---- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/include/hw/misc/unimp.h b/include/hw/misc/unimp.h index XXXXXXX..XXXXXXX 100644 --- a/include/hw/misc/unimp.h +++ b/include/hw/misc/unimp.h @@ -XXX,XX +XXX,XX @@ typedef struct { SysBusDevice parent_obj; MemoryRegion iomem; + unsigned offset_fmt_width; char *name; uint64_t size; } UnimplementedDeviceState; diff --git a/hw/misc/unimp.c b/hw/misc/unimp.c index XXXXXXX..XXXXXXX 100644 --- a/hw/misc/unimp.c +++ b/hw/misc/unimp.c @@ -XXX,XX +XXX,XX @@ static uint64_t unimp_read(void *opaque, hwaddr offset, unsigned size) UnimplementedDeviceState *s = UNIMPLEMENTED_DEVICE(opaque); qemu_log_mask(LOG_UNIMP, "%s: unimplemented device read " - "(size %d, offset 0x%" HWADDR_PRIx ")\n", - s->name, size, offset); + "(size %d, offset 0x%0*" HWADDR_PRIx ")\n", + s->name, size, s->offset_fmt_width, offset); return 0; } @@ -XXX,XX +XXX,XX @@ static void unimp_write(void *opaque, hwaddr offset, UnimplementedDeviceState *s = UNIMPLEMENTED_DEVICE(opaque); qemu_log_mask(LOG_UNIMP, "%s: unimplemented device write " - "(size %d, offset 0x%" HWADDR_PRIx + "(size %d, offset 0x%0*" HWADDR_PRIx ", value 0x%0*" PRIx64 ")\n", - s->name, size, offset, size << 1, value); + s->name, size, s->offset_fmt_width, offset, size << 1, value); } static const MemoryRegionOps unimp_ops = { @@ -XXX,XX +XXX,XX @@ static void unimp_realize(DeviceState *dev, Error **errp) return; } + s->offset_fmt_width = DIV_ROUND_UP(64 - clz64(s->size - 1), 4); + memory_region_init_io(&s->iomem, OBJECT(s), &unimp_ops, s, s->name, s->size); sysbus_init_mmio(SYS_BUS_DEVICE(s), &s->iomem); -- 2.20.1 From: Eduardo Habkost <ehabkost@redhat.com> TYPE_ARM_SSE is a TYPE_SYS_BUS_DEVICE subclass, but ARMSSEClass::parent_class is declared as DeviceClass. It never caused any problems by pure luck: We were not setting class_size for TYPE_ARM_SSE, so class_size of TYPE_SYS_BUS_DEVICE was being used (sizeof(SysBusDeviceClass)). This made the system allocate enough memory for TYPE_ARM_SSE devices even though ARMSSEClass was too small for a sysbus device. Additionally, the ARMSSEClass::info field ended up at the same offset as SysBusDeviceClass::explicit_ofw_unit_address. This would make sysbus_get_fw_dev_path() crash for the device. Luckily, sysbus_get_fw_dev_path() never gets called for TYPE_ARM_SSE devices, because qdev_get_fw_dev_path() is only used by the boot device code, and TYPE_ARM_SSE devices don't appear at the fw_boot_order list. Signed-off-by: Eduardo Habkost <ehabkost@redhat.com> Message-id: 20200826181006.4097163-1-ehabkost@redhat.com Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- include/hw/arm/armsse.h | 2 +- hw/arm/armsse.c | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/include/hw/arm/armsse.h b/include/hw/arm/armsse.h index XXXXXXX..XXXXXXX 100644 --- a/include/hw/arm/armsse.h +++ b/include/hw/arm/armsse.h @@ -XXX,XX +XXX,XX @@ typedef struct ARMSSE { typedef struct ARMSSEInfo ARMSSEInfo; typedef struct ARMSSEClass { - DeviceClass parent_class; + SysBusDeviceClass parent_class; const ARMSSEInfo *info; } ARMSSEClass; diff --git a/hw/arm/armsse.c b/hw/arm/armsse.c index XXXXXXX..XXXXXXX 100644 --- a/hw/arm/armsse.c +++ b/hw/arm/armsse.c @@ -XXX,XX +XXX,XX @@ static const TypeInfo armsse_info = { .name = TYPE_ARMSSE, .parent = TYPE_SYS_BUS_DEVICE, .instance_size = sizeof(ARMSSE), + .class_size = sizeof(ARMSSEClass), .instance_init = armsse_init, .abstract = true, .interfaces = (InterfaceInfo[]) { -- 2.20.1 From: Richard Henderson <richard.henderson@linaro.org> Add left-shift to match the existing right-shift. Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Message-id: 20200815013145.539409-2-richard.henderson@linaro.org Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- include/qemu/int128.h | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/include/qemu/int128.h b/include/qemu/int128.h index XXXXXXX..XXXXXXX 100644 --- a/include/qemu/int128.h +++ b/include/qemu/int128.h @@ -XXX,XX +XXX,XX @@ static inline Int128 int128_rshift(Int128 a, int n) return a >> n; } +static inline Int128 int128_lshift(Int128 a, int n) +{ + return a << n; +} + static inline Int128 int128_add(Int128 a, Int128 b) { return a + b; @@ -XXX,XX +XXX,XX @@ static inline Int128 int128_rshift(Int128 a, int n) } } +static inline Int128 int128_lshift(Int128 a, int n) +{ + uint64_t l = a.lo << (n & 63); + if (n >= 64) { + return int128_make128(0, l); + } else if (n > 0) { + return int128_make128(l, (a.hi << n) | (a.lo >> (64 - n))); + } + return a; +} + static inline Int128 int128_add(Int128 a, Int128 b) { uint64_t lo = a.lo + b.lo; -- 2.20.1 From: Richard Henderson <richard.henderson@linaro.org> Model the new function on gen_gvec_fn2 in translate-a64.c, but indicating which kind of register and in which order. Since there is only one user of do_vector2_z, fold it into do_mov_z. Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Message-id: 20200815013145.539409-3-richard.henderson@linaro.org Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- target/arm/translate-sve.c | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/translate-sve.c +++ b/target/arm/translate-sve.c @@ -XXX,XX +XXX,XX @@ static int pred_gvec_reg_size(DisasContext *s) } /* Invoke a vector expander on two Zregs. */ -static bool do_vector2_z(DisasContext *s, GVecGen2Fn *gvec_fn, - int esz, int rd, int rn) + +static void gen_gvec_fn_zz(DisasContext *s, GVecGen2Fn *gvec_fn, + int esz, int rd, int rn) { - if (sve_access_check(s)) { - unsigned vsz = vec_full_reg_size(s); - gvec_fn(esz, vec_full_reg_offset(s, rd), - vec_full_reg_offset(s, rn), vsz, vsz); - } - return true; + unsigned vsz = vec_full_reg_size(s); + gvec_fn(esz, vec_full_reg_offset(s, rd), + vec_full_reg_offset(s, rn), vsz, vsz); } /* Invoke a vector expander on three Zregs. */ @@ -XXX,XX +XXX,XX @@ static bool do_vector3_z(DisasContext *s, GVecGen3Fn *gvec_fn, /* Invoke a vector move on two Zregs. */ static bool do_mov_z(DisasContext *s, int rd, int rn) { - return do_vector2_z(s, tcg_gen_gvec_mov, 0, rd, rn); + if (sve_access_check(s)) { + gen_gvec_fn_zz(s, tcg_gen_gvec_mov, MO_8, rd, rn); + } + return true; } /* Initialize a Zreg with replications of a 64-bit immediate. */ -- 2.20.1 From: Richard Henderson <richard.henderson@linaro.org> Model gen_gvec_fn_zzz on gen_gvec_fn3 in translate-a64.c, but indicating which kind of register and in which order. Model do_zzz_fn on the other do_foo functions that take an argument set and verify sve enabled. Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Message-id: 20200815013145.539409-4-richard.henderson@linaro.org Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- target/arm/translate-sve.c | 43 +++++++++++++++++++++----------------- 1 file changed, 24 insertions(+), 19 deletions(-) diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/translate-sve.c +++ b/target/arm/translate-sve.c @@ -XXX,XX +XXX,XX @@ static void gen_gvec_fn_zz(DisasContext *s, GVecGen2Fn *gvec_fn, } /* Invoke a vector expander on three Zregs. */ -static bool do_vector3_z(DisasContext *s, GVecGen3Fn *gvec_fn, - int esz, int rd, int rn, int rm) +static void gen_gvec_fn_zzz(DisasContext *s, GVecGen3Fn *gvec_fn, + int esz, int rd, int rn, int rm) { - if (sve_access_check(s)) { - unsigned vsz = vec_full_reg_size(s); - gvec_fn(esz, vec_full_reg_offset(s, rd), - vec_full_reg_offset(s, rn), - vec_full_reg_offset(s, rm), vsz, vsz); - } - return true; + unsigned vsz = vec_full_reg_size(s); + gvec_fn(esz, vec_full_reg_offset(s, rd), + vec_full_reg_offset(s, rn), + vec_full_reg_offset(s, rm), vsz, vsz); } /* Invoke a vector move on two Zregs. */ @@ -XXX,XX +XXX,XX @@ const uint64_t pred_esz_masks[4] = { *** SVE Logical - Unpredicated Group */ +static bool do_zzz_fn(DisasContext *s, arg_rrr_esz *a, GVecGen3Fn *gvec_fn) +{ + if (sve_access_check(s)) { + gen_gvec_fn_zzz(s, gvec_fn, a->esz, a->rd, a->rn, a->rm); + } + return true; +} + static bool trans_AND_zzz(DisasContext *s, arg_rrr_esz *a) { - return do_vector3_z(s, tcg_gen_gvec_and, 0, a->rd, a->rn, a->rm); + return do_zzz_fn(s, a, tcg_gen_gvec_and); } static bool trans_ORR_zzz(DisasContext *s, arg_rrr_esz *a) { - return do_vector3_z(s, tcg_gen_gvec_or, 0, a->rd, a->rn, a->rm); + return do_zzz_fn(s, a, tcg_gen_gvec_or); } static bool trans_EOR_zzz(DisasContext *s, arg_rrr_esz *a) { - return do_vector3_z(s, tcg_gen_gvec_xor, 0, a->rd, a->rn, a->rm); + return do_zzz_fn(s, a, tcg_gen_gvec_xor); } static bool trans_BIC_zzz(DisasContext *s, arg_rrr_esz *a) { - return do_vector3_z(s, tcg_gen_gvec_andc, 0, a->rd, a->rn, a->rm); + return do_zzz_fn(s, a, tcg_gen_gvec_andc); } /* @@ -XXX,XX +XXX,XX @@ static bool trans_BIC_zzz(DisasContext *s, arg_rrr_esz *a) static bool trans_ADD_zzz(DisasContext *s, arg_rrr_esz *a) { - return do_vector3_z(s, tcg_gen_gvec_add, a->esz, a->rd, a->rn, a->rm); + return do_zzz_fn(s, a, tcg_gen_gvec_add); } static bool trans_SUB_zzz(DisasContext *s, arg_rrr_esz *a) { - return do_vector3_z(s, tcg_gen_gvec_sub, a->esz, a->rd, a->rn, a->rm); + return do_zzz_fn(s, a, tcg_gen_gvec_sub); } static bool trans_SQADD_zzz(DisasContext *s, arg_rrr_esz *a) { - return do_vector3_z(s, tcg_gen_gvec_ssadd, a->esz, a->rd, a->rn, a->rm); + return do_zzz_fn(s, a, tcg_gen_gvec_ssadd); } static bool trans_SQSUB_zzz(DisasContext *s, arg_rrr_esz *a) { - return do_vector3_z(s, tcg_gen_gvec_sssub, a->esz, a->rd, a->rn, a->rm); + return do_zzz_fn(s, a, tcg_gen_gvec_sssub); } static bool trans_UQADD_zzz(DisasContext *s, arg_rrr_esz *a) { - return do_vector3_z(s, tcg_gen_gvec_usadd, a->esz, a->rd, a->rn, a->rm); + return do_zzz_fn(s, a, tcg_gen_gvec_usadd); } static bool trans_UQSUB_zzz(DisasContext *s, arg_rrr_esz *a) { - return do_vector3_z(s, tcg_gen_gvec_ussub, a->esz, a->rd, a->rn, a->rm); + return do_zzz_fn(s, a, tcg_gen_gvec_ussub); } /* -- 2.20.1 From: Richard Henderson <richard.henderson@linaro.org> We want to ensure that access is checked by the time we ask for a specific fp/vector register. We want to ensure that we do not emit two lots of code to raise an exception. But sometimes it's difficult to cleanly organize the code such that we never pass through sve_check_access exactly once. Allow multiple calls so long as the result is true, that is, no exception to be raised. Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Message-id: 20200815013145.539409-5-richard.henderson@linaro.org Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- target/arm/translate.h | 1 + target/arm/translate-a64.c | 27 ++++++++++++++++----------- 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/target/arm/translate.h b/target/arm/translate.h index XXXXXXX..XXXXXXX 100644 --- a/target/arm/translate.h +++ b/target/arm/translate.h @@ -XXX,XX +XXX,XX @@ typedef struct DisasContext { * that it is set at the point where we actually touch the FP regs. */ bool fp_access_checked; + bool sve_access_checked; /* ARMv8 single-step state (this is distinct from the QEMU gdbstub * single-step support). */ diff --git a/target/arm/translate-a64.c b/target/arm/translate-a64.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/translate-a64.c +++ b/target/arm/translate-a64.c @@ -XXX,XX +XXX,XX @@ static void do_vec_ld(DisasContext *s, int destidx, int element, * unallocated-encoding checks (otherwise the syndrome information * for the resulting exception will be incorrect). */ -static inline bool fp_access_check(DisasContext *s) +static bool fp_access_check(DisasContext *s) { - assert(!s->fp_access_checked); - s->fp_access_checked = true; + if (s->fp_excp_el) { + assert(!s->fp_access_checked); + s->fp_access_checked = true; - if (!s->fp_excp_el) { - return true; + gen_exception_insn(s, s->pc_curr, EXCP_UDEF, + syn_fp_access_trap(1, 0xe, false), s->fp_excp_el); + return false; } - - gen_exception_insn(s, s->pc_curr, EXCP_UDEF, - syn_fp_access_trap(1, 0xe, false), s->fp_excp_el); - return false; + s->fp_access_checked = true; + return true; } /* Check that SVE access is enabled. If it is, return true. @@ -XXX,XX +XXX,XX @@ static inline bool fp_access_check(DisasContext *s) bool sve_access_check(DisasContext *s) { if (s->sve_excp_el) { - gen_exception_insn(s, s->pc_curr, EXCP_UDEF, syn_sve_access_trap(), - s->sve_excp_el); + assert(!s->sve_access_checked); + s->sve_access_checked = true; + + gen_exception_insn(s, s->pc_curr, EXCP_UDEF, + syn_sve_access_trap(), s->sve_excp_el); return false; } + s->sve_access_checked = true; return fp_access_check(s); } @@ -XXX,XX +XXX,XX @@ static void disas_a64_insn(CPUARMState *env, DisasContext *s) s->base.pc_next += 4; s->fp_access_checked = false; + s->sve_access_checked = false; if (dc_isar_feature(aa64_bti, s)) { if (s->base.num_insns == 1) { -- 2.20.1 From: Richard Henderson <richard.henderson@linaro.org> This is the only user of the function. Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Message-id: 20200815013145.539409-6-richard.henderson@linaro.org Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- target/arm/translate-sve.c | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/translate-sve.c +++ b/target/arm/translate-sve.c @@ -XXX,XX +XXX,XX @@ static void do_dupi_z(DisasContext *s, int rd, uint64_t word) tcg_gen_gvec_dup_imm(MO_64, vec_full_reg_offset(s, rd), vsz, vsz, word); } -/* Invoke a vector expander on two Pregs. */ -static bool do_vector2_p(DisasContext *s, GVecGen2Fn *gvec_fn, - int esz, int rd, int rn) -{ - if (sve_access_check(s)) { - unsigned psz = pred_gvec_reg_size(s); - gvec_fn(esz, pred_full_reg_offset(s, rd), - pred_full_reg_offset(s, rn), psz, psz); - } - return true; -} - /* Invoke a vector expander on three Pregs. */ static bool do_vector3_p(DisasContext *s, GVecGen3Fn *gvec_fn, int esz, int rd, int rn, int rm) @@ -XXX,XX +XXX,XX @@ static bool do_vecop4_p(DisasContext *s, const GVecGen4 *gvec_op, /* Invoke a vector move on two Pregs. */ static bool do_mov_p(DisasContext *s, int rd, int rn) { - return do_vector2_p(s, tcg_gen_gvec_mov, 0, rd, rn); + if (sve_access_check(s)) { + unsigned psz = pred_gvec_reg_size(s); + tcg_gen_gvec_mov(MO_8, pred_full_reg_offset(s, rd), + pred_full_reg_offset(s, rn), psz, psz); + } + return true; } /* Set the cpu flags as per a return from an SVE helper. */ -- 2.20.1 From: Richard Henderson <richard.henderson@linaro.org> Move the check for !S into do_pppp_flags, which allows to merge in do_vecop4_p. Split out gen_gvec_fn_ppp without sve_access_check, to mirror gen_gvec_fn_zzz. Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Message-id: 20200815013145.539409-7-richard.henderson@linaro.org Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- target/arm/translate-sve.c | 111 ++++++++++++++----------------------- 1 file changed, 43 insertions(+), 68 deletions(-) diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/translate-sve.c +++ b/target/arm/translate-sve.c @@ -XXX,XX +XXX,XX @@ static void do_dupi_z(DisasContext *s, int rd, uint64_t word) } /* Invoke a vector expander on three Pregs. */ -static bool do_vector3_p(DisasContext *s, GVecGen3Fn *gvec_fn, - int esz, int rd, int rn, int rm) +static void gen_gvec_fn_ppp(DisasContext *s, GVecGen3Fn *gvec_fn, + int rd, int rn, int rm) { - if (sve_access_check(s)) { - unsigned psz = pred_gvec_reg_size(s); - gvec_fn(esz, pred_full_reg_offset(s, rd), - pred_full_reg_offset(s, rn), - pred_full_reg_offset(s, rm), psz, psz); - } - return true; -} - -/* Invoke a vector operation on four Pregs. */ -static bool do_vecop4_p(DisasContext *s, const GVecGen4 *gvec_op, - int rd, int rn, int rm, int rg) -{ - if (sve_access_check(s)) { - unsigned psz = pred_gvec_reg_size(s); - tcg_gen_gvec_4(pred_full_reg_offset(s, rd), - pred_full_reg_offset(s, rn), - pred_full_reg_offset(s, rm), - pred_full_reg_offset(s, rg), - psz, psz, gvec_op); - } - return true; + unsigned psz = pred_gvec_reg_size(s); + gvec_fn(MO_64, pred_full_reg_offset(s, rd), + pred_full_reg_offset(s, rn), + pred_full_reg_offset(s, rm), psz, psz); } /* Invoke a vector move on two Pregs. */ @@ -XXX,XX +XXX,XX @@ static bool do_pppp_flags(DisasContext *s, arg_rprr_s *a, int mofs = pred_full_reg_offset(s, a->rm); int gofs = pred_full_reg_offset(s, a->pg); + if (!a->s) { + tcg_gen_gvec_4(dofs, nofs, mofs, gofs, psz, psz, gvec_op); + return true; + } + if (psz == 8) { /* Do the operation and the flags generation in temps. */ TCGv_i64 pd = tcg_temp_new_i64(); @@ -XXX,XX +XXX,XX @@ static bool trans_AND_pppp(DisasContext *s, arg_rprr_s *a) .fno = gen_helper_sve_and_pppp, .prefer_i64 = TCG_TARGET_REG_BITS == 64, }; - if (a->s) { - return do_pppp_flags(s, a, &op); - } else if (a->rn == a->rm) { - if (a->pg == a->rn) { - return do_mov_p(s, a->rd, a->rn); - } else { - return do_vector3_p(s, tcg_gen_gvec_and, 0, a->rd, a->rn, a->pg); + + if (!a->s) { + if (!sve_access_check(s)) { + return true; + } + if (a->rn == a->rm) { + if (a->pg == a->rn) { + do_mov_p(s, a->rd, a->rn); + } else { + gen_gvec_fn_ppp(s, tcg_gen_gvec_and, a->rd, a->rn, a->pg); + } + return true; + } else if (a->pg == a->rn || a->pg == a->rm) { + gen_gvec_fn_ppp(s, tcg_gen_gvec_and, a->rd, a->rn, a->rm); + return true; } - } else if (a->pg == a->rn || a->pg == a->rm) { - return do_vector3_p(s, tcg_gen_gvec_and, 0, a->rd, a->rn, a->rm); - } else { - return do_vecop4_p(s, &op, a->rd, a->rn, a->rm, a->pg); } + return do_pppp_flags(s, a, &op); } static void gen_bic_pg_i64(TCGv_i64 pd, TCGv_i64 pn, TCGv_i64 pm, TCGv_i64 pg) @@ -XXX,XX +XXX,XX @@ static bool trans_BIC_pppp(DisasContext *s, arg_rprr_s *a) .fno = gen_helper_sve_bic_pppp, .prefer_i64 = TCG_TARGET_REG_BITS == 64, }; - if (a->s) { - return do_pppp_flags(s, a, &op); - } else if (a->pg == a->rn) { - return do_vector3_p(s, tcg_gen_gvec_andc, 0, a->rd, a->rn, a->rm); - } else { - return do_vecop4_p(s, &op, a->rd, a->rn, a->rm, a->pg); + + if (!a->s && a->pg == a->rn) { + if (sve_access_check(s)) { + gen_gvec_fn_ppp(s, tcg_gen_gvec_andc, a->rd, a->rn, a->rm); + } + return true; } + return do_pppp_flags(s, a, &op); } static void gen_eor_pg_i64(TCGv_i64 pd, TCGv_i64 pn, TCGv_i64 pm, TCGv_i64 pg) @@ -XXX,XX +XXX,XX @@ static bool trans_EOR_pppp(DisasContext *s, arg_rprr_s *a) .fno = gen_helper_sve_eor_pppp, .prefer_i64 = TCG_TARGET_REG_BITS == 64, }; - if (a->s) { - return do_pppp_flags(s, a, &op); - } else { - return do_vecop4_p(s, &op, a->rd, a->rn, a->rm, a->pg); - } + return do_pppp_flags(s, a, &op); } static void gen_sel_pg_i64(TCGv_i64 pd, TCGv_i64 pn, TCGv_i64 pm, TCGv_i64 pg) @@ -XXX,XX +XXX,XX @@ static bool trans_SEL_pppp(DisasContext *s, arg_rprr_s *a) .fno = gen_helper_sve_sel_pppp, .prefer_i64 = TCG_TARGET_REG_BITS == 64, }; + if (a->s) { return false; - } else { - return do_vecop4_p(s, &op, a->rd, a->rn, a->rm, a->pg); } + return do_pppp_flags(s, a, &op); } static void gen_orr_pg_i64(TCGv_i64 pd, TCGv_i64 pn, TCGv_i64 pm, TCGv_i64 pg) @@ -XXX,XX +XXX,XX @@ static bool trans_ORR_pppp(DisasContext *s, arg_rprr_s *a) .fno = gen_helper_sve_orr_pppp, .prefer_i64 = TCG_TARGET_REG_BITS == 64, }; - if (a->s) { - return do_pppp_flags(s, a, &op); - } else if (a->pg == a->rn && a->rn == a->rm) { + + if (!a->s && a->pg == a->rn && a->rn == a->rm) { return do_mov_p(s, a->rd, a->rn); - } else { - return do_vecop4_p(s, &op, a->rd, a->rn, a->rm, a->pg); } + return do_pppp_flags(s, a, &op); } static void gen_orn_pg_i64(TCGv_i64 pd, TCGv_i64 pn, TCGv_i64 pm, TCGv_i64 pg) @@ -XXX,XX +XXX,XX @@ static bool trans_ORN_pppp(DisasContext *s, arg_rprr_s *a) .fno = gen_helper_sve_orn_pppp, .prefer_i64 = TCG_TARGET_REG_BITS == 64, }; - if (a->s) { - return do_pppp_flags(s, a, &op); - } else { - return do_vecop4_p(s, &op, a->rd, a->rn, a->rm, a->pg); - } + return do_pppp_flags(s, a, &op); } static void gen_nor_pg_i64(TCGv_i64 pd, TCGv_i64 pn, TCGv_i64 pm, TCGv_i64 pg) @@ -XXX,XX +XXX,XX @@ static bool trans_NOR_pppp(DisasContext *s, arg_rprr_s *a) .fno = gen_helper_sve_nor_pppp, .prefer_i64 = TCG_TARGET_REG_BITS == 64, }; - if (a->s) { - return do_pppp_flags(s, a, &op); - } else { - return do_vecop4_p(s, &op, a->rd, a->rn, a->rm, a->pg); - } + return do_pppp_flags(s, a, &op); } static void gen_nand_pg_i64(TCGv_i64 pd, TCGv_i64 pn, TCGv_i64 pm, TCGv_i64 pg) @@ -XXX,XX +XXX,XX @@ static bool trans_NAND_pppp(DisasContext *s, arg_rprr_s *a) .fno = gen_helper_sve_nand_pppp, .prefer_i64 = TCG_TARGET_REG_BITS == 64, }; - if (a->s) { - return do_pppp_flags(s, a, &op); - } else { - return do_vecop4_p(s, &op, a->rd, a->rn, a->rm, a->pg); - } + return do_pppp_flags(s, a, &op); } /* -- 2.20.1 From: Richard Henderson <richard.henderson@linaro.org> The gvec operation was added after the initial implementation of the SEL instruction and was missed in the conversion. Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Message-id: 20200815013145.539409-8-richard.henderson@linaro.org Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- target/arm/translate-sve.c | 31 ++++++++----------------------- 1 file changed, 8 insertions(+), 23 deletions(-) diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/translate-sve.c +++ b/target/arm/translate-sve.c @@ -XXX,XX +XXX,XX @@ static bool trans_EOR_pppp(DisasContext *s, arg_rprr_s *a) return do_pppp_flags(s, a, &op); } -static void gen_sel_pg_i64(TCGv_i64 pd, TCGv_i64 pn, TCGv_i64 pm, TCGv_i64 pg) -{ - tcg_gen_and_i64(pn, pn, pg); - tcg_gen_andc_i64(pm, pm, pg); - tcg_gen_or_i64(pd, pn, pm); -} - -static void gen_sel_pg_vec(unsigned vece, TCGv_vec pd, TCGv_vec pn, - TCGv_vec pm, TCGv_vec pg) -{ - tcg_gen_and_vec(vece, pn, pn, pg); - tcg_gen_andc_vec(vece, pm, pm, pg); - tcg_gen_or_vec(vece, pd, pn, pm); -} - static bool trans_SEL_pppp(DisasContext *s, arg_rprr_s *a) { - static const GVecGen4 op = { - .fni8 = gen_sel_pg_i64, - .fniv = gen_sel_pg_vec, - .fno = gen_helper_sve_sel_pppp, - .prefer_i64 = TCG_TARGET_REG_BITS == 64, - }; - if (a->s) { return false; } - return do_pppp_flags(s, a, &op); + if (sve_access_check(s)) { + unsigned psz = pred_gvec_reg_size(s); + tcg_gen_gvec_bitsel(MO_8, pred_full_reg_offset(s, a->rd), + pred_full_reg_offset(s, a->pg), + pred_full_reg_offset(s, a->rn), + pred_full_reg_offset(s, a->rm), psz, psz); + } + return true; } static void gen_orr_pg_i64(TCGv_i64 pd, TCGv_i64 pn, TCGv_i64 pm, TCGv_i64 pg) -- 2.20.1 From: Richard Henderson <richard.henderson@linaro.org> Model after gen_gvec_fn_zzz et al. Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Message-id: 20200815013145.539409-9-richard.henderson@linaro.org Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- target/arm/translate-sve.c | 35 ++++++++++++++++------------------- 1 file changed, 16 insertions(+), 19 deletions(-) diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/translate-sve.c +++ b/target/arm/translate-sve.c @@ -XXX,XX +XXX,XX @@ static int pred_gvec_reg_size(DisasContext *s) return size_for_gvec(pred_full_reg_size(s)); } -/* Invoke a vector expander on two Zregs. */ +/* Invoke an out-of-line helper on 3 Zregs and a predicate. */ +static void gen_gvec_ool_zzzp(DisasContext *s, gen_helper_gvec_4 *fn, + int rd, int rn, int rm, int pg, int data) +{ + unsigned vsz = vec_full_reg_size(s); + tcg_gen_gvec_4_ool(vec_full_reg_offset(s, rd), + vec_full_reg_offset(s, rn), + vec_full_reg_offset(s, rm), + pred_full_reg_offset(s, pg), + vsz, vsz, data, fn); +} +/* Invoke a vector expander on two Zregs. */ static void gen_gvec_fn_zz(DisasContext *s, GVecGen2Fn *gvec_fn, int esz, int rd, int rn) { @@ -XXX,XX +XXX,XX @@ static bool trans_UQSUB_zzz(DisasContext *s, arg_rrr_esz *a) static bool do_zpzz_ool(DisasContext *s, arg_rprr_esz *a, gen_helper_gvec_4 *fn) { - unsigned vsz = vec_full_reg_size(s); if (fn == NULL) { return false; } if (sve_access_check(s)) { - tcg_gen_gvec_4_ool(vec_full_reg_offset(s, a->rd), - vec_full_reg_offset(s, a->rn), - vec_full_reg_offset(s, a->rm), - pred_full_reg_offset(s, a->pg), - vsz, vsz, 0, fn); + gen_gvec_ool_zzzp(s, fn, a->rd, a->rn, a->rm, a->pg, 0); } return true; } @@ -XXX,XX +XXX,XX @@ static void do_sel_z(DisasContext *s, int rd, int rn, int rm, int pg, int esz) gen_helper_sve_sel_zpzz_b, gen_helper_sve_sel_zpzz_h, gen_helper_sve_sel_zpzz_s, gen_helper_sve_sel_zpzz_d }; - unsigned vsz = vec_full_reg_size(s); - tcg_gen_gvec_4_ool(vec_full_reg_offset(s, rd), - vec_full_reg_offset(s, rn), - vec_full_reg_offset(s, rm), - pred_full_reg_offset(s, pg), - vsz, vsz, 0, fns[esz]); + gen_gvec_ool_zzzp(s, fns[esz], rd, rn, rm, pg, 0); } #define DO_ZPZZ(NAME, name) \ @@ -XXX,XX +XXX,XX @@ static bool trans_RBIT(DisasContext *s, arg_rpr_esz *a) static bool trans_SPLICE(DisasContext *s, arg_rprr_esz *a) { if (sve_access_check(s)) { - unsigned vsz = vec_full_reg_size(s); - tcg_gen_gvec_4_ool(vec_full_reg_offset(s, a->rd), - vec_full_reg_offset(s, a->rn), - vec_full_reg_offset(s, a->rm), - pred_full_reg_offset(s, a->pg), - vsz, vsz, a->esz, gen_helper_sve_splice); + gen_gvec_ool_zzzp(s, gen_helper_sve_splice, + a->rd, a->rn, a->rm, a->pg, 0); } return true; } -- 2.20.1 From: Richard Henderson <richard.henderson@linaro.org> The existing clr functions have only one vector argument, and so can only clear in place. The existing movz functions have two vector arguments, and so can clear while moving. Merge them, with a flag that controls the sense of active vs inactive elements being cleared. Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Message-id: 20200815013145.539409-10-richard.henderson@linaro.org Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- target/arm/helper-sve.h | 5 --- target/arm/sve_helper.c | 70 ++++++++------------------------------ target/arm/translate-sve.c | 53 +++++++++++------------------ 3 files changed, 34 insertions(+), 94 deletions(-) diff --git a/target/arm/helper-sve.h b/target/arm/helper-sve.h index XXXXXXX..XXXXXXX 100644 --- a/target/arm/helper-sve.h +++ b/target/arm/helper-sve.h @@ -XXX,XX +XXX,XX @@ DEF_HELPER_FLAGS_3(sve_uminv_h, TCG_CALL_NO_RWG, i64, ptr, ptr, i32) DEF_HELPER_FLAGS_3(sve_uminv_s, TCG_CALL_NO_RWG, i64, ptr, ptr, i32) DEF_HELPER_FLAGS_3(sve_uminv_d, TCG_CALL_NO_RWG, i64, ptr, ptr, i32) -DEF_HELPER_FLAGS_3(sve_clr_b, TCG_CALL_NO_RWG, void, ptr, ptr, i32) -DEF_HELPER_FLAGS_3(sve_clr_h, TCG_CALL_NO_RWG, void, ptr, ptr, i32) -DEF_HELPER_FLAGS_3(sve_clr_s, TCG_CALL_NO_RWG, void, ptr, ptr, i32) -DEF_HELPER_FLAGS_3(sve_clr_d, TCG_CALL_NO_RWG, void, ptr, ptr, i32) - DEF_HELPER_FLAGS_4(sve_movz_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) DEF_HELPER_FLAGS_4(sve_movz_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) DEF_HELPER_FLAGS_4(sve_movz_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) diff --git a/target/arm/sve_helper.c b/target/arm/sve_helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/sve_helper.c +++ b/target/arm/sve_helper.c @@ -XXX,XX +XXX,XX @@ uint32_t HELPER(sve_pnext)(void *vd, void *vg, uint32_t pred_desc) return flags; } -/* Store zero into every active element of Zd. We will use this for two - * and three-operand predicated instructions for which logic dictates a - * zero result. In particular, logical shift by element size, which is - * otherwise undefined on the host. - * - * For element sizes smaller than uint64_t, we use tables to expand - * the N bits of the controlling predicate to a byte mask, and clear - * those bytes. +/* + * Copy Zn into Zd, and store zero into inactive elements. + * If inv, store zeros into the active elements. */ -void HELPER(sve_clr_b)(void *vd, void *vg, uint32_t desc) -{ - intptr_t i, opr_sz = simd_oprsz(desc) / 8; - uint64_t *d = vd; - uint8_t *pg = vg; - for (i = 0; i < opr_sz; i += 1) { - d[i] &= ~expand_pred_b(pg[H1(i)]); - } -} - -void HELPER(sve_clr_h)(void *vd, void *vg, uint32_t desc) -{ - intptr_t i, opr_sz = simd_oprsz(desc) / 8; - uint64_t *d = vd; - uint8_t *pg = vg; - for (i = 0; i < opr_sz; i += 1) { - d[i] &= ~expand_pred_h(pg[H1(i)]); - } -} - -void HELPER(sve_clr_s)(void *vd, void *vg, uint32_t desc) -{ - intptr_t i, opr_sz = simd_oprsz(desc) / 8; - uint64_t *d = vd; - uint8_t *pg = vg; - for (i = 0; i < opr_sz; i += 1) { - d[i] &= ~expand_pred_s(pg[H1(i)]); - } -} - -void HELPER(sve_clr_d)(void *vd, void *vg, uint32_t desc) -{ - intptr_t i, opr_sz = simd_oprsz(desc) / 8; - uint64_t *d = vd; - uint8_t *pg = vg; - for (i = 0; i < opr_sz; i += 1) { - if (pg[H1(i)] & 1) { - d[i] = 0; - } - } -} - -/* Copy Zn into Zd, and store zero into inactive elements. */ void HELPER(sve_movz_b)(void *vd, void *vn, void *vg, uint32_t desc) { intptr_t i, opr_sz = simd_oprsz(desc) / 8; + uint64_t inv = -(uint64_t)(simd_data(desc) & 1); uint64_t *d = vd, *n = vn; uint8_t *pg = vg; + for (i = 0; i < opr_sz; i += 1) { - d[i] = n[i] & expand_pred_b(pg[H1(i)]); + d[i] = n[i] & (expand_pred_b(pg[H1(i)]) ^ inv); } } void HELPER(sve_movz_h)(void *vd, void *vn, void *vg, uint32_t desc) { intptr_t i, opr_sz = simd_oprsz(desc) / 8; + uint64_t inv = -(uint64_t)(simd_data(desc) & 1); uint64_t *d = vd, *n = vn; uint8_t *pg = vg; + for (i = 0; i < opr_sz; i += 1) { - d[i] = n[i] & expand_pred_h(pg[H1(i)]); + d[i] = n[i] & (expand_pred_h(pg[H1(i)]) ^ inv); } } void HELPER(sve_movz_s)(void *vd, void *vn, void *vg, uint32_t desc) { intptr_t i, opr_sz = simd_oprsz(desc) / 8; + uint64_t inv = -(uint64_t)(simd_data(desc) & 1); uint64_t *d = vd, *n = vn; uint8_t *pg = vg; + for (i = 0; i < opr_sz; i += 1) { - d[i] = n[i] & expand_pred_s(pg[H1(i)]); + d[i] = n[i] & (expand_pred_s(pg[H1(i)]) ^ inv); } } @@ -XXX,XX +XXX,XX @@ void HELPER(sve_movz_d)(void *vd, void *vn, void *vg, uint32_t desc) intptr_t i, opr_sz = simd_oprsz(desc) / 8; uint64_t *d = vd, *n = vn; uint8_t *pg = vg; + uint8_t inv = simd_data(desc); + for (i = 0; i < opr_sz; i += 1) { - d[i] = n[i] & -(uint64_t)(pg[H1(i)] & 1); + d[i] = n[i] & -(uint64_t)((pg[H1(i)] ^ inv) & 1); } } diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/translate-sve.c +++ b/target/arm/translate-sve.c @@ -XXX,XX +XXX,XX @@ static bool trans_SADDV(DisasContext *s, arg_rpr_esz *a) *** SVE Shift by Immediate - Predicated Group */ -/* Store zero into every active element of Zd. We will use this for two - * and three-operand predicated instructions for which logic dictates a - * zero result. +/* + * Copy Zn into Zd, storing zeros into inactive elements. + * If invert, store zeros into the active elements. */ -static bool do_clr_zp(DisasContext *s, int rd, int pg, int esz) -{ - static gen_helper_gvec_2 * const fns[4] = { - gen_helper_sve_clr_b, gen_helper_sve_clr_h, - gen_helper_sve_clr_s, gen_helper_sve_clr_d, - }; - if (sve_access_check(s)) { - unsigned vsz = vec_full_reg_size(s); - tcg_gen_gvec_2_ool(vec_full_reg_offset(s, rd), - pred_full_reg_offset(s, pg), - vsz, vsz, 0, fns[esz]); - } - return true; -} - -/* Copy Zn into Zd, storing zeros into inactive elements. */ -static void do_movz_zpz(DisasContext *s, int rd, int rn, int pg, int esz) +static bool do_movz_zpz(DisasContext *s, int rd, int rn, int pg, + int esz, bool invert) { static gen_helper_gvec_3 * const fns[4] = { gen_helper_sve_movz_b, gen_helper_sve_movz_h, gen_helper_sve_movz_s, gen_helper_sve_movz_d, }; - unsigned vsz = vec_full_reg_size(s); - tcg_gen_gvec_3_ool(vec_full_reg_offset(s, rd), - vec_full_reg_offset(s, rn), - pred_full_reg_offset(s, pg), - vsz, vsz, 0, fns[esz]); + + if (sve_access_check(s)) { + unsigned vsz = vec_full_reg_size(s); + tcg_gen_gvec_3_ool(vec_full_reg_offset(s, rd), + vec_full_reg_offset(s, rn), + pred_full_reg_offset(s, pg), + vsz, vsz, invert, fns[esz]); + } + return true; } static bool do_zpzi_ool(DisasContext *s, arg_rpri_esz *a, @@ -XXX,XX +XXX,XX @@ static bool trans_LSR_zpzi(DisasContext *s, arg_rpri_esz *a) /* Shift by element size is architecturally valid. For logical shifts, it is a zeroing operation. */ if (a->imm >= (8 << a->esz)) { - return do_clr_zp(s, a->rd, a->pg, a->esz); + return do_movz_zpz(s, a->rd, a->rd, a->pg, a->esz, true); } else { return do_zpzi_ool(s, a, fns[a->esz]); } @@ -XXX,XX +XXX,XX @@ static bool trans_LSL_zpzi(DisasContext *s, arg_rpri_esz *a) /* Shift by element size is architecturally valid. For logical shifts, it is a zeroing operation. */ if (a->imm >= (8 << a->esz)) { - return do_clr_zp(s, a->rd, a->pg, a->esz); + return do_movz_zpz(s, a->rd, a->rd, a->pg, a->esz, true); } else { return do_zpzi_ool(s, a, fns[a->esz]); } @@ -XXX,XX +XXX,XX @@ static bool trans_ASRD(DisasContext *s, arg_rpri_esz *a) /* Shift by element size is architecturally valid. For arithmetic right shift for division, it is a zeroing operation. */ if (a->imm >= (8 << a->esz)) { - return do_clr_zp(s, a->rd, a->pg, a->esz); + return do_movz_zpz(s, a->rd, a->rd, a->pg, a->esz, true); } else { return do_zpzi_ool(s, a, fns[a->esz]); } @@ -XXX,XX +XXX,XX @@ static bool trans_LD1R_zpri(DisasContext *s, arg_rpri_load *a) /* Zero the inactive elements. */ gen_set_label(over); - do_movz_zpz(s, a->rd, a->rd, a->pg, esz); - return true; + return do_movz_zpz(s, a->rd, a->rd, a->pg, esz, false); } static void do_st_zpa(DisasContext *s, int zt, int pg, TCGv_i64 addr, @@ -XXX,XX +XXX,XX @@ static bool trans_MOVPRFX_m(DisasContext *s, arg_rpr_esz *a) static bool trans_MOVPRFX_z(DisasContext *s, arg_rpr_esz *a) { - if (sve_access_check(s)) { - do_movz_zpz(s, a->rd, a->rn, a->pg, a->esz); - } - return true; + return do_movz_zpz(s, a->rd, a->rn, a->pg, a->esz, false); } -- 2.20.1 From: Richard Henderson <richard.henderson@linaro.org> Model after gen_gvec_fn_zzz et al. Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Message-id: 20200815013145.539409-11-richard.henderson@linaro.org Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- target/arm/translate-sve.c | 29 ++++++++++++++--------------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/translate-sve.c +++ b/target/arm/translate-sve.c @@ -XXX,XX +XXX,XX @@ static int pred_gvec_reg_size(DisasContext *s) return size_for_gvec(pred_full_reg_size(s)); } +/* Invoke an out-of-line helper on 2 Zregs and a predicate. */ +static void gen_gvec_ool_zzp(DisasContext *s, gen_helper_gvec_3 *fn, + int rd, int rn, int pg, int data) +{ + unsigned vsz = vec_full_reg_size(s); + tcg_gen_gvec_3_ool(vec_full_reg_offset(s, rd), + vec_full_reg_offset(s, rn), + pred_full_reg_offset(s, pg), + vsz, vsz, data, fn); +} + /* Invoke an out-of-line helper on 3 Zregs and a predicate. */ static void gen_gvec_ool_zzzp(DisasContext *s, gen_helper_gvec_4 *fn, int rd, int rn, int rm, int pg, int data) @@ -XXX,XX +XXX,XX @@ static bool do_zpz_ool(DisasContext *s, arg_rpr_esz *a, gen_helper_gvec_3 *fn) return false; } if (sve_access_check(s)) { - unsigned vsz = vec_full_reg_size(s); - tcg_gen_gvec_3_ool(vec_full_reg_offset(s, a->rd), - vec_full_reg_offset(s, a->rn), - pred_full_reg_offset(s, a->pg), - vsz, vsz, 0, fn); + gen_gvec_ool_zzp(s, fn, a->rd, a->rn, a->pg, 0); } return true; } @@ -XXX,XX +XXX,XX @@ static bool do_movz_zpz(DisasContext *s, int rd, int rn, int pg, }; if (sve_access_check(s)) { - unsigned vsz = vec_full_reg_size(s); - tcg_gen_gvec_3_ool(vec_full_reg_offset(s, rd), - vec_full_reg_offset(s, rn), - pred_full_reg_offset(s, pg), - vsz, vsz, invert, fns[esz]); + gen_gvec_ool_zzp(s, fns[esz], rd, rn, pg, invert); } return true; } @@ -XXX,XX +XXX,XX @@ static bool do_zpzi_ool(DisasContext *s, arg_rpri_esz *a, gen_helper_gvec_3 *fn) { if (sve_access_check(s)) { - unsigned vsz = vec_full_reg_size(s); - tcg_gen_gvec_3_ool(vec_full_reg_offset(s, a->rd), - vec_full_reg_offset(s, a->rn), - pred_full_reg_offset(s, a->pg), - vsz, vsz, a->imm, fn); + gen_gvec_ool_zzp(s, fn, a->rd, a->rn, a->pg, a->imm); } return true; } -- 2.20.1 From: Richard Henderson <richard.henderson@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Message-id: 20200815013145.539409-12-richard.henderson@linaro.org Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- target/arm/translate-sve.c | 53 +++++++++++++------------------------- 1 file changed, 18 insertions(+), 35 deletions(-) diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/translate-sve.c +++ b/target/arm/translate-sve.c @@ -XXX,XX +XXX,XX @@ static int pred_gvec_reg_size(DisasContext *s) return size_for_gvec(pred_full_reg_size(s)); } +/* Invoke an out-of-line helper on 3 Zregs. */ +static void gen_gvec_ool_zzz(DisasContext *s, gen_helper_gvec_3 *fn, + int rd, int rn, int rm, int data) +{ + unsigned vsz = vec_full_reg_size(s); + tcg_gen_gvec_3_ool(vec_full_reg_offset(s, rd), + vec_full_reg_offset(s, rn), + vec_full_reg_offset(s, rm), + vsz, vsz, data, fn); +} + /* Invoke an out-of-line helper on 2 Zregs and a predicate. */ static void gen_gvec_ool_zzp(DisasContext *s, gen_helper_gvec_3 *fn, int rd, int rn, int pg, int data) @@ -XXX,XX +XXX,XX @@ static bool do_zzw_ool(DisasContext *s, arg_rrr_esz *a, gen_helper_gvec_3 *fn) return false; } if (sve_access_check(s)) { - unsigned vsz = vec_full_reg_size(s); - tcg_gen_gvec_3_ool(vec_full_reg_offset(s, a->rd), - vec_full_reg_offset(s, a->rn), - vec_full_reg_offset(s, a->rm), - vsz, vsz, 0, fn); + gen_gvec_ool_zzz(s, fn, a->rd, a->rn, a->rm, 0); } return true; } @@ -XXX,XX +XXX,XX @@ static bool trans_RDVL(DisasContext *s, arg_RDVL *a) static bool do_adr(DisasContext *s, arg_rrri *a, gen_helper_gvec_3 *fn) { if (sve_access_check(s)) { - unsigned vsz = vec_full_reg_size(s); - tcg_gen_gvec_3_ool(vec_full_reg_offset(s, a->rd), - vec_full_reg_offset(s, a->rn), - vec_full_reg_offset(s, a->rm), - vsz, vsz, a->imm, fn); + gen_gvec_ool_zzz(s, fn, a->rd, a->rn, a->rm, a->imm); } return true; } @@ -XXX,XX +XXX,XX @@ static bool trans_FTSSEL(DisasContext *s, arg_rrr_esz *a) return false; } if (sve_access_check(s)) { - unsigned vsz = vec_full_reg_size(s); - tcg_gen_gvec_3_ool(vec_full_reg_offset(s, a->rd), - vec_full_reg_offset(s, a->rn), - vec_full_reg_offset(s, a->rm), - vsz, vsz, 0, fns[a->esz]); + gen_gvec_ool_zzz(s, fns[a->esz], a->rd, a->rn, a->rm, 0); } return true; } @@ -XXX,XX +XXX,XX @@ static bool trans_TBL(DisasContext *s, arg_rrr_esz *a) }; if (sve_access_check(s)) { - unsigned vsz = vec_full_reg_size(s); - tcg_gen_gvec_3_ool(vec_full_reg_offset(s, a->rd), - vec_full_reg_offset(s, a->rn), - vec_full_reg_offset(s, a->rm), - vsz, vsz, 0, fns[a->esz]); + gen_gvec_ool_zzz(s, fns[a->esz], a->rd, a->rn, a->rm, 0); } return true; } @@ -XXX,XX +XXX,XX @@ static bool do_zzz_data_ool(DisasContext *s, arg_rrr_esz *a, int data, gen_helper_gvec_3 *fn) { if (sve_access_check(s)) { - unsigned vsz = vec_full_reg_size(s); - tcg_gen_gvec_3_ool(vec_full_reg_offset(s, a->rd), - vec_full_reg_offset(s, a->rn), - vec_full_reg_offset(s, a->rm), - vsz, vsz, data, fn); + gen_gvec_ool_zzz(s, fn, a->rd, a->rn, a->rm, data); } return true; } @@ -XXX,XX +XXX,XX @@ static bool trans_DOT_zzz(DisasContext *s, arg_DOT_zzz *a) }; if (sve_access_check(s)) { - unsigned vsz = vec_full_reg_size(s); - tcg_gen_gvec_3_ool(vec_full_reg_offset(s, a->rd), - vec_full_reg_offset(s, a->rn), - vec_full_reg_offset(s, a->rm), - vsz, vsz, 0, fns[a->u][a->sz]); + gen_gvec_ool_zzz(s, fns[a->u][a->sz], a->rd, a->rn, a->rm, 0); } return true; } @@ -XXX,XX +XXX,XX @@ static bool trans_DOT_zzx(DisasContext *s, arg_DOT_zzx *a) }; if (sve_access_check(s)) { - unsigned vsz = vec_full_reg_size(s); - tcg_gen_gvec_3_ool(vec_full_reg_offset(s, a->rd), - vec_full_reg_offset(s, a->rn), - vec_full_reg_offset(s, a->rm), - vsz, vsz, a->index, fns[a->u][a->sz]); + gen_gvec_ool_zzz(s, fns[a->u][a->sz], a->rd, a->rn, a->rm, a->index); } return true; } -- 2.20.1 From: Richard Henderson <richard.henderson@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Message-id: 20200815013145.539409-13-richard.henderson@linaro.org Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- target/arm/translate-sve.c | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/translate-sve.c +++ b/target/arm/translate-sve.c @@ -XXX,XX +XXX,XX @@ static int pred_gvec_reg_size(DisasContext *s) return size_for_gvec(pred_full_reg_size(s)); } +/* Invoke an out-of-line helper on 2 Zregs. */ +static void gen_gvec_ool_zz(DisasContext *s, gen_helper_gvec_2 *fn, + int rd, int rn, int data) +{ + unsigned vsz = vec_full_reg_size(s); + tcg_gen_gvec_2_ool(vec_full_reg_offset(s, rd), + vec_full_reg_offset(s, rn), + vsz, vsz, data, fn); +} + /* Invoke an out-of-line helper on 3 Zregs. */ static void gen_gvec_ool_zzz(DisasContext *s, gen_helper_gvec_3 *fn, int rd, int rn, int rm, int data) @@ -XXX,XX +XXX,XX @@ static bool trans_FEXPA(DisasContext *s, arg_rr_esz *a) return false; } if (sve_access_check(s)) { - unsigned vsz = vec_full_reg_size(s); - tcg_gen_gvec_2_ool(vec_full_reg_offset(s, a->rd), - vec_full_reg_offset(s, a->rn), - vsz, vsz, 0, fns[a->esz]); + gen_gvec_ool_zz(s, fns[a->esz], a->rd, a->rn, 0); } return true; } @@ -XXX,XX +XXX,XX @@ static bool trans_REV_v(DisasContext *s, arg_rr_esz *a) }; if (sve_access_check(s)) { - unsigned vsz = vec_full_reg_size(s); - tcg_gen_gvec_2_ool(vec_full_reg_offset(s, a->rd), - vec_full_reg_offset(s, a->rn), - vsz, vsz, 0, fns[a->esz]); + gen_gvec_ool_zz(s, fns[a->esz], a->rd, a->rn, 0); } return true; } -- 2.20.1 From: Richard Henderson <richard.henderson@linaro.org> Rather than require the user to fill in the immediate (shl or shr), create full formats that include the immediate. Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Message-id: 20200815013145.539409-14-richard.henderson@linaro.org Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- target/arm/sve.decode | 35 ++++++++++++++++------------------- 1 file changed, 16 insertions(+), 19 deletions(-) diff --git a/target/arm/sve.decode b/target/arm/sve.decode index XXXXXXX..XXXXXXX 100644 --- a/target/arm/sve.decode +++ b/target/arm/sve.decode @@ -XXX,XX +XXX,XX @@ @rd_rn_i6 ........ ... rn:5 ..... imm:s6 rd:5 &rri # Two register operand, one immediate operand, with predicate, -# element size encoded as TSZHL. User must fill in imm. -@rdn_pg_tszimm ........ .. ... ... ... pg:3 ..... rd:5 \ - &rpri_esz rn=%reg_movprfx esz=%tszimm_esz +# element size encoded as TSZHL. +@rdn_pg_tszimm_shl ........ .. ... ... ... pg:3 ..... rd:5 \ + &rpri_esz rn=%reg_movprfx esz=%tszimm_esz imm=%tszimm_shl +@rdn_pg_tszimm_shr ........ .. ... ... ... pg:3 ..... rd:5 \ + &rpri_esz rn=%reg_movprfx esz=%tszimm_esz imm=%tszimm_shr # Similarly without predicate. -@rd_rn_tszimm ........ .. ... ... ...... rn:5 rd:5 \ - &rri_esz esz=%tszimm16_esz +@rd_rn_tszimm_shl ........ .. ... ... ...... rn:5 rd:5 \ + &rri_esz esz=%tszimm16_esz imm=%tszimm16_shl +@rd_rn_tszimm_shr ........ .. ... ... ...... rn:5 rd:5 \ + &rri_esz esz=%tszimm16_esz imm=%tszimm16_shr # Two register operand, one immediate operand, with 4-bit predicate. # User must fill in imm. @@ -XXX,XX +XXX,XX @@ UMINV 00000100 .. 001 011 001 ... ..... ..... @rd_pg_rn ### SVE Shift by Immediate - Predicated Group # SVE bitwise shift by immediate (predicated) -ASR_zpzi 00000100 .. 000 000 100 ... .. ... ..... \ - @rdn_pg_tszimm imm=%tszimm_shr -LSR_zpzi 00000100 .. 000 001 100 ... .. ... ..... \ - @rdn_pg_tszimm imm=%tszimm_shr -LSL_zpzi 00000100 .. 000 011 100 ... .. ... ..... \ - @rdn_pg_tszimm imm=%tszimm_shl -ASRD 00000100 .. 000 100 100 ... .. ... ..... \ - @rdn_pg_tszimm imm=%tszimm_shr +ASR_zpzi 00000100 .. 000 000 100 ... .. ... ..... @rdn_pg_tszimm_shr +LSR_zpzi 00000100 .. 000 001 100 ... .. ... ..... @rdn_pg_tszimm_shr +LSL_zpzi 00000100 .. 000 011 100 ... .. ... ..... @rdn_pg_tszimm_shl +ASRD 00000100 .. 000 100 100 ... .. ... ..... @rdn_pg_tszimm_shr # SVE bitwise shift by vector (predicated) ASR_zpzz 00000100 .. 010 000 100 ... ..... ..... @rdn_pg_rm @@ -XXX,XX +XXX,XX @@ RDVL 00000100 101 11111 01010 imm:s6 rd:5 ### SVE Bitwise Shift - Unpredicated Group # SVE bitwise shift by immediate (unpredicated) -ASR_zzi 00000100 .. 1 ..... 1001 00 ..... ..... \ - @rd_rn_tszimm imm=%tszimm16_shr -LSR_zzi 00000100 .. 1 ..... 1001 01 ..... ..... \ - @rd_rn_tszimm imm=%tszimm16_shr -LSL_zzi 00000100 .. 1 ..... 1001 11 ..... ..... \ - @rd_rn_tszimm imm=%tszimm16_shl +ASR_zzi 00000100 .. 1 ..... 1001 00 ..... ..... @rd_rn_tszimm_shr +LSR_zzi 00000100 .. 1 ..... 1001 01 ..... ..... @rd_rn_tszimm_shr +LSL_zzi 00000100 .. 1 ..... 1001 11 ..... ..... @rd_rn_tszimm_shl # SVE bitwise shift by wide elements (unpredicated) # Note esz != 3 -- 2.20.1 From: Richard Henderson <richard.henderson@linaro.org> Unify add/sub helpers and add a parameter for rounding. This will allow saturating non-rounding to reuse this code. Signed-off-by: Richard Henderson <richard.henderson@linaro.org> [PMM: fixed accidental use of '=' rather than '+=' in do_sqrdmlah_s] Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Message-id: 20200815013145.539409-15-richard.henderson@linaro.org Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- target/arm/vec_helper.c | 80 +++++++++++++++-------------------------- 1 file changed, 29 insertions(+), 51 deletions(-) diff --git a/target/arm/vec_helper.c b/target/arm/vec_helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/vec_helper.c +++ b/target/arm/vec_helper.c @@ -XXX,XX +XXX,XX @@ #endif /* Signed saturating rounding doubling multiply-accumulate high half, 16-bit */ -static int16_t inl_qrdmlah_s16(int16_t src1, int16_t src2, - int16_t src3, uint32_t *sat) +static int16_t do_sqrdmlah_h(int16_t src1, int16_t src2, int16_t src3, + bool neg, bool round, uint32_t *sat) { - /* Simplify: + /* + * Simplify: * = ((a3 << 16) + ((e1 * e2) << 1) + (1 << 15)) >> 16 * = ((a3 << 15) + (e1 * e2) + (1 << 14)) >> 15 */ int32_t ret = (int32_t)src1 * src2; - ret = ((int32_t)src3 << 15) + ret + (1 << 14); + if (neg) { + ret = -ret; + } + ret += ((int32_t)src3 << 15) + (round << 14); ret >>= 15; + if (ret != (int16_t)ret) { *sat = 1; - ret = (ret < 0 ? -0x8000 : 0x7fff); + ret = (ret < 0 ? INT16_MIN : INT16_MAX); } return ret; } @@ -XXX,XX +XXX,XX @@ uint32_t HELPER(neon_qrdmlah_s16)(CPUARMState *env, uint32_t src1, uint32_t src2, uint32_t src3) { uint32_t *sat = &env->vfp.qc[0]; - uint16_t e1 = inl_qrdmlah_s16(src1, src2, src3, sat); - uint16_t e2 = inl_qrdmlah_s16(src1 >> 16, src2 >> 16, src3 >> 16, sat); + uint16_t e1 = do_sqrdmlah_h(src1, src2, src3, false, true, sat); + uint16_t e2 = do_sqrdmlah_h(src1 >> 16, src2 >> 16, src3 >> 16, + false, true, sat); return deposit32(e1, 16, 16, e2); } @@ -XXX,XX +XXX,XX @@ void HELPER(gvec_qrdmlah_s16)(void *vd, void *vn, void *vm, uintptr_t i; for (i = 0; i < opr_sz / 2; ++i) { - d[i] = inl_qrdmlah_s16(n[i], m[i], d[i], vq); + d[i] = do_sqrdmlah_h(n[i], m[i], d[i], false, true, vq); } clear_tail(d, opr_sz, simd_maxsz(desc)); } -/* Signed saturating rounding doubling multiply-subtract high half, 16-bit */ -static int16_t inl_qrdmlsh_s16(int16_t src1, int16_t src2, - int16_t src3, uint32_t *sat) -{ - /* Similarly, using subtraction: - * = ((a3 << 16) - ((e1 * e2) << 1) + (1 << 15)) >> 16 - * = ((a3 << 15) - (e1 * e2) + (1 << 14)) >> 15 - */ - int32_t ret = (int32_t)src1 * src2; - ret = ((int32_t)src3 << 15) - ret + (1 << 14); - ret >>= 15; - if (ret != (int16_t)ret) { - *sat = 1; - ret = (ret < 0 ? -0x8000 : 0x7fff); - } - return ret; -} - uint32_t HELPER(neon_qrdmlsh_s16)(CPUARMState *env, uint32_t src1, uint32_t src2, uint32_t src3) { uint32_t *sat = &env->vfp.qc[0]; - uint16_t e1 = inl_qrdmlsh_s16(src1, src2, src3, sat); - uint16_t e2 = inl_qrdmlsh_s16(src1 >> 16, src2 >> 16, src3 >> 16, sat); + uint16_t e1 = do_sqrdmlah_h(src1, src2, src3, true, true, sat); + uint16_t e2 = do_sqrdmlah_h(src1 >> 16, src2 >> 16, src3 >> 16, + true, true, sat); return deposit32(e1, 16, 16, e2); } @@ -XXX,XX +XXX,XX @@ void HELPER(gvec_qrdmlsh_s16)(void *vd, void *vn, void *vm, uintptr_t i; for (i = 0; i < opr_sz / 2; ++i) { - d[i] = inl_qrdmlsh_s16(n[i], m[i], d[i], vq); + d[i] = do_sqrdmlah_h(n[i], m[i], d[i], true, true, vq); } clear_tail(d, opr_sz, simd_maxsz(desc)); } /* Signed saturating rounding doubling multiply-accumulate high half, 32-bit */ -static int32_t inl_qrdmlah_s32(int32_t src1, int32_t src2, - int32_t src3, uint32_t *sat) +static int32_t do_sqrdmlah_s(int32_t src1, int32_t src2, int32_t src3, + bool neg, bool round, uint32_t *sat) { /* Simplify similarly to int_qrdmlah_s16 above. */ int64_t ret = (int64_t)src1 * src2; - ret = ((int64_t)src3 << 31) + ret + (1 << 30); + if (neg) { + ret = -ret; + } + ret += ((int64_t)src3 << 31) + (round << 30); ret >>= 31; + if (ret != (int32_t)ret) { *sat = 1; ret = (ret < 0 ? INT32_MIN : INT32_MAX); @@ -XXX,XX +XXX,XX @@ uint32_t HELPER(neon_qrdmlah_s32)(CPUARMState *env, int32_t src1, int32_t src2, int32_t src3) { uint32_t *sat = &env->vfp.qc[0]; - return inl_qrdmlah_s32(src1, src2, src3, sat); + return do_sqrdmlah_s(src1, src2, src3, false, true, sat); } void HELPER(gvec_qrdmlah_s32)(void *vd, void *vn, void *vm, @@ -XXX,XX +XXX,XX @@ void HELPER(gvec_qrdmlah_s32)(void *vd, void *vn, void *vm, uintptr_t i; for (i = 0; i < opr_sz / 4; ++i) { - d[i] = inl_qrdmlah_s32(n[i], m[i], d[i], vq); + d[i] = do_sqrdmlah_s(n[i], m[i], d[i], false, true, vq); } clear_tail(d, opr_sz, simd_maxsz(desc)); } -/* Signed saturating rounding doubling multiply-subtract high half, 32-bit */ -static int32_t inl_qrdmlsh_s32(int32_t src1, int32_t src2, - int32_t src3, uint32_t *sat) -{ - /* Simplify similarly to int_qrdmlsh_s16 above. */ - int64_t ret = (int64_t)src1 * src2; - ret = ((int64_t)src3 << 31) - ret + (1 << 30); - ret >>= 31; - if (ret != (int32_t)ret) { - *sat = 1; - ret = (ret < 0 ? INT32_MIN : INT32_MAX); - } - return ret; -} - uint32_t HELPER(neon_qrdmlsh_s32)(CPUARMState *env, int32_t src1, int32_t src2, int32_t src3) { uint32_t *sat = &env->vfp.qc[0]; - return inl_qrdmlsh_s32(src1, src2, src3, sat); + return do_sqrdmlah_s(src1, src2, src3, true, true, sat); } void HELPER(gvec_qrdmlsh_s32)(void *vd, void *vn, void *vm, @@ -XXX,XX +XXX,XX @@ void HELPER(gvec_qrdmlsh_s32)(void *vd, void *vn, void *vm, uintptr_t i; for (i = 0; i < opr_sz / 4; ++i) { - d[i] = inl_qrdmlsh_s32(n[i], m[i], d[i], vq); + d[i] = do_sqrdmlah_s(n[i], m[i], d[i], true, true, vq); } clear_tail(d, opr_sz, simd_maxsz(desc)); } -- 2.20.1 From: Richard Henderson <richard.henderson@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Message-id: 20200815013145.539409-19-richard.henderson@linaro.org Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- target/arm/helper.h | 4 ++++ target/arm/translate-a64.c | 16 ++++++++++++++++ target/arm/vec_helper.c | 29 +++++++++++++++++++++++++---- 3 files changed, 45 insertions(+), 4 deletions(-) diff --git a/target/arm/helper.h b/target/arm/helper.h index XXXXXXX..XXXXXXX 100644 --- a/target/arm/helper.h +++ b/target/arm/helper.h @@ -XXX,XX +XXX,XX @@ DEF_HELPER_FLAGS_4(gvec_uaba_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) DEF_HELPER_FLAGS_4(gvec_uaba_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) DEF_HELPER_FLAGS_4(gvec_uaba_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(gvec_mul_idx_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(gvec_mul_idx_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_4(gvec_mul_idx_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) + #ifdef TARGET_AARCH64 #include "helper-a64.h" #include "helper-sve.h" diff --git a/target/arm/translate-a64.c b/target/arm/translate-a64.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/translate-a64.c +++ b/target/arm/translate-a64.c @@ -XXX,XX +XXX,XX @@ static void disas_simd_indexed(DisasContext *s, uint32_t insn) data, gen_helper_gvec_fmlal_idx_a64); } return; + + case 0x08: /* MUL */ + if (!is_long && !is_scalar) { + static gen_helper_gvec_3 * const fns[3] = { + gen_helper_gvec_mul_idx_h, + gen_helper_gvec_mul_idx_s, + gen_helper_gvec_mul_idx_d, + }; + tcg_gen_gvec_3_ool(vec_full_reg_offset(s, rd), + vec_full_reg_offset(s, rn), + vec_full_reg_offset(s, rm), + is_q ? 16 : 8, vec_full_reg_size(s), + index, fns[size - 1]); + return; + } + break; } if (size == 3) { diff --git a/target/arm/vec_helper.c b/target/arm/vec_helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/vec_helper.c +++ b/target/arm/vec_helper.c @@ -XXX,XX +XXX,XX @@ DO_3OP(gvec_rsqrts_d, helper_rsqrtsf_f64, float64) */ #define DO_MUL_IDX(NAME, TYPE, H) \ +void HELPER(NAME)(void *vd, void *vn, void *vm, uint32_t desc) \ +{ \ + intptr_t i, j, oprsz = simd_oprsz(desc), segment = 16 / sizeof(TYPE); \ + intptr_t idx = simd_data(desc); \ + TYPE *d = vd, *n = vn, *m = vm; \ + for (i = 0; i < oprsz / sizeof(TYPE); i += segment) { \ + TYPE mm = m[H(i + idx)]; \ + for (j = 0; j < segment; j++) { \ + d[i + j] = n[i + j] * mm; \ + } \ + } \ + clear_tail(d, oprsz, simd_maxsz(desc)); \ +} + +DO_MUL_IDX(gvec_mul_idx_h, uint16_t, H2) +DO_MUL_IDX(gvec_mul_idx_s, uint32_t, H4) +DO_MUL_IDX(gvec_mul_idx_d, uint64_t, ) + +#undef DO_MUL_IDX + +#define DO_FMUL_IDX(NAME, TYPE, H) \ void HELPER(NAME)(void *vd, void *vn, void *vm, void *stat, uint32_t desc) \ { \ intptr_t i, j, oprsz = simd_oprsz(desc), segment = 16 / sizeof(TYPE); \ @@ -XXX,XX +XXX,XX @@ void HELPER(NAME)(void *vd, void *vn, void *vm, void *stat, uint32_t desc) \ clear_tail(d, oprsz, simd_maxsz(desc)); \ } -DO_MUL_IDX(gvec_fmul_idx_h, float16, H2) -DO_MUL_IDX(gvec_fmul_idx_s, float32, H4) -DO_MUL_IDX(gvec_fmul_idx_d, float64, ) +DO_FMUL_IDX(gvec_fmul_idx_h, float16, H2) +DO_FMUL_IDX(gvec_fmul_idx_s, float32, H4) +DO_FMUL_IDX(gvec_fmul_idx_d, float64, ) -#undef DO_MUL_IDX +#undef DO_FMUL_IDX #define DO_FMLA_IDX(NAME, TYPE, H) \ void HELPER(NAME)(void *vd, void *vn, void *vm, void *va, \ -- 2.20.1 From: Richard Henderson <richard.henderson@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Message-id: 20200815013145.539409-20-richard.henderson@linaro.org Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- target/arm/helper.h | 14 ++++++++++++++ target/arm/translate-a64.c | 34 ++++++++++++++++++++++++++++++++++ target/arm/vec_helper.c | 25 +++++++++++++++++++++++++ 3 files changed, 73 insertions(+) diff --git a/target/arm/helper.h b/target/arm/helper.h index XXXXXXX..XXXXXXX 100644 --- a/target/arm/helper.h +++ b/target/arm/helper.h @@ -XXX,XX +XXX,XX @@ DEF_HELPER_FLAGS_4(gvec_mul_idx_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) DEF_HELPER_FLAGS_4(gvec_mul_idx_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) DEF_HELPER_FLAGS_4(gvec_mul_idx_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(gvec_mla_idx_h, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(gvec_mla_idx_s, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(gvec_mla_idx_d, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) + +DEF_HELPER_FLAGS_5(gvec_mls_idx_h, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(gvec_mls_idx_s, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(gvec_mls_idx_d, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) + #ifdef TARGET_AARCH64 #include "helper-a64.h" #include "helper-sve.h" diff --git a/target/arm/translate-a64.c b/target/arm/translate-a64.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/translate-a64.c +++ b/target/arm/translate-a64.c @@ -XXX,XX +XXX,XX @@ static void disas_simd_indexed(DisasContext *s, uint32_t insn) return; } break; + + case 0x10: /* MLA */ + if (!is_long && !is_scalar) { + static gen_helper_gvec_4 * const fns[3] = { + gen_helper_gvec_mla_idx_h, + gen_helper_gvec_mla_idx_s, + gen_helper_gvec_mla_idx_d, + }; + tcg_gen_gvec_4_ool(vec_full_reg_offset(s, rd), + vec_full_reg_offset(s, rn), + vec_full_reg_offset(s, rm), + vec_full_reg_offset(s, rd), + is_q ? 16 : 8, vec_full_reg_size(s), + index, fns[size - 1]); + return; + } + break; + + case 0x14: /* MLS */ + if (!is_long && !is_scalar) { + static gen_helper_gvec_4 * const fns[3] = { + gen_helper_gvec_mls_idx_h, + gen_helper_gvec_mls_idx_s, + gen_helper_gvec_mls_idx_d, + }; + tcg_gen_gvec_4_ool(vec_full_reg_offset(s, rd), + vec_full_reg_offset(s, rn), + vec_full_reg_offset(s, rm), + vec_full_reg_offset(s, rd), + is_q ? 16 : 8, vec_full_reg_size(s), + index, fns[size - 1]); + return; + } + break; } if (size == 3) { diff --git a/target/arm/vec_helper.c b/target/arm/vec_helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/vec_helper.c +++ b/target/arm/vec_helper.c @@ -XXX,XX +XXX,XX @@ DO_MUL_IDX(gvec_mul_idx_d, uint64_t, ) #undef DO_MUL_IDX +#define DO_MLA_IDX(NAME, TYPE, OP, H) \ +void HELPER(NAME)(void *vd, void *vn, void *vm, void *va, uint32_t desc) \ +{ \ + intptr_t i, j, oprsz = simd_oprsz(desc), segment = 16 / sizeof(TYPE); \ + intptr_t idx = simd_data(desc); \ + TYPE *d = vd, *n = vn, *m = vm, *a = va; \ + for (i = 0; i < oprsz / sizeof(TYPE); i += segment) { \ + TYPE mm = m[H(i + idx)]; \ + for (j = 0; j < segment; j++) { \ + d[i + j] = a[i + j] OP n[i + j] * mm; \ + } \ + } \ + clear_tail(d, oprsz, simd_maxsz(desc)); \ +} + +DO_MLA_IDX(gvec_mla_idx_h, uint16_t, +, H2) +DO_MLA_IDX(gvec_mla_idx_s, uint32_t, +, H4) +DO_MLA_IDX(gvec_mla_idx_d, uint64_t, +, ) + +DO_MLA_IDX(gvec_mls_idx_h, uint16_t, -, H2) +DO_MLA_IDX(gvec_mls_idx_s, uint32_t, -, H4) +DO_MLA_IDX(gvec_mls_idx_d, uint64_t, -, ) + +#undef DO_MLA_IDX + #define DO_FMUL_IDX(NAME, TYPE, H) \ void HELPER(NAME)(void *vd, void *vn, void *vm, void *stat, uint32_t desc) \ { \ -- 2.20.1 From: Richard Henderson <richard.henderson@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Message-id: 20200815013145.539409-21-richard.henderson@linaro.org Signed-off-by: Peter Maydell <peter.maydell@linaro.org> --- target/arm/helper.h | 10 ++++++++ target/arm/translate-a64.c | 33 ++++++++++++++++++-------- target/arm/vec_helper.c | 48 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 81 insertions(+), 10 deletions(-) diff --git a/target/arm/helper.h b/target/arm/helper.h index XXXXXXX..XXXXXXX 100644 --- a/target/arm/helper.h +++ b/target/arm/helper.h @@ -XXX,XX +XXX,XX @@ DEF_HELPER_FLAGS_5(gvec_mls_idx_s, TCG_CALL_NO_RWG, DEF_HELPER_FLAGS_5(gvec_mls_idx_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(neon_sqdmulh_h, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(neon_sqdmulh_s, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) + +DEF_HELPER_FLAGS_5(neon_sqrdmulh_h, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(neon_sqrdmulh_s, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) + #ifdef TARGET_AARCH64 #include "helper-a64.h" #include "helper-sve.h" diff --git a/target/arm/translate-a64.c b/target/arm/translate-a64.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/translate-a64.c +++ b/target/arm/translate-a64.c @@ -XXX,XX +XXX,XX @@ static void gen_gvec_op3_fpst(DisasContext *s, bool is_q, int rd, int rn, tcg_temp_free_ptr(fpst); } +/* Expand a 3-operand + qc + operation using an out-of-line helper. */ +static void gen_gvec_op3_qc(DisasContext *s, bool is_q, int rd, int rn, + int rm, gen_helper_gvec_3_ptr *fn) +{ + TCGv_ptr qc_ptr = tcg_temp_new_ptr(); + + tcg_gen_addi_ptr(qc_ptr, cpu_env, offsetof(CPUARMState, vfp.qc)); + tcg_gen_gvec_3_ptr(vec_full_reg_offset(s, rd), + vec_full_reg_offset(s, rn), + vec_full_reg_offset(s, rm), qc_ptr, + is_q ? 16 : 8, vec_full_reg_size(s), 0, fn); + tcg_temp_free_ptr(qc_ptr); +} + /* Set ZF and NF based on a 64 bit result. This is alas fiddlier * than the 32 bit equivalent. */ @@ -XXX,XX +XXX,XX @@ static void disas_simd_3same_int(DisasContext *s, uint32_t insn) gen_gvec_fn3(s, is_q, rd, rn, rm, gen_gvec_mla, size); } return; + case 0x16: /* SQDMULH, SQRDMULH */ + { + static gen_helper_gvec_3_ptr * const fns[2][2] = { + { gen_helper_neon_sqdmulh_h, gen_helper_neon_sqrdmulh_h }, + { gen_helper_neon_sqdmulh_s, gen_helper_neon_sqrdmulh_s }, + }; + gen_gvec_op3_qc(s, is_q, rd, rn, rm, fns[size - 1][u]); + } + return; case 0x11: if (!u) { /* CMTST */ gen_gvec_fn3(s, is_q, rd, rn, rm, gen_gvec_cmtst, size); @@ -XXX,XX +XXX,XX @@ static void disas_simd_3same_int(DisasContext *s, uint32_t insn) genenvfn = fns[size][u]; break; } - case 0x16: /* SQDMULH, SQRDMULH */ - { - static NeonGenTwoOpEnvFn * const fns[2][2] = { - { gen_helper_neon_qdmulh_s16, gen_helper_neon_qrdmulh_s16 }, - { gen_helper_neon_qdmulh_s32, gen_helper_neon_qrdmulh_s32 }, - }; - assert(size == 1 || size == 2); - genenvfn = fns[size - 1][u]; - break; - } default: g_assert_not_reached(); } diff --git a/target/arm/vec_helper.c b/target/arm/vec_helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/vec_helper.c +++ b/target/arm/vec_helper.c @@ -XXX,XX +XXX,XX @@ void HELPER(gvec_qrdmlsh_s16)(void *vd, void *vn, void *vm, clear_tail(d, opr_sz, simd_maxsz(desc)); } +void HELPER(neon_sqdmulh_h)(void *vd, void *vn, void *vm, + void *vq, uint32_t desc) +{ + intptr_t i, opr_sz = simd_oprsz(desc); + int16_t *d = vd, *n = vn, *m = vm; + + for (i = 0; i < opr_sz / 2; ++i) { + d[i] = do_sqrdmlah_h(n[i], m[i], 0, false, false, vq); + } + clear_tail(d, opr_sz, simd_maxsz(desc)); +} + +void HELPER(neon_sqrdmulh_h)(void *vd, void *vn, void *vm, + void *vq, uint32_t desc) +{ + intptr_t i, opr_sz = simd_oprsz(desc); + int16_t *d = vd, *n = vn, *m = vm; + + for (i = 0; i < opr_sz / 2; ++i) { + d[i] = do_sqrdmlah_h(n[i], m[i], 0, false, true, vq); + } + clear_tail(d, opr_sz, simd_maxsz(desc)); +} + /* Signed saturating rounding doubling multiply-accumulate high half, 32-bit */ static int32_t do_sqrdmlah_s(int32_t src1, int32_t src2, int32_t src3, bool neg, bool round, uint32_t *sat) @@ -XXX,XX +XXX,XX @@ void HELPER(gvec_qrdmlsh_s32)(void *vd, void *vn, void *vm, clear_tail(d, opr_sz, simd_maxsz(desc)); } +void HELPER(neon_sqdmulh_s)(void *vd, void *vn, void *vm, + void *vq, uint32_t desc) +{ + intptr_t i, opr_sz = simd_oprsz(desc); + int32_t *d = vd, *n = vn, *m = vm; + + for (i = 0; i < opr_sz / 4; ++i) { + d[i] = do_sqrdmlah_s(n[i], m[i], 0, false, false, vq); + } + clear_tail(d, opr_sz, simd_maxsz(desc)); +} + +void HELPER(neon_sqrdmulh_s)(void *vd, void *vn, void *vm, + void *vq, uint32_t desc) +{ + intptr_t i, opr_sz = simd_oprsz(desc); + int32_t *d = vd, *n = vn, *m = vm; + + for (i = 0; i < opr_sz / 4; ++i) { + d[i] = do_sqrdmlah_s(n[i], m[i], 0, false, true, vq); + } + clear_tail(d, opr_sz, simd_maxsz(desc)); +} + /* Integer 8 and 16-bit dot-product. * * Note that for the loops herein, host endianness does not matter -- 2.20.1