Based-on: 20260429222445.26301-1-richard.henderson@linaro.org ("[PULL 00/48] fpu patch queue") Changes for v2: - Merge 48 patches - Applied r-b - Split partsN(muladd_scalbn) into partsN(muladd) + partsN(scalbn). - Improvements to partsN(muladd). - Export partsN(scalbn). - Introduce exp_scalbn. - Handle exp overflow during rounding. r~ Richard Henderson (40): fpu: Inline float8_e4m3_pack_raw to single caller fpu: Inline float8_e5m2_pack_raw into single caller fpu: Inline float16_pack_raw into callers fpu: Inline bfloat16_pack_raw into callers fpu: Inline float32_pack_raw into callers fpu: Inline float64_pack_raw into callers fpu: Mark unpack_raw64 QEMU_ALWAYS_INLINE fpu: Mark pack_raw64 QEMU_ALWAYS_INLINE fpu: Split FloatParts{64,128} to softfloat-parts.h fpu: Export FloatFmt structures fpu: Export unpack_canonical and round_pack_canonical routines fpu: Return struct from parts{64,128}_default_nan fpu: Return struct from parts{64,128}_silence_nan fpu: Return struct from parts{64,128}_return_nan fpu: Sink exp_bias adjustment in float64r32_pack_raw fpu: Return struct from parts{64,128}_pick_nan fpu: Return struct from parts{64,128}_div fpu: Return struct from parts{64,128}_round_to_int fpu: Use parts64_round_to_int in parts_s390_divide_to_integer fpu: Export default_nan, pick_nan, return_nan routines fpu: Introduce parts64_round_to_fmt fpu: Return struct from parts{64,128}_pick_nan_muladd fpu: Introduce record_denormals_used fpu: Split scalbn from partsN(muladd_scalbn) fpu: Reorganize partsN(muladd) fpu: Return struct from parts{64,128}_muladd fpu: Drop QEMU_FLATTEN from muladd routines fpu: Export parts{64,128}_compare fpu: Return struct from parts{64,128}_mul fpu: Hoist nan check in partsN_addsub fpu: Return struct from parts{64,128}_addsub fpu: Simplify 0 +/- N case in parts_addsub target/s390x: Move float{32,64}_s390_divide_to_integer target/arm: Use FloatParts64 in bfdotadd_ebf target/arm: Drop oddstatus from is_ebf and bfdotadd_ebf target/arm: Use FloatParts64 in f16_dotadd fpu: Return struct from parts{64,128}_scalbn host-utils: Introduce signed saturation primitives fpu: Saturate the exponent in uncanon_normal fpu: Introduce exp_scalbn include/fpu/softfloat-parts.h | 229 +++++++++++ include/fpu/softfloat.h | 11 - include/qemu/host-utils.h | 60 +++ target/arm/tcg/vec_internal.h | 12 +- accel/tcg/tcg-runtime-gvec.c | 20 +- fpu/softfloat.c | 689 +++++++++++---------------------- target/arm/tcg/sme_helper.c | 102 ++--- target/arm/tcg/vec_helper.c | 127 +++--- target/s390x/tcg/fpu_helper.c | 135 +++++++ fpu/softfloat-parts.c.inc | 463 ++++++++++------------ fpu/softfloat-specialize.c.inc | 42 +- 11 files changed, 1005 insertions(+), 885 deletions(-) create mode 100644 include/fpu/softfloat-parts.h -- 2.43.0 Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- fpu/softfloat.c | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/fpu/softfloat.c b/fpu/softfloat.c index XXXXXXX..XXXXXXX 100644 --- a/fpu/softfloat.c +++ b/fpu/softfloat.c @@ -XXX,XX +XXX,XX @@ static uint64_t pack_raw64(const FloatParts64 *p, const FloatFmt *fmt) return ret; } -static float8_e4m3 QEMU_FLATTEN float8_e4m3_pack_raw(const FloatParts64 *p) -{ - return pack_raw64(p, &float8_e4m3_params); -} - static float8_e5m2 QEMU_FLATTEN float8_e5m2_pack_raw(const FloatParts64 *p) { return pack_raw64(p, &float8_e5m2_params); @@ -XXX,XX +XXX,XX @@ static float8_e4m3 float8_e4m3_round_pack_canonical(FloatParts64 *p, bool saturate) { parts64_uncanon(p, s, &float8_e4m3_params, saturate); - return float8_e4m3_pack_raw(p); + return pack_raw64(p, &float8_e4m3_params); } static float8_e5m2 float8_e5m2_round_pack_canonical(FloatParts64 *p, -- 2.43.0 Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- fpu/softfloat.c | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/fpu/softfloat.c b/fpu/softfloat.c index XXXXXXX..XXXXXXX 100644 --- a/fpu/softfloat.c +++ b/fpu/softfloat.c @@ -XXX,XX +XXX,XX @@ static uint64_t pack_raw64(const FloatParts64 *p, const FloatFmt *fmt) return ret; } -static float8_e5m2 QEMU_FLATTEN float8_e5m2_pack_raw(const FloatParts64 *p) -{ - return pack_raw64(p, &float8_e5m2_params); -} - static float16 QEMU_FLATTEN float16_pack_raw(const FloatParts64 *p) { return make_float16(pack_raw64(p, &float16_params)); @@ -XXX,XX +XXX,XX @@ static float8_e5m2 float8_e5m2_round_pack_canonical(FloatParts64 *p, bool saturate) { parts64_uncanon(p, s, &float8_e5m2_params, saturate); - return float8_e5m2_pack_raw(p); + return pack_raw64(p, &float8_e5m2_params); } static float16 float16a_round_pack_canonical(FloatParts64 *p, -- 2.43.0 Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- fpu/softfloat.c | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/fpu/softfloat.c b/fpu/softfloat.c index XXXXXXX..XXXXXXX 100644 --- a/fpu/softfloat.c +++ b/fpu/softfloat.c @@ -XXX,XX +XXX,XX @@ static uint64_t pack_raw64(const FloatParts64 *p, const FloatFmt *fmt) return ret; } -static float16 QEMU_FLATTEN float16_pack_raw(const FloatParts64 *p) -{ - return make_float16(pack_raw64(p, &float16_params)); -} - static bfloat16 QEMU_FLATTEN bfloat16_pack_raw(const FloatParts64 *p) { return pack_raw64(p, &bfloat16_params); @@ -XXX,XX +XXX,XX @@ static float16 float16a_round_pack_canonical(FloatParts64 *p, const FloatFmt *params) { parts64_uncanon(p, s, params, false); - return float16_pack_raw(p); + return pack_raw64(p, &float16_params); } static float16 float16_round_pack_canonical(FloatParts64 *p, @@ -XXX,XX +XXX,XX @@ float16_muladd_scalbn(float16 a, float16 b, float16 c, if ((flags & float_muladd_negate_result) && !is_nan(pr->cls)) { pr->sign ^= 1; } - return float16_pack_raw(pr); + return pack_raw64(pr, &float16_params); } float16 float16_muladd(float16 a, float16 b, float16 c, @@ -XXX,XX +XXX,XX @@ float16 float16_default_nan(float_status *status) parts64_default_nan(&p, status); p.frac >>= float16_params.frac_shift; - return float16_pack_raw(&p); + return pack_raw64(&p, &float16_params); } float32 float32_default_nan(float_status *status) @@ -XXX,XX +XXX,XX @@ float16 float16_silence_nan(float16 a, float_status *status) p.frac <<= float16_params.frac_shift; parts64_silence_nan(&p, status); p.frac >>= float16_params.frac_shift; - return float16_pack_raw(&p); + return pack_raw64(&p, &float16_params); } float32 float32_silence_nan(float32 a, float_status *status) -- 2.43.0 Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- fpu/softfloat.c | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/fpu/softfloat.c b/fpu/softfloat.c index XXXXXXX..XXXXXXX 100644 --- a/fpu/softfloat.c +++ b/fpu/softfloat.c @@ -XXX,XX +XXX,XX @@ static uint64_t pack_raw64(const FloatParts64 *p, const FloatFmt *fmt) return ret; } -static bfloat16 QEMU_FLATTEN bfloat16_pack_raw(const FloatParts64 *p) -{ - return pack_raw64(p, &bfloat16_params); -} - static float32 QEMU_FLATTEN float32_pack_raw(const FloatParts64 *p) { return make_float32(pack_raw64(p, &float32_params)); @@ -XXX,XX +XXX,XX @@ static bfloat16 bfloat16_round_pack_canonical(FloatParts64 *p, float_status *s) { parts64_uncanon(p, s, &bfloat16_params, false); - return bfloat16_pack_raw(p); + return pack_raw64(p, &bfloat16_params); } static FloatParts64 float32_unpack_canonical(float32 f, float_status *s) @@ -XXX,XX +XXX,XX @@ bfloat16 QEMU_FLATTEN bfloat16_muladd(bfloat16 a, bfloat16 b, bfloat16 c, if ((flags & float_muladd_negate_result) && !is_nan(pr->cls)) { pr->sign ^= 1; } - return bfloat16_pack_raw(pr); + return pack_raw64(pr, &bfloat16_params); } float128 QEMU_FLATTEN float128_muladd(float128 a, float128 b, float128 c, @@ -XXX,XX +XXX,XX @@ bfloat16 bfloat16_default_nan(float_status *status) parts64_default_nan(&p, status); p.frac >>= bfloat16_params.frac_shift; - return bfloat16_pack_raw(&p); + return pack_raw64(&p, &bfloat16_params); } /*---------------------------------------------------------------------------- @@ -XXX,XX +XXX,XX @@ bfloat16 bfloat16_silence_nan(bfloat16 a, float_status *status) p.frac <<= bfloat16_params.frac_shift; parts64_silence_nan(&p, status); p.frac >>= bfloat16_params.frac_shift; - return bfloat16_pack_raw(&p); + return pack_raw64(&p, &bfloat16_params); } float128 float128_silence_nan(float128 a, float_status *status) -- 2.43.0 Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- fpu/softfloat.c | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/fpu/softfloat.c b/fpu/softfloat.c index XXXXXXX..XXXXXXX 100644 --- a/fpu/softfloat.c +++ b/fpu/softfloat.c @@ -XXX,XX +XXX,XX @@ static uint64_t pack_raw64(const FloatParts64 *p, const FloatFmt *fmt) return ret; } -static float32 QEMU_FLATTEN float32_pack_raw(const FloatParts64 *p) -{ - return make_float32(pack_raw64(p, &float32_params)); -} - static float64 QEMU_FLATTEN float64_pack_raw(const FloatParts64 *p) { return make_float64(pack_raw64(p, &float64_params)); @@ -XXX,XX +XXX,XX @@ static float32 float32_round_pack_canonical(FloatParts64 *p, float_status *s) { parts64_uncanon(p, s, &float32_params, false); - return float32_pack_raw(p); + return pack_raw64(p, &float32_params); } static FloatParts64 float64_unpack_canonical(float64 f, float_status *s) @@ -XXX,XX +XXX,XX @@ float32_muladd_scalbn(float32 a, float32 b, float32 c, if ((flags & float_muladd_negate_result) && !is_nan(pr->cls)) { pr->sign ^= 1; } - return float32_pack_raw(pr); + return pack_raw64(pr, &float32_params); } float64 QEMU_SOFTFLOAT_ATTR @@ -XXX,XX +XXX,XX @@ float32 float32_default_nan(float_status *status) parts64_default_nan(&p, status); p.frac >>= float32_params.frac_shift; - return float32_pack_raw(&p); + return pack_raw64(&p, &float32_params); } float64 float64_default_nan(float_status *status) @@ -XXX,XX +XXX,XX @@ float32 float32_silence_nan(float32 a, float_status *status) p.frac <<= float32_params.frac_shift; parts64_silence_nan(&p, status); p.frac >>= float32_params.frac_shift; - return float32_pack_raw(&p); + return pack_raw64(&p, &float32_params); } float64 float64_silence_nan(float64 a, float_status *status) -- 2.43.0 Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- fpu/softfloat.c | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/fpu/softfloat.c b/fpu/softfloat.c index XXXXXXX..XXXXXXX 100644 --- a/fpu/softfloat.c +++ b/fpu/softfloat.c @@ -XXX,XX +XXX,XX @@ static uint64_t pack_raw64(const FloatParts64 *p, const FloatFmt *fmt) return ret; } -static float64 QEMU_FLATTEN float64_pack_raw(const FloatParts64 *p) -{ - return make_float64(pack_raw64(p, &float64_params)); -} - static float128 QEMU_FLATTEN float128_pack_raw(const FloatParts128 *p) { const int f_size = float128_params.frac_size - 64; @@ -XXX,XX +XXX,XX @@ static float64 float64_round_pack_canonical(FloatParts64 *p, float_status *s) { parts64_uncanon(p, s, &float64_params, false); - return float64_pack_raw(p); + return pack_raw64(p, &float64_params); } static float64 float64r32_pack_raw(FloatParts64 *p) @@ -XXX,XX +XXX,XX @@ static float64 float64r32_pack_raw(FloatParts64 *p) g_assert_not_reached(); } - return float64_pack_raw(p); + return pack_raw64(p, &float64_params); } static float64 float64r32_round_pack_canonical(FloatParts64 *p, @@ -XXX,XX +XXX,XX @@ float64_muladd_scalbn(float64 a, float64 b, float64 c, if ((flags & float_muladd_negate_result) && !is_nan(pr->cls)) { pr->sign ^= 1; } - return float64_pack_raw(pr); + return pack_raw64(pr, &float64_params); } static bool force_soft_fma; @@ -XXX,XX +XXX,XX @@ float64 float64_default_nan(float_status *status) parts64_default_nan(&p, status); p.frac >>= float64_params.frac_shift; - return float64_pack_raw(&p); + return pack_raw64(&p, &float64_params); } float128 float128_default_nan(float_status *status) @@ -XXX,XX +XXX,XX @@ float64 float64_silence_nan(float64 a, float_status *status) p.frac <<= float64_params.frac_shift; parts64_silence_nan(&p, status); p.frac >>= float64_params.frac_shift; - return float64_pack_raw(&p); + return pack_raw64(&p, &float64_params); } bfloat16 bfloat16_silence_nan(bfloat16 a, float_status *status) -- 2.43.0 This is almost always used with a constant FloatFmt, so inlining pulls the constants into the shifts. Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- fpu/softfloat.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fpu/softfloat.c b/fpu/softfloat.c index XXXXXXX..XXXXXXX 100644 --- a/fpu/softfloat.c +++ b/fpu/softfloat.c @@ -XXX,XX +XXX,XX @@ static const FloatFmt floatx80_params[3] = { }; /* Unpack a float to parts, but do not canonicalize. */ -static FloatParts64 unpack_raw64(const FloatFmt *fmt, uint64_t raw) +static inline QEMU_ALWAYS_INLINE +FloatParts64 unpack_raw64(const FloatFmt *fmt, uint64_t raw) { const int f_size = fmt->frac_size; const int e_size = fmt->exp_size; -- 2.43.0 This is almost always used with a constant FloatFmt, so inlining pulls the constants into the shifts. Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- fpu/softfloat.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fpu/softfloat.c b/fpu/softfloat.c index XXXXXXX..XXXXXXX 100644 --- a/fpu/softfloat.c +++ b/fpu/softfloat.c @@ -XXX,XX +XXX,XX @@ static FloatParts128 float128_unpack_raw(float128 f) } /* Pack a float from parts, but do not canonicalize. */ -static uint64_t pack_raw64(const FloatParts64 *p, const FloatFmt *fmt) +static inline uint64_t QEMU_ALWAYS_INLINE +pack_raw64(const FloatParts64 *p, const FloatFmt *fmt) { const int f_size = fmt->frac_size; const int e_size = fmt->exp_size; -- 2.43.0 Begin exposing the intermediate representation of softfloat. Start with just the representation structures. Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- include/fpu/softfloat-parts.h | 88 +++++++++++++++++++++++++++++++++++ fpu/softfloat.c | 69 +-------------------------- 2 files changed, 90 insertions(+), 67 deletions(-) create mode 100644 include/fpu/softfloat-parts.h diff --git a/include/fpu/softfloat-parts.h b/include/fpu/softfloat-parts.h new file mode 100644 index XXXXXXX..XXXXXXX --- /dev/null +++ b/include/fpu/softfloat-parts.h @@ -XXX,XX +XXX,XX @@ +/* + * Floating point intermediate representation + * + * The code in this source file is derived from release 2a of the SoftFloat + * IEC/IEEE Floating-point Arithmetic Package. Those parts of the code (and + * some later contributions) are provided under that license, as detailed below. + * It has subsequently been modified by contributors to the QEMU Project, + * so some portions are provided under: + * the SoftFloat-2a license + * the BSD license + * GPL-v2-or-later + * + * Any future contributions to this file after December 1st 2014 will be + * taken to be licensed under the Softfloat-2a license unless specifically + * indicated otherwise. + */ + +#ifndef SOFTFLOAT_PARTS_H +#define SOFTFLOAT_PARTS_H + +/* + * Classify a floating point number. Everything above float_class_qnan + * is a NaN so cls >= float_class_qnan is any NaN. + * + * Note that we canonicalize denormals, so most code should treat + * class_normal and class_denormal identically. + */ + +typedef enum __attribute__ ((__packed__)) { + float_class_unclassified, + float_class_zero, + float_class_normal, + float_class_denormal, /* input was a non-squashed denormal */ + float_class_inf, + float_class_qnan, /* all NaNs from here */ + float_class_snan, +} FloatClass; + +#define float_cmask(bit) (1u << (bit)) + +enum { + float_cmask_zero = float_cmask(float_class_zero), + float_cmask_normal = float_cmask(float_class_normal), + float_cmask_denormal = float_cmask(float_class_denormal), + float_cmask_inf = float_cmask(float_class_inf), + float_cmask_qnan = float_cmask(float_class_qnan), + float_cmask_snan = float_cmask(float_class_snan), + + float_cmask_infzero = float_cmask_zero | float_cmask_inf, + float_cmask_anynan = float_cmask_qnan | float_cmask_snan, + float_cmask_anynorm = float_cmask_normal | float_cmask_denormal, +}; + +/* + * Structure holding all of the decomposed parts of a float. + * The exponent is unbiased and the fraction is normalized. + * + * The fraction words are stored in big-endian word ordering, + * so that truncation from a larger format to a smaller format + * can be done simply by ignoring subsequent elements. + */ + +typedef struct { + FloatClass cls; + bool sign; + int32_t exp; + union { + /* Routines that know the structure may reference the singular name. */ + uint64_t frac; + /* + * Routines expanded with multiple structures reference "hi" and "lo" + * depending on the operation. In FloatParts64, "hi" and "lo" are + * both the same word and aliased here. + */ + uint64_t frac_hi; + uint64_t frac_lo; + }; +} FloatParts64; + +typedef struct { + FloatClass cls; + bool sign; + int32_t exp; + uint64_t frac_hi; + uint64_t frac_lo; +} FloatParts128; + +#endif diff --git a/fpu/softfloat.c b/fpu/softfloat.c index XXXXXXX..XXXXXXX 100644 --- a/fpu/softfloat.c +++ b/fpu/softfloat.c @@ -XXX,XX +XXX,XX @@ this code that are retained. #include <math.h> #include "qemu/bitops.h" #include "fpu/softfloat.h" +#include "fpu/softfloat-parts.h" /* We only need stdlib for abort() */ @@ -XXX,XX +XXX,XX @@ float64_gen2(float64 xa, float64 xb, float_status *s, return soft(ua.s, ub.s, s); } -/* - * Classify a floating point number. Everything above float_class_qnan - * is a NaN so cls >= float_class_qnan is any NaN. - * - * Note that we canonicalize denormals, so most code should treat - * class_normal and class_denormal identically. - */ - -typedef enum __attribute__ ((__packed__)) { - float_class_unclassified, - float_class_zero, - float_class_normal, - float_class_denormal, /* input was a non-squashed denormal */ - float_class_inf, - float_class_qnan, /* all NaNs from here */ - float_class_snan, -} FloatClass; - -#define float_cmask(bit) (1u << (bit)) - -enum { - float_cmask_zero = float_cmask(float_class_zero), - float_cmask_normal = float_cmask(float_class_normal), - float_cmask_denormal = float_cmask(float_class_denormal), - float_cmask_inf = float_cmask(float_class_inf), - float_cmask_qnan = float_cmask(float_class_qnan), - float_cmask_snan = float_cmask(float_class_snan), - - float_cmask_infzero = float_cmask_zero | float_cmask_inf, - float_cmask_anynan = float_cmask_qnan | float_cmask_snan, - float_cmask_anynorm = float_cmask_normal | float_cmask_denormal, -}; - /* Flags for parts_minmax. */ enum { /* Set for minimum; clear for maximum. */ @@ -XXX,XX +XXX,XX @@ static inline bool is_anynorm(FloatClass c) return float_cmask(c) & float_cmask_anynorm; } -/* - * Structure holding all of the decomposed parts of a float. - * The exponent is unbiased and the fraction is normalized. - * - * The fraction words are stored in big-endian word ordering, - * so that truncation from a larger format to a smaller format - * can be done simply by ignoring subsequent elements. - */ - -typedef struct { - FloatClass cls; - bool sign; - int32_t exp; - union { - /* Routines that know the structure may reference the singular name. */ - uint64_t frac; - /* - * Routines expanded with multiple structures reference "hi" and "lo" - * depending on the operation. In FloatParts64, "hi" and "lo" are - * both the same word and aliased here. - */ - uint64_t frac_hi; - uint64_t frac_lo; - }; -} FloatParts64; - -typedef struct { - FloatClass cls; - bool sign; - int32_t exp; - uint64_t frac_hi; - uint64_t frac_lo; -} FloatParts128; - +/* FloatParts256 is entirely internal, for parts128_mul* */ typedef struct { FloatClass cls; bool sign; -- 2.43.0 Export most of the FloatFmt structures. Skip float16_params_ahp and the floatx80 precisions. Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- include/fpu/softfloat-parts.h | 48 ++++++++++++++++++++++++++++++ fpu/softfloat.c | 55 +++++------------------------------ 2 files changed, 56 insertions(+), 47 deletions(-) diff --git a/include/fpu/softfloat-parts.h b/include/fpu/softfloat-parts.h index XXXXXXX..XXXXXXX 100644 --- a/include/fpu/softfloat-parts.h +++ b/include/fpu/softfloat-parts.h @@ -XXX,XX +XXX,XX @@ #ifndef SOFTFLOAT_PARTS_H #define SOFTFLOAT_PARTS_H +/* Format-specific handling of exp == exp_max */ +typedef enum __attribute__((__packed__)) { + /* exp==max, frac==0 ? infinity : nan; this is ieee standard. */ + float_expmax_ieee, + /* exp==max is a normal number; no infinity or nan representation. */ + float_expmax_normal, + /* exp==max, frac==max ? nan : normal; no infinity representation. */ + float_expmax_e4m3, +} FloatFmtExpMaxKind; + +/* + * Structure holding all of the relevant parameters for a format. + * exp_size: the size of the exponent field + * exp_bias: the offset applied to the exponent field + * exp_max: the maximum normalised exponent + * frac_size: the size of the fraction field + * frac_shift: shift to normalise the fraction with DECOMPOSED_BINARY_POINT + * The following are computed based the size of fraction + * round_mask: bits below lsb which must be rounded + * The following optional modifiers are available: + * exp_max_kind: affects how exp == exp_max is interpreted + * has_explicit_bit: has an explicit integer bit; this affects whether + * the float_status floatx80_behaviour handling applies + * overflow_raises_invalid: for float_expmax_normal, raise invalid + * instead of overflow. + */ +typedef struct { + int exp_size; + int exp_bias; + int exp_re_bias; + int exp_max; + int frac_size; + int frac_shift; + FloatFmtExpMaxKind exp_max_kind; + bool has_explicit_bit; + bool overflow_raises_invalid; + uint64_t round_mask; +} FloatFmt; + +extern const FloatFmt float4_e2m1_params; +extern const FloatFmt float8_e4m3_params; +extern const FloatFmt float8_e5m2_params; +extern const FloatFmt float16_params; +extern const FloatFmt bfloat16_params; +extern const FloatFmt float32_params; +extern const FloatFmt float64_params; +extern const FloatFmt float128_params; + /* * Classify a floating point number. Everything above float_class_qnan * is a NaN so cls >= float_class_qnan is any NaN. diff --git a/fpu/softfloat.c b/fpu/softfloat.c index XXXXXXX..XXXXXXX 100644 --- a/fpu/softfloat.c +++ b/fpu/softfloat.c @@ -XXX,XX +XXX,XX @@ typedef struct { #define DECOMPOSED_BINARY_POINT 63 #define DECOMPOSED_IMPLICIT_BIT (1ull << DECOMPOSED_BINARY_POINT) -/* Format-specific handling of exp == exp_max */ -typedef enum __attribute__((__packed__)) { - /* exp==max, frac==0 ? infinity : nan; this is ieee standard. */ - float_expmax_ieee, - /* exp==max is a normal number; no infinity or nan representation. */ - float_expmax_normal, - /* exp==max, frac==max ? nan : normal; no infinity representation. */ - float_expmax_e4m3, -} FloatFmtExpMaxKind; - -/* - * Structure holding all of the relevant parameters for a format. - * exp_size: the size of the exponent field - * exp_bias: the offset applied to the exponent field - * exp_max: the maximum normalised exponent - * frac_size: the size of the fraction field - * frac_shift: shift to normalise the fraction with DECOMPOSED_BINARY_POINT - * The following are computed based the size of fraction - * round_mask: bits below lsb which must be rounded - * The following optional modifiers are available: - * exp_max_kind: affects how exp == exp_max is interpreted - * has_explicit_bit: has an explicit integer bit; this affects whether - * the float_status floatx80_behaviour handling applies - * overflow_raises_invalid: for float_expmax_normal, raise invalid - * instead of overflow. - */ -typedef struct { - int exp_size; - int exp_bias; - int exp_re_bias; - int exp_max; - int frac_size; - int frac_shift; - FloatFmtExpMaxKind exp_max_kind; - bool has_explicit_bit; - bool overflow_raises_invalid; - uint64_t round_mask; -} FloatFmt; - /* Expand fields based on the size of exponent and fraction */ #define FLOAT_PARAMS_(E) \ .exp_size = E, \ @@ -XXX,XX +XXX,XX @@ typedef struct { .frac_shift = (-F - 1) & 63, \ .round_mask = (1ull << ((-F - 1) & 63)) - 1 -static const FloatFmt float4_e2m1_params = { +const FloatFmt float4_e2m1_params = { FLOAT_PARAMS(2, 1), .exp_max_kind = float_expmax_normal, }; -static const FloatFmt float8_e4m3_params = { +const FloatFmt float8_e4m3_params = { FLOAT_PARAMS(4, 3), .exp_max_kind = float_expmax_e4m3 }; @@ -XXX,XX +XXX,XX @@ static const FloatFmt float8_e4m3_params = { /* 110 << frac_shift, with the implicit bit set */ #define E4M3_NORMAL_FRAC_MAX 0xe000000000000000ull -static const FloatFmt float8_e5m2_params = { +const FloatFmt float8_e5m2_params = { FLOAT_PARAMS(5, 2) }; -static const FloatFmt float16_params = { +const FloatFmt float16_params = { FLOAT_PARAMS(5, 10) }; @@ -XXX,XX +XXX,XX @@ static const FloatFmt float16_params_ahp = { .overflow_raises_invalid = true, }; -static const FloatFmt bfloat16_params = { +const FloatFmt bfloat16_params = { FLOAT_PARAMS(8, 7) }; -static const FloatFmt float32_params = { +const FloatFmt float32_params = { FLOAT_PARAMS(8, 23) }; -static const FloatFmt float64_params = { +const FloatFmt float64_params = { FLOAT_PARAMS(11, 52) }; -static const FloatFmt float128_params = { +const FloatFmt float128_params = { FLOAT_PARAMS(15, 112) }; -- 2.43.0 Export the unpacking and repacking into the various formats. Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- include/fpu/softfloat-parts.h | 30 ++++++++++++++++++++++ fpu/softfloat.c | 47 ++++++++++++++--------------------- 2 files changed, 49 insertions(+), 28 deletions(-) diff --git a/include/fpu/softfloat-parts.h b/include/fpu/softfloat-parts.h index XXXXXXX..XXXXXXX 100644 --- a/include/fpu/softfloat-parts.h +++ b/include/fpu/softfloat-parts.h @@ -XXX,XX +XXX,XX @@ typedef struct { uint64_t frac_lo; } FloatParts128; +/* + * Unpack routines from a specific floating-point format. + */ + +FloatParts64 float4_e2m1_unpack_canonical(float4_e2m1 f, float_status *s); +FloatParts64 float8_e4m3_unpack_canonical(float8_e4m3 f, float_status *s); +FloatParts64 float8_e5m2_unpack_canonical(float8_e5m2 f, float_status *s); +FloatParts64 float16_unpack_canonical(float16 f, float_status *s); +FloatParts64 bfloat16_unpack_canonical(bfloat16 f, float_status *s); +FloatParts64 float32_unpack_canonical(float32 f, float_status *s); +FloatParts64 float64_unpack_canonical(float64 f, float_status *s); +FloatParts128 float128_unpack_canonical(float128 f, float_status *s); +/* Returns false if the encoding is invalid. */ +bool floatx80_unpack_canonical(FloatParts128 *p, floatx80 f, float_status *s); + +/* + * Pack routines to a specific floating-point format. + */ + +float8_e4m3 float8_e4m3_round_pack_canonical(FloatParts64 *p, float_status *s, + bool saturate); +float8_e5m2 float8_e5m2_round_pack_canonical(FloatParts64 *p, float_status *s, + bool saturate); +float16 float16_round_pack_canonical(FloatParts64 *p, float_status *s); +bfloat16 bfloat16_round_pack_canonical(FloatParts64 *p, float_status *s); +float32 float32_round_pack_canonical(FloatParts64 *p, float_status *s); +float64 float64_round_pack_canonical(FloatParts64 *p, float_status *s); +float128 float128_round_pack_canonical(FloatParts128 *p, float_status *s); +floatx80 floatx80_round_pack_canonical(FloatParts128 *p, float_status *s); + #endif diff --git a/fpu/softfloat.c b/fpu/softfloat.c index XXXXXXX..XXXXXXX 100644 --- a/fpu/softfloat.c +++ b/fpu/softfloat.c @@ -XXX,XX +XXX,XX @@ static const uint16_t rsqrt_tab[128] = { * Pack/unpack routines with a specific FloatFmt. */ -static FloatParts64 float4_e2m1_unpack_canonical(float4_e2m1 f, float_status *s) +FloatParts64 float4_e2m1_unpack_canonical(float4_e2m1 f, float_status *s) { FloatParts64 p = unpack_raw64(&float4_e2m1_params, f); parts64_canonicalize(&p, s, &float4_e2m1_params); return p; } -static FloatParts64 float8_e4m3_unpack_canonical(float8_e4m3 f, float_status *s) +FloatParts64 float8_e4m3_unpack_canonical(float8_e4m3 f, float_status *s) { FloatParts64 p = unpack_raw64(&float8_e4m3_params, f); parts64_canonicalize(&p, s, &float8_e4m3_params); return p; } -static FloatParts64 float8_e5m2_unpack_canonical(float8_e5m2 f, float_status *s) +FloatParts64 float8_e5m2_unpack_canonical(float8_e5m2 f, float_status *s) { FloatParts64 p = unpack_raw64(&float8_e5m2_params, f); parts64_canonicalize(&p, s, &float8_e5m2_params); @@ -XXX,XX +XXX,XX @@ static FloatParts64 float16a_unpack_canonical(float16 f, float_status *s, return p; } -static FloatParts64 float16_unpack_canonical(float16 f, float_status *s) +FloatParts64 float16_unpack_canonical(float16 f, float_status *s) { return float16a_unpack_canonical(f, s, &float16_params); } -static FloatParts64 bfloat16_unpack_canonical(bfloat16 f, float_status *s) +FloatParts64 bfloat16_unpack_canonical(bfloat16 f, float_status *s) { FloatParts64 p = unpack_raw64(&bfloat16_params, f); parts64_canonicalize(&p, s, &bfloat16_params); return p; } -static float8_e4m3 float8_e4m3_round_pack_canonical(FloatParts64 *p, - float_status *s, - bool saturate) +float8_e4m3 float8_e4m3_round_pack_canonical(FloatParts64 *p, float_status *s, + bool saturate) { parts64_uncanon(p, s, &float8_e4m3_params, saturate); return pack_raw64(p, &float8_e4m3_params); } -static float8_e5m2 float8_e5m2_round_pack_canonical(FloatParts64 *p, - float_status *s, - bool saturate) +float8_e5m2 float8_e5m2_round_pack_canonical(FloatParts64 *p, float_status *s, + bool saturate) { parts64_uncanon(p, s, &float8_e5m2_params, saturate); return pack_raw64(p, &float8_e5m2_params); @@ -XXX,XX +XXX,XX @@ static float16 float16a_round_pack_canonical(FloatParts64 *p, return pack_raw64(p, &float16_params); } -static float16 float16_round_pack_canonical(FloatParts64 *p, - float_status *s) +float16 float16_round_pack_canonical(FloatParts64 *p, float_status *s) { return float16a_round_pack_canonical(p, s, &float16_params); } -static bfloat16 bfloat16_round_pack_canonical(FloatParts64 *p, - float_status *s) +bfloat16 bfloat16_round_pack_canonical(FloatParts64 *p, float_status *s) { parts64_uncanon(p, s, &bfloat16_params, false); return pack_raw64(p, &bfloat16_params); } -static FloatParts64 float32_unpack_canonical(float32 f, float_status *s) +FloatParts64 float32_unpack_canonical(float32 f, float_status *s) { FloatParts64 p = unpack_raw64(&float32_params, f); parts64_canonicalize(&p, s, &float32_params); return p; } -static float32 float32_round_pack_canonical(FloatParts64 *p, - float_status *s) +float32 float32_round_pack_canonical(FloatParts64 *p, float_status *s) { parts64_uncanon(p, s, &float32_params, false); return pack_raw64(p, &float32_params); } -static FloatParts64 float64_unpack_canonical(float64 f, float_status *s) +FloatParts64 float64_unpack_canonical(float64 f, float_status *s) { FloatParts64 p = unpack_raw64(&float64_params, f); parts64_canonicalize(&p, s, &float64_params); return p; } -static float64 float64_round_pack_canonical(FloatParts64 *p, - float_status *s) +float64 float64_round_pack_canonical(FloatParts64 *p, float_status *s) { parts64_uncanon(p, s, &float64_params, false); return pack_raw64(p, &float64_params); @@ -XXX,XX +XXX,XX @@ static float64 float64r32_round_pack_canonical(FloatParts64 *p, return float64r32_pack_raw(p); } -static FloatParts128 float128_unpack_canonical(float128 f, float_status *s) +FloatParts128 float128_unpack_canonical(float128 f, float_status *s) { FloatParts128 p = float128_unpack_raw(f); parts128_canonicalize(&p, s, &float128_params); return p; } -static float128 float128_round_pack_canonical(FloatParts128 *p, - float_status *s) +float128 float128_round_pack_canonical(FloatParts128 *p, float_status *s) { parts128_uncanon(p, s, &float128_params, false); return float128_pack_raw(p); } /* Returns false if the encoding is invalid. */ -static bool floatx80_unpack_canonical(FloatParts128 *p, floatx80 f, - float_status *s) +bool floatx80_unpack_canonical(FloatParts128 *p, floatx80 f, float_status *s) { /* Ensure rounding precision is set before beginning. */ switch (s->floatx80_rounding_precision) { @@ -XXX,XX +XXX,XX @@ static bool floatx80_unpack_canonical(FloatParts128 *p, floatx80 f, return true; } -static floatx80 floatx80_round_pack_canonical(FloatParts128 *p, - float_status *s) +floatx80 floatx80_round_pack_canonical(FloatParts128 *p, float_status *s) { const FloatFmt *fmt = &floatx80_params[s->floatx80_rounding_precision]; uint64_t frac; -- 2.43.0 Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- fpu/softfloat.c | 29 ++++++++++++----------------- fpu/softfloat-parts.c.inc | 22 +++++++++++----------- fpu/softfloat-specialize.c.inc | 22 ++++++++++------------ 3 files changed, 33 insertions(+), 40 deletions(-) diff --git a/fpu/softfloat.c b/fpu/softfloat.c index XXXXXXX..XXXXXXX 100644 --- a/fpu/softfloat.c +++ b/fpu/softfloat.c @@ -XXX,XX +XXX,XX @@ float32 floatx80_to_float32(floatx80 a, float_status *s) if (floatx80_unpack_canonical(&p128, a, s)) { p64 = parts128_to_parts64(&p128, s); } else { - parts64_default_nan(&p64, s); + p64 = parts64_default_nan(s); } return float32_round_pack_canonical(&p64, s); } @@ -XXX,XX +XXX,XX @@ float64 floatx80_to_float64(floatx80 a, float_status *s) if (floatx80_unpack_canonical(&p128, a, s)) { p64 = parts128_to_parts64(&p128, s); } else { - parts64_default_nan(&p64, s); + p64 = parts64_default_nan(s); } return float64_round_pack_canonical(&p64, s); } @@ -XXX,XX +XXX,XX @@ float128 floatx80_to_float128(floatx80 a, float_status *s) if (floatx80_unpack_canonical(&p, a, s)) { parts128_float_to_float(&p, s); } else { - parts128_default_nan(&p, s); + p = parts128_default_nan(s); } return float128_round_pack_canonical(&p, s); } @@ -XXX,XX +XXX,XX @@ static int32_t floatx80_to_int32_scalbn(floatx80 a, FloatRoundMode rmode, FloatParts128 p; if (!floatx80_unpack_canonical(&p, a, s)) { - parts128_default_nan(&p, s); + p = parts128_default_nan(s); } return parts128_float_to_sint(&p, rmode, scale, INT32_MIN, INT32_MAX, s); } @@ -XXX,XX +XXX,XX @@ static int64_t floatx80_to_int64_scalbn(floatx80 a, FloatRoundMode rmode, FloatParts128 p; if (!floatx80_unpack_canonical(&p, a, s)) { - parts128_default_nan(&p, s); + p = parts128_default_nan(s); } return parts128_float_to_sint(&p, rmode, scale, INT64_MIN, INT64_MAX, s); } @@ -XXX,XX +XXX,XX @@ static void parts64_log2(FloatParts64 *a, float_status *s, const FloatFmt *fmt) d_nan: float_raise(float_flag_invalid, s); - parts64_default_nan(a, s); + *a = parts64_default_nan(s); } float32 float32_log2(float32 a, float_status *status) @@ -XXX,XX +XXX,XX @@ float64 float64_log2(float64 a, float_status *status) float16 float16_default_nan(float_status *status) { - FloatParts64 p; + FloatParts64 p = parts64_default_nan(status); - parts64_default_nan(&p, status); p.frac >>= float16_params.frac_shift; return pack_raw64(&p, &float16_params); } float32 float32_default_nan(float_status *status) { - FloatParts64 p; + FloatParts64 p = parts64_default_nan(status); - parts64_default_nan(&p, status); p.frac >>= float32_params.frac_shift; return pack_raw64(&p, &float32_params); } float64 float64_default_nan(float_status *status) { - FloatParts64 p; + FloatParts64 p = parts64_default_nan(status); - parts64_default_nan(&p, status); p.frac >>= float64_params.frac_shift; return pack_raw64(&p, &float64_params); } float128 float128_default_nan(float_status *status) { - FloatParts128 p; + FloatParts128 p = parts128_default_nan(status); - parts128_default_nan(&p, status); frac128_shr(&p, float128_params.frac_shift); return float128_pack_raw(&p); } bfloat16 bfloat16_default_nan(float_status *status) { - FloatParts64 p; + FloatParts64 p = parts64_default_nan(status); - parts64_default_nan(&p, status); p.frac >>= bfloat16_params.frac_shift; return pack_raw64(&p, &bfloat16_params); } @@ -XXX,XX +XXX,XX @@ static void parts_s390_divide_to_integer(FloatParts64 *a, FloatParts64 *b, *n = *r; *cc = 1; } else if (a->cls == float_class_inf || b->cls == float_class_zero) { - parts64_default_nan(r, status); + *r = parts64_default_nan(status); *n = *r; *cc = 1; status->float_exception_flags |= float_flag_invalid; diff --git a/fpu/softfloat-parts.c.inc b/fpu/softfloat-parts.c.inc index XXXXXXX..XXXXXXX 100644 --- a/fpu/softfloat-parts.c.inc +++ b/fpu/softfloat-parts.c.inc @@ -XXX,XX +XXX,XX @@ static void partsN(return_nan)(FloatPartsN *a, float_status *s) case float_class_snan: float_raise(float_flag_invalid | float_flag_invalid_snan, s); if (s->default_nan_mode) { - partsN(default_nan)(a, s); + *a = partsN(default_nan)(s); } else { partsN(silence_nan)(a, s); } break; case float_class_qnan: if (s->default_nan_mode) { - partsN(default_nan)(a, s); + *a = partsN(default_nan)(s); } break; default: @@ -XXX,XX +XXX,XX @@ static FloatPartsN *partsN(pick_nan)(FloatPartsN *a, FloatPartsN *b, } if (s->default_nan_mode) { - partsN(default_nan)(a, s); + *a = partsN(default_nan)(s); return a; } @@ -XXX,XX +XXX,XX @@ static FloatPartsN *partsN(pick_nan_muladd)(FloatPartsN *a, FloatPartsN *b, return ret; default_nan: - partsN(default_nan)(a, s); + *a = partsN(default_nan)(s); return a; } @@ -XXX,XX +XXX,XX @@ static void partsN(uncanon_e4m3_overflow)(FloatPartsN *p, float_status *s, p->exp = fmt->exp_max; p->frac_hi = E4M3_NORMAL_FRAC_MAX; } else { - partsN(default_nan)(p, s); + *p = partsN(default_nan)(s); } } @@ -XXX,XX +XXX,XX @@ static FloatPartsN *partsN(addsub)(FloatPartsN *a, FloatPartsN *b, } /* Inf - Inf */ float_raise(float_flag_invalid | float_flag_invalid_isi, s); - partsN(default_nan)(a, s); + *a = partsN(default_nan)(s); return a; } } else { @@ -XXX,XX +XXX,XX @@ static FloatPartsN *partsN(mul)(FloatPartsN *a, FloatPartsN *b, /* Inf * Zero == NaN */ if (unlikely(ab_mask == float_cmask_infzero)) { float_raise(float_flag_invalid | float_flag_invalid_imz, s); - partsN(default_nan)(a, s); + *a = partsN(default_nan)(s); return a; } @@ -XXX,XX +XXX,XX @@ static FloatPartsN *partsN(muladd_scalbn)(FloatPartsN *a, FloatPartsN *b, goto finish_sign; d_nan: - partsN(default_nan)(a, s); + *a = partsN(default_nan)(s); return a; } @@ -XXX,XX +XXX,XX @@ static FloatPartsN *partsN(div)(FloatPartsN *a, FloatPartsN *b, return a; d_nan: - partsN(default_nan)(a, s); + *a = partsN(default_nan)(s); return a; } @@ -XXX,XX +XXX,XX @@ static FloatPartsN *partsN(modrem)(FloatPartsN *a, FloatPartsN *b, /* Inf % N; N % 0 */ if (a->cls == float_class_inf || b->cls == float_class_zero) { float_raise(float_flag_invalid, s); - partsN(default_nan)(a, s); + *a = partsN(default_nan)(s); return a; } @@ -XXX,XX +XXX,XX @@ static void partsN(sqrt)(FloatPartsN *a, float_status *status, d_nan: float_raise(float_flag_invalid | float_flag_invalid_sqrt, status); - partsN(default_nan)(a, status); + *a = partsN(default_nan)(status); } /* diff --git a/fpu/softfloat-specialize.c.inc b/fpu/softfloat-specialize.c.inc index XXXXXXX..XXXXXXX 100644 --- a/fpu/softfloat-specialize.c.inc +++ b/fpu/softfloat-specialize.c.inc @@ -XXX,XX +XXX,XX @@ static bool parts_is_snan_frac(uint64_t frac, float_status *status) | The pattern for a default generated deconstructed floating-point NaN. *----------------------------------------------------------------------------*/ -static void parts64_default_nan(FloatParts64 *p, float_status *status) +static FloatParts64 parts64_default_nan(float_status *status) { bool sign = 0; uint64_t frac; @@ -XXX,XX +XXX,XX @@ static void parts64_default_nan(FloatParts64 *p, float_status *status) frac = deposit64(0, DECOMPOSED_BINARY_POINT - 7, 7, dnan_pattern); frac = deposit64(frac, 0, DECOMPOSED_BINARY_POINT - 7, -(dnan_pattern & 1)); - *p = (FloatParts64) { + return (FloatParts64) { .cls = float_class_qnan, .sign = sign, .exp = INT_MAX, @@ -XXX,XX +XXX,XX @@ static void parts64_default_nan(FloatParts64 *p, float_status *status) }; } -static void parts128_default_nan(FloatParts128 *p, float_status *status) +static FloatParts128 parts128_default_nan(float_status *status) { /* * Extrapolate from the choices made by parts64_default_nan to fill * in the quad-floating format. If the low bit is set, assume we * want to set all non-snan bits. */ - FloatParts64 p64; - parts64_default_nan(&p64, status); + FloatParts64 p64 = parts64_default_nan(status); - *p = (FloatParts128) { + return (FloatParts128) { .cls = float_class_qnan, .sign = p64.sign, .exp = INT_MAX, @@ -XXX,XX +XXX,XX @@ static void parts128_silence_nan(FloatParts128 *p, float_status *status) *----------------------------------------------------------------------------*/ floatx80 floatx80_default_nan(float_status *status) { - floatx80 r; /* * Extrapolate from the choices made by parts64_default_nan to fill * in the floatx80 format. We assume that floatx80's explicit * integer bit is always set (this is true for i386 and m68k, * which are the only real users of this format). */ - FloatParts64 p64; - parts64_default_nan(&p64, status); + FloatParts64 p64 = parts64_default_nan(status); - r.high = 0x7FFF | (p64.sign << 15); - r.low = (1ULL << DECOMPOSED_BINARY_POINT) | p64.frac; - return r; + return (floatx80) { + .high = 0x7FFF | (p64.sign << 15), + .low = (1ULL << DECOMPOSED_BINARY_POINT) | p64.frac, + }; } /*---------------------------------------------------------------------------- -- 2.43.0 Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- fpu/softfloat.c | 10 +++++----- fpu/softfloat-parts.c.inc | 6 +++--- fpu/softfloat-specialize.c.inc | 20 ++++++++++++++------ 3 files changed, 22 insertions(+), 14 deletions(-) diff --git a/fpu/softfloat.c b/fpu/softfloat.c index XXXXXXX..XXXXXXX 100644 --- a/fpu/softfloat.c +++ b/fpu/softfloat.c @@ -XXX,XX +XXX,XX @@ float16 float16_silence_nan(float16 a, float_status *status) FloatParts64 p = unpack_raw64(&float16_params, a); p.frac <<= float16_params.frac_shift; - parts64_silence_nan(&p, status); + p = parts64_silence_nan(&p, status); p.frac >>= float16_params.frac_shift; return pack_raw64(&p, &float16_params); } @@ -XXX,XX +XXX,XX @@ float32 float32_silence_nan(float32 a, float_status *status) FloatParts64 p = unpack_raw64(&float32_params, a); p.frac <<= float32_params.frac_shift; - parts64_silence_nan(&p, status); + p = parts64_silence_nan(&p, status); p.frac >>= float32_params.frac_shift; return pack_raw64(&p, &float32_params); } @@ -XXX,XX +XXX,XX @@ float64 float64_silence_nan(float64 a, float_status *status) FloatParts64 p = unpack_raw64(&float64_params, a); p.frac <<= float64_params.frac_shift; - parts64_silence_nan(&p, status); + p = parts64_silence_nan(&p, status); p.frac >>= float64_params.frac_shift; return pack_raw64(&p, &float64_params); } @@ -XXX,XX +XXX,XX @@ bfloat16 bfloat16_silence_nan(bfloat16 a, float_status *status) FloatParts64 p = unpack_raw64(&bfloat16_params, a); p.frac <<= bfloat16_params.frac_shift; - parts64_silence_nan(&p, status); + p = parts64_silence_nan(&p, status); p.frac >>= bfloat16_params.frac_shift; return pack_raw64(&p, &bfloat16_params); } @@ -XXX,XX +XXX,XX @@ float128 float128_silence_nan(float128 a, float_status *status) FloatParts128 p = float128_unpack_raw(a); frac128_shl(&p, float128_params.frac_shift); - parts128_silence_nan(&p, status); + p = parts128_silence_nan(&p, status); frac128_shr(&p, float128_params.frac_shift); return float128_pack_raw(&p); } diff --git a/fpu/softfloat-parts.c.inc b/fpu/softfloat-parts.c.inc index XXXXXXX..XXXXXXX 100644 --- a/fpu/softfloat-parts.c.inc +++ b/fpu/softfloat-parts.c.inc @@ -XXX,XX +XXX,XX @@ static void partsN(return_nan)(FloatPartsN *a, float_status *s) if (s->default_nan_mode) { *a = partsN(default_nan)(s); } else { - partsN(silence_nan)(a, s); + *a = partsN(silence_nan)(a, s); } break; case float_class_qnan: @@ -XXX,XX +XXX,XX @@ static FloatPartsN *partsN(pick_nan)(FloatPartsN *a, FloatPartsN *b, } if (is_snan(ret->cls)) { - partsN(silence_nan)(ret, s); + *ret = partsN(silence_nan)(ret, s); } return ret; } @@ -XXX,XX +XXX,XX @@ static FloatPartsN *partsN(pick_nan_muladd)(FloatPartsN *a, FloatPartsN *b, } if (is_snan(ret->cls)) { - partsN(silence_nan)(ret, s); + *ret = partsN(silence_nan)(ret, s); } return ret; diff --git a/fpu/softfloat-specialize.c.inc b/fpu/softfloat-specialize.c.inc index XXXXXXX..XXXXXXX 100644 --- a/fpu/softfloat-specialize.c.inc +++ b/fpu/softfloat-specialize.c.inc @@ -XXX,XX +XXX,XX @@ static uint64_t parts_silence_nan_frac(uint64_t frac, float_status *status) return frac; } -static void parts64_silence_nan(FloatParts64 *p, float_status *status) +static FloatParts64 parts64_silence_nan(const FloatParts64 *p, + float_status *status) { - p->frac = parts_silence_nan_frac(p->frac, status); - p->cls = float_class_qnan; + FloatParts64 r = *p; + + r.frac = parts_silence_nan_frac(r.frac, status); + r.cls = float_class_qnan; + return r; } -static void parts128_silence_nan(FloatParts128 *p, float_status *status) +static FloatParts128 parts128_silence_nan(const FloatParts128 *p, + float_status *status) { - p->frac_hi = parts_silence_nan_frac(p->frac_hi, status); - p->cls = float_class_qnan; + FloatParts128 r = *p; + + r.frac_hi = parts_silence_nan_frac(r.frac_hi, status); + r.cls = float_class_qnan; + return r; } /*---------------------------------------------------------------------------- -- 2.43.0 Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- fpu/softfloat.c | 14 +++++++------- fpu/softfloat-parts.c.inc | 15 ++++++++------- 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/fpu/softfloat.c b/fpu/softfloat.c index XXXXXXX..XXXXXXX 100644 --- a/fpu/softfloat.c +++ b/fpu/softfloat.c @@ -XXX,XX +XXX,XX @@ static void parts_float_to_e5m2(FloatParts64 *a, float_status *s, bool saturate) switch (a->cls) { case float_class_snan: case float_class_qnan: - parts64_return_nan(a, s); + *a = parts64_return_nan(a, s); break; case float_class_inf: @@ -XXX,XX +XXX,XX @@ static void parts_float_to_e5m2(FloatParts64 *a, float_status *s, bool saturate) static void parts64_float_to_float(FloatParts64 *a, float_status *s) { if (is_nan(a->cls)) { - parts64_return_nan(a, s); + *a = parts64_return_nan(a, s); } if (a->cls == float_class_denormal) { float_raise(float_flag_input_denormal_used, s); @@ -XXX,XX +XXX,XX @@ static void parts64_float_to_float(FloatParts64 *a, float_status *s) static void parts128_float_to_float(FloatParts128 *a, float_status *s) { if (is_nan(a->cls)) { - parts128_return_nan(a, s); + *a = parts128_return_nan(a, s); } if (a->cls == float_class_denormal) { float_raise(float_flag_input_denormal_used, s); @@ -XXX,XX +XXX,XX @@ static FloatParts64 parts128_to_parts64(FloatParts128 *b, float_status *s) case float_class_qnan: /* Discard the low bits of the NaN. */ r.frac = b->frac_hi; - parts64_return_nan(&r, s); + r = parts64_return_nan(&r, s); break; default: break; @@ -XXX,XX +XXX,XX @@ static FloatParts128 parts64_to_parts128(FloatParts64 *b, float_status *s) switch (r.cls) { case float_class_qnan: case float_class_snan: - parts128_return_nan(&r, s); + r = parts128_return_nan(&r, s); break; case float_class_denormal: float_raise(float_flag_input_denormal_used, s); @@ -XXX,XX +XXX,XX @@ static void parts64_log2(FloatParts64 *a, float_status *s, const FloatFmt *fmt) break; case float_class_snan: case float_class_qnan: - parts64_return_nan(a, s); + *a = parts64_return_nan(a, s); return; case float_class_zero: float_raise(float_flag_divbyzero, s); @@ -XXX,XX +XXX,XX @@ float32 float32_exp2(float32 a, float_status *status) break; case float_class_snan: case float_class_qnan: - parts64_return_nan(&xp, status); + xp = parts64_return_nan(&xp, status); return float32_round_pack_canonical(&xp, status); case float_class_inf: return xp.sign ? float32_zero : a; diff --git a/fpu/softfloat-parts.c.inc b/fpu/softfloat-parts.c.inc index XXXXXXX..XXXXXXX 100644 --- a/fpu/softfloat-parts.c.inc +++ b/fpu/softfloat-parts.c.inc @@ -XXX,XX +XXX,XX @@ * indicated otherwise. */ -static void partsN(return_nan)(FloatPartsN *a, float_status *s) +static FloatPartsN partsN(return_nan)(const FloatPartsN *a, float_status *s) { switch (a->cls) { case float_class_snan: float_raise(float_flag_invalid | float_flag_invalid_snan, s); if (s->default_nan_mode) { - *a = partsN(default_nan)(s); + return partsN(default_nan)(s); } else { - *a = partsN(silence_nan)(a, s); + return partsN(silence_nan)(a, s); } break; case float_class_qnan: if (s->default_nan_mode) { - *a = partsN(default_nan)(s); + return partsN(default_nan)(s); } break; default: g_assert_not_reached(); } + return *a; } static FloatPartsN *partsN(pick_nan)(FloatPartsN *a, FloatPartsN *b, @@ -XXX,XX +XXX,XX @@ static void partsN(sqrt)(FloatPartsN *a, float_status *status, break; case float_class_snan: case float_class_qnan: - partsN(return_nan)(a, status); + *a = partsN(return_nan)(a, status); return; case float_class_zero: return; @@ -XXX,XX +XXX,XX @@ static void partsN(round_to_int)(FloatPartsN *a, FloatRoundMode rmode, switch (a->cls) { case float_class_qnan: case float_class_snan: - partsN(return_nan)(a, s); + *a = partsN(return_nan)(a, s); break; case float_class_zero: case float_class_inf: @@ -XXX,XX +XXX,XX @@ static void partsN(scalbn)(FloatPartsN *a, int n, float_status *s) switch (a->cls) { case float_class_snan: case float_class_qnan: - partsN(return_nan)(a, s); + *a = partsN(return_nan)(a, s); break; case float_class_zero: case float_class_inf: -- 2.43.0 Share the float32->float64 bias adjustment from both the normal and denormal paths. Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- fpu/softfloat.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/fpu/softfloat.c b/fpu/softfloat.c index XXXXXXX..XXXXXXX 100644 --- a/fpu/softfloat.c +++ b/fpu/softfloat.c @@ -XXX,XX +XXX,XX @@ static float64 float64r32_pack_raw(FloatParts64 *p) * in normalized form for float64. Adjust, per canonicalize. */ int shift = frac64_normalize(p); - p->exp = (float32_params.frac_shift - - float32_params.exp_bias - shift + 1 + - float64_params.exp_bias); frac64_shr(p, float64_params.frac_shift); + p->exp = float32_params.frac_shift - shift + 1; } else { frac64_shl(p, float32_params.frac_shift - float64_params.frac_shift); - p->exp += float64_params.exp_bias - float32_params.exp_bias; } + p->exp += float64_params.exp_bias - float32_params.exp_bias; break; case float_class_snan: case float_class_qnan: -- 2.43.0 Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- fpu/softfloat.c | 8 ++++---- fpu/softfloat-parts.c.inc | 28 ++++++++++++++++------------ 2 files changed, 20 insertions(+), 16 deletions(-) diff --git a/fpu/softfloat.c b/fpu/softfloat.c index XXXXXXX..XXXXXXX 100644 --- a/fpu/softfloat.c +++ b/fpu/softfloat.c @@ -XXX,XX +XXX,XX @@ void normalizeFloatx80Subnormal(uint64_t aSig, int32_t *zExpPtr, floatx80 propagateFloatx80NaN(floatx80 a, floatx80 b, float_status *status) { - FloatParts128 pa, pb, *pr; + FloatParts128 pa, pb; if (!floatx80_unpack_canonical(&pa, a, status) || !floatx80_unpack_canonical(&pb, b, status)) { return floatx80_default_nan(status); } - pr = parts128_pick_nan(&pa, &pb, status); - return floatx80_round_pack_canonical(pr, status); + pa = parts128_pick_nan(&pa, &pb, status); + return floatx80_round_pack_canonical(&pa, status); } /*---------------------------------------------------------------------------- @@ -XXX,XX +XXX,XX @@ static void parts_s390_divide_to_integer(FloatParts64 *a, FloatParts64 *b, { /* POp table "Results: DIVIDE TO INTEGER (Part 1 of 2)" */ if ((float_cmask(a->cls) | float_cmask(b->cls)) & float_cmask_anynan) { - *r = *parts64_pick_nan(a, b, status); + *r = parts64_pick_nan(a, b, status); *n = *r; *cc = 1; } else if (a->cls == float_class_inf || b->cls == float_class_zero) { diff --git a/fpu/softfloat-parts.c.inc b/fpu/softfloat-parts.c.inc index XXXXXXX..XXXXXXX 100644 --- a/fpu/softfloat-parts.c.inc +++ b/fpu/softfloat-parts.c.inc @@ -XXX,XX +XXX,XX @@ static FloatPartsN partsN(return_nan)(const FloatPartsN *a, float_status *s) return *a; } -static FloatPartsN *partsN(pick_nan)(FloatPartsN *a, FloatPartsN *b, - float_status *s) +static FloatPartsN partsN(pick_nan)(const FloatPartsN *a, const FloatPartsN *b, + float_status *s) { bool have_snan = false; - FloatPartsN *ret; + const FloatPartsN *ret; int cmp; if (is_snan(a->cls) || is_snan(b->cls)) { @@ -XXX,XX +XXX,XX @@ static FloatPartsN *partsN(pick_nan)(FloatPartsN *a, FloatPartsN *b, } if (s->default_nan_mode) { - *a = partsN(default_nan)(s); - return a; + return partsN(default_nan)(s); } switch (s->float_2nan_prop_rule) { @@ -XXX,XX +XXX,XX @@ static FloatPartsN *partsN(pick_nan)(FloatPartsN *a, FloatPartsN *b, } if (is_snan(ret->cls)) { - *ret = partsN(silence_nan)(ret, s); + return partsN(silence_nan)(ret, s); } - return ret; + return *ret; } static FloatPartsN *partsN(pick_nan_muladd)(FloatPartsN *a, FloatPartsN *b, @@ -XXX,XX +XXX,XX @@ static FloatPartsN *partsN(addsub)(FloatPartsN *a, FloatPartsN *b, return b; p_nan: - return partsN(pick_nan)(a, b, s); + *a = partsN(pick_nan)(a, b, s); + return a; } /* @@ -XXX,XX +XXX,XX @@ static FloatPartsN *partsN(mul)(FloatPartsN *a, FloatPartsN *b, } if (unlikely(ab_mask & float_cmask_anynan)) { - return partsN(pick_nan)(a, b, s); + *a = partsN(pick_nan)(a, b, s); + return a; } /* Multiply by 0 or Inf */ @@ -XXX,XX +XXX,XX @@ static FloatPartsN *partsN(div)(FloatPartsN *a, FloatPartsN *b, /* All the NaN cases */ if (unlikely(ab_mask & float_cmask_anynan)) { - return partsN(pick_nan)(a, b, s); + *a = partsN(pick_nan)(a, b, s); + return a; } if ((ab_mask & float_cmask_denormal) && b->cls != float_class_zero) { @@ -XXX,XX +XXX,XX @@ static FloatPartsN *partsN(modrem)(FloatPartsN *a, FloatPartsN *b, /* All the NaN cases */ if (unlikely(ab_mask & float_cmask_anynan)) { - return partsN(pick_nan)(a, b, s); + *a = partsN(pick_nan)(a, b, s); + return a; } /* Inf % N; N % 0 */ @@ -XXX,XX +XXX,XX @@ static FloatPartsN *partsN(minmax)(FloatPartsN *a, FloatPartsN *b, return is_nan(a->cls) ? b : a; } - return partsN(pick_nan)(a, b, s); + *a = partsN(pick_nan)(a, b, s); + return a; } if (ab_mask & float_cmask_denormal) { -- 2.43.0 At the same time, export. Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- include/fpu/softfloat-parts.h | 9 +++++++ fpu/softfloat.c | 34 +++++++++++++------------- fpu/softfloat-parts.c.inc | 45 ++++++++++++++++------------------- 3 files changed, 46 insertions(+), 42 deletions(-) diff --git a/include/fpu/softfloat-parts.h b/include/fpu/softfloat-parts.h index XXXXXXX..XXXXXXX 100644 --- a/include/fpu/softfloat-parts.h +++ b/include/fpu/softfloat-parts.h @@ -XXX,XX +XXX,XX @@ float64 float64_round_pack_canonical(FloatParts64 *p, float_status *s); float128 float128_round_pack_canonical(FloatParts128 *p, float_status *s); floatx80 floatx80_round_pack_canonical(FloatParts128 *p, float_status *s); +/* + * Operations + */ + +FloatParts64 parts64_div(const FloatParts64 *a, const FloatParts64 *b, + float_status *s); +FloatParts128 parts128_div(const FloatParts128 *a, const FloatParts128 *b, + float_status *s); + #endif diff --git a/fpu/softfloat.c b/fpu/softfloat.c index XXXXXXX..XXXXXXX 100644 --- a/fpu/softfloat.c +++ b/fpu/softfloat.c @@ -XXX,XX +XXX,XX @@ float16 float16_div(float16 a, float16 b, float_status *status) { FloatParts64 pa = float16_unpack_canonical(a, status); FloatParts64 pb = float16_unpack_canonical(b, status); - FloatParts64 *pr = parts64_div(&pa, &pb, status); + FloatParts64 pr = parts64_div(&pa, &pb, status); - return float16_round_pack_canonical(pr, status); + return float16_round_pack_canonical(&pr, status); } static float32 QEMU_SOFTFLOAT_ATTR @@ -XXX,XX +XXX,XX @@ soft_f32_div(float32 a, float32 b, float_status *status) { FloatParts64 pa = float32_unpack_canonical(a, status); FloatParts64 pb = float32_unpack_canonical(b, status); - FloatParts64 *pr = parts64_div(&pa, &pb, status); + FloatParts64 pr = parts64_div(&pa, &pb, status); - return float32_round_pack_canonical(pr, status); + return float32_round_pack_canonical(&pr, status); } static float64 QEMU_SOFTFLOAT_ATTR @@ -XXX,XX +XXX,XX @@ soft_f64_div(float64 a, float64 b, float_status *status) { FloatParts64 pa = float64_unpack_canonical(a, status); FloatParts64 pb = float64_unpack_canonical(b, status); - FloatParts64 *pr = parts64_div(&pa, &pb, status); + FloatParts64 pr = parts64_div(&pa, &pb, status); - return float64_round_pack_canonical(pr, status); + return float64_round_pack_canonical(&pr, status); } static float hard_f32_div(float a, float b) @@ -XXX,XX +XXX,XX @@ float64 float64r32_div(float64 a, float64 b, float_status *status) { FloatParts64 pa = float64_unpack_canonical(a, status); FloatParts64 pb = float64_unpack_canonical(b, status); - FloatParts64 *pr = parts64_div(&pa, &pb, status); + FloatParts64 pr = parts64_div(&pa, &pb, status); - return float64r32_round_pack_canonical(pr, status); + return float64r32_round_pack_canonical(&pr, status); } bfloat16 QEMU_FLATTEN @@ -XXX,XX +XXX,XX @@ bfloat16_div(bfloat16 a, bfloat16 b, float_status *status) { FloatParts64 pa = bfloat16_unpack_canonical(a, status); FloatParts64 pb = bfloat16_unpack_canonical(b, status); - FloatParts64 *pr = parts64_div(&pa, &pb, status); + FloatParts64 pr = parts64_div(&pa, &pb, status); - return bfloat16_round_pack_canonical(pr, status); + return bfloat16_round_pack_canonical(&pr, status); } float128 QEMU_FLATTEN @@ -XXX,XX +XXX,XX @@ float128_div(float128 a, float128 b, float_status *status) { FloatParts128 pa = float128_unpack_canonical(a, status); FloatParts128 pb = float128_unpack_canonical(b, status); - FloatParts128 *pr = parts128_div(&pa, &pb, status); + FloatParts128 pr = parts128_div(&pa, &pb, status); - return float128_round_pack_canonical(pr, status); + return float128_round_pack_canonical(&pr, status); } floatx80 floatx80_div(floatx80 a, floatx80 b, float_status *status) { - FloatParts128 pa, pb, *pr; + FloatParts128 pa, pb; if (!floatx80_unpack_canonical(&pa, a, status) || !floatx80_unpack_canonical(&pb, b, status)) { return floatx80_default_nan(status); } - pr = parts128_div(&pa, &pb, status); - return floatx80_round_pack_canonical(pr, status); + pa = parts128_div(&pa, &pb, status); + return floatx80_round_pack_canonical(&pa, status); } /* @@ -XXX,XX +XXX,XX @@ static void parts_s390_divide_to_integer(FloatParts64 *a, FloatParts64 *b, uint32_t r_flags; /* Compute precise quotient */ - q_buf = *a; - q = parts64_div(&q_buf, b, status); + q_buf = parts64_div(a, b, status); + q = &q_buf; /* * Check whether two closest integers can be precisely represented, diff --git a/fpu/softfloat-parts.c.inc b/fpu/softfloat-parts.c.inc index XXXXXXX..XXXXXXX 100644 --- a/fpu/softfloat-parts.c.inc +++ b/fpu/softfloat-parts.c.inc @@ -XXX,XX +XXX,XX @@ static FloatPartsN *partsN(muladd_scalbn)(FloatPartsN *a, FloatPartsN *b, * corresponding value `b'. The operation is performed according to * the IEC/IEEE Standard for Binary Floating-Point Arithmetic. */ -static FloatPartsN *partsN(div)(FloatPartsN *a, FloatPartsN *b, - float_status *s) +FloatPartsN partsN(div)(const FloatPartsN *a, const FloatPartsN *b, + float_status *s) { int ab_mask = float_cmask(a->cls) | float_cmask(b->cls); - bool sign = a->sign ^ b->sign; + FloatPartsN r = *a; + + r.sign ^= b->sign; + r.exp -= b->exp; if (likely(cmask_is_only_normals(ab_mask))) { if (ab_mask & float_cmask_denormal) { float_raise(float_flag_input_denormal_used, s); } - a->sign = sign; - a->exp -= b->exp + fracN(div)(a, b); - return a; + r.exp -= fracN(div)(&r, b); + return r; } /* 0/0 or Inf/Inf => NaN */ if (unlikely(ab_mask == float_cmask_zero)) { float_raise(float_flag_invalid | float_flag_invalid_zdz, s); - goto d_nan; + return partsN(default_nan)(s); } if (unlikely(ab_mask == float_cmask_inf)) { float_raise(float_flag_invalid | float_flag_invalid_idi, s); - goto d_nan; + return partsN(default_nan)(s); } /* All the NaN cases */ if (unlikely(ab_mask & float_cmask_anynan)) { - *a = partsN(pick_nan)(a, b, s); - return a; + return partsN(pick_nan)(a, b, s); } if ((ab_mask & float_cmask_denormal) && b->cls != float_class_zero) { float_raise(float_flag_input_denormal_used, s); } - a->sign = sign; - /* Inf / X */ - if (a->cls == float_class_inf) { - return a; + if (r.cls == float_class_inf) { + return r; } /* 0 / X */ - if (a->cls == float_class_zero) { - return a; + if (r.cls == float_class_zero) { + return r; } /* X / Inf */ if (b->cls == float_class_inf) { - a->cls = float_class_zero; - return a; + r.cls = float_class_zero; + return r; } /* X / 0 => Inf */ - g_assert(b->cls == float_class_zero); + assert(b->cls == float_class_zero); float_raise(float_flag_divbyzero, s); - a->cls = float_class_inf; - return a; - - d_nan: - *a = partsN(default_nan)(s); - return a; + r.cls = float_class_inf; + return r; } /* -- 2.43.0 At the same time, export. Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- include/fpu/softfloat-parts.h | 9 +++++++++ fpu/softfloat.c | 19 ++++++++++++------- fpu/softfloat-parts.c.inc | 17 ++++++++++------- 3 files changed, 31 insertions(+), 14 deletions(-) diff --git a/include/fpu/softfloat-parts.h b/include/fpu/softfloat-parts.h index XXXXXXX..XXXXXXX 100644 --- a/include/fpu/softfloat-parts.h +++ b/include/fpu/softfloat-parts.h @@ -XXX,XX +XXX,XX @@ FloatParts64 parts64_div(const FloatParts64 *a, const FloatParts64 *b, FloatParts128 parts128_div(const FloatParts128 *a, const FloatParts128 *b, float_status *s); +FloatParts64 parts64_round_to_int(const FloatParts64 *a, + FloatRoundMode rmode, + int scale, float_status *s, + const FloatFmt *fmt); +FloatParts128 parts128_round_to_int(const FloatParts128 *a, + FloatRoundMode rmode, + int scale, float_status *s, + const FloatFmt *fmt); + #endif diff --git a/fpu/softfloat.c b/fpu/softfloat.c index XXXXXXX..XXXXXXX 100644 --- a/fpu/softfloat.c +++ b/fpu/softfloat.c @@ -XXX,XX +XXX,XX @@ float16 float16_round_to_int(float16 a, float_status *s) { FloatParts64 p = float16_unpack_canonical(a, s); - parts64_round_to_int(&p, s->float_rounding_mode, 0, s, &float16_params); + p = parts64_round_to_int(&p, s->float_rounding_mode, 0, s, + &float16_params); return float16_round_pack_canonical(&p, s); } @@ -XXX,XX +XXX,XX @@ float32 float32_round_to_int(float32 a, float_status *s) { FloatParts64 p = float32_unpack_canonical(a, s); - parts64_round_to_int(&p, s->float_rounding_mode, 0, s, &float32_params); + p = parts64_round_to_int(&p, s->float_rounding_mode, 0, s, + &float32_params); return float32_round_pack_canonical(&p, s); } @@ -XXX,XX +XXX,XX @@ float64 float64_round_to_int(float64 a, float_status *s) { FloatParts64 p = float64_unpack_canonical(a, s); - parts64_round_to_int(&p, s->float_rounding_mode, 0, s, &float64_params); + p = parts64_round_to_int(&p, s->float_rounding_mode, 0, s, + &float64_params); return float64_round_pack_canonical(&p, s); } @@ -XXX,XX +XXX,XX @@ bfloat16 bfloat16_round_to_int(bfloat16 a, float_status *s) { FloatParts64 p = bfloat16_unpack_canonical(a, s); - parts64_round_to_int(&p, s->float_rounding_mode, 0, s, &bfloat16_params); + p = parts64_round_to_int(&p, s->float_rounding_mode, 0, s, + &bfloat16_params); return bfloat16_round_pack_canonical(&p, s); } @@ -XXX,XX +XXX,XX @@ float128 float128_round_to_int(float128 a, float_status *s) { FloatParts128 p = float128_unpack_canonical(a, s); - parts128_round_to_int(&p, s->float_rounding_mode, 0, s, &float128_params); + p = parts128_round_to_int(&p, s->float_rounding_mode, 0, s, + &float128_params); return float128_round_pack_canonical(&p, s); } @@ -XXX,XX +XXX,XX @@ floatx80 floatx80_round_to_int(floatx80 a, float_status *status) return floatx80_default_nan(status); } - parts128_round_to_int(&p, status->float_rounding_mode, 0, status, - &floatx80_params[status->floatx80_rounding_precision]); + p = parts128_round_to_int(&p, status->float_rounding_mode, 0, status, + &floatx80_params[status->floatx80_rounding_precision]); return floatx80_round_pack_canonical(&p, status); } diff --git a/fpu/softfloat-parts.c.inc b/fpu/softfloat-parts.c.inc index XXXXXXX..XXXXXXX 100644 --- a/fpu/softfloat-parts.c.inc +++ b/fpu/softfloat-parts.c.inc @@ -XXX,XX +XXX,XX @@ static bool partsN(round_to_int_normal)(FloatPartsN *a, FloatRoundMode rmode, return true; } -static void partsN(round_to_int)(FloatPartsN *a, FloatRoundMode rmode, +FloatPartsN partsN(round_to_int)(const FloatPartsN *a, + FloatRoundMode rmode, int scale, float_status *s, const FloatFmt *fmt) { switch (a->cls) { case float_class_qnan: case float_class_snan: - *a = partsN(return_nan)(a, s); - break; + return partsN(return_nan)(a, s); case float_class_zero: case float_class_inf: - break; + return *a; case float_class_normal: case float_class_denormal: - if (partsN(round_to_int_normal)(a, rmode, scale, fmt->frac_size)) { - float_raise(float_flag_inexact, s); + { + FloatPartsN r = *a; + if (partsN(round_to_int_normal)(&r, rmode, scale, fmt->frac_size)) { + float_raise(float_flag_inexact, s); + } + return r; } - break; default: g_assert_not_reached(); } -- 2.43.0 We will not expose parts_round_to_int_normal. Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- fpu/softfloat.c | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/fpu/softfloat.c b/fpu/softfloat.c index XXXXXXX..XXXXXXX 100644 --- a/fpu/softfloat.c +++ b/fpu/softfloat.c @@ -XXX,XX +XXX,XX @@ static void parts_s390_divide_to_integer(FloatParts64 *a, FloatParts64 *b, * Rounding of partial quotient may be inexact. This is the whole point * of distinguishing partial quotients, so ignore the exception. */ - *n = *q; - parts64_round_to_int_normal(n, - is_q_smallish - ? final_quotient_rounding_mode - : float_round_to_zero, - 0, fmt->frac_size); + *n = parts64_round_to_int(q, + is_q_smallish + ? final_quotient_rounding_mode + : float_round_to_zero, + 0, status, fmt); /* Compute precise remainder */ r_precise_buf = *b; -- 2.43.0 Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- include/fpu/softfloat-parts.h | 15 +++++++++++++++ fpu/softfloat-parts.c.inc | 6 +++--- fpu/softfloat-specialize.c.inc | 4 ++-- 3 files changed, 20 insertions(+), 5 deletions(-) diff --git a/include/fpu/softfloat-parts.h b/include/fpu/softfloat-parts.h index XXXXXXX..XXXXXXX 100644 --- a/include/fpu/softfloat-parts.h +++ b/include/fpu/softfloat-parts.h @@ -XXX,XX +XXX,XX @@ float64 float64_round_pack_canonical(FloatParts64 *p, float_status *s); float128 float128_round_pack_canonical(FloatParts128 *p, float_status *s); floatx80 floatx80_round_pack_canonical(FloatParts128 *p, float_status *s); +/* + * NaN handling + */ + +FloatParts64 parts64_default_nan(float_status *status); +FloatParts128 parts128_default_nan(float_status *status); + +FloatParts64 parts64_pick_nan(const FloatParts64 *, const FloatParts64 *, + float_status *); +FloatParts128 parts128_pick_nan(const FloatParts128 *, const FloatParts128 *, + float_status *); + +FloatParts64 parts64_return_nan(const FloatParts64 *a, float_status *s); +FloatParts128 parts128_return_nan(const FloatParts128 *a, float_status *s); + /* * Operations */ diff --git a/fpu/softfloat-parts.c.inc b/fpu/softfloat-parts.c.inc index XXXXXXX..XXXXXXX 100644 --- a/fpu/softfloat-parts.c.inc +++ b/fpu/softfloat-parts.c.inc @@ -XXX,XX +XXX,XX @@ * indicated otherwise. */ -static FloatPartsN partsN(return_nan)(const FloatPartsN *a, float_status *s) +FloatPartsN partsN(return_nan)(const FloatPartsN *a, float_status *s) { switch (a->cls) { case float_class_snan: @@ -XXX,XX +XXX,XX @@ static FloatPartsN partsN(return_nan)(const FloatPartsN *a, float_status *s) return *a; } -static FloatPartsN partsN(pick_nan)(const FloatPartsN *a, const FloatPartsN *b, - float_status *s) +FloatPartsN partsN(pick_nan)(const FloatPartsN *a, const FloatPartsN *b, + float_status *s) { bool have_snan = false; const FloatPartsN *ret; diff --git a/fpu/softfloat-specialize.c.inc b/fpu/softfloat-specialize.c.inc index XXXXXXX..XXXXXXX 100644 --- a/fpu/softfloat-specialize.c.inc +++ b/fpu/softfloat-specialize.c.inc @@ -XXX,XX +XXX,XX @@ static bool parts_is_snan_frac(uint64_t frac, float_status *status) | The pattern for a default generated deconstructed floating-point NaN. *----------------------------------------------------------------------------*/ -static FloatParts64 parts64_default_nan(float_status *status) +FloatParts64 parts64_default_nan(float_status *status) { bool sign = 0; uint64_t frac; @@ -XXX,XX +XXX,XX @@ static FloatParts64 parts64_default_nan(float_status *status) }; } -static FloatParts128 parts128_default_nan(float_status *status) +FloatParts128 parts128_default_nan(float_status *status) { /* * Extrapolate from the choices made by parts64_default_nan to fill -- 2.43.0 Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- include/fpu/softfloat-parts.h | 3 +++ fpu/softfloat.c | 22 +++++++++++++++++++--- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/include/fpu/softfloat-parts.h b/include/fpu/softfloat-parts.h index XXXXXXX..XXXXXXX 100644 --- a/include/fpu/softfloat-parts.h +++ b/include/fpu/softfloat-parts.h @@ -XXX,XX +XXX,XX @@ FloatParts128 parts128_round_to_int(const FloatParts128 *a, int scale, float_status *s, const FloatFmt *fmt); +FloatParts64 parts64_round_to_fmt(const FloatParts64 *p, float_status *s, + const FloatFmt *fmt); + #endif diff --git a/fpu/softfloat.c b/fpu/softfloat.c index XXXXXXX..XXXXXXX 100644 --- a/fpu/softfloat.c +++ b/fpu/softfloat.c @@ -XXX,XX +XXX,XX @@ float64 float64_round_pack_canonical(FloatParts64 *p, float_status *s) return pack_raw64(p, &float64_params); } +/* + * Round to Fmt while remaining canonicalized. + */ +FloatParts64 parts64_round_to_fmt(const FloatParts64 *p, float_status *s, + const FloatFmt *fmt) +{ + FloatParts64 r = *p; + + parts64_uncanon(&r, s, fmt, false); + /* + * We normally expect uncanon to be followed by pack_raw, + * so we don't actually crop the bits. Do so now. + */ + r.frac &= MAKE_64BIT_MASK(0, fmt->frac_size); + parts64_canonicalize(&r, s, fmt); + return r; +} + static float64 float64r32_pack_raw(FloatParts64 *p) { /* @@ -XXX,XX +XXX,XX @@ static void parts_s390_divide_to_integer(FloatParts64 *a, FloatParts64 *b, /* Round remainder to the target format */ *r = *r_precise; status->float_exception_flags = 0; - parts64_uncanon(r, status, fmt, false); + *r = parts64_round_to_fmt(r, status, fmt); r_flags = status->float_exception_flags; - r->frac &= (1ULL << fmt->frac_size) - 1; - parts64_canonicalize(r, status, fmt); /* POp table "Results: DIVIDE TO INTEGER (Part 2 of 2)" */ if (is_q_smallish) { -- 2.43.0 Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- fpu/softfloat-parts.c.inc | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/fpu/softfloat-parts.c.inc b/fpu/softfloat-parts.c.inc index XXXXXXX..XXXXXXX 100644 --- a/fpu/softfloat-parts.c.inc +++ b/fpu/softfloat-parts.c.inc @@ -XXX,XX +XXX,XX @@ FloatPartsN partsN(pick_nan)(const FloatPartsN *a, const FloatPartsN *b, return *ret; } -static FloatPartsN *partsN(pick_nan_muladd)(FloatPartsN *a, FloatPartsN *b, - FloatPartsN *c, float_status *s, - int ab_mask, int abc_mask) +static FloatPartsN partsN(pick_nan_muladd)(const FloatPartsN *a, + const FloatPartsN *b, + const FloatPartsN *c, + float_status *s, + int ab_mask, int abc_mask) { bool infzero = (ab_mask == float_cmask_infzero); bool have_snan = (abc_mask & float_cmask_snan); - FloatPartsN *ret; + const FloatPartsN *ret; if (unlikely(have_snan)) { float_raise(float_flag_invalid | float_flag_invalid_snan, s); @@ -XXX,XX +XXX,XX @@ static FloatPartsN *partsN(pick_nan_muladd)(FloatPartsN *a, FloatPartsN *b, } ret = c; } else { - FloatPartsN *val[R_3NAN_1ST_MASK + 1] = { a, b, c }; + const FloatPartsN *val[R_3NAN_1ST_MASK + 1] = { a, b, c }; Float3NaNPropRule rule = s->float_3nan_prop_rule; assert(rule != float_3nan_prop_none); @@ -XXX,XX +XXX,XX @@ static FloatPartsN *partsN(pick_nan_muladd)(FloatPartsN *a, FloatPartsN *b, } if (is_snan(ret->cls)) { - *ret = partsN(silence_nan)(ret, s); + return partsN(silence_nan)(ret, s); } - return ret; + return *ret; default_nan: - *a = partsN(default_nan)(s); - return a; + return partsN(default_nan)(s); } /* @@ -XXX,XX +XXX,XX @@ static FloatPartsN *partsN(muladd_scalbn)(FloatPartsN *a, FloatPartsN *b, * off to the target-specific pick-a-NaN routine. */ if (unlikely(abc_mask & float_cmask_anynan)) { - return partsN(pick_nan_muladd)(a, b, c, s, ab_mask, abc_mask); + *a = partsN(pick_nan_muladd)(a, b, c, s, ab_mask, abc_mask); + return a; } if (flags & float_muladd_negate_c) { -- 2.43.0 Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- fpu/softfloat.c | 8 +++++++ fpu/softfloat-parts.c.inc | 49 +++++++++++---------------------------- 2 files changed, 22 insertions(+), 35 deletions(-) diff --git a/fpu/softfloat.c b/fpu/softfloat.c index XXXXXXX..XXXXXXX 100644 --- a/fpu/softfloat.c +++ b/fpu/softfloat.c @@ -XXX,XX +XXX,XX @@ static inline bool is_anynorm(FloatClass c) return float_cmask(c) & float_cmask_anynorm; } +/* Record when denormals have been used. */ +static void record_denormals_used(int mask, float_status *s) +{ + if (unlikely(mask & float_cmask_denormal)) { + float_raise(float_flag_input_denormal_used, s); + } +} + /* FloatParts256 is entirely internal, for parts128_mul* */ typedef struct { FloatClass cls; diff --git a/fpu/softfloat-parts.c.inc b/fpu/softfloat-parts.c.inc index XXXXXXX..XXXXXXX 100644 --- a/fpu/softfloat-parts.c.inc +++ b/fpu/softfloat-parts.c.inc @@ -XXX,XX +XXX,XX @@ static FloatPartsN *partsN(addsub)(FloatPartsN *a, FloatPartsN *b, * For addition and subtraction, we will consume an * input denormal unless the other input is a NaN. */ - if ((ab_mask & (float_cmask_denormal | float_cmask_anynan)) == - float_cmask_denormal) { - float_raise(float_flag_input_denormal_used, s); + if (!(ab_mask & float_cmask_anynan)) { + record_denormals_used(ab_mask, s); } if (a->sign != b_sign) { @@ -XXX,XX +XXX,XX @@ static FloatPartsN *partsN(mul)(FloatPartsN *a, FloatPartsN *b, if (likely(cmask_is_only_normals(ab_mask))) { FloatPartsW tmp; - if (ab_mask & float_cmask_denormal) { - float_raise(float_flag_input_denormal_used, s); - } + record_denormals_used(ab_mask, s); fracN(mulw)(&tmp, a, b); fracN(truncjam)(a, &tmp); @@ -XXX,XX +XXX,XX @@ static FloatPartsN *partsN(mul)(FloatPartsN *a, FloatPartsN *b, } /* Multiply by 0 or Inf */ - if (ab_mask & float_cmask_denormal) { - float_raise(float_flag_input_denormal_used, s); - } + record_denormals_used(ab_mask, s); if (ab_mask & float_cmask_inf) { a->cls = float_class_inf; @@ -XXX,XX +XXX,XX @@ static FloatPartsN *partsN(muladd_scalbn)(FloatPartsN *a, FloatPartsN *b, * this matches the set of cases where we consumed a * denormal input. */ - if (abc_mask & float_cmask_denormal) { - float_raise(float_flag_input_denormal_used, s); - } + record_denormals_used(abc_mask, s); return a; return_sub_zero: @@ -XXX,XX +XXX,XX @@ FloatPartsN partsN(div)(const FloatPartsN *a, const FloatPartsN *b, r.exp -= b->exp; if (likely(cmask_is_only_normals(ab_mask))) { - if (ab_mask & float_cmask_denormal) { - float_raise(float_flag_input_denormal_used, s); - } + record_denormals_used(ab_mask, s); r.exp -= fracN(div)(&r, b); return r; } @@ -XXX,XX +XXX,XX @@ FloatPartsN partsN(div)(const FloatPartsN *a, const FloatPartsN *b, return partsN(pick_nan)(a, b, s); } - if ((ab_mask & float_cmask_denormal) && b->cls != float_class_zero) { - float_raise(float_flag_input_denormal_used, s); + if (b->cls != float_class_zero) { + record_denormals_used(ab_mask, s); } /* Inf / X */ @@ -XXX,XX +XXX,XX @@ static FloatPartsN *partsN(modrem)(FloatPartsN *a, FloatPartsN *b, int ab_mask = float_cmask(a->cls) | float_cmask(b->cls); if (likely(cmask_is_only_normals(ab_mask))) { - if (ab_mask & float_cmask_denormal) { - float_raise(float_flag_input_denormal_used, s); - } + record_denormals_used(ab_mask, s); fracN(modrem)(a, b, mod_quot); return a; } @@ -XXX,XX +XXX,XX @@ static FloatPartsN *partsN(modrem)(FloatPartsN *a, FloatPartsN *b, return a; } - if (ab_mask & float_cmask_denormal) { - float_raise(float_flag_input_denormal_used, s); - } + record_denormals_used(ab_mask, s); /* N % Inf; 0 % N */ g_assert(b->cls == float_class_inf || a->cls == float_class_zero); @@ -XXX,XX +XXX,XX @@ static FloatPartsN *partsN(minmax)(FloatPartsN *a, FloatPartsN *b, if ((flags & (minmax_isnum | minmax_isnumber)) && !(ab_mask & float_cmask_snan) && (ab_mask & ~float_cmask_qnan)) { - if (ab_mask & float_cmask_denormal) { - float_raise(float_flag_input_denormal_used, s); - } + record_denormals_used(ab_mask, s); return is_nan(a->cls) ? b : a; } @@ -XXX,XX +XXX,XX @@ static FloatPartsN *partsN(minmax)(FloatPartsN *a, FloatPartsN *b, return a; } - if (ab_mask & float_cmask_denormal) { - float_raise(float_flag_input_denormal_used, s); - } + record_denormals_used(ab_mask, s); a_exp = a->exp; b_exp = b->exp; @@ -XXX,XX +XXX,XX @@ static FloatRelation partsN(compare)(FloatPartsN *a, FloatPartsN *b, if (likely(cmask_is_only_normals(ab_mask))) { FloatRelation cmp; - if (ab_mask & float_cmask_denormal) { - float_raise(float_flag_input_denormal_used, s); - } + record_denormals_used(ab_mask, s); if (a->sign != b->sign) { goto a_sign; @@ -XXX,XX +XXX,XX @@ static FloatRelation partsN(compare)(FloatPartsN *a, FloatPartsN *b, return float_relation_unordered; } - if (ab_mask & float_cmask_denormal) { - float_raise(float_flag_input_denormal_used, s); - } + record_denormals_used(ab_mask, s); if (ab_mask & float_cmask_zero) { if (ab_mask == float_cmask_zero) { -- 2.43.0 Handle the scaling separately with parts64_scalbn. Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- fpu/softfloat.c | 37 ++++++++++++++++++++++++------------- fpu/softfloat-parts.c.inc | 20 ++++++++++---------- 2 files changed, 34 insertions(+), 23 deletions(-) diff --git a/fpu/softfloat.c b/fpu/softfloat.c index XXXXXXX..XXXXXXX 100644 --- a/fpu/softfloat.c +++ b/fpu/softfloat.c @@ -XXX,XX +XXX,XX @@ float16_muladd_scalbn(float16 a, float16 b, float16 c, FloatParts64 pa = float16_unpack_canonical(a, status); FloatParts64 pb = float16_unpack_canonical(b, status); FloatParts64 pc = float16_unpack_canonical(c, status); - FloatParts64 *pr = - parts64_muladd_scalbn(&pa, &pb, &pc, scale, flags, status); + FloatParts64 *pr = parts64_muladd(&pa, &pb, &pc, flags, status); - /* Round before applying negate result. */ + /* Before rounding, scale. */ + if (scale) { + parts64_scalbn(pr, scale, status); + } parts64_uncanon(pr, status, &float16_params, false); + /* After rounding, apply negate result, especially for -0.0. */ if ((flags & float_muladd_negate_result) && !is_nan(pr->cls)) { pr->sign ^= 1; } @@ -XXX,XX +XXX,XX @@ float32_muladd_scalbn(float32 a, float32 b, float32 c, FloatParts64 pa = float32_unpack_canonical(a, status); FloatParts64 pb = float32_unpack_canonical(b, status); FloatParts64 pc = float32_unpack_canonical(c, status); - FloatParts64 *pr = parts64_muladd_scalbn(&pa, &pb, &pc, scale, flags, status); + FloatParts64 *pr = parts64_muladd(&pa, &pb, &pc, flags, status); - /* Round before applying negate result. */ + /* Before rounding, scale. */ + if (scale) { + parts64_scalbn(pr, scale, status); + } parts64_uncanon(pr, status, &float32_params, false); + /* After rounding, apply negate result, especially for -0.0. */ if ((flags & float_muladd_negate_result) && !is_nan(pr->cls)) { pr->sign ^= 1; } @@ -XXX,XX +XXX,XX @@ float64_muladd_scalbn(float64 a, float64 b, float64 c, FloatParts64 pa = float64_unpack_canonical(a, status); FloatParts64 pb = float64_unpack_canonical(b, status); FloatParts64 pc = float64_unpack_canonical(c, status); - FloatParts64 *pr = parts64_muladd_scalbn(&pa, &pb, &pc, scale, flags, status); + FloatParts64 *pr = parts64_muladd(&pa, &pb, &pc, flags, status); - /* Round before applying negate result. */ + /* Before rounding, scale. */ + if (scale) { + parts64_scalbn(pr, scale, status); + } parts64_uncanon(pr, status, &float64_params, false); + /* After rounding, apply negate result, especially for -0.0. */ if ((flags & float_muladd_negate_result) && !is_nan(pr->cls)) { pr->sign ^= 1; } @@ -XXX,XX +XXX,XX @@ float64 float64r32_muladd(float64 a, float64 b, float64 c, FloatParts64 pa = float64_unpack_canonical(a, status); FloatParts64 pb = float64_unpack_canonical(b, status); FloatParts64 pc = float64_unpack_canonical(c, status); - FloatParts64 *pr = parts64_muladd_scalbn(&pa, &pb, &pc, 0, flags, status); + FloatParts64 *pr = parts64_muladd(&pa, &pb, &pc, flags, status); /* Round before applying negate result. */ parts64_uncanon(pr, status, &float32_params, false); @@ -XXX,XX +XXX,XX @@ bfloat16 QEMU_FLATTEN bfloat16_muladd(bfloat16 a, bfloat16 b, bfloat16 c, FloatParts64 pa = bfloat16_unpack_canonical(a, status); FloatParts64 pb = bfloat16_unpack_canonical(b, status); FloatParts64 pc = bfloat16_unpack_canonical(c, status); - FloatParts64 *pr = parts64_muladd_scalbn(&pa, &pb, &pc, 0, flags, status); + FloatParts64 *pr = parts64_muladd(&pa, &pb, &pc, flags, status); /* Round before applying negate result. */ parts64_uncanon(pr, status, &bfloat16_params, false); @@ -XXX,XX +XXX,XX @@ float128 QEMU_FLATTEN float128_muladd(float128 a, float128 b, float128 c, FloatParts128 pa = float128_unpack_canonical(a, status); FloatParts128 pb = float128_unpack_canonical(b, status); FloatParts128 pc = float128_unpack_canonical(c, status); - FloatParts128 *pr = parts128_muladd_scalbn(&pa, &pb, &pc, 0, flags, status); + FloatParts128 *pr = parts128_muladd(&pa, &pb, &pc, flags, status); /* Round before applying negate result. */ parts128_uncanon(pr, status, &float128_params, false); @@ -XXX,XX +XXX,XX @@ float32 float32_exp2(float32 a, float_status *status) rp = float64_unpack_canonical(float64_one, status); for (int i = 0; i < 15; i++) { tp = float64_unpack_canonical(float32_exp2_coefficients[i], status); - rp = *parts64_muladd_scalbn(&tp, &xnp, &rp, 0, 0, status); + rp = *parts64_muladd(&tp, &xnp, &rp, 0, status); xnp = *parts64_mul(&xnp, &xp, status); } @@ -XXX,XX +XXX,XX @@ static void parts_s390_divide_to_integer(FloatParts64 *a, FloatParts64 *b, /* Compute precise remainder */ r_precise_buf = *b; - r_precise = parts64_muladd_scalbn(&r_precise_buf, n, a, 0, - float_muladd_negate_product, status); + r_precise = parts64_muladd(&r_precise_buf, n, a, + float_muladd_negate_product, status); /* Round remainder to the target format */ *r = *r_precise; diff --git a/fpu/softfloat-parts.c.inc b/fpu/softfloat-parts.c.inc index XXXXXXX..XXXXXXX 100644 --- a/fpu/softfloat-parts.c.inc +++ b/fpu/softfloat-parts.c.inc @@ -XXX,XX +XXX,XX @@ static FloatPartsN *partsN(mul)(FloatPartsN *a, FloatPartsN *b, * `b' then adding 'c', with no intermediate rounding step after the * multiplication. The operation is performed according to the * IEC/IEEE Standard for Binary Floating-Point Arithmetic 754-2008. - * The flags argument allows the caller to select negation of the - * addend, the intermediate product, or the final result. (The - * difference between this and having the caller do a separate - * negation is that negating externally will flip the sign bit on NaNs.) + * The flags argument allows the caller to select negation of the addend + * or the intermediate product. (The difference between this and having + * the caller do a separate negation is that negating externally will + * flip the sign bit on NaNs.) Note that float_muladd_negate_result + * is not applied here, and should be handled separately after rounding + * chooses the final sign of 0.0. * * Requires A and C extracted into a double-sized structure to provide the * extra space for the widening multiply. */ -static FloatPartsN *partsN(muladd_scalbn)(FloatPartsN *a, FloatPartsN *b, - FloatPartsN *c, int scale, - int flags, float_status *s) +static FloatPartsN *partsN(muladd)(FloatPartsN *a, FloatPartsN *b, + FloatPartsN *c, + int flags, float_status *s) { int ab_mask, abc_mask; FloatPartsW p_widen, c_widen; @@ -XXX,XX +XXX,XX @@ static FloatPartsN *partsN(muladd_scalbn)(FloatPartsN *a, FloatPartsN *b, g_assert(ab_mask & float_cmask_zero); if (is_anynorm(c->cls)) { *a = *c; - goto return_normal; + goto finish_sign; } if (c->cls == float_class_zero) { if (flags & float_muladd_suppress_add_product_zero) { @@ -XXX,XX +XXX,XX @@ static FloatPartsN *partsN(muladd_scalbn)(FloatPartsN *a, FloatPartsN *b, a->sign = p_widen.sign; a->exp = p_widen.exp; - return_normal: - a->exp += scale; finish_sign: /* * All result types except for "return the default NaN -- 2.43.0 Check the likely case of normal product and normal or zero addend first; shift NaN and infinity detection down; end with zero product + addend. Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- fpu/softfloat-parts.c.inc | 155 +++++++++++++++++--------------------- 1 file changed, 70 insertions(+), 85 deletions(-) diff --git a/fpu/softfloat-parts.c.inc b/fpu/softfloat-parts.c.inc index XXXXXXX..XXXXXXX 100644 --- a/fpu/softfloat-parts.c.inc +++ b/fpu/softfloat-parts.c.inc @@ -XXX,XX +XXX,XX @@ static FloatPartsN *partsN(muladd)(FloatPartsN *a, FloatPartsN *b, FloatPartsN *c, int flags, float_status *s) { - int ab_mask, abc_mask; - FloatPartsW p_widen, c_widen; + int ab_mask = float_cmask(a->cls) | float_cmask(b->cls); + int c_mask = float_cmask(c->cls); + int abc_mask = ab_mask | c_mask; + bool c_sign = c->sign ^ !!(flags & float_muladd_negate_c); + bool p_sign = a->sign ^ b->sign ^ !!(flags & float_muladd_negate_product); - ab_mask = float_cmask(a->cls) | float_cmask(b->cls); - abc_mask = float_cmask(c->cls) | ab_mask; + /* + * The "likely" case is A and B normal, so that the product is normal, + * and C normal or zero so that the result is normal. + */ + int likely_mask = ab_mask | (c_mask & ~float_cmask_zero); + if (likely(cmask_is_only_normals(likely_mask))) { + record_denormals_used(abc_mask, s); + + /* Perform the multiplication step. */ + FloatPartsW p_widen = { .sign = p_sign, .exp = a->exp + b->exp + 1 }; + fracN(mulw)(&p_widen, a, b); + if (!(p_widen.frac_hi & DECOMPOSED_IMPLICIT_BIT)) { + fracW(add)(&p_widen, &p_widen, &p_widen); + p_widen.exp -= 1; + } + + /* Perform the addition step. */ + if (!(c_mask & float_cmask_zero)) { + /* Zero-extend C to less significant bits. */ + FloatPartsW c_widen = { .sign = c_sign, .exp = c->exp }; + fracN(widen)(&c_widen, c); + + if (p_sign == c_sign) { + partsW(add_normal)(&p_widen, &c_widen); + } else if (!partsW(sub_normal)(&p_widen, &c_widen)) { + goto return_sub_zero; + } + } + + /* Narrow with sticky bit, for proper rounding later. */ + fracN(truncjam)(a, &p_widen); + a->sign = p_widen.sign; + a->exp = p_widen.exp; + return a; + } /* * It is implementation-defined whether the cases of (0,inf,qnan) @@ -XXX,XX +XXX,XX @@ static FloatPartsN *partsN(muladd)(FloatPartsN *a, FloatPartsN *b, return a; } - if (flags & float_muladd_negate_c) { - c->sign ^= 1; + if (unlikely(ab_mask == float_cmask_infzero)) { + /* Inf * Zero == NaN */ + float_raise(float_flag_invalid | float_flag_invalid_imz, s); + goto d_nan; } - /* Compute the sign of the product into A. */ - a->sign ^= b->sign; - if (flags & float_muladd_negate_product) { - a->sign ^= 1; - } - - if (unlikely(!cmask_is_only_normals(ab_mask))) { - if (unlikely(ab_mask == float_cmask_infzero)) { - float_raise(float_flag_invalid | float_flag_invalid_imz, s); + if (unlikely(ab_mask & float_cmask_inf)) { + if ((c_mask & float_cmask_inf) && p_sign != c_sign) { + /* Inf - Inf == NaN */ + float_raise(float_flag_invalid | float_flag_invalid_isi, s); goto d_nan; } - - if (ab_mask & float_cmask_inf) { - if (c->cls == float_class_inf && a->sign != c->sign) { - float_raise(float_flag_invalid | float_flag_invalid_isi, s); - goto d_nan; - } - goto return_inf; - } - - g_assert(ab_mask & float_cmask_zero); - if (is_anynorm(c->cls)) { - *a = *c; - goto finish_sign; - } - if (c->cls == float_class_zero) { - if (flags & float_muladd_suppress_add_product_zero) { - a->sign = c->sign; - } else if (a->sign != c->sign) { - goto return_sub_zero; - } - goto return_zero; - } - g_assert(c->cls == float_class_inf); + /* Inf + C == Inf */ + record_denormals_used(abc_mask, s); + a->sign = p_sign; + a->cls = float_class_inf; + return a; } - - if (unlikely(c->cls == float_class_inf)) { - a->sign = c->sign; - goto return_inf; - } - - /* Perform the multiplication step. */ - p_widen.sign = a->sign; - p_widen.exp = a->exp + b->exp + 1; - fracN(mulw)(&p_widen, a, b); - if (!(p_widen.frac_hi & DECOMPOSED_IMPLICIT_BIT)) { - fracW(add)(&p_widen, &p_widen, &p_widen); - p_widen.exp -= 1; - } - - /* Perform the addition step. */ - if (c->cls != float_class_zero) { - /* Zero-extend C to less significant bits. */ - fracN(widen)(&c_widen, c); - c_widen.exp = c->exp; - - if (a->sign == c->sign) { - partsW(add_normal)(&p_widen, &c_widen); - } else if (!partsW(sub_normal)(&p_widen, &c_widen)) { - goto return_sub_zero; - } - } - - /* Narrow with sticky bit, for proper rounding later. */ - fracN(truncjam)(a, &p_widen); - a->sign = p_widen.sign; - a->exp = p_widen.exp; - - finish_sign: - /* - * All result types except for "return the default NaN - * because this is an Invalid Operation" go through here; - * this matches the set of cases where we consumed a - * denormal input. - */ record_denormals_used(abc_mask, s); - return a; + + /* Only remaining case is zero product. */ + assert(ab_mask & float_cmask_zero); + + /* + * 0 + C == C, + * except for 0 - 0, which needs special rounding, + * except for when we want to suppress this addition step. + */ + if (!(c_mask & float_cmask_zero) + || p_sign == c_sign + || (flags & float_muladd_suppress_add_product_zero)) { + c->sign = c_sign; + return c; + } return_sub_zero: + /* 0 - 0 == -0 for round_down, +0 otherwise. */ a->sign = s->float_rounding_mode == float_round_down; - return_zero: a->cls = float_class_zero; - goto finish_sign; - - return_inf: - a->cls = float_class_inf; - goto finish_sign; + return a; d_nan: *a = partsN(default_nan)(s); -- 2.43.0 At the same time, export. Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- include/fpu/softfloat-parts.h | 9 ++++ fpu/softfloat.c | 83 +++++++++++++++++------------------ fpu/softfloat-parts.c.inc | 38 ++++++++-------- 3 files changed, 69 insertions(+), 61 deletions(-) diff --git a/include/fpu/softfloat-parts.h b/include/fpu/softfloat-parts.h index XXXXXXX..XXXXXXX 100644 --- a/include/fpu/softfloat-parts.h +++ b/include/fpu/softfloat-parts.h @@ -XXX,XX +XXX,XX @@ FloatParts64 parts64_div(const FloatParts64 *a, const FloatParts64 *b, FloatParts128 parts128_div(const FloatParts128 *a, const FloatParts128 *b, float_status *s); +FloatParts64 parts64_muladd(const FloatParts64 *a, + const FloatParts64 *b, + const FloatParts64 *c, + int flags, float_status *s); +FloatParts128 parts128_muladd(const FloatParts128 *a, + const FloatParts128 *b, + const FloatParts128 *c, + int flags, float_status *s); + FloatParts64 parts64_round_to_int(const FloatParts64 *a, FloatRoundMode rmode, int scale, float_status *s, diff --git a/fpu/softfloat.c b/fpu/softfloat.c index XXXXXXX..XXXXXXX 100644 --- a/fpu/softfloat.c +++ b/fpu/softfloat.c @@ -XXX,XX +XXX,XX @@ float16_muladd_scalbn(float16 a, float16 b, float16 c, FloatParts64 pa = float16_unpack_canonical(a, status); FloatParts64 pb = float16_unpack_canonical(b, status); FloatParts64 pc = float16_unpack_canonical(c, status); - FloatParts64 *pr = parts64_muladd(&pa, &pb, &pc, flags, status); + FloatParts64 pr = parts64_muladd(&pa, &pb, &pc, flags, status); /* Before rounding, scale. */ if (scale) { - parts64_scalbn(pr, scale, status); + parts64_scalbn(&pr, scale, status); } - parts64_uncanon(pr, status, &float16_params, false); + parts64_uncanon(&pr, status, &float16_params, false); /* After rounding, apply negate result, especially for -0.0. */ - if ((flags & float_muladd_negate_result) && !is_nan(pr->cls)) { - pr->sign ^= 1; + if ((flags & float_muladd_negate_result) && !is_nan(pr.cls)) { + pr.sign ^= 1; } - return pack_raw64(pr, &float16_params); + return pack_raw64(&pr, &float16_params); } float16 float16_muladd(float16 a, float16 b, float16 c, @@ -XXX,XX +XXX,XX @@ float32_muladd_scalbn(float32 a, float32 b, float32 c, FloatParts64 pa = float32_unpack_canonical(a, status); FloatParts64 pb = float32_unpack_canonical(b, status); FloatParts64 pc = float32_unpack_canonical(c, status); - FloatParts64 *pr = parts64_muladd(&pa, &pb, &pc, flags, status); + FloatParts64 pr = parts64_muladd(&pa, &pb, &pc, flags, status); /* Before rounding, scale. */ if (scale) { - parts64_scalbn(pr, scale, status); + parts64_scalbn(&pr, scale, status); } - parts64_uncanon(pr, status, &float32_params, false); + parts64_uncanon(&pr, status, &float32_params, false); /* After rounding, apply negate result, especially for -0.0. */ - if ((flags & float_muladd_negate_result) && !is_nan(pr->cls)) { - pr->sign ^= 1; + if ((flags & float_muladd_negate_result) && !is_nan(pr.cls)) { + pr.sign ^= 1; } - return pack_raw64(pr, &float32_params); + return pack_raw64(&pr, &float32_params); } float64 QEMU_SOFTFLOAT_ATTR @@ -XXX,XX +XXX,XX @@ float64_muladd_scalbn(float64 a, float64 b, float64 c, FloatParts64 pa = float64_unpack_canonical(a, status); FloatParts64 pb = float64_unpack_canonical(b, status); FloatParts64 pc = float64_unpack_canonical(c, status); - FloatParts64 *pr = parts64_muladd(&pa, &pb, &pc, flags, status); + FloatParts64 pr = parts64_muladd(&pa, &pb, &pc, flags, status); /* Before rounding, scale. */ if (scale) { - parts64_scalbn(pr, scale, status); + parts64_scalbn(&pr, scale, status); } - parts64_uncanon(pr, status, &float64_params, false); + parts64_uncanon(&pr, status, &float64_params, false); /* After rounding, apply negate result, especially for -0.0. */ - if ((flags & float_muladd_negate_result) && !is_nan(pr->cls)) { - pr->sign ^= 1; + if ((flags & float_muladd_negate_result) && !is_nan(pr.cls)) { + pr.sign ^= 1; } - return pack_raw64(pr, &float64_params); + return pack_raw64(&pr, &float64_params); } static bool force_soft_fma; @@ -XXX,XX +XXX,XX @@ float64 float64r32_muladd(float64 a, float64 b, float64 c, FloatParts64 pa = float64_unpack_canonical(a, status); FloatParts64 pb = float64_unpack_canonical(b, status); FloatParts64 pc = float64_unpack_canonical(c, status); - FloatParts64 *pr = parts64_muladd(&pa, &pb, &pc, flags, status); + FloatParts64 pr = parts64_muladd(&pa, &pb, &pc, flags, status); /* Round before applying negate result. */ - parts64_uncanon(pr, status, &float32_params, false); - if ((flags & float_muladd_negate_result) && !is_nan(pr->cls)) { - pr->sign ^= 1; + parts64_uncanon(&pr, status, &float32_params, false); + if ((flags & float_muladd_negate_result) && !is_nan(pr.cls)) { + pr.sign ^= 1; } - return float64r32_pack_raw(pr); + return float64r32_pack_raw(&pr); } bfloat16 QEMU_FLATTEN bfloat16_muladd(bfloat16 a, bfloat16 b, bfloat16 c, @@ -XXX,XX +XXX,XX @@ bfloat16 QEMU_FLATTEN bfloat16_muladd(bfloat16 a, bfloat16 b, bfloat16 c, FloatParts64 pa = bfloat16_unpack_canonical(a, status); FloatParts64 pb = bfloat16_unpack_canonical(b, status); FloatParts64 pc = bfloat16_unpack_canonical(c, status); - FloatParts64 *pr = parts64_muladd(&pa, &pb, &pc, flags, status); + FloatParts64 pr = parts64_muladd(&pa, &pb, &pc, flags, status); /* Round before applying negate result. */ - parts64_uncanon(pr, status, &bfloat16_params, false); - if ((flags & float_muladd_negate_result) && !is_nan(pr->cls)) { - pr->sign ^= 1; + parts64_uncanon(&pr, status, &bfloat16_params, false); + if ((flags & float_muladd_negate_result) && !is_nan(pr.cls)) { + pr.sign ^= 1; } - return pack_raw64(pr, &bfloat16_params); + return pack_raw64(&pr, &bfloat16_params); } float128 QEMU_FLATTEN float128_muladd(float128 a, float128 b, float128 c, @@ -XXX,XX +XXX,XX @@ float128 QEMU_FLATTEN float128_muladd(float128 a, float128 b, float128 c, FloatParts128 pa = float128_unpack_canonical(a, status); FloatParts128 pb = float128_unpack_canonical(b, status); FloatParts128 pc = float128_unpack_canonical(c, status); - FloatParts128 *pr = parts128_muladd(&pa, &pb, &pc, flags, status); + FloatParts128 pr = parts128_muladd(&pa, &pb, &pc, flags, status); /* Round before applying negate result. */ - parts128_uncanon(pr, status, &float128_params, false); - if ((flags & float_muladd_negate_result) && !is_nan(pr->cls)) { - pr->sign ^= 1; + parts128_uncanon(&pr, status, &float128_params, false); + if ((flags & float_muladd_negate_result) && !is_nan(pr.cls)) { + pr.sign ^= 1; } - return float128_pack_raw(pr); + return float128_pack_raw(&pr); } /* @@ -XXX,XX +XXX,XX @@ float32 float32_exp2(float32 a, float_status *status) rp = float64_unpack_canonical(float64_one, status); for (int i = 0; i < 15; i++) { tp = float64_unpack_canonical(float32_exp2_coefficients[i], status); - rp = *parts64_muladd(&tp, &xnp, &rp, 0, status); + rp = parts64_muladd(&tp, &xnp, &rp, 0, status); xnp = *parts64_mul(&xnp, &xp, status); } @@ -XXX,XX +XXX,XX @@ static void parts_s390_divide_to_integer(FloatParts64 *a, FloatParts64 *b, n->sign = a->sign ^ b->sign; *cc = 0; } else { - FloatParts64 *q, q_buf, *r_precise, r_precise_buf; + FloatParts64 *q, q_buf, r_precise; int float_exception_flags = 0; bool is_q_smallish; uint32_t r_flags; @@ -XXX,XX +XXX,XX @@ static void parts_s390_divide_to_integer(FloatParts64 *a, FloatParts64 *b, 0, status, fmt); /* Compute precise remainder */ - r_precise_buf = *b; - r_precise = parts64_muladd(&r_precise_buf, n, a, + r_precise = parts64_muladd(b, n, a, float_muladd_negate_product, status); /* Round remainder to the target format */ - *r = *r_precise; + *r = r_precise; status->float_exception_flags = 0; *r = parts64_round_to_fmt(r, status, fmt); r_flags = status->float_exception_flags; @@ -XXX,XX +XXX,XX @@ static void parts_s390_divide_to_integer(FloatParts64 *a, FloatParts64 *b, * toward zero) or incremented. */ saved_r_sign = r->sign; - saved_r_precise_sign = r_precise->sign; + saved_r_precise_sign = r_precise.sign; r->sign = false; - r_precise->sign = false; - if (parts64_compare(r, r_precise, status, true) < + r_precise.sign = false; + if (parts64_compare(r, &r_precise, status, true) < float_relation_equal) { *dxc = 0x8; } else { *dxc = 0xc; } r->sign = saved_r_sign; - r_precise->sign = saved_r_precise_sign; + r_precise.sign = saved_r_precise_sign; } } } diff --git a/fpu/softfloat-parts.c.inc b/fpu/softfloat-parts.c.inc index XXXXXXX..XXXXXXX 100644 --- a/fpu/softfloat-parts.c.inc +++ b/fpu/softfloat-parts.c.inc @@ -XXX,XX +XXX,XX @@ static FloatPartsN *partsN(mul)(FloatPartsN *a, FloatPartsN *b, * Requires A and C extracted into a double-sized structure to provide the * extra space for the widening multiply. */ -static FloatPartsN *partsN(muladd)(FloatPartsN *a, FloatPartsN *b, - FloatPartsN *c, - int flags, float_status *s) +FloatPartsN partsN(muladd)(const FloatPartsN *a, const FloatPartsN *b, + const FloatPartsN *c, int flags, float_status *s) { int ab_mask = float_cmask(a->cls) | float_cmask(b->cls); int c_mask = float_cmask(c->cls); @@ -XXX,XX +XXX,XX @@ static FloatPartsN *partsN(muladd)(FloatPartsN *a, FloatPartsN *b, } /* Narrow with sticky bit, for proper rounding later. */ - fracN(truncjam)(a, &p_widen); - a->sign = p_widen.sign; - a->exp = p_widen.exp; - return a; + FloatPartsN r = { + .sign = p_widen.sign, + .exp = p_widen.exp, + .cls = float_class_normal, + }; + fracN(truncjam)(&r, &p_widen); + return r; } /* @@ -XXX,XX +XXX,XX @@ static FloatPartsN *partsN(muladd)(FloatPartsN *a, FloatPartsN *b, * off to the target-specific pick-a-NaN routine. */ if (unlikely(abc_mask & float_cmask_anynan)) { - *a = partsN(pick_nan_muladd)(a, b, c, s, ab_mask, abc_mask); - return a; + return partsN(pick_nan_muladd)(a, b, c, s, ab_mask, abc_mask); } if (unlikely(ab_mask == float_cmask_infzero)) { @@ -XXX,XX +XXX,XX @@ static FloatPartsN *partsN(muladd)(FloatPartsN *a, FloatPartsN *b, } /* Inf + C == Inf */ record_denormals_used(abc_mask, s); - a->sign = p_sign; - a->cls = float_class_inf; - return a; + return (FloatPartsN){ .sign = p_sign, .cls = float_class_inf }; } record_denormals_used(abc_mask, s); @@ -XXX,XX +XXX,XX @@ static FloatPartsN *partsN(muladd)(FloatPartsN *a, FloatPartsN *b, if (!(c_mask & float_cmask_zero) || p_sign == c_sign || (flags & float_muladd_suppress_add_product_zero)) { - c->sign = c_sign; - return c; + FloatPartsN r = *c; + r.sign = c_sign; + return r; } return_sub_zero: /* 0 - 0 == -0 for round_down, +0 otherwise. */ - a->sign = s->float_rounding_mode == float_round_down; - a->cls = float_class_zero; - return a; + return (FloatPartsN){ + .sign = s->float_rounding_mode == float_round_down, + .cls = float_class_zero + }; d_nan: - *a = partsN(default_nan)(s); - return a; + return partsN(default_nan)(s); } /* -- 2.43.0 The float16, bfloat16 and float128 paths are not so important that they require forced optimization. Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- fpu/softfloat.c | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/fpu/softfloat.c b/fpu/softfloat.c index XXXXXXX..XXXXXXX 100644 --- a/fpu/softfloat.c +++ b/fpu/softfloat.c @@ -XXX,XX +XXX,XX @@ floatx80_mul(floatx80 a, floatx80 b, float_status *status) * Fused multiply-add */ -float16 QEMU_FLATTEN -float16_muladd_scalbn(float16 a, float16 b, float16 c, - int scale, int flags, float_status *status) +float16 float16_muladd_scalbn(float16 a, float16 b, float16 c, + int scale, int flags, float_status *status) { FloatParts64 pa = float16_unpack_canonical(a, status); FloatParts64 pb = float16_unpack_canonical(b, status); @@ -XXX,XX +XXX,XX @@ float64 float64r32_muladd(float64 a, float64 b, float64 c, return float64r32_pack_raw(&pr); } -bfloat16 QEMU_FLATTEN bfloat16_muladd(bfloat16 a, bfloat16 b, bfloat16 c, - int flags, float_status *status) +bfloat16 bfloat16_muladd(bfloat16 a, bfloat16 b, bfloat16 c, + int flags, float_status *status) { FloatParts64 pa = bfloat16_unpack_canonical(a, status); FloatParts64 pb = bfloat16_unpack_canonical(b, status); @@ -XXX,XX +XXX,XX @@ bfloat16 QEMU_FLATTEN bfloat16_muladd(bfloat16 a, bfloat16 b, bfloat16 c, return pack_raw64(&pr, &bfloat16_params); } -float128 QEMU_FLATTEN float128_muladd(float128 a, float128 b, float128 c, - int flags, float_status *status) +float128 float128_muladd(float128 a, float128 b, float128 c, + int flags, float_status *status) { FloatParts128 pa = float128_unpack_canonical(a, status); FloatParts128 pb = float128_unpack_canonical(b, status); -- 2.43.0 At the same time, constify the inputs. Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- include/fpu/softfloat-parts.h | 5 +++++ fpu/softfloat-parts.c.inc | 4 ++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/include/fpu/softfloat-parts.h b/include/fpu/softfloat-parts.h index XXXXXXX..XXXXXXX 100644 --- a/include/fpu/softfloat-parts.h +++ b/include/fpu/softfloat-parts.h @@ -XXX,XX +XXX,XX @@ FloatParts128 parts128_return_nan(const FloatParts128 *a, float_status *s); * Operations */ +FloatRelation parts64_compare(const FloatParts64 *a, const FloatParts64 *b, + float_status *s, bool quiet); +FloatRelation parts128_compare(const FloatParts128 *a, const FloatParts128 *b, + float_status *s, bool quiet); + FloatParts64 parts64_div(const FloatParts64 *a, const FloatParts64 *b, float_status *s); FloatParts128 parts128_div(const FloatParts128 *a, const FloatParts128 *b, diff --git a/fpu/softfloat-parts.c.inc b/fpu/softfloat-parts.c.inc index XXXXXXX..XXXXXXX 100644 --- a/fpu/softfloat-parts.c.inc +++ b/fpu/softfloat-parts.c.inc @@ -XXX,XX +XXX,XX @@ static FloatPartsN *partsN(minmax)(FloatPartsN *a, FloatPartsN *b, /* * Floating point compare */ -static FloatRelation partsN(compare)(FloatPartsN *a, FloatPartsN *b, - float_status *s, bool is_quiet) +FloatRelation partsN(compare)(const FloatPartsN *a, const FloatPartsN *b, + float_status *s, bool is_quiet) { int ab_mask = float_cmask(a->cls) | float_cmask(b->cls); -- 2.43.0 At the same time, export. Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- include/fpu/softfloat-parts.h | 5 +++++ fpu/softfloat.c | 34 +++++++++++++++++----------------- fpu/softfloat-parts.c.inc | 35 ++++++++++++++++------------------- 3 files changed, 38 insertions(+), 36 deletions(-) diff --git a/include/fpu/softfloat-parts.h b/include/fpu/softfloat-parts.h index XXXXXXX..XXXXXXX 100644 --- a/include/fpu/softfloat-parts.h +++ b/include/fpu/softfloat-parts.h @@ -XXX,XX +XXX,XX @@ FloatParts64 parts64_div(const FloatParts64 *a, const FloatParts64 *b, FloatParts128 parts128_div(const FloatParts128 *a, const FloatParts128 *b, float_status *s); +FloatParts64 parts64_mul(const FloatParts64 *a, const FloatParts64 *b, + float_status *s); +FloatParts128 parts128_mul(const FloatParts128 *a, const FloatParts128 *b, + float_status *s); + FloatParts64 parts64_muladd(const FloatParts64 *a, const FloatParts64 *b, const FloatParts64 *c, diff --git a/fpu/softfloat.c b/fpu/softfloat.c index XXXXXXX..XXXXXXX 100644 --- a/fpu/softfloat.c +++ b/fpu/softfloat.c @@ -XXX,XX +XXX,XX @@ float16 QEMU_FLATTEN float16_mul(float16 a, float16 b, float_status *status) { FloatParts64 pa = float16_unpack_canonical(a, status); FloatParts64 pb = float16_unpack_canonical(b, status); - FloatParts64 *pr = parts64_mul(&pa, &pb, status); + FloatParts64 pr = parts64_mul(&pa, &pb, status); - return float16_round_pack_canonical(pr, status); + return float16_round_pack_canonical(&pr, status); } static float32 QEMU_SOFTFLOAT_ATTR @@ -XXX,XX +XXX,XX @@ soft_f32_mul(float32 a, float32 b, float_status *status) { FloatParts64 pa = float32_unpack_canonical(a, status); FloatParts64 pb = float32_unpack_canonical(b, status); - FloatParts64 *pr = parts64_mul(&pa, &pb, status); + FloatParts64 pr = parts64_mul(&pa, &pb, status); - return float32_round_pack_canonical(pr, status); + return float32_round_pack_canonical(&pr, status); } static float64 QEMU_SOFTFLOAT_ATTR @@ -XXX,XX +XXX,XX @@ soft_f64_mul(float64 a, float64 b, float_status *status) { FloatParts64 pa = float64_unpack_canonical(a, status); FloatParts64 pb = float64_unpack_canonical(b, status); - FloatParts64 *pr = parts64_mul(&pa, &pb, status); + FloatParts64 pr = parts64_mul(&pa, &pb, status); - return float64_round_pack_canonical(pr, status); + return float64_round_pack_canonical(&pr, status); } static float hard_f32_mul(float a, float b) @@ -XXX,XX +XXX,XX @@ float64 float64r32_mul(float64 a, float64 b, float_status *status) { FloatParts64 pa = float64_unpack_canonical(a, status); FloatParts64 pb = float64_unpack_canonical(b, status); - FloatParts64 *pr = parts64_mul(&pa, &pb, status); + FloatParts64 pr = parts64_mul(&pa, &pb, status); - return float64r32_round_pack_canonical(pr, status); + return float64r32_round_pack_canonical(&pr, status); } bfloat16 QEMU_FLATTEN @@ -XXX,XX +XXX,XX @@ bfloat16_mul(bfloat16 a, bfloat16 b, float_status *status) { FloatParts64 pa = bfloat16_unpack_canonical(a, status); FloatParts64 pb = bfloat16_unpack_canonical(b, status); - FloatParts64 *pr = parts64_mul(&pa, &pb, status); + FloatParts64 pr = parts64_mul(&pa, &pb, status); - return bfloat16_round_pack_canonical(pr, status); + return bfloat16_round_pack_canonical(&pr, status); } float128 QEMU_FLATTEN @@ -XXX,XX +XXX,XX @@ float128_mul(float128 a, float128 b, float_status *status) { FloatParts128 pa = float128_unpack_canonical(a, status); FloatParts128 pb = float128_unpack_canonical(b, status); - FloatParts128 *pr = parts128_mul(&pa, &pb, status); + FloatParts128 pr = parts128_mul(&pa, &pb, status); - return float128_round_pack_canonical(pr, status); + return float128_round_pack_canonical(&pr, status); } floatx80 QEMU_FLATTEN floatx80_mul(floatx80 a, floatx80 b, float_status *status) { - FloatParts128 pa, pb, *pr; + FloatParts128 pa, pb; if (!floatx80_unpack_canonical(&pa, a, status) || !floatx80_unpack_canonical(&pb, b, status)) { return floatx80_default_nan(status); } - pr = parts128_mul(&pa, &pb, status); - return floatx80_round_pack_canonical(pr, status); + pa = parts128_mul(&pa, &pb, status); + return floatx80_round_pack_canonical(&pa, status); } /* @@ -XXX,XX +XXX,XX @@ float32 float32_exp2(float32 a, float_status *status) float_raise(float_flag_inexact, status); tp = float64_unpack_canonical(float64_ln2, status); - xp = *parts64_mul(&xp, &tp, status); + xp = parts64_mul(&xp, &tp, status); xnp = xp; rp = float64_unpack_canonical(float64_one, status); for (int i = 0; i < 15; i++) { tp = float64_unpack_canonical(float32_exp2_coefficients[i], status); rp = parts64_muladd(&tp, &xnp, &rp, 0, status); - xnp = *parts64_mul(&xnp, &xp, status); + xnp = parts64_mul(&xnp, &xp, status); } return float32_round_pack_canonical(&rp, status); diff --git a/fpu/softfloat-parts.c.inc b/fpu/softfloat-parts.c.inc index XXXXXXX..XXXXXXX 100644 --- a/fpu/softfloat-parts.c.inc +++ b/fpu/softfloat-parts.c.inc @@ -XXX,XX +XXX,XX @@ static FloatPartsN *partsN(addsub)(FloatPartsN *a, FloatPartsN *b, * `b'. The operation is performed according to the IEC/IEEE Standard * for Binary Floating-Point Arithmetic. */ -static FloatPartsN *partsN(mul)(FloatPartsN *a, FloatPartsN *b, - float_status *s) +FloatPartsN partsN(mul)(const FloatPartsN *a, const FloatPartsN *b, + float_status *s) { int ab_mask = float_cmask(a->cls) | float_cmask(b->cls); bool sign = a->sign ^ b->sign; if (likely(cmask_is_only_normals(ab_mask))) { FloatPartsW tmp; + FloatPartsN r = { + .cls = float_class_normal, + .sign = sign, + .exp = a->exp + b->exp + 1, + }; record_denormals_used(ab_mask, s); fracN(mulw)(&tmp, a, b); - fracN(truncjam)(a, &tmp); + fracN(truncjam)(&r, &tmp); - a->exp += b->exp + 1; - if (!(a->frac_hi & DECOMPOSED_IMPLICIT_BIT)) { - fracN(add)(a, a, a); - a->exp -= 1; + if (!(r.frac_hi & DECOMPOSED_IMPLICIT_BIT)) { + fracN(add)(&r, &r, &r); + r.exp -= 1; } - a->sign = sign; - return a; + return r; } /* Inf * Zero == NaN */ if (unlikely(ab_mask == float_cmask_infzero)) { float_raise(float_flag_invalid | float_flag_invalid_imz, s); - *a = partsN(default_nan)(s); - return a; + return partsN(default_nan)(s); } if (unlikely(ab_mask & float_cmask_anynan)) { - *a = partsN(pick_nan)(a, b, s); - return a; + return partsN(pick_nan)(a, b, s); } /* Multiply by 0 or Inf */ record_denormals_used(ab_mask, s); if (ab_mask & float_cmask_inf) { - a->cls = float_class_inf; - a->sign = sign; - return a; + return (FloatPartsN){ .cls = float_class_inf, .sign = sign }; } g_assert(ab_mask & float_cmask_zero); - a->cls = float_class_zero; - a->sign = sign; - return a; + return (FloatPartsN){ .cls = float_class_zero, .sign = sign }; } /* -- 2.43.0 The nan test had been down below because it was unlikely. But if we have to have one anyway because of denormals, we might as well take care of them right away. Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- fpu/softfloat-parts.c.inc | 21 ++++++--------------- 1 file changed, 6 insertions(+), 15 deletions(-) diff --git a/fpu/softfloat-parts.c.inc b/fpu/softfloat-parts.c.inc index XXXXXXX..XXXXXXX 100644 --- a/fpu/softfloat-parts.c.inc +++ b/fpu/softfloat-parts.c.inc @@ -XXX,XX +XXX,XX @@ static FloatPartsN *partsN(addsub)(FloatPartsN *a, FloatPartsN *b, bool b_sign = b->sign ^ subtract; int ab_mask = float_cmask(a->cls) | float_cmask(b->cls); + if (unlikely(ab_mask & float_cmask_anynan)) { + *a = partsN(pick_nan)(a, b, s); + return a; + } + /* * For addition and subtraction, we will consume an * input denormal unless the other input is a NaN. */ - if (!(ab_mask & float_cmask_anynan)) { - record_denormals_used(ab_mask, s); - } + record_denormals_used(ab_mask, s); if (a->sign != b_sign) { /* Subtraction */ @@ -XXX,XX +XXX,XX @@ static FloatPartsN *partsN(addsub)(FloatPartsN *a, FloatPartsN *b, return a; } - if (unlikely(ab_mask & float_cmask_anynan)) { - goto p_nan; - } - if (ab_mask & float_cmask_inf) { if (a->cls != float_class_inf) { /* N - Inf */ @@ -XXX,XX +XXX,XX @@ static FloatPartsN *partsN(addsub)(FloatPartsN *a, FloatPartsN *b, return a; } - if (unlikely(ab_mask & float_cmask_anynan)) { - goto p_nan; - } - if (ab_mask & float_cmask_inf) { a->cls = float_class_inf; return a; @@ -XXX,XX +XXX,XX @@ static FloatPartsN *partsN(addsub)(FloatPartsN *a, FloatPartsN *b, return_b: b->sign = b_sign; return b; - - p_nan: - *a = partsN(pick_nan)(a, b, s); - return a; } /* -- 2.43.0 At the same time, export. Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- include/fpu/softfloat-parts.h | 5 ++++ fpu/softfloat.c | 30 +++++++++++------------ fpu/softfloat-parts.c.inc | 45 ++++++++++++++++++----------------- 3 files changed, 43 insertions(+), 37 deletions(-) diff --git a/include/fpu/softfloat-parts.h b/include/fpu/softfloat-parts.h index XXXXXXX..XXXXXXX 100644 --- a/include/fpu/softfloat-parts.h +++ b/include/fpu/softfloat-parts.h @@ -XXX,XX +XXX,XX @@ FloatParts128 parts128_return_nan(const FloatParts128 *a, float_status *s); * Operations */ +FloatParts64 parts64_addsub(const FloatParts64 *a, const FloatParts64 *b, + float_status *s, bool subtract); +FloatParts128 parts128_addsub(const FloatParts128 *a, const FloatParts128 *b, + float_status *s, bool subtract); + FloatRelation parts64_compare(const FloatParts64 *a, const FloatParts64 *b, float_status *s, bool quiet); FloatRelation parts128_compare(const FloatParts128 *a, const FloatParts128 *b, diff --git a/fpu/softfloat.c b/fpu/softfloat.c index XXXXXXX..XXXXXXX 100644 --- a/fpu/softfloat.c +++ b/fpu/softfloat.c @@ -XXX,XX +XXX,XX @@ float16_addsub(float16 a, float16 b, float_status *status, bool subtract) { FloatParts64 pa = float16_unpack_canonical(a, status); FloatParts64 pb = float16_unpack_canonical(b, status); - FloatParts64 *pr = parts64_addsub(&pa, &pb, status, subtract); + FloatParts64 pr = parts64_addsub(&pa, &pb, status, subtract); - return float16_round_pack_canonical(pr, status); + return float16_round_pack_canonical(&pr, status); } float16 float16_add(float16 a, float16 b, float_status *status) @@ -XXX,XX +XXX,XX @@ soft_f32_addsub(float32 a, float32 b, float_status *status, bool subtract) { FloatParts64 pa = float32_unpack_canonical(a, status); FloatParts64 pb = float32_unpack_canonical(b, status); - FloatParts64 *pr = parts64_addsub(&pa, &pb, status, subtract); + FloatParts64 pr = parts64_addsub(&pa, &pb, status, subtract); - return float32_round_pack_canonical(pr, status); + return float32_round_pack_canonical(&pr, status); } static float32 soft_f32_add(float32 a, float32 b, float_status *status) @@ -XXX,XX +XXX,XX @@ soft_f64_addsub(float64 a, float64 b, float_status *status, bool subtract) { FloatParts64 pa = float64_unpack_canonical(a, status); FloatParts64 pb = float64_unpack_canonical(b, status); - FloatParts64 *pr = parts64_addsub(&pa, &pb, status, subtract); + FloatParts64 pr = parts64_addsub(&pa, &pb, status, subtract); - return float64_round_pack_canonical(pr, status); + return float64_round_pack_canonical(&pr, status); } static float64 soft_f64_add(float64 a, float64 b, float_status *status) @@ -XXX,XX +XXX,XX @@ static float64 float64r32_addsub(float64 a, float64 b, float_status *status, { FloatParts64 pa = float64_unpack_canonical(a, status); FloatParts64 pb = float64_unpack_canonical(b, status); - FloatParts64 *pr = parts64_addsub(&pa, &pb, status, subtract); + FloatParts64 pr = parts64_addsub(&pa, &pb, status, subtract); - return float64r32_round_pack_canonical(pr, status); + return float64r32_round_pack_canonical(&pr, status); } float64 float64r32_add(float64 a, float64 b, float_status *status) @@ -XXX,XX +XXX,XX @@ bfloat16_addsub(bfloat16 a, bfloat16 b, float_status *status, bool subtract) { FloatParts64 pa = bfloat16_unpack_canonical(a, status); FloatParts64 pb = bfloat16_unpack_canonical(b, status); - FloatParts64 *pr = parts64_addsub(&pa, &pb, status, subtract); + FloatParts64 pr = parts64_addsub(&pa, &pb, status, subtract); - return bfloat16_round_pack_canonical(pr, status); + return bfloat16_round_pack_canonical(&pr, status); } bfloat16 bfloat16_add(bfloat16 a, bfloat16 b, float_status *status) @@ -XXX,XX +XXX,XX @@ float128_addsub(float128 a, float128 b, float_status *status, bool subtract) { FloatParts128 pa = float128_unpack_canonical(a, status); FloatParts128 pb = float128_unpack_canonical(b, status); - FloatParts128 *pr = parts128_addsub(&pa, &pb, status, subtract); + FloatParts128 pr = parts128_addsub(&pa, &pb, status, subtract); - return float128_round_pack_canonical(pr, status); + return float128_round_pack_canonical(&pr, status); } float128 float128_add(float128 a, float128 b, float_status *status) @@ -XXX,XX +XXX,XX @@ float128 float128_sub(float128 a, float128 b, float_status *status) static floatx80 QEMU_FLATTEN floatx80_addsub(floatx80 a, floatx80 b, float_status *status, bool subtract) { - FloatParts128 pa, pb, *pr; + FloatParts128 pa, pb; if (!floatx80_unpack_canonical(&pa, a, status) || !floatx80_unpack_canonical(&pb, b, status)) { return floatx80_default_nan(status); } - pr = parts128_addsub(&pa, &pb, status, subtract); - return floatx80_round_pack_canonical(pr, status); + pa = parts128_addsub(&pa, &pb, status, subtract); + return floatx80_round_pack_canonical(&pa, status); } floatx80 floatx80_add(floatx80 a, floatx80 b, float_status *status) diff --git a/fpu/softfloat-parts.c.inc b/fpu/softfloat-parts.c.inc index XXXXXXX..XXXXXXX 100644 --- a/fpu/softfloat-parts.c.inc +++ b/fpu/softfloat-parts.c.inc @@ -XXX,XX +XXX,XX @@ static void partsN(uncanon)(FloatPartsN *p, float_status *s, * according to the IEC/IEEE Standard for Binary Floating-Point * Arithmetic. */ -static FloatPartsN *partsN(addsub)(FloatPartsN *a, FloatPartsN *b, - float_status *s, bool subtract) +FloatPartsN partsN(addsub)(const FloatPartsN *a_orig, + const FloatPartsN *b_orig, + float_status *s, bool subtract) { - bool b_sign = b->sign ^ subtract; - int ab_mask = float_cmask(a->cls) | float_cmask(b->cls); + int ab_mask = float_cmask(a_orig->cls) | float_cmask(b_orig->cls); if (unlikely(ab_mask & float_cmask_anynan)) { - *a = partsN(pick_nan)(a, b, s); - return a; + return partsN(pick_nan)(a_orig, b_orig, s); } /* @@ -XXX,XX +XXX,XX @@ static FloatPartsN *partsN(addsub)(FloatPartsN *a, FloatPartsN *b, */ record_denormals_used(ab_mask, s); - if (a->sign != b_sign) { + FloatPartsN a = *a_orig; + FloatPartsN b = *b_orig; + + b.sign ^= subtract; + + if (a.sign != b.sign) { /* Subtraction */ if (likely(cmask_is_only_normals(ab_mask))) { - if (partsN(sub_normal)(a, b)) { + if (partsN(sub_normal)(&a, &b)) { return a; } /* Subtract was exact, fall through to set sign. */ @@ -XXX,XX +XXX,XX @@ static FloatPartsN *partsN(addsub)(FloatPartsN *a, FloatPartsN *b, } if (ab_mask == float_cmask_zero) { - a->sign = s->float_rounding_mode == float_round_down; + a.sign = s->float_rounding_mode == float_round_down; return a; } if (ab_mask & float_cmask_inf) { - if (a->cls != float_class_inf) { + if (a.cls != float_class_inf) { /* N - Inf */ - goto return_b; + return b; } - if (b->cls != float_class_inf) { + if (b.cls != float_class_inf) { /* Inf - N */ return a; } /* Inf - Inf */ float_raise(float_flag_invalid | float_flag_invalid_isi, s); - *a = partsN(default_nan)(s); - return a; + return partsN(default_nan)(s); } } else { /* Addition */ if (likely(cmask_is_only_normals(ab_mask))) { - partsN(add_normal)(a, b); + partsN(add_normal)(&a, &b); return a; } @@ -XXX,XX +XXX,XX @@ static FloatPartsN *partsN(addsub)(FloatPartsN *a, FloatPartsN *b, } if (ab_mask & float_cmask_inf) { - a->cls = float_class_inf; + a.cls = float_class_inf; return a; } } - if (b->cls == float_class_zero) { - g_assert(is_anynorm(a->cls)); + if (b.cls == float_class_zero) { + g_assert(is_anynorm(a.cls)); return a; } - g_assert(a->cls == float_class_zero); - g_assert(is_anynorm(b->cls)); - return_b: - b->sign = b_sign; + g_assert(a.cls == float_class_zero); + g_assert(is_anynorm(b.cls)); return b; } -- 2.43.0 Consolidate the tests for zero and anynorm. Add comments for a few cases. Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- fpu/softfloat-parts.c.inc | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/fpu/softfloat-parts.c.inc b/fpu/softfloat-parts.c.inc index XXXXXXX..XXXXXXX 100644 --- a/fpu/softfloat-parts.c.inc +++ b/fpu/softfloat-parts.c.inc @@ -XXX,XX +XXX,XX @@ FloatPartsN partsN(addsub)(const FloatPartsN *a_orig, } if (ab_mask == float_cmask_zero) { + /* 0 - 0 */ a.sign = s->float_rounding_mode == float_round_down; return a; } @@ -XXX,XX +XXX,XX @@ FloatPartsN partsN(addsub)(const FloatPartsN *a_orig, } if (ab_mask == float_cmask_zero) { + /* 0 + 0 */ return a; } if (ab_mask & float_cmask_inf) { + /* N + Inf or Inf + N */ a.cls = float_class_inf; return a; } } - if (b.cls == float_class_zero) { - g_assert(is_anynorm(a.cls)); - return a; - } - - g_assert(a.cls == float_class_zero); - g_assert(is_anynorm(b.cls)); - return b; + /* 0 +/- N or N +/- 0 */ + assert((ab_mask & float_cmask_zero) && (ab_mask & float_cmask_anynorm)); + return b.cls == float_class_zero ? a : b; } /* -- 2.43.0 Now that we've exposed enough infrastructure, this can be implemented in the backend that needs it. Reviewed-by: Ilya Leoshkevich <iii@linux.ibm.com> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- include/fpu/softfloat.h | 11 --- fpu/softfloat.c | 137 ---------------------------------- target/s390x/tcg/fpu_helper.c | 135 +++++++++++++++++++++++++++++++++ 3 files changed, 135 insertions(+), 148 deletions(-) diff --git a/include/fpu/softfloat.h b/include/fpu/softfloat.h index XXXXXXX..XXXXXXX 100644 --- a/include/fpu/softfloat.h +++ b/include/fpu/softfloat.h @@ -XXX,XX +XXX,XX @@ static inline bool float128_unordered_quiet(float128 a, float128 b, *----------------------------------------------------------------------------*/ float128 float128_default_nan(float_status *status); -#define DECLARE_S390_DIVIDE_TO_INTEGER(floatN) \ -void floatN ## _s390_divide_to_integer(floatN a, floatN b, \ - int final_quotient_rounding_mode, \ - bool mask_underflow, bool mask_inexact, \ - floatN *r, floatN *n, \ - uint32_t *cc, int *dxc, \ - float_status *status) -DECLARE_S390_DIVIDE_TO_INTEGER(float32); -DECLARE_S390_DIVIDE_TO_INTEGER(float64); - - #endif /* SOFTFLOAT_H */ diff --git a/fpu/softfloat.c b/fpu/softfloat.c index XXXXXXX..XXXXXXX 100644 --- a/fpu/softfloat.c +++ b/fpu/softfloat.c @@ -XXX,XX +XXX,XX @@ floatx80 floatx80_round(floatx80 a, float_status *status) return floatx80_round_pack_canonical(&p, status); } -static void parts_s390_divide_to_integer(FloatParts64 *a, FloatParts64 *b, - int final_quotient_rounding_mode, - bool mask_underflow, bool mask_inexact, - const FloatFmt *fmt, - FloatParts64 *r, FloatParts64 *n, - uint32_t *cc, int *dxc, - float_status *status) -{ - /* POp table "Results: DIVIDE TO INTEGER (Part 1 of 2)" */ - if ((float_cmask(a->cls) | float_cmask(b->cls)) & float_cmask_anynan) { - *r = parts64_pick_nan(a, b, status); - *n = *r; - *cc = 1; - } else if (a->cls == float_class_inf || b->cls == float_class_zero) { - *r = parts64_default_nan(status); - *n = *r; - *cc = 1; - status->float_exception_flags |= float_flag_invalid; - } else if (b->cls == float_class_inf) { - *r = *a; - n->cls = float_class_zero; - n->sign = a->sign ^ b->sign; - *cc = 0; - } else { - FloatParts64 *q, q_buf, r_precise; - int float_exception_flags = 0; - bool is_q_smallish; - uint32_t r_flags; - - /* Compute precise quotient */ - q_buf = parts64_div(a, b, status); - q = &q_buf; - - /* - * Check whether two closest integers can be precisely represented, - * i.e., all their bits fit into the fractional part. - */ - is_q_smallish = q->exp < (fmt->frac_size + 1); - - /* - * Final quotient is rounded using final-quotient-rounding method, and - * partial quotient is rounded toward zero. - * - * Rounding of partial quotient may be inexact. This is the whole point - * of distinguishing partial quotients, so ignore the exception. - */ - *n = parts64_round_to_int(q, - is_q_smallish - ? final_quotient_rounding_mode - : float_round_to_zero, - 0, status, fmt); - - /* Compute precise remainder */ - r_precise = parts64_muladd(b, n, a, - float_muladd_negate_product, status); - - /* Round remainder to the target format */ - *r = r_precise; - status->float_exception_flags = 0; - *r = parts64_round_to_fmt(r, status, fmt); - r_flags = status->float_exception_flags; - - /* POp table "Results: DIVIDE TO INTEGER (Part 2 of 2)" */ - if (is_q_smallish) { - if (r->cls != float_class_zero) { - if (r->exp < 2 - (1 << (fmt->exp_size - 1))) { - if (mask_underflow) { - float_exception_flags |= float_flag_underflow; - *dxc = 0x10; - r->exp += fmt->exp_re_bias; - } - } else if (r_flags & float_flag_inexact) { - float_exception_flags |= float_flag_inexact; - if (mask_inexact) { - bool saved_r_sign, saved_r_precise_sign; - - /* - * Check whether remainder was truncated (rounded - * toward zero) or incremented. - */ - saved_r_sign = r->sign; - saved_r_precise_sign = r_precise.sign; - r->sign = false; - r_precise.sign = false; - if (parts64_compare(r, &r_precise, status, true) < - float_relation_equal) { - *dxc = 0x8; - } else { - *dxc = 0xc; - } - r->sign = saved_r_sign; - r_precise.sign = saved_r_precise_sign; - } - } - } - *cc = 0; - } else if (n->exp > (1 << (fmt->exp_size - 1)) - 1) { - n->exp -= fmt->exp_re_bias; - *cc = r->cls == float_class_zero ? 1 : 3; - } else { - *cc = r->cls == float_class_zero ? 0 : 2; - } - - /* Adjust signs of zero results */ - if (r->cls == float_class_zero) { - r->sign = a->sign; - } - if (n->cls == float_class_zero) { - n->sign = a->sign ^ b->sign; - } - - status->float_exception_flags = float_exception_flags; - } -} - -#define DEFINE_S390_DIVIDE_TO_INTEGER(floatN) \ -void floatN ## _s390_divide_to_integer(floatN a, floatN b, \ - int final_quotient_rounding_mode, \ - bool mask_underflow, bool mask_inexact, \ - floatN *r, floatN *n, \ - uint32_t *cc, int *dxc, \ - float_status *status) \ -{ \ - FloatParts64 pa = floatN ## _unpack_canonical(a, status); \ - FloatParts64 pb = floatN ## _unpack_canonical(b, status); \ - FloatParts64 pr, pn; \ - parts_s390_divide_to_integer(&pa, &pb, final_quotient_rounding_mode, \ - mask_underflow, mask_inexact, \ - &floatN ## _params, \ - &pr, &pn, cc, dxc, status); \ - *r = floatN ## _round_pack_canonical(&pr, status); \ - *n = floatN ## _round_pack_canonical(&pn, status); \ -} - -DEFINE_S390_DIVIDE_TO_INTEGER(float32) -DEFINE_S390_DIVIDE_TO_INTEGER(float64) - static void __attribute__((constructor)) softfloat_init(void) { union_float64 ua, ub, uc, ur; diff --git a/target/s390x/tcg/fpu_helper.c b/target/s390x/tcg/fpu_helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/s390x/tcg/fpu_helper.c +++ b/target/s390x/tcg/fpu_helper.c @@ -XXX,XX +XXX,XX @@ #include "tcg_s390x.h" #include "exec/helper-proto.h" #include "fpu/softfloat.h" +#include "fpu/softfloat-parts.h" /* #define DEBUG_HELPER */ #ifdef DEBUG_HELPER @@ -XXX,XX +XXX,XX @@ Int128 HELPER(dxb)(CPUS390XState *env, Int128 a, Int128 b) return RET128(ret); } +static void parts_s390_divide_to_integer(FloatParts64 *a, FloatParts64 *b, + int final_quotient_rounding_mode, + bool mask_underflow, bool mask_inexact, + const FloatFmt *fmt, + FloatParts64 *r, FloatParts64 *n, + uint32_t *cc, int *dxc, + float_status *status) +{ + /* POp table "Results: DIVIDE TO INTEGER (Part 1 of 2)" */ + if ((float_cmask(a->cls) | float_cmask(b->cls)) & float_cmask_anynan) { + *r = parts64_pick_nan(a, b, status); + *n = *r; + *cc = 1; + } else if (a->cls == float_class_inf || b->cls == float_class_zero) { + *r = parts64_default_nan(status); + *n = *r; + *cc = 1; + status->float_exception_flags |= float_flag_invalid; + } else if (b->cls == float_class_inf) { + *r = *a; + n->cls = float_class_zero; + n->sign = a->sign ^ b->sign; + *cc = 0; + } else { + FloatParts64 *q, q_buf, r_precise; + int float_exception_flags = 0; + bool is_q_smallish; + uint32_t r_flags; + + /* Compute precise quotient */ + q_buf = parts64_div(a, b, status); + q = &q_buf; + + /* + * Check whether two closest integers can be precisely represented, + * i.e., all their bits fit into the fractional part. + */ + is_q_smallish = q->exp < (fmt->frac_size + 1); + + /* + * Final quotient is rounded using final-quotient-rounding method, and + * partial quotient is rounded toward zero. + * + * Rounding of partial quotient may be inexact. This is the whole point + * of distinguishing partial quotients, so ignore the exception. + */ + *n = parts64_round_to_int(q, + is_q_smallish + ? final_quotient_rounding_mode + : float_round_to_zero, + 0, status, fmt); + + /* Compute precise remainder */ + r_precise = parts64_muladd(b, n, a, + float_muladd_negate_product, status); + + /* Round remainder to the target format */ + *r = r_precise; + status->float_exception_flags = 0; + *r = parts64_round_to_fmt(r, status, fmt); + r_flags = status->float_exception_flags; + + /* POp table "Results: DIVIDE TO INTEGER (Part 2 of 2)" */ + if (is_q_smallish) { + if (r->cls != float_class_zero) { + if (r->exp < 2 - (1 << (fmt->exp_size - 1))) { + if (mask_underflow) { + float_exception_flags |= float_flag_underflow; + *dxc = 0x10; + r->exp += fmt->exp_re_bias; + } + } else if (r_flags & float_flag_inexact) { + float_exception_flags |= float_flag_inexact; + if (mask_inexact) { + bool saved_r_sign, saved_r_precise_sign; + + /* + * Check whether remainder was truncated (rounded + * toward zero) or incremented. + */ + saved_r_sign = r->sign; + saved_r_precise_sign = r_precise.sign; + r->sign = false; + r_precise.sign = false; + if (parts64_compare(r, &r_precise, status, true) < + float_relation_equal) { + *dxc = 0x8; + } else { + *dxc = 0xc; + } + r->sign = saved_r_sign; + r_precise.sign = saved_r_precise_sign; + } + } + } + *cc = 0; + } else if (n->exp > (1 << (fmt->exp_size - 1)) - 1) { + n->exp -= fmt->exp_re_bias; + *cc = r->cls == float_class_zero ? 1 : 3; + } else { + *cc = r->cls == float_class_zero ? 0 : 2; + } + + /* Adjust signs of zero results */ + if (r->cls == float_class_zero) { + r->sign = a->sign; + } + if (n->cls == float_class_zero) { + n->sign = a->sign ^ b->sign; + } + + status->float_exception_flags = float_exception_flags; + } +} + +#define DEFINE_S390_DIVIDE_TO_INTEGER(floatN) \ +static void floatN ## _s390_divide_to_integer(floatN a, floatN b, \ + int final_quotient_rounding_mode, bool mask_underflow, bool mask_inexact, \ + floatN *r, floatN *n, uint32_t *cc, int *dxc, float_status *status) \ +{ \ + FloatParts64 pa = floatN ## _unpack_canonical(a, status); \ + FloatParts64 pb = floatN ## _unpack_canonical(b, status); \ + FloatParts64 pr, pn; \ + parts_s390_divide_to_integer(&pa, &pb, final_quotient_rounding_mode, \ + mask_underflow, mask_inexact, \ + &floatN ## _params, \ + &pr, &pn, cc, dxc, status); \ + *r = floatN ## _round_pack_canonical(&pr, status); \ + *n = floatN ## _round_pack_canonical(&pn, status); \ +} + +DEFINE_S390_DIVIDE_TO_INTEGER(float32) +DEFINE_S390_DIVIDE_TO_INTEGER(float64) + void HELPER(dib)(CPUS390XState *env, uint32_t r1, uint32_t r2, uint32_t r3, uint32_t m4, uint32_t bits) { -- 2.43.0 Use softfloat-parts.h so that we can more naturally perform the required operations witha single rounding step. This happens to also simplify the NaN detection step. Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- target/arm/tcg/vec_helper.c | 77 +++++++++++++++++++------------------ 1 file changed, 40 insertions(+), 37 deletions(-) diff --git a/target/arm/tcg/vec_helper.c b/target/arm/tcg/vec_helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/tcg/vec_helper.c +++ b/target/arm/tcg/vec_helper.c @@ -XXX,XX +XXX,XX @@ #include "helper.h" #include "tcg/tcg-gvec-desc.h" #include "fpu/softfloat.h" +#include "fpu/softfloat-parts.h" #include "qemu/int128.h" #include "crypto/clmul.h" #include "vec_internal.h" @@ -XXX,XX +XXX,XX @@ float32 bfdotadd(float32 sum, uint32_t e1, uint32_t e2, float_status *fpst) float32 bfdotadd_ebf(float32 sum, uint32_t e1, uint32_t e2, float_status *fpst, float_status *fpst_odd) { + /* Unpack two BFloat16 into two Float32, trivially. */ float32 s1r = e1 << 16; float32 s1c = e1 & 0xffff0000u; float32 s2r = e2 << 16; float32 s2c = e2 & 0xffff0000u; float32 t32; + /* + * Compare f16_dotadd() in sme_helper.c, but here we have + * bfloat16 inputs. In particular that means that we do not + * want the FPCR.FZ16 flush semantics, so we use the normal + * float_status for the input handling here. + */ + FloatParts64 p1r = float32_unpack_canonical(s1r, fpst); + FloatParts64 p1c = float32_unpack_canonical(s1c, fpst); + FloatParts64 p2r = float32_unpack_canonical(s2r, fpst); + FloatParts64 p2c = float32_unpack_canonical(s2c, fpst); + + int all_mask = (float_cmask(p1r.cls) | float_cmask(p1c.cls) | + float_cmask(p1r.cls) | float_cmask(p1c.cls)); + /* C.f. FPProcessNaNs4 */ - if (float32_is_any_nan(s1r) || float32_is_any_nan(s1c) || - float32_is_any_nan(s2r) || float32_is_any_nan(s2c)) { - if (float32_is_signaling_nan(s1r, fpst)) { - t32 = s1r; - } else if (float32_is_signaling_nan(s1c, fpst)) { - t32 = s1c; - } else if (float32_is_signaling_nan(s2r, fpst)) { - t32 = s2r; - } else if (float32_is_signaling_nan(s2c, fpst)) { - t32 = s2c; - } else if (float32_is_any_nan(s1r)) { - t32 = s1r; - } else if (float32_is_any_nan(s1c)) { - t32 = s1c; - } else if (float32_is_any_nan(s2r)) { - t32 = s2r; + if (unlikely(all_mask & float_cmask_anynan)) { + if (unlikely(all_mask & float_cmask_snan)) { + if (p1r.cls == float_class_snan) { + t32 = s1r; + } else if (p1c.cls == float_class_snan) { + t32 = s1c; + } else if (p2r.cls == float_class_snan) { + t32 = s2r; + } else { + t32 = s2c; + } } else { - t32 = s2c; + if (p1r.cls == float_class_qnan) { + t32 = s1r; + } else if (p1c.cls == float_class_qnan) { + t32 = s1c; + } else if (p2r.cls == float_class_qnan) { + t32 = s2r; + } else { + t32 = s2c; + } } /* * FPConvertNaN(FPProcessNaN(t32)) will be done as part * of the final addition below. */ } else { - /* - * Compare f16_dotadd() in sme_helper.c, but here we have - * bfloat16 inputs. In particular that means that we do not - * want the FPCR.FZ16 flush semantics, so we use the normal - * float_status for the input handling here. - */ - float64 e1r = float32_to_float64(s1r, fpst); - float64 e1c = float32_to_float64(s1c, fpst); - float64 e2r = float32_to_float64(s2r, fpst); - float64 e2c = float32_to_float64(s2c, fpst); - float64 t64; - /* * The ARM pseudocode function FPDot performs both multiplies - * and the add with a single rounding operation. Emulate this - * by performing the first multiply in round-to-odd, then doing - * the second multiply as fused multiply-add, and rounding to - * float32 all in one step. + * and the add with a single rounding operation. */ - t64 = float64_mul(e1r, e2r, fpst_odd); - t64 = float64r32_muladd(e1c, e2c, t64, 0, fpst); + FloatParts64 tmp = parts64_mul(&p1r, &p2r, fpst); + tmp = parts64_muladd(&p1c, &p2c, &tmp, 0, fpst); - /* This conversion is exact, because we've already rounded. */ - t32 = float64_to_float32(t64, fpst); + t32 = float32_round_pack_canonical(&tmp, fpst); } /* The final accumulation step is not fused. */ -- 2.43.0 This argument is no longer used. Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- target/arm/tcg/vec_internal.h | 12 +++------ target/arm/tcg/sme_helper.c | 6 ++--- target/arm/tcg/vec_helper.c | 50 +++++++++++++++-------------------- 3 files changed, 29 insertions(+), 39 deletions(-) diff --git a/target/arm/tcg/vec_internal.h b/target/arm/tcg/vec_internal.h index XXXXXXX..XXXXXXX 100644 --- a/target/arm/tcg/vec_internal.h +++ b/target/arm/tcg/vec_internal.h @@ -XXX,XX +XXX,XX @@ float32 bfdotadd(float32 sum, uint32_t e1, uint32_t e2, float_status *fpst); * @sum: addend * @e1, @e2: multiplicand vectors * @fpst: floating-point status to use - * @fpst_odd: floating-point status to use for round-to-odd operations * * BFloat16 2-way dot product of @e1 & @e2, accumulating with @sum. * The @e1 and @e2 operands correspond to the 32-bit source vector @@ -XXX,XX +XXX,XX @@ float32 bfdotadd(float32 sum, uint32_t e1, uint32_t e2, float_status *fpst); * Corresponds to the ARM pseudocode function BFDotAdd, specialized * for the FPCR.EBF == 1 case. */ -float32 bfdotadd_ebf(float32 sum, uint32_t e1, uint32_t e2, - float_status *fpst, float_status *fpst_odd); +float32 bfdotadd_ebf(float32 sum, uint32_t e1, uint32_t e2, float_status *fpst); /** * is_ebf: * @env: CPU state * @statusp: pointer to floating point status to fill in - * @oddstatusp: pointer to floating point status to fill in for round-to-odd * * Determine whether a BFDotAdd operation should use FPCR.EBF = 0 - * or FPCR.EBF = 1 semantics. On return, has initialized *statusp - * and *oddstatusp to suitable float_status arguments to use with either - * bfdotadd() or bfdotadd_ebf(). + * or FPCR.EBF = 1 semantics. On return, has initialized *statusp as suitable + * for float_status arguments to either bfdotadd() or bfdotadd_ebf(). * Returns true for EBF = 1, false for EBF = 0. (The caller should use this * to decide whether to call bfdotadd() or bfdotadd_ebf().) */ -bool is_ebf(CPUARMState *env, float_status *statusp, float_status *oddstatusp); +bool is_ebf(CPUARMState *env, float_status *statusp); /* * Negate as for FPCR.AH=1 -- do not negate NaNs. diff --git a/target/arm/tcg/sme_helper.c b/target/arm/tcg/sme_helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/tcg/sme_helper.c +++ b/target/arm/tcg/sme_helper.c @@ -XXX,XX +XXX,XX @@ static void do_bfmopa_w(void *vza, void *vzn, void *vzm, uint32_t desc, uint32_t negx, bool ah_neg) { intptr_t row, col, oprsz = simd_maxsz(desc); - float_status fpst, fpst_odd; + float_status fpst; - if (is_ebf(env, &fpst, &fpst_odd)) { + if (is_ebf(env, &fpst)) { for (row = 0; row < oprsz; ) { uint16_t prow = pn[H2(row >> 4)]; do { @@ -XXX,XX +XXX,XX @@ static void do_bfmopa_w(void *vza, void *vzn, void *vzm, uint32_t m = *(uint32_t *)(vzm + H1_4(col)); m = f16mop_adj_pair(m, pcol, 0); - *a = bfdotadd_ebf(*a, n, m, &fpst, &fpst_odd); + *a = bfdotadd_ebf(*a, n, m, &fpst); } col += 4; pcol >>= 4; diff --git a/target/arm/tcg/vec_helper.c b/target/arm/tcg/vec_helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/tcg/vec_helper.c +++ b/target/arm/tcg/vec_helper.c @@ -XXX,XX +XXX,XX @@ DO_MMLA_B(gvec_usmmla_b, do_usmmla_b) * BFloat16 Dot Product */ -bool is_ebf(CPUARMState *env, float_status *statusp, float_status *oddstatusp) +bool is_ebf(CPUARMState *env, float_status *statusp) { /* * For BFDOT, BFMMLA, etc, the behaviour depends on FPCR.EBF. @@ -XXX,XX +XXX,XX @@ bool is_ebf(CPUARMState *env, float_status *statusp, float_status *oddstatusp) *statusp = env->vfp.fp_status[is_a64(env) ? FPST_A64 : FPST_A32]; set_default_nan_mode(true, statusp); - if (ebf) { - /* EBF=1 needs to do a step with round-to-odd semantics */ - *oddstatusp = *statusp; - set_float_rounding_mode(float_round_to_odd, oddstatusp); - } else { + if (!ebf) { set_flush_to_zero(true, statusp); set_flush_inputs_to_zero(true, statusp); set_float_rounding_mode(float_round_to_odd_inf, statusp); @@ -XXX,XX +XXX,XX @@ float32 bfdotadd(float32 sum, uint32_t e1, uint32_t e2, float_status *fpst) return t1; } -float32 bfdotadd_ebf(float32 sum, uint32_t e1, uint32_t e2, - float_status *fpst, float_status *fpst_odd) +float32 bfdotadd_ebf(float32 sum, uint32_t e1, uint32_t e2, float_status *fpst) { /* Unpack two BFloat16 into two Float32, trivially. */ float32 s1r = e1 << 16; @@ -XXX,XX +XXX,XX @@ void HELPER(gvec_bfdot)(void *vd, void *vn, void *vm, void *va, intptr_t i, opr_sz = simd_oprsz(desc); float32 *d = vd, *a = va; uint32_t *n = vn, *m = vm; - float_status fpst, fpst_odd; + float_status fpst; - if (is_ebf(env, &fpst, &fpst_odd)) { + if (is_ebf(env, &fpst)) { for (i = 0; i < opr_sz / 4; ++i) { - d[i] = bfdotadd_ebf(a[i], n[i], m[i], &fpst, &fpst_odd); + d[i] = bfdotadd_ebf(a[i], n[i], m[i], &fpst); } } else { for (i = 0; i < opr_sz / 4; ++i) { @@ -XXX,XX +XXX,XX @@ void HELPER(gvec_bfdot_idx)(void *vd, void *vn, void *vm, intptr_t eltspersegment = MIN(16 / 4, elements); float32 *d = vd, *a = va; uint32_t *n = vn, *m = vm; - float_status fpst, fpst_odd; + float_status fpst; - if (is_ebf(env, &fpst, &fpst_odd)) { + if (is_ebf(env, &fpst)) { for (i = 0; i < elements; i += eltspersegment) { uint32_t m_idx = m[i + H4(index)]; for (j = i; j < i + eltspersegment; j++) { - d[j] = bfdotadd_ebf(a[j], n[j], m_idx, &fpst, &fpst_odd); + d[j] = bfdotadd_ebf(a[j], n[j], m_idx, &fpst); } } } else { @@ -XXX,XX +XXX,XX @@ void HELPER(sme2_bfvdot_idx)(void *vd, void *vn, void *vm, uint16_t *n0 = vn; uint16_t *n1 = vn + sizeof(ARMVectorReg); uint32_t *m = vm; - float_status fpst, fpst_odd; + float_status fpst; - if (is_ebf(env, &fpst, &fpst_odd)) { + if (is_ebf(env, &fpst)) { for (i = 0; i < elements; i += eltspersegment) { uint32_t m_idx = m[i + H4(idx)]; for (j = 0; j < eltspersegment; j++) { uint32_t nn = (n0[H2(2 * (i + j) + sel)]) | (n1[H2(2 * (i + j) + sel)] << 16); - d[i + H4(j)] = bfdotadd_ebf(a[i + H4(j)], nn, m_idx, - &fpst, &fpst_odd); + d[i + H4(j)] = bfdotadd_ebf(a[i + H4(j)], nn, m_idx, &fpst); } } } else { @@ -XXX,XX +XXX,XX @@ void HELPER(gvec_bfmmla)(void *vd, void *vn, void *vm, void *va, intptr_t s, opr_sz = simd_oprsz(desc); float32 *d = vd, *a = va; uint32_t *n = vn, *m = vm; - float_status fpst, fpst_odd; + float_status fpst; - if (is_ebf(env, &fpst, &fpst_odd)) { + if (is_ebf(env, &fpst)) { for (s = 0; s < opr_sz / 4; s += 4) { float32 sum00, sum01, sum10, sum11; @@ -XXX,XX +XXX,XX @@ void HELPER(gvec_bfmmla)(void *vd, void *vn, void *vm, void *va, * i j i k j k */ sum00 = a[s + H4(0 + 0)]; - sum00 = bfdotadd_ebf(sum00, n[s + H4(0 + 0)], m[s + H4(0 + 0)], &fpst, &fpst_odd); - sum00 = bfdotadd_ebf(sum00, n[s + H4(0 + 1)], m[s + H4(0 + 1)], &fpst, &fpst_odd); + sum00 = bfdotadd_ebf(sum00, n[s + H4(0 + 0)], m[s + H4(0 + 0)], &fpst); + sum00 = bfdotadd_ebf(sum00, n[s + H4(0 + 1)], m[s + H4(0 + 1)], &fpst); sum01 = a[s + H4(0 + 1)]; - sum01 = bfdotadd_ebf(sum01, n[s + H4(0 + 0)], m[s + H4(2 + 0)], &fpst, &fpst_odd); - sum01 = bfdotadd_ebf(sum01, n[s + H4(0 + 1)], m[s + H4(2 + 1)], &fpst, &fpst_odd); + sum01 = bfdotadd_ebf(sum01, n[s + H4(0 + 0)], m[s + H4(2 + 0)], &fpst); + sum01 = bfdotadd_ebf(sum01, n[s + H4(0 + 1)], m[s + H4(2 + 1)], &fpst); sum10 = a[s + H4(2 + 0)]; - sum10 = bfdotadd_ebf(sum10, n[s + H4(2 + 0)], m[s + H4(0 + 0)], &fpst, &fpst_odd); - sum10 = bfdotadd_ebf(sum10, n[s + H4(2 + 1)], m[s + H4(0 + 1)], &fpst, &fpst_odd); + sum10 = bfdotadd_ebf(sum10, n[s + H4(2 + 0)], m[s + H4(0 + 0)], &fpst); + sum10 = bfdotadd_ebf(sum10, n[s + H4(2 + 1)], m[s + H4(0 + 1)], &fpst); sum11 = a[s + H4(2 + 1)]; - sum11 = bfdotadd_ebf(sum11, n[s + H4(2 + 0)], m[s + H4(2 + 0)], &fpst, &fpst_odd); - sum11 = bfdotadd_ebf(sum11, n[s + H4(2 + 1)], m[s + H4(2 + 1)], &fpst, &fpst_odd); + sum11 = bfdotadd_ebf(sum11, n[s + H4(2 + 0)], m[s + H4(2 + 0)], &fpst); + sum11 = bfdotadd_ebf(sum11, n[s + H4(2 + 1)], m[s + H4(2 + 1)], &fpst); d[s + H4(0 + 0)] = sum00; d[s + H4(0 + 1)] = sum01; -- 2.43.0 Use softfloat-parts.h so that we can more naturally perform the required operations witha single rounding step. This happens to also simplify the NaN detection step. Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- target/arm/tcg/sme_helper.c | 96 ++++++++++++++++--------------------- 1 file changed, 40 insertions(+), 56 deletions(-) diff --git a/target/arm/tcg/sme_helper.c b/target/arm/tcg/sme_helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/tcg/sme_helper.c +++ b/target/arm/tcg/sme_helper.c @@ -XXX,XX +XXX,XX @@ #include "accel/tcg/helper-retaddr.h" #include "qemu/int128.h" #include "fpu/softfloat.h" +#include "fpu/softfloat-parts.h" #include "vec_internal.h" #include "sve_ldst_internal.h" @@ -XXX,XX +XXX,XX @@ static inline uint32_t bf16mop_ah_neg_adj_pair(uint32_t pair, uint32_t pg) } static float32 f16_dotadd(float32 sum, uint32_t e1, uint32_t e2, - float_status *s_f16, float_status *s_std, - float_status *s_odd) + float_status *s_f16, float_status *s_std) { /* - * We need three different float_status for different parts of this + * We need two different float_status for different parts of this * operation: * - the input conversion of the float16 values must use the * f16-specific float_status, so that the FPCR.FZ16 control is applied * - operations on float32 including the final accumulation must use * the normal float_status, so that FPCR.FZ is applied - * - we have pre-set-up copy of s_std which is set to round-to-odd, - * for the multiply (see below) */ float16 h1r = e1 & 0xffff; float16 h1c = e1 >> 16; @@ -XXX,XX +XXX,XX @@ static float32 f16_dotadd(float32 sum, uint32_t e1, uint32_t e2, float16 h2c = e2 >> 16; float32 t32; + FloatParts64 p1r = float16_unpack_canonical(h1r, s_f16); + FloatParts64 p1c = float16_unpack_canonical(h1c, s_f16); + FloatParts64 p2r = float16_unpack_canonical(h2r, s_f16); + FloatParts64 p2c = float16_unpack_canonical(h2c, s_f16); + + int all_mask = (float_cmask(p1r.cls) | float_cmask(p1c.cls) | + float_cmask(p1r.cls) | float_cmask(p1c.cls)); + /* C.f. FPProcessNaNs4 */ - if (float16_is_any_nan(h1r) || float16_is_any_nan(h1c) || - float16_is_any_nan(h2r) || float16_is_any_nan(h2c)) { + if (unlikely(all_mask & float_cmask_anynan)) { float16 t16; - if (float16_is_signaling_nan(h1r, s_f16)) { - t16 = h1r; - } else if (float16_is_signaling_nan(h1c, s_f16)) { - t16 = h1c; - } else if (float16_is_signaling_nan(h2r, s_f16)) { - t16 = h2r; - } else if (float16_is_signaling_nan(h2c, s_f16)) { - t16 = h2c; - } else if (float16_is_any_nan(h1r)) { - t16 = h1r; - } else if (float16_is_any_nan(h1c)) { - t16 = h1c; - } else if (float16_is_any_nan(h2r)) { - t16 = h2r; + if (unlikely(all_mask & float_cmask_snan)) { + if (p1r.cls == float_class_snan) { + t16 = h1r; + } else if (p1c.cls == float_class_snan) { + t16 = h1c; + } else if (p2r.cls == float_class_snan) { + t16 = h2r; + } else { + t16 = h2c; + } } else { - t16 = h2c; + if (p1r.cls == float_class_qnan) { + t16 = h1r; + } else if (p1c.cls == float_class_qnan) { + t16 = h1c; + } else if (p2r.cls == float_class_qnan) { + t16 = h2r; + } else { + t16 = h2c; + } } t32 = float16_to_float32(t16, true, s_f16); } else { - float64 e1r = float16_to_float64(h1r, true, s_f16); - float64 e1c = float16_to_float64(h1c, true, s_f16); - float64 e2r = float16_to_float64(h2r, true, s_f16); - float64 e2c = float16_to_float64(h2c, true, s_f16); - float64 t64; - /* * The ARM pseudocode function FPDot performs both multiplies - * and the add with a single rounding operation. Emulate this - * by performing the first multiply in round-to-odd, then doing - * the second multiply as fused multiply-add, and rounding to - * float32 all in one step. + * and the add with a single rounding operation. */ - t64 = float64_mul(e1r, e2r, s_odd); - t64 = float64r32_muladd(e1c, e2c, t64, 0, s_std); + FloatParts64 tmp = parts64_mul(&p1r, &p2r, s_std); + tmp = parts64_muladd(&p1c, &p2c, &tmp, 0, s_std); - /* This conversion is exact, because we've already rounded. */ - t32 = float64_to_float32(t64, s_std); + t32 = float32_round_pack_canonical(&tmp, s_std); } /* The final accumulation step is not fused. */ @@ -XXX,XX +XXX,XX @@ static void do_fmopa_w_h(void *vza, void *vzn, void *vzm, uint16_t *pn, uint32_t negx, bool ah_neg) { intptr_t row, col, oprsz = simd_maxsz(desc); - float_status fpst_odd = env->vfp.fp_status[FPST_ZA]; - - set_float_rounding_mode(float_round_to_odd, &fpst_odd); for (row = 0; row < oprsz; ) { uint16_t prow = pn[H2(row >> 4)]; @@ -XXX,XX +XXX,XX @@ static void do_fmopa_w_h(void *vza, void *vzn, void *vzm, uint16_t *pn, m = f16mop_adj_pair(m, pcol, 0); *a = f16_dotadd(*a, n, m, &env->vfp.fp_status[FPST_ZA_F16], - &env->vfp.fp_status[FPST_ZA], - &fpst_odd); + &env->vfp.fp_status[FPST_ZA]); } col += 4; pcol >>= 4; @@ -XXX,XX +XXX,XX @@ void HELPER(sme2_fdot_h)(void *vd, void *vn, void *vm, void *va, bool za = extract32(desc, SIMD_DATA_SHIFT, 1); float_status *fpst_std = &env->vfp.fp_status[za ? FPST_ZA : FPST_A64]; float_status *fpst_f16 = &env->vfp.fp_status[za ? FPST_ZA_F16 : FPST_A64_F16]; - float_status fpst_odd = *fpst_std; float32 *d = vd, *a = va; uint32_t *n = vn, *m = vm; - set_float_rounding_mode(float_round_to_odd, &fpst_odd); - for (i = 0; i < oprsz / sizeof(float32); ++i) { d[H4(i)] = f16_dotadd(a[H4(i)], n[H4(i)], m[H4(i)], - fpst_f16, fpst_std, &fpst_odd); + fpst_f16, fpst_std); } } @@ -XXX,XX +XXX,XX @@ void HELPER(sme2_fdot_idx_h)(void *vd, void *vn, void *vm, void *va, bool za = extract32(desc, SIMD_DATA_SHIFT + 2, 1); float_status *fpst_std = &env->vfp.fp_status[za ? FPST_ZA : FPST_A64]; float_status *fpst_f16 = &env->vfp.fp_status[za ? FPST_ZA_F16 : FPST_A64_F16]; - float_status fpst_odd = *fpst_std; float32 *d = vd, *a = va; uint32_t *n = vn, *m = (uint32_t *)vm + H4(idx); - set_float_rounding_mode(float_round_to_odd, &fpst_odd); - for (i = 0; i < elements; i += eltspersegment) { uint32_t mm = m[i]; for (j = 0; j < eltspersegment; ++j) { d[H4(i + j)] = f16_dotadd(a[H4(i + j)], n[H4(i + j)], mm, - fpst_f16, fpst_std, &fpst_odd); + fpst_f16, fpst_std); } } } @@ -XXX,XX +XXX,XX @@ void HELPER(sme2_fvdot_idx_h)(void *vd, void *vn, void *vm, void *va, intptr_t eltspersegment = MIN(4, elements); int idx = extract32(desc, SIMD_DATA_SHIFT, 2); int sel = extract32(desc, SIMD_DATA_SHIFT + 2, 1); - float_status fpst_odd, *fpst_std, *fpst_f16; float32 *d = vd, *a = va; uint16_t *n0 = vn; uint16_t *n1 = vn + sizeof(ARMVectorReg); uint32_t *m = (uint32_t *)vm + H4(idx); - fpst_std = &env->vfp.fp_status[FPST_ZA]; - fpst_f16 = &env->vfp.fp_status[FPST_ZA_F16]; - fpst_odd = *fpst_std; - set_float_rounding_mode(float_round_to_odd, &fpst_odd); - for (i = 0; i < elements; i += eltspersegment) { uint32_t mm = m[i]; for (j = 0; j < eltspersegment; ++j) { uint32_t nn = (n0[H2(2 * (i + j) + sel)]) | (n1[H2(2 * (i + j) + sel)] << 16); d[i + H4(j)] = f16_dotadd(a[i + H4(j)], nn, mm, - fpst_f16, fpst_std, &fpst_odd); + &env->vfp.fp_status[FPST_ZA_F16], + &env->vfp.fp_status[FPST_ZA]); } } } -- 2.43.0 At the same time, export. Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- include/fpu/softfloat-parts.h | 3 +++ fpu/softfloat-parts.c.inc | 14 ++++++++------ 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/include/fpu/softfloat-parts.h b/include/fpu/softfloat-parts.h index XXXXXXX..XXXXXXX 100644 --- a/include/fpu/softfloat-parts.h +++ b/include/fpu/softfloat-parts.h @@ -XXX,XX +XXX,XX @@ FloatParts128 parts128_round_to_int(const FloatParts128 *a, FloatParts64 parts64_round_to_fmt(const FloatParts64 *p, float_status *s, const FloatFmt *fmt); +FloatParts64 parts64_scalbn(const FloatParts64 *a, int n, float_status *s); +FloatParts128 parts128_scalbn(const FloatParts128 *a, int n, float_status *s); + #endif diff --git a/fpu/softfloat-parts.c.inc b/fpu/softfloat-parts.c.inc index XXXXXXX..XXXXXXX 100644 --- a/fpu/softfloat-parts.c.inc +++ b/fpu/softfloat-parts.c.inc @@ -XXX,XX +XXX,XX @@ FloatRelation partsN(compare)(const FloatPartsN *a, const FloatPartsN *b, /* * Multiply A by 2 raised to the power N. */ -static void partsN(scalbn)(FloatPartsN *a, int n, float_status *s) +FloatPartsN partsN(scalbn)(const FloatPartsN *a, int n, float_status *s) { switch (a->cls) { case float_class_snan: case float_class_qnan: - *a = partsN(return_nan)(a, s); - break; + return partsN(return_nan)(a, s); case float_class_zero: case float_class_inf: - break; + return *a; case float_class_denormal: float_raise(float_flag_input_denormal_used, s); /* fall through */ case float_class_normal: - a->exp += MIN(MAX(n, -0x10000), 0x10000); - break; + { + FloatPartsN r = *a; + r.exp += MIN(MAX(n, -0x10000), 0x10000); + return r; + } default: g_assert_not_reached(); } -- 2.43.0 Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- include/qemu/host-utils.h | 60 ++++++++++++++++++++++++++++++++++++ accel/tcg/tcg-runtime-gvec.c | 20 +++--------- 2 files changed, 64 insertions(+), 16 deletions(-) diff --git a/include/qemu/host-utils.h b/include/qemu/host-utils.h index XXXXXXX..XXXXXXX 100644 --- a/include/qemu/host-utils.h +++ b/include/qemu/host-utils.h @@ -XXX,XX +XXX,XX @@ static inline bool umul64_overflow(uint64_t x, uint64_t y, uint64_t *ret) return __builtin_mul_overflow(x, y, ret); } +/** + * sadd32_saturate - addition with saturation + * @x, @y: addends + * + * Computes @x + @y, and saturates rathern than truncating the result. + */ +static inline int32_t sadd32_saturate(int32_t x, int32_t y) +{ + int32_t ret; + if (sadd32_overflow(x, y, &ret)) { + ret = y < 0 ? INT32_MIN : INT32_MAX; + } + return ret; +} + +/** + * sadd64_saturate - addition with saturation + * @x, @y: addends + * + * Computes @x + @y, and saturates rathern than truncating the result. + */ +static inline int64_t sadd64_saturate(int64_t x, int64_t y) +{ + int64_t ret; + if (sadd64_overflow(x, y, &ret)) { + ret = y < 0 ? INT64_MIN : INT64_MAX; + } + return ret; +} + +/** + * ssub32_saturate - subtraction with saturation + * @x, @y: addends + * + * Computes @x + @y, and saturates rathern than truncating the result. + */ +static inline bool ssub32_saturate(int32_t x, int32_t y) +{ + int32_t ret; + if (ssub32_overflow(x, y, &ret)) { + ret = x < 0 ? INT32_MAX : INT32_MIN; + } + return ret; +} + +/** + * ssub64_saturate - subtraction with saturation + * @x, @y: addends + * + * Computes @x + @y, and saturates rathern than truncating the result. + */ +static inline bool ssub64_saturate(int64_t x, int64_t y) +{ + int64_t ret; + if (ssub64_overflow(x, y, &ret)) { + ret = x < 0 ? INT64_MAX : INT64_MIN; + } + return ret; +} + /* * Unsigned 128x64 multiplication. * Returns true if the result got truncated to 128 bits. diff --git a/accel/tcg/tcg-runtime-gvec.c b/accel/tcg/tcg-runtime-gvec.c index XXXXXXX..XXXXXXX 100644 --- a/accel/tcg/tcg-runtime-gvec.c +++ b/accel/tcg/tcg-runtime-gvec.c @@ -XXX,XX +XXX,XX @@ void HELPER(gvec_ssadd32)(void *d, void *a, void *b, uint32_t desc) for (i = 0; i < oprsz; i += sizeof(int32_t)) { int32_t ai = *(int32_t *)(a + i); int32_t bi = *(int32_t *)(b + i); - int32_t di; - if (sadd32_overflow(ai, bi, &di)) { - di = (di < 0 ? INT32_MAX : INT32_MIN); - } + int32_t di = sadd32_saturate(ai, bi); *(int32_t *)(d + i) = di; } clear_high(d, oprsz, desc); @@ -XXX,XX +XXX,XX @@ void HELPER(gvec_ssadd64)(void *d, void *a, void *b, uint32_t desc) for (i = 0; i < oprsz; i += sizeof(int64_t)) { int64_t ai = *(int64_t *)(a + i); int64_t bi = *(int64_t *)(b + i); - int64_t di; - if (sadd64_overflow(ai, bi, &di)) { - di = (di < 0 ? INT64_MAX : INT64_MIN); - } + int64_t di = sadd64_saturate(ai, bi); *(int64_t *)(d + i) = di; } clear_high(d, oprsz, desc); @@ -XXX,XX +XXX,XX @@ void HELPER(gvec_sssub32)(void *d, void *a, void *b, uint32_t desc) for (i = 0; i < oprsz; i += sizeof(int32_t)) { int32_t ai = *(int32_t *)(a + i); int32_t bi = *(int32_t *)(b + i); - int32_t di; - if (ssub32_overflow(ai, bi, &di)) { - di = (di < 0 ? INT32_MAX : INT32_MIN); - } + int32_t di = ssub32_saturate(ai, bi); *(int32_t *)(d + i) = di; } clear_high(d, oprsz, desc); @@ -XXX,XX +XXX,XX @@ void HELPER(gvec_sssub64)(void *d, void *a, void *b, uint32_t desc) for (i = 0; i < oprsz; i += sizeof(int64_t)) { int64_t ai = *(int64_t *)(a + i); int64_t bi = *(int64_t *)(b + i); - int64_t di; - if (ssub64_overflow(ai, bi, &di)) { - di = (di < 0 ? INT64_MAX : INT64_MIN); - } + int64_t di = ssub64_saturate(ai, bi); *(int64_t *)(d + i) = di; } clear_high(d, oprsz, desc); -- 2.43.0 Notice the conflict between saturation and rebias_overflow, which should never happen. Otherwise, saturate to INT32_MAX and allow the usual overflow to max, infinity, or dnan. Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- fpu/softfloat-parts.c.inc | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/fpu/softfloat-parts.c.inc b/fpu/softfloat-parts.c.inc index XXXXXXX..XXXXXXX 100644 --- a/fpu/softfloat-parts.c.inc +++ b/fpu/softfloat-parts.c.inc @@ -XXX,XX +XXX,XX @@ static void partsN(uncanon_normal)(FloatPartsN *p, float_status *s, g_assert_not_reached(); } - exp = p->exp + fmt->exp_bias; + /* Because exp_bias is positive, we can only overflow past INT_MAX. */ + if (sadd32_overflow(p->exp, fmt->exp_bias, &exp)) { + /* + * rebias_overflow wants to compute a modulo exponent, which + * conflicts with saturation. That said, saturation can only + * happen with scalbn, which is not a PowerPC operation. + */ + assert(!s->rebias_overflow); + exp = INT32_MAX; + } + if (likely(exp > 0)) { if (p->frac_lo & round_mask) { flags |= float_flag_inexact; -- 2.43.0 Avoid exponent overflow as well as checking that we don't lose information with opposing scaling. Use it in partsN(scalbn) and partsN(round_to_int_normal). Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- fpu/softfloat.c | 17 +++++++++++++++++ fpu/softfloat-parts.c.inc | 5 ++--- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/fpu/softfloat.c b/fpu/softfloat.c index XXXXXXX..XXXXXXX 100644 --- a/fpu/softfloat.c +++ b/fpu/softfloat.c @@ -XXX,XX +XXX,XX @@ static float128 QEMU_FLATTEN float128_pack_raw(const FloatParts128 *p) *----------------------------------------------------------------------------*/ #include "softfloat-specialize.c.inc" +static int32_t exp_scalbn(int32_t exp, int32_t scale) +{ + /* + * Catch chains of scaling which lose information. + * In particular, if the exponent has been saturated, + * do not allow it to become unsaturated. + */ + if (unlikely(exp == INT32_MAX)) { + assert(scale >= 0); + } else if (unlikely(exp == INT32_MIN)) { + assert(scale <= 0); + } else { + exp = sadd32_saturate(exp, scale); + } + return exp; +} + /* * Helper functions for softfloat-parts.c.inc, per-size operations. */ diff --git a/fpu/softfloat-parts.c.inc b/fpu/softfloat-parts.c.inc index XXXXXXX..XXXXXXX 100644 --- a/fpu/softfloat-parts.c.inc +++ b/fpu/softfloat-parts.c.inc @@ -XXX,XX +XXX,XX @@ static bool partsN(round_to_int_normal)(FloatPartsN *a, FloatRoundMode rmode, uint64_t frac_lsb, frac_lsbm1, rnd_even_mask, rnd_mask, inc; int shift_adj; - scale = MIN(MAX(scale, -0x10000), 0x10000); - a->exp += scale; + a->exp = exp_scalbn(a->exp, scale); if (a->exp < 0) { bool one; @@ -XXX,XX +XXX,XX @@ FloatPartsN partsN(scalbn)(const FloatPartsN *a, int n, float_status *s) case float_class_normal: { FloatPartsN r = *a; - r.exp += MIN(MAX(n, -0x10000), 0x10000); + r.exp = exp_scalbn(r.exp, n); return r; } default: -- 2.43.0 Fixes for scalbn and muladd. r~ Richard Henderson (12): fpu: Return struct from parts{64,128}_scalbn fpu: Reorganize partsN(muladd) fpu: Return struct from parts{64,128}_muladd fpu: Hoist nan check in partsN_addsub fpu: Simplify 0 +/- N case in parts_addsub fpu: Use parts64_round_to_int in parts_s390_divide_to_integer target/s390x: Move float{32,64}_s390_divide_to_integer target/arm: Use FloatParts64 in bfdotadd_ebf target/arm: Drop oddstatus from is_ebf and bfdotadd_ebf target/arm: Use FloatParts64 in f16_dotadd fpu: Saturate the exponent in uncanon_normal fpu: Introduce exp_scalbn include/fpu/softfloat-parts.h | 12 ++ include/fpu/softfloat.h | 11 -- target/arm/tcg/vec_internal.h | 12 +- fpu/softfloat.c | 236 ++++++++-------------------------- target/arm/tcg/sme_helper.c | 102 +++++++-------- target/arm/tcg/vec_helper.c | 127 +++++++++--------- target/s390x/tcg/fpu_helper.c | 135 +++++++++++++++++++ fpu/softfloat-parts.c.inc | 236 ++++++++++++++++------------------ 8 files changed, 424 insertions(+), 447 deletions(-) -- 2.43.0 At the same time, export. Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- include/fpu/softfloat-parts.h | 3 +++ fpu/softfloat.c | 18 +++++++++--------- fpu/softfloat-parts.c.inc | 14 ++++++++------ 3 files changed, 20 insertions(+), 15 deletions(-) diff --git a/include/fpu/softfloat-parts.h b/include/fpu/softfloat-parts.h index XXXXXXX..XXXXXXX 100644 --- a/include/fpu/softfloat-parts.h +++ b/include/fpu/softfloat-parts.h @@ -XXX,XX +XXX,XX @@ FloatParts128 parts128_round_to_int(const FloatParts128 *a, FloatParts64 parts64_round_to_fmt(const FloatParts64 *p, float_status *s, const FloatFmt *fmt); +FloatParts64 parts64_scalbn(const FloatParts64 *a, int n, float_status *s); +FloatParts128 parts128_scalbn(const FloatParts128 *a, int n, float_status *s); + #endif diff --git a/fpu/softfloat.c b/fpu/softfloat.c index XXXXXXX..XXXXXXX 100644 --- a/fpu/softfloat.c +++ b/fpu/softfloat.c @@ -XXX,XX +XXX,XX @@ float16 float16_muladd_scalbn(float16 a, float16 b, float16 c, /* Before rounding, scale. */ if (scale) { - parts64_scalbn(pr, scale, status); + *pr = parts64_scalbn(pr, scale, status); } parts64_uncanon(pr, status, &float16_params, false); /* After rounding, apply negate result, especially for -0.0. */ @@ -XXX,XX +XXX,XX @@ float32_muladd_scalbn(float32 a, float32 b, float32 c, /* Before rounding, scale. */ if (scale) { - parts64_scalbn(pr, scale, status); + *pr = parts64_scalbn(pr, scale, status); } parts64_uncanon(pr, status, &float32_params, false); /* After rounding, apply negate result, especially for -0.0. */ @@ -XXX,XX +XXX,XX @@ float64_muladd_scalbn(float64 a, float64 b, float64 c, /* Before rounding, scale. */ if (scale) { - parts64_scalbn(pr, scale, status); + *pr = parts64_scalbn(pr, scale, status); } parts64_uncanon(pr, status, &float64_params, false); /* After rounding, apply negate result, especially for -0.0. */ @@ -XXX,XX +XXX,XX @@ float16 float16_scalbn(float16 a, int n, float_status *status) { FloatParts64 p = float16_unpack_canonical(a, status); - parts64_scalbn(&p, n, status); + p = parts64_scalbn(&p, n, status); return float16_round_pack_canonical(&p, status); } @@ -XXX,XX +XXX,XX @@ float32 float32_scalbn(float32 a, int n, float_status *status) { FloatParts64 p = float32_unpack_canonical(a, status); - parts64_scalbn(&p, n, status); + p = parts64_scalbn(&p, n, status); return float32_round_pack_canonical(&p, status); } @@ -XXX,XX +XXX,XX @@ float64 float64_scalbn(float64 a, int n, float_status *status) { FloatParts64 p = float64_unpack_canonical(a, status); - parts64_scalbn(&p, n, status); + p = parts64_scalbn(&p, n, status); return float64_round_pack_canonical(&p, status); } @@ -XXX,XX +XXX,XX @@ bfloat16 bfloat16_scalbn(bfloat16 a, int n, float_status *status) { FloatParts64 p = bfloat16_unpack_canonical(a, status); - parts64_scalbn(&p, n, status); + p = parts64_scalbn(&p, n, status); return bfloat16_round_pack_canonical(&p, status); } @@ -XXX,XX +XXX,XX @@ float128 float128_scalbn(float128 a, int n, float_status *status) { FloatParts128 p = float128_unpack_canonical(a, status); - parts128_scalbn(&p, n, status); + p = parts128_scalbn(&p, n, status); return float128_round_pack_canonical(&p, status); } @@ -XXX,XX +XXX,XX @@ floatx80 floatx80_scalbn(floatx80 a, int n, float_status *status) if (!floatx80_unpack_canonical(&p, a, status)) { return floatx80_default_nan(status); } - parts128_scalbn(&p, n, status); + p = parts128_scalbn(&p, n, status); return floatx80_round_pack_canonical(&p, status); } diff --git a/fpu/softfloat-parts.c.inc b/fpu/softfloat-parts.c.inc index XXXXXXX..XXXXXXX 100644 --- a/fpu/softfloat-parts.c.inc +++ b/fpu/softfloat-parts.c.inc @@ -XXX,XX +XXX,XX @@ FloatRelation partsN(compare)(const FloatPartsN *a, const FloatPartsN *b, /* * Multiply A by 2 raised to the power N. */ -static void partsN(scalbn)(FloatPartsN *a, int n, float_status *s) +FloatPartsN partsN(scalbn)(const FloatPartsN *a, int n, float_status *s) { switch (a->cls) { case float_class_snan: case float_class_qnan: - *a = partsN(return_nan)(a, s); - break; + return partsN(return_nan)(a, s); case float_class_zero: case float_class_inf: - break; + return *a; case float_class_denormal: float_raise(float_flag_input_denormal_used, s); /* fall through */ case float_class_normal: - a->exp += MIN(MAX(n, -0x10000), 0x10000); - break; + { + FloatPartsN r = *a; + r.exp += MIN(MAX(n, -0x10000), 0x10000); + return r; + } default: g_assert_not_reached(); } -- 2.43.0 Check the likely case of normal product and normal or zero addend first; shift NaN and infinity detection down; end with zero product + addend. Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- fpu/softfloat-parts.c.inc | 156 +++++++++++++++++--------------------- 1 file changed, 71 insertions(+), 85 deletions(-) diff --git a/fpu/softfloat-parts.c.inc b/fpu/softfloat-parts.c.inc index XXXXXXX..XXXXXXX 100644 --- a/fpu/softfloat-parts.c.inc +++ b/fpu/softfloat-parts.c.inc @@ -XXX,XX +XXX,XX @@ static FloatPartsN *partsN(muladd)(FloatPartsN *a, FloatPartsN *b, FloatPartsN *c, int flags, float_status *s) { - int ab_mask, abc_mask; - FloatPartsW p_widen, c_widen; + int ab_mask = float_cmask(a->cls) | float_cmask(b->cls); + int c_mask = float_cmask(c->cls); + int abc_mask = ab_mask | c_mask; + bool c_sign = c->sign ^ !!(flags & float_muladd_negate_c); + bool p_sign = a->sign ^ b->sign ^ !!(flags & float_muladd_negate_product); - ab_mask = float_cmask(a->cls) | float_cmask(b->cls); - abc_mask = float_cmask(c->cls) | ab_mask; + /* + * The "likely" case is A and B normal, so that the product is normal, + * and C normal or zero so that the result is normal. + */ + int likely_mask = ab_mask | (c_mask & ~float_cmask_zero); + if (likely(cmask_is_only_normals(likely_mask))) { + record_denormals_used(abc_mask, s); + + /* Perform the multiplication step. */ + FloatPartsW p_widen = { .sign = p_sign, .exp = a->exp + b->exp + 1 }; + fracN(mulw)(&p_widen, a, b); + if (!(p_widen.frac_hi & DECOMPOSED_IMPLICIT_BIT)) { + fracW(add)(&p_widen, &p_widen, &p_widen); + p_widen.exp -= 1; + } + + /* Perform the addition step. */ + if (!(c_mask & float_cmask_zero)) { + /* Zero-extend C to less significant bits. */ + FloatPartsW c_widen = { .sign = c_sign, .exp = c->exp }; + fracN(widen)(&c_widen, c); + + if (p_sign == c_sign) { + partsW(add_normal)(&p_widen, &c_widen); + } else if (!partsW(sub_normal)(&p_widen, &c_widen)) { + goto return_sub_zero; + } + } + + /* Narrow with sticky bit, for proper rounding later. */ + fracN(truncjam)(a, &p_widen); + a->sign = p_widen.sign; + a->exp = p_widen.exp; + return a; + } /* * It is implementation-defined whether the cases of (0,inf,qnan) @@ -XXX,XX +XXX,XX @@ static FloatPartsN *partsN(muladd)(FloatPartsN *a, FloatPartsN *b, return a; } - if (flags & float_muladd_negate_c) { - c->sign ^= 1; + if (unlikely(ab_mask == float_cmask_infzero)) { + /* Inf * Zero == NaN */ + float_raise(float_flag_invalid | float_flag_invalid_imz, s); + goto d_nan; } - /* Compute the sign of the product into A. */ - a->sign ^= b->sign; - if (flags & float_muladd_negate_product) { - a->sign ^= 1; - } - - if (unlikely(!cmask_is_only_normals(ab_mask))) { - if (unlikely(ab_mask == float_cmask_infzero)) { - float_raise(float_flag_invalid | float_flag_invalid_imz, s); + if (unlikely(ab_mask & float_cmask_inf)) { + if ((c_mask & float_cmask_inf) && p_sign != c_sign) { + /* Inf - Inf == NaN */ + float_raise(float_flag_invalid | float_flag_invalid_isi, s); goto d_nan; } - - if (ab_mask & float_cmask_inf) { - if (c->cls == float_class_inf && a->sign != c->sign) { - float_raise(float_flag_invalid | float_flag_invalid_isi, s); - goto d_nan; - } - goto return_inf; - } - - g_assert(ab_mask & float_cmask_zero); - if (is_anynorm(c->cls)) { - *a = *c; - goto finish_sign; - } - if (c->cls == float_class_zero) { - if (flags & float_muladd_suppress_add_product_zero) { - a->sign = c->sign; - } else if (a->sign != c->sign) { - goto return_sub_zero; - } - goto return_zero; - } - g_assert(c->cls == float_class_inf); + /* Inf + C == Inf */ + record_denormals_used(abc_mask, s); + a->sign = p_sign; + a->cls = float_class_inf; + return a; } - - if (unlikely(c->cls == float_class_inf)) { - a->sign = c->sign; - goto return_inf; - } - - /* Perform the multiplication step. */ - p_widen.sign = a->sign; - p_widen.exp = a->exp + b->exp + 1; - fracN(mulw)(&p_widen, a, b); - if (!(p_widen.frac_hi & DECOMPOSED_IMPLICIT_BIT)) { - fracW(add)(&p_widen, &p_widen, &p_widen); - p_widen.exp -= 1; - } - - /* Perform the addition step. */ - if (c->cls != float_class_zero) { - /* Zero-extend C to less significant bits. */ - fracN(widen)(&c_widen, c); - c_widen.exp = c->exp; - - if (a->sign == c->sign) { - partsW(add_normal)(&p_widen, &c_widen); - } else if (!partsW(sub_normal)(&p_widen, &c_widen)) { - goto return_sub_zero; - } - } - - /* Narrow with sticky bit, for proper rounding later. */ - fracN(truncjam)(a, &p_widen); - a->sign = p_widen.sign; - a->exp = p_widen.exp; - - finish_sign: - /* - * All result types except for "return the default NaN - * because this is an Invalid Operation" go through here; - * this matches the set of cases where we consumed a - * denormal input. - */ record_denormals_used(abc_mask, s); - return a; + + /* Only remaining cases are zero product or inf addend. */ + assert((ab_mask & float_cmask_zero) | (c_mask & float_cmask_inf)); + + /* + * P + Inf == Inf, or + * 0 + C == C, + * except for 0 - 0, which needs special rounding, + * except for when we want to suppress this addition step. + */ + if (!(c_mask & float_cmask_zero) + || p_sign == c_sign + || (flags & float_muladd_suppress_add_product_zero)) { + c->sign = c_sign; + return c; + } return_sub_zero: + /* 0 - 0 == -0 for round_down, +0 otherwise. */ a->sign = s->float_rounding_mode == float_round_down; - return_zero: a->cls = float_class_zero; - goto finish_sign; - - return_inf: - a->cls = float_class_inf; - goto finish_sign; + return a; d_nan: *a = partsN(default_nan)(s); -- 2.43.0 At the same time, export. Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- include/fpu/softfloat-parts.h | 9 ++++ fpu/softfloat.c | 83 +++++++++++++++++------------------ fpu/softfloat-parts.c.inc | 38 ++++++++-------- 3 files changed, 69 insertions(+), 61 deletions(-) diff --git a/include/fpu/softfloat-parts.h b/include/fpu/softfloat-parts.h index XXXXXXX..XXXXXXX 100644 --- a/include/fpu/softfloat-parts.h +++ b/include/fpu/softfloat-parts.h @@ -XXX,XX +XXX,XX @@ FloatParts64 parts64_mul(const FloatParts64 *a, const FloatParts64 *b, FloatParts128 parts128_mul(const FloatParts128 *a, const FloatParts128 *b, float_status *s); +FloatParts64 parts64_muladd(const FloatParts64 *a, + const FloatParts64 *b, + const FloatParts64 *c, + int flags, float_status *s); +FloatParts128 parts128_muladd(const FloatParts128 *a, + const FloatParts128 *b, + const FloatParts128 *c, + int flags, float_status *s); + FloatParts64 parts64_round_to_int(const FloatParts64 *a, FloatRoundMode rmode, int scale, float_status *s, diff --git a/fpu/softfloat.c b/fpu/softfloat.c index XXXXXXX..XXXXXXX 100644 --- a/fpu/softfloat.c +++ b/fpu/softfloat.c @@ -XXX,XX +XXX,XX @@ float16 float16_muladd_scalbn(float16 a, float16 b, float16 c, FloatParts64 pa = float16_unpack_canonical(a, status); FloatParts64 pb = float16_unpack_canonical(b, status); FloatParts64 pc = float16_unpack_canonical(c, status); - FloatParts64 *pr = parts64_muladd(&pa, &pb, &pc, flags, status); + FloatParts64 pr = parts64_muladd(&pa, &pb, &pc, flags, status); /* Before rounding, scale. */ if (scale) { - *pr = parts64_scalbn(pr, scale, status); + pr = parts64_scalbn(&pr, scale, status); } - parts64_uncanon(pr, status, &float16_params, false); + parts64_uncanon(&pr, status, &float16_params, false); /* After rounding, apply negate result, especially for -0.0. */ - if ((flags & float_muladd_negate_result) && !is_nan(pr->cls)) { - pr->sign ^= 1; + if ((flags & float_muladd_negate_result) && !is_nan(pr.cls)) { + pr.sign ^= 1; } - return pack_raw64(pr, &float16_params); + return pack_raw64(&pr, &float16_params); } float16 float16_muladd(float16 a, float16 b, float16 c, @@ -XXX,XX +XXX,XX @@ float32_muladd_scalbn(float32 a, float32 b, float32 c, FloatParts64 pa = float32_unpack_canonical(a, status); FloatParts64 pb = float32_unpack_canonical(b, status); FloatParts64 pc = float32_unpack_canonical(c, status); - FloatParts64 *pr = parts64_muladd(&pa, &pb, &pc, flags, status); + FloatParts64 pr = parts64_muladd(&pa, &pb, &pc, flags, status); /* Before rounding, scale. */ if (scale) { - *pr = parts64_scalbn(pr, scale, status); + pr = parts64_scalbn(&pr, scale, status); } - parts64_uncanon(pr, status, &float32_params, false); + parts64_uncanon(&pr, status, &float32_params, false); /* After rounding, apply negate result, especially for -0.0. */ - if ((flags & float_muladd_negate_result) && !is_nan(pr->cls)) { - pr->sign ^= 1; + if ((flags & float_muladd_negate_result) && !is_nan(pr.cls)) { + pr.sign ^= 1; } - return pack_raw64(pr, &float32_params); + return pack_raw64(&pr, &float32_params); } float64 QEMU_SOFTFLOAT_ATTR @@ -XXX,XX +XXX,XX @@ float64_muladd_scalbn(float64 a, float64 b, float64 c, FloatParts64 pa = float64_unpack_canonical(a, status); FloatParts64 pb = float64_unpack_canonical(b, status); FloatParts64 pc = float64_unpack_canonical(c, status); - FloatParts64 *pr = parts64_muladd(&pa, &pb, &pc, flags, status); + FloatParts64 pr = parts64_muladd(&pa, &pb, &pc, flags, status); /* Before rounding, scale. */ if (scale) { - *pr = parts64_scalbn(pr, scale, status); + pr = parts64_scalbn(&pr, scale, status); } - parts64_uncanon(pr, status, &float64_params, false); + parts64_uncanon(&pr, status, &float64_params, false); /* After rounding, apply negate result, especially for -0.0. */ - if ((flags & float_muladd_negate_result) && !is_nan(pr->cls)) { - pr->sign ^= 1; + if ((flags & float_muladd_negate_result) && !is_nan(pr.cls)) { + pr.sign ^= 1; } - return pack_raw64(pr, &float64_params); + return pack_raw64(&pr, &float64_params); } static bool force_soft_fma; @@ -XXX,XX +XXX,XX @@ float64 float64r32_muladd(float64 a, float64 b, float64 c, FloatParts64 pa = float64_unpack_canonical(a, status); FloatParts64 pb = float64_unpack_canonical(b, status); FloatParts64 pc = float64_unpack_canonical(c, status); - FloatParts64 *pr = parts64_muladd(&pa, &pb, &pc, flags, status); + FloatParts64 pr = parts64_muladd(&pa, &pb, &pc, flags, status); /* Round before applying negate result. */ - parts64_uncanon(pr, status, &float32_params, false); - if ((flags & float_muladd_negate_result) && !is_nan(pr->cls)) { - pr->sign ^= 1; + parts64_uncanon(&pr, status, &float32_params, false); + if ((flags & float_muladd_negate_result) && !is_nan(pr.cls)) { + pr.sign ^= 1; } - return float64r32_pack_raw(pr); + return float64r32_pack_raw(&pr); } bfloat16 bfloat16_muladd(bfloat16 a, bfloat16 b, bfloat16 c, @@ -XXX,XX +XXX,XX @@ bfloat16 bfloat16_muladd(bfloat16 a, bfloat16 b, bfloat16 c, FloatParts64 pa = bfloat16_unpack_canonical(a, status); FloatParts64 pb = bfloat16_unpack_canonical(b, status); FloatParts64 pc = bfloat16_unpack_canonical(c, status); - FloatParts64 *pr = parts64_muladd(&pa, &pb, &pc, flags, status); + FloatParts64 pr = parts64_muladd(&pa, &pb, &pc, flags, status); /* Round before applying negate result. */ - parts64_uncanon(pr, status, &bfloat16_params, false); - if ((flags & float_muladd_negate_result) && !is_nan(pr->cls)) { - pr->sign ^= 1; + parts64_uncanon(&pr, status, &bfloat16_params, false); + if ((flags & float_muladd_negate_result) && !is_nan(pr.cls)) { + pr.sign ^= 1; } - return pack_raw64(pr, &bfloat16_params); + return pack_raw64(&pr, &bfloat16_params); } float128 float128_muladd(float128 a, float128 b, float128 c, @@ -XXX,XX +XXX,XX @@ float128 float128_muladd(float128 a, float128 b, float128 c, FloatParts128 pa = float128_unpack_canonical(a, status); FloatParts128 pb = float128_unpack_canonical(b, status); FloatParts128 pc = float128_unpack_canonical(c, status); - FloatParts128 *pr = parts128_muladd(&pa, &pb, &pc, flags, status); + FloatParts128 pr = parts128_muladd(&pa, &pb, &pc, flags, status); /* Round before applying negate result. */ - parts128_uncanon(pr, status, &float128_params, false); - if ((flags & float_muladd_negate_result) && !is_nan(pr->cls)) { - pr->sign ^= 1; + parts128_uncanon(&pr, status, &float128_params, false); + if ((flags & float_muladd_negate_result) && !is_nan(pr.cls)) { + pr.sign ^= 1; } - return float128_pack_raw(pr); + return float128_pack_raw(&pr); } /* @@ -XXX,XX +XXX,XX @@ float32 float32_exp2(float32 a, float_status *status) rp = float64_unpack_canonical(float64_one, status); for (int i = 0; i < 15; i++) { tp = float64_unpack_canonical(float32_exp2_coefficients[i], status); - rp = *parts64_muladd(&tp, &xnp, &rp, 0, status); + rp = parts64_muladd(&tp, &xnp, &rp, 0, status); xnp = parts64_mul(&xnp, &xp, status); } @@ -XXX,XX +XXX,XX @@ static void parts_s390_divide_to_integer(FloatParts64 *a, FloatParts64 *b, n->sign = a->sign ^ b->sign; *cc = 0; } else { - FloatParts64 *q, q_buf, *r_precise, r_precise_buf; + FloatParts64 *q, q_buf, r_precise; int float_exception_flags = 0; bool is_q_smallish; uint32_t r_flags; @@ -XXX,XX +XXX,XX @@ static void parts_s390_divide_to_integer(FloatParts64 *a, FloatParts64 *b, 0, fmt->frac_size); /* Compute precise remainder */ - r_precise_buf = *b; - r_precise = parts64_muladd(&r_precise_buf, n, a, + r_precise = parts64_muladd(b, n, a, float_muladd_negate_product, status); /* Round remainder to the target format */ - *r = *r_precise; + *r = r_precise; status->float_exception_flags = 0; *r = parts64_round_to_fmt(r, status, fmt); r_flags = status->float_exception_flags; @@ -XXX,XX +XXX,XX @@ static void parts_s390_divide_to_integer(FloatParts64 *a, FloatParts64 *b, * toward zero) or incremented. */ saved_r_sign = r->sign; - saved_r_precise_sign = r_precise->sign; + saved_r_precise_sign = r_precise.sign; r->sign = false; - r_precise->sign = false; - if (parts64_compare(r, r_precise, status, true) < + r_precise.sign = false; + if (parts64_compare(r, &r_precise, status, true) < float_relation_equal) { *dxc = 0x8; } else { *dxc = 0xc; } r->sign = saved_r_sign; - r_precise->sign = saved_r_precise_sign; + r_precise.sign = saved_r_precise_sign; } } } diff --git a/fpu/softfloat-parts.c.inc b/fpu/softfloat-parts.c.inc index XXXXXXX..XXXXXXX 100644 --- a/fpu/softfloat-parts.c.inc +++ b/fpu/softfloat-parts.c.inc @@ -XXX,XX +XXX,XX @@ FloatPartsN partsN(mul)(const FloatPartsN *a, const FloatPartsN *b, * Requires A and C extracted into a double-sized structure to provide the * extra space for the widening multiply. */ -static FloatPartsN *partsN(muladd)(FloatPartsN *a, FloatPartsN *b, - FloatPartsN *c, - int flags, float_status *s) +FloatPartsN partsN(muladd)(const FloatPartsN *a, const FloatPartsN *b, + const FloatPartsN *c, int flags, float_status *s) { int ab_mask = float_cmask(a->cls) | float_cmask(b->cls); int c_mask = float_cmask(c->cls); @@ -XXX,XX +XXX,XX @@ static FloatPartsN *partsN(muladd)(FloatPartsN *a, FloatPartsN *b, } /* Narrow with sticky bit, for proper rounding later. */ - fracN(truncjam)(a, &p_widen); - a->sign = p_widen.sign; - a->exp = p_widen.exp; - return a; + FloatPartsN r = { + .sign = p_widen.sign, + .exp = p_widen.exp, + .cls = float_class_normal, + }; + fracN(truncjam)(&r, &p_widen); + return r; } /* @@ -XXX,XX +XXX,XX @@ static FloatPartsN *partsN(muladd)(FloatPartsN *a, FloatPartsN *b, * off to the target-specific pick-a-NaN routine. */ if (unlikely(abc_mask & float_cmask_anynan)) { - *a = partsN(pick_nan_muladd)(a, b, c, s, ab_mask, abc_mask); - return a; + return partsN(pick_nan_muladd)(a, b, c, s, ab_mask, abc_mask); } if (unlikely(ab_mask == float_cmask_infzero)) { @@ -XXX,XX +XXX,XX @@ static FloatPartsN *partsN(muladd)(FloatPartsN *a, FloatPartsN *b, } /* Inf + C == Inf */ record_denormals_used(abc_mask, s); - a->sign = p_sign; - a->cls = float_class_inf; - return a; + return (FloatPartsN){ .sign = p_sign, .cls = float_class_inf }; } record_denormals_used(abc_mask, s); @@ -XXX,XX +XXX,XX @@ static FloatPartsN *partsN(muladd)(FloatPartsN *a, FloatPartsN *b, if (!(c_mask & float_cmask_zero) || p_sign == c_sign || (flags & float_muladd_suppress_add_product_zero)) { - c->sign = c_sign; - return c; + FloatPartsN r = *c; + r.sign = c_sign; + return r; } return_sub_zero: /* 0 - 0 == -0 for round_down, +0 otherwise. */ - a->sign = s->float_rounding_mode == float_round_down; - a->cls = float_class_zero; - return a; + return (FloatPartsN){ + .sign = s->float_rounding_mode == float_round_down, + .cls = float_class_zero + }; d_nan: - *a = partsN(default_nan)(s); - return a; + return partsN(default_nan)(s); } /* -- 2.43.0 The nan test had been down below because it was unlikely. But if we have to have one anyway because of denormals, we might as well take care of them right away. Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- fpu/softfloat-parts.c.inc | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) diff --git a/fpu/softfloat-parts.c.inc b/fpu/softfloat-parts.c.inc index XXXXXXX..XXXXXXX 100644 --- a/fpu/softfloat-parts.c.inc +++ b/fpu/softfloat-parts.c.inc @@ -XXX,XX +XXX,XX @@ FloatPartsN partsN(addsub)(const FloatPartsN *a_orig, { int ab_mask = float_cmask(a_orig->cls) | float_cmask(b_orig->cls); + if (unlikely(ab_mask & float_cmask_anynan)) { + return partsN(pick_nan)(a_orig, b_orig, s); + } + /* * For addition and subtraction, we will consume an * input denormal unless the other input is a NaN. */ - if (!(ab_mask & float_cmask_anynan)) { - record_denormals_used(ab_mask, s); - } + record_denormals_used(ab_mask, s); FloatPartsN a = *a_orig; FloatPartsN b = *b_orig; @@ -XXX,XX +XXX,XX @@ FloatPartsN partsN(addsub)(const FloatPartsN *a_orig, return a; } - if (unlikely(ab_mask & float_cmask_anynan)) { - goto p_nan; - } - if (ab_mask & float_cmask_inf) { if (a.cls != float_class_inf) { /* N - Inf */ @@ -XXX,XX +XXX,XX @@ FloatPartsN partsN(addsub)(const FloatPartsN *a_orig, return a; } - if (unlikely(ab_mask & float_cmask_anynan)) { - goto p_nan; - } - if (ab_mask & float_cmask_inf) { a.cls = float_class_inf; return a; @@ -XXX,XX +XXX,XX @@ FloatPartsN partsN(addsub)(const FloatPartsN *a_orig, g_assert(a.cls == float_class_zero); g_assert(is_anynorm(b.cls)); return b; - - p_nan: - return partsN(pick_nan)(a_orig, b_orig, s); } /* -- 2.43.0 Consolidate the tests for zero and anynorm. Add comments for a few cases. Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- fpu/softfloat-parts.c.inc | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/fpu/softfloat-parts.c.inc b/fpu/softfloat-parts.c.inc index XXXXXXX..XXXXXXX 100644 --- a/fpu/softfloat-parts.c.inc +++ b/fpu/softfloat-parts.c.inc @@ -XXX,XX +XXX,XX @@ FloatPartsN partsN(addsub)(const FloatPartsN *a_orig, } if (ab_mask == float_cmask_zero) { + /* 0 - 0 */ a.sign = s->float_rounding_mode == float_round_down; return a; } @@ -XXX,XX +XXX,XX @@ FloatPartsN partsN(addsub)(const FloatPartsN *a_orig, } if (ab_mask == float_cmask_zero) { + /* 0 + 0 */ return a; } if (ab_mask & float_cmask_inf) { + /* N + Inf or Inf + N */ a.cls = float_class_inf; return a; } } - if (b.cls == float_class_zero) { - g_assert(is_anynorm(a.cls)); - return a; - } - - g_assert(a.cls == float_class_zero); - g_assert(is_anynorm(b.cls)); - return b; + /* 0 +/- N or N +/- 0 */ + assert((ab_mask & float_cmask_zero) && (ab_mask & float_cmask_anynorm)); + return b.cls == float_class_zero ? a : b; } /* -- 2.43.0 We will not expose parts_round_to_int_normal. Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- fpu/softfloat.c | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/fpu/softfloat.c b/fpu/softfloat.c index XXXXXXX..XXXXXXX 100644 --- a/fpu/softfloat.c +++ b/fpu/softfloat.c @@ -XXX,XX +XXX,XX @@ static void parts_s390_divide_to_integer(FloatParts64 *a, FloatParts64 *b, * Rounding of partial quotient may be inexact. This is the whole point * of distinguishing partial quotients, so ignore the exception. */ - *n = *q; - parts64_round_to_int_normal(n, - is_q_smallish - ? final_quotient_rounding_mode - : float_round_to_zero, - 0, fmt->frac_size); + *n = parts64_round_to_int(q, + is_q_smallish + ? final_quotient_rounding_mode + : float_round_to_zero, + 0, status, fmt); /* Compute precise remainder */ r_precise = parts64_muladd(b, n, a, -- 2.43.0 Now that we've exposed enough infrastructure, this can be implemented in the backend that needs it. Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org> Reviewed-by: Ilya Leoshkevich <iii@linux.ibm.com> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- include/fpu/softfloat.h | 11 --- fpu/softfloat.c | 137 ---------------------------------- target/s390x/tcg/fpu_helper.c | 135 +++++++++++++++++++++++++++++++++ 3 files changed, 135 insertions(+), 148 deletions(-) diff --git a/include/fpu/softfloat.h b/include/fpu/softfloat.h index XXXXXXX..XXXXXXX 100644 --- a/include/fpu/softfloat.h +++ b/include/fpu/softfloat.h @@ -XXX,XX +XXX,XX @@ static inline bool float128_unordered_quiet(float128 a, float128 b, *----------------------------------------------------------------------------*/ float128 float128_default_nan(float_status *status); -#define DECLARE_S390_DIVIDE_TO_INTEGER(floatN) \ -void floatN ## _s390_divide_to_integer(floatN a, floatN b, \ - int final_quotient_rounding_mode, \ - bool mask_underflow, bool mask_inexact, \ - floatN *r, floatN *n, \ - uint32_t *cc, int *dxc, \ - float_status *status) -DECLARE_S390_DIVIDE_TO_INTEGER(float32); -DECLARE_S390_DIVIDE_TO_INTEGER(float64); - - #endif /* SOFTFLOAT_H */ diff --git a/fpu/softfloat.c b/fpu/softfloat.c index XXXXXXX..XXXXXXX 100644 --- a/fpu/softfloat.c +++ b/fpu/softfloat.c @@ -XXX,XX +XXX,XX @@ floatx80 floatx80_round(floatx80 a, float_status *status) return floatx80_round_pack_canonical(&p, status); } -static void parts_s390_divide_to_integer(FloatParts64 *a, FloatParts64 *b, - int final_quotient_rounding_mode, - bool mask_underflow, bool mask_inexact, - const FloatFmt *fmt, - FloatParts64 *r, FloatParts64 *n, - uint32_t *cc, int *dxc, - float_status *status) -{ - /* POp table "Results: DIVIDE TO INTEGER (Part 1 of 2)" */ - if ((float_cmask(a->cls) | float_cmask(b->cls)) & float_cmask_anynan) { - *r = parts64_pick_nan(a, b, status); - *n = *r; - *cc = 1; - } else if (a->cls == float_class_inf || b->cls == float_class_zero) { - *r = parts64_default_nan(status); - *n = *r; - *cc = 1; - status->float_exception_flags |= float_flag_invalid; - } else if (b->cls == float_class_inf) { - *r = *a; - n->cls = float_class_zero; - n->sign = a->sign ^ b->sign; - *cc = 0; - } else { - FloatParts64 *q, q_buf, r_precise; - int float_exception_flags = 0; - bool is_q_smallish; - uint32_t r_flags; - - /* Compute precise quotient */ - q_buf = parts64_div(a, b, status); - q = &q_buf; - - /* - * Check whether two closest integers can be precisely represented, - * i.e., all their bits fit into the fractional part. - */ - is_q_smallish = q->exp < (fmt->frac_size + 1); - - /* - * Final quotient is rounded using final-quotient-rounding method, and - * partial quotient is rounded toward zero. - * - * Rounding of partial quotient may be inexact. This is the whole point - * of distinguishing partial quotients, so ignore the exception. - */ - *n = parts64_round_to_int(q, - is_q_smallish - ? final_quotient_rounding_mode - : float_round_to_zero, - 0, status, fmt); - - /* Compute precise remainder */ - r_precise = parts64_muladd(b, n, a, - float_muladd_negate_product, status); - - /* Round remainder to the target format */ - *r = r_precise; - status->float_exception_flags = 0; - *r = parts64_round_to_fmt(r, status, fmt); - r_flags = status->float_exception_flags; - - /* POp table "Results: DIVIDE TO INTEGER (Part 2 of 2)" */ - if (is_q_smallish) { - if (r->cls != float_class_zero) { - if (r->exp < 2 - (1 << (fmt->exp_size - 1))) { - if (mask_underflow) { - float_exception_flags |= float_flag_underflow; - *dxc = 0x10; - r->exp += fmt->exp_re_bias; - } - } else if (r_flags & float_flag_inexact) { - float_exception_flags |= float_flag_inexact; - if (mask_inexact) { - bool saved_r_sign, saved_r_precise_sign; - - /* - * Check whether remainder was truncated (rounded - * toward zero) or incremented. - */ - saved_r_sign = r->sign; - saved_r_precise_sign = r_precise.sign; - r->sign = false; - r_precise.sign = false; - if (parts64_compare(r, &r_precise, status, true) < - float_relation_equal) { - *dxc = 0x8; - } else { - *dxc = 0xc; - } - r->sign = saved_r_sign; - r_precise.sign = saved_r_precise_sign; - } - } - } - *cc = 0; - } else if (n->exp > (1 << (fmt->exp_size - 1)) - 1) { - n->exp -= fmt->exp_re_bias; - *cc = r->cls == float_class_zero ? 1 : 3; - } else { - *cc = r->cls == float_class_zero ? 0 : 2; - } - - /* Adjust signs of zero results */ - if (r->cls == float_class_zero) { - r->sign = a->sign; - } - if (n->cls == float_class_zero) { - n->sign = a->sign ^ b->sign; - } - - status->float_exception_flags = float_exception_flags; - } -} - -#define DEFINE_S390_DIVIDE_TO_INTEGER(floatN) \ -void floatN ## _s390_divide_to_integer(floatN a, floatN b, \ - int final_quotient_rounding_mode, \ - bool mask_underflow, bool mask_inexact, \ - floatN *r, floatN *n, \ - uint32_t *cc, int *dxc, \ - float_status *status) \ -{ \ - FloatParts64 pa = floatN ## _unpack_canonical(a, status); \ - FloatParts64 pb = floatN ## _unpack_canonical(b, status); \ - FloatParts64 pr, pn; \ - parts_s390_divide_to_integer(&pa, &pb, final_quotient_rounding_mode, \ - mask_underflow, mask_inexact, \ - &floatN ## _params, \ - &pr, &pn, cc, dxc, status); \ - *r = floatN ## _round_pack_canonical(&pr, status); \ - *n = floatN ## _round_pack_canonical(&pn, status); \ -} - -DEFINE_S390_DIVIDE_TO_INTEGER(float32) -DEFINE_S390_DIVIDE_TO_INTEGER(float64) - static void __attribute__((constructor)) softfloat_init(void) { union_float64 ua, ub, uc, ur; diff --git a/target/s390x/tcg/fpu_helper.c b/target/s390x/tcg/fpu_helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/s390x/tcg/fpu_helper.c +++ b/target/s390x/tcg/fpu_helper.c @@ -XXX,XX +XXX,XX @@ #include "tcg_s390x.h" #include "exec/helper-proto.h" #include "fpu/softfloat.h" +#include "fpu/softfloat-parts.h" /* #define DEBUG_HELPER */ #ifdef DEBUG_HELPER @@ -XXX,XX +XXX,XX @@ Int128 HELPER(dxb)(CPUS390XState *env, Int128 a, Int128 b) return RET128(ret); } +static void parts_s390_divide_to_integer(FloatParts64 *a, FloatParts64 *b, + int final_quotient_rounding_mode, + bool mask_underflow, bool mask_inexact, + const FloatFmt *fmt, + FloatParts64 *r, FloatParts64 *n, + uint32_t *cc, int *dxc, + float_status *status) +{ + /* POp table "Results: DIVIDE TO INTEGER (Part 1 of 2)" */ + if ((float_cmask(a->cls) | float_cmask(b->cls)) & float_cmask_anynan) { + *r = parts64_pick_nan(a, b, status); + *n = *r; + *cc = 1; + } else if (a->cls == float_class_inf || b->cls == float_class_zero) { + *r = parts64_default_nan(status); + *n = *r; + *cc = 1; + status->float_exception_flags |= float_flag_invalid; + } else if (b->cls == float_class_inf) { + *r = *a; + n->cls = float_class_zero; + n->sign = a->sign ^ b->sign; + *cc = 0; + } else { + FloatParts64 *q, q_buf, r_precise; + int float_exception_flags = 0; + bool is_q_smallish; + uint32_t r_flags; + + /* Compute precise quotient */ + q_buf = parts64_div(a, b, status); + q = &q_buf; + + /* + * Check whether two closest integers can be precisely represented, + * i.e., all their bits fit into the fractional part. + */ + is_q_smallish = q->exp < (fmt->frac_size + 1); + + /* + * Final quotient is rounded using final-quotient-rounding method, and + * partial quotient is rounded toward zero. + * + * Rounding of partial quotient may be inexact. This is the whole point + * of distinguishing partial quotients, so ignore the exception. + */ + *n = parts64_round_to_int(q, + is_q_smallish + ? final_quotient_rounding_mode + : float_round_to_zero, + 0, status, fmt); + + /* Compute precise remainder */ + r_precise = parts64_muladd(b, n, a, + float_muladd_negate_product, status); + + /* Round remainder to the target format */ + *r = r_precise; + status->float_exception_flags = 0; + *r = parts64_round_to_fmt(r, status, fmt); + r_flags = status->float_exception_flags; + + /* POp table "Results: DIVIDE TO INTEGER (Part 2 of 2)" */ + if (is_q_smallish) { + if (r->cls != float_class_zero) { + if (r->exp < 2 - (1 << (fmt->exp_size - 1))) { + if (mask_underflow) { + float_exception_flags |= float_flag_underflow; + *dxc = 0x10; + r->exp += fmt->exp_re_bias; + } + } else if (r_flags & float_flag_inexact) { + float_exception_flags |= float_flag_inexact; + if (mask_inexact) { + bool saved_r_sign, saved_r_precise_sign; + + /* + * Check whether remainder was truncated (rounded + * toward zero) or incremented. + */ + saved_r_sign = r->sign; + saved_r_precise_sign = r_precise.sign; + r->sign = false; + r_precise.sign = false; + if (parts64_compare(r, &r_precise, status, true) < + float_relation_equal) { + *dxc = 0x8; + } else { + *dxc = 0xc; + } + r->sign = saved_r_sign; + r_precise.sign = saved_r_precise_sign; + } + } + } + *cc = 0; + } else if (n->exp > (1 << (fmt->exp_size - 1)) - 1) { + n->exp -= fmt->exp_re_bias; + *cc = r->cls == float_class_zero ? 1 : 3; + } else { + *cc = r->cls == float_class_zero ? 0 : 2; + } + + /* Adjust signs of zero results */ + if (r->cls == float_class_zero) { + r->sign = a->sign; + } + if (n->cls == float_class_zero) { + n->sign = a->sign ^ b->sign; + } + + status->float_exception_flags = float_exception_flags; + } +} + +#define DEFINE_S390_DIVIDE_TO_INTEGER(floatN) \ +static void floatN ## _s390_divide_to_integer(floatN a, floatN b, \ + int final_quotient_rounding_mode, bool mask_underflow, bool mask_inexact, \ + floatN *r, floatN *n, uint32_t *cc, int *dxc, float_status *status) \ +{ \ + FloatParts64 pa = floatN ## _unpack_canonical(a, status); \ + FloatParts64 pb = floatN ## _unpack_canonical(b, status); \ + FloatParts64 pr, pn; \ + parts_s390_divide_to_integer(&pa, &pb, final_quotient_rounding_mode, \ + mask_underflow, mask_inexact, \ + &floatN ## _params, \ + &pr, &pn, cc, dxc, status); \ + *r = floatN ## _round_pack_canonical(&pr, status); \ + *n = floatN ## _round_pack_canonical(&pn, status); \ +} + +DEFINE_S390_DIVIDE_TO_INTEGER(float32) +DEFINE_S390_DIVIDE_TO_INTEGER(float64) + void HELPER(dib)(CPUS390XState *env, uint32_t r1, uint32_t r2, uint32_t r3, uint32_t m4, uint32_t bits) { -- 2.43.0 Use softfloat-parts.h so that we can more naturally perform the required operations witha single rounding step. This happens to also simplify the NaN detection step. Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- target/arm/tcg/vec_helper.c | 77 +++++++++++++++++++------------------ 1 file changed, 40 insertions(+), 37 deletions(-) diff --git a/target/arm/tcg/vec_helper.c b/target/arm/tcg/vec_helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/tcg/vec_helper.c +++ b/target/arm/tcg/vec_helper.c @@ -XXX,XX +XXX,XX @@ #include "helper.h" #include "tcg/tcg-gvec-desc.h" #include "fpu/softfloat.h" +#include "fpu/softfloat-parts.h" #include "qemu/int128.h" #include "crypto/clmul.h" #include "vec_internal.h" @@ -XXX,XX +XXX,XX @@ float32 bfdotadd(float32 sum, uint32_t e1, uint32_t e2, float_status *fpst) float32 bfdotadd_ebf(float32 sum, uint32_t e1, uint32_t e2, float_status *fpst, float_status *fpst_odd) { + /* Unpack two BFloat16 into two Float32, trivially. */ float32 s1r = e1 << 16; float32 s1c = e1 & 0xffff0000u; float32 s2r = e2 << 16; float32 s2c = e2 & 0xffff0000u; float32 t32; + /* + * Compare f16_dotadd() in sme_helper.c, but here we have + * bfloat16 inputs. In particular that means that we do not + * want the FPCR.FZ16 flush semantics, so we use the normal + * float_status for the input handling here. + */ + FloatParts64 p1r = float32_unpack_canonical(s1r, fpst); + FloatParts64 p1c = float32_unpack_canonical(s1c, fpst); + FloatParts64 p2r = float32_unpack_canonical(s2r, fpst); + FloatParts64 p2c = float32_unpack_canonical(s2c, fpst); + + int all_mask = (float_cmask(p1r.cls) | float_cmask(p1c.cls) | + float_cmask(p1r.cls) | float_cmask(p1c.cls)); + /* C.f. FPProcessNaNs4 */ - if (float32_is_any_nan(s1r) || float32_is_any_nan(s1c) || - float32_is_any_nan(s2r) || float32_is_any_nan(s2c)) { - if (float32_is_signaling_nan(s1r, fpst)) { - t32 = s1r; - } else if (float32_is_signaling_nan(s1c, fpst)) { - t32 = s1c; - } else if (float32_is_signaling_nan(s2r, fpst)) { - t32 = s2r; - } else if (float32_is_signaling_nan(s2c, fpst)) { - t32 = s2c; - } else if (float32_is_any_nan(s1r)) { - t32 = s1r; - } else if (float32_is_any_nan(s1c)) { - t32 = s1c; - } else if (float32_is_any_nan(s2r)) { - t32 = s2r; + if (unlikely(all_mask & float_cmask_anynan)) { + if (unlikely(all_mask & float_cmask_snan)) { + if (p1r.cls == float_class_snan) { + t32 = s1r; + } else if (p1c.cls == float_class_snan) { + t32 = s1c; + } else if (p2r.cls == float_class_snan) { + t32 = s2r; + } else { + t32 = s2c; + } } else { - t32 = s2c; + if (p1r.cls == float_class_qnan) { + t32 = s1r; + } else if (p1c.cls == float_class_qnan) { + t32 = s1c; + } else if (p2r.cls == float_class_qnan) { + t32 = s2r; + } else { + t32 = s2c; + } } /* * FPConvertNaN(FPProcessNaN(t32)) will be done as part * of the final addition below. */ } else { - /* - * Compare f16_dotadd() in sme_helper.c, but here we have - * bfloat16 inputs. In particular that means that we do not - * want the FPCR.FZ16 flush semantics, so we use the normal - * float_status for the input handling here. - */ - float64 e1r = float32_to_float64(s1r, fpst); - float64 e1c = float32_to_float64(s1c, fpst); - float64 e2r = float32_to_float64(s2r, fpst); - float64 e2c = float32_to_float64(s2c, fpst); - float64 t64; - /* * The ARM pseudocode function FPDot performs both multiplies - * and the add with a single rounding operation. Emulate this - * by performing the first multiply in round-to-odd, then doing - * the second multiply as fused multiply-add, and rounding to - * float32 all in one step. + * and the add with a single rounding operation. */ - t64 = float64_mul(e1r, e2r, fpst_odd); - t64 = float64r32_muladd(e1c, e2c, t64, 0, fpst); + FloatParts64 tmp = parts64_mul(&p1r, &p2r, fpst); + tmp = parts64_muladd(&p1c, &p2c, &tmp, 0, fpst); - /* This conversion is exact, because we've already rounded. */ - t32 = float64_to_float32(t64, fpst); + t32 = float32_round_pack_canonical(&tmp, fpst); } /* The final accumulation step is not fused. */ -- 2.43.0 This argument is no longer used. Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- target/arm/tcg/vec_internal.h | 12 +++------ target/arm/tcg/sme_helper.c | 6 ++--- target/arm/tcg/vec_helper.c | 50 +++++++++++++++-------------------- 3 files changed, 29 insertions(+), 39 deletions(-) diff --git a/target/arm/tcg/vec_internal.h b/target/arm/tcg/vec_internal.h index XXXXXXX..XXXXXXX 100644 --- a/target/arm/tcg/vec_internal.h +++ b/target/arm/tcg/vec_internal.h @@ -XXX,XX +XXX,XX @@ float32 bfdotadd(float32 sum, uint32_t e1, uint32_t e2, float_status *fpst); * @sum: addend * @e1, @e2: multiplicand vectors * @fpst: floating-point status to use - * @fpst_odd: floating-point status to use for round-to-odd operations * * BFloat16 2-way dot product of @e1 & @e2, accumulating with @sum. * The @e1 and @e2 operands correspond to the 32-bit source vector @@ -XXX,XX +XXX,XX @@ float32 bfdotadd(float32 sum, uint32_t e1, uint32_t e2, float_status *fpst); * Corresponds to the ARM pseudocode function BFDotAdd, specialized * for the FPCR.EBF == 1 case. */ -float32 bfdotadd_ebf(float32 sum, uint32_t e1, uint32_t e2, - float_status *fpst, float_status *fpst_odd); +float32 bfdotadd_ebf(float32 sum, uint32_t e1, uint32_t e2, float_status *fpst); /** * is_ebf: * @env: CPU state * @statusp: pointer to floating point status to fill in - * @oddstatusp: pointer to floating point status to fill in for round-to-odd * * Determine whether a BFDotAdd operation should use FPCR.EBF = 0 - * or FPCR.EBF = 1 semantics. On return, has initialized *statusp - * and *oddstatusp to suitable float_status arguments to use with either - * bfdotadd() or bfdotadd_ebf(). + * or FPCR.EBF = 1 semantics. On return, has initialized *statusp as suitable + * for float_status arguments to either bfdotadd() or bfdotadd_ebf(). * Returns true for EBF = 1, false for EBF = 0. (The caller should use this * to decide whether to call bfdotadd() or bfdotadd_ebf().) */ -bool is_ebf(CPUARMState *env, float_status *statusp, float_status *oddstatusp); +bool is_ebf(CPUARMState *env, float_status *statusp); /* * Negate as for FPCR.AH=1 -- do not negate NaNs. diff --git a/target/arm/tcg/sme_helper.c b/target/arm/tcg/sme_helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/tcg/sme_helper.c +++ b/target/arm/tcg/sme_helper.c @@ -XXX,XX +XXX,XX @@ static void do_bfmopa_w(void *vza, void *vzn, void *vzm, uint32_t desc, uint32_t negx, bool ah_neg) { intptr_t row, col, oprsz = simd_maxsz(desc); - float_status fpst, fpst_odd; + float_status fpst; - if (is_ebf(env, &fpst, &fpst_odd)) { + if (is_ebf(env, &fpst)) { for (row = 0; row < oprsz; ) { uint16_t prow = pn[H2(row >> 4)]; do { @@ -XXX,XX +XXX,XX @@ static void do_bfmopa_w(void *vza, void *vzn, void *vzm, uint32_t m = *(uint32_t *)(vzm + H1_4(col)); m = f16mop_adj_pair(m, pcol, 0); - *a = bfdotadd_ebf(*a, n, m, &fpst, &fpst_odd); + *a = bfdotadd_ebf(*a, n, m, &fpst); } col += 4; pcol >>= 4; diff --git a/target/arm/tcg/vec_helper.c b/target/arm/tcg/vec_helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/tcg/vec_helper.c +++ b/target/arm/tcg/vec_helper.c @@ -XXX,XX +XXX,XX @@ DO_MMLA_B(gvec_usmmla_b, do_usmmla_b) * BFloat16 Dot Product */ -bool is_ebf(CPUARMState *env, float_status *statusp, float_status *oddstatusp) +bool is_ebf(CPUARMState *env, float_status *statusp) { /* * For BFDOT, BFMMLA, etc, the behaviour depends on FPCR.EBF. @@ -XXX,XX +XXX,XX @@ bool is_ebf(CPUARMState *env, float_status *statusp, float_status *oddstatusp) *statusp = env->vfp.fp_status[is_a64(env) ? FPST_A64 : FPST_A32]; set_default_nan_mode(true, statusp); - if (ebf) { - /* EBF=1 needs to do a step with round-to-odd semantics */ - *oddstatusp = *statusp; - set_float_rounding_mode(float_round_to_odd, oddstatusp); - } else { + if (!ebf) { set_flush_to_zero(true, statusp); set_flush_inputs_to_zero(true, statusp); set_float_rounding_mode(float_round_to_odd_inf, statusp); @@ -XXX,XX +XXX,XX @@ float32 bfdotadd(float32 sum, uint32_t e1, uint32_t e2, float_status *fpst) return t1; } -float32 bfdotadd_ebf(float32 sum, uint32_t e1, uint32_t e2, - float_status *fpst, float_status *fpst_odd) +float32 bfdotadd_ebf(float32 sum, uint32_t e1, uint32_t e2, float_status *fpst) { /* Unpack two BFloat16 into two Float32, trivially. */ float32 s1r = e1 << 16; @@ -XXX,XX +XXX,XX @@ void HELPER(gvec_bfdot)(void *vd, void *vn, void *vm, void *va, intptr_t i, opr_sz = simd_oprsz(desc); float32 *d = vd, *a = va; uint32_t *n = vn, *m = vm; - float_status fpst, fpst_odd; + float_status fpst; - if (is_ebf(env, &fpst, &fpst_odd)) { + if (is_ebf(env, &fpst)) { for (i = 0; i < opr_sz / 4; ++i) { - d[i] = bfdotadd_ebf(a[i], n[i], m[i], &fpst, &fpst_odd); + d[i] = bfdotadd_ebf(a[i], n[i], m[i], &fpst); } } else { for (i = 0; i < opr_sz / 4; ++i) { @@ -XXX,XX +XXX,XX @@ void HELPER(gvec_bfdot_idx)(void *vd, void *vn, void *vm, intptr_t eltspersegment = MIN(16 / 4, elements); float32 *d = vd, *a = va; uint32_t *n = vn, *m = vm; - float_status fpst, fpst_odd; + float_status fpst; - if (is_ebf(env, &fpst, &fpst_odd)) { + if (is_ebf(env, &fpst)) { for (i = 0; i < elements; i += eltspersegment) { uint32_t m_idx = m[i + H4(index)]; for (j = i; j < i + eltspersegment; j++) { - d[j] = bfdotadd_ebf(a[j], n[j], m_idx, &fpst, &fpst_odd); + d[j] = bfdotadd_ebf(a[j], n[j], m_idx, &fpst); } } } else { @@ -XXX,XX +XXX,XX @@ void HELPER(sme2_bfvdot_idx)(void *vd, void *vn, void *vm, uint16_t *n0 = vn; uint16_t *n1 = vn + sizeof(ARMVectorReg); uint32_t *m = vm; - float_status fpst, fpst_odd; + float_status fpst; - if (is_ebf(env, &fpst, &fpst_odd)) { + if (is_ebf(env, &fpst)) { for (i = 0; i < elements; i += eltspersegment) { uint32_t m_idx = m[i + H4(idx)]; for (j = 0; j < eltspersegment; j++) { uint32_t nn = (n0[H2(2 * (i + j) + sel)]) | (n1[H2(2 * (i + j) + sel)] << 16); - d[i + H4(j)] = bfdotadd_ebf(a[i + H4(j)], nn, m_idx, - &fpst, &fpst_odd); + d[i + H4(j)] = bfdotadd_ebf(a[i + H4(j)], nn, m_idx, &fpst); } } } else { @@ -XXX,XX +XXX,XX @@ void HELPER(gvec_bfmmla)(void *vd, void *vn, void *vm, void *va, intptr_t s, opr_sz = simd_oprsz(desc); float32 *d = vd, *a = va; uint32_t *n = vn, *m = vm; - float_status fpst, fpst_odd; + float_status fpst; - if (is_ebf(env, &fpst, &fpst_odd)) { + if (is_ebf(env, &fpst)) { for (s = 0; s < opr_sz / 4; s += 4) { float32 sum00, sum01, sum10, sum11; @@ -XXX,XX +XXX,XX @@ void HELPER(gvec_bfmmla)(void *vd, void *vn, void *vm, void *va, * i j i k j k */ sum00 = a[s + H4(0 + 0)]; - sum00 = bfdotadd_ebf(sum00, n[s + H4(0 + 0)], m[s + H4(0 + 0)], &fpst, &fpst_odd); - sum00 = bfdotadd_ebf(sum00, n[s + H4(0 + 1)], m[s + H4(0 + 1)], &fpst, &fpst_odd); + sum00 = bfdotadd_ebf(sum00, n[s + H4(0 + 0)], m[s + H4(0 + 0)], &fpst); + sum00 = bfdotadd_ebf(sum00, n[s + H4(0 + 1)], m[s + H4(0 + 1)], &fpst); sum01 = a[s + H4(0 + 1)]; - sum01 = bfdotadd_ebf(sum01, n[s + H4(0 + 0)], m[s + H4(2 + 0)], &fpst, &fpst_odd); - sum01 = bfdotadd_ebf(sum01, n[s + H4(0 + 1)], m[s + H4(2 + 1)], &fpst, &fpst_odd); + sum01 = bfdotadd_ebf(sum01, n[s + H4(0 + 0)], m[s + H4(2 + 0)], &fpst); + sum01 = bfdotadd_ebf(sum01, n[s + H4(0 + 1)], m[s + H4(2 + 1)], &fpst); sum10 = a[s + H4(2 + 0)]; - sum10 = bfdotadd_ebf(sum10, n[s + H4(2 + 0)], m[s + H4(0 + 0)], &fpst, &fpst_odd); - sum10 = bfdotadd_ebf(sum10, n[s + H4(2 + 1)], m[s + H4(0 + 1)], &fpst, &fpst_odd); + sum10 = bfdotadd_ebf(sum10, n[s + H4(2 + 0)], m[s + H4(0 + 0)], &fpst); + sum10 = bfdotadd_ebf(sum10, n[s + H4(2 + 1)], m[s + H4(0 + 1)], &fpst); sum11 = a[s + H4(2 + 1)]; - sum11 = bfdotadd_ebf(sum11, n[s + H4(2 + 0)], m[s + H4(2 + 0)], &fpst, &fpst_odd); - sum11 = bfdotadd_ebf(sum11, n[s + H4(2 + 1)], m[s + H4(2 + 1)], &fpst, &fpst_odd); + sum11 = bfdotadd_ebf(sum11, n[s + H4(2 + 0)], m[s + H4(2 + 0)], &fpst); + sum11 = bfdotadd_ebf(sum11, n[s + H4(2 + 1)], m[s + H4(2 + 1)], &fpst); d[s + H4(0 + 0)] = sum00; d[s + H4(0 + 1)] = sum01; -- 2.43.0 Use softfloat-parts.h so that we can more naturally perform the required operations witha single rounding step. This happens to also simplify the NaN detection step. Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- target/arm/tcg/sme_helper.c | 96 ++++++++++++++++--------------------- 1 file changed, 40 insertions(+), 56 deletions(-) diff --git a/target/arm/tcg/sme_helper.c b/target/arm/tcg/sme_helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/tcg/sme_helper.c +++ b/target/arm/tcg/sme_helper.c @@ -XXX,XX +XXX,XX @@ #include "accel/tcg/helper-retaddr.h" #include "qemu/int128.h" #include "fpu/softfloat.h" +#include "fpu/softfloat-parts.h" #include "vec_internal.h" #include "sve_ldst_internal.h" @@ -XXX,XX +XXX,XX @@ static inline uint32_t bf16mop_ah_neg_adj_pair(uint32_t pair, uint32_t pg) } static float32 f16_dotadd(float32 sum, uint32_t e1, uint32_t e2, - float_status *s_f16, float_status *s_std, - float_status *s_odd) + float_status *s_f16, float_status *s_std) { /* - * We need three different float_status for different parts of this + * We need two different float_status for different parts of this * operation: * - the input conversion of the float16 values must use the * f16-specific float_status, so that the FPCR.FZ16 control is applied * - operations on float32 including the final accumulation must use * the normal float_status, so that FPCR.FZ is applied - * - we have pre-set-up copy of s_std which is set to round-to-odd, - * for the multiply (see below) */ float16 h1r = e1 & 0xffff; float16 h1c = e1 >> 16; @@ -XXX,XX +XXX,XX @@ static float32 f16_dotadd(float32 sum, uint32_t e1, uint32_t e2, float16 h2c = e2 >> 16; float32 t32; + FloatParts64 p1r = float16_unpack_canonical(h1r, s_f16); + FloatParts64 p1c = float16_unpack_canonical(h1c, s_f16); + FloatParts64 p2r = float16_unpack_canonical(h2r, s_f16); + FloatParts64 p2c = float16_unpack_canonical(h2c, s_f16); + + int all_mask = (float_cmask(p1r.cls) | float_cmask(p1c.cls) | + float_cmask(p1r.cls) | float_cmask(p1c.cls)); + /* C.f. FPProcessNaNs4 */ - if (float16_is_any_nan(h1r) || float16_is_any_nan(h1c) || - float16_is_any_nan(h2r) || float16_is_any_nan(h2c)) { + if (unlikely(all_mask & float_cmask_anynan)) { float16 t16; - if (float16_is_signaling_nan(h1r, s_f16)) { - t16 = h1r; - } else if (float16_is_signaling_nan(h1c, s_f16)) { - t16 = h1c; - } else if (float16_is_signaling_nan(h2r, s_f16)) { - t16 = h2r; - } else if (float16_is_signaling_nan(h2c, s_f16)) { - t16 = h2c; - } else if (float16_is_any_nan(h1r)) { - t16 = h1r; - } else if (float16_is_any_nan(h1c)) { - t16 = h1c; - } else if (float16_is_any_nan(h2r)) { - t16 = h2r; + if (unlikely(all_mask & float_cmask_snan)) { + if (p1r.cls == float_class_snan) { + t16 = h1r; + } else if (p1c.cls == float_class_snan) { + t16 = h1c; + } else if (p2r.cls == float_class_snan) { + t16 = h2r; + } else { + t16 = h2c; + } } else { - t16 = h2c; + if (p1r.cls == float_class_qnan) { + t16 = h1r; + } else if (p1c.cls == float_class_qnan) { + t16 = h1c; + } else if (p2r.cls == float_class_qnan) { + t16 = h2r; + } else { + t16 = h2c; + } } t32 = float16_to_float32(t16, true, s_f16); } else { - float64 e1r = float16_to_float64(h1r, true, s_f16); - float64 e1c = float16_to_float64(h1c, true, s_f16); - float64 e2r = float16_to_float64(h2r, true, s_f16); - float64 e2c = float16_to_float64(h2c, true, s_f16); - float64 t64; - /* * The ARM pseudocode function FPDot performs both multiplies - * and the add with a single rounding operation. Emulate this - * by performing the first multiply in round-to-odd, then doing - * the second multiply as fused multiply-add, and rounding to - * float32 all in one step. + * and the add with a single rounding operation. */ - t64 = float64_mul(e1r, e2r, s_odd); - t64 = float64r32_muladd(e1c, e2c, t64, 0, s_std); + FloatParts64 tmp = parts64_mul(&p1r, &p2r, s_std); + tmp = parts64_muladd(&p1c, &p2c, &tmp, 0, s_std); - /* This conversion is exact, because we've already rounded. */ - t32 = float64_to_float32(t64, s_std); + t32 = float32_round_pack_canonical(&tmp, s_std); } /* The final accumulation step is not fused. */ @@ -XXX,XX +XXX,XX @@ static void do_fmopa_w_h(void *vza, void *vzn, void *vzm, uint16_t *pn, uint32_t negx, bool ah_neg) { intptr_t row, col, oprsz = simd_maxsz(desc); - float_status fpst_odd = env->vfp.fp_status[FPST_ZA]; - - set_float_rounding_mode(float_round_to_odd, &fpst_odd); for (row = 0; row < oprsz; ) { uint16_t prow = pn[H2(row >> 4)]; @@ -XXX,XX +XXX,XX @@ static void do_fmopa_w_h(void *vza, void *vzn, void *vzm, uint16_t *pn, m = f16mop_adj_pair(m, pcol, 0); *a = f16_dotadd(*a, n, m, &env->vfp.fp_status[FPST_ZA_F16], - &env->vfp.fp_status[FPST_ZA], - &fpst_odd); + &env->vfp.fp_status[FPST_ZA]); } col += 4; pcol >>= 4; @@ -XXX,XX +XXX,XX @@ void HELPER(sme2_fdot_h)(void *vd, void *vn, void *vm, void *va, bool za = extract32(desc, SIMD_DATA_SHIFT, 1); float_status *fpst_std = &env->vfp.fp_status[za ? FPST_ZA : FPST_A64]; float_status *fpst_f16 = &env->vfp.fp_status[za ? FPST_ZA_F16 : FPST_A64_F16]; - float_status fpst_odd = *fpst_std; float32 *d = vd, *a = va; uint32_t *n = vn, *m = vm; - set_float_rounding_mode(float_round_to_odd, &fpst_odd); - for (i = 0; i < oprsz / sizeof(float32); ++i) { d[H4(i)] = f16_dotadd(a[H4(i)], n[H4(i)], m[H4(i)], - fpst_f16, fpst_std, &fpst_odd); + fpst_f16, fpst_std); } } @@ -XXX,XX +XXX,XX @@ void HELPER(sme2_fdot_idx_h)(void *vd, void *vn, void *vm, void *va, bool za = extract32(desc, SIMD_DATA_SHIFT + 2, 1); float_status *fpst_std = &env->vfp.fp_status[za ? FPST_ZA : FPST_A64]; float_status *fpst_f16 = &env->vfp.fp_status[za ? FPST_ZA_F16 : FPST_A64_F16]; - float_status fpst_odd = *fpst_std; float32 *d = vd, *a = va; uint32_t *n = vn, *m = (uint32_t *)vm + H4(idx); - set_float_rounding_mode(float_round_to_odd, &fpst_odd); - for (i = 0; i < elements; i += eltspersegment) { uint32_t mm = m[i]; for (j = 0; j < eltspersegment; ++j) { d[H4(i + j)] = f16_dotadd(a[H4(i + j)], n[H4(i + j)], mm, - fpst_f16, fpst_std, &fpst_odd); + fpst_f16, fpst_std); } } } @@ -XXX,XX +XXX,XX @@ void HELPER(sme2_fvdot_idx_h)(void *vd, void *vn, void *vm, void *va, intptr_t eltspersegment = MIN(4, elements); int idx = extract32(desc, SIMD_DATA_SHIFT, 2); int sel = extract32(desc, SIMD_DATA_SHIFT + 2, 1); - float_status fpst_odd, *fpst_std, *fpst_f16; float32 *d = vd, *a = va; uint16_t *n0 = vn; uint16_t *n1 = vn + sizeof(ARMVectorReg); uint32_t *m = (uint32_t *)vm + H4(idx); - fpst_std = &env->vfp.fp_status[FPST_ZA]; - fpst_f16 = &env->vfp.fp_status[FPST_ZA_F16]; - fpst_odd = *fpst_std; - set_float_rounding_mode(float_round_to_odd, &fpst_odd); - for (i = 0; i < elements; i += eltspersegment) { uint32_t mm = m[i]; for (j = 0; j < eltspersegment; ++j) { uint32_t nn = (n0[H2(2 * (i + j) + sel)]) | (n1[H2(2 * (i + j) + sel)] << 16); d[i + H4(j)] = f16_dotadd(a[i + H4(j)], nn, mm, - fpst_f16, fpst_std, &fpst_odd); + &env->vfp.fp_status[FPST_ZA_F16], + &env->vfp.fp_status[FPST_ZA]); } } } -- 2.43.0 Notice the conflict between saturation and rebias_overflow, which should never happen. Otherwise, saturate to INT32_MAX and allow the usual overflow to max, infinity, or dnan. Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- fpu/softfloat-parts.c.inc | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/fpu/softfloat-parts.c.inc b/fpu/softfloat-parts.c.inc index XXXXXXX..XXXXXXX 100644 --- a/fpu/softfloat-parts.c.inc +++ b/fpu/softfloat-parts.c.inc @@ -XXX,XX +XXX,XX @@ static void partsN(uncanon_normal)(FloatPartsN *p, float_status *s, g_assert_not_reached(); } - exp = p->exp + fmt->exp_bias; + /* Because exp_bias is positive, we can only overflow past INT_MAX. */ + if (sadd32_overflow(p->exp, fmt->exp_bias, &exp)) { + /* + * rebias_overflow wants to compute a modulo exponent, which + * conflicts with saturation. That said, saturation can only + * happen with scalbn, which is not a PowerPC operation. + */ + assert(!s->rebias_overflow); + exp = INT32_MAX; + } + if (likely(exp > 0)) { if (p->frac_lo & round_mask) { flags |= float_flag_inexact; -- 2.43.0 Avoid exponent overflow as well as checking that we don't lose information with opposing scaling. Use it in partsN(scalbn) and partsN(round_to_int_normal). Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- fpu/softfloat.c | 17 +++++++++++++++++ fpu/softfloat-parts.c.inc | 5 ++--- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/fpu/softfloat.c b/fpu/softfloat.c index XXXXXXX..XXXXXXX 100644 --- a/fpu/softfloat.c +++ b/fpu/softfloat.c @@ -XXX,XX +XXX,XX @@ static float128 QEMU_FLATTEN float128_pack_raw(const FloatParts128 *p) *----------------------------------------------------------------------------*/ #include "softfloat-specialize.c.inc" +static int32_t exp_scalbn(int32_t exp, int32_t scale) +{ + /* + * Catch chains of scaling which lose information. + * In particular, if the exponent has been saturated, + * do not allow it to become unsaturated. + */ + if (unlikely(exp == INT32_MAX)) { + assert(scale >= 0); + } else if (unlikely(exp == INT32_MIN)) { + assert(scale <= 0); + } else { + exp = sadd32_saturate(exp, scale); + } + return exp; +} + /* * Helper functions for softfloat-parts.c.inc, per-size operations. */ diff --git a/fpu/softfloat-parts.c.inc b/fpu/softfloat-parts.c.inc index XXXXXXX..XXXXXXX 100644 --- a/fpu/softfloat-parts.c.inc +++ b/fpu/softfloat-parts.c.inc @@ -XXX,XX +XXX,XX @@ static bool partsN(round_to_int_normal)(FloatPartsN *a, FloatRoundMode rmode, uint64_t frac_lsb, frac_lsbm1, rnd_even_mask, rnd_mask, inc; int shift_adj; - scale = MIN(MAX(scale, -0x10000), 0x10000); - a->exp += scale; + a->exp = exp_scalbn(a->exp, scale); if (a->exp < 0) { bool one; @@ -XXX,XX +XXX,XX @@ FloatPartsN partsN(scalbn)(const FloatPartsN *a, int n, float_status *s) case float_class_normal: { FloatPartsN r = *a; - r.exp += MIN(MAX(n, -0x10000), 0x10000); + r.exp = exp_scalbn(r.exp, n); return r; } default: -- 2.43.0