The following changes since commit c52d69e7dbaaed0ffdef8125e79218672c30161d: Merge remote-tracking branch 'remotes/cschoenebeck/tags/pull-9p-20211027' into staging (2021-10-27 11:45:18 -0700) are available in the Git repository at: https://gitlab.com/rth7680/qemu.git tags/pull-tcg-20211027 for you to fetch changes up to 820c025f0dcacf2f3c12735b1f162893fbfa7bc6: tcg/optimize: Propagate sign info for shifting (2021-10-27 17:11:23 -0700) ---------------------------------------------------------------- Improvements to qemu/int128 Fixes for 128/64 division. Cleanup tcg/optimize.c Optimize redundant sign extensions ---------------------------------------------------------------- Frédéric Pétrot (1): qemu/int128: Add int128_{not,xor} Luis Pires (4): host-utils: move checks out of divu128/divs128 host-utils: move udiv_qrnnd() to host-utils host-utils: add 128-bit quotient support to divu128/divs128 host-utils: add unit tests for divu128/divs128 Richard Henderson (51): tcg/optimize: Rename "mask" to "z_mask" tcg/optimize: Split out OptContext tcg/optimize: Remove do_default label tcg/optimize: Change tcg_opt_gen_{mov,movi} interface tcg/optimize: Move prev_mb into OptContext tcg/optimize: Split out init_arguments tcg/optimize: Split out copy_propagate tcg/optimize: Split out fold_call tcg/optimize: Drop nb_oargs, nb_iargs locals tcg/optimize: Change fail return for do_constant_folding_cond* tcg/optimize: Return true from tcg_opt_gen_{mov,movi} tcg/optimize: Split out finish_folding tcg/optimize: Use a boolean to avoid a mass of continues tcg/optimize: Split out fold_mb, fold_qemu_{ld,st} tcg/optimize: Split out fold_const{1,2} tcg/optimize: Split out fold_setcond2 tcg/optimize: Split out fold_brcond2 tcg/optimize: Split out fold_brcond tcg/optimize: Split out fold_setcond tcg/optimize: Split out fold_mulu2_i32 tcg/optimize: Split out fold_addsub2_i32 tcg/optimize: Split out fold_movcond tcg/optimize: Split out fold_extract2 tcg/optimize: Split out fold_extract, fold_sextract tcg/optimize: Split out fold_deposit tcg/optimize: Split out fold_count_zeros tcg/optimize: Split out fold_bswap tcg/optimize: Split out fold_dup, fold_dup2 tcg/optimize: Split out fold_mov tcg/optimize: Split out fold_xx_to_i tcg/optimize: Split out fold_xx_to_x tcg/optimize: Split out fold_xi_to_i tcg/optimize: Add type to OptContext tcg/optimize: Split out fold_to_not tcg/optimize: Split out fold_sub_to_neg tcg/optimize: Split out fold_xi_to_x tcg/optimize: Split out fold_ix_to_i tcg/optimize: Split out fold_masks tcg/optimize: Expand fold_mulu2_i32 to all 4-arg multiplies tcg/optimize: Expand fold_addsub2_i32 to 64-bit ops tcg/optimize: Sink commutative operand swapping into fold functions tcg/optimize: Stop forcing z_mask to "garbage" for 32-bit values tcg/optimize: Use fold_xx_to_i for orc tcg/optimize: Use fold_xi_to_x for mul tcg/optimize: Use fold_xi_to_x for div tcg/optimize: Use fold_xx_to_i for rem tcg/optimize: Optimize sign extensions tcg/optimize: Propagate sign info for logical operations tcg/optimize: Propagate sign info for setcond tcg/optimize: Propagate sign info for bit counting tcg/optimize: Propagate sign info for shifting include/fpu/softfloat-macros.h | 82 -- include/hw/clock.h | 5 +- include/qemu/host-utils.h | 121 +- include/qemu/int128.h | 20 + target/ppc/int_helper.c | 23 +- tcg/optimize.c | 2644 ++++++++++++++++++++++++---------------- tests/unit/test-div128.c | 197 +++ util/host-utils.c | 147 ++- tests/unit/meson.build | 1 + 9 files changed, 2053 insertions(+), 1187 deletions(-) create mode 100644 tests/unit/test-div128.c From: Frédéric Pétrot <frederic.petrot@univ-grenoble-alpes.fr> Addition of not and xor on 128-bit integers. Signed-off-by: Frédéric Pétrot <frederic.petrot@univ-grenoble-alpes.fr> Co-authored-by: Fabien Portas <fabien.portas@grenoble-inp.org> Message-Id: <20211025122818.168890-3-frederic.petrot@univ-grenoble-alpes.fr> [rth: Split out logical operations.] Reviewed-by: Richard Henderson <richard.henderson@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- include/qemu/int128.h | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/include/qemu/int128.h b/include/qemu/int128.h index XXXXXXX..XXXXXXX 100644 --- a/include/qemu/int128.h +++ b/include/qemu/int128.h @@ -XXX,XX +XXX,XX @@ static inline Int128 int128_exts64(int64_t a) return a; } +static inline Int128 int128_not(Int128 a) +{ + return ~a; +} + static inline Int128 int128_and(Int128 a, Int128 b) { return a & b; @@ -XXX,XX +XXX,XX @@ static inline Int128 int128_or(Int128 a, Int128 b) return a | b; } +static inline Int128 int128_xor(Int128 a, Int128 b) +{ + return a ^ b; +} + static inline Int128 int128_rshift(Int128 a, int n) { return a >> n; @@ -XXX,XX +XXX,XX @@ static inline Int128 int128_exts64(int64_t a) return int128_make128(a, (a < 0) ? -1 : 0); } +static inline Int128 int128_not(Int128 a) +{ + return int128_make128(~a.lo, ~a.hi); +} + static inline Int128 int128_and(Int128 a, Int128 b) { return int128_make128(a.lo & b.lo, a.hi & b.hi); @@ -XXX,XX +XXX,XX @@ static inline Int128 int128_or(Int128 a, Int128 b) return int128_make128(a.lo | b.lo, a.hi | b.hi); } +static inline Int128 int128_xor(Int128 a, Int128 b) +{ + return int128_make128(a.lo ^ b.lo, a.hi ^ b.hi); +} + static inline Int128 int128_rshift(Int128 a, int n) { int64_t h; -- 2.25.1 From: Luis Pires <luis.pires@eldorado.org.br> In preparation for changing the divu128/divs128 implementations to allow for quotients larger than 64 bits, move the div-by-zero and overflow checks to the callers. Signed-off-by: Luis Pires <luis.pires@eldorado.org.br> Reviewed-by: Richard Henderson <richard.henderson@linaro.org> Message-Id: <20211025191154.350831-2-luis.pires@eldorado.org.br> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- include/hw/clock.h | 5 +++-- include/qemu/host-utils.h | 34 ++++++++++++--------------------- target/ppc/int_helper.c | 14 +++++++++----- util/host-utils.c | 40 ++++++++++++++++++--------------------- 4 files changed, 42 insertions(+), 51 deletions(-) diff --git a/include/hw/clock.h b/include/hw/clock.h index XXXXXXX..XXXXXXX 100644 --- a/include/hw/clock.h +++ b/include/hw/clock.h @@ -XXX,XX +XXX,XX @@ static inline uint64_t clock_ns_to_ticks(const Clock *clk, uint64_t ns) return 0; } /* - * Ignore divu128() return value as we've caught div-by-zero and don't - * need different behaviour for overflow. + * BUG: when CONFIG_INT128 is not defined, the current implementation of + * divu128 does not return a valid truncated quotient, so the result will + * be wrong. */ divu128(&lo, &hi, clk->period); return lo; diff --git a/include/qemu/host-utils.h b/include/qemu/host-utils.h index XXXXXXX..XXXXXXX 100644 --- a/include/qemu/host-utils.h +++ b/include/qemu/host-utils.h @@ -XXX,XX +XXX,XX @@ static inline uint64_t muldiv64(uint64_t a, uint32_t b, uint32_t c) return (__int128_t)a * b / c; } -static inline int divu128(uint64_t *plow, uint64_t *phigh, uint64_t divisor) +static inline void divu128(uint64_t *plow, uint64_t *phigh, uint64_t divisor) { - if (divisor == 0) { - return 1; - } else { - __uint128_t dividend = ((__uint128_t)*phigh << 64) | *plow; - __uint128_t result = dividend / divisor; - *plow = result; - *phigh = dividend % divisor; - return result > UINT64_MAX; - } + __uint128_t dividend = ((__uint128_t)*phigh << 64) | *plow; + __uint128_t result = dividend / divisor; + *plow = result; + *phigh = dividend % divisor; } -static inline int divs128(int64_t *plow, int64_t *phigh, int64_t divisor) +static inline void divs128(int64_t *plow, int64_t *phigh, int64_t divisor) { - if (divisor == 0) { - return 1; - } else { - __int128_t dividend = ((__int128_t)*phigh << 64) | (uint64_t)*plow; - __int128_t result = dividend / divisor; - *plow = result; - *phigh = dividend % divisor; - return result != *plow; - } + __int128_t dividend = ((__int128_t)*phigh << 64) | (uint64_t)*plow; + __int128_t result = dividend / divisor; + *plow = result; + *phigh = dividend % divisor; } #else void muls64(uint64_t *plow, uint64_t *phigh, int64_t a, int64_t b); void mulu64(uint64_t *plow, uint64_t *phigh, uint64_t a, uint64_t b); -int divu128(uint64_t *plow, uint64_t *phigh, uint64_t divisor); -int divs128(int64_t *plow, int64_t *phigh, int64_t divisor); +void divu128(uint64_t *plow, uint64_t *phigh, uint64_t divisor); +void divs128(int64_t *plow, int64_t *phigh, int64_t divisor); static inline uint64_t muldiv64(uint64_t a, uint32_t b, uint32_t c) { diff --git a/target/ppc/int_helper.c b/target/ppc/int_helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/ppc/int_helper.c +++ b/target/ppc/int_helper.c @@ -XXX,XX +XXX,XX @@ uint64_t helper_divdeu(CPUPPCState *env, uint64_t ra, uint64_t rb, uint32_t oe) uint64_t rt = 0; int overflow = 0; - overflow = divu128(&rt, &ra, rb); - - if (unlikely(overflow)) { + if (unlikely(rb == 0 || ra >= rb)) { + overflow = 1; rt = 0; /* Undefined */ + } else { + divu128(&rt, &ra, rb); } if (oe) { @@ -XXX,XX +XXX,XX @@ uint64_t helper_divde(CPUPPCState *env, uint64_t rau, uint64_t rbu, uint32_t oe) int64_t rt = 0; int64_t ra = (int64_t)rau; int64_t rb = (int64_t)rbu; - int overflow = divs128(&rt, &ra, rb); + int overflow = 0; - if (unlikely(overflow)) { + if (unlikely(rb == 0 || uabs64(ra) >= uabs64(rb))) { + overflow = 1; rt = 0; /* Undefined */ + } else { + divs128(&rt, &ra, rb); } if (oe) { diff --git a/util/host-utils.c b/util/host-utils.c index XXXXXXX..XXXXXXX 100644 --- a/util/host-utils.c +++ b/util/host-utils.c @@ -XXX,XX +XXX,XX @@ void muls64 (uint64_t *plow, uint64_t *phigh, int64_t a, int64_t b) *phigh = rh; } -/* Unsigned 128x64 division. Returns 1 if overflow (divide by zero or */ -/* quotient exceeds 64 bits). Otherwise returns quotient via plow and */ -/* remainder via phigh. */ -int divu128(uint64_t *plow, uint64_t *phigh, uint64_t divisor) +/* + * Unsigned 128-by-64 division. Returns quotient via plow and + * remainder via phigh. + * The result must fit in 64 bits (plow) - otherwise, the result + * is undefined. + * This function will cause a division by zero if passed a zero divisor. + */ +void divu128(uint64_t *plow, uint64_t *phigh, uint64_t divisor) { uint64_t dhi = *phigh; uint64_t dlo = *plow; unsigned i; uint64_t carry = 0; - if (divisor == 0) { - return 1; - } else if (dhi == 0) { + if (divisor == 0 || dhi == 0) { *plow = dlo / divisor; *phigh = dlo % divisor; - return 0; - } else if (dhi >= divisor) { - return 1; } else { for (i = 0; i < 64; i++) { @@ -XXX,XX +XXX,XX @@ int divu128(uint64_t *plow, uint64_t *phigh, uint64_t divisor) *plow = dlo; *phigh = dhi; - return 0; } } -int divs128(int64_t *plow, int64_t *phigh, int64_t divisor) +/* + * Signed 128-by-64 division. Returns quotient via plow and + * remainder via phigh. + * The result must fit in 64 bits (plow) - otherwise, the result + * is undefined. + * This function will cause a division by zero if passed a zero divisor. + */ +void divs128(int64_t *plow, int64_t *phigh, int64_t divisor) { int sgn_dvdnd = *phigh < 0; int sgn_divsr = divisor < 0; - int overflow = 0; if (sgn_dvdnd) { *plow = ~(*plow); @@ -XXX,XX +XXX,XX @@ int divs128(int64_t *plow, int64_t *phigh, int64_t divisor) divisor = 0 - divisor; } - overflow = divu128((uint64_t *)plow, (uint64_t *)phigh, (uint64_t)divisor); + divu128((uint64_t *)plow, (uint64_t *)phigh, (uint64_t)divisor); if (sgn_dvdnd ^ sgn_divsr) { *plow = 0 - *plow; } - - if (!overflow) { - if ((*plow < 0) ^ (sgn_dvdnd ^ sgn_divsr)) { - overflow = 1; - } - } - - return overflow; } #endif -- 2.25.1 From: Luis Pires <luis.pires@eldorado.org.br> Move udiv_qrnnd() from include/fpu/softfloat-macros.h to host-utils, so it can be reused by divu128(). Signed-off-by: Luis Pires <luis.pires@eldorado.org.br> Reviewed-by: Richard Henderson <richard.henderson@linaro.org> Message-Id: <20211025191154.350831-3-luis.pires@eldorado.org.br> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- include/fpu/softfloat-macros.h | 82 ---------------------------------- include/qemu/host-utils.h | 81 +++++++++++++++++++++++++++++++++ 2 files changed, 81 insertions(+), 82 deletions(-) diff --git a/include/fpu/softfloat-macros.h b/include/fpu/softfloat-macros.h index XXXXXXX..XXXXXXX 100644 --- a/include/fpu/softfloat-macros.h +++ b/include/fpu/softfloat-macros.h @@ -XXX,XX +XXX,XX @@ * so some portions are provided under: * the SoftFloat-2a license * the BSD license - * GPL-v2-or-later * * Any future contributions to this file after December 1st 2014 will be * taken to be licensed under the Softfloat-2a license unless specifically @@ -XXX,XX +XXX,XX @@ this code that are retained. * THE POSSIBILITY OF SUCH DAMAGE. */ -/* Portions of this work are licensed under the terms of the GNU GPL, - * version 2 or later. See the COPYING file in the top-level directory. - */ - #ifndef FPU_SOFTFLOAT_MACROS_H #define FPU_SOFTFLOAT_MACROS_H @@ -XXX,XX +XXX,XX @@ static inline uint64_t estimateDiv128To64(uint64_t a0, uint64_t a1, uint64_t b) } -/* From the GNU Multi Precision Library - longlong.h __udiv_qrnnd - * (https://gmplib.org/repo/gmp/file/tip/longlong.h) - * - * Licensed under the GPLv2/LGPLv3 - */ -static inline uint64_t udiv_qrnnd(uint64_t *r, uint64_t n1, - uint64_t n0, uint64_t d) -{ -#if defined(__x86_64__) - uint64_t q; - asm("divq %4" : "=a"(q), "=d"(*r) : "0"(n0), "1"(n1), "rm"(d)); - return q; -#elif defined(__s390x__) && !defined(__clang__) - /* Need to use a TImode type to get an even register pair for DLGR. */ - unsigned __int128 n = (unsigned __int128)n1 << 64 | n0; - asm("dlgr %0, %1" : "+r"(n) : "r"(d)); - *r = n >> 64; - return n; -#elif defined(_ARCH_PPC64) && defined(_ARCH_PWR7) - /* From Power ISA 2.06, programming note for divdeu. */ - uint64_t q1, q2, Q, r1, r2, R; - asm("divdeu %0,%2,%4; divdu %1,%3,%4" - : "=&r"(q1), "=r"(q2) - : "r"(n1), "r"(n0), "r"(d)); - r1 = -(q1 * d); /* low part of (n1<<64) - (q1 * d) */ - r2 = n0 - (q2 * d); - Q = q1 + q2; - R = r1 + r2; - if (R >= d || R < r2) { /* overflow implies R > d */ - Q += 1; - R -= d; - } - *r = R; - return Q; -#else - uint64_t d0, d1, q0, q1, r1, r0, m; - - d0 = (uint32_t)d; - d1 = d >> 32; - - r1 = n1 % d1; - q1 = n1 / d1; - m = q1 * d0; - r1 = (r1 << 32) | (n0 >> 32); - if (r1 < m) { - q1 -= 1; - r1 += d; - if (r1 >= d) { - if (r1 < m) { - q1 -= 1; - r1 += d; - } - } - } - r1 -= m; - - r0 = r1 % d1; - q0 = r1 / d1; - m = q0 * d0; - r0 = (r0 << 32) | (uint32_t)n0; - if (r0 < m) { - q0 -= 1; - r0 += d; - if (r0 >= d) { - if (r0 < m) { - q0 -= 1; - r0 += d; - } - } - } - r0 -= m; - - *r = r0; - return (q1 << 32) | q0; -#endif -} - /*---------------------------------------------------------------------------- | Returns an approximation to the square root of the 32-bit significand given | by `a'. Considered as an integer, `a' must be at least 2^31. If bit 0 of diff --git a/include/qemu/host-utils.h b/include/qemu/host-utils.h index XXXXXXX..XXXXXXX 100644 --- a/include/qemu/host-utils.h +++ b/include/qemu/host-utils.h @@ -XXX,XX +XXX,XX @@ * THE SOFTWARE. */ +/* Portions of this work are licensed under the terms of the GNU GPL, + * version 2 or later. See the COPYING file in the top-level directory. + */ + #ifndef HOST_UTILS_H #define HOST_UTILS_H @@ -XXX,XX +XXX,XX @@ void urshift(uint64_t *plow, uint64_t *phigh, int32_t shift); */ void ulshift(uint64_t *plow, uint64_t *phigh, int32_t shift, bool *overflow); +/* From the GNU Multi Precision Library - longlong.h __udiv_qrnnd + * (https://gmplib.org/repo/gmp/file/tip/longlong.h) + * + * Licensed under the GPLv2/LGPLv3 + */ +static inline uint64_t udiv_qrnnd(uint64_t *r, uint64_t n1, + uint64_t n0, uint64_t d) +{ +#if defined(__x86_64__) + uint64_t q; + asm("divq %4" : "=a"(q), "=d"(*r) : "0"(n0), "1"(n1), "rm"(d)); + return q; +#elif defined(__s390x__) && !defined(__clang__) + /* Need to use a TImode type to get an even register pair for DLGR. */ + unsigned __int128 n = (unsigned __int128)n1 << 64 | n0; + asm("dlgr %0, %1" : "+r"(n) : "r"(d)); + *r = n >> 64; + return n; +#elif defined(_ARCH_PPC64) && defined(_ARCH_PWR7) + /* From Power ISA 2.06, programming note for divdeu. */ + uint64_t q1, q2, Q, r1, r2, R; + asm("divdeu %0,%2,%4; divdu %1,%3,%4" + : "=&r"(q1), "=r"(q2) + : "r"(n1), "r"(n0), "r"(d)); + r1 = -(q1 * d); /* low part of (n1<<64) - (q1 * d) */ + r2 = n0 - (q2 * d); + Q = q1 + q2; + R = r1 + r2; + if (R >= d || R < r2) { /* overflow implies R > d */ + Q += 1; + R -= d; + } + *r = R; + return Q; +#else + uint64_t d0, d1, q0, q1, r1, r0, m; + + d0 = (uint32_t)d; + d1 = d >> 32; + + r1 = n1 % d1; + q1 = n1 / d1; + m = q1 * d0; + r1 = (r1 << 32) | (n0 >> 32); + if (r1 < m) { + q1 -= 1; + r1 += d; + if (r1 >= d) { + if (r1 < m) { + q1 -= 1; + r1 += d; + } + } + } + r1 -= m; + + r0 = r1 % d1; + q0 = r1 / d1; + m = q0 * d0; + r0 = (r0 << 32) | (uint32_t)n0; + if (r0 < m) { + q0 -= 1; + r0 += d; + if (r0 >= d) { + if (r0 < m) { + q0 -= 1; + r0 += d; + } + } + } + r0 -= m; + + *r = r0; + return (q1 << 32) | q0; +#endif +} + #endif -- 2.25.1 From: Luis Pires <luis.pires@eldorado.org.br> These will be used to implement new decimal floating point instructions from Power ISA 3.1. The remainder is now returned directly by divu128/divs128, freeing up phigh to receive the high 64 bits of the quotient. Signed-off-by: Luis Pires <luis.pires@eldorado.org.br> Reviewed-by: Richard Henderson <richard.henderson@linaro.org> Message-Id: <20211025191154.350831-4-luis.pires@eldorado.org.br> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- include/hw/clock.h | 6 +- include/qemu/host-utils.h | 20 ++++-- target/ppc/int_helper.c | 9 +-- util/host-utils.c | 133 +++++++++++++++++++++++++------------- 4 files changed, 108 insertions(+), 60 deletions(-) diff --git a/include/hw/clock.h b/include/hw/clock.h index XXXXXXX..XXXXXXX 100644 --- a/include/hw/clock.h +++ b/include/hw/clock.h @@ -XXX,XX +XXX,XX @@ static inline uint64_t clock_ns_to_ticks(const Clock *clk, uint64_t ns) if (clk->period == 0) { return 0; } - /* - * BUG: when CONFIG_INT128 is not defined, the current implementation of - * divu128 does not return a valid truncated quotient, so the result will - * be wrong. - */ + divu128(&lo, &hi, clk->period); return lo; } diff --git a/include/qemu/host-utils.h b/include/qemu/host-utils.h index XXXXXXX..XXXXXXX 100644 --- a/include/qemu/host-utils.h +++ b/include/qemu/host-utils.h @@ -XXX,XX +XXX,XX @@ static inline uint64_t muldiv64(uint64_t a, uint32_t b, uint32_t c) return (__int128_t)a * b / c; } -static inline void divu128(uint64_t *plow, uint64_t *phigh, uint64_t divisor) +static inline uint64_t divu128(uint64_t *plow, uint64_t *phigh, + uint64_t divisor) { __uint128_t dividend = ((__uint128_t)*phigh << 64) | *plow; __uint128_t result = dividend / divisor; + *plow = result; - *phigh = dividend % divisor; + *phigh = result >> 64; + return dividend % divisor; } -static inline void divs128(int64_t *plow, int64_t *phigh, int64_t divisor) +static inline int64_t divs128(uint64_t *plow, int64_t *phigh, + int64_t divisor) { - __int128_t dividend = ((__int128_t)*phigh << 64) | (uint64_t)*plow; + __int128_t dividend = ((__int128_t)*phigh << 64) | *plow; __int128_t result = dividend / divisor; + *plow = result; - *phigh = dividend % divisor; + *phigh = result >> 64; + return dividend % divisor; } #else void muls64(uint64_t *plow, uint64_t *phigh, int64_t a, int64_t b); void mulu64(uint64_t *plow, uint64_t *phigh, uint64_t a, uint64_t b); -void divu128(uint64_t *plow, uint64_t *phigh, uint64_t divisor); -void divs128(int64_t *plow, int64_t *phigh, int64_t divisor); +uint64_t divu128(uint64_t *plow, uint64_t *phigh, uint64_t divisor); +int64_t divs128(uint64_t *plow, int64_t *phigh, int64_t divisor); static inline uint64_t muldiv64(uint64_t a, uint32_t b, uint32_t c) { diff --git a/target/ppc/int_helper.c b/target/ppc/int_helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/ppc/int_helper.c +++ b/target/ppc/int_helper.c @@ -XXX,XX +XXX,XX @@ uint64_t helper_divdeu(CPUPPCState *env, uint64_t ra, uint64_t rb, uint32_t oe) uint64_t helper_divde(CPUPPCState *env, uint64_t rau, uint64_t rbu, uint32_t oe) { - int64_t rt = 0; + uint64_t rt = 0; int64_t ra = (int64_t)rau; int64_t rb = (int64_t)rbu; int overflow = 0; @@ -XXX,XX +XXX,XX @@ uint32_t helper_bcdcfsq(ppc_avr_t *r, ppc_avr_t *b, uint32_t ps) int cr; uint64_t lo_value; uint64_t hi_value; + uint64_t rem; ppc_avr_t ret = { .u64 = { 0, 0 } }; if (b->VsrSD(0) < 0) { @@ -XXX,XX +XXX,XX @@ uint32_t helper_bcdcfsq(ppc_avr_t *r, ppc_avr_t *b, uint32_t ps) * In that case, we leave r unchanged. */ } else { - divu128(&lo_value, &hi_value, 1000000000000000ULL); + rem = divu128(&lo_value, &hi_value, 1000000000000000ULL); - for (i = 1; i < 16; hi_value /= 10, i++) { - bcd_put_digit(&ret, hi_value % 10, i); + for (i = 1; i < 16; rem /= 10, i++) { + bcd_put_digit(&ret, rem % 10, i); } for (; i < 32; lo_value /= 10, i++) { diff --git a/util/host-utils.c b/util/host-utils.c index XXXXXXX..XXXXXXX 100644 --- a/util/host-utils.c +++ b/util/host-utils.c @@ -XXX,XX +XXX,XX @@ void muls64 (uint64_t *plow, uint64_t *phigh, int64_t a, int64_t b) } /* - * Unsigned 128-by-64 division. Returns quotient via plow and - * remainder via phigh. - * The result must fit in 64 bits (plow) - otherwise, the result - * is undefined. - * This function will cause a division by zero if passed a zero divisor. + * Unsigned 128-by-64 division. + * Returns the remainder. + * Returns quotient via plow and phigh. + * Also returns the remainder via the function return value. */ -void divu128(uint64_t *plow, uint64_t *phigh, uint64_t divisor) +uint64_t divu128(uint64_t *plow, uint64_t *phigh, uint64_t divisor) { uint64_t dhi = *phigh; uint64_t dlo = *plow; - unsigned i; - uint64_t carry = 0; + uint64_t rem, dhighest; + int sh; if (divisor == 0 || dhi == 0) { *plow = dlo / divisor; - *phigh = dlo % divisor; + *phigh = 0; + return dlo % divisor; } else { + sh = clz64(divisor); - for (i = 0; i < 64; i++) { - carry = dhi >> 63; - dhi = (dhi << 1) | (dlo >> 63); - if (carry || (dhi >= divisor)) { - dhi -= divisor; - carry = 1; - } else { - carry = 0; + if (dhi < divisor) { + if (sh != 0) { + /* normalize the divisor, shifting the dividend accordingly */ + divisor <<= sh; + dhi = (dhi << sh) | (dlo >> (64 - sh)); + dlo <<= sh; } - dlo = (dlo << 1) | carry; + + *phigh = 0; + *plow = udiv_qrnnd(&rem, dhi, dlo, divisor); + } else { + if (sh != 0) { + /* normalize the divisor, shifting the dividend accordingly */ + divisor <<= sh; + dhighest = dhi >> (64 - sh); + dhi = (dhi << sh) | (dlo >> (64 - sh)); + dlo <<= sh; + + *phigh = udiv_qrnnd(&dhi, dhighest, dhi, divisor); + } else { + /** + * dhi >= divisor + * Since the MSB of divisor is set (sh == 0), + * (dhi - divisor) < divisor + * + * Thus, the high part of the quotient is 1, and we can + * calculate the low part with a single call to udiv_qrnnd + * after subtracting divisor from dhi + */ + dhi -= divisor; + *phigh = 1; + } + + *plow = udiv_qrnnd(&rem, dhi, dlo, divisor); } - *plow = dlo; - *phigh = dhi; + /* + * since the dividend/divisor might have been normalized, + * the remainder might also have to be shifted back + */ + return rem >> sh; } } /* - * Signed 128-by-64 division. Returns quotient via plow and - * remainder via phigh. - * The result must fit in 64 bits (plow) - otherwise, the result - * is undefined. - * This function will cause a division by zero if passed a zero divisor. + * Signed 128-by-64 division. + * Returns quotient via plow and phigh. + * Also returns the remainder via the function return value. */ -void divs128(int64_t *plow, int64_t *phigh, int64_t divisor) +int64_t divs128(uint64_t *plow, int64_t *phigh, int64_t divisor) { - int sgn_dvdnd = *phigh < 0; - int sgn_divsr = divisor < 0; + bool neg_quotient = false, neg_remainder = false; + uint64_t unsig_hi = *phigh, unsig_lo = *plow; + uint64_t rem; - if (sgn_dvdnd) { - *plow = ~(*plow); - *phigh = ~(*phigh); - if (*plow == (int64_t)-1) { + if (*phigh < 0) { + neg_quotient = !neg_quotient; + neg_remainder = !neg_remainder; + + if (unsig_lo == 0) { + unsig_hi = -unsig_hi; + } else { + unsig_hi = ~unsig_hi; + unsig_lo = -unsig_lo; + } + } + + if (divisor < 0) { + neg_quotient = !neg_quotient; + + divisor = -divisor; + } + + rem = divu128(&unsig_lo, &unsig_hi, (uint64_t)divisor); + + if (neg_quotient) { + if (unsig_lo == 0) { + *phigh = -unsig_hi; *plow = 0; - (*phigh)++; - } else { - (*plow)++; - } + } else { + *phigh = ~unsig_hi; + *plow = -unsig_lo; + } + } else { + *phigh = unsig_hi; + *plow = unsig_lo; } - if (sgn_divsr) { - divisor = 0 - divisor; - } - - divu128((uint64_t *)plow, (uint64_t *)phigh, (uint64_t)divisor); - - if (sgn_dvdnd ^ sgn_divsr) { - *plow = 0 - *plow; + if (neg_remainder) { + return -rem; + } else { + return rem; } } #endif -- 2.25.1 From: Luis Pires <luis.pires@eldorado.org.br> Signed-off-by: Luis Pires <luis.pires@eldorado.org.br> Reviewed-by: Richard Henderson <richard.henderson@linaro.org> Message-Id: <20211025191154.350831-5-luis.pires@eldorado.org.br> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tests/unit/test-div128.c | 197 +++++++++++++++++++++++++++++++++++++++ tests/unit/meson.build | 1 + 2 files changed, 198 insertions(+) create mode 100644 tests/unit/test-div128.c diff --git a/tests/unit/test-div128.c b/tests/unit/test-div128.c new file mode 100644 index XXXXXXX..XXXXXXX --- /dev/null +++ b/tests/unit/test-div128.c @@ -XXX,XX +XXX,XX @@ +/* + * Test 128-bit division functions + * + * Copyright (c) 2021 Instituto de Pesquisas Eldorado (eldorado.org.br) + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, see <http://www.gnu.org/licenses/>. + */ + +#include "qemu/osdep.h" +#include "qemu/host-utils.h" + +typedef struct { + uint64_t high; + uint64_t low; + uint64_t rhigh; + uint64_t rlow; + uint64_t divisor; + uint64_t remainder; +} test_data_unsigned; + +typedef struct { + int64_t high; + uint64_t low; + int64_t rhigh; + uint64_t rlow; + int64_t divisor; + int64_t remainder; +} test_data_signed; + +static const test_data_unsigned test_table_unsigned[] = { + /* Dividend fits in 64 bits */ + { 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000000000000ULL, 0x0000000000000000ULL, + 0x0000000000000001ULL, 0x0000000000000000ULL}, + { 0x0000000000000000ULL, 0x0000000000000001ULL, + 0x0000000000000000ULL, 0x0000000000000001ULL, + 0x0000000000000001ULL, 0x0000000000000000ULL}, + { 0x0000000000000000ULL, 0x0000000000000003ULL, + 0x0000000000000000ULL, 0x0000000000000001ULL, + 0x0000000000000002ULL, 0x0000000000000001ULL}, + { 0x0000000000000000ULL, 0x8000000000000000ULL, + 0x0000000000000000ULL, 0x8000000000000000ULL, + 0x0000000000000001ULL, 0x0000000000000000ULL}, + { 0x0000000000000000ULL, 0xa000000000000000ULL, + 0x0000000000000000ULL, 0x0000000000000002ULL, + 0x4000000000000000ULL, 0x2000000000000000ULL}, + { 0x0000000000000000ULL, 0x8000000000000000ULL, + 0x0000000000000000ULL, 0x0000000000000001ULL, + 0x8000000000000000ULL, 0x0000000000000000ULL}, + + /* Dividend > 64 bits, with MSB 0 */ + { 0x123456789abcdefeULL, 0xefedcba987654321ULL, + 0x123456789abcdefeULL, 0xefedcba987654321ULL, + 0x0000000000000001ULL, 0x0000000000000000ULL}, + { 0x123456789abcdefeULL, 0xefedcba987654321ULL, + 0x0000000000000001ULL, 0x000000000000000dULL, + 0x123456789abcdefeULL, 0x03456789abcdf03bULL}, + { 0x123456789abcdefeULL, 0xefedcba987654321ULL, + 0x0123456789abcdefULL, 0xeefedcba98765432ULL, + 0x0000000000000010ULL, 0x0000000000000001ULL}, + + /* Dividend > 64 bits, with MSB 1 */ + { 0xfeeddccbbaa99887ULL, 0x766554433221100fULL, + 0xfeeddccbbaa99887ULL, 0x766554433221100fULL, + 0x0000000000000001ULL, 0x0000000000000000ULL}, + { 0xfeeddccbbaa99887ULL, 0x766554433221100fULL, + 0x0000000000000001ULL, 0x0000000000000000ULL, + 0xfeeddccbbaa99887ULL, 0x766554433221100fULL}, + { 0xfeeddccbbaa99887ULL, 0x766554433221100fULL, + 0x0feeddccbbaa9988ULL, 0x7766554433221100ULL, + 0x0000000000000010ULL, 0x000000000000000fULL}, + { 0xfeeddccbbaa99887ULL, 0x766554433221100fULL, + 0x000000000000000eULL, 0x00f0f0f0f0f0f35aULL, + 0x123456789abcdefeULL, 0x0f8922bc55ef90c3ULL}, + + /** + * Divisor == 64 bits, with MSB 1 + * and high 64 bits of dividend >= divisor + * (for testing normalization) + */ + { 0xfeeddccbbaa99887ULL, 0x766554433221100fULL, + 0x0000000000000001ULL, 0x0000000000000000ULL, + 0xfeeddccbbaa99887ULL, 0x766554433221100fULL}, + { 0xfeeddccbbaa99887ULL, 0x766554433221100fULL, + 0x0000000000000001ULL, 0xfddbb9977553310aULL, + 0x8000000000000001ULL, 0x78899aabbccddf05ULL}, + + /* Dividend > 64 bits, divisor almost as big */ + { 0x0000000000000001ULL, 0x23456789abcdef01ULL, + 0x0000000000000000ULL, 0x000000000000000fULL, + 0x123456789abcdefeULL, 0x123456789abcde1fULL}, +}; + +static const test_data_signed test_table_signed[] = { + /* Positive dividend, positive/negative divisors */ + { 0x0000000000000000LL, 0x0000000000bc614eULL, + 0x0000000000000000LL, 0x0000000000bc614eULL, + 0x0000000000000001LL, 0x0000000000000000LL}, + { 0x0000000000000000LL, 0x0000000000bc614eULL, + 0xffffffffffffffffLL, 0xffffffffff439eb2ULL, + 0xffffffffffffffffLL, 0x0000000000000000LL}, + { 0x0000000000000000LL, 0x0000000000bc614eULL, + 0x0000000000000000LL, 0x00000000005e30a7ULL, + 0x0000000000000002LL, 0x0000000000000000LL}, + { 0x0000000000000000LL, 0x0000000000bc614eULL, + 0xffffffffffffffffLL, 0xffffffffffa1cf59ULL, + 0xfffffffffffffffeLL, 0x0000000000000000LL}, + { 0x0000000000000000LL, 0x0000000000bc614eULL, + 0x0000000000000000LL, 0x0000000000178c29ULL, + 0x0000000000000008LL, 0x0000000000000006LL}, + { 0x0000000000000000LL, 0x0000000000bc614eULL, + 0xffffffffffffffffLL, 0xffffffffffe873d7ULL, + 0xfffffffffffffff8LL, 0x0000000000000006LL}, + { 0x0000000000000000LL, 0x0000000000bc614eULL, + 0x0000000000000000LL, 0x000000000000550dULL, + 0x0000000000000237LL, 0x0000000000000183LL}, + { 0x0000000000000000LL, 0x0000000000bc614eULL, + 0xffffffffffffffffLL, 0xffffffffffffaaf3ULL, + 0xfffffffffffffdc9LL, 0x0000000000000183LL}, + + /* Negative dividend, positive/negative divisors */ + { 0xffffffffffffffffLL, 0xffffffffff439eb2ULL, + 0xffffffffffffffffLL, 0xffffffffff439eb2ULL, + 0x0000000000000001LL, 0x0000000000000000LL}, + { 0xffffffffffffffffLL, 0xffffffffff439eb2ULL, + 0x0000000000000000LL, 0x0000000000bc614eULL, + 0xffffffffffffffffLL, 0x0000000000000000LL}, + { 0xffffffffffffffffLL, 0xffffffffff439eb2ULL, + 0xffffffffffffffffLL, 0xffffffffffa1cf59ULL, + 0x0000000000000002LL, 0x0000000000000000LL}, + { 0xffffffffffffffffLL, 0xffffffffff439eb2ULL, + 0x0000000000000000LL, 0x00000000005e30a7ULL, + 0xfffffffffffffffeLL, 0x0000000000000000LL}, + { 0xffffffffffffffffLL, 0xffffffffff439eb2ULL, + 0xffffffffffffffffLL, 0xffffffffffe873d7ULL, + 0x0000000000000008LL, 0xfffffffffffffffaLL}, + { 0xffffffffffffffffLL, 0xffffffffff439eb2ULL, + 0x0000000000000000LL, 0x0000000000178c29ULL, + 0xfffffffffffffff8LL, 0xfffffffffffffffaLL}, + { 0xffffffffffffffffLL, 0xffffffffff439eb2ULL, + 0xffffffffffffffffLL, 0xffffffffffffaaf3ULL, + 0x0000000000000237LL, 0xfffffffffffffe7dLL}, + { 0xffffffffffffffffLL, 0xffffffffff439eb2ULL, + 0x0000000000000000LL, 0x000000000000550dULL, + 0xfffffffffffffdc9LL, 0xfffffffffffffe7dLL}, +}; + +static void test_divu128(void) +{ + int i; + uint64_t rem; + test_data_unsigned tmp; + + for (i = 0; i < ARRAY_SIZE(test_table_unsigned); ++i) { + tmp = test_table_unsigned[i]; + + rem = divu128(&tmp.low, &tmp.high, tmp.divisor); + g_assert_cmpuint(tmp.low, ==, tmp.rlow); + g_assert_cmpuint(tmp.high, ==, tmp.rhigh); + g_assert_cmpuint(rem, ==, tmp.remainder); + } +} + +static void test_divs128(void) +{ + int i; + int64_t rem; + test_data_signed tmp; + + for (i = 0; i < ARRAY_SIZE(test_table_signed); ++i) { + tmp = test_table_signed[i]; + + rem = divs128(&tmp.low, &tmp.high, tmp.divisor); + g_assert_cmpuint(tmp.low, ==, tmp.rlow); + g_assert_cmpuint(tmp.high, ==, tmp.rhigh); + g_assert_cmpuint(rem, ==, tmp.remainder); + } +} + +int main(int argc, char **argv) +{ + g_test_init(&argc, &argv, NULL); + g_test_add_func("/host-utils/test_divu128", test_divu128); + g_test_add_func("/host-utils/test_divs128", test_divs128); + return g_test_run(); +} diff --git a/tests/unit/meson.build b/tests/unit/meson.build index XXXXXXX..XXXXXXX 100644 --- a/tests/unit/meson.build +++ b/tests/unit/meson.build @@ -XXX,XX +XXX,XX @@ tests = { # all code tested by test-x86-cpuid is inside topology.h 'test-x86-cpuid': [], 'test-cutils': [], + 'test-div128': [], 'test-shift128': [], 'test-mul64': [], # all code tested by test-int128 is inside int128.h -- 2.25.1 Prepare for tracking different masks by renaming this one. Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 142 +++++++++++++++++++++++++------------------------ 1 file changed, 72 insertions(+), 70 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ typedef struct TempOptInfo { TCGTemp *prev_copy; TCGTemp *next_copy; uint64_t val; - uint64_t mask; + uint64_t z_mask; /* mask bit is 0 if and only if value bit is 0 */ } TempOptInfo; static inline TempOptInfo *ts_info(TCGTemp *ts) @@ -XXX,XX +XXX,XX @@ static void reset_ts(TCGTemp *ts) ti->next_copy = ts; ti->prev_copy = ts; ti->is_const = false; - ti->mask = -1; + ti->z_mask = -1; } static void reset_temp(TCGArg arg) @@ -XXX,XX +XXX,XX @@ static void init_ts_info(TCGTempSet *temps_used, TCGTemp *ts) if (ts->kind == TEMP_CONST) { ti->is_const = true; ti->val = ts->val; - ti->mask = ts->val; + ti->z_mask = ts->val; if (TCG_TARGET_REG_BITS > 32 && ts->type == TCG_TYPE_I32) { /* High bits of a 32-bit quantity are garbage. */ - ti->mask |= ~0xffffffffull; + ti->z_mask |= ~0xffffffffull; } } else { ti->is_const = false; - ti->mask = -1; + ti->z_mask = -1; } } @@ -XXX,XX +XXX,XX @@ static void tcg_opt_gen_mov(TCGContext *s, TCGOp *op, TCGArg dst, TCGArg src) const TCGOpDef *def; TempOptInfo *di; TempOptInfo *si; - uint64_t mask; + uint64_t z_mask; TCGOpcode new_op; if (ts_are_copies(dst_ts, src_ts)) { @@ -XXX,XX +XXX,XX @@ static void tcg_opt_gen_mov(TCGContext *s, TCGOp *op, TCGArg dst, TCGArg src) op->args[0] = dst; op->args[1] = src; - mask = si->mask; + z_mask = si->z_mask; if (TCG_TARGET_REG_BITS > 32 && new_op == INDEX_op_mov_i32) { /* High bits of the destination are now garbage. */ - mask |= ~0xffffffffull; + z_mask |= ~0xffffffffull; } - di->mask = mask; + di->z_mask = z_mask; if (src_ts->type == dst_ts->type) { TempOptInfo *ni = ts_info(si->next_copy); @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) } QTAILQ_FOREACH_SAFE(op, &s->ops, link, op_next) { - uint64_t mask, partmask, affected, tmp; + uint64_t z_mask, partmask, affected, tmp; int nb_oargs, nb_iargs; TCGOpcode opc = op->opc; const TCGOpDef *def = &tcg_op_defs[opc]; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) /* Simplify using known-zero bits. Currently only ops with a single output argument is supported. */ - mask = -1; + z_mask = -1; affected = -1; switch (opc) { CASE_OP_32_64(ext8s): - if ((arg_info(op->args[1])->mask & 0x80) != 0) { + if ((arg_info(op->args[1])->z_mask & 0x80) != 0) { break; } QEMU_FALLTHROUGH; CASE_OP_32_64(ext8u): - mask = 0xff; + z_mask = 0xff; goto and_const; CASE_OP_32_64(ext16s): - if ((arg_info(op->args[1])->mask & 0x8000) != 0) { + if ((arg_info(op->args[1])->z_mask & 0x8000) != 0) { break; } QEMU_FALLTHROUGH; CASE_OP_32_64(ext16u): - mask = 0xffff; + z_mask = 0xffff; goto and_const; case INDEX_op_ext32s_i64: - if ((arg_info(op->args[1])->mask & 0x80000000) != 0) { + if ((arg_info(op->args[1])->z_mask & 0x80000000) != 0) { break; } QEMU_FALLTHROUGH; case INDEX_op_ext32u_i64: - mask = 0xffffffffU; + z_mask = 0xffffffffU; goto and_const; CASE_OP_32_64(and): - mask = arg_info(op->args[2])->mask; + z_mask = arg_info(op->args[2])->z_mask; if (arg_is_const(op->args[2])) { and_const: - affected = arg_info(op->args[1])->mask & ~mask; + affected = arg_info(op->args[1])->z_mask & ~z_mask; } - mask = arg_info(op->args[1])->mask & mask; + z_mask = arg_info(op->args[1])->z_mask & z_mask; break; case INDEX_op_ext_i32_i64: - if ((arg_info(op->args[1])->mask & 0x80000000) != 0) { + if ((arg_info(op->args[1])->z_mask & 0x80000000) != 0) { break; } QEMU_FALLTHROUGH; case INDEX_op_extu_i32_i64: /* We do not compute affected as it is a size changing op. */ - mask = (uint32_t)arg_info(op->args[1])->mask; + z_mask = (uint32_t)arg_info(op->args[1])->z_mask; break; CASE_OP_32_64(andc): /* Known-zeros does not imply known-ones. Therefore unless op->args[2] is constant, we can't infer anything from it. */ if (arg_is_const(op->args[2])) { - mask = ~arg_info(op->args[2])->mask; + z_mask = ~arg_info(op->args[2])->z_mask; goto and_const; } /* But we certainly know nothing outside args[1] may be set. */ - mask = arg_info(op->args[1])->mask; + z_mask = arg_info(op->args[1])->z_mask; break; case INDEX_op_sar_i32: if (arg_is_const(op->args[2])) { tmp = arg_info(op->args[2])->val & 31; - mask = (int32_t)arg_info(op->args[1])->mask >> tmp; + z_mask = (int32_t)arg_info(op->args[1])->z_mask >> tmp; } break; case INDEX_op_sar_i64: if (arg_is_const(op->args[2])) { tmp = arg_info(op->args[2])->val & 63; - mask = (int64_t)arg_info(op->args[1])->mask >> tmp; + z_mask = (int64_t)arg_info(op->args[1])->z_mask >> tmp; } break; case INDEX_op_shr_i32: if (arg_is_const(op->args[2])) { tmp = arg_info(op->args[2])->val & 31; - mask = (uint32_t)arg_info(op->args[1])->mask >> tmp; + z_mask = (uint32_t)arg_info(op->args[1])->z_mask >> tmp; } break; case INDEX_op_shr_i64: if (arg_is_const(op->args[2])) { tmp = arg_info(op->args[2])->val & 63; - mask = (uint64_t)arg_info(op->args[1])->mask >> tmp; + z_mask = (uint64_t)arg_info(op->args[1])->z_mask >> tmp; } break; case INDEX_op_extrl_i64_i32: - mask = (uint32_t)arg_info(op->args[1])->mask; + z_mask = (uint32_t)arg_info(op->args[1])->z_mask; break; case INDEX_op_extrh_i64_i32: - mask = (uint64_t)arg_info(op->args[1])->mask >> 32; + z_mask = (uint64_t)arg_info(op->args[1])->z_mask >> 32; break; CASE_OP_32_64(shl): if (arg_is_const(op->args[2])) { tmp = arg_info(op->args[2])->val & (TCG_TARGET_REG_BITS - 1); - mask = arg_info(op->args[1])->mask << tmp; + z_mask = arg_info(op->args[1])->z_mask << tmp; } break; CASE_OP_32_64(neg): /* Set to 1 all bits to the left of the rightmost. */ - mask = -(arg_info(op->args[1])->mask - & -arg_info(op->args[1])->mask); + z_mask = -(arg_info(op->args[1])->z_mask + & -arg_info(op->args[1])->z_mask); break; CASE_OP_32_64(deposit): - mask = deposit64(arg_info(op->args[1])->mask, - op->args[3], op->args[4], - arg_info(op->args[2])->mask); + z_mask = deposit64(arg_info(op->args[1])->z_mask, + op->args[3], op->args[4], + arg_info(op->args[2])->z_mask); break; CASE_OP_32_64(extract): - mask = extract64(arg_info(op->args[1])->mask, - op->args[2], op->args[3]); + z_mask = extract64(arg_info(op->args[1])->z_mask, + op->args[2], op->args[3]); if (op->args[2] == 0) { - affected = arg_info(op->args[1])->mask & ~mask; + affected = arg_info(op->args[1])->z_mask & ~z_mask; } break; CASE_OP_32_64(sextract): - mask = sextract64(arg_info(op->args[1])->mask, - op->args[2], op->args[3]); - if (op->args[2] == 0 && (tcg_target_long)mask >= 0) { - affected = arg_info(op->args[1])->mask & ~mask; + z_mask = sextract64(arg_info(op->args[1])->z_mask, + op->args[2], op->args[3]); + if (op->args[2] == 0 && (tcg_target_long)z_mask >= 0) { + affected = arg_info(op->args[1])->z_mask & ~z_mask; } break; CASE_OP_32_64(or): CASE_OP_32_64(xor): - mask = arg_info(op->args[1])->mask | arg_info(op->args[2])->mask; + z_mask = arg_info(op->args[1])->z_mask + | arg_info(op->args[2])->z_mask; break; case INDEX_op_clz_i32: case INDEX_op_ctz_i32: - mask = arg_info(op->args[2])->mask | 31; + z_mask = arg_info(op->args[2])->z_mask | 31; break; case INDEX_op_clz_i64: case INDEX_op_ctz_i64: - mask = arg_info(op->args[2])->mask | 63; + z_mask = arg_info(op->args[2])->z_mask | 63; break; case INDEX_op_ctpop_i32: - mask = 32 | 31; + z_mask = 32 | 31; break; case INDEX_op_ctpop_i64: - mask = 64 | 63; + z_mask = 64 | 63; break; CASE_OP_32_64(setcond): case INDEX_op_setcond2_i32: - mask = 1; + z_mask = 1; break; CASE_OP_32_64(movcond): - mask = arg_info(op->args[3])->mask | arg_info(op->args[4])->mask; + z_mask = arg_info(op->args[3])->z_mask + | arg_info(op->args[4])->z_mask; break; CASE_OP_32_64(ld8u): - mask = 0xff; + z_mask = 0xff; break; CASE_OP_32_64(ld16u): - mask = 0xffff; + z_mask = 0xffff; break; case INDEX_op_ld32u_i64: - mask = 0xffffffffu; + z_mask = 0xffffffffu; break; CASE_OP_32_64(qemu_ld): @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) MemOpIdx oi = op->args[nb_oargs + nb_iargs]; MemOp mop = get_memop(oi); if (!(mop & MO_SIGN)) { - mask = (2ULL << ((8 << (mop & MO_SIZE)) - 1)) - 1; + z_mask = (2ULL << ((8 << (mop & MO_SIZE)) - 1)) - 1; } } break; CASE_OP_32_64(bswap16): - mask = arg_info(op->args[1])->mask; - if (mask <= 0xffff) { + z_mask = arg_info(op->args[1])->z_mask; + if (z_mask <= 0xffff) { op->args[2] |= TCG_BSWAP_IZ; } - mask = bswap16(mask); + z_mask = bswap16(z_mask); switch (op->args[2] & (TCG_BSWAP_OZ | TCG_BSWAP_OS)) { case TCG_BSWAP_OZ: break; case TCG_BSWAP_OS: - mask = (int16_t)mask; + z_mask = (int16_t)z_mask; break; default: /* undefined high bits */ - mask |= MAKE_64BIT_MASK(16, 48); + z_mask |= MAKE_64BIT_MASK(16, 48); break; } break; case INDEX_op_bswap32_i64: - mask = arg_info(op->args[1])->mask; - if (mask <= 0xffffffffu) { + z_mask = arg_info(op->args[1])->z_mask; + if (z_mask <= 0xffffffffu) { op->args[2] |= TCG_BSWAP_IZ; } - mask = bswap32(mask); + z_mask = bswap32(z_mask); switch (op->args[2] & (TCG_BSWAP_OZ | TCG_BSWAP_OS)) { case TCG_BSWAP_OZ: break; case TCG_BSWAP_OS: - mask = (int32_t)mask; + z_mask = (int32_t)z_mask; break; default: /* undefined high bits */ - mask |= MAKE_64BIT_MASK(32, 32); + z_mask |= MAKE_64BIT_MASK(32, 32); break; } break; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) /* 32-bit ops generate 32-bit results. For the result is zero test below, we can ignore high bits, but for further optimizations we need to record that the high bits contain garbage. */ - partmask = mask; + partmask = z_mask; if (!(def->flags & TCG_OPF_64BIT)) { - mask |= ~(tcg_target_ulong)0xffffffffu; + z_mask |= ~(tcg_target_ulong)0xffffffffu; partmask &= 0xffffffffu; affected &= 0xffffffffu; } @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) vs the high word of the input. */ do_setcond_high: reset_temp(op->args[0]); - arg_info(op->args[0])->mask = 1; + arg_info(op->args[0])->z_mask = 1; op->opc = INDEX_op_setcond_i32; op->args[1] = op->args[2]; op->args[2] = op->args[4]; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) } do_setcond_low: reset_temp(op->args[0]); - arg_info(op->args[0])->mask = 1; + arg_info(op->args[0])->z_mask = 1; op->opc = INDEX_op_setcond_i32; op->args[2] = op->args[3]; op->args[3] = op->args[5]; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) /* Default case: we know nothing about operation (or were unable to compute the operation result) so no propagation is done. We trash everything if the operation is the end of a basic - block, otherwise we only trash the output args. "mask" is + block, otherwise we only trash the output args. "z_mask" is the non-zero bits mask for the first output arg. */ if (def->flags & TCG_OPF_BB_END) { memset(&temps_used, 0, sizeof(temps_used)); @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) /* Save the corresponding known-zero bits mask for the first output argument (only one supported so far). */ if (i == 0) { - arg_info(op->args[i])->mask = mask; + arg_info(op->args[i])->z_mask = z_mask; } } } -- 2.25.1 Provide what will become a larger context for splitting the very large tcg_optimize function. Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 77 ++++++++++++++++++++++++++------------------------ 1 file changed, 40 insertions(+), 37 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ typedef struct TempOptInfo { uint64_t z_mask; /* mask bit is 0 if and only if value bit is 0 */ } TempOptInfo; +typedef struct OptContext { + TCGTempSet temps_used; +} OptContext; + static inline TempOptInfo *ts_info(TCGTemp *ts) { return ts->state_ptr; @@ -XXX,XX +XXX,XX @@ static void reset_temp(TCGArg arg) } /* Initialize and activate a temporary. */ -static void init_ts_info(TCGTempSet *temps_used, TCGTemp *ts) +static void init_ts_info(OptContext *ctx, TCGTemp *ts) { size_t idx = temp_idx(ts); TempOptInfo *ti; - if (test_bit(idx, temps_used->l)) { + if (test_bit(idx, ctx->temps_used.l)) { return; } - set_bit(idx, temps_used->l); + set_bit(idx, ctx->temps_used.l); ti = ts->state_ptr; if (ti == NULL) { @@ -XXX,XX +XXX,XX @@ static void init_ts_info(TCGTempSet *temps_used, TCGTemp *ts) } } -static void init_arg_info(TCGTempSet *temps_used, TCGArg arg) +static void init_arg_info(OptContext *ctx, TCGArg arg) { - init_ts_info(temps_used, arg_temp(arg)); + init_ts_info(ctx, arg_temp(arg)); } static TCGTemp *find_better_copy(TCGContext *s, TCGTemp *ts) @@ -XXX,XX +XXX,XX @@ static void tcg_opt_gen_mov(TCGContext *s, TCGOp *op, TCGArg dst, TCGArg src) } } -static void tcg_opt_gen_movi(TCGContext *s, TCGTempSet *temps_used, +static void tcg_opt_gen_movi(TCGContext *s, OptContext *ctx, TCGOp *op, TCGArg dst, uint64_t val) { const TCGOpDef *def = &tcg_op_defs[op->opc]; @@ -XXX,XX +XXX,XX @@ static void tcg_opt_gen_movi(TCGContext *s, TCGTempSet *temps_used, /* Convert movi to mov with constant temp. */ tv = tcg_constant_internal(type, val); - init_ts_info(temps_used, tv); + init_ts_info(ctx, tv); tcg_opt_gen_mov(s, op, dst, temp_arg(tv)); } @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) { int nb_temps, nb_globals, i; TCGOp *op, *op_next, *prev_mb = NULL; - TCGTempSet temps_used; + OptContext ctx = {}; /* Array VALS has an element for each temp. If this temp holds a constant then its value is kept in VALS' element. @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) nb_temps = s->nb_temps; nb_globals = s->nb_globals; - memset(&temps_used, 0, sizeof(temps_used)); for (i = 0; i < nb_temps; ++i) { s->temps[i].state_ptr = NULL; } @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) for (i = 0; i < nb_oargs + nb_iargs; i++) { TCGTemp *ts = arg_temp(op->args[i]); if (ts) { - init_ts_info(&temps_used, ts); + init_ts_info(&ctx, ts); } } } else { nb_oargs = def->nb_oargs; nb_iargs = def->nb_iargs; for (i = 0; i < nb_oargs + nb_iargs; i++) { - init_arg_info(&temps_used, op->args[i]); + init_arg_info(&ctx, op->args[i]); } } @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) CASE_OP_32_64(rotr): if (arg_is_const(op->args[1]) && arg_info(op->args[1])->val == 0) { - tcg_opt_gen_movi(s, &temps_used, op, op->args[0], 0); + tcg_opt_gen_movi(s, &ctx, op, op->args[0], 0); continue; } break; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) if (partmask == 0) { tcg_debug_assert(nb_oargs == 1); - tcg_opt_gen_movi(s, &temps_used, op, op->args[0], 0); + tcg_opt_gen_movi(s, &ctx, op, op->args[0], 0); continue; } if (affected == 0) { @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) CASE_OP_32_64(mulsh): if (arg_is_const(op->args[2]) && arg_info(op->args[2])->val == 0) { - tcg_opt_gen_movi(s, &temps_used, op, op->args[0], 0); + tcg_opt_gen_movi(s, &ctx, op, op->args[0], 0); continue; } break; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) CASE_OP_32_64_VEC(sub): CASE_OP_32_64_VEC(xor): if (args_are_copies(op->args[1], op->args[2])) { - tcg_opt_gen_movi(s, &temps_used, op, op->args[0], 0); + tcg_opt_gen_movi(s, &ctx, op, op->args[0], 0); continue; } break; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) if (arg_is_const(op->args[1])) { tmp = arg_info(op->args[1])->val; tmp = dup_const(TCGOP_VECE(op), tmp); - tcg_opt_gen_movi(s, &temps_used, op, op->args[0], tmp); + tcg_opt_gen_movi(s, &ctx, op, op->args[0], tmp); break; } goto do_default; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) case INDEX_op_dup2_vec: assert(TCG_TARGET_REG_BITS == 32); if (arg_is_const(op->args[1]) && arg_is_const(op->args[2])) { - tcg_opt_gen_movi(s, &temps_used, op, op->args[0], + tcg_opt_gen_movi(s, &ctx, op, op->args[0], deposit64(arg_info(op->args[1])->val, 32, 32, arg_info(op->args[2])->val)); break; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) case INDEX_op_extrh_i64_i32: if (arg_is_const(op->args[1])) { tmp = do_constant_folding(opc, arg_info(op->args[1])->val, 0); - tcg_opt_gen_movi(s, &temps_used, op, op->args[0], tmp); + tcg_opt_gen_movi(s, &ctx, op, op->args[0], tmp); break; } goto do_default; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) if (arg_is_const(op->args[1])) { tmp = do_constant_folding(opc, arg_info(op->args[1])->val, op->args[2]); - tcg_opt_gen_movi(s, &temps_used, op, op->args[0], tmp); + tcg_opt_gen_movi(s, &ctx, op, op->args[0], tmp); break; } goto do_default; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) if (arg_is_const(op->args[1]) && arg_is_const(op->args[2])) { tmp = do_constant_folding(opc, arg_info(op->args[1])->val, arg_info(op->args[2])->val); - tcg_opt_gen_movi(s, &temps_used, op, op->args[0], tmp); + tcg_opt_gen_movi(s, &ctx, op, op->args[0], tmp); break; } goto do_default; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) TCGArg v = arg_info(op->args[1])->val; if (v != 0) { tmp = do_constant_folding(opc, v, 0); - tcg_opt_gen_movi(s, &temps_used, op, op->args[0], tmp); + tcg_opt_gen_movi(s, &ctx, op, op->args[0], tmp); } else { tcg_opt_gen_mov(s, op, op->args[0], op->args[2]); } @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) tmp = deposit64(arg_info(op->args[1])->val, op->args[3], op->args[4], arg_info(op->args[2])->val); - tcg_opt_gen_movi(s, &temps_used, op, op->args[0], tmp); + tcg_opt_gen_movi(s, &ctx, op, op->args[0], tmp); break; } goto do_default; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) if (arg_is_const(op->args[1])) { tmp = extract64(arg_info(op->args[1])->val, op->args[2], op->args[3]); - tcg_opt_gen_movi(s, &temps_used, op, op->args[0], tmp); + tcg_opt_gen_movi(s, &ctx, op, op->args[0], tmp); break; } goto do_default; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) if (arg_is_const(op->args[1])) { tmp = sextract64(arg_info(op->args[1])->val, op->args[2], op->args[3]); - tcg_opt_gen_movi(s, &temps_used, op, op->args[0], tmp); + tcg_opt_gen_movi(s, &ctx, op, op->args[0], tmp); break; } goto do_default; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) tmp = (int32_t)(((uint32_t)v1 >> shr) | ((uint32_t)v2 << (32 - shr))); } - tcg_opt_gen_movi(s, &temps_used, op, op->args[0], tmp); + tcg_opt_gen_movi(s, &ctx, op, op->args[0], tmp); break; } goto do_default; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) tmp = do_constant_folding_cond(opc, op->args[1], op->args[2], op->args[3]); if (tmp != 2) { - tcg_opt_gen_movi(s, &temps_used, op, op->args[0], tmp); + tcg_opt_gen_movi(s, &ctx, op, op->args[0], tmp); break; } goto do_default; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) op->args[1], op->args[2]); if (tmp != 2) { if (tmp) { - memset(&temps_used, 0, sizeof(temps_used)); + memset(&ctx.temps_used, 0, sizeof(ctx.temps_used)); op->opc = INDEX_op_br; op->args[0] = op->args[3]; } else { @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) rl = op->args[0]; rh = op->args[1]; - tcg_opt_gen_movi(s, &temps_used, op, rl, (int32_t)a); - tcg_opt_gen_movi(s, &temps_used, op2, rh, (int32_t)(a >> 32)); + tcg_opt_gen_movi(s, &ctx, op, rl, (int32_t)a); + tcg_opt_gen_movi(s, &ctx, op2, rh, (int32_t)(a >> 32)); break; } goto do_default; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) rl = op->args[0]; rh = op->args[1]; - tcg_opt_gen_movi(s, &temps_used, op, rl, (int32_t)r); - tcg_opt_gen_movi(s, &temps_used, op2, rh, (int32_t)(r >> 32)); + tcg_opt_gen_movi(s, &ctx, op, rl, (int32_t)r); + tcg_opt_gen_movi(s, &ctx, op2, rh, (int32_t)(r >> 32)); break; } goto do_default; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) if (tmp != 2) { if (tmp) { do_brcond_true: - memset(&temps_used, 0, sizeof(temps_used)); + memset(&ctx.temps_used, 0, sizeof(ctx.temps_used)); op->opc = INDEX_op_br; op->args[0] = op->args[5]; } else { @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) /* Simplify LT/GE comparisons vs zero to a single compare vs the high word of the input. */ do_brcond_high: - memset(&temps_used, 0, sizeof(temps_used)); + memset(&ctx.temps_used, 0, sizeof(ctx.temps_used)); op->opc = INDEX_op_brcond_i32; op->args[0] = op->args[1]; op->args[1] = op->args[3]; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) goto do_default; } do_brcond_low: - memset(&temps_used, 0, sizeof(temps_used)); + memset(&ctx.temps_used, 0, sizeof(ctx.temps_used)); op->opc = INDEX_op_brcond_i32; op->args[1] = op->args[2]; op->args[2] = op->args[4]; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) op->args[5]); if (tmp != 2) { do_setcond_const: - tcg_opt_gen_movi(s, &temps_used, op, op->args[0], tmp); + tcg_opt_gen_movi(s, &ctx, op, op->args[0], tmp); } else if ((op->args[5] == TCG_COND_LT || op->args[5] == TCG_COND_GE) && arg_is_const(op->args[3]) @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) if (!(tcg_call_flags(op) & (TCG_CALL_NO_READ_GLOBALS | TCG_CALL_NO_WRITE_GLOBALS))) { for (i = 0; i < nb_globals; i++) { - if (test_bit(i, temps_used.l)) { + if (test_bit(i, ctx.temps_used.l)) { reset_ts(&s->temps[i]); } } @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) block, otherwise we only trash the output args. "z_mask" is the non-zero bits mask for the first output arg. */ if (def->flags & TCG_OPF_BB_END) { - memset(&temps_used, 0, sizeof(temps_used)); + memset(&ctx.temps_used, 0, sizeof(ctx.temps_used)); } else { do_reset_output: for (i = 0; i < nb_oargs; i++) { -- 2.25.1 Break the final cleanup clause out of the main switch statement. When fully folding an opcode to mov/movi, use "continue" to process the next opcode, else break to fall into the final cleanup. Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 190 ++++++++++++++++++++++++------------------------- 1 file changed, 94 insertions(+), 96 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) switch (opc) { CASE_OP_32_64_VEC(mov): tcg_opt_gen_mov(s, op, op->args[0], op->args[1]); - break; + continue; case INDEX_op_dup_vec: if (arg_is_const(op->args[1])) { tmp = arg_info(op->args[1])->val; tmp = dup_const(TCGOP_VECE(op), tmp); tcg_opt_gen_movi(s, &ctx, op, op->args[0], tmp); - break; + continue; } - goto do_default; + break; case INDEX_op_dup2_vec: assert(TCG_TARGET_REG_BITS == 32); @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) tcg_opt_gen_movi(s, &ctx, op, op->args[0], deposit64(arg_info(op->args[1])->val, 32, 32, arg_info(op->args[2])->val)); - break; + continue; } else if (args_are_copies(op->args[1], op->args[2])) { op->opc = INDEX_op_dup_vec; TCGOP_VECE(op) = MO_32; nb_iargs = 1; } - goto do_default; + break; CASE_OP_32_64(not): CASE_OP_32_64(neg): @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) if (arg_is_const(op->args[1])) { tmp = do_constant_folding(opc, arg_info(op->args[1])->val, 0); tcg_opt_gen_movi(s, &ctx, op, op->args[0], tmp); - break; + continue; } - goto do_default; + break; CASE_OP_32_64(bswap16): CASE_OP_32_64(bswap32): @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) tmp = do_constant_folding(opc, arg_info(op->args[1])->val, op->args[2]); tcg_opt_gen_movi(s, &ctx, op, op->args[0], tmp); - break; + continue; } - goto do_default; + break; CASE_OP_32_64(add): CASE_OP_32_64(sub): @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) tmp = do_constant_folding(opc, arg_info(op->args[1])->val, arg_info(op->args[2])->val); tcg_opt_gen_movi(s, &ctx, op, op->args[0], tmp); - break; + continue; } - goto do_default; + break; CASE_OP_32_64(clz): CASE_OP_32_64(ctz): @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) } else { tcg_opt_gen_mov(s, op, op->args[0], op->args[2]); } - break; + continue; } - goto do_default; + break; CASE_OP_32_64(deposit): if (arg_is_const(op->args[1]) && arg_is_const(op->args[2])) { @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) op->args[3], op->args[4], arg_info(op->args[2])->val); tcg_opt_gen_movi(s, &ctx, op, op->args[0], tmp); - break; + continue; } - goto do_default; + break; CASE_OP_32_64(extract): if (arg_is_const(op->args[1])) { tmp = extract64(arg_info(op->args[1])->val, op->args[2], op->args[3]); tcg_opt_gen_movi(s, &ctx, op, op->args[0], tmp); - break; + continue; } - goto do_default; + break; CASE_OP_32_64(sextract): if (arg_is_const(op->args[1])) { tmp = sextract64(arg_info(op->args[1])->val, op->args[2], op->args[3]); tcg_opt_gen_movi(s, &ctx, op, op->args[0], tmp); - break; + continue; } - goto do_default; + break; CASE_OP_32_64(extract2): if (arg_is_const(op->args[1]) && arg_is_const(op->args[2])) { @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) ((uint32_t)v2 << (32 - shr))); } tcg_opt_gen_movi(s, &ctx, op, op->args[0], tmp); - break; + continue; } - goto do_default; + break; CASE_OP_32_64(setcond): tmp = do_constant_folding_cond(opc, op->args[1], op->args[2], op->args[3]); if (tmp != 2) { tcg_opt_gen_movi(s, &ctx, op, op->args[0], tmp); - break; + continue; } - goto do_default; + break; CASE_OP_32_64(brcond): tmp = do_constant_folding_cond(opc, op->args[0], op->args[1], op->args[2]); - if (tmp != 2) { - if (tmp) { - memset(&ctx.temps_used, 0, sizeof(ctx.temps_used)); - op->opc = INDEX_op_br; - op->args[0] = op->args[3]; - } else { - tcg_op_remove(s, op); - } + switch (tmp) { + case 0: + tcg_op_remove(s, op); + continue; + case 1: + memset(&ctx.temps_used, 0, sizeof(ctx.temps_used)); + op->opc = opc = INDEX_op_br; + op->args[0] = op->args[3]; break; } - goto do_default; + break; CASE_OP_32_64(movcond): tmp = do_constant_folding_cond(opc, op->args[1], op->args[2], op->args[5]); if (tmp != 2) { tcg_opt_gen_mov(s, op, op->args[0], op->args[4-tmp]); - break; + continue; } if (arg_is_const(op->args[3]) && arg_is_const(op->args[4])) { uint64_t tv = arg_info(op->args[3])->val; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) if (fv == 1 && tv == 0) { cond = tcg_invert_cond(cond); } else if (!(tv == 1 && fv == 0)) { - goto do_default; + break; } op->args[3] = cond; op->opc = opc = (opc == INDEX_op_movcond_i32 @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) : INDEX_op_setcond_i64); nb_iargs = 2; } - goto do_default; + break; case INDEX_op_add2_i32: case INDEX_op_sub2_i32: @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) rh = op->args[1]; tcg_opt_gen_movi(s, &ctx, op, rl, (int32_t)a); tcg_opt_gen_movi(s, &ctx, op2, rh, (int32_t)(a >> 32)); - break; + continue; } - goto do_default; + break; case INDEX_op_mulu2_i32: if (arg_is_const(op->args[2]) && arg_is_const(op->args[3])) { @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) rh = op->args[1]; tcg_opt_gen_movi(s, &ctx, op, rl, (int32_t)r); tcg_opt_gen_movi(s, &ctx, op2, rh, (int32_t)(r >> 32)); - break; + continue; } - goto do_default; + break; case INDEX_op_brcond2_i32: tmp = do_constant_folding_cond2(&op->args[0], &op->args[2], op->args[4]); - if (tmp != 2) { - if (tmp) { - do_brcond_true: - memset(&ctx.temps_used, 0, sizeof(ctx.temps_used)); - op->opc = INDEX_op_br; - op->args[0] = op->args[5]; - } else { + if (tmp == 0) { do_brcond_false: - tcg_op_remove(s, op); - } - } else if ((op->args[4] == TCG_COND_LT - || op->args[4] == TCG_COND_GE) - && arg_is_const(op->args[2]) - && arg_info(op->args[2])->val == 0 - && arg_is_const(op->args[3]) - && arg_info(op->args[3])->val == 0) { + tcg_op_remove(s, op); + continue; + } + if (tmp == 1) { + do_brcond_true: + op->opc = opc = INDEX_op_br; + op->args[0] = op->args[5]; + break; + } + if ((op->args[4] == TCG_COND_LT || op->args[4] == TCG_COND_GE) + && arg_is_const(op->args[2]) + && arg_info(op->args[2])->val == 0 + && arg_is_const(op->args[3]) + && arg_info(op->args[3])->val == 0) { /* Simplify LT/GE comparisons vs zero to a single compare vs the high word of the input. */ do_brcond_high: - memset(&ctx.temps_used, 0, sizeof(ctx.temps_used)); - op->opc = INDEX_op_brcond_i32; + op->opc = opc = INDEX_op_brcond_i32; op->args[0] = op->args[1]; op->args[1] = op->args[3]; op->args[2] = op->args[4]; op->args[3] = op->args[5]; - } else if (op->args[4] == TCG_COND_EQ) { + break; + } + if (op->args[4] == TCG_COND_EQ) { /* Simplify EQ comparisons where one of the pairs can be simplified. */ tmp = do_constant_folding_cond(INDEX_op_brcond_i32, @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) if (tmp == 0) { goto do_brcond_false; } else if (tmp != 1) { - goto do_default; + break; } do_brcond_low: memset(&ctx.temps_used, 0, sizeof(ctx.temps_used)); @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) op->args[1] = op->args[2]; op->args[2] = op->args[4]; op->args[3] = op->args[5]; - } else if (op->args[4] == TCG_COND_NE) { + break; + } + if (op->args[4] == TCG_COND_NE) { /* Simplify NE comparisons where one of the pairs can be simplified. */ tmp = do_constant_folding_cond(INDEX_op_brcond_i32, @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) } else if (tmp == 1) { goto do_brcond_true; } - goto do_default; - } else { - goto do_default; } break; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) if (tmp != 2) { do_setcond_const: tcg_opt_gen_movi(s, &ctx, op, op->args[0], tmp); - } else if ((op->args[5] == TCG_COND_LT - || op->args[5] == TCG_COND_GE) - && arg_is_const(op->args[3]) - && arg_info(op->args[3])->val == 0 - && arg_is_const(op->args[4]) - && arg_info(op->args[4])->val == 0) { + continue; + } + if ((op->args[5] == TCG_COND_LT || op->args[5] == TCG_COND_GE) + && arg_is_const(op->args[3]) + && arg_info(op->args[3])->val == 0 + && arg_is_const(op->args[4]) + && arg_info(op->args[4])->val == 0) { /* Simplify LT/GE comparisons vs zero to a single compare vs the high word of the input. */ do_setcond_high: @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) op->args[1] = op->args[2]; op->args[2] = op->args[4]; op->args[3] = op->args[5]; - } else if (op->args[5] == TCG_COND_EQ) { + break; + } + if (op->args[5] == TCG_COND_EQ) { /* Simplify EQ comparisons where one of the pairs can be simplified. */ tmp = do_constant_folding_cond(INDEX_op_setcond_i32, @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) if (tmp == 0) { goto do_setcond_high; } else if (tmp != 1) { - goto do_default; + break; } do_setcond_low: reset_temp(op->args[0]); @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) op->opc = INDEX_op_setcond_i32; op->args[2] = op->args[3]; op->args[3] = op->args[5]; - } else if (op->args[5] == TCG_COND_NE) { + break; + } + if (op->args[5] == TCG_COND_NE) { /* Simplify NE comparisons where one of the pairs can be simplified. */ tmp = do_constant_folding_cond(INDEX_op_setcond_i32, @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) } else if (tmp == 1) { goto do_setcond_const; } - goto do_default; - } else { - goto do_default; } break; - case INDEX_op_call: - if (!(tcg_call_flags(op) + default: + break; + } + + /* Some of the folding above can change opc. */ + opc = op->opc; + def = &tcg_op_defs[opc]; + if (def->flags & TCG_OPF_BB_END) { + memset(&ctx.temps_used, 0, sizeof(ctx.temps_used)); + } else { + if (opc == INDEX_op_call && + !(tcg_call_flags(op) & (TCG_CALL_NO_READ_GLOBALS | TCG_CALL_NO_WRITE_GLOBALS))) { for (i = 0; i < nb_globals; i++) { if (test_bit(i, ctx.temps_used.l)) { @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) } } } - goto do_reset_output; - default: - do_default: - /* Default case: we know nothing about operation (or were unable - to compute the operation result) so no propagation is done. - We trash everything if the operation is the end of a basic - block, otherwise we only trash the output args. "z_mask" is - the non-zero bits mask for the first output arg. */ - if (def->flags & TCG_OPF_BB_END) { - memset(&ctx.temps_used, 0, sizeof(ctx.temps_used)); - } else { - do_reset_output: - for (i = 0; i < nb_oargs; i++) { - reset_temp(op->args[i]); - /* Save the corresponding known-zero bits mask for the - first output argument (only one supported so far). */ - if (i == 0) { - arg_info(op->args[i])->z_mask = z_mask; - } + for (i = 0; i < nb_oargs; i++) { + reset_temp(op->args[i]); + /* Save the corresponding known-zero bits mask for the + first output argument (only one supported so far). */ + if (i == 0) { + arg_info(op->args[i])->z_mask = z_mask; } } - break; } /* Eliminate duplicate and redundant fence instructions. */ -- 2.25.1 Adjust the interface to take the OptContext parameter instead of TCGContext or both. Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 67 +++++++++++++++++++++++++------------------------- 1 file changed, 34 insertions(+), 33 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ typedef struct TempOptInfo { } TempOptInfo; typedef struct OptContext { + TCGContext *tcg; TCGTempSet temps_used; } OptContext; @@ -XXX,XX +XXX,XX @@ static bool args_are_copies(TCGArg arg1, TCGArg arg2) return ts_are_copies(arg_temp(arg1), arg_temp(arg2)); } -static void tcg_opt_gen_mov(TCGContext *s, TCGOp *op, TCGArg dst, TCGArg src) +static void tcg_opt_gen_mov(OptContext *ctx, TCGOp *op, TCGArg dst, TCGArg src) { TCGTemp *dst_ts = arg_temp(dst); TCGTemp *src_ts = arg_temp(src); @@ -XXX,XX +XXX,XX @@ static void tcg_opt_gen_mov(TCGContext *s, TCGOp *op, TCGArg dst, TCGArg src) TCGOpcode new_op; if (ts_are_copies(dst_ts, src_ts)) { - tcg_op_remove(s, op); + tcg_op_remove(ctx->tcg, op); return; } @@ -XXX,XX +XXX,XX @@ static void tcg_opt_gen_mov(TCGContext *s, TCGOp *op, TCGArg dst, TCGArg src) } } -static void tcg_opt_gen_movi(TCGContext *s, OptContext *ctx, - TCGOp *op, TCGArg dst, uint64_t val) +static void tcg_opt_gen_movi(OptContext *ctx, TCGOp *op, + TCGArg dst, uint64_t val) { const TCGOpDef *def = &tcg_op_defs[op->opc]; TCGType type; @@ -XXX,XX +XXX,XX @@ static void tcg_opt_gen_movi(TCGContext *s, OptContext *ctx, /* Convert movi to mov with constant temp. */ tv = tcg_constant_internal(type, val); init_ts_info(ctx, tv); - tcg_opt_gen_mov(s, op, dst, temp_arg(tv)); + tcg_opt_gen_mov(ctx, op, dst, temp_arg(tv)); } static uint64_t do_constant_folding_2(TCGOpcode op, uint64_t x, uint64_t y) @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) { int nb_temps, nb_globals, i; TCGOp *op, *op_next, *prev_mb = NULL; - OptContext ctx = {}; + OptContext ctx = { .tcg = s }; /* Array VALS has an element for each temp. If this temp holds a constant then its value is kept in VALS' element. @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) CASE_OP_32_64(rotr): if (arg_is_const(op->args[1]) && arg_info(op->args[1])->val == 0) { - tcg_opt_gen_movi(s, &ctx, op, op->args[0], 0); + tcg_opt_gen_movi(&ctx, op, op->args[0], 0); continue; } break; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) if (!arg_is_const(op->args[1]) && arg_is_const(op->args[2]) && arg_info(op->args[2])->val == 0) { - tcg_opt_gen_mov(s, op, op->args[0], op->args[1]); + tcg_opt_gen_mov(&ctx, op, op->args[0], op->args[1]); continue; } break; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) if (!arg_is_const(op->args[1]) && arg_is_const(op->args[2]) && arg_info(op->args[2])->val == -1) { - tcg_opt_gen_mov(s, op, op->args[0], op->args[1]); + tcg_opt_gen_mov(&ctx, op, op->args[0], op->args[1]); continue; } break; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) if (partmask == 0) { tcg_debug_assert(nb_oargs == 1); - tcg_opt_gen_movi(s, &ctx, op, op->args[0], 0); + tcg_opt_gen_movi(&ctx, op, op->args[0], 0); continue; } if (affected == 0) { tcg_debug_assert(nb_oargs == 1); - tcg_opt_gen_mov(s, op, op->args[0], op->args[1]); + tcg_opt_gen_mov(&ctx, op, op->args[0], op->args[1]); continue; } @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) CASE_OP_32_64(mulsh): if (arg_is_const(op->args[2]) && arg_info(op->args[2])->val == 0) { - tcg_opt_gen_movi(s, &ctx, op, op->args[0], 0); + tcg_opt_gen_movi(&ctx, op, op->args[0], 0); continue; } break; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) CASE_OP_32_64_VEC(or): CASE_OP_32_64_VEC(and): if (args_are_copies(op->args[1], op->args[2])) { - tcg_opt_gen_mov(s, op, op->args[0], op->args[1]); + tcg_opt_gen_mov(&ctx, op, op->args[0], op->args[1]); continue; } break; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) CASE_OP_32_64_VEC(sub): CASE_OP_32_64_VEC(xor): if (args_are_copies(op->args[1], op->args[2])) { - tcg_opt_gen_movi(s, &ctx, op, op->args[0], 0); + tcg_opt_gen_movi(&ctx, op, op->args[0], 0); continue; } break; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) allocator where needed and possible. Also detect copies. */ switch (opc) { CASE_OP_32_64_VEC(mov): - tcg_opt_gen_mov(s, op, op->args[0], op->args[1]); + tcg_opt_gen_mov(&ctx, op, op->args[0], op->args[1]); continue; case INDEX_op_dup_vec: if (arg_is_const(op->args[1])) { tmp = arg_info(op->args[1])->val; tmp = dup_const(TCGOP_VECE(op), tmp); - tcg_opt_gen_movi(s, &ctx, op, op->args[0], tmp); + tcg_opt_gen_movi(&ctx, op, op->args[0], tmp); continue; } break; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) case INDEX_op_dup2_vec: assert(TCG_TARGET_REG_BITS == 32); if (arg_is_const(op->args[1]) && arg_is_const(op->args[2])) { - tcg_opt_gen_movi(s, &ctx, op, op->args[0], + tcg_opt_gen_movi(&ctx, op, op->args[0], deposit64(arg_info(op->args[1])->val, 32, 32, arg_info(op->args[2])->val)); continue; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) case INDEX_op_extrh_i64_i32: if (arg_is_const(op->args[1])) { tmp = do_constant_folding(opc, arg_info(op->args[1])->val, 0); - tcg_opt_gen_movi(s, &ctx, op, op->args[0], tmp); + tcg_opt_gen_movi(&ctx, op, op->args[0], tmp); continue; } break; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) if (arg_is_const(op->args[1])) { tmp = do_constant_folding(opc, arg_info(op->args[1])->val, op->args[2]); - tcg_opt_gen_movi(s, &ctx, op, op->args[0], tmp); + tcg_opt_gen_movi(&ctx, op, op->args[0], tmp); continue; } break; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) if (arg_is_const(op->args[1]) && arg_is_const(op->args[2])) { tmp = do_constant_folding(opc, arg_info(op->args[1])->val, arg_info(op->args[2])->val); - tcg_opt_gen_movi(s, &ctx, op, op->args[0], tmp); + tcg_opt_gen_movi(&ctx, op, op->args[0], tmp); continue; } break; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) TCGArg v = arg_info(op->args[1])->val; if (v != 0) { tmp = do_constant_folding(opc, v, 0); - tcg_opt_gen_movi(s, &ctx, op, op->args[0], tmp); + tcg_opt_gen_movi(&ctx, op, op->args[0], tmp); } else { - tcg_opt_gen_mov(s, op, op->args[0], op->args[2]); + tcg_opt_gen_mov(&ctx, op, op->args[0], op->args[2]); } continue; } @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) tmp = deposit64(arg_info(op->args[1])->val, op->args[3], op->args[4], arg_info(op->args[2])->val); - tcg_opt_gen_movi(s, &ctx, op, op->args[0], tmp); + tcg_opt_gen_movi(&ctx, op, op->args[0], tmp); continue; } break; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) if (arg_is_const(op->args[1])) { tmp = extract64(arg_info(op->args[1])->val, op->args[2], op->args[3]); - tcg_opt_gen_movi(s, &ctx, op, op->args[0], tmp); + tcg_opt_gen_movi(&ctx, op, op->args[0], tmp); continue; } break; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) if (arg_is_const(op->args[1])) { tmp = sextract64(arg_info(op->args[1])->val, op->args[2], op->args[3]); - tcg_opt_gen_movi(s, &ctx, op, op->args[0], tmp); + tcg_opt_gen_movi(&ctx, op, op->args[0], tmp); continue; } break; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) tmp = (int32_t)(((uint32_t)v1 >> shr) | ((uint32_t)v2 << (32 - shr))); } - tcg_opt_gen_movi(s, &ctx, op, op->args[0], tmp); + tcg_opt_gen_movi(&ctx, op, op->args[0], tmp); continue; } break; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) tmp = do_constant_folding_cond(opc, op->args[1], op->args[2], op->args[3]); if (tmp != 2) { - tcg_opt_gen_movi(s, &ctx, op, op->args[0], tmp); + tcg_opt_gen_movi(&ctx, op, op->args[0], tmp); continue; } break; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) tmp = do_constant_folding_cond(opc, op->args[1], op->args[2], op->args[5]); if (tmp != 2) { - tcg_opt_gen_mov(s, op, op->args[0], op->args[4-tmp]); + tcg_opt_gen_mov(&ctx, op, op->args[0], op->args[4-tmp]); continue; } if (arg_is_const(op->args[3]) && arg_is_const(op->args[4])) { @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) rl = op->args[0]; rh = op->args[1]; - tcg_opt_gen_movi(s, &ctx, op, rl, (int32_t)a); - tcg_opt_gen_movi(s, &ctx, op2, rh, (int32_t)(a >> 32)); + tcg_opt_gen_movi(&ctx, op, rl, (int32_t)a); + tcg_opt_gen_movi(&ctx, op2, rh, (int32_t)(a >> 32)); continue; } break; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) rl = op->args[0]; rh = op->args[1]; - tcg_opt_gen_movi(s, &ctx, op, rl, (int32_t)r); - tcg_opt_gen_movi(s, &ctx, op2, rh, (int32_t)(r >> 32)); + tcg_opt_gen_movi(&ctx, op, rl, (int32_t)r); + tcg_opt_gen_movi(&ctx, op2, rh, (int32_t)(r >> 32)); continue; } break; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) op->args[5]); if (tmp != 2) { do_setcond_const: - tcg_opt_gen_movi(s, &ctx, op, op->args[0], tmp); + tcg_opt_gen_movi(&ctx, op, op->args[0], tmp); continue; } if ((op->args[5] == TCG_COND_LT || op->args[5] == TCG_COND_GE) -- 2.25.1 This will expose the variable to subroutines that will be broken out of tcg_optimize. Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ typedef struct TempOptInfo { typedef struct OptContext { TCGContext *tcg; + TCGOp *prev_mb; TCGTempSet temps_used; } OptContext; @@ -XXX,XX +XXX,XX @@ static bool swap_commutative2(TCGArg *p1, TCGArg *p2) void tcg_optimize(TCGContext *s) { int nb_temps, nb_globals, i; - TCGOp *op, *op_next, *prev_mb = NULL; + TCGOp *op, *op_next; OptContext ctx = { .tcg = s }; /* Array VALS has an element for each temp. @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) } /* Eliminate duplicate and redundant fence instructions. */ - if (prev_mb) { + if (ctx.prev_mb) { switch (opc) { case INDEX_op_mb: /* Merge two barriers of the same type into one, @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) * barrier. This is stricter than specified but for * the purposes of TCG is better than not optimizing. */ - prev_mb->args[0] |= op->args[0]; + ctx.prev_mb->args[0] |= op->args[0]; tcg_op_remove(s, op); break; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) case INDEX_op_qemu_st_i64: case INDEX_op_call: /* Opcodes that touch guest memory stop the optimization. */ - prev_mb = NULL; + ctx.prev_mb = NULL; break; } } else if (opc == INDEX_op_mb) { - prev_mb = op; + ctx.prev_mb = op; } } } -- 2.25.1 There was no real reason for calls to have separate code here. Unify init for calls vs non-calls using the call path, which handles TCG_CALL_DUMMY_ARG. Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static void init_ts_info(OptContext *ctx, TCGTemp *ts) } } -static void init_arg_info(OptContext *ctx, TCGArg arg) -{ - init_ts_info(ctx, arg_temp(arg)); -} - static TCGTemp *find_better_copy(TCGContext *s, TCGTemp *ts) { TCGTemp *i, *g, *l; @@ -XXX,XX +XXX,XX @@ static bool swap_commutative2(TCGArg *p1, TCGArg *p2) return false; } +static void init_arguments(OptContext *ctx, TCGOp *op, int nb_args) +{ + for (int i = 0; i < nb_args; i++) { + TCGTemp *ts = arg_temp(op->args[i]); + if (ts) { + init_ts_info(ctx, ts); + } + } +} + /* Propagate constants and copies, fold constant expressions. */ void tcg_optimize(TCGContext *s) { @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) if (opc == INDEX_op_call) { nb_oargs = TCGOP_CALLO(op); nb_iargs = TCGOP_CALLI(op); - for (i = 0; i < nb_oargs + nb_iargs; i++) { - TCGTemp *ts = arg_temp(op->args[i]); - if (ts) { - init_ts_info(&ctx, ts); - } - } } else { nb_oargs = def->nb_oargs; nb_iargs = def->nb_iargs; - for (i = 0; i < nb_oargs + nb_iargs; i++) { - init_arg_info(&ctx, op->args[i]); - } } + init_arguments(&ctx, op, nb_oargs + nb_iargs); /* Do copy propagation */ for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) { -- 2.25.1 Continue splitting tcg_optimize. Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static void init_arguments(OptContext *ctx, TCGOp *op, int nb_args) } } +static void copy_propagate(OptContext *ctx, TCGOp *op, + int nb_oargs, int nb_iargs) +{ + TCGContext *s = ctx->tcg; + + for (int i = nb_oargs; i < nb_oargs + nb_iargs; i++) { + TCGTemp *ts = arg_temp(op->args[i]); + if (ts && ts_is_copy(ts)) { + op->args[i] = temp_arg(find_better_copy(s, ts)); + } + } +} + /* Propagate constants and copies, fold constant expressions. */ void tcg_optimize(TCGContext *s) { @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) nb_iargs = def->nb_iargs; } init_arguments(&ctx, op, nb_oargs + nb_iargs); - - /* Do copy propagation */ - for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) { - TCGTemp *ts = arg_temp(op->args[i]); - if (ts && ts_is_copy(ts)) { - op->args[i] = temp_arg(find_better_copy(s, ts)); - } - } + copy_propagate(&ctx, op, nb_oargs, nb_iargs); /* For commutative operations make constant second argument */ switch (opc) { -- 2.25.1 Calls are special in that they have a variable number of arguments, and need to be able to clobber globals. Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 63 ++++++++++++++++++++++++++++++++------------------ 1 file changed, 41 insertions(+), 22 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static void copy_propagate(OptContext *ctx, TCGOp *op, } } +static bool fold_call(OptContext *ctx, TCGOp *op) +{ + TCGContext *s = ctx->tcg; + int nb_oargs = TCGOP_CALLO(op); + int nb_iargs = TCGOP_CALLI(op); + int flags, i; + + init_arguments(ctx, op, nb_oargs + nb_iargs); + copy_propagate(ctx, op, nb_oargs, nb_iargs); + + /* If the function reads or writes globals, reset temp data. */ + flags = tcg_call_flags(op); + if (!(flags & (TCG_CALL_NO_READ_GLOBALS | TCG_CALL_NO_WRITE_GLOBALS))) { + int nb_globals = s->nb_globals; + + for (i = 0; i < nb_globals; i++) { + if (test_bit(i, ctx->temps_used.l)) { + reset_ts(&ctx->tcg->temps[i]); + } + } + } + + /* Reset temp data for outputs. */ + for (i = 0; i < nb_oargs; i++) { + reset_temp(op->args[i]); + } + + /* Stop optimizing MB across calls. */ + ctx->prev_mb = NULL; + return true; +} + /* Propagate constants and copies, fold constant expressions. */ void tcg_optimize(TCGContext *s) { - int nb_temps, nb_globals, i; + int nb_temps, i; TCGOp *op, *op_next; OptContext ctx = { .tcg = s }; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) available through the doubly linked circular list. */ nb_temps = s->nb_temps; - nb_globals = s->nb_globals; - for (i = 0; i < nb_temps; ++i) { s->temps[i].state_ptr = NULL; } @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) uint64_t z_mask, partmask, affected, tmp; int nb_oargs, nb_iargs; TCGOpcode opc = op->opc; - const TCGOpDef *def = &tcg_op_defs[opc]; + const TCGOpDef *def; - /* Count the arguments, and initialize the temps that are - going to be used */ + /* Calls are special. */ if (opc == INDEX_op_call) { - nb_oargs = TCGOP_CALLO(op); - nb_iargs = TCGOP_CALLI(op); - } else { - nb_oargs = def->nb_oargs; - nb_iargs = def->nb_iargs; + fold_call(&ctx, op); + continue; } + + def = &tcg_op_defs[opc]; + nb_oargs = def->nb_oargs; + nb_iargs = def->nb_iargs; init_arguments(&ctx, op, nb_oargs + nb_iargs); copy_propagate(&ctx, op, nb_oargs, nb_iargs); @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) if (def->flags & TCG_OPF_BB_END) { memset(&ctx.temps_used, 0, sizeof(ctx.temps_used)); } else { - if (opc == INDEX_op_call && - !(tcg_call_flags(op) - & (TCG_CALL_NO_READ_GLOBALS | TCG_CALL_NO_WRITE_GLOBALS))) { - for (i = 0; i < nb_globals; i++) { - if (test_bit(i, ctx.temps_used.l)) { - reset_ts(&s->temps[i]); - } - } - } - for (i = 0; i < nb_oargs; i++) { reset_temp(op->args[i]); /* Save the corresponding known-zero bits mask for the @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) case INDEX_op_qemu_st_i32: case INDEX_op_qemu_st8_i32: case INDEX_op_qemu_st_i64: - case INDEX_op_call: /* Opcodes that touch guest memory stop the optimization. */ ctx.prev_mb = NULL; break; -- 2.25.1 Rather than try to keep these up-to-date across folding, re-read nb_oargs at the end, after re-reading the opcode. A couple of asserts need dropping, but that will take care of itself as we split the function further. Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) QTAILQ_FOREACH_SAFE(op, &s->ops, link, op_next) { uint64_t z_mask, partmask, affected, tmp; - int nb_oargs, nb_iargs; TCGOpcode opc = op->opc; const TCGOpDef *def; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) } def = &tcg_op_defs[opc]; - nb_oargs = def->nb_oargs; - nb_iargs = def->nb_iargs; - init_arguments(&ctx, op, nb_oargs + nb_iargs); - copy_propagate(&ctx, op, nb_oargs, nb_iargs); + init_arguments(&ctx, op, def->nb_oargs + def->nb_iargs); + copy_propagate(&ctx, op, def->nb_oargs, def->nb_iargs); /* For commutative operations make constant second argument */ switch (opc) { @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) CASE_OP_32_64(qemu_ld): { - MemOpIdx oi = op->args[nb_oargs + nb_iargs]; + MemOpIdx oi = op->args[def->nb_oargs + def->nb_iargs]; MemOp mop = get_memop(oi); if (!(mop & MO_SIGN)) { z_mask = (2ULL << ((8 << (mop & MO_SIZE)) - 1)) - 1; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) } if (partmask == 0) { - tcg_debug_assert(nb_oargs == 1); tcg_opt_gen_movi(&ctx, op, op->args[0], 0); continue; } if (affected == 0) { - tcg_debug_assert(nb_oargs == 1); tcg_opt_gen_mov(&ctx, op, op->args[0], op->args[1]); continue; } @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) } else if (args_are_copies(op->args[1], op->args[2])) { op->opc = INDEX_op_dup_vec; TCGOP_VECE(op) = MO_32; - nb_iargs = 1; } break; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) op->opc = opc = (opc == INDEX_op_movcond_i32 ? INDEX_op_setcond_i32 : INDEX_op_setcond_i64); - nb_iargs = 2; } break; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) if (def->flags & TCG_OPF_BB_END) { memset(&ctx.temps_used, 0, sizeof(ctx.temps_used)); } else { + int nb_oargs = def->nb_oargs; for (i = 0; i < nb_oargs; i++) { reset_temp(op->args[i]); /* Save the corresponding known-zero bits mask for the -- 2.25.1 Return -1 instead of 2 for failure, so that we can use comparisons against 0 for all cases. Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 145 +++++++++++++++++++++++++------------------------ 1 file changed, 74 insertions(+), 71 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool do_constant_folding_cond_eq(TCGCond c) } } -/* Return 2 if the condition can't be simplified, and the result - of the condition (0 or 1) if it can */ -static TCGArg do_constant_folding_cond(TCGOpcode op, TCGArg x, - TCGArg y, TCGCond c) +/* + * Return -1 if the condition can't be simplified, + * and the result of the condition (0 or 1) if it can. + */ +static int do_constant_folding_cond(TCGOpcode op, TCGArg x, + TCGArg y, TCGCond c) { uint64_t xv = arg_info(x)->val; uint64_t yv = arg_info(y)->val; @@ -XXX,XX +XXX,XX @@ static TCGArg do_constant_folding_cond(TCGOpcode op, TCGArg x, case TCG_COND_GEU: return 1; default: - return 2; + return -1; } } - return 2; + return -1; } -/* Return 2 if the condition can't be simplified, and the result - of the condition (0 or 1) if it can */ -static TCGArg do_constant_folding_cond2(TCGArg *p1, TCGArg *p2, TCGCond c) +/* + * Return -1 if the condition can't be simplified, + * and the result of the condition (0 or 1) if it can. + */ +static int do_constant_folding_cond2(TCGArg *p1, TCGArg *p2, TCGCond c) { TCGArg al = p1[0], ah = p1[1]; TCGArg bl = p2[0], bh = p2[1]; @@ -XXX,XX +XXX,XX @@ static TCGArg do_constant_folding_cond2(TCGArg *p1, TCGArg *p2, TCGCond c) if (args_are_copies(al, bl) && args_are_copies(ah, bh)) { return do_constant_folding_cond_eq(c); } - return 2; + return -1; } static bool swap_commutative(TCGArg dest, TCGArg *p1, TCGArg *p2) @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) break; CASE_OP_32_64(setcond): - tmp = do_constant_folding_cond(opc, op->args[1], - op->args[2], op->args[3]); - if (tmp != 2) { - tcg_opt_gen_movi(&ctx, op, op->args[0], tmp); + i = do_constant_folding_cond(opc, op->args[1], + op->args[2], op->args[3]); + if (i >= 0) { + tcg_opt_gen_movi(&ctx, op, op->args[0], i); continue; } break; CASE_OP_32_64(brcond): - tmp = do_constant_folding_cond(opc, op->args[0], - op->args[1], op->args[2]); - switch (tmp) { - case 0: + i = do_constant_folding_cond(opc, op->args[0], + op->args[1], op->args[2]); + if (i == 0) { tcg_op_remove(s, op); continue; - case 1: + } else if (i > 0) { memset(&ctx.temps_used, 0, sizeof(ctx.temps_used)); op->opc = opc = INDEX_op_br; op->args[0] = op->args[3]; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) break; CASE_OP_32_64(movcond): - tmp = do_constant_folding_cond(opc, op->args[1], - op->args[2], op->args[5]); - if (tmp != 2) { - tcg_opt_gen_mov(&ctx, op, op->args[0], op->args[4-tmp]); + i = do_constant_folding_cond(opc, op->args[1], + op->args[2], op->args[5]); + if (i >= 0) { + tcg_opt_gen_mov(&ctx, op, op->args[0], op->args[4 - i]); continue; } if (arg_is_const(op->args[3]) && arg_is_const(op->args[4])) { @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) break; case INDEX_op_brcond2_i32: - tmp = do_constant_folding_cond2(&op->args[0], &op->args[2], - op->args[4]); - if (tmp == 0) { + i = do_constant_folding_cond2(&op->args[0], &op->args[2], + op->args[4]); + if (i == 0) { do_brcond_false: tcg_op_remove(s, op); continue; } - if (tmp == 1) { + if (i > 0) { do_brcond_true: op->opc = opc = INDEX_op_br; op->args[0] = op->args[5]; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) if (op->args[4] == TCG_COND_EQ) { /* Simplify EQ comparisons where one of the pairs can be simplified. */ - tmp = do_constant_folding_cond(INDEX_op_brcond_i32, - op->args[0], op->args[2], - TCG_COND_EQ); - if (tmp == 0) { + i = do_constant_folding_cond(INDEX_op_brcond_i32, + op->args[0], op->args[2], + TCG_COND_EQ); + if (i == 0) { goto do_brcond_false; - } else if (tmp == 1) { + } else if (i > 0) { goto do_brcond_high; } - tmp = do_constant_folding_cond(INDEX_op_brcond_i32, - op->args[1], op->args[3], - TCG_COND_EQ); - if (tmp == 0) { + i = do_constant_folding_cond(INDEX_op_brcond_i32, + op->args[1], op->args[3], + TCG_COND_EQ); + if (i == 0) { goto do_brcond_false; - } else if (tmp != 1) { + } else if (i < 0) { break; } do_brcond_low: @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) if (op->args[4] == TCG_COND_NE) { /* Simplify NE comparisons where one of the pairs can be simplified. */ - tmp = do_constant_folding_cond(INDEX_op_brcond_i32, - op->args[0], op->args[2], - TCG_COND_NE); - if (tmp == 0) { + i = do_constant_folding_cond(INDEX_op_brcond_i32, + op->args[0], op->args[2], + TCG_COND_NE); + if (i == 0) { goto do_brcond_high; - } else if (tmp == 1) { + } else if (i > 0) { goto do_brcond_true; } - tmp = do_constant_folding_cond(INDEX_op_brcond_i32, - op->args[1], op->args[3], - TCG_COND_NE); - if (tmp == 0) { + i = do_constant_folding_cond(INDEX_op_brcond_i32, + op->args[1], op->args[3], + TCG_COND_NE); + if (i == 0) { goto do_brcond_low; - } else if (tmp == 1) { + } else if (i > 0) { goto do_brcond_true; } } break; case INDEX_op_setcond2_i32: - tmp = do_constant_folding_cond2(&op->args[1], &op->args[3], - op->args[5]); - if (tmp != 2) { + i = do_constant_folding_cond2(&op->args[1], &op->args[3], + op->args[5]); + if (i >= 0) { do_setcond_const: - tcg_opt_gen_movi(&ctx, op, op->args[0], tmp); + tcg_opt_gen_movi(&ctx, op, op->args[0], i); continue; } if ((op->args[5] == TCG_COND_LT || op->args[5] == TCG_COND_GE) @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) if (op->args[5] == TCG_COND_EQ) { /* Simplify EQ comparisons where one of the pairs can be simplified. */ - tmp = do_constant_folding_cond(INDEX_op_setcond_i32, - op->args[1], op->args[3], - TCG_COND_EQ); - if (tmp == 0) { + i = do_constant_folding_cond(INDEX_op_setcond_i32, + op->args[1], op->args[3], + TCG_COND_EQ); + if (i == 0) { goto do_setcond_const; - } else if (tmp == 1) { + } else if (i > 0) { goto do_setcond_high; } - tmp = do_constant_folding_cond(INDEX_op_setcond_i32, - op->args[2], op->args[4], - TCG_COND_EQ); - if (tmp == 0) { + i = do_constant_folding_cond(INDEX_op_setcond_i32, + op->args[2], op->args[4], + TCG_COND_EQ); + if (i == 0) { goto do_setcond_high; - } else if (tmp != 1) { + } else if (i < 0) { break; } do_setcond_low: @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) if (op->args[5] == TCG_COND_NE) { /* Simplify NE comparisons where one of the pairs can be simplified. */ - tmp = do_constant_folding_cond(INDEX_op_setcond_i32, - op->args[1], op->args[3], - TCG_COND_NE); - if (tmp == 0) { + i = do_constant_folding_cond(INDEX_op_setcond_i32, + op->args[1], op->args[3], + TCG_COND_NE); + if (i == 0) { goto do_setcond_high; - } else if (tmp == 1) { + } else if (i > 0) { goto do_setcond_const; } - tmp = do_constant_folding_cond(INDEX_op_setcond_i32, - op->args[2], op->args[4], - TCG_COND_NE); - if (tmp == 0) { + i = do_constant_folding_cond(INDEX_op_setcond_i32, + op->args[2], op->args[4], + TCG_COND_NE); + if (i == 0) { goto do_setcond_low; - } else if (tmp == 1) { + } else if (i > 0) { goto do_setcond_const; } } -- 2.25.1 This will allow callers to tail call to these functions and return true indicating processing complete. Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool args_are_copies(TCGArg arg1, TCGArg arg2) return ts_are_copies(arg_temp(arg1), arg_temp(arg2)); } -static void tcg_opt_gen_mov(OptContext *ctx, TCGOp *op, TCGArg dst, TCGArg src) +static bool tcg_opt_gen_mov(OptContext *ctx, TCGOp *op, TCGArg dst, TCGArg src) { TCGTemp *dst_ts = arg_temp(dst); TCGTemp *src_ts = arg_temp(src); @@ -XXX,XX +XXX,XX @@ static void tcg_opt_gen_mov(OptContext *ctx, TCGOp *op, TCGArg dst, TCGArg src) if (ts_are_copies(dst_ts, src_ts)) { tcg_op_remove(ctx->tcg, op); - return; + return true; } reset_ts(dst_ts); @@ -XXX,XX +XXX,XX @@ static void tcg_opt_gen_mov(OptContext *ctx, TCGOp *op, TCGArg dst, TCGArg src) di->is_const = si->is_const; di->val = si->val; } + return true; } -static void tcg_opt_gen_movi(OptContext *ctx, TCGOp *op, +static bool tcg_opt_gen_movi(OptContext *ctx, TCGOp *op, TCGArg dst, uint64_t val) { const TCGOpDef *def = &tcg_op_defs[op->opc]; @@ -XXX,XX +XXX,XX @@ static void tcg_opt_gen_movi(OptContext *ctx, TCGOp *op, /* Convert movi to mov with constant temp. */ tv = tcg_constant_internal(type, val); init_ts_info(ctx, tv); - tcg_opt_gen_mov(ctx, op, dst, temp_arg(tv)); + return tcg_opt_gen_mov(ctx, op, dst, temp_arg(tv)); } static uint64_t do_constant_folding_2(TCGOpcode op, uint64_t x, uint64_t y) -- 2.25.1 Copy z_mask into OptContext, for writeback to the first output within the new function. Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 49 +++++++++++++++++++++++++++++++++---------------- 1 file changed, 33 insertions(+), 16 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ typedef struct OptContext { TCGContext *tcg; TCGOp *prev_mb; TCGTempSet temps_used; + + /* In flight values from optimization. */ + uint64_t z_mask; } OptContext; static inline TempOptInfo *ts_info(TCGTemp *ts) @@ -XXX,XX +XXX,XX @@ static void copy_propagate(OptContext *ctx, TCGOp *op, } } +static void finish_folding(OptContext *ctx, TCGOp *op) +{ + const TCGOpDef *def = &tcg_op_defs[op->opc]; + int i, nb_oargs; + + /* + * For an opcode that ends a BB, reset all temp data. + * We do no cross-BB optimization. + */ + if (def->flags & TCG_OPF_BB_END) { + memset(&ctx->temps_used, 0, sizeof(ctx->temps_used)); + ctx->prev_mb = NULL; + return; + } + + nb_oargs = def->nb_oargs; + for (i = 0; i < nb_oargs; i++) { + reset_temp(op->args[i]); + /* + * Save the corresponding known-zero bits mask for the + * first output argument (only one supported so far). + */ + if (i == 0) { + arg_info(op->args[i])->z_mask = ctx->z_mask; + } + } +} + static bool fold_call(OptContext *ctx, TCGOp *op) { TCGContext *s = ctx->tcg; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) partmask &= 0xffffffffu; affected &= 0xffffffffu; } + ctx.z_mask = z_mask; if (partmask == 0) { tcg_opt_gen_movi(&ctx, op, op->args[0], 0); @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) break; } - /* Some of the folding above can change opc. */ - opc = op->opc; - def = &tcg_op_defs[opc]; - if (def->flags & TCG_OPF_BB_END) { - memset(&ctx.temps_used, 0, sizeof(ctx.temps_used)); - } else { - int nb_oargs = def->nb_oargs; - for (i = 0; i < nb_oargs; i++) { - reset_temp(op->args[i]); - /* Save the corresponding known-zero bits mask for the - first output argument (only one supported so far). */ - if (i == 0) { - arg_info(op->args[i])->z_mask = z_mask; - } - } - } + finish_folding(&ctx, op); /* Eliminate duplicate and redundant fence instructions. */ if (ctx.prev_mb) { -- 2.25.1 Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) uint64_t z_mask, partmask, affected, tmp; TCGOpcode opc = op->opc; const TCGOpDef *def; + bool done = false; /* Calls are special. */ if (opc == INDEX_op_call) { @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) allocator where needed and possible. Also detect copies. */ switch (opc) { CASE_OP_32_64_VEC(mov): - tcg_opt_gen_mov(&ctx, op, op->args[0], op->args[1]); - continue; + done = tcg_opt_gen_mov(&ctx, op, op->args[0], op->args[1]); + break; case INDEX_op_dup_vec: if (arg_is_const(op->args[1])) { @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) break; } - finish_folding(&ctx, op); + if (!done) { + finish_folding(&ctx, op); + } /* Eliminate duplicate and redundant fence instructions. */ if (ctx.prev_mb) { -- 2.25.1 This puts the separate mb optimization into the same framework as the others. While fold_qemu_{ld,st} are currently identical, that won't last as more code gets moved. Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 89 +++++++++++++++++++++++++++++--------------------- 1 file changed, 51 insertions(+), 38 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_call(OptContext *ctx, TCGOp *op) return true; } +static bool fold_mb(OptContext *ctx, TCGOp *op) +{ + /* Eliminate duplicate and redundant fence instructions. */ + if (ctx->prev_mb) { + /* + * Merge two barriers of the same type into one, + * or a weaker barrier into a stronger one, + * or two weaker barriers into a stronger one. + * mb X; mb Y => mb X|Y + * mb; strl => mb; st + * ldaq; mb => ld; mb + * ldaq; strl => ld; mb; st + * Other combinations are also merged into a strong + * barrier. This is stricter than specified but for + * the purposes of TCG is better than not optimizing. + */ + ctx->prev_mb->args[0] |= op->args[0]; + tcg_op_remove(ctx->tcg, op); + } else { + ctx->prev_mb = op; + } + return true; +} + +static bool fold_qemu_ld(OptContext *ctx, TCGOp *op) +{ + /* Opcodes that touch guest memory stop the mb optimization. */ + ctx->prev_mb = NULL; + return false; +} + +static bool fold_qemu_st(OptContext *ctx, TCGOp *op) +{ + /* Opcodes that touch guest memory stop the mb optimization. */ + ctx->prev_mb = NULL; + return false; +} + /* Propagate constants and copies, fold constant expressions. */ void tcg_optimize(TCGContext *s) { @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) } break; + case INDEX_op_mb: + done = fold_mb(&ctx, op); + break; + case INDEX_op_qemu_ld_i32: + case INDEX_op_qemu_ld_i64: + done = fold_qemu_ld(&ctx, op); + break; + case INDEX_op_qemu_st_i32: + case INDEX_op_qemu_st8_i32: + case INDEX_op_qemu_st_i64: + done = fold_qemu_st(&ctx, op); + break; + default: break; } @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) if (!done) { finish_folding(&ctx, op); } - - /* Eliminate duplicate and redundant fence instructions. */ - if (ctx.prev_mb) { - switch (opc) { - case INDEX_op_mb: - /* Merge two barriers of the same type into one, - * or a weaker barrier into a stronger one, - * or two weaker barriers into a stronger one. - * mb X; mb Y => mb X|Y - * mb; strl => mb; st - * ldaq; mb => ld; mb - * ldaq; strl => ld; mb; st - * Other combinations are also merged into a strong - * barrier. This is stricter than specified but for - * the purposes of TCG is better than not optimizing. - */ - ctx.prev_mb->args[0] |= op->args[0]; - tcg_op_remove(s, op); - break; - - default: - /* Opcodes that end the block stop the optimization. */ - if ((def->flags & TCG_OPF_BB_END) == 0) { - break; - } - /* fallthru */ - case INDEX_op_qemu_ld_i32: - case INDEX_op_qemu_ld_i64: - case INDEX_op_qemu_st_i32: - case INDEX_op_qemu_st8_i32: - case INDEX_op_qemu_st_i64: - /* Opcodes that touch guest memory stop the optimization. */ - ctx.prev_mb = NULL; - break; - } - } else if (opc == INDEX_op_mb) { - ctx.prev_mb = op; - } } } -- 2.25.1 Split out a whole bunch of placeholder functions, which are currently identical. That won't last as more code gets moved. Use CASE_32_64_VEC for some logical operators that previously missed the addition of vectors. Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 271 +++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 219 insertions(+), 52 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static void finish_folding(OptContext *ctx, TCGOp *op) } } +/* + * The fold_* functions return true when processing is complete, + * usually by folding the operation to a constant or to a copy, + * and calling tcg_opt_gen_{mov,movi}. They may do other things, + * like collect information about the value produced, for use in + * optimizing a subsequent operation. + * + * These first fold_* functions are all helpers, used by other + * folders for more specific operations. + */ + +static bool fold_const1(OptContext *ctx, TCGOp *op) +{ + if (arg_is_const(op->args[1])) { + uint64_t t; + + t = arg_info(op->args[1])->val; + t = do_constant_folding(op->opc, t, 0); + return tcg_opt_gen_movi(ctx, op, op->args[0], t); + } + return false; +} + +static bool fold_const2(OptContext *ctx, TCGOp *op) +{ + if (arg_is_const(op->args[1]) && arg_is_const(op->args[2])) { + uint64_t t1 = arg_info(op->args[1])->val; + uint64_t t2 = arg_info(op->args[2])->val; + + t1 = do_constant_folding(op->opc, t1, t2); + return tcg_opt_gen_movi(ctx, op, op->args[0], t1); + } + return false; +} + +/* + * These outermost fold_<op> functions are sorted alphabetically. + */ + +static bool fold_add(OptContext *ctx, TCGOp *op) +{ + return fold_const2(ctx, op); +} + +static bool fold_and(OptContext *ctx, TCGOp *op) +{ + return fold_const2(ctx, op); +} + +static bool fold_andc(OptContext *ctx, TCGOp *op) +{ + return fold_const2(ctx, op); +} + static bool fold_call(OptContext *ctx, TCGOp *op) { TCGContext *s = ctx->tcg; @@ -XXX,XX +XXX,XX @@ static bool fold_call(OptContext *ctx, TCGOp *op) return true; } +static bool fold_ctpop(OptContext *ctx, TCGOp *op) +{ + return fold_const1(ctx, op); +} + +static bool fold_divide(OptContext *ctx, TCGOp *op) +{ + return fold_const2(ctx, op); +} + +static bool fold_eqv(OptContext *ctx, TCGOp *op) +{ + return fold_const2(ctx, op); +} + +static bool fold_exts(OptContext *ctx, TCGOp *op) +{ + return fold_const1(ctx, op); +} + +static bool fold_extu(OptContext *ctx, TCGOp *op) +{ + return fold_const1(ctx, op); +} + static bool fold_mb(OptContext *ctx, TCGOp *op) { /* Eliminate duplicate and redundant fence instructions. */ @@ -XXX,XX +XXX,XX @@ static bool fold_mb(OptContext *ctx, TCGOp *op) return true; } +static bool fold_mul(OptContext *ctx, TCGOp *op) +{ + return fold_const2(ctx, op); +} + +static bool fold_mul_highpart(OptContext *ctx, TCGOp *op) +{ + return fold_const2(ctx, op); +} + +static bool fold_nand(OptContext *ctx, TCGOp *op) +{ + return fold_const2(ctx, op); +} + +static bool fold_neg(OptContext *ctx, TCGOp *op) +{ + return fold_const1(ctx, op); +} + +static bool fold_nor(OptContext *ctx, TCGOp *op) +{ + return fold_const2(ctx, op); +} + +static bool fold_not(OptContext *ctx, TCGOp *op) +{ + return fold_const1(ctx, op); +} + +static bool fold_or(OptContext *ctx, TCGOp *op) +{ + return fold_const2(ctx, op); +} + +static bool fold_orc(OptContext *ctx, TCGOp *op) +{ + return fold_const2(ctx, op); +} + static bool fold_qemu_ld(OptContext *ctx, TCGOp *op) { /* Opcodes that touch guest memory stop the mb optimization. */ @@ -XXX,XX +XXX,XX @@ static bool fold_qemu_st(OptContext *ctx, TCGOp *op) return false; } +static bool fold_remainder(OptContext *ctx, TCGOp *op) +{ + return fold_const2(ctx, op); +} + +static bool fold_shift(OptContext *ctx, TCGOp *op) +{ + return fold_const2(ctx, op); +} + +static bool fold_sub(OptContext *ctx, TCGOp *op) +{ + return fold_const2(ctx, op); +} + +static bool fold_xor(OptContext *ctx, TCGOp *op) +{ + return fold_const2(ctx, op); +} + /* Propagate constants and copies, fold constant expressions. */ void tcg_optimize(TCGContext *s) { @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) } break; - CASE_OP_32_64(not): - CASE_OP_32_64(neg): - CASE_OP_32_64(ext8s): - CASE_OP_32_64(ext8u): - CASE_OP_32_64(ext16s): - CASE_OP_32_64(ext16u): - CASE_OP_32_64(ctpop): - case INDEX_op_ext32s_i64: - case INDEX_op_ext32u_i64: - case INDEX_op_ext_i32_i64: - case INDEX_op_extu_i32_i64: - case INDEX_op_extrl_i64_i32: - case INDEX_op_extrh_i64_i32: - if (arg_is_const(op->args[1])) { - tmp = do_constant_folding(opc, arg_info(op->args[1])->val, 0); - tcg_opt_gen_movi(&ctx, op, op->args[0], tmp); - continue; - } - break; - CASE_OP_32_64(bswap16): CASE_OP_32_64(bswap32): case INDEX_op_bswap64_i64: @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) } break; - CASE_OP_32_64(add): - CASE_OP_32_64(sub): - CASE_OP_32_64(mul): - CASE_OP_32_64(or): - CASE_OP_32_64(and): - CASE_OP_32_64(xor): - CASE_OP_32_64(shl): - CASE_OP_32_64(shr): - CASE_OP_32_64(sar): - CASE_OP_32_64(rotl): - CASE_OP_32_64(rotr): - CASE_OP_32_64(andc): - CASE_OP_32_64(orc): - CASE_OP_32_64(eqv): - CASE_OP_32_64(nand): - CASE_OP_32_64(nor): - CASE_OP_32_64(muluh): - CASE_OP_32_64(mulsh): - CASE_OP_32_64(div): - CASE_OP_32_64(divu): - CASE_OP_32_64(rem): - CASE_OP_32_64(remu): - if (arg_is_const(op->args[1]) && arg_is_const(op->args[2])) { - tmp = do_constant_folding(opc, arg_info(op->args[1])->val, - arg_info(op->args[2])->val); - tcg_opt_gen_movi(&ctx, op, op->args[0], tmp); - continue; - } - break; - CASE_OP_32_64(clz): CASE_OP_32_64(ctz): if (arg_is_const(op->args[1])) { @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) } break; + default: + break; + + /* ---------------------------------------------------------- */ + /* Sorted alphabetically by opcode as much as possible. */ + + CASE_OP_32_64_VEC(add): + done = fold_add(&ctx, op); + break; + CASE_OP_32_64_VEC(and): + done = fold_and(&ctx, op); + break; + CASE_OP_32_64_VEC(andc): + done = fold_andc(&ctx, op); + break; + CASE_OP_32_64(ctpop): + done = fold_ctpop(&ctx, op); + break; + CASE_OP_32_64(div): + CASE_OP_32_64(divu): + done = fold_divide(&ctx, op); + break; + CASE_OP_32_64(eqv): + done = fold_eqv(&ctx, op); + break; + CASE_OP_32_64(ext8s): + CASE_OP_32_64(ext16s): + case INDEX_op_ext32s_i64: + case INDEX_op_ext_i32_i64: + done = fold_exts(&ctx, op); + break; + CASE_OP_32_64(ext8u): + CASE_OP_32_64(ext16u): + case INDEX_op_ext32u_i64: + case INDEX_op_extu_i32_i64: + case INDEX_op_extrl_i64_i32: + case INDEX_op_extrh_i64_i32: + done = fold_extu(&ctx, op); + break; case INDEX_op_mb: done = fold_mb(&ctx, op); break; + CASE_OP_32_64(mul): + done = fold_mul(&ctx, op); + break; + CASE_OP_32_64(mulsh): + CASE_OP_32_64(muluh): + done = fold_mul_highpart(&ctx, op); + break; + CASE_OP_32_64(nand): + done = fold_nand(&ctx, op); + break; + CASE_OP_32_64(neg): + done = fold_neg(&ctx, op); + break; + CASE_OP_32_64(nor): + done = fold_nor(&ctx, op); + break; + CASE_OP_32_64_VEC(not): + done = fold_not(&ctx, op); + break; + CASE_OP_32_64_VEC(or): + done = fold_or(&ctx, op); + break; + CASE_OP_32_64_VEC(orc): + done = fold_orc(&ctx, op); + break; case INDEX_op_qemu_ld_i32: case INDEX_op_qemu_ld_i64: done = fold_qemu_ld(&ctx, op); @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) case INDEX_op_qemu_st_i64: done = fold_qemu_st(&ctx, op); break; - - default: + CASE_OP_32_64(rem): + CASE_OP_32_64(remu): + done = fold_remainder(&ctx, op); + break; + CASE_OP_32_64(rotl): + CASE_OP_32_64(rotr): + CASE_OP_32_64(sar): + CASE_OP_32_64(shl): + CASE_OP_32_64(shr): + done = fold_shift(&ctx, op); + break; + CASE_OP_32_64_VEC(sub): + done = fold_sub(&ctx, op); + break; + CASE_OP_32_64_VEC(xor): + done = fold_xor(&ctx, op); break; } -- 2.25.1 Reduce some code duplication by folding the NE and EQ cases. Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 145 ++++++++++++++++++++++++------------------------- 1 file changed, 72 insertions(+), 73 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_remainder(OptContext *ctx, TCGOp *op) return fold_const2(ctx, op); } +static bool fold_setcond2(OptContext *ctx, TCGOp *op) +{ + TCGCond cond = op->args[5]; + int i = do_constant_folding_cond2(&op->args[1], &op->args[3], cond); + int inv = 0; + + if (i >= 0) { + goto do_setcond_const; + } + + switch (cond) { + case TCG_COND_LT: + case TCG_COND_GE: + /* + * Simplify LT/GE comparisons vs zero to a single compare + * vs the high word of the input. + */ + if (arg_is_const(op->args[3]) && arg_info(op->args[3])->val == 0 && + arg_is_const(op->args[4]) && arg_info(op->args[4])->val == 0) { + goto do_setcond_high; + } + break; + + case TCG_COND_NE: + inv = 1; + QEMU_FALLTHROUGH; + case TCG_COND_EQ: + /* + * Simplify EQ/NE comparisons where one of the pairs + * can be simplified. + */ + i = do_constant_folding_cond(INDEX_op_setcond_i32, op->args[1], + op->args[3], cond); + switch (i ^ inv) { + case 0: + goto do_setcond_const; + case 1: + goto do_setcond_high; + } + + i = do_constant_folding_cond(INDEX_op_setcond_i32, op->args[2], + op->args[4], cond); + switch (i ^ inv) { + case 0: + goto do_setcond_const; + case 1: + op->args[2] = op->args[3]; + op->args[3] = cond; + op->opc = INDEX_op_setcond_i32; + break; + } + break; + + default: + break; + + do_setcond_high: + op->args[1] = op->args[2]; + op->args[2] = op->args[4]; + op->args[3] = cond; + op->opc = INDEX_op_setcond_i32; + break; + } + return false; + + do_setcond_const: + return tcg_opt_gen_movi(ctx, op, op->args[0], i); +} + static bool fold_shift(OptContext *ctx, TCGOp *op) { return fold_const2(ctx, op); @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) } break; - case INDEX_op_setcond2_i32: - i = do_constant_folding_cond2(&op->args[1], &op->args[3], - op->args[5]); - if (i >= 0) { - do_setcond_const: - tcg_opt_gen_movi(&ctx, op, op->args[0], i); - continue; - } - if ((op->args[5] == TCG_COND_LT || op->args[5] == TCG_COND_GE) - && arg_is_const(op->args[3]) - && arg_info(op->args[3])->val == 0 - && arg_is_const(op->args[4]) - && arg_info(op->args[4])->val == 0) { - /* Simplify LT/GE comparisons vs zero to a single compare - vs the high word of the input. */ - do_setcond_high: - reset_temp(op->args[0]); - arg_info(op->args[0])->z_mask = 1; - op->opc = INDEX_op_setcond_i32; - op->args[1] = op->args[2]; - op->args[2] = op->args[4]; - op->args[3] = op->args[5]; - break; - } - if (op->args[5] == TCG_COND_EQ) { - /* Simplify EQ comparisons where one of the pairs - can be simplified. */ - i = do_constant_folding_cond(INDEX_op_setcond_i32, - op->args[1], op->args[3], - TCG_COND_EQ); - if (i == 0) { - goto do_setcond_const; - } else if (i > 0) { - goto do_setcond_high; - } - i = do_constant_folding_cond(INDEX_op_setcond_i32, - op->args[2], op->args[4], - TCG_COND_EQ); - if (i == 0) { - goto do_setcond_high; - } else if (i < 0) { - break; - } - do_setcond_low: - reset_temp(op->args[0]); - arg_info(op->args[0])->z_mask = 1; - op->opc = INDEX_op_setcond_i32; - op->args[2] = op->args[3]; - op->args[3] = op->args[5]; - break; - } - if (op->args[5] == TCG_COND_NE) { - /* Simplify NE comparisons where one of the pairs - can be simplified. */ - i = do_constant_folding_cond(INDEX_op_setcond_i32, - op->args[1], op->args[3], - TCG_COND_NE); - if (i == 0) { - goto do_setcond_high; - } else if (i > 0) { - goto do_setcond_const; - } - i = do_constant_folding_cond(INDEX_op_setcond_i32, - op->args[2], op->args[4], - TCG_COND_NE); - if (i == 0) { - goto do_setcond_low; - } else if (i > 0) { - goto do_setcond_const; - } - } - break; - default: break; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) CASE_OP_32_64(shr): done = fold_shift(&ctx, op); break; + case INDEX_op_setcond2_i32: + done = fold_setcond2(&ctx, op); + break; CASE_OP_32_64_VEC(sub): done = fold_sub(&ctx, op); break; -- 2.25.1 Reduce some code duplication by folding the NE and EQ cases. Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 159 +++++++++++++++++++++++++------------------------ 1 file changed, 81 insertions(+), 78 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_andc(OptContext *ctx, TCGOp *op) return fold_const2(ctx, op); } +static bool fold_brcond2(OptContext *ctx, TCGOp *op) +{ + TCGCond cond = op->args[4]; + int i = do_constant_folding_cond2(&op->args[0], &op->args[2], cond); + TCGArg label = op->args[5]; + int inv = 0; + + if (i >= 0) { + goto do_brcond_const; + } + + switch (cond) { + case TCG_COND_LT: + case TCG_COND_GE: + /* + * Simplify LT/GE comparisons vs zero to a single compare + * vs the high word of the input. + */ + if (arg_is_const(op->args[2]) && arg_info(op->args[2])->val == 0 && + arg_is_const(op->args[3]) && arg_info(op->args[3])->val == 0) { + goto do_brcond_high; + } + break; + + case TCG_COND_NE: + inv = 1; + QEMU_FALLTHROUGH; + case TCG_COND_EQ: + /* + * Simplify EQ/NE comparisons where one of the pairs + * can be simplified. + */ + i = do_constant_folding_cond(INDEX_op_brcond_i32, op->args[0], + op->args[2], cond); + switch (i ^ inv) { + case 0: + goto do_brcond_const; + case 1: + goto do_brcond_high; + } + + i = do_constant_folding_cond(INDEX_op_brcond_i32, op->args[1], + op->args[3], cond); + switch (i ^ inv) { + case 0: + goto do_brcond_const; + case 1: + op->opc = INDEX_op_brcond_i32; + op->args[1] = op->args[2]; + op->args[2] = cond; + op->args[3] = label; + break; + } + break; + + default: + break; + + do_brcond_high: + op->opc = INDEX_op_brcond_i32; + op->args[0] = op->args[1]; + op->args[1] = op->args[3]; + op->args[2] = cond; + op->args[3] = label; + break; + + do_brcond_const: + if (i == 0) { + tcg_op_remove(ctx->tcg, op); + return true; + } + op->opc = INDEX_op_br; + op->args[0] = label; + break; + } + return false; +} + static bool fold_call(OptContext *ctx, TCGOp *op) { TCGContext *s = ctx->tcg; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) } break; - case INDEX_op_brcond2_i32: - i = do_constant_folding_cond2(&op->args[0], &op->args[2], - op->args[4]); - if (i == 0) { - do_brcond_false: - tcg_op_remove(s, op); - continue; - } - if (i > 0) { - do_brcond_true: - op->opc = opc = INDEX_op_br; - op->args[0] = op->args[5]; - break; - } - if ((op->args[4] == TCG_COND_LT || op->args[4] == TCG_COND_GE) - && arg_is_const(op->args[2]) - && arg_info(op->args[2])->val == 0 - && arg_is_const(op->args[3]) - && arg_info(op->args[3])->val == 0) { - /* Simplify LT/GE comparisons vs zero to a single compare - vs the high word of the input. */ - do_brcond_high: - op->opc = opc = INDEX_op_brcond_i32; - op->args[0] = op->args[1]; - op->args[1] = op->args[3]; - op->args[2] = op->args[4]; - op->args[3] = op->args[5]; - break; - } - if (op->args[4] == TCG_COND_EQ) { - /* Simplify EQ comparisons where one of the pairs - can be simplified. */ - i = do_constant_folding_cond(INDEX_op_brcond_i32, - op->args[0], op->args[2], - TCG_COND_EQ); - if (i == 0) { - goto do_brcond_false; - } else if (i > 0) { - goto do_brcond_high; - } - i = do_constant_folding_cond(INDEX_op_brcond_i32, - op->args[1], op->args[3], - TCG_COND_EQ); - if (i == 0) { - goto do_brcond_false; - } else if (i < 0) { - break; - } - do_brcond_low: - memset(&ctx.temps_used, 0, sizeof(ctx.temps_used)); - op->opc = INDEX_op_brcond_i32; - op->args[1] = op->args[2]; - op->args[2] = op->args[4]; - op->args[3] = op->args[5]; - break; - } - if (op->args[4] == TCG_COND_NE) { - /* Simplify NE comparisons where one of the pairs - can be simplified. */ - i = do_constant_folding_cond(INDEX_op_brcond_i32, - op->args[0], op->args[2], - TCG_COND_NE); - if (i == 0) { - goto do_brcond_high; - } else if (i > 0) { - goto do_brcond_true; - } - i = do_constant_folding_cond(INDEX_op_brcond_i32, - op->args[1], op->args[3], - TCG_COND_NE); - if (i == 0) { - goto do_brcond_low; - } else if (i > 0) { - goto do_brcond_true; - } - } - break; - default: break; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) CASE_OP_32_64_VEC(andc): done = fold_andc(&ctx, op); break; + case INDEX_op_brcond2_i32: + done = fold_brcond2(&ctx, op); + break; CASE_OP_32_64(ctpop): done = fold_ctpop(&ctx, op); break; -- 2.25.1 Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 33 +++++++++++++++++++-------------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_andc(OptContext *ctx, TCGOp *op) return fold_const2(ctx, op); } +static bool fold_brcond(OptContext *ctx, TCGOp *op) +{ + TCGCond cond = op->args[2]; + int i = do_constant_folding_cond(op->opc, op->args[0], op->args[1], cond); + + if (i == 0) { + tcg_op_remove(ctx->tcg, op); + return true; + } + if (i > 0) { + op->opc = INDEX_op_br; + op->args[0] = op->args[3]; + } + return false; +} + static bool fold_brcond2(OptContext *ctx, TCGOp *op) { TCGCond cond = op->args[4]; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) } break; - CASE_OP_32_64(brcond): - i = do_constant_folding_cond(opc, op->args[0], - op->args[1], op->args[2]); - if (i == 0) { - tcg_op_remove(s, op); - continue; - } else if (i > 0) { - memset(&ctx.temps_used, 0, sizeof(ctx.temps_used)); - op->opc = opc = INDEX_op_br; - op->args[0] = op->args[3]; - break; - } - break; - CASE_OP_32_64(movcond): i = do_constant_folding_cond(opc, op->args[1], op->args[2], op->args[5]); @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) CASE_OP_32_64_VEC(andc): done = fold_andc(&ctx, op); break; + CASE_OP_32_64(brcond): + done = fold_brcond(&ctx, op); + break; case INDEX_op_brcond2_i32: done = fold_brcond2(&ctx, op); break; -- 2.25.1 Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_remainder(OptContext *ctx, TCGOp *op) return fold_const2(ctx, op); } +static bool fold_setcond(OptContext *ctx, TCGOp *op) +{ + TCGCond cond = op->args[3]; + int i = do_constant_folding_cond(op->opc, op->args[1], op->args[2], cond); + + if (i >= 0) { + return tcg_opt_gen_movi(ctx, op, op->args[0], i); + } + return false; +} + static bool fold_setcond2(OptContext *ctx, TCGOp *op) { TCGCond cond = op->args[5]; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) } break; - CASE_OP_32_64(setcond): - i = do_constant_folding_cond(opc, op->args[1], - op->args[2], op->args[3]); - if (i >= 0) { - tcg_opt_gen_movi(&ctx, op, op->args[0], i); - continue; - } - break; - CASE_OP_32_64(movcond): i = do_constant_folding_cond(opc, op->args[1], op->args[2], op->args[5]); @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) CASE_OP_32_64(shr): done = fold_shift(&ctx, op); break; + CASE_OP_32_64(setcond): + done = fold_setcond(&ctx, op); + break; case INDEX_op_setcond2_i32: done = fold_setcond2(&ctx, op); break; -- 2.25.1 Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 37 +++++++++++++++++++++---------------- 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_mul_highpart(OptContext *ctx, TCGOp *op) return fold_const2(ctx, op); } +static bool fold_mulu2_i32(OptContext *ctx, TCGOp *op) +{ + if (arg_is_const(op->args[2]) && arg_is_const(op->args[3])) { + uint32_t a = arg_info(op->args[2])->val; + uint32_t b = arg_info(op->args[3])->val; + uint64_t r = (uint64_t)a * b; + TCGArg rl, rh; + TCGOp *op2 = tcg_op_insert_before(ctx->tcg, op, INDEX_op_mov_i32); + + rl = op->args[0]; + rh = op->args[1]; + tcg_opt_gen_movi(ctx, op, rl, (int32_t)r); + tcg_opt_gen_movi(ctx, op2, rh, (int32_t)(r >> 32)); + return true; + } + return false; +} + static bool fold_nand(OptContext *ctx, TCGOp *op) { return fold_const2(ctx, op); @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) } break; - case INDEX_op_mulu2_i32: - if (arg_is_const(op->args[2]) && arg_is_const(op->args[3])) { - uint32_t a = arg_info(op->args[2])->val; - uint32_t b = arg_info(op->args[3])->val; - uint64_t r = (uint64_t)a * b; - TCGArg rl, rh; - TCGOp *op2 = tcg_op_insert_before(s, op, INDEX_op_mov_i32); - - rl = op->args[0]; - rh = op->args[1]; - tcg_opt_gen_movi(&ctx, op, rl, (int32_t)r); - tcg_opt_gen_movi(&ctx, op2, rh, (int32_t)(r >> 32)); - continue; - } - break; - default: break; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) CASE_OP_32_64(muluh): done = fold_mul_highpart(&ctx, op); break; + case INDEX_op_mulu2_i32: + done = fold_mulu2_i32(&ctx, op); + break; CASE_OP_32_64(nand): done = fold_nand(&ctx, op); break; -- 2.25.1 Add two additional helpers, fold_add2_i32 and fold_sub2_i32 which will not be simple wrappers forever. Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 70 +++++++++++++++++++++++++++++++------------------- 1 file changed, 44 insertions(+), 26 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_add(OptContext *ctx, TCGOp *op) return fold_const2(ctx, op); } +static bool fold_addsub2_i32(OptContext *ctx, TCGOp *op, bool add) +{ + if (arg_is_const(op->args[2]) && arg_is_const(op->args[3]) && + arg_is_const(op->args[4]) && arg_is_const(op->args[5])) { + uint32_t al = arg_info(op->args[2])->val; + uint32_t ah = arg_info(op->args[3])->val; + uint32_t bl = arg_info(op->args[4])->val; + uint32_t bh = arg_info(op->args[5])->val; + uint64_t a = ((uint64_t)ah << 32) | al; + uint64_t b = ((uint64_t)bh << 32) | bl; + TCGArg rl, rh; + TCGOp *op2 = tcg_op_insert_before(ctx->tcg, op, INDEX_op_mov_i32); + + if (add) { + a += b; + } else { + a -= b; + } + + rl = op->args[0]; + rh = op->args[1]; + tcg_opt_gen_movi(ctx, op, rl, (int32_t)a); + tcg_opt_gen_movi(ctx, op2, rh, (int32_t)(a >> 32)); + return true; + } + return false; +} + +static bool fold_add2_i32(OptContext *ctx, TCGOp *op) +{ + return fold_addsub2_i32(ctx, op, true); +} + static bool fold_and(OptContext *ctx, TCGOp *op) { return fold_const2(ctx, op); @@ -XXX,XX +XXX,XX @@ static bool fold_sub(OptContext *ctx, TCGOp *op) return fold_const2(ctx, op); } +static bool fold_sub2_i32(OptContext *ctx, TCGOp *op) +{ + return fold_addsub2_i32(ctx, op, false); +} + static bool fold_xor(OptContext *ctx, TCGOp *op) { return fold_const2(ctx, op); @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) } break; - case INDEX_op_add2_i32: - case INDEX_op_sub2_i32: - if (arg_is_const(op->args[2]) && arg_is_const(op->args[3]) - && arg_is_const(op->args[4]) && arg_is_const(op->args[5])) { - uint32_t al = arg_info(op->args[2])->val; - uint32_t ah = arg_info(op->args[3])->val; - uint32_t bl = arg_info(op->args[4])->val; - uint32_t bh = arg_info(op->args[5])->val; - uint64_t a = ((uint64_t)ah << 32) | al; - uint64_t b = ((uint64_t)bh << 32) | bl; - TCGArg rl, rh; - TCGOp *op2 = tcg_op_insert_before(s, op, INDEX_op_mov_i32); - - if (opc == INDEX_op_add2_i32) { - a += b; - } else { - a -= b; - } - - rl = op->args[0]; - rh = op->args[1]; - tcg_opt_gen_movi(&ctx, op, rl, (int32_t)a); - tcg_opt_gen_movi(&ctx, op2, rh, (int32_t)(a >> 32)); - continue; - } - break; default: break; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) CASE_OP_32_64_VEC(add): done = fold_add(&ctx, op); break; + case INDEX_op_add2_i32: + done = fold_add2_i32(&ctx, op); + break; CASE_OP_32_64_VEC(and): done = fold_and(&ctx, op); break; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) CASE_OP_32_64_VEC(sub): done = fold_sub(&ctx, op); break; + case INDEX_op_sub2_i32: + done = fold_sub2_i32(&ctx, op); + break; CASE_OP_32_64_VEC(xor): done = fold_xor(&ctx, op); break; -- 2.25.1 Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 56 ++++++++++++++++++++++++++++---------------------- 1 file changed, 31 insertions(+), 25 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_mb(OptContext *ctx, TCGOp *op) return true; } +static bool fold_movcond(OptContext *ctx, TCGOp *op) +{ + TCGOpcode opc = op->opc; + TCGCond cond = op->args[5]; + int i = do_constant_folding_cond(opc, op->args[1], op->args[2], cond); + + if (i >= 0) { + return tcg_opt_gen_mov(ctx, op, op->args[0], op->args[4 - i]); + } + + if (arg_is_const(op->args[3]) && arg_is_const(op->args[4])) { + uint64_t tv = arg_info(op->args[3])->val; + uint64_t fv = arg_info(op->args[4])->val; + + opc = (opc == INDEX_op_movcond_i32 + ? INDEX_op_setcond_i32 : INDEX_op_setcond_i64); + + if (tv == 1 && fv == 0) { + op->opc = opc; + op->args[3] = cond; + } else if (fv == 1 && tv == 0) { + op->opc = opc; + op->args[3] = tcg_invert_cond(cond); + } + } + return false; +} + static bool fold_mul(OptContext *ctx, TCGOp *op) { return fold_const2(ctx, op); @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) } break; - CASE_OP_32_64(movcond): - i = do_constant_folding_cond(opc, op->args[1], - op->args[2], op->args[5]); - if (i >= 0) { - tcg_opt_gen_mov(&ctx, op, op->args[0], op->args[4 - i]); - continue; - } - if (arg_is_const(op->args[3]) && arg_is_const(op->args[4])) { - uint64_t tv = arg_info(op->args[3])->val; - uint64_t fv = arg_info(op->args[4])->val; - TCGCond cond = op->args[5]; - - if (fv == 1 && tv == 0) { - cond = tcg_invert_cond(cond); - } else if (!(tv == 1 && fv == 0)) { - break; - } - op->args[3] = cond; - op->opc = opc = (opc == INDEX_op_movcond_i32 - ? INDEX_op_setcond_i32 - : INDEX_op_setcond_i64); - } - break; - - default: break; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) case INDEX_op_mb: done = fold_mb(&ctx, op); break; + CASE_OP_32_64(movcond): + done = fold_movcond(&ctx, op); + break; CASE_OP_32_64(mul): done = fold_mul(&ctx, op); break; -- 2.25.1 Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 39 ++++++++++++++++++++++----------------- 1 file changed, 22 insertions(+), 17 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_eqv(OptContext *ctx, TCGOp *op) return fold_const2(ctx, op); } +static bool fold_extract2(OptContext *ctx, TCGOp *op) +{ + if (arg_is_const(op->args[1]) && arg_is_const(op->args[2])) { + uint64_t v1 = arg_info(op->args[1])->val; + uint64_t v2 = arg_info(op->args[2])->val; + int shr = op->args[3]; + + if (op->opc == INDEX_op_extract2_i64) { + v1 >>= shr; + v2 <<= 64 - shr; + } else { + v1 = (uint32_t)v1 >> shr; + v2 = (int32_t)v2 << (32 - shr); + } + return tcg_opt_gen_movi(ctx, op, op->args[0], v1 | v2); + } + return false; +} + static bool fold_exts(OptContext *ctx, TCGOp *op) { return fold_const1(ctx, op); @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) } break; - CASE_OP_32_64(extract2): - if (arg_is_const(op->args[1]) && arg_is_const(op->args[2])) { - uint64_t v1 = arg_info(op->args[1])->val; - uint64_t v2 = arg_info(op->args[2])->val; - int shr = op->args[3]; - - if (opc == INDEX_op_extract2_i64) { - tmp = (v1 >> shr) | (v2 << (64 - shr)); - } else { - tmp = (int32_t)(((uint32_t)v1 >> shr) | - ((uint32_t)v2 << (32 - shr))); - } - tcg_opt_gen_movi(&ctx, op, op->args[0], tmp); - continue; - } - break; - default: break; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) CASE_OP_32_64(eqv): done = fold_eqv(&ctx, op); break; + CASE_OP_32_64(extract2): + done = fold_extract2(&ctx, op); + break; CASE_OP_32_64(ext8s): CASE_OP_32_64(ext16s): case INDEX_op_ext32s_i64: -- 2.25.1 Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 48 ++++++++++++++++++++++++++++++------------------ 1 file changed, 30 insertions(+), 18 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_eqv(OptContext *ctx, TCGOp *op) return fold_const2(ctx, op); } +static bool fold_extract(OptContext *ctx, TCGOp *op) +{ + if (arg_is_const(op->args[1])) { + uint64_t t; + + t = arg_info(op->args[1])->val; + t = extract64(t, op->args[2], op->args[3]); + return tcg_opt_gen_movi(ctx, op, op->args[0], t); + } + return false; +} + static bool fold_extract2(OptContext *ctx, TCGOp *op) { if (arg_is_const(op->args[1]) && arg_is_const(op->args[2])) { @@ -XXX,XX +XXX,XX @@ static bool fold_setcond2(OptContext *ctx, TCGOp *op) return tcg_opt_gen_movi(ctx, op, op->args[0], i); } +static bool fold_sextract(OptContext *ctx, TCGOp *op) +{ + if (arg_is_const(op->args[1])) { + uint64_t t; + + t = arg_info(op->args[1])->val; + t = sextract64(t, op->args[2], op->args[3]); + return tcg_opt_gen_movi(ctx, op, op->args[0], t); + } + return false; +} + static bool fold_shift(OptContext *ctx, TCGOp *op) { return fold_const2(ctx, op); @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) } break; - CASE_OP_32_64(extract): - if (arg_is_const(op->args[1])) { - tmp = extract64(arg_info(op->args[1])->val, - op->args[2], op->args[3]); - tcg_opt_gen_movi(&ctx, op, op->args[0], tmp); - continue; - } - break; - - CASE_OP_32_64(sextract): - if (arg_is_const(op->args[1])) { - tmp = sextract64(arg_info(op->args[1])->val, - op->args[2], op->args[3]); - tcg_opt_gen_movi(&ctx, op, op->args[0], tmp); - continue; - } - break; - default: break; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) CASE_OP_32_64(eqv): done = fold_eqv(&ctx, op); break; + CASE_OP_32_64(extract): + done = fold_extract(&ctx, op); + break; CASE_OP_32_64(extract2): done = fold_extract2(&ctx, op); break; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) case INDEX_op_setcond2_i32: done = fold_setcond2(&ctx, op); break; + CASE_OP_32_64(sextract): + done = fold_sextract(&ctx, op); + break; CASE_OP_32_64_VEC(sub): done = fold_sub(&ctx, op); break; -- 2.25.1 Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_ctpop(OptContext *ctx, TCGOp *op) return fold_const1(ctx, op); } +static bool fold_deposit(OptContext *ctx, TCGOp *op) +{ + if (arg_is_const(op->args[1]) && arg_is_const(op->args[2])) { + uint64_t t1 = arg_info(op->args[1])->val; + uint64_t t2 = arg_info(op->args[2])->val; + + t1 = deposit64(t1, op->args[3], op->args[4], t2); + return tcg_opt_gen_movi(ctx, op, op->args[0], t1); + } + return false; +} + static bool fold_divide(OptContext *ctx, TCGOp *op) { return fold_const2(ctx, op); @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) } break; - CASE_OP_32_64(deposit): - if (arg_is_const(op->args[1]) && arg_is_const(op->args[2])) { - tmp = deposit64(arg_info(op->args[1])->val, - op->args[3], op->args[4], - arg_info(op->args[2])->val); - tcg_opt_gen_movi(&ctx, op, op->args[0], tmp); - continue; - } - break; - default: break; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) CASE_OP_32_64(ctpop): done = fold_ctpop(&ctx, op); break; + CASE_OP_32_64(deposit): + done = fold_deposit(&ctx, op); + break; CASE_OP_32_64(div): CASE_OP_32_64(divu): done = fold_divide(&ctx, op); -- 2.25.1 Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 32 ++++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_call(OptContext *ctx, TCGOp *op) return true; } +static bool fold_count_zeros(OptContext *ctx, TCGOp *op) +{ + if (arg_is_const(op->args[1])) { + uint64_t t = arg_info(op->args[1])->val; + + if (t != 0) { + t = do_constant_folding(op->opc, t, 0); + return tcg_opt_gen_movi(ctx, op, op->args[0], t); + } + return tcg_opt_gen_mov(ctx, op, op->args[0], op->args[2]); + } + return false; +} + static bool fold_ctpop(OptContext *ctx, TCGOp *op) { return fold_const1(ctx, op); @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) } break; - CASE_OP_32_64(clz): - CASE_OP_32_64(ctz): - if (arg_is_const(op->args[1])) { - TCGArg v = arg_info(op->args[1])->val; - if (v != 0) { - tmp = do_constant_folding(opc, v, 0); - tcg_opt_gen_movi(&ctx, op, op->args[0], tmp); - } else { - tcg_opt_gen_mov(&ctx, op, op->args[0], op->args[2]); - } - continue; - } - break; - default: break; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) case INDEX_op_brcond2_i32: done = fold_brcond2(&ctx, op); break; + CASE_OP_32_64(clz): + CASE_OP_32_64(ctz): + done = fold_count_zeros(&ctx, op); + break; CASE_OP_32_64(ctpop): done = fold_ctpop(&ctx, op); break; -- 2.25.1 Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_brcond2(OptContext *ctx, TCGOp *op) return false; } +static bool fold_bswap(OptContext *ctx, TCGOp *op) +{ + if (arg_is_const(op->args[1])) { + uint64_t t = arg_info(op->args[1])->val; + + t = do_constant_folding(op->opc, t, op->args[2]); + return tcg_opt_gen_movi(ctx, op, op->args[0], t); + } + return false; +} + static bool fold_call(OptContext *ctx, TCGOp *op) { TCGContext *s = ctx->tcg; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) } break; - CASE_OP_32_64(bswap16): - CASE_OP_32_64(bswap32): - case INDEX_op_bswap64_i64: - if (arg_is_const(op->args[1])) { - tmp = do_constant_folding(opc, arg_info(op->args[1])->val, - op->args[2]); - tcg_opt_gen_movi(&ctx, op, op->args[0], tmp); - continue; - } - break; - default: break; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) case INDEX_op_brcond2_i32: done = fold_brcond2(&ctx, op); break; + CASE_OP_32_64(bswap16): + CASE_OP_32_64(bswap32): + case INDEX_op_bswap64_i64: + done = fold_bswap(&ctx, op); + break; CASE_OP_32_64(clz): CASE_OP_32_64(ctz): done = fold_count_zeros(&ctx, op); -- 2.25.1 Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 53 +++++++++++++++++++++++++++++--------------------- 1 file changed, 31 insertions(+), 22 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_divide(OptContext *ctx, TCGOp *op) return fold_const2(ctx, op); } +static bool fold_dup(OptContext *ctx, TCGOp *op) +{ + if (arg_is_const(op->args[1])) { + uint64_t t = arg_info(op->args[1])->val; + t = dup_const(TCGOP_VECE(op), t); + return tcg_opt_gen_movi(ctx, op, op->args[0], t); + } + return false; +} + +static bool fold_dup2(OptContext *ctx, TCGOp *op) +{ + if (arg_is_const(op->args[1]) && arg_is_const(op->args[2])) { + uint64_t t = deposit64(arg_info(op->args[1])->val, 32, 32, + arg_info(op->args[2])->val); + return tcg_opt_gen_movi(ctx, op, op->args[0], t); + } + + if (args_are_copies(op->args[1], op->args[2])) { + op->opc = INDEX_op_dup_vec; + TCGOP_VECE(op) = MO_32; + } + return false; +} + static bool fold_eqv(OptContext *ctx, TCGOp *op) { return fold_const2(ctx, op); @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) done = tcg_opt_gen_mov(&ctx, op, op->args[0], op->args[1]); break; - case INDEX_op_dup_vec: - if (arg_is_const(op->args[1])) { - tmp = arg_info(op->args[1])->val; - tmp = dup_const(TCGOP_VECE(op), tmp); - tcg_opt_gen_movi(&ctx, op, op->args[0], tmp); - continue; - } - break; - - case INDEX_op_dup2_vec: - assert(TCG_TARGET_REG_BITS == 32); - if (arg_is_const(op->args[1]) && arg_is_const(op->args[2])) { - tcg_opt_gen_movi(&ctx, op, op->args[0], - deposit64(arg_info(op->args[1])->val, 32, 32, - arg_info(op->args[2])->val)); - continue; - } else if (args_are_copies(op->args[1], op->args[2])) { - op->opc = INDEX_op_dup_vec; - TCGOP_VECE(op) = MO_32; - } - break; - default: break; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) CASE_OP_32_64(divu): done = fold_divide(&ctx, op); break; + case INDEX_op_dup_vec: + done = fold_dup(&ctx, op); + break; + case INDEX_op_dup2_vec: + done = fold_dup2(&ctx, op); + break; CASE_OP_32_64(eqv): done = fold_eqv(&ctx, op); break; -- 2.25.1 This is the final entry in the main switch that was in a different form. After this, we have the option to convert the switch into a function dispatch table. Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_mb(OptContext *ctx, TCGOp *op) return true; } +static bool fold_mov(OptContext *ctx, TCGOp *op) +{ + return tcg_opt_gen_mov(ctx, op, op->args[0], op->args[1]); +} + static bool fold_movcond(OptContext *ctx, TCGOp *op) { TCGOpcode opc = op->opc; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) break; } - /* Propagate constants through copy operations and do constant - folding. Constants will be substituted to arguments by register - allocator where needed and possible. Also detect copies. */ + /* + * Process each opcode. + * Sorted alphabetically by opcode as much as possible. + */ switch (opc) { - CASE_OP_32_64_VEC(mov): - done = tcg_opt_gen_mov(&ctx, op, op->args[0], op->args[1]); - break; - - default: - break; - - /* ---------------------------------------------------------- */ - /* Sorted alphabetically by opcode as much as possible. */ - CASE_OP_32_64_VEC(add): done = fold_add(&ctx, op); break; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) case INDEX_op_mb: done = fold_mb(&ctx, op); break; + CASE_OP_32_64_VEC(mov): + done = fold_mov(&ctx, op); + break; CASE_OP_32_64(movcond): done = fold_movcond(&ctx, op); break; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) CASE_OP_32_64_VEC(xor): done = fold_xor(&ctx, op); break; + default: + break; } if (!done) { -- 2.25.1 Pull the "op r, a, a => movi r, 0" optimization into a function, and use it in the outer opcode fold functions. Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 41 ++++++++++++++++++++++++----------------- 1 file changed, 24 insertions(+), 17 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_const2(OptContext *ctx, TCGOp *op) return false; } +/* If the binary operation has both arguments equal, fold to @i. */ +static bool fold_xx_to_i(OptContext *ctx, TCGOp *op, uint64_t i) +{ + if (args_are_copies(op->args[1], op->args[2])) { + return tcg_opt_gen_movi(ctx, op, op->args[0], i); + } + return false; +} + /* * These outermost fold_<op> functions are sorted alphabetically. */ @@ -XXX,XX +XXX,XX @@ static bool fold_and(OptContext *ctx, TCGOp *op) static bool fold_andc(OptContext *ctx, TCGOp *op) { - return fold_const2(ctx, op); + if (fold_const2(ctx, op) || + fold_xx_to_i(ctx, op, 0)) { + return true; + } + return false; } static bool fold_brcond(OptContext *ctx, TCGOp *op) @@ -XXX,XX +XXX,XX @@ static bool fold_shift(OptContext *ctx, TCGOp *op) static bool fold_sub(OptContext *ctx, TCGOp *op) { - return fold_const2(ctx, op); + if (fold_const2(ctx, op) || + fold_xx_to_i(ctx, op, 0)) { + return true; + } + return false; } static bool fold_sub2_i32(OptContext *ctx, TCGOp *op) @@ -XXX,XX +XXX,XX @@ static bool fold_sub2_i32(OptContext *ctx, TCGOp *op) static bool fold_xor(OptContext *ctx, TCGOp *op) { - return fold_const2(ctx, op); + if (fold_const2(ctx, op) || + fold_xx_to_i(ctx, op, 0)) { + return true; + } + return false; } /* Propagate constants and copies, fold constant expressions. */ @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) break; } - /* Simplify expression for "op r, a, a => movi r, 0" cases */ - switch (opc) { - CASE_OP_32_64_VEC(andc): - CASE_OP_32_64_VEC(sub): - CASE_OP_32_64_VEC(xor): - if (args_are_copies(op->args[1], op->args[2])) { - tcg_opt_gen_movi(&ctx, op, op->args[0], 0); - continue; - } - break; - default: - break; - } - /* * Process each opcode. * Sorted alphabetically by opcode as much as possible. -- 2.25.1 Pull the "op r, a, a => mov r, a" optimization into a function, and use it in the outer opcode fold functions. Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 39 ++++++++++++++++++++++++--------------- 1 file changed, 24 insertions(+), 15 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_xx_to_i(OptContext *ctx, TCGOp *op, uint64_t i) return false; } +/* If the binary operation has both arguments equal, fold to identity. */ +static bool fold_xx_to_x(OptContext *ctx, TCGOp *op) +{ + if (args_are_copies(op->args[1], op->args[2])) { + return tcg_opt_gen_mov(ctx, op, op->args[0], op->args[1]); + } + return false; +} + /* * These outermost fold_<op> functions are sorted alphabetically. + * + * The ordering of the transformations should be: + * 1) those that produce a constant + * 2) those that produce a copy + * 3) those that produce information about the result value. */ static bool fold_add(OptContext *ctx, TCGOp *op) @@ -XXX,XX +XXX,XX @@ static bool fold_add2_i32(OptContext *ctx, TCGOp *op) static bool fold_and(OptContext *ctx, TCGOp *op) { - return fold_const2(ctx, op); + if (fold_const2(ctx, op) || + fold_xx_to_x(ctx, op)) { + return true; + } + return false; } static bool fold_andc(OptContext *ctx, TCGOp *op) @@ -XXX,XX +XXX,XX @@ static bool fold_not(OptContext *ctx, TCGOp *op) static bool fold_or(OptContext *ctx, TCGOp *op) { - return fold_const2(ctx, op); + if (fold_const2(ctx, op) || + fold_xx_to_x(ctx, op)) { + return true; + } + return false; } static bool fold_orc(OptContext *ctx, TCGOp *op) @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) break; } - /* Simplify expression for "op r, a, a => mov r, a" cases */ - switch (opc) { - CASE_OP_32_64_VEC(or): - CASE_OP_32_64_VEC(and): - if (args_are_copies(op->args[1], op->args[2])) { - tcg_opt_gen_mov(&ctx, op, op->args[0], op->args[1]); - continue; - } - break; - default: - break; - } - /* * Process each opcode. * Sorted alphabetically by opcode as much as possible. -- 2.25.1 Pull the "op r, a, 0 => movi r, 0" optimization into a function, and use it in the outer opcode fold functions. Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 38 ++++++++++++++++++++------------------ 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_const2(OptContext *ctx, TCGOp *op) return false; } +/* If the binary operation has second argument @i, fold to @i. */ +static bool fold_xi_to_i(OptContext *ctx, TCGOp *op, uint64_t i) +{ + if (arg_is_const(op->args[2]) && arg_info(op->args[2])->val == i) { + return tcg_opt_gen_movi(ctx, op, op->args[0], i); + } + return false; +} + /* If the binary operation has both arguments equal, fold to @i. */ static bool fold_xx_to_i(OptContext *ctx, TCGOp *op, uint64_t i) { @@ -XXX,XX +XXX,XX @@ static bool fold_add2_i32(OptContext *ctx, TCGOp *op) static bool fold_and(OptContext *ctx, TCGOp *op) { if (fold_const2(ctx, op) || + fold_xi_to_i(ctx, op, 0) || fold_xx_to_x(ctx, op)) { return true; } @@ -XXX,XX +XXX,XX @@ static bool fold_movcond(OptContext *ctx, TCGOp *op) static bool fold_mul(OptContext *ctx, TCGOp *op) { - return fold_const2(ctx, op); + if (fold_const2(ctx, op) || + fold_xi_to_i(ctx, op, 0)) { + return true; + } + return false; } static bool fold_mul_highpart(OptContext *ctx, TCGOp *op) { - return fold_const2(ctx, op); + if (fold_const2(ctx, op) || + fold_xi_to_i(ctx, op, 0)) { + return true; + } + return false; } static bool fold_mulu2_i32(OptContext *ctx, TCGOp *op) @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) continue; } - /* Simplify expression for "op r, a, 0 => movi r, 0" cases */ - switch (opc) { - CASE_OP_32_64_VEC(and): - CASE_OP_32_64_VEC(mul): - CASE_OP_32_64(muluh): - CASE_OP_32_64(mulsh): - if (arg_is_const(op->args[2]) - && arg_info(op->args[2])->val == 0) { - tcg_opt_gen_movi(&ctx, op, op->args[0], 0); - continue; - } - break; - default: - break; - } - /* * Process each opcode. * Sorted alphabetically by opcode as much as possible. -- 2.25.1 Compute the type of the operation early. There are at least 4 places that used a def->flags ladder to determine the type of the operation being optimized. There were two places that assumed !TCG_OPF_64BIT means TCG_TYPE_I32, and so could potentially compute incorrect results for vector operations. Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 149 +++++++++++++++++++++++++++++-------------------- 1 file changed, 89 insertions(+), 60 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ typedef struct OptContext { /* In flight values from optimization. */ uint64_t z_mask; + TCGType type; } OptContext; static inline TempOptInfo *ts_info(TCGTemp *ts) @@ -XXX,XX +XXX,XX @@ static bool tcg_opt_gen_mov(OptContext *ctx, TCGOp *op, TCGArg dst, TCGArg src) { TCGTemp *dst_ts = arg_temp(dst); TCGTemp *src_ts = arg_temp(src); - const TCGOpDef *def; TempOptInfo *di; TempOptInfo *si; uint64_t z_mask; @@ -XXX,XX +XXX,XX @@ static bool tcg_opt_gen_mov(OptContext *ctx, TCGOp *op, TCGArg dst, TCGArg src) reset_ts(dst_ts); di = ts_info(dst_ts); si = ts_info(src_ts); - def = &tcg_op_defs[op->opc]; - if (def->flags & TCG_OPF_VECTOR) { - new_op = INDEX_op_mov_vec; - } else if (def->flags & TCG_OPF_64BIT) { - new_op = INDEX_op_mov_i64; - } else { + + switch (ctx->type) { + case TCG_TYPE_I32: new_op = INDEX_op_mov_i32; + break; + case TCG_TYPE_I64: + new_op = INDEX_op_mov_i64; + break; + case TCG_TYPE_V64: + case TCG_TYPE_V128: + case TCG_TYPE_V256: + /* TCGOP_VECL and TCGOP_VECE remain unchanged. */ + new_op = INDEX_op_mov_vec; + break; + default: + g_assert_not_reached(); } op->opc = new_op; - /* TCGOP_VECL and TCGOP_VECE remain unchanged. */ op->args[0] = dst; op->args[1] = src; @@ -XXX,XX +XXX,XX @@ static bool tcg_opt_gen_mov(OptContext *ctx, TCGOp *op, TCGArg dst, TCGArg src) static bool tcg_opt_gen_movi(OptContext *ctx, TCGOp *op, TCGArg dst, uint64_t val) { - const TCGOpDef *def = &tcg_op_defs[op->opc]; - TCGType type; - TCGTemp *tv; - - if (def->flags & TCG_OPF_VECTOR) { - type = TCGOP_VECL(op) + TCG_TYPE_V64; - } else if (def->flags & TCG_OPF_64BIT) { - type = TCG_TYPE_I64; - } else { - type = TCG_TYPE_I32; - } - /* Convert movi to mov with constant temp. */ - tv = tcg_constant_internal(type, val); + TCGTemp *tv = tcg_constant_internal(ctx->type, val); + init_ts_info(ctx, tv); return tcg_opt_gen_mov(ctx, op, dst, temp_arg(tv)); } @@ -XXX,XX +XXX,XX @@ static uint64_t do_constant_folding_2(TCGOpcode op, uint64_t x, uint64_t y) } } -static uint64_t do_constant_folding(TCGOpcode op, uint64_t x, uint64_t y) +static uint64_t do_constant_folding(TCGOpcode op, TCGType type, + uint64_t x, uint64_t y) { - const TCGOpDef *def = &tcg_op_defs[op]; uint64_t res = do_constant_folding_2(op, x, y); - if (!(def->flags & TCG_OPF_64BIT)) { + if (type == TCG_TYPE_I32) { res = (int32_t)res; } return res; @@ -XXX,XX +XXX,XX @@ static bool do_constant_folding_cond_eq(TCGCond c) * Return -1 if the condition can't be simplified, * and the result of the condition (0 or 1) if it can. */ -static int do_constant_folding_cond(TCGOpcode op, TCGArg x, +static int do_constant_folding_cond(TCGType type, TCGArg x, TCGArg y, TCGCond c) { uint64_t xv = arg_info(x)->val; uint64_t yv = arg_info(y)->val; if (arg_is_const(x) && arg_is_const(y)) { - const TCGOpDef *def = &tcg_op_defs[op]; - tcg_debug_assert(!(def->flags & TCG_OPF_VECTOR)); - if (def->flags & TCG_OPF_64BIT) { - return do_constant_folding_cond_64(xv, yv, c); - } else { + switch (type) { + case TCG_TYPE_I32: return do_constant_folding_cond_32(xv, yv, c); + case TCG_TYPE_I64: + return do_constant_folding_cond_64(xv, yv, c); + default: + /* Only scalar comparisons are optimizable */ + return -1; } } else if (args_are_copies(x, y)) { return do_constant_folding_cond_eq(c); @@ -XXX,XX +XXX,XX @@ static bool fold_const1(OptContext *ctx, TCGOp *op) uint64_t t; t = arg_info(op->args[1])->val; - t = do_constant_folding(op->opc, t, 0); + t = do_constant_folding(op->opc, ctx->type, t, 0); return tcg_opt_gen_movi(ctx, op, op->args[0], t); } return false; @@ -XXX,XX +XXX,XX @@ static bool fold_const2(OptContext *ctx, TCGOp *op) uint64_t t1 = arg_info(op->args[1])->val; uint64_t t2 = arg_info(op->args[2])->val; - t1 = do_constant_folding(op->opc, t1, t2); + t1 = do_constant_folding(op->opc, ctx->type, t1, t2); return tcg_opt_gen_movi(ctx, op, op->args[0], t1); } return false; @@ -XXX,XX +XXX,XX @@ static bool fold_andc(OptContext *ctx, TCGOp *op) static bool fold_brcond(OptContext *ctx, TCGOp *op) { TCGCond cond = op->args[2]; - int i = do_constant_folding_cond(op->opc, op->args[0], op->args[1], cond); + int i = do_constant_folding_cond(ctx->type, op->args[0], op->args[1], cond); if (i == 0) { tcg_op_remove(ctx->tcg, op); @@ -XXX,XX +XXX,XX @@ static bool fold_brcond2(OptContext *ctx, TCGOp *op) * Simplify EQ/NE comparisons where one of the pairs * can be simplified. */ - i = do_constant_folding_cond(INDEX_op_brcond_i32, op->args[0], + i = do_constant_folding_cond(TCG_TYPE_I32, op->args[0], op->args[2], cond); switch (i ^ inv) { case 0: @@ -XXX,XX +XXX,XX @@ static bool fold_brcond2(OptContext *ctx, TCGOp *op) goto do_brcond_high; } - i = do_constant_folding_cond(INDEX_op_brcond_i32, op->args[1], + i = do_constant_folding_cond(TCG_TYPE_I32, op->args[1], op->args[3], cond); switch (i ^ inv) { case 0: @@ -XXX,XX +XXX,XX @@ static bool fold_bswap(OptContext *ctx, TCGOp *op) if (arg_is_const(op->args[1])) { uint64_t t = arg_info(op->args[1])->val; - t = do_constant_folding(op->opc, t, op->args[2]); + t = do_constant_folding(op->opc, ctx->type, t, op->args[2]); return tcg_opt_gen_movi(ctx, op, op->args[0], t); } return false; @@ -XXX,XX +XXX,XX @@ static bool fold_count_zeros(OptContext *ctx, TCGOp *op) uint64_t t = arg_info(op->args[1])->val; if (t != 0) { - t = do_constant_folding(op->opc, t, 0); + t = do_constant_folding(op->opc, ctx->type, t, 0); return tcg_opt_gen_movi(ctx, op, op->args[0], t); } return tcg_opt_gen_mov(ctx, op, op->args[0], op->args[2]); @@ -XXX,XX +XXX,XX @@ static bool fold_mov(OptContext *ctx, TCGOp *op) static bool fold_movcond(OptContext *ctx, TCGOp *op) { - TCGOpcode opc = op->opc; TCGCond cond = op->args[5]; - int i = do_constant_folding_cond(opc, op->args[1], op->args[2], cond); + int i = do_constant_folding_cond(ctx->type, op->args[1], op->args[2], cond); if (i >= 0) { return tcg_opt_gen_mov(ctx, op, op->args[0], op->args[4 - i]); @@ -XXX,XX +XXX,XX @@ static bool fold_movcond(OptContext *ctx, TCGOp *op) if (arg_is_const(op->args[3]) && arg_is_const(op->args[4])) { uint64_t tv = arg_info(op->args[3])->val; uint64_t fv = arg_info(op->args[4])->val; + TCGOpcode opc; - opc = (opc == INDEX_op_movcond_i32 - ? INDEX_op_setcond_i32 : INDEX_op_setcond_i64); + switch (ctx->type) { + case TCG_TYPE_I32: + opc = INDEX_op_setcond_i32; + break; + case TCG_TYPE_I64: + opc = INDEX_op_setcond_i64; + break; + default: + g_assert_not_reached(); + } if (tv == 1 && fv == 0) { op->opc = opc; @@ -XXX,XX +XXX,XX @@ static bool fold_remainder(OptContext *ctx, TCGOp *op) static bool fold_setcond(OptContext *ctx, TCGOp *op) { TCGCond cond = op->args[3]; - int i = do_constant_folding_cond(op->opc, op->args[1], op->args[2], cond); + int i = do_constant_folding_cond(ctx->type, op->args[1], op->args[2], cond); if (i >= 0) { return tcg_opt_gen_movi(ctx, op, op->args[0], i); @@ -XXX,XX +XXX,XX @@ static bool fold_setcond2(OptContext *ctx, TCGOp *op) * Simplify EQ/NE comparisons where one of the pairs * can be simplified. */ - i = do_constant_folding_cond(INDEX_op_setcond_i32, op->args[1], + i = do_constant_folding_cond(TCG_TYPE_I32, op->args[1], op->args[3], cond); switch (i ^ inv) { case 0: @@ -XXX,XX +XXX,XX @@ static bool fold_setcond2(OptContext *ctx, TCGOp *op) goto do_setcond_high; } - i = do_constant_folding_cond(INDEX_op_setcond_i32, op->args[2], + i = do_constant_folding_cond(TCG_TYPE_I32, op->args[2], op->args[4], cond); switch (i ^ inv) { case 0: @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) init_arguments(&ctx, op, def->nb_oargs + def->nb_iargs); copy_propagate(&ctx, op, def->nb_oargs, def->nb_iargs); + /* Pre-compute the type of the operation. */ + if (def->flags & TCG_OPF_VECTOR) { + ctx.type = TCG_TYPE_V64 + TCGOP_VECL(op); + } else if (def->flags & TCG_OPF_64BIT) { + ctx.type = TCG_TYPE_I64; + } else { + ctx.type = TCG_TYPE_I32; + } + /* For commutative operations make constant second argument */ switch (opc) { CASE_OP_32_64_VEC(add): @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) /* Proceed with possible constant folding. */ break; } - if (opc == INDEX_op_sub_i32) { + switch (ctx.type) { + case TCG_TYPE_I32: neg_op = INDEX_op_neg_i32; have_neg = TCG_TARGET_HAS_neg_i32; - } else if (opc == INDEX_op_sub_i64) { + break; + case TCG_TYPE_I64: neg_op = INDEX_op_neg_i64; have_neg = TCG_TARGET_HAS_neg_i64; - } else if (TCG_TARGET_HAS_neg_vec) { - TCGType type = TCGOP_VECL(op) + TCG_TYPE_V64; - unsigned vece = TCGOP_VECE(op); - neg_op = INDEX_op_neg_vec; - have_neg = tcg_can_emit_vec_op(neg_op, type, vece) > 0; - } else { break; + case TCG_TYPE_V64: + case TCG_TYPE_V128: + case TCG_TYPE_V256: + neg_op = INDEX_op_neg_vec; + have_neg = tcg_can_emit_vec_op(neg_op, ctx.type, + TCGOP_VECE(op)) > 0; + break; + default: + g_assert_not_reached(); } if (!have_neg) { break; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) TCGOpcode not_op; bool have_not; - if (def->flags & TCG_OPF_VECTOR) { - not_op = INDEX_op_not_vec; - have_not = TCG_TARGET_HAS_not_vec; - } else if (def->flags & TCG_OPF_64BIT) { - not_op = INDEX_op_not_i64; - have_not = TCG_TARGET_HAS_not_i64; - } else { + switch (ctx.type) { + case TCG_TYPE_I32: not_op = INDEX_op_not_i32; have_not = TCG_TARGET_HAS_not_i32; + break; + case TCG_TYPE_I64: + not_op = INDEX_op_not_i64; + have_not = TCG_TARGET_HAS_not_i64; + break; + case TCG_TYPE_V64: + case TCG_TYPE_V128: + case TCG_TYPE_V256: + not_op = INDEX_op_not_vec; + have_not = TCG_TARGET_HAS_not_vec; + break; + default: + g_assert_not_reached(); } if (!have_not) { break; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) below, we can ignore high bits, but for further optimizations we need to record that the high bits contain garbage. */ partmask = z_mask; - if (!(def->flags & TCG_OPF_64BIT)) { + if (ctx.type == TCG_TYPE_I32) { z_mask |= ~(tcg_target_ulong)0xffffffffu; partmask &= 0xffffffffu; affected &= 0xffffffffu; -- 2.25.1 Split out the conditional conversion from a more complex logical operation to a simple NOT. Create a couple more helpers to make this easy for the outer-most logical operations. Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 158 +++++++++++++++++++++++++++---------------------- 1 file changed, 86 insertions(+), 72 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_const2(OptContext *ctx, TCGOp *op) return false; } +/* + * Convert @op to NOT, if NOT is supported by the host. + * Return true f the conversion is successful, which will still + * indicate that the processing is complete. + */ +static bool fold_not(OptContext *ctx, TCGOp *op); +static bool fold_to_not(OptContext *ctx, TCGOp *op, int idx) +{ + TCGOpcode not_op; + bool have_not; + + switch (ctx->type) { + case TCG_TYPE_I32: + not_op = INDEX_op_not_i32; + have_not = TCG_TARGET_HAS_not_i32; + break; + case TCG_TYPE_I64: + not_op = INDEX_op_not_i64; + have_not = TCG_TARGET_HAS_not_i64; + break; + case TCG_TYPE_V64: + case TCG_TYPE_V128: + case TCG_TYPE_V256: + not_op = INDEX_op_not_vec; + have_not = TCG_TARGET_HAS_not_vec; + break; + default: + g_assert_not_reached(); + } + if (have_not) { + op->opc = not_op; + op->args[1] = op->args[idx]; + return fold_not(ctx, op); + } + return false; +} + +/* If the binary operation has first argument @i, fold to NOT. */ +static bool fold_ix_to_not(OptContext *ctx, TCGOp *op, uint64_t i) +{ + if (arg_is_const(op->args[1]) && arg_info(op->args[1])->val == i) { + return fold_to_not(ctx, op, 2); + } + return false; +} + /* If the binary operation has second argument @i, fold to @i. */ static bool fold_xi_to_i(OptContext *ctx, TCGOp *op, uint64_t i) { @@ -XXX,XX +XXX,XX @@ static bool fold_xi_to_i(OptContext *ctx, TCGOp *op, uint64_t i) return false; } +/* If the binary operation has second argument @i, fold to NOT. */ +static bool fold_xi_to_not(OptContext *ctx, TCGOp *op, uint64_t i) +{ + if (arg_is_const(op->args[2]) && arg_info(op->args[2])->val == i) { + return fold_to_not(ctx, op, 1); + } + return false; +} + /* If the binary operation has both arguments equal, fold to @i. */ static bool fold_xx_to_i(OptContext *ctx, TCGOp *op, uint64_t i) { @@ -XXX,XX +XXX,XX @@ static bool fold_and(OptContext *ctx, TCGOp *op) static bool fold_andc(OptContext *ctx, TCGOp *op) { if (fold_const2(ctx, op) || - fold_xx_to_i(ctx, op, 0)) { + fold_xx_to_i(ctx, op, 0) || + fold_ix_to_not(ctx, op, -1)) { return true; } return false; @@ -XXX,XX +XXX,XX @@ static bool fold_dup2(OptContext *ctx, TCGOp *op) static bool fold_eqv(OptContext *ctx, TCGOp *op) { - return fold_const2(ctx, op); + if (fold_const2(ctx, op) || + fold_xi_to_not(ctx, op, 0)) { + return true; + } + return false; } static bool fold_extract(OptContext *ctx, TCGOp *op) @@ -XXX,XX +XXX,XX @@ static bool fold_mulu2_i32(OptContext *ctx, TCGOp *op) static bool fold_nand(OptContext *ctx, TCGOp *op) { - return fold_const2(ctx, op); + if (fold_const2(ctx, op) || + fold_xi_to_not(ctx, op, -1)) { + return true; + } + return false; } static bool fold_neg(OptContext *ctx, TCGOp *op) @@ -XXX,XX +XXX,XX @@ static bool fold_neg(OptContext *ctx, TCGOp *op) static bool fold_nor(OptContext *ctx, TCGOp *op) { - return fold_const2(ctx, op); + if (fold_const2(ctx, op) || + fold_xi_to_not(ctx, op, 0)) { + return true; + } + return false; } static bool fold_not(OptContext *ctx, TCGOp *op) { - return fold_const1(ctx, op); + if (fold_const1(ctx, op)) { + return true; + } + + /* Because of fold_to_not, we want to always return true, via finish. */ + finish_folding(ctx, op); + return true; } static bool fold_or(OptContext *ctx, TCGOp *op) @@ -XXX,XX +XXX,XX @@ static bool fold_or(OptContext *ctx, TCGOp *op) static bool fold_orc(OptContext *ctx, TCGOp *op) { - return fold_const2(ctx, op); + if (fold_const2(ctx, op) || + fold_ix_to_not(ctx, op, 0)) { + return true; + } + return false; } static bool fold_qemu_ld(OptContext *ctx, TCGOp *op) @@ -XXX,XX +XXX,XX @@ static bool fold_sub2_i32(OptContext *ctx, TCGOp *op) static bool fold_xor(OptContext *ctx, TCGOp *op) { if (fold_const2(ctx, op) || - fold_xx_to_i(ctx, op, 0)) { + fold_xx_to_i(ctx, op, 0) || + fold_xi_to_not(ctx, op, -1)) { return true; } return false; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) } } break; - CASE_OP_32_64_VEC(xor): - CASE_OP_32_64(nand): - if (!arg_is_const(op->args[1]) - && arg_is_const(op->args[2]) - && arg_info(op->args[2])->val == -1) { - i = 1; - goto try_not; - } - break; - CASE_OP_32_64(nor): - if (!arg_is_const(op->args[1]) - && arg_is_const(op->args[2]) - && arg_info(op->args[2])->val == 0) { - i = 1; - goto try_not; - } - break; - CASE_OP_32_64_VEC(andc): - if (!arg_is_const(op->args[2]) - && arg_is_const(op->args[1]) - && arg_info(op->args[1])->val == -1) { - i = 2; - goto try_not; - } - break; - CASE_OP_32_64_VEC(orc): - CASE_OP_32_64(eqv): - if (!arg_is_const(op->args[2]) - && arg_is_const(op->args[1]) - && arg_info(op->args[1])->val == 0) { - i = 2; - goto try_not; - } - break; - try_not: - { - TCGOpcode not_op; - bool have_not; - - switch (ctx.type) { - case TCG_TYPE_I32: - not_op = INDEX_op_not_i32; - have_not = TCG_TARGET_HAS_not_i32; - break; - case TCG_TYPE_I64: - not_op = INDEX_op_not_i64; - have_not = TCG_TARGET_HAS_not_i64; - break; - case TCG_TYPE_V64: - case TCG_TYPE_V128: - case TCG_TYPE_V256: - not_op = INDEX_op_not_vec; - have_not = TCG_TARGET_HAS_not_vec; - break; - default: - g_assert_not_reached(); - } - if (!have_not) { - break; - } - op->opc = not_op; - reset_temp(op->args[0]); - op->args[1] = op->args[i]; - continue; - } default: break; } -- 2.25.1 Even though there is only one user, place this more complex conversion into its own helper. Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 89 ++++++++++++++++++++++++++------------------------ 1 file changed, 47 insertions(+), 42 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_nand(OptContext *ctx, TCGOp *op) static bool fold_neg(OptContext *ctx, TCGOp *op) { - return fold_const1(ctx, op); + if (fold_const1(ctx, op)) { + return true; + } + /* + * Because of fold_sub_to_neg, we want to always return true, + * via finish_folding. + */ + finish_folding(ctx, op); + return true; } static bool fold_nor(OptContext *ctx, TCGOp *op) @@ -XXX,XX +XXX,XX @@ static bool fold_shift(OptContext *ctx, TCGOp *op) return fold_const2(ctx, op); } +static bool fold_sub_to_neg(OptContext *ctx, TCGOp *op) +{ + TCGOpcode neg_op; + bool have_neg; + + if (!arg_is_const(op->args[1]) || arg_info(op->args[1])->val != 0) { + return false; + } + + switch (ctx->type) { + case TCG_TYPE_I32: + neg_op = INDEX_op_neg_i32; + have_neg = TCG_TARGET_HAS_neg_i32; + break; + case TCG_TYPE_I64: + neg_op = INDEX_op_neg_i64; + have_neg = TCG_TARGET_HAS_neg_i64; + break; + case TCG_TYPE_V64: + case TCG_TYPE_V128: + case TCG_TYPE_V256: + neg_op = INDEX_op_neg_vec; + have_neg = (TCG_TARGET_HAS_neg_vec && + tcg_can_emit_vec_op(neg_op, ctx->type, TCGOP_VECE(op)) > 0); + break; + default: + g_assert_not_reached(); + } + if (have_neg) { + op->opc = neg_op; + op->args[1] = op->args[2]; + return fold_neg(ctx, op); + } + return false; +} + static bool fold_sub(OptContext *ctx, TCGOp *op) { if (fold_const2(ctx, op) || - fold_xx_to_i(ctx, op, 0)) { + fold_xx_to_i(ctx, op, 0) || + fold_sub_to_neg(ctx, op)) { return true; } return false; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) continue; } break; - CASE_OP_32_64_VEC(sub): - { - TCGOpcode neg_op; - bool have_neg; - - if (arg_is_const(op->args[2])) { - /* Proceed with possible constant folding. */ - break; - } - switch (ctx.type) { - case TCG_TYPE_I32: - neg_op = INDEX_op_neg_i32; - have_neg = TCG_TARGET_HAS_neg_i32; - break; - case TCG_TYPE_I64: - neg_op = INDEX_op_neg_i64; - have_neg = TCG_TARGET_HAS_neg_i64; - break; - case TCG_TYPE_V64: - case TCG_TYPE_V128: - case TCG_TYPE_V256: - neg_op = INDEX_op_neg_vec; - have_neg = tcg_can_emit_vec_op(neg_op, ctx.type, - TCGOP_VECE(op)) > 0; - break; - default: - g_assert_not_reached(); - } - if (!have_neg) { - break; - } - if (arg_is_const(op->args[1]) - && arg_info(op->args[1])->val == 0) { - op->opc = neg_op; - reset_temp(op->args[0]); - op->args[1] = op->args[2]; - continue; - } - } - break; default: break; } -- 2.25.1 Pull the "op r, a, i => mov r, a" optimization into a function, and use them in the outer-most logical operations. Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 61 +++++++++++++++++++++----------------------------- 1 file changed, 26 insertions(+), 35 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_xi_to_i(OptContext *ctx, TCGOp *op, uint64_t i) return false; } +/* If the binary operation has second argument @i, fold to identity. */ +static bool fold_xi_to_x(OptContext *ctx, TCGOp *op, uint64_t i) +{ + if (arg_is_const(op->args[2]) && arg_info(op->args[2])->val == i) { + return tcg_opt_gen_mov(ctx, op, op->args[0], op->args[1]); + } + return false; +} + /* If the binary operation has second argument @i, fold to NOT. */ static bool fold_xi_to_not(OptContext *ctx, TCGOp *op, uint64_t i) { @@ -XXX,XX +XXX,XX @@ static bool fold_xx_to_x(OptContext *ctx, TCGOp *op) static bool fold_add(OptContext *ctx, TCGOp *op) { - return fold_const2(ctx, op); + if (fold_const2(ctx, op) || + fold_xi_to_x(ctx, op, 0)) { + return true; + } + return false; } static bool fold_addsub2_i32(OptContext *ctx, TCGOp *op, bool add) @@ -XXX,XX +XXX,XX @@ static bool fold_and(OptContext *ctx, TCGOp *op) { if (fold_const2(ctx, op) || fold_xi_to_i(ctx, op, 0) || + fold_xi_to_x(ctx, op, -1) || fold_xx_to_x(ctx, op)) { return true; } @@ -XXX,XX +XXX,XX @@ static bool fold_andc(OptContext *ctx, TCGOp *op) { if (fold_const2(ctx, op) || fold_xx_to_i(ctx, op, 0) || + fold_xi_to_x(ctx, op, 0) || fold_ix_to_not(ctx, op, -1)) { return true; } @@ -XXX,XX +XXX,XX @@ static bool fold_dup2(OptContext *ctx, TCGOp *op) static bool fold_eqv(OptContext *ctx, TCGOp *op) { if (fold_const2(ctx, op) || + fold_xi_to_x(ctx, op, -1) || fold_xi_to_not(ctx, op, 0)) { return true; } @@ -XXX,XX +XXX,XX @@ static bool fold_not(OptContext *ctx, TCGOp *op) static bool fold_or(OptContext *ctx, TCGOp *op) { if (fold_const2(ctx, op) || + fold_xi_to_x(ctx, op, 0) || fold_xx_to_x(ctx, op)) { return true; } @@ -XXX,XX +XXX,XX @@ static bool fold_or(OptContext *ctx, TCGOp *op) static bool fold_orc(OptContext *ctx, TCGOp *op) { if (fold_const2(ctx, op) || + fold_xi_to_x(ctx, op, -1) || fold_ix_to_not(ctx, op, 0)) { return true; } @@ -XXX,XX +XXX,XX @@ static bool fold_sextract(OptContext *ctx, TCGOp *op) static bool fold_shift(OptContext *ctx, TCGOp *op) { - return fold_const2(ctx, op); + if (fold_const2(ctx, op) || + fold_xi_to_x(ctx, op, 0)) { + return true; + } + return false; } static bool fold_sub_to_neg(OptContext *ctx, TCGOp *op) @@ -XXX,XX +XXX,XX @@ static bool fold_sub(OptContext *ctx, TCGOp *op) { if (fold_const2(ctx, op) || fold_xx_to_i(ctx, op, 0) || + fold_xi_to_x(ctx, op, 0) || fold_sub_to_neg(ctx, op)) { return true; } @@ -XXX,XX +XXX,XX @@ static bool fold_xor(OptContext *ctx, TCGOp *op) { if (fold_const2(ctx, op) || fold_xx_to_i(ctx, op, 0) || + fold_xi_to_x(ctx, op, 0) || fold_xi_to_not(ctx, op, -1)) { return true; } @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) break; } - /* Simplify expression for "op r, a, const => mov r, a" cases */ - switch (opc) { - CASE_OP_32_64_VEC(add): - CASE_OP_32_64_VEC(sub): - CASE_OP_32_64_VEC(or): - CASE_OP_32_64_VEC(xor): - CASE_OP_32_64_VEC(andc): - CASE_OP_32_64(shl): - CASE_OP_32_64(shr): - CASE_OP_32_64(sar): - CASE_OP_32_64(rotl): - CASE_OP_32_64(rotr): - if (!arg_is_const(op->args[1]) - && arg_is_const(op->args[2]) - && arg_info(op->args[2])->val == 0) { - tcg_opt_gen_mov(&ctx, op, op->args[0], op->args[1]); - continue; - } - break; - CASE_OP_32_64_VEC(and): - CASE_OP_32_64_VEC(orc): - CASE_OP_32_64(eqv): - if (!arg_is_const(op->args[1]) - && arg_is_const(op->args[2]) - && arg_info(op->args[2])->val == -1) { - tcg_opt_gen_mov(&ctx, op, op->args[0], op->args[1]); - continue; - } - break; - default: - break; - } - /* Simplify using known-zero bits. Currently only ops with a single output argument is supported. */ z_mask = -1; -- 2.25.1 Pull the "op r, 0, b => movi r, 0" optimization into a function, and use it in fold_shift. Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 28 ++++++++++------------------ 1 file changed, 10 insertions(+), 18 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_to_not(OptContext *ctx, TCGOp *op, int idx) return false; } +/* If the binary operation has first argument @i, fold to @i. */ +static bool fold_ix_to_i(OptContext *ctx, TCGOp *op, uint64_t i) +{ + if (arg_is_const(op->args[1]) && arg_info(op->args[1])->val == i) { + return tcg_opt_gen_movi(ctx, op, op->args[0], i); + } + return false; +} + /* If the binary operation has first argument @i, fold to NOT. */ static bool fold_ix_to_not(OptContext *ctx, TCGOp *op, uint64_t i) { @@ -XXX,XX +XXX,XX @@ static bool fold_sextract(OptContext *ctx, TCGOp *op) static bool fold_shift(OptContext *ctx, TCGOp *op) { if (fold_const2(ctx, op) || + fold_ix_to_i(ctx, op, 0) || fold_xi_to_x(ctx, op, 0)) { return true; } @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) break; } - /* Simplify expressions for "shift/rot r, 0, a => movi r, 0", - and "sub r, 0, a => neg r, a" case. */ - switch (opc) { - CASE_OP_32_64(shl): - CASE_OP_32_64(shr): - CASE_OP_32_64(sar): - CASE_OP_32_64(rotl): - CASE_OP_32_64(rotr): - if (arg_is_const(op->args[1]) - && arg_info(op->args[1])->val == 0) { - tcg_opt_gen_movi(&ctx, op, op->args[0], 0); - continue; - } - break; - default: - break; - } - /* Simplify using known-zero bits. Currently only ops with a single output argument is supported. */ z_mask = -1; -- 2.25.1 Move all of the known-zero optimizations into the per-opcode functions. Use fold_masks when there is a possibility of the result being determined, and simply set ctx->z_mask otherwise. Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 545 ++++++++++++++++++++++++++----------------------- 1 file changed, 294 insertions(+), 251 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ typedef struct OptContext { TCGTempSet temps_used; /* In flight values from optimization. */ - uint64_t z_mask; + uint64_t a_mask; /* mask bit is 0 iff value identical to first input */ + uint64_t z_mask; /* mask bit is 0 iff value bit is 0 */ TCGType type; } OptContext; @@ -XXX,XX +XXX,XX @@ static bool fold_const2(OptContext *ctx, TCGOp *op) return false; } +static bool fold_masks(OptContext *ctx, TCGOp *op) +{ + uint64_t a_mask = ctx->a_mask; + uint64_t z_mask = ctx->z_mask; + + /* + * 32-bit ops generate 32-bit results. For the result is zero test + * below, we can ignore high bits, but for further optimizations we + * need to record that the high bits contain garbage. + */ + if (ctx->type == TCG_TYPE_I32) { + ctx->z_mask |= MAKE_64BIT_MASK(32, 32); + a_mask &= MAKE_64BIT_MASK(0, 32); + z_mask &= MAKE_64BIT_MASK(0, 32); + } + + if (z_mask == 0) { + return tcg_opt_gen_movi(ctx, op, op->args[0], 0); + } + if (a_mask == 0) { + return tcg_opt_gen_mov(ctx, op, op->args[0], op->args[1]); + } + return false; +} + /* * Convert @op to NOT, if NOT is supported by the host. * Return true f the conversion is successful, which will still @@ -XXX,XX +XXX,XX @@ static bool fold_add2_i32(OptContext *ctx, TCGOp *op) static bool fold_and(OptContext *ctx, TCGOp *op) { + uint64_t z1, z2; + if (fold_const2(ctx, op) || fold_xi_to_i(ctx, op, 0) || fold_xi_to_x(ctx, op, -1) || fold_xx_to_x(ctx, op)) { return true; } - return false; + + z1 = arg_info(op->args[1])->z_mask; + z2 = arg_info(op->args[2])->z_mask; + ctx->z_mask = z1 & z2; + + /* + * Known-zeros does not imply known-ones. Therefore unless + * arg2 is constant, we can't infer affected bits from it. + */ + if (arg_is_const(op->args[2])) { + ctx->a_mask = z1 & ~z2; + } + + return fold_masks(ctx, op); } static bool fold_andc(OptContext *ctx, TCGOp *op) { + uint64_t z1; + if (fold_const2(ctx, op) || fold_xx_to_i(ctx, op, 0) || fold_xi_to_x(ctx, op, 0) || fold_ix_to_not(ctx, op, -1)) { return true; } - return false; + + z1 = arg_info(op->args[1])->z_mask; + + /* + * Known-zeros does not imply known-ones. Therefore unless + * arg2 is constant, we can't infer anything from it. + */ + if (arg_is_const(op->args[2])) { + uint64_t z2 = ~arg_info(op->args[2])->z_mask; + ctx->a_mask = z1 & ~z2; + z1 &= z2; + } + ctx->z_mask = z1; + + return fold_masks(ctx, op); } static bool fold_brcond(OptContext *ctx, TCGOp *op) @@ -XXX,XX +XXX,XX @@ static bool fold_brcond2(OptContext *ctx, TCGOp *op) static bool fold_bswap(OptContext *ctx, TCGOp *op) { + uint64_t z_mask, sign; + if (arg_is_const(op->args[1])) { uint64_t t = arg_info(op->args[1])->val; t = do_constant_folding(op->opc, ctx->type, t, op->args[2]); return tcg_opt_gen_movi(ctx, op, op->args[0], t); } - return false; + + z_mask = arg_info(op->args[1])->z_mask; + switch (op->opc) { + case INDEX_op_bswap16_i32: + case INDEX_op_bswap16_i64: + z_mask = bswap16(z_mask); + sign = INT16_MIN; + break; + case INDEX_op_bswap32_i32: + case INDEX_op_bswap32_i64: + z_mask = bswap32(z_mask); + sign = INT32_MIN; + break; + case INDEX_op_bswap64_i64: + z_mask = bswap64(z_mask); + sign = INT64_MIN; + break; + default: + g_assert_not_reached(); + } + + switch (op->args[2] & (TCG_BSWAP_OZ | TCG_BSWAP_OS)) { + case TCG_BSWAP_OZ: + break; + case TCG_BSWAP_OS: + /* If the sign bit may be 1, force all the bits above to 1. */ + if (z_mask & sign) { + z_mask |= sign; + } + break; + default: + /* The high bits are undefined: force all bits above the sign to 1. */ + z_mask |= sign << 1; + break; + } + ctx->z_mask = z_mask; + + return fold_masks(ctx, op); } static bool fold_call(OptContext *ctx, TCGOp *op) @@ -XXX,XX +XXX,XX @@ static bool fold_call(OptContext *ctx, TCGOp *op) static bool fold_count_zeros(OptContext *ctx, TCGOp *op) { + uint64_t z_mask; + if (arg_is_const(op->args[1])) { uint64_t t = arg_info(op->args[1])->val; @@ -XXX,XX +XXX,XX @@ static bool fold_count_zeros(OptContext *ctx, TCGOp *op) } return tcg_opt_gen_mov(ctx, op, op->args[0], op->args[2]); } + + switch (ctx->type) { + case TCG_TYPE_I32: + z_mask = 31; + break; + case TCG_TYPE_I64: + z_mask = 63; + break; + default: + g_assert_not_reached(); + } + ctx->z_mask = arg_info(op->args[2])->z_mask | z_mask; + return false; } static bool fold_ctpop(OptContext *ctx, TCGOp *op) { - return fold_const1(ctx, op); + if (fold_const1(ctx, op)) { + return true; + } + + switch (ctx->type) { + case TCG_TYPE_I32: + ctx->z_mask = 32 | 31; + break; + case TCG_TYPE_I64: + ctx->z_mask = 64 | 63; + break; + default: + g_assert_not_reached(); + } + return false; } static bool fold_deposit(OptContext *ctx, TCGOp *op) @@ -XXX,XX +XXX,XX @@ static bool fold_deposit(OptContext *ctx, TCGOp *op) t1 = deposit64(t1, op->args[3], op->args[4], t2); return tcg_opt_gen_movi(ctx, op, op->args[0], t1); } + + ctx->z_mask = deposit64(arg_info(op->args[1])->z_mask, + op->args[3], op->args[4], + arg_info(op->args[2])->z_mask); return false; } @@ -XXX,XX +XXX,XX @@ static bool fold_eqv(OptContext *ctx, TCGOp *op) static bool fold_extract(OptContext *ctx, TCGOp *op) { + uint64_t z_mask_old, z_mask; + if (arg_is_const(op->args[1])) { uint64_t t; @@ -XXX,XX +XXX,XX @@ static bool fold_extract(OptContext *ctx, TCGOp *op) t = extract64(t, op->args[2], op->args[3]); return tcg_opt_gen_movi(ctx, op, op->args[0], t); } - return false; + + z_mask_old = arg_info(op->args[1])->z_mask; + z_mask = extract64(z_mask_old, op->args[2], op->args[3]); + if (op->args[2] == 0) { + ctx->a_mask = z_mask_old ^ z_mask; + } + ctx->z_mask = z_mask; + + return fold_masks(ctx, op); } static bool fold_extract2(OptContext *ctx, TCGOp *op) @@ -XXX,XX +XXX,XX @@ static bool fold_extract2(OptContext *ctx, TCGOp *op) static bool fold_exts(OptContext *ctx, TCGOp *op) { - return fold_const1(ctx, op); + uint64_t z_mask_old, z_mask, sign; + bool type_change = false; + + if (fold_const1(ctx, op)) { + return true; + } + + z_mask_old = z_mask = arg_info(op->args[1])->z_mask; + + switch (op->opc) { + CASE_OP_32_64(ext8s): + sign = INT8_MIN; + z_mask = (uint8_t)z_mask; + break; + CASE_OP_32_64(ext16s): + sign = INT16_MIN; + z_mask = (uint16_t)z_mask; + break; + case INDEX_op_ext_i32_i64: + type_change = true; + QEMU_FALLTHROUGH; + case INDEX_op_ext32s_i64: + sign = INT32_MIN; + z_mask = (uint32_t)z_mask; + break; + default: + g_assert_not_reached(); + } + + if (z_mask & sign) { + z_mask |= sign; + } else if (!type_change) { + ctx->a_mask = z_mask_old ^ z_mask; + } + ctx->z_mask = z_mask; + + return fold_masks(ctx, op); } static bool fold_extu(OptContext *ctx, TCGOp *op) { - return fold_const1(ctx, op); + uint64_t z_mask_old, z_mask; + bool type_change = false; + + if (fold_const1(ctx, op)) { + return true; + } + + z_mask_old = z_mask = arg_info(op->args[1])->z_mask; + + switch (op->opc) { + CASE_OP_32_64(ext8u): + z_mask = (uint8_t)z_mask; + break; + CASE_OP_32_64(ext16u): + z_mask = (uint16_t)z_mask; + break; + case INDEX_op_extrl_i64_i32: + case INDEX_op_extu_i32_i64: + type_change = true; + QEMU_FALLTHROUGH; + case INDEX_op_ext32u_i64: + z_mask = (uint32_t)z_mask; + break; + case INDEX_op_extrh_i64_i32: + type_change = true; + z_mask >>= 32; + break; + default: + g_assert_not_reached(); + } + + ctx->z_mask = z_mask; + if (!type_change) { + ctx->a_mask = z_mask_old ^ z_mask; + } + return fold_masks(ctx, op); } static bool fold_mb(OptContext *ctx, TCGOp *op) @@ -XXX,XX +XXX,XX @@ static bool fold_movcond(OptContext *ctx, TCGOp *op) return tcg_opt_gen_mov(ctx, op, op->args[0], op->args[4 - i]); } + ctx->z_mask = arg_info(op->args[3])->z_mask + | arg_info(op->args[4])->z_mask; + if (arg_is_const(op->args[3]) && arg_is_const(op->args[4])) { uint64_t tv = arg_info(op->args[3])->val; uint64_t fv = arg_info(op->args[4])->val; @@ -XXX,XX +XXX,XX @@ static bool fold_nand(OptContext *ctx, TCGOp *op) static bool fold_neg(OptContext *ctx, TCGOp *op) { + uint64_t z_mask; + if (fold_const1(ctx, op)) { return true; } + + /* Set to 1 all bits to the left of the rightmost. */ + z_mask = arg_info(op->args[1])->z_mask; + ctx->z_mask = -(z_mask & -z_mask); + /* * Because of fold_sub_to_neg, we want to always return true, * via finish_folding. @@ -XXX,XX +XXX,XX @@ static bool fold_or(OptContext *ctx, TCGOp *op) fold_xx_to_x(ctx, op)) { return true; } - return false; + + ctx->z_mask = arg_info(op->args[1])->z_mask + | arg_info(op->args[2])->z_mask; + return fold_masks(ctx, op); } static bool fold_orc(OptContext *ctx, TCGOp *op) @@ -XXX,XX +XXX,XX @@ static bool fold_orc(OptContext *ctx, TCGOp *op) static bool fold_qemu_ld(OptContext *ctx, TCGOp *op) { + const TCGOpDef *def = &tcg_op_defs[op->opc]; + MemOpIdx oi = op->args[def->nb_oargs + def->nb_iargs]; + MemOp mop = get_memop(oi); + int width = 8 * memop_size(mop); + + if (!(mop & MO_SIGN) && width < 64) { + ctx->z_mask = MAKE_64BIT_MASK(0, width); + } + /* Opcodes that touch guest memory stop the mb optimization. */ ctx->prev_mb = NULL; return false; @@ -XXX,XX +XXX,XX @@ static bool fold_setcond(OptContext *ctx, TCGOp *op) if (i >= 0) { return tcg_opt_gen_movi(ctx, op, op->args[0], i); } + + ctx->z_mask = 1; return false; } @@ -XXX,XX +XXX,XX @@ static bool fold_setcond2(OptContext *ctx, TCGOp *op) op->opc = INDEX_op_setcond_i32; break; } + + ctx->z_mask = 1; return false; do_setcond_const: @@ -XXX,XX +XXX,XX @@ static bool fold_setcond2(OptContext *ctx, TCGOp *op) static bool fold_sextract(OptContext *ctx, TCGOp *op) { + int64_t z_mask_old, z_mask; + if (arg_is_const(op->args[1])) { uint64_t t; @@ -XXX,XX +XXX,XX @@ static bool fold_sextract(OptContext *ctx, TCGOp *op) t = sextract64(t, op->args[2], op->args[3]); return tcg_opt_gen_movi(ctx, op, op->args[0], t); } - return false; + + z_mask_old = arg_info(op->args[1])->z_mask; + z_mask = sextract64(z_mask_old, op->args[2], op->args[3]); + if (op->args[2] == 0 && z_mask >= 0) { + ctx->a_mask = z_mask_old ^ z_mask; + } + ctx->z_mask = z_mask; + + return fold_masks(ctx, op); } static bool fold_shift(OptContext *ctx, TCGOp *op) @@ -XXX,XX +XXX,XX @@ static bool fold_shift(OptContext *ctx, TCGOp *op) fold_xi_to_x(ctx, op, 0)) { return true; } + + if (arg_is_const(op->args[2])) { + ctx->z_mask = do_constant_folding(op->opc, ctx->type, + arg_info(op->args[1])->z_mask, + arg_info(op->args[2])->val); + return fold_masks(ctx, op); + } return false; } @@ -XXX,XX +XXX,XX @@ static bool fold_sub2_i32(OptContext *ctx, TCGOp *op) return fold_addsub2_i32(ctx, op, false); } +static bool fold_tcg_ld(OptContext *ctx, TCGOp *op) +{ + /* We can't do any folding with a load, but we can record bits. */ + switch (op->opc) { + CASE_OP_32_64(ld8u): + ctx->z_mask = MAKE_64BIT_MASK(0, 8); + break; + CASE_OP_32_64(ld16u): + ctx->z_mask = MAKE_64BIT_MASK(0, 16); + break; + case INDEX_op_ld32u_i64: + ctx->z_mask = MAKE_64BIT_MASK(0, 32); + break; + default: + g_assert_not_reached(); + } + return false; +} + static bool fold_xor(OptContext *ctx, TCGOp *op) { if (fold_const2(ctx, op) || @@ -XXX,XX +XXX,XX @@ static bool fold_xor(OptContext *ctx, TCGOp *op) fold_xi_to_not(ctx, op, -1)) { return true; } - return false; + + ctx->z_mask = arg_info(op->args[1])->z_mask + | arg_info(op->args[2])->z_mask; + return fold_masks(ctx, op); } /* Propagate constants and copies, fold constant expressions. */ @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) } QTAILQ_FOREACH_SAFE(op, &s->ops, link, op_next) { - uint64_t z_mask, partmask, affected, tmp; TCGOpcode opc = op->opc; const TCGOpDef *def; bool done = false; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) break; } - /* Simplify using known-zero bits. Currently only ops with a single - output argument is supported. */ - z_mask = -1; - affected = -1; - switch (opc) { - CASE_OP_32_64(ext8s): - if ((arg_info(op->args[1])->z_mask & 0x80) != 0) { - break; - } - QEMU_FALLTHROUGH; - CASE_OP_32_64(ext8u): - z_mask = 0xff; - goto and_const; - CASE_OP_32_64(ext16s): - if ((arg_info(op->args[1])->z_mask & 0x8000) != 0) { - break; - } - QEMU_FALLTHROUGH; - CASE_OP_32_64(ext16u): - z_mask = 0xffff; - goto and_const; - case INDEX_op_ext32s_i64: - if ((arg_info(op->args[1])->z_mask & 0x80000000) != 0) { - break; - } - QEMU_FALLTHROUGH; - case INDEX_op_ext32u_i64: - z_mask = 0xffffffffU; - goto and_const; - - CASE_OP_32_64(and): - z_mask = arg_info(op->args[2])->z_mask; - if (arg_is_const(op->args[2])) { - and_const: - affected = arg_info(op->args[1])->z_mask & ~z_mask; - } - z_mask = arg_info(op->args[1])->z_mask & z_mask; - break; - - case INDEX_op_ext_i32_i64: - if ((arg_info(op->args[1])->z_mask & 0x80000000) != 0) { - break; - } - QEMU_FALLTHROUGH; - case INDEX_op_extu_i32_i64: - /* We do not compute affected as it is a size changing op. */ - z_mask = (uint32_t)arg_info(op->args[1])->z_mask; - break; - - CASE_OP_32_64(andc): - /* Known-zeros does not imply known-ones. Therefore unless - op->args[2] is constant, we can't infer anything from it. */ - if (arg_is_const(op->args[2])) { - z_mask = ~arg_info(op->args[2])->z_mask; - goto and_const; - } - /* But we certainly know nothing outside args[1] may be set. */ - z_mask = arg_info(op->args[1])->z_mask; - break; - - case INDEX_op_sar_i32: - if (arg_is_const(op->args[2])) { - tmp = arg_info(op->args[2])->val & 31; - z_mask = (int32_t)arg_info(op->args[1])->z_mask >> tmp; - } - break; - case INDEX_op_sar_i64: - if (arg_is_const(op->args[2])) { - tmp = arg_info(op->args[2])->val & 63; - z_mask = (int64_t)arg_info(op->args[1])->z_mask >> tmp; - } - break; - - case INDEX_op_shr_i32: - if (arg_is_const(op->args[2])) { - tmp = arg_info(op->args[2])->val & 31; - z_mask = (uint32_t)arg_info(op->args[1])->z_mask >> tmp; - } - break; - case INDEX_op_shr_i64: - if (arg_is_const(op->args[2])) { - tmp = arg_info(op->args[2])->val & 63; - z_mask = (uint64_t)arg_info(op->args[1])->z_mask >> tmp; - } - break; - - case INDEX_op_extrl_i64_i32: - z_mask = (uint32_t)arg_info(op->args[1])->z_mask; - break; - case INDEX_op_extrh_i64_i32: - z_mask = (uint64_t)arg_info(op->args[1])->z_mask >> 32; - break; - - CASE_OP_32_64(shl): - if (arg_is_const(op->args[2])) { - tmp = arg_info(op->args[2])->val & (TCG_TARGET_REG_BITS - 1); - z_mask = arg_info(op->args[1])->z_mask << tmp; - } - break; - - CASE_OP_32_64(neg): - /* Set to 1 all bits to the left of the rightmost. */ - z_mask = -(arg_info(op->args[1])->z_mask - & -arg_info(op->args[1])->z_mask); - break; - - CASE_OP_32_64(deposit): - z_mask = deposit64(arg_info(op->args[1])->z_mask, - op->args[3], op->args[4], - arg_info(op->args[2])->z_mask); - break; - - CASE_OP_32_64(extract): - z_mask = extract64(arg_info(op->args[1])->z_mask, - op->args[2], op->args[3]); - if (op->args[2] == 0) { - affected = arg_info(op->args[1])->z_mask & ~z_mask; - } - break; - CASE_OP_32_64(sextract): - z_mask = sextract64(arg_info(op->args[1])->z_mask, - op->args[2], op->args[3]); - if (op->args[2] == 0 && (tcg_target_long)z_mask >= 0) { - affected = arg_info(op->args[1])->z_mask & ~z_mask; - } - break; - - CASE_OP_32_64(or): - CASE_OP_32_64(xor): - z_mask = arg_info(op->args[1])->z_mask - | arg_info(op->args[2])->z_mask; - break; - - case INDEX_op_clz_i32: - case INDEX_op_ctz_i32: - z_mask = arg_info(op->args[2])->z_mask | 31; - break; - - case INDEX_op_clz_i64: - case INDEX_op_ctz_i64: - z_mask = arg_info(op->args[2])->z_mask | 63; - break; - - case INDEX_op_ctpop_i32: - z_mask = 32 | 31; - break; - case INDEX_op_ctpop_i64: - z_mask = 64 | 63; - break; - - CASE_OP_32_64(setcond): - case INDEX_op_setcond2_i32: - z_mask = 1; - break; - - CASE_OP_32_64(movcond): - z_mask = arg_info(op->args[3])->z_mask - | arg_info(op->args[4])->z_mask; - break; - - CASE_OP_32_64(ld8u): - z_mask = 0xff; - break; - CASE_OP_32_64(ld16u): - z_mask = 0xffff; - break; - case INDEX_op_ld32u_i64: - z_mask = 0xffffffffu; - break; - - CASE_OP_32_64(qemu_ld): - { - MemOpIdx oi = op->args[def->nb_oargs + def->nb_iargs]; - MemOp mop = get_memop(oi); - if (!(mop & MO_SIGN)) { - z_mask = (2ULL << ((8 << (mop & MO_SIZE)) - 1)) - 1; - } - } - break; - - CASE_OP_32_64(bswap16): - z_mask = arg_info(op->args[1])->z_mask; - if (z_mask <= 0xffff) { - op->args[2] |= TCG_BSWAP_IZ; - } - z_mask = bswap16(z_mask); - switch (op->args[2] & (TCG_BSWAP_OZ | TCG_BSWAP_OS)) { - case TCG_BSWAP_OZ: - break; - case TCG_BSWAP_OS: - z_mask = (int16_t)z_mask; - break; - default: /* undefined high bits */ - z_mask |= MAKE_64BIT_MASK(16, 48); - break; - } - break; - - case INDEX_op_bswap32_i64: - z_mask = arg_info(op->args[1])->z_mask; - if (z_mask <= 0xffffffffu) { - op->args[2] |= TCG_BSWAP_IZ; - } - z_mask = bswap32(z_mask); - switch (op->args[2] & (TCG_BSWAP_OZ | TCG_BSWAP_OS)) { - case TCG_BSWAP_OZ: - break; - case TCG_BSWAP_OS: - z_mask = (int32_t)z_mask; - break; - default: /* undefined high bits */ - z_mask |= MAKE_64BIT_MASK(32, 32); - break; - } - break; - - default: - break; - } - - /* 32-bit ops generate 32-bit results. For the result is zero test - below, we can ignore high bits, but for further optimizations we - need to record that the high bits contain garbage. */ - partmask = z_mask; - if (ctx.type == TCG_TYPE_I32) { - z_mask |= ~(tcg_target_ulong)0xffffffffu; - partmask &= 0xffffffffu; - affected &= 0xffffffffu; - } - ctx.z_mask = z_mask; - - if (partmask == 0) { - tcg_opt_gen_movi(&ctx, op, op->args[0], 0); - continue; - } - if (affected == 0) { - tcg_opt_gen_mov(&ctx, op, op->args[0], op->args[1]); - continue; - } + /* Assume all bits affected, and no bits known zero. */ + ctx.a_mask = -1; + ctx.z_mask = -1; /* * Process each opcode. @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) case INDEX_op_extrh_i64_i32: done = fold_extu(&ctx, op); break; + CASE_OP_32_64(ld8u): + CASE_OP_32_64(ld16u): + case INDEX_op_ld32u_i64: + done = fold_tcg_ld(&ctx, op); + break; case INDEX_op_mb: done = fold_mb(&ctx, op); break; -- 2.25.1 Rename to fold_multiply2, and handle muls2_i32, mulu2_i64, and muls2_i64. Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 44 +++++++++++++++++++++++++++++++++++--------- 1 file changed, 35 insertions(+), 9 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_mul_highpart(OptContext *ctx, TCGOp *op) return false; } -static bool fold_mulu2_i32(OptContext *ctx, TCGOp *op) +static bool fold_multiply2(OptContext *ctx, TCGOp *op) { if (arg_is_const(op->args[2]) && arg_is_const(op->args[3])) { - uint32_t a = arg_info(op->args[2])->val; - uint32_t b = arg_info(op->args[3])->val; - uint64_t r = (uint64_t)a * b; + uint64_t a = arg_info(op->args[2])->val; + uint64_t b = arg_info(op->args[3])->val; + uint64_t h, l; TCGArg rl, rh; - TCGOp *op2 = tcg_op_insert_before(ctx->tcg, op, INDEX_op_mov_i32); + TCGOp *op2; + + switch (op->opc) { + case INDEX_op_mulu2_i32: + l = (uint64_t)(uint32_t)a * (uint32_t)b; + h = (int32_t)(l >> 32); + l = (int32_t)l; + break; + case INDEX_op_muls2_i32: + l = (int64_t)(int32_t)a * (int32_t)b; + h = l >> 32; + l = (int32_t)l; + break; + case INDEX_op_mulu2_i64: + mulu64(&l, &h, a, b); + break; + case INDEX_op_muls2_i64: + muls64(&l, &h, a, b); + break; + default: + g_assert_not_reached(); + } rl = op->args[0]; rh = op->args[1]; - tcg_opt_gen_movi(ctx, op, rl, (int32_t)r); - tcg_opt_gen_movi(ctx, op2, rh, (int32_t)(r >> 32)); + + /* The proper opcode is supplied by tcg_opt_gen_mov. */ + op2 = tcg_op_insert_before(ctx->tcg, op, 0); + + tcg_opt_gen_movi(ctx, op, rl, l); + tcg_opt_gen_movi(ctx, op2, rh, h); return true; } return false; @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) CASE_OP_32_64(muluh): done = fold_mul_highpart(&ctx, op); break; - case INDEX_op_mulu2_i32: - done = fold_mulu2_i32(&ctx, op); + CASE_OP_32_64(muls2): + CASE_OP_32_64(mulu2): + done = fold_multiply2(&ctx, op); break; CASE_OP_32_64(nand): done = fold_nand(&ctx, op); -- 2.25.1 Rename to fold_addsub2. Use Int128 to implement the wider operation. Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 65 ++++++++++++++++++++++++++++++++++---------------- 1 file changed, 44 insertions(+), 21 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ */ #include "qemu/osdep.h" +#include "qemu/int128.h" #include "tcg/tcg-op.h" #include "tcg-internal.h" @@ -XXX,XX +XXX,XX @@ static bool fold_add(OptContext *ctx, TCGOp *op) return false; } -static bool fold_addsub2_i32(OptContext *ctx, TCGOp *op, bool add) +static bool fold_addsub2(OptContext *ctx, TCGOp *op, bool add) { if (arg_is_const(op->args[2]) && arg_is_const(op->args[3]) && arg_is_const(op->args[4]) && arg_is_const(op->args[5])) { - uint32_t al = arg_info(op->args[2])->val; - uint32_t ah = arg_info(op->args[3])->val; - uint32_t bl = arg_info(op->args[4])->val; - uint32_t bh = arg_info(op->args[5])->val; - uint64_t a = ((uint64_t)ah << 32) | al; - uint64_t b = ((uint64_t)bh << 32) | bl; + uint64_t al = arg_info(op->args[2])->val; + uint64_t ah = arg_info(op->args[3])->val; + uint64_t bl = arg_info(op->args[4])->val; + uint64_t bh = arg_info(op->args[5])->val; TCGArg rl, rh; - TCGOp *op2 = tcg_op_insert_before(ctx->tcg, op, INDEX_op_mov_i32); + TCGOp *op2; - if (add) { - a += b; + if (ctx->type == TCG_TYPE_I32) { + uint64_t a = deposit64(al, 32, 32, ah); + uint64_t b = deposit64(bl, 32, 32, bh); + + if (add) { + a += b; + } else { + a -= b; + } + + al = sextract64(a, 0, 32); + ah = sextract64(a, 32, 32); } else { - a -= b; + Int128 a = int128_make128(al, ah); + Int128 b = int128_make128(bl, bh); + + if (add) { + a = int128_add(a, b); + } else { + a = int128_sub(a, b); + } + + al = int128_getlo(a); + ah = int128_gethi(a); } rl = op->args[0]; rh = op->args[1]; - tcg_opt_gen_movi(ctx, op, rl, (int32_t)a); - tcg_opt_gen_movi(ctx, op2, rh, (int32_t)(a >> 32)); + + /* The proper opcode is supplied by tcg_opt_gen_mov. */ + op2 = tcg_op_insert_before(ctx->tcg, op, 0); + + tcg_opt_gen_movi(ctx, op, rl, al); + tcg_opt_gen_movi(ctx, op2, rh, ah); return true; } return false; } -static bool fold_add2_i32(OptContext *ctx, TCGOp *op) +static bool fold_add2(OptContext *ctx, TCGOp *op) { - return fold_addsub2_i32(ctx, op, true); + return fold_addsub2(ctx, op, true); } static bool fold_and(OptContext *ctx, TCGOp *op) @@ -XXX,XX +XXX,XX @@ static bool fold_sub(OptContext *ctx, TCGOp *op) return false; } -static bool fold_sub2_i32(OptContext *ctx, TCGOp *op) +static bool fold_sub2(OptContext *ctx, TCGOp *op) { - return fold_addsub2_i32(ctx, op, false); + return fold_addsub2(ctx, op, false); } static bool fold_tcg_ld(OptContext *ctx, TCGOp *op) @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) CASE_OP_32_64_VEC(add): done = fold_add(&ctx, op); break; - case INDEX_op_add2_i32: - done = fold_add2_i32(&ctx, op); + CASE_OP_32_64(add2): + done = fold_add2(&ctx, op); break; CASE_OP_32_64_VEC(and): done = fold_and(&ctx, op); @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) CASE_OP_32_64_VEC(sub): done = fold_sub(&ctx, op); break; - case INDEX_op_sub2_i32: - done = fold_sub2_i32(&ctx, op); + CASE_OP_32_64(sub2): + done = fold_sub2(&ctx, op); break; CASE_OP_32_64_VEC(xor): done = fold_xor(&ctx, op); -- 2.25.1 Most of these are handled by creating a fold_const2_commutative to handle all of the binary operators. The rest were already handled on a case-by-case basis in the switch, and have their own fold function in which to place the call. We now have only one major switch on TCGOpcode. Introduce NO_DEST and a block comment for swap_commutative in order to make the handling of brcond and movcond opcodes cleaner. Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 142 ++++++++++++++++++++++++------------------------- 1 file changed, 70 insertions(+), 72 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static int do_constant_folding_cond2(TCGArg *p1, TCGArg *p2, TCGCond c) return -1; } +/** + * swap_commutative: + * @dest: TCGArg of the destination argument, or NO_DEST. + * @p1: first paired argument + * @p2: second paired argument + * + * If *@p1 is a constant and *@p2 is not, swap. + * If *@p2 matches @dest, swap. + * Return true if a swap was performed. + */ + +#define NO_DEST temp_arg(NULL) + static bool swap_commutative(TCGArg dest, TCGArg *p1, TCGArg *p2) { TCGArg a1 = *p1, a2 = *p2; @@ -XXX,XX +XXX,XX @@ static bool fold_const2(OptContext *ctx, TCGOp *op) return false; } +static bool fold_const2_commutative(OptContext *ctx, TCGOp *op) +{ + swap_commutative(op->args[0], &op->args[1], &op->args[2]); + return fold_const2(ctx, op); +} + static bool fold_masks(OptContext *ctx, TCGOp *op) { uint64_t a_mask = ctx->a_mask; @@ -XXX,XX +XXX,XX @@ static bool fold_xx_to_x(OptContext *ctx, TCGOp *op) static bool fold_add(OptContext *ctx, TCGOp *op) { - if (fold_const2(ctx, op) || + if (fold_const2_commutative(ctx, op) || fold_xi_to_x(ctx, op, 0)) { return true; } @@ -XXX,XX +XXX,XX @@ static bool fold_addsub2(OptContext *ctx, TCGOp *op, bool add) static bool fold_add2(OptContext *ctx, TCGOp *op) { + /* Note that the high and low parts may be independently swapped. */ + swap_commutative(op->args[0], &op->args[2], &op->args[4]); + swap_commutative(op->args[1], &op->args[3], &op->args[5]); + return fold_addsub2(ctx, op, true); } @@ -XXX,XX +XXX,XX @@ static bool fold_and(OptContext *ctx, TCGOp *op) { uint64_t z1, z2; - if (fold_const2(ctx, op) || + if (fold_const2_commutative(ctx, op) || fold_xi_to_i(ctx, op, 0) || fold_xi_to_x(ctx, op, -1) || fold_xx_to_x(ctx, op)) { @@ -XXX,XX +XXX,XX @@ static bool fold_andc(OptContext *ctx, TCGOp *op) static bool fold_brcond(OptContext *ctx, TCGOp *op) { TCGCond cond = op->args[2]; - int i = do_constant_folding_cond(ctx->type, op->args[0], op->args[1], cond); + int i; + if (swap_commutative(NO_DEST, &op->args[0], &op->args[1])) { + op->args[2] = cond = tcg_swap_cond(cond); + } + + i = do_constant_folding_cond(ctx->type, op->args[0], op->args[1], cond); if (i == 0) { tcg_op_remove(ctx->tcg, op); return true; @@ -XXX,XX +XXX,XX @@ static bool fold_brcond(OptContext *ctx, TCGOp *op) static bool fold_brcond2(OptContext *ctx, TCGOp *op) { TCGCond cond = op->args[4]; - int i = do_constant_folding_cond2(&op->args[0], &op->args[2], cond); TCGArg label = op->args[5]; - int inv = 0; + int i, inv = 0; + if (swap_commutative2(&op->args[0], &op->args[2])) { + op->args[4] = cond = tcg_swap_cond(cond); + } + + i = do_constant_folding_cond2(&op->args[0], &op->args[2], cond); if (i >= 0) { goto do_brcond_const; } @@ -XXX,XX +XXX,XX @@ static bool fold_dup2(OptContext *ctx, TCGOp *op) static bool fold_eqv(OptContext *ctx, TCGOp *op) { - if (fold_const2(ctx, op) || + if (fold_const2_commutative(ctx, op) || fold_xi_to_x(ctx, op, -1) || fold_xi_to_not(ctx, op, 0)) { return true; @@ -XXX,XX +XXX,XX @@ static bool fold_mov(OptContext *ctx, TCGOp *op) static bool fold_movcond(OptContext *ctx, TCGOp *op) { TCGCond cond = op->args[5]; - int i = do_constant_folding_cond(ctx->type, op->args[1], op->args[2], cond); + int i; + if (swap_commutative(NO_DEST, &op->args[1], &op->args[2])) { + op->args[5] = cond = tcg_swap_cond(cond); + } + /* + * Canonicalize the "false" input reg to match the destination reg so + * that the tcg backend can implement a "move if true" operation. + */ + if (swap_commutative(op->args[0], &op->args[4], &op->args[3])) { + op->args[5] = cond = tcg_invert_cond(cond); + } + + i = do_constant_folding_cond(ctx->type, op->args[1], op->args[2], cond); if (i >= 0) { return tcg_opt_gen_mov(ctx, op, op->args[0], op->args[4 - i]); } @@ -XXX,XX +XXX,XX @@ static bool fold_mul(OptContext *ctx, TCGOp *op) static bool fold_mul_highpart(OptContext *ctx, TCGOp *op) { - if (fold_const2(ctx, op) || + if (fold_const2_commutative(ctx, op) || fold_xi_to_i(ctx, op, 0)) { return true; } @@ -XXX,XX +XXX,XX @@ static bool fold_mul_highpart(OptContext *ctx, TCGOp *op) static bool fold_multiply2(OptContext *ctx, TCGOp *op) { + swap_commutative(op->args[0], &op->args[2], &op->args[3]); + if (arg_is_const(op->args[2]) && arg_is_const(op->args[3])) { uint64_t a = arg_info(op->args[2])->val; uint64_t b = arg_info(op->args[3])->val; @@ -XXX,XX +XXX,XX @@ static bool fold_multiply2(OptContext *ctx, TCGOp *op) static bool fold_nand(OptContext *ctx, TCGOp *op) { - if (fold_const2(ctx, op) || + if (fold_const2_commutative(ctx, op) || fold_xi_to_not(ctx, op, -1)) { return true; } @@ -XXX,XX +XXX,XX @@ static bool fold_neg(OptContext *ctx, TCGOp *op) static bool fold_nor(OptContext *ctx, TCGOp *op) { - if (fold_const2(ctx, op) || + if (fold_const2_commutative(ctx, op) || fold_xi_to_not(ctx, op, 0)) { return true; } @@ -XXX,XX +XXX,XX @@ static bool fold_not(OptContext *ctx, TCGOp *op) static bool fold_or(OptContext *ctx, TCGOp *op) { - if (fold_const2(ctx, op) || + if (fold_const2_commutative(ctx, op) || fold_xi_to_x(ctx, op, 0) || fold_xx_to_x(ctx, op)) { return true; @@ -XXX,XX +XXX,XX @@ static bool fold_remainder(OptContext *ctx, TCGOp *op) static bool fold_setcond(OptContext *ctx, TCGOp *op) { TCGCond cond = op->args[3]; - int i = do_constant_folding_cond(ctx->type, op->args[1], op->args[2], cond); + int i; + if (swap_commutative(op->args[0], &op->args[1], &op->args[2])) { + op->args[3] = cond = tcg_swap_cond(cond); + } + + i = do_constant_folding_cond(ctx->type, op->args[1], op->args[2], cond); if (i >= 0) { return tcg_opt_gen_movi(ctx, op, op->args[0], i); } @@ -XXX,XX +XXX,XX @@ static bool fold_setcond(OptContext *ctx, TCGOp *op) static bool fold_setcond2(OptContext *ctx, TCGOp *op) { TCGCond cond = op->args[5]; - int i = do_constant_folding_cond2(&op->args[1], &op->args[3], cond); - int inv = 0; + int i, inv = 0; + if (swap_commutative2(&op->args[1], &op->args[3])) { + op->args[5] = cond = tcg_swap_cond(cond); + } + + i = do_constant_folding_cond2(&op->args[1], &op->args[3], cond); if (i >= 0) { goto do_setcond_const; } @@ -XXX,XX +XXX,XX @@ static bool fold_tcg_ld(OptContext *ctx, TCGOp *op) static bool fold_xor(OptContext *ctx, TCGOp *op) { - if (fold_const2(ctx, op) || + if (fold_const2_commutative(ctx, op) || fold_xx_to_i(ctx, op, 0) || fold_xi_to_x(ctx, op, 0) || fold_xi_to_not(ctx, op, -1)) { @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) ctx.type = TCG_TYPE_I32; } - /* For commutative operations make constant second argument */ - switch (opc) { - CASE_OP_32_64_VEC(add): - CASE_OP_32_64_VEC(mul): - CASE_OP_32_64_VEC(and): - CASE_OP_32_64_VEC(or): - CASE_OP_32_64_VEC(xor): - CASE_OP_32_64(eqv): - CASE_OP_32_64(nand): - CASE_OP_32_64(nor): - CASE_OP_32_64(muluh): - CASE_OP_32_64(mulsh): - swap_commutative(op->args[0], &op->args[1], &op->args[2]); - break; - CASE_OP_32_64(brcond): - if (swap_commutative(-1, &op->args[0], &op->args[1])) { - op->args[2] = tcg_swap_cond(op->args[2]); - } - break; - CASE_OP_32_64(setcond): - if (swap_commutative(op->args[0], &op->args[1], &op->args[2])) { - op->args[3] = tcg_swap_cond(op->args[3]); - } - break; - CASE_OP_32_64(movcond): - if (swap_commutative(-1, &op->args[1], &op->args[2])) { - op->args[5] = tcg_swap_cond(op->args[5]); - } - /* For movcond, we canonicalize the "false" input reg to match - the destination reg so that the tcg backend can implement - a "move if true" operation. */ - if (swap_commutative(op->args[0], &op->args[4], &op->args[3])) { - op->args[5] = tcg_invert_cond(op->args[5]); - } - break; - CASE_OP_32_64(add2): - swap_commutative(op->args[0], &op->args[2], &op->args[4]); - swap_commutative(op->args[1], &op->args[3], &op->args[5]); - break; - CASE_OP_32_64(mulu2): - CASE_OP_32_64(muls2): - swap_commutative(op->args[0], &op->args[2], &op->args[3]); - break; - case INDEX_op_brcond2_i32: - if (swap_commutative2(&op->args[0], &op->args[2])) { - op->args[4] = tcg_swap_cond(op->args[4]); - } - break; - case INDEX_op_setcond2_i32: - if (swap_commutative2(&op->args[1], &op->args[3])) { - op->args[5] = tcg_swap_cond(op->args[5]); - } - break; - default: - break; - } - /* Assume all bits affected, and no bits known zero. */ ctx.a_mask = -1; ctx.z_mask = -1; -- 2.25.1 This "garbage" setting pre-dates the addition of the type changing opcodes INDEX_op_ext_i32_i64, INDEX_op_extu_i32_i64, and INDEX_op_extr{l,h}_i64_i32. So now we have a definitive points at which to adjust z_mask to eliminate such bits from the 32-bit operands. Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 35 ++++++++++++++++------------------- 1 file changed, 16 insertions(+), 19 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static void init_ts_info(OptContext *ctx, TCGTemp *ts) ti->is_const = true; ti->val = ts->val; ti->z_mask = ts->val; - if (TCG_TARGET_REG_BITS > 32 && ts->type == TCG_TYPE_I32) { - /* High bits of a 32-bit quantity are garbage. */ - ti->z_mask |= ~0xffffffffull; - } } else { ti->is_const = false; ti->z_mask = -1; @@ -XXX,XX +XXX,XX @@ static bool tcg_opt_gen_mov(OptContext *ctx, TCGOp *op, TCGArg dst, TCGArg src) TCGTemp *src_ts = arg_temp(src); TempOptInfo *di; TempOptInfo *si; - uint64_t z_mask; TCGOpcode new_op; if (ts_are_copies(dst_ts, src_ts)) { @@ -XXX,XX +XXX,XX @@ static bool tcg_opt_gen_mov(OptContext *ctx, TCGOp *op, TCGArg dst, TCGArg src) op->args[0] = dst; op->args[1] = src; - z_mask = si->z_mask; - if (TCG_TARGET_REG_BITS > 32 && new_op == INDEX_op_mov_i32) { - /* High bits of the destination are now garbage. */ - z_mask |= ~0xffffffffull; - } - di->z_mask = z_mask; + di->z_mask = si->z_mask; if (src_ts->type == dst_ts->type) { TempOptInfo *ni = ts_info(si->next_copy); @@ -XXX,XX +XXX,XX @@ static bool tcg_opt_gen_mov(OptContext *ctx, TCGOp *op, TCGArg dst, TCGArg src) static bool tcg_opt_gen_movi(OptContext *ctx, TCGOp *op, TCGArg dst, uint64_t val) { - /* Convert movi to mov with constant temp. */ - TCGTemp *tv = tcg_constant_internal(ctx->type, val); + TCGTemp *tv; + if (ctx->type == TCG_TYPE_I32) { + val = (int32_t)val; + } + + /* Convert movi to mov with constant temp. */ + tv = tcg_constant_internal(ctx->type, val); init_ts_info(ctx, tv); return tcg_opt_gen_mov(ctx, op, dst, temp_arg(tv)); } @@ -XXX,XX +XXX,XX @@ static bool fold_masks(OptContext *ctx, TCGOp *op) uint64_t z_mask = ctx->z_mask; /* - * 32-bit ops generate 32-bit results. For the result is zero test - * below, we can ignore high bits, but for further optimizations we - * need to record that the high bits contain garbage. + * 32-bit ops generate 32-bit results, which for the purpose of + * simplifying tcg are sign-extended. Certainly that's how we + * represent our constants elsewhere. Note that the bits will + * be reset properly for a 64-bit value when encountering the + * type changing opcodes. */ if (ctx->type == TCG_TYPE_I32) { - ctx->z_mask |= MAKE_64BIT_MASK(32, 32); - a_mask &= MAKE_64BIT_MASK(0, 32); - z_mask &= MAKE_64BIT_MASK(0, 32); + a_mask = (int32_t)a_mask; + z_mask = (int32_t)z_mask; + ctx->z_mask = z_mask; } if (z_mask == 0) { -- 2.25.1 Recognize the constant function for or-complement. Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 1 + 1 file changed, 1 insertion(+) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_or(OptContext *ctx, TCGOp *op) static bool fold_orc(OptContext *ctx, TCGOp *op) { if (fold_const2(ctx, op) || + fold_xx_to_i(ctx, op, -1) || fold_xi_to_x(ctx, op, -1) || fold_ix_to_not(ctx, op, 0)) { return true; -- 2.25.1 Recognize the identity function for low-part multiply. Suggested-by: Luis Pires <luis.pires@eldorado.org.br> Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_movcond(OptContext *ctx, TCGOp *op) static bool fold_mul(OptContext *ctx, TCGOp *op) { if (fold_const2(ctx, op) || - fold_xi_to_i(ctx, op, 0)) { + fold_xi_to_i(ctx, op, 0) || + fold_xi_to_x(ctx, op, 1)) { return true; } return false; -- 2.25.1 Recognize the identity function for division. Suggested-by: Luis Pires <luis.pires@eldorado.org.br> Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_deposit(OptContext *ctx, TCGOp *op) static bool fold_divide(OptContext *ctx, TCGOp *op) { - return fold_const2(ctx, op); + if (fold_const2(ctx, op) || + fold_xi_to_x(ctx, op, 1)) { + return true; + } + return false; } static bool fold_dup(OptContext *ctx, TCGOp *op) -- 2.25.1 Recognize the constant function for remainder. Suggested-by: Luis Pires <luis.pires@eldorado.org.br> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_qemu_st(OptContext *ctx, TCGOp *op) static bool fold_remainder(OptContext *ctx, TCGOp *op) { - return fold_const2(ctx, op); + if (fold_const2(ctx, op) || + fold_xx_to_i(ctx, op, 0)) { + return true; + } + return false; } static bool fold_setcond(OptContext *ctx, TCGOp *op) -- 2.25.1 Certain targets, like riscv, produce signed 32-bit results. This can lead to lots of redundant extensions as values are manipulated. Begin by tracking only the obvious sign-extensions, and converting them to simple copies when possible. Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 123 ++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 102 insertions(+), 21 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ typedef struct TempOptInfo { TCGTemp *next_copy; uint64_t val; uint64_t z_mask; /* mask bit is 0 if and only if value bit is 0 */ + uint64_t s_mask; /* a left-aligned mask of clrsb(value) bits. */ } TempOptInfo; typedef struct OptContext { @@ -XXX,XX +XXX,XX @@ typedef struct OptContext { /* In flight values from optimization. */ uint64_t a_mask; /* mask bit is 0 iff value identical to first input */ uint64_t z_mask; /* mask bit is 0 iff value bit is 0 */ + uint64_t s_mask; /* mask of clrsb(value) bits */ TCGType type; } OptContext; +/* Calculate the smask for a specific value. */ +static uint64_t smask_from_value(uint64_t value) +{ + int rep = clrsb64(value); + return ~(~0ull >> rep); +} + +/* + * Calculate the smask for a given set of known-zeros. + * If there are lots of zeros on the left, we can consider the remainder + * an unsigned field, and thus the corresponding signed field is one bit + * larger. + */ +static uint64_t smask_from_zmask(uint64_t zmask) +{ + /* + * Only the 0 bits are significant for zmask, thus the msb itself + * must be zero, else we have no sign information. + */ + int rep = clz64(zmask); + if (rep == 0) { + return 0; + } + rep -= 1; + return ~(~0ull >> rep); +} + static inline TempOptInfo *ts_info(TCGTemp *ts) { return ts->state_ptr; @@ -XXX,XX +XXX,XX @@ static void reset_ts(TCGTemp *ts) ti->prev_copy = ts; ti->is_const = false; ti->z_mask = -1; + ti->s_mask = 0; } static void reset_temp(TCGArg arg) @@ -XXX,XX +XXX,XX @@ static void init_ts_info(OptContext *ctx, TCGTemp *ts) ti->is_const = true; ti->val = ts->val; ti->z_mask = ts->val; + ti->s_mask = smask_from_value(ts->val); } else { ti->is_const = false; ti->z_mask = -1; + ti->s_mask = 0; } } @@ -XXX,XX +XXX,XX @@ static bool tcg_opt_gen_mov(OptContext *ctx, TCGOp *op, TCGArg dst, TCGArg src) op->args[1] = src; di->z_mask = si->z_mask; + di->s_mask = si->s_mask; if (src_ts->type == dst_ts->type) { TempOptInfo *ni = ts_info(si->next_copy); @@ -XXX,XX +XXX,XX @@ static void finish_folding(OptContext *ctx, TCGOp *op) nb_oargs = def->nb_oargs; for (i = 0; i < nb_oargs; i++) { - reset_temp(op->args[i]); + TCGTemp *ts = arg_temp(op->args[i]); + reset_ts(ts); /* - * Save the corresponding known-zero bits mask for the + * Save the corresponding known-zero/sign bits mask for the * first output argument (only one supported so far). */ if (i == 0) { - arg_info(op->args[i])->z_mask = ctx->z_mask; + ts_info(ts)->z_mask = ctx->z_mask; + ts_info(ts)->s_mask = ctx->s_mask; } } } @@ -XXX,XX +XXX,XX @@ static bool fold_masks(OptContext *ctx, TCGOp *op) { uint64_t a_mask = ctx->a_mask; uint64_t z_mask = ctx->z_mask; + uint64_t s_mask = ctx->s_mask; /* * 32-bit ops generate 32-bit results, which for the purpose of @@ -XXX,XX +XXX,XX @@ static bool fold_masks(OptContext *ctx, TCGOp *op) if (ctx->type == TCG_TYPE_I32) { a_mask = (int32_t)a_mask; z_mask = (int32_t)z_mask; + s_mask |= MAKE_64BIT_MASK(32, 32); ctx->z_mask = z_mask; + ctx->s_mask = s_mask; } if (z_mask == 0) { @@ -XXX,XX +XXX,XX @@ static bool fold_brcond2(OptContext *ctx, TCGOp *op) static bool fold_bswap(OptContext *ctx, TCGOp *op) { - uint64_t z_mask, sign; + uint64_t z_mask, s_mask, sign; if (arg_is_const(op->args[1])) { uint64_t t = arg_info(op->args[1])->val; @@ -XXX,XX +XXX,XX @@ static bool fold_bswap(OptContext *ctx, TCGOp *op) } z_mask = arg_info(op->args[1])->z_mask; + switch (op->opc) { case INDEX_op_bswap16_i32: case INDEX_op_bswap16_i64: @@ -XXX,XX +XXX,XX @@ static bool fold_bswap(OptContext *ctx, TCGOp *op) default: g_assert_not_reached(); } + s_mask = smask_from_zmask(z_mask); switch (op->args[2] & (TCG_BSWAP_OZ | TCG_BSWAP_OS)) { case TCG_BSWAP_OZ: @@ -XXX,XX +XXX,XX @@ static bool fold_bswap(OptContext *ctx, TCGOp *op) /* If the sign bit may be 1, force all the bits above to 1. */ if (z_mask & sign) { z_mask |= sign; + s_mask = sign << 1; } break; default: /* The high bits are undefined: force all bits above the sign to 1. */ z_mask |= sign << 1; + s_mask = 0; break; } ctx->z_mask = z_mask; + ctx->s_mask = s_mask; return fold_masks(ctx, op); } @@ -XXX,XX +XXX,XX @@ static bool fold_eqv(OptContext *ctx, TCGOp *op) static bool fold_extract(OptContext *ctx, TCGOp *op) { uint64_t z_mask_old, z_mask; + int pos = op->args[2]; + int len = op->args[3]; if (arg_is_const(op->args[1])) { uint64_t t; t = arg_info(op->args[1])->val; - t = extract64(t, op->args[2], op->args[3]); + t = extract64(t, pos, len); return tcg_opt_gen_movi(ctx, op, op->args[0], t); } z_mask_old = arg_info(op->args[1])->z_mask; - z_mask = extract64(z_mask_old, op->args[2], op->args[3]); - if (op->args[2] == 0) { + z_mask = extract64(z_mask_old, pos, len); + if (pos == 0) { ctx->a_mask = z_mask_old ^ z_mask; } ctx->z_mask = z_mask; + ctx->s_mask = smask_from_zmask(z_mask); return fold_masks(ctx, op); } @@ -XXX,XX +XXX,XX @@ static bool fold_extract2(OptContext *ctx, TCGOp *op) static bool fold_exts(OptContext *ctx, TCGOp *op) { - uint64_t z_mask_old, z_mask, sign; + uint64_t s_mask_old, s_mask, z_mask, sign; bool type_change = false; if (fold_const1(ctx, op)) { return true; } - z_mask_old = z_mask = arg_info(op->args[1])->z_mask; + z_mask = arg_info(op->args[1])->z_mask; + s_mask = arg_info(op->args[1])->s_mask; + s_mask_old = s_mask; switch (op->opc) { CASE_OP_32_64(ext8s): @@ -XXX,XX +XXX,XX @@ static bool fold_exts(OptContext *ctx, TCGOp *op) if (z_mask & sign) { z_mask |= sign; - } else if (!type_change) { - ctx->a_mask = z_mask_old ^ z_mask; } + s_mask |= sign << 1; + ctx->z_mask = z_mask; + ctx->s_mask = s_mask; + if (!type_change) { + ctx->a_mask = s_mask & ~s_mask_old; + } return fold_masks(ctx, op); } @@ -XXX,XX +XXX,XX @@ static bool fold_extu(OptContext *ctx, TCGOp *op) } ctx->z_mask = z_mask; + ctx->s_mask = smask_from_zmask(z_mask); if (!type_change) { ctx->a_mask = z_mask_old ^ z_mask; } @@ -XXX,XX +XXX,XX @@ static bool fold_qemu_ld(OptContext *ctx, TCGOp *op) MemOp mop = get_memop(oi); int width = 8 * memop_size(mop); - if (!(mop & MO_SIGN) && width < 64) { - ctx->z_mask = MAKE_64BIT_MASK(0, width); + if (width < 64) { + ctx->s_mask = MAKE_64BIT_MASK(width, 64 - width); + if (!(mop & MO_SIGN)) { + ctx->z_mask = MAKE_64BIT_MASK(0, width); + ctx->s_mask <<= 1; + } } /* Opcodes that touch guest memory stop the mb optimization. */ @@ -XXX,XX +XXX,XX @@ static bool fold_setcond2(OptContext *ctx, TCGOp *op) static bool fold_sextract(OptContext *ctx, TCGOp *op) { - int64_t z_mask_old, z_mask; + uint64_t z_mask, s_mask, s_mask_old; + int pos = op->args[2]; + int len = op->args[3]; if (arg_is_const(op->args[1])) { uint64_t t; t = arg_info(op->args[1])->val; - t = sextract64(t, op->args[2], op->args[3]); + t = sextract64(t, pos, len); return tcg_opt_gen_movi(ctx, op, op->args[0], t); } - z_mask_old = arg_info(op->args[1])->z_mask; - z_mask = sextract64(z_mask_old, op->args[2], op->args[3]); - if (op->args[2] == 0 && z_mask >= 0) { - ctx->a_mask = z_mask_old ^ z_mask; - } + z_mask = arg_info(op->args[1])->z_mask; + z_mask = sextract64(z_mask, pos, len); ctx->z_mask = z_mask; + s_mask_old = arg_info(op->args[1])->s_mask; + s_mask = sextract64(s_mask_old, pos, len); + s_mask |= MAKE_64BIT_MASK(len, 64 - len); + ctx->s_mask = s_mask; + + if (pos == 0) { + ctx->a_mask = s_mask & ~s_mask_old; + } + return fold_masks(ctx, op); } @@ -XXX,XX +XXX,XX @@ static bool fold_tcg_ld(OptContext *ctx, TCGOp *op) { /* We can't do any folding with a load, but we can record bits. */ switch (op->opc) { + CASE_OP_32_64(ld8s): + ctx->s_mask = MAKE_64BIT_MASK(8, 56); + break; CASE_OP_32_64(ld8u): ctx->z_mask = MAKE_64BIT_MASK(0, 8); + ctx->s_mask = MAKE_64BIT_MASK(9, 55); + break; + CASE_OP_32_64(ld16s): + ctx->s_mask = MAKE_64BIT_MASK(16, 48); break; CASE_OP_32_64(ld16u): ctx->z_mask = MAKE_64BIT_MASK(0, 16); + ctx->s_mask = MAKE_64BIT_MASK(17, 47); + break; + case INDEX_op_ld32s_i64: + ctx->s_mask = MAKE_64BIT_MASK(32, 32); break; case INDEX_op_ld32u_i64: ctx->z_mask = MAKE_64BIT_MASK(0, 32); + ctx->s_mask = MAKE_64BIT_MASK(33, 31); break; default: g_assert_not_reached(); @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) ctx.type = TCG_TYPE_I32; } - /* Assume all bits affected, and no bits known zero. */ + /* Assume all bits affected, no bits known zero, no sign reps. */ ctx.a_mask = -1; ctx.z_mask = -1; + ctx.s_mask = 0; /* * Process each opcode. @@ -XXX,XX +XXX,XX @@ void tcg_optimize(TCGContext *s) case INDEX_op_extrh_i64_i32: done = fold_extu(&ctx, op); break; + CASE_OP_32_64(ld8s): CASE_OP_32_64(ld8u): + CASE_OP_32_64(ld16s): CASE_OP_32_64(ld16u): + case INDEX_op_ld32s_i64: case INDEX_op_ld32u_i64: done = fold_tcg_ld(&ctx, op); break; -- 2.25.1 Sign repetitions are perforce all identical, whether they are 1 or 0. Bitwise operations preserve the relative quantity of the repetitions. Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_and(OptContext *ctx, TCGOp *op) z2 = arg_info(op->args[2])->z_mask; ctx->z_mask = z1 & z2; + /* + * Sign repetitions are perforce all identical, whether they are 1 or 0. + * Bitwise operations preserve the relative quantity of the repetitions. + */ + ctx->s_mask = arg_info(op->args[1])->s_mask + & arg_info(op->args[2])->s_mask; + /* * Known-zeros does not imply known-ones. Therefore unless * arg2 is constant, we can't infer affected bits from it. @@ -XXX,XX +XXX,XX @@ static bool fold_andc(OptContext *ctx, TCGOp *op) } ctx->z_mask = z1; + ctx->s_mask = arg_info(op->args[1])->s_mask + & arg_info(op->args[2])->s_mask; return fold_masks(ctx, op); } @@ -XXX,XX +XXX,XX @@ static bool fold_eqv(OptContext *ctx, TCGOp *op) fold_xi_to_not(ctx, op, 0)) { return true; } + + ctx->s_mask = arg_info(op->args[1])->s_mask + & arg_info(op->args[2])->s_mask; return false; } @@ -XXX,XX +XXX,XX @@ static bool fold_movcond(OptContext *ctx, TCGOp *op) ctx->z_mask = arg_info(op->args[3])->z_mask | arg_info(op->args[4])->z_mask; + ctx->s_mask = arg_info(op->args[3])->s_mask + & arg_info(op->args[4])->s_mask; if (arg_is_const(op->args[3]) && arg_is_const(op->args[4])) { uint64_t tv = arg_info(op->args[3])->val; @@ -XXX,XX +XXX,XX @@ static bool fold_nand(OptContext *ctx, TCGOp *op) fold_xi_to_not(ctx, op, -1)) { return true; } + + ctx->s_mask = arg_info(op->args[1])->s_mask + & arg_info(op->args[2])->s_mask; return false; } @@ -XXX,XX +XXX,XX @@ static bool fold_nor(OptContext *ctx, TCGOp *op) fold_xi_to_not(ctx, op, 0)) { return true; } + + ctx->s_mask = arg_info(op->args[1])->s_mask + & arg_info(op->args[2])->s_mask; return false; } @@ -XXX,XX +XXX,XX @@ static bool fold_not(OptContext *ctx, TCGOp *op) return true; } + ctx->s_mask = arg_info(op->args[1])->s_mask; + /* Because of fold_to_not, we want to always return true, via finish. */ finish_folding(ctx, op); return true; @@ -XXX,XX +XXX,XX @@ static bool fold_or(OptContext *ctx, TCGOp *op) ctx->z_mask = arg_info(op->args[1])->z_mask | arg_info(op->args[2])->z_mask; + ctx->s_mask = arg_info(op->args[1])->s_mask + & arg_info(op->args[2])->s_mask; return fold_masks(ctx, op); } @@ -XXX,XX +XXX,XX @@ static bool fold_orc(OptContext *ctx, TCGOp *op) fold_ix_to_not(ctx, op, 0)) { return true; } + + ctx->s_mask = arg_info(op->args[1])->s_mask + & arg_info(op->args[2])->s_mask; return false; } @@ -XXX,XX +XXX,XX @@ static bool fold_xor(OptContext *ctx, TCGOp *op) ctx->z_mask = arg_info(op->args[1])->z_mask | arg_info(op->args[2])->z_mask; + ctx->s_mask = arg_info(op->args[1])->s_mask + & arg_info(op->args[2])->s_mask; return fold_masks(ctx, op); } -- 2.25.1 The result is either 0 or 1, which means that we have a 2 bit signed result, and thus 62 bits of sign. For clarity, use the smask_from_zmask function. Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_setcond(OptContext *ctx, TCGOp *op) } ctx->z_mask = 1; + ctx->s_mask = smask_from_zmask(1); return false; } @@ -XXX,XX +XXX,XX @@ static bool fold_setcond2(OptContext *ctx, TCGOp *op) } ctx->z_mask = 1; + ctx->s_mask = smask_from_zmask(1); return false; do_setcond_const: -- 2.25.1 The results are generally 6 bit unsigned values, though the count leading and trailing bits may produce any value for a zero input. Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static bool fold_count_zeros(OptContext *ctx, TCGOp *op) g_assert_not_reached(); } ctx->z_mask = arg_info(op->args[2])->z_mask | z_mask; - + ctx->s_mask = smask_from_zmask(ctx->z_mask); return false; } @@ -XXX,XX +XXX,XX @@ static bool fold_ctpop(OptContext *ctx, TCGOp *op) default: g_assert_not_reached(); } + ctx->s_mask = smask_from_zmask(ctx->z_mask); return false; } -- 2.25.1 For constant shifts, we can simply shift the s_mask. For variable shifts, we know that sar does not reduce the s_mask, which helps for sequences like ext32s_i64 t, in sar_i64 t, t, v ext32s_i64 out, t allowing the final extend to be eliminated. Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Reviewed-by: Luis Pires <luis.pires@eldorado.org.br> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/optimize.c | 50 +++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 47 insertions(+), 3 deletions(-) diff --git a/tcg/optimize.c b/tcg/optimize.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -XXX,XX +XXX,XX @@ static uint64_t smask_from_zmask(uint64_t zmask) return ~(~0ull >> rep); } +/* + * Recreate a properly left-aligned smask after manipulation. + * Some bit-shuffling, particularly shifts and rotates, may + * retain sign bits on the left, but may scatter disconnected + * sign bits on the right. Retain only what remains to the left. + */ +static uint64_t smask_from_smask(int64_t smask) +{ + /* Only the 1 bits are significant for smask */ + return smask_from_zmask(~smask); +} + static inline TempOptInfo *ts_info(TCGTemp *ts) { return ts->state_ptr; @@ -XXX,XX +XXX,XX @@ static bool fold_sextract(OptContext *ctx, TCGOp *op) static bool fold_shift(OptContext *ctx, TCGOp *op) { + uint64_t s_mask, z_mask, sign; + if (fold_const2(ctx, op) || fold_ix_to_i(ctx, op, 0) || fold_xi_to_x(ctx, op, 0)) { return true; } + s_mask = arg_info(op->args[1])->s_mask; + z_mask = arg_info(op->args[1])->z_mask; + if (arg_is_const(op->args[2])) { - ctx->z_mask = do_constant_folding(op->opc, ctx->type, - arg_info(op->args[1])->z_mask, - arg_info(op->args[2])->val); + int sh = arg_info(op->args[2])->val; + + ctx->z_mask = do_constant_folding(op->opc, ctx->type, z_mask, sh); + + s_mask = do_constant_folding(op->opc, ctx->type, s_mask, sh); + ctx->s_mask = smask_from_smask(s_mask); + return fold_masks(ctx, op); } + + switch (op->opc) { + CASE_OP_32_64(sar): + /* + * Arithmetic right shift will not reduce the number of + * input sign repetitions. + */ + ctx->s_mask = s_mask; + break; + CASE_OP_32_64(shr): + /* + * If the sign bit is known zero, then logical right shift + * will not reduced the number of input sign repetitions. + */ + sign = (s_mask & -s_mask) >> 1; + if (!(z_mask & sign)) { + ctx->s_mask = s_mask; + } + break; + default: + break; + } + return false; } -- 2.25.1 The following changes since commit 75d30fde55485b965a1168a21d016dd07b50ed32: Merge tag 'block-pull-request' of https://gitlab.com/stefanha/qemu into staging (2022-10-30 15:07:25 -0400) are available in the Git repository at: https://gitlab.com/rth7680/qemu.git tags/pull-tcg-20221031 for you to fetch changes up to cb375590983fc3d23600d02ba05a05d34fe44150: target/i386: Expand eflags updates inline (2022-10-31 11:39:10 +1100) ---------------------------------------------------------------- Remove sparc32plus support from tcg/sparc. target/i386: Use cpu_unwind_state_data for tpr access. target/i386: Expand eflags updates inline ---------------------------------------------------------------- Icenowy Zheng (1): tcg/tci: fix logic error when registering helpers via FFI Richard Henderson (10): tcg/sparc: Remove support for sparc32plus tcg/sparc64: Rename from tcg/sparc tcg/sparc64: Remove sparc32plus constraints accel/tcg: Introduce cpu_unwind_state_data target/i386: Use cpu_unwind_state_data for tpr access target/openrisc: Always exit after mtspr npc target/openrisc: Use cpu_unwind_state_data for mfspr accel/tcg: Remove will_exit argument from cpu_restore_state accel/tcg: Remove reset_icount argument from cpu_restore_state_from_tb target/i386: Expand eflags updates inline meson.build | 4 +- accel/tcg/internal.h | 4 +- include/exec/exec-all.h | 24 ++- target/i386/helper.h | 5 - tcg/{sparc => sparc64}/tcg-target-con-set.h | 16 +- tcg/{sparc => sparc64}/tcg-target-con-str.h | 3 - tcg/{sparc => sparc64}/tcg-target.h | 11 -- accel/tcg/cpu-exec-common.c | 2 +- accel/tcg/tb-maint.c | 4 +- accel/tcg/translate-all.c | 91 +++++---- target/alpha/helper.c | 2 +- target/alpha/mem_helper.c | 2 +- target/arm/op_helper.c | 2 +- target/arm/tlb_helper.c | 8 +- target/cris/helper.c | 2 +- target/i386/helper.c | 21 ++- target/i386/tcg/cc_helper.c | 41 ----- target/i386/tcg/sysemu/svm_helper.c | 2 +- target/i386/tcg/translate.c | 30 ++- target/m68k/op_helper.c | 4 +- target/microblaze/helper.c | 2 +- target/nios2/op_helper.c | 2 +- target/openrisc/sys_helper.c | 17 +- target/ppc/excp_helper.c | 2 +- target/s390x/tcg/excp_helper.c | 2 +- target/tricore/op_helper.c | 2 +- target/xtensa/helper.c | 6 +- tcg/tcg.c | 81 +------- tcg/{sparc => sparc64}/tcg-target.c.inc | 275 ++++++++-------------------- MAINTAINERS | 2 +- 30 files changed, 232 insertions(+), 437 deletions(-) rename tcg/{sparc => sparc64}/tcg-target-con-set.h (69%) rename tcg/{sparc => sparc64}/tcg-target-con-str.h (77%) rename tcg/{sparc => sparc64}/tcg-target.h (95%) rename tcg/{sparc => sparc64}/tcg-target.c.inc (91%) Since 9b9c37c36439, we have only supported sparc64 cpus. Debian and Gentoo now only support 64-bit sparc64 userland, so it is time to drop the 32-bit sparc64 userland: sparc32plus. Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/sparc/tcg-target.h | 11 --- tcg/tcg.c | 75 +---------------- tcg/sparc/tcg-target.c.inc | 166 +++++++------------------------------ 3 files changed, 33 insertions(+), 219 deletions(-) diff --git a/tcg/sparc/tcg-target.h b/tcg/sparc/tcg-target.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/sparc/tcg-target.h +++ b/tcg/sparc/tcg-target.h @@ -XXX,XX +XXX,XX @@ #ifndef SPARC_TCG_TARGET_H #define SPARC_TCG_TARGET_H -#define TCG_TARGET_REG_BITS 64 - #define TCG_TARGET_INSN_UNIT_SIZE 4 #define TCG_TARGET_TLB_DISPLACEMENT_BITS 32 #define TCG_TARGET_NB_REGS 32 @@ -XXX,XX +XXX,XX @@ typedef enum { /* used for function call generation */ #define TCG_REG_CALL_STACK TCG_REG_O6 -#ifdef __arch64__ #define TCG_TARGET_STACK_BIAS 2047 #define TCG_TARGET_STACK_ALIGN 16 #define TCG_TARGET_CALL_STACK_OFFSET (128 + 6*8 + TCG_TARGET_STACK_BIAS) -#else -#define TCG_TARGET_STACK_BIAS 0 -#define TCG_TARGET_STACK_ALIGN 8 -#define TCG_TARGET_CALL_STACK_OFFSET (64 + 4 + 6*4) -#endif - -#ifdef __arch64__ #define TCG_TARGET_EXTEND_ARGS 1 -#endif #if defined(__VIS__) && __VIS__ >= 0x300 #define use_vis3_instructions 1 diff --git a/tcg/tcg.c b/tcg/tcg.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/tcg.c +++ b/tcg/tcg.c @@ -XXX,XX +XXX,XX @@ void tcg_gen_callN(void *func, TCGTemp *ret, int nargs, TCGTemp **args) } #endif -#if defined(__sparc__) && !defined(__arch64__) \ - && !defined(CONFIG_TCG_INTERPRETER) - /* We have 64-bit values in one register, but need to pass as two - separate parameters. Split them. */ - int orig_typemask = typemask; - int orig_nargs = nargs; - TCGv_i64 retl, reth; - TCGTemp *split_args[MAX_OPC_PARAM]; - - retl = NULL; - reth = NULL; - typemask = 0; - for (i = real_args = 0; i < nargs; ++i) { - int argtype = extract32(orig_typemask, (i + 1) * 3, 3); - bool is_64bit = (argtype & ~1) == dh_typecode_i64; - - if (is_64bit) { - TCGv_i64 orig = temp_tcgv_i64(args[i]); - TCGv_i32 h = tcg_temp_new_i32(); - TCGv_i32 l = tcg_temp_new_i32(); - tcg_gen_extr_i64_i32(l, h, orig); - split_args[real_args++] = tcgv_i32_temp(h); - typemask |= dh_typecode_i32 << (real_args * 3); - split_args[real_args++] = tcgv_i32_temp(l); - typemask |= dh_typecode_i32 << (real_args * 3); - } else { - split_args[real_args++] = args[i]; - typemask |= argtype << (real_args * 3); - } - } - nargs = real_args; - args = split_args; -#elif defined(TCG_TARGET_EXTEND_ARGS) && TCG_TARGET_REG_BITS == 64 +#if defined(TCG_TARGET_EXTEND_ARGS) && TCG_TARGET_REG_BITS == 64 for (i = 0; i < nargs; ++i) { int argtype = extract32(typemask, (i + 1) * 3, 3); bool is_32bit = (argtype & ~1) == dh_typecode_i32; @@ -XXX,XX +XXX,XX @@ void tcg_gen_callN(void *func, TCGTemp *ret, int nargs, TCGTemp **args) pi = 0; if (ret != NULL) { -#if defined(__sparc__) && !defined(__arch64__) \ - && !defined(CONFIG_TCG_INTERPRETER) - if ((typemask & 6) == dh_typecode_i64) { - /* The 32-bit ABI is going to return the 64-bit value in - the %o0/%o1 register pair. Prepare for this by using - two return temporaries, and reassemble below. */ - retl = tcg_temp_new_i64(); - reth = tcg_temp_new_i64(); - op->args[pi++] = tcgv_i64_arg(reth); - op->args[pi++] = tcgv_i64_arg(retl); - nb_rets = 2; - } else { - op->args[pi++] = temp_arg(ret); - nb_rets = 1; - } -#else if (TCG_TARGET_REG_BITS < 64 && (typemask & 6) == dh_typecode_i64) { #if HOST_BIG_ENDIAN op->args[pi++] = temp_arg(ret + 1); @@ -XXX,XX +XXX,XX @@ void tcg_gen_callN(void *func, TCGTemp *ret, int nargs, TCGTemp **args) op->args[pi++] = temp_arg(ret); nb_rets = 1; } -#endif } else { nb_rets = 0; } @@ -XXX,XX +XXX,XX @@ void tcg_gen_callN(void *func, TCGTemp *ret, int nargs, TCGTemp **args) tcg_debug_assert(TCGOP_CALLI(op) == real_args); tcg_debug_assert(pi <= ARRAY_SIZE(op->args)); -#if defined(__sparc__) && !defined(__arch64__) \ - && !defined(CONFIG_TCG_INTERPRETER) - /* Free all of the parts we allocated above. */ - for (i = real_args = 0; i < orig_nargs; ++i) { - int argtype = extract32(orig_typemask, (i + 1) * 3, 3); - bool is_64bit = (argtype & ~1) == dh_typecode_i64; - - if (is_64bit) { - tcg_temp_free_internal(args[real_args++]); - tcg_temp_free_internal(args[real_args++]); - } else { - real_args++; - } - } - if ((orig_typemask & 6) == dh_typecode_i64) { - /* The 32-bit ABI returned two 32-bit pieces. Re-assemble them. - Note that describing these as TCGv_i64 eliminates an unnecessary - zero-extension that tcg_gen_concat_i32_i64 would create. */ - tcg_gen_concat32_i64(temp_tcgv_i64(ret), retl, reth); - tcg_temp_free_i64(retl); - tcg_temp_free_i64(reth); - } -#elif defined(TCG_TARGET_EXTEND_ARGS) && TCG_TARGET_REG_BITS == 64 +#if defined(TCG_TARGET_EXTEND_ARGS) && TCG_TARGET_REG_BITS == 64 for (i = 0; i < nargs; ++i) { int argtype = extract32(typemask, (i + 1) * 3, 3); bool is_32bit = (argtype & ~1) == dh_typecode_i32; diff --git a/tcg/sparc/tcg-target.c.inc b/tcg/sparc/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/sparc/tcg-target.c.inc +++ b/tcg/sparc/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ * THE SOFTWARE. */ +/* We only support generating code for 64-bit mode. */ +#ifndef __arch64__ +#error "unsupported code generation mode" +#endif + #include "../tcg-pool.c.inc" #ifdef CONFIG_DEBUG_TCG @@ -XXX,XX +XXX,XX @@ static const char * const tcg_target_reg_names[TCG_TARGET_NB_REGS] = { }; #endif -#ifdef __arch64__ -# define SPARC64 1 -#else -# define SPARC64 0 -#endif - #define TCG_CT_CONST_S11 0x100 #define TCG_CT_CONST_S13 0x200 #define TCG_CT_CONST_ZERO 0x400 @@ -XXX,XX +XXX,XX @@ static const char * const tcg_target_reg_names[TCG_TARGET_NB_REGS] = { * high bits of the %i and %l registers garbage at all times. */ #define ALL_GENERAL_REGS MAKE_64BIT_MASK(0, 32) -#if SPARC64 # define ALL_GENERAL_REGS64 ALL_GENERAL_REGS -#else -# define ALL_GENERAL_REGS64 MAKE_64BIT_MASK(0, 16) -#endif #define ALL_QLDST_REGS (ALL_GENERAL_REGS & ~SOFTMMU_RESERVE_REGS) #define ALL_QLDST_REGS64 (ALL_GENERAL_REGS64 & ~SOFTMMU_RESERVE_REGS) @@ -XXX,XX +XXX,XX @@ static bool check_fit_i32(int32_t val, unsigned int bits) } #define check_fit_tl check_fit_i64 -#if SPARC64 -# define check_fit_ptr check_fit_i64 -#else -# define check_fit_ptr check_fit_i32 -#endif +#define check_fit_ptr check_fit_i64 static bool patch_reloc(tcg_insn_unit *src_rw, int type, intptr_t value, intptr_t addend) @@ -XXX,XX +XXX,XX @@ static void tcg_out_sety(TCGContext *s, TCGReg rs) tcg_out32(s, WRY | INSN_RS1(TCG_REG_G0) | INSN_RS2(rs)); } -static void tcg_out_rdy(TCGContext *s, TCGReg rd) -{ - tcg_out32(s, RDY | INSN_RD(rd)); -} - static void tcg_out_div32(TCGContext *s, TCGReg rd, TCGReg rs1, int32_t val2, int val2const, int uns) { @@ -XXX,XX +XXX,XX @@ static void emit_extend(TCGContext *s, TCGReg r, int op) tcg_out_arithi(s, r, r, 16, SHIFT_SRL); break; case MO_32: - if (SPARC64) { - tcg_out_arith(s, r, r, 0, SHIFT_SRL); - } + tcg_out_arith(s, r, r, 0, SHIFT_SRL); break; case MO_64: break; @@ -XXX,XX +XXX,XX @@ static void build_trampolines(TCGContext *s) }; int i; - TCGReg ra; for (i = 0; i < ARRAY_SIZE(qemu_ld_helpers); ++i) { if (qemu_ld_helpers[i] == NULL) { @@ -XXX,XX +XXX,XX @@ static void build_trampolines(TCGContext *s) } qemu_ld_trampoline[i] = tcg_splitwx_to_rx(s->code_ptr); - if (SPARC64 || TARGET_LONG_BITS == 32) { - ra = TCG_REG_O3; - } else { - /* Install the high part of the address. */ - tcg_out_arithi(s, TCG_REG_O1, TCG_REG_O2, 32, SHIFT_SRLX); - ra = TCG_REG_O4; - } - /* Set the retaddr operand. */ - tcg_out_mov(s, TCG_TYPE_PTR, ra, TCG_REG_O7); + tcg_out_mov(s, TCG_TYPE_PTR, TCG_REG_O3, TCG_REG_O7); /* Tail call. */ tcg_out_jmpl_const(s, qemu_ld_helpers[i], true, true); /* delay slot -- set the env argument */ @@ -XXX,XX +XXX,XX @@ static void build_trampolines(TCGContext *s) } qemu_st_trampoline[i] = tcg_splitwx_to_rx(s->code_ptr); - if (SPARC64) { - emit_extend(s, TCG_REG_O2, i); - ra = TCG_REG_O4; - } else { - ra = TCG_REG_O1; - if (TARGET_LONG_BITS == 64) { - /* Install the high part of the address. */ - tcg_out_arithi(s, ra, ra + 1, 32, SHIFT_SRLX); - ra += 2; - } else { - ra += 1; - } - if ((i & MO_SIZE) == MO_64) { - /* Install the high part of the data. */ - tcg_out_arithi(s, ra, ra + 1, 32, SHIFT_SRLX); - ra += 2; - } else { - emit_extend(s, ra, i); - ra += 1; - } - /* Skip the oi argument. */ - ra += 1; - } - + emit_extend(s, TCG_REG_O2, i); + /* Set the retaddr operand. */ - if (ra >= TCG_REG_O6) { - tcg_out_st(s, TCG_TYPE_PTR, TCG_REG_O7, TCG_REG_CALL_STACK, - TCG_TARGET_CALL_STACK_OFFSET); - } else { - tcg_out_mov(s, TCG_TYPE_PTR, ra, TCG_REG_O7); - } + tcg_out_mov(s, TCG_TYPE_PTR, TCG_REG_O4, TCG_REG_O7); /* Tail call. */ tcg_out_jmpl_const(s, qemu_st_helpers[i], true, true); @@ -XXX,XX +XXX,XX @@ static void build_trampolines(TCGContext *s) qemu_unalign_st_trampoline = tcg_splitwx_to_rx(s->code_ptr); } - if (!SPARC64 && TARGET_LONG_BITS == 64) { - /* Install the high part of the address. */ - tcg_out_arithi(s, TCG_REG_O1, TCG_REG_O2, 32, SHIFT_SRLX); - } - /* Tail call. */ tcg_out_jmpl_const(s, helper, true, true); /* delay slot -- set the env argument */ @@ -XXX,XX +XXX,XX @@ static TCGReg tcg_out_tlb_load(TCGContext *s, TCGReg addr, int mem_index, tcg_out_cmp(s, r0, r2, 0); /* If the guest address must be zero-extended, do so now. */ - if (SPARC64 && TARGET_LONG_BITS == 32) { + if (TARGET_LONG_BITS == 32) { tcg_out_arithi(s, r0, addr, 0, SHIFT_SRL); return r0; } @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_ld(TCGContext *s, TCGReg data, TCGReg addr, #ifdef CONFIG_SOFTMMU unsigned memi = get_mmuidx(oi); - TCGReg addrz, param; + TCGReg addrz; const tcg_insn_unit *func; addrz = tcg_out_tlb_load(s, addr, memi, memop, @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_ld(TCGContext *s, TCGReg data, TCGReg addr, /* TLB Miss. */ - param = TCG_REG_O1; - if (!SPARC64 && TARGET_LONG_BITS == 64) { - /* Skip the high-part; we'll perform the extract in the trampoline. */ - param++; - } - tcg_out_mov(s, TCG_TYPE_REG, param++, addrz); + tcg_out_mov(s, TCG_TYPE_REG, TCG_REG_O1, addrz); /* We use the helpers to extend SB and SW data, leaving the case of SL needing explicit extending below. */ @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_ld(TCGContext *s, TCGReg data, TCGReg addr, tcg_debug_assert(func != NULL); tcg_out_call_nodelay(s, func, false); /* delay slot */ - tcg_out_movi(s, TCG_TYPE_I32, param, oi); + tcg_out_movi(s, TCG_TYPE_I32, TCG_REG_O2, oi); - /* Recall that all of the helpers return 64-bit results. - Which complicates things for sparcv8plus. */ - if (SPARC64) { - /* We let the helper sign-extend SB and SW, but leave SL for here. */ - if (is_64 && (memop & MO_SSIZE) == MO_SL) { - tcg_out_arithi(s, data, TCG_REG_O0, 0, SHIFT_SRA); - } else { - tcg_out_mov(s, TCG_TYPE_REG, data, TCG_REG_O0); - } + /* We let the helper sign-extend SB and SW, but leave SL for here. */ + if (is_64 && (memop & MO_SSIZE) == MO_SL) { + tcg_out_arithi(s, data, TCG_REG_O0, 0, SHIFT_SRA); } else { - if ((memop & MO_SIZE) == MO_64) { - tcg_out_arithi(s, TCG_REG_O0, TCG_REG_O0, 32, SHIFT_SLLX); - tcg_out_arithi(s, TCG_REG_O1, TCG_REG_O1, 0, SHIFT_SRL); - tcg_out_arith(s, data, TCG_REG_O0, TCG_REG_O1, ARITH_OR); - } else if (is_64) { - /* Re-extend from 32-bit rather than reassembling when we - know the high register must be an extension. */ - tcg_out_arithi(s, data, TCG_REG_O1, 0, - memop & MO_SIGN ? SHIFT_SRA : SHIFT_SRL); - } else { - tcg_out_mov(s, TCG_TYPE_I32, data, TCG_REG_O1); - } + tcg_out_mov(s, TCG_TYPE_REG, data, TCG_REG_O0); } *label_ptr |= INSN_OFF19(tcg_ptr_byte_diff(s->code_ptr, label_ptr)); @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_ld(TCGContext *s, TCGReg data, TCGReg addr, unsigned s_bits = memop & MO_SIZE; unsigned t_bits; - if (SPARC64 && TARGET_LONG_BITS == 32) { + if (TARGET_LONG_BITS == 32) { tcg_out_arithi(s, TCG_REG_T1, addr, 0, SHIFT_SRL); addr = TCG_REG_T1; } @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_ld(TCGContext *s, TCGReg data, TCGReg addr, * operation in the delay slot, and failure need only invoke the * handler for SIGBUS. */ - TCGReg arg_low = TCG_REG_O1 + (!SPARC64 && TARGET_LONG_BITS == 64); tcg_out_call_nodelay(s, qemu_unalign_ld_trampoline, false); /* delay slot -- move to low part of argument reg */ - tcg_out_mov_delay(s, arg_low, addr); + tcg_out_mov_delay(s, TCG_REG_O1, addr); } else { /* Underalignment: load by pieces of minimum alignment. */ int ld_opc, a_size, s_size, i; @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_st(TCGContext *s, TCGReg data, TCGReg addr, #ifdef CONFIG_SOFTMMU unsigned memi = get_mmuidx(oi); - TCGReg addrz, param; + TCGReg addrz; const tcg_insn_unit *func; addrz = tcg_out_tlb_load(s, addr, memi, memop, @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_st(TCGContext *s, TCGReg data, TCGReg addr, /* TLB Miss. */ - param = TCG_REG_O1; - if (!SPARC64 && TARGET_LONG_BITS == 64) { - /* Skip the high-part; we'll perform the extract in the trampoline. */ - param++; - } - tcg_out_mov(s, TCG_TYPE_REG, param++, addrz); - if (!SPARC64 && (memop & MO_SIZE) == MO_64) { - /* Skip the high-part; we'll perform the extract in the trampoline. */ - param++; - } - tcg_out_mov(s, TCG_TYPE_REG, param++, data); + tcg_out_mov(s, TCG_TYPE_REG, TCG_REG_O1, addrz); + tcg_out_mov(s, TCG_TYPE_REG, TCG_REG_O2, data); func = qemu_st_trampoline[memop & (MO_BSWAP | MO_SIZE)]; tcg_debug_assert(func != NULL); tcg_out_call_nodelay(s, func, false); /* delay slot */ - tcg_out_movi(s, TCG_TYPE_I32, param, oi); + tcg_out_movi(s, TCG_TYPE_I32, TCG_REG_O3, oi); *label_ptr |= INSN_OFF19(tcg_ptr_byte_diff(s->code_ptr, label_ptr)); #else @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_st(TCGContext *s, TCGReg data, TCGReg addr, unsigned s_bits = memop & MO_SIZE; unsigned t_bits; - if (SPARC64 && TARGET_LONG_BITS == 32) { + if (TARGET_LONG_BITS == 32) { tcg_out_arithi(s, TCG_REG_T1, addr, 0, SHIFT_SRL); addr = TCG_REG_T1; } @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_st(TCGContext *s, TCGReg data, TCGReg addr, * operation in the delay slot, and failure need only invoke the * handler for SIGBUS. */ - TCGReg arg_low = TCG_REG_O1 + (!SPARC64 && TARGET_LONG_BITS == 64); tcg_out_call_nodelay(s, qemu_unalign_st_trampoline, false); /* delay slot -- move to low part of argument reg */ - tcg_out_mov_delay(s, arg_low, addr); + tcg_out_mov_delay(s, TCG_REG_O1, addr); } else { /* Underalignment: store by pieces of minimum alignment. */ int st_opc, a_size, s_size, i; @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, case INDEX_op_muls2_i32: c = ARITH_SMUL; do_mul2: - /* The 32-bit multiply insns produce a full 64-bit result. If the - destination register can hold it, we can avoid the slower RDY. */ + /* The 32-bit multiply insns produce a full 64-bit result. */ tcg_out_arithc(s, a0, a2, args[3], const_args[3], c); - if (SPARC64 || a0 <= TCG_REG_O7) { - tcg_out_arithi(s, a1, a0, 32, SHIFT_SRLX); - } else { - tcg_out_rdy(s, a1); - } + tcg_out_arithi(s, a1, a0, 32, SHIFT_SRLX); break; case INDEX_op_qemu_ld_i32: @@ -XXX,XX +XXX,XX @@ static void tcg_target_init(TCGContext *s) tcg_regset_set_reg(s->reserved_regs, TCG_REG_T2); /* for internal use */ } -#if SPARC64 -# define ELF_HOST_MACHINE EM_SPARCV9 -#else -# define ELF_HOST_MACHINE EM_SPARC32PLUS -# define ELF_HOST_FLAGS EF_SPARC_32PLUS -#endif +#define ELF_HOST_MACHINE EM_SPARCV9 typedef struct { DebugFrameHeader h; - uint8_t fde_def_cfa[SPARC64 ? 4 : 2]; + uint8_t fde_def_cfa[4]; uint8_t fde_win_save; uint8_t fde_ret_save[3]; } DebugFrame; @@ -XXX,XX +XXX,XX @@ static const DebugFrame debug_frame = { .h.fde.len = sizeof(DebugFrame) - offsetof(DebugFrame, h.fde.cie_offset), .fde_def_cfa = { -#if SPARC64 12, 30, /* DW_CFA_def_cfa i6, 2047 */ (2047 & 0x7f) | 0x80, (2047 >> 7) -#else - 13, 30 /* DW_CFA_def_cfa_register i6 */ -#endif }, .fde_win_save = 0x2d, /* DW_CFA_GNU_window_save */ .fde_ret_save = { 9, 15, 31 }, /* DW_CFA_register o7, i7 */ -- 2.34.1 Emphasize that we only support full 64-bit code generation. Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- meson.build | 4 +--- tcg/{sparc => sparc64}/tcg-target-con-set.h | 0 tcg/{sparc => sparc64}/tcg-target-con-str.h | 0 tcg/{sparc => sparc64}/tcg-target.h | 0 tcg/{sparc => sparc64}/tcg-target.c.inc | 0 MAINTAINERS | 2 +- 6 files changed, 2 insertions(+), 4 deletions(-) rename tcg/{sparc => sparc64}/tcg-target-con-set.h (100%) rename tcg/{sparc => sparc64}/tcg-target-con-str.h (100%) rename tcg/{sparc => sparc64}/tcg-target.h (100%) rename tcg/{sparc => sparc64}/tcg-target.c.inc (100%) diff --git a/meson.build b/meson.build index XXXXXXX..XXXXXXX 100644 --- a/meson.build +++ b/meson.build @@ -XXX,XX +XXX,XX @@ qapi_trace_events = [] bsd_oses = ['gnu/kfreebsd', 'freebsd', 'netbsd', 'openbsd', 'dragonfly', 'darwin'] supported_oses = ['windows', 'freebsd', 'netbsd', 'openbsd', 'darwin', 'sunos', 'linux'] supported_cpus = ['ppc', 'ppc64', 's390x', 'riscv', 'x86', 'x86_64', - 'arm', 'aarch64', 'loongarch64', 'mips', 'mips64', 'sparc', 'sparc64'] + 'arm', 'aarch64', 'loongarch64', 'mips', 'mips64', 'sparc64'] cpu = host_machine.cpu_family() @@ -XXX,XX +XXX,XX @@ if get_option('tcg').allowed() endif if get_option('tcg_interpreter') tcg_arch = 'tci' - elif host_arch == 'sparc64' - tcg_arch = 'sparc' elif host_arch == 'x86_64' tcg_arch = 'i386' elif host_arch == 'ppc64' diff --git a/tcg/sparc/tcg-target-con-set.h b/tcg/sparc64/tcg-target-con-set.h similarity index 100% rename from tcg/sparc/tcg-target-con-set.h rename to tcg/sparc64/tcg-target-con-set.h diff --git a/tcg/sparc/tcg-target-con-str.h b/tcg/sparc64/tcg-target-con-str.h similarity index 100% rename from tcg/sparc/tcg-target-con-str.h rename to tcg/sparc64/tcg-target-con-str.h diff --git a/tcg/sparc/tcg-target.h b/tcg/sparc64/tcg-target.h similarity index 100% rename from tcg/sparc/tcg-target.h rename to tcg/sparc64/tcg-target.h diff --git a/tcg/sparc/tcg-target.c.inc b/tcg/sparc64/tcg-target.c.inc similarity index 100% rename from tcg/sparc/tcg-target.c.inc rename to tcg/sparc64/tcg-target.c.inc diff --git a/MAINTAINERS b/MAINTAINERS index XXXXXXX..XXXXXXX 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -XXX,XX +XXX,XX @@ L: qemu-s390x@nongnu.org SPARC TCG target S: Odd Fixes -F: tcg/sparc/ +F: tcg/sparc64/ F: disas/sparc.c TCI TCG target -- 2.34.1 With sparc64 we need not distinguish between registers that can hold 32-bit values and those that can hold 64-bit values. Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/sparc64/tcg-target-con-set.h | 16 +---- tcg/sparc64/tcg-target-con-str.h | 3 - tcg/sparc64/tcg-target.c.inc | 109 ++++++++++++------------------- 3 files changed, 44 insertions(+), 84 deletions(-) diff --git a/tcg/sparc64/tcg-target-con-set.h b/tcg/sparc64/tcg-target-con-set.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/sparc64/tcg-target-con-set.h +++ b/tcg/sparc64/tcg-target-con-set.h @@ -XXX,XX +XXX,XX @@ */ C_O0_I1(r) C_O0_I2(rZ, r) -C_O0_I2(RZ, r) C_O0_I2(rZ, rJ) -C_O0_I2(RZ, RJ) -C_O0_I2(sZ, A) -C_O0_I2(SZ, A) -C_O1_I1(r, A) -C_O1_I1(R, A) +C_O0_I2(sZ, s) +C_O1_I1(r, s) C_O1_I1(r, r) -C_O1_I1(r, R) -C_O1_I1(R, r) -C_O1_I1(R, R) -C_O1_I2(R, R, R) +C_O1_I2(r, r, r) C_O1_I2(r, rZ, rJ) -C_O1_I2(R, RZ, RJ) C_O1_I4(r, rZ, rJ, rI, 0) -C_O1_I4(R, RZ, RJ, RI, 0) C_O2_I2(r, r, rZ, rJ) -C_O2_I4(R, R, RZ, RZ, RJ, RI) C_O2_I4(r, r, rZ, rZ, rJ, rJ) diff --git a/tcg/sparc64/tcg-target-con-str.h b/tcg/sparc64/tcg-target-con-str.h index XXXXXXX..XXXXXXX 100644 --- a/tcg/sparc64/tcg-target-con-str.h +++ b/tcg/sparc64/tcg-target-con-str.h @@ -XXX,XX +XXX,XX @@ * REGS(letter, register_mask) */ REGS('r', ALL_GENERAL_REGS) -REGS('R', ALL_GENERAL_REGS64) REGS('s', ALL_QLDST_REGS) -REGS('S', ALL_QLDST_REGS64) -REGS('A', TARGET_LONG_BITS == 64 ? ALL_QLDST_REGS64 : ALL_QLDST_REGS) /* * Define constraint letters for constants: diff --git a/tcg/sparc64/tcg-target.c.inc b/tcg/sparc64/tcg-target.c.inc index XXXXXXX..XXXXXXX 100644 --- a/tcg/sparc64/tcg-target.c.inc +++ b/tcg/sparc64/tcg-target.c.inc @@ -XXX,XX +XXX,XX @@ static const char * const tcg_target_reg_names[TCG_TARGET_NB_REGS] = { #else #define SOFTMMU_RESERVE_REGS 0 #endif - -/* - * Note that sparcv8plus can only hold 64 bit quantities in %g and %o - * registers. These are saved manually by the kernel in full 64-bit - * slots. The %i and %l registers are saved by the register window - * mechanism, which only allocates space for 32 bits. Given that this - * window spill/fill can happen on any signal, we must consider the - * high bits of the %i and %l registers garbage at all times. - */ #define ALL_GENERAL_REGS MAKE_64BIT_MASK(0, 32) -# define ALL_GENERAL_REGS64 ALL_GENERAL_REGS #define ALL_QLDST_REGS (ALL_GENERAL_REGS & ~SOFTMMU_RESERVE_REGS) -#define ALL_QLDST_REGS64 (ALL_GENERAL_REGS64 & ~SOFTMMU_RESERVE_REGS) /* Define some temporary registers. T2 is used for constant generation. */ #define TCG_REG_T1 TCG_REG_G1 @@ -XXX,XX +XXX,XX @@ static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode op) return C_O0_I1(r); case INDEX_op_ld8u_i32: + case INDEX_op_ld8u_i64: case INDEX_op_ld8s_i32: + case INDEX_op_ld8s_i64: case INDEX_op_ld16u_i32: + case INDEX_op_ld16u_i64: case INDEX_op_ld16s_i32: + case INDEX_op_ld16s_i64: case INDEX_op_ld_i32: + case INDEX_op_ld32u_i64: + case INDEX_op_ld32s_i64: + case INDEX_op_ld_i64: case INDEX_op_neg_i32: + case INDEX_op_neg_i64: case INDEX_op_not_i32: + case INDEX_op_not_i64: + case INDEX_op_ext32s_i64: + case INDEX_op_ext32u_i64: + case INDEX_op_ext_i32_i64: + case INDEX_op_extu_i32_i64: + case INDEX_op_extrl_i64_i32: + case INDEX_op_extrh_i64_i32: return C_O1_I1(r, r); case INDEX_op_st8_i32: + case INDEX_op_st8_i64: case INDEX_op_st16_i32: + case INDEX_op_st16_i64: case INDEX_op_st_i32: + case INDEX_op_st32_i64: + case INDEX_op_st_i64: return C_O0_I2(rZ, r); case INDEX_op_add_i32: + case INDEX_op_add_i64: case INDEX_op_mul_i32: + case INDEX_op_mul_i64: case INDEX_op_div_i32: + case INDEX_op_div_i64: case INDEX_op_divu_i32: + case INDEX_op_divu_i64: case INDEX_op_sub_i32: + case INDEX_op_sub_i64: case INDEX_op_and_i32: + case INDEX_op_and_i64: case INDEX_op_andc_i32: + case INDEX_op_andc_i64: case INDEX_op_or_i32: + case INDEX_op_or_i64: case INDEX_op_orc_i32: + case INDEX_op_orc_i64: case INDEX_op_xor_i32: + case INDEX_op_xor_i64: case INDEX_op_shl_i32: + case INDEX_op_shl_i64: case INDEX_op_shr_i32: + case INDEX_op_shr_i64: case INDEX_op_sar_i32: + case INDEX_op_sar_i64: case INDEX_op_setcond_i32: + case INDEX_op_setcond_i64: return C_O1_I2(r, rZ, rJ); case INDEX_op_brcond_i32: + case INDEX_op_brcond_i64: return C_O0_I2(rZ, rJ); case INDEX_op_movcond_i32: + case INDEX_op_movcond_i64: return C_O1_I4(r, rZ, rJ, rI, 0); case INDEX_op_add2_i32: + case INDEX_op_add2_i64: case INDEX_op_sub2_i32: + case INDEX_op_sub2_i64: return C_O2_I4(r, r, rZ, rZ, rJ, rJ); case INDEX_op_mulu2_i32: case INDEX_op_muls2_i32: return C_O2_I2(r, r, rZ, rJ); - - case INDEX_op_ld8u_i64: - case INDEX_op_ld8s_i64: - case INDEX_op_ld16u_i64: - case INDEX_op_ld16s_i64: - case INDEX_op_ld32u_i64: - case INDEX_op_ld32s_i64: - case INDEX_op_ld_i64: - case INDEX_op_ext_i32_i64: - case INDEX_op_extu_i32_i64: - return C_O1_I1(R, r); - - case INDEX_op_st8_i64: - case INDEX_op_st16_i64: - case INDEX_op_st32_i64: - case INDEX_op_st_i64: - return C_O0_I2(RZ, r); - - case INDEX_op_add_i64: - case INDEX_op_mul_i64: - case INDEX_op_div_i64: - case INDEX_op_divu_i64: - case INDEX_op_sub_i64: - case INDEX_op_and_i64: - case INDEX_op_andc_i64: - case INDEX_op_or_i64: - case INDEX_op_orc_i64: - case INDEX_op_xor_i64: - case INDEX_op_shl_i64: - case INDEX_op_shr_i64: - case INDEX_op_sar_i64: - case INDEX_op_setcond_i64: - return C_O1_I2(R, RZ, RJ); - - case INDEX_op_neg_i64: - case INDEX_op_not_i64: - case INDEX_op_ext32s_i64: - case INDEX_op_ext32u_i64: - return C_O1_I1(R, R); - - case INDEX_op_extrl_i64_i32: - case INDEX_op_extrh_i64_i32: - return C_O1_I1(r, R); - - case INDEX_op_brcond_i64: - return C_O0_I2(RZ, RJ); - case INDEX_op_movcond_i64: - return C_O1_I4(R, RZ, RJ, RI, 0); - case INDEX_op_add2_i64: - case INDEX_op_sub2_i64: - return C_O2_I4(R, R, RZ, RZ, RJ, RI); case INDEX_op_muluh_i64: - return C_O1_I2(R, R, R); + return C_O1_I2(r, r, r); case INDEX_op_qemu_ld_i32: - return C_O1_I1(r, A); case INDEX_op_qemu_ld_i64: - return C_O1_I1(R, A); + return C_O1_I1(r, s); case INDEX_op_qemu_st_i32: - return C_O0_I2(sZ, A); case INDEX_op_qemu_st_i64: - return C_O0_I2(SZ, A); + return C_O0_I2(sZ, s); default: g_assert_not_reached(); @@ -XXX,XX +XXX,XX @@ static void tcg_target_init(TCGContext *s) #endif tcg_target_available_regs[TCG_TYPE_I32] = ALL_GENERAL_REGS; - tcg_target_available_regs[TCG_TYPE_I64] = ALL_GENERAL_REGS64; + tcg_target_available_regs[TCG_TYPE_I64] = ALL_GENERAL_REGS; tcg_target_call_clobber_regs = 0; tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_G1); -- 2.34.1 From: Icenowy Zheng <uwu@icenowy.me> When registering helpers via FFI for TCI, the inner loop that iterates parameters of the helper reuses (and thus pollutes) the same variable used by the outer loop that iterates all helpers, thus made some helpers unregistered. Fix this logic error by using a dedicated temporary variable for the inner loop. Fixes: 22f15579fa ("tcg: Build ffi data structures for helpers") Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org> Signed-off-by: Icenowy Zheng <uwu@icenowy.me> Message-Id: <20221028072145.1593205-1-uwu@icenowy.me> [rth: Move declaration of j to the for loop itself] Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- tcg/tcg.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tcg/tcg.c b/tcg/tcg.c index XXXXXXX..XXXXXXX 100644 --- a/tcg/tcg.c +++ b/tcg/tcg.c @@ -XXX,XX +XXX,XX @@ static void tcg_context_init(unsigned max_cpus) if (nargs != 0) { ca->cif.arg_types = ca->args; - for (i = 0; i < nargs; ++i) { - int typecode = extract32(typemask, (i + 1) * 3, 3); - ca->args[i] = typecode_to_ffi[typecode]; + for (int j = 0; j < nargs; ++j) { + int typecode = extract32(typemask, (j + 1) * 3, 3); + ca->args[j] = typecode_to_ffi[typecode]; } } -- 2.34.1 Add a way to examine the unwind data without actually restoring the data back into env. Reviewed-by: Claudio Fontana <cfontana@suse.de> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- accel/tcg/internal.h | 4 +-- include/exec/exec-all.h | 21 ++++++++--- accel/tcg/translate-all.c | 74 ++++++++++++++++++++++++++------------- 3 files changed, 68 insertions(+), 31 deletions(-) diff --git a/accel/tcg/internal.h b/accel/tcg/internal.h index XXXXXXX..XXXXXXX 100644 --- a/accel/tcg/internal.h +++ b/accel/tcg/internal.h @@ -XXX,XX +XXX,XX @@ void tb_reset_jump(TranslationBlock *tb, int n); TranslationBlock *tb_link_page(TranslationBlock *tb, tb_page_addr_t phys_pc, tb_page_addr_t phys_page2); bool tb_invalidate_phys_page_unwind(tb_page_addr_t addr, uintptr_t pc); -int cpu_restore_state_from_tb(CPUState *cpu, TranslationBlock *tb, - uintptr_t searched_pc, bool reset_icount); +void cpu_restore_state_from_tb(CPUState *cpu, TranslationBlock *tb, + uintptr_t host_pc, bool reset_icount); /* Return the current PC from CPU, which may be cached in TB. */ static inline target_ulong log_pc(CPUState *cpu, const TranslationBlock *tb) diff --git a/include/exec/exec-all.h b/include/exec/exec-all.h index XXXXXXX..XXXXXXX 100644 --- a/include/exec/exec-all.h +++ b/include/exec/exec-all.h @@ -XXX,XX +XXX,XX @@ typedef ram_addr_t tb_page_addr_t; #define TB_PAGE_ADDR_FMT RAM_ADDR_FMT #endif +/** + * cpu_unwind_state_data: + * @cpu: the cpu context + * @host_pc: the host pc within the translation + * @data: output data + * + * Attempt to load the the unwind state for a host pc occurring in + * translated code. If @host_pc is not in translated code, the + * function returns false; otherwise @data is loaded. + * This is the same unwind info as given to restore_state_to_opc. + */ +bool cpu_unwind_state_data(CPUState *cpu, uintptr_t host_pc, uint64_t *data); + /** * cpu_restore_state: - * @cpu: the vCPU state is to be restore to - * @searched_pc: the host PC the fault occurred at + * @cpu: the cpu context + * @host_pc: the host pc within the translation * @will_exit: true if the TB executed will be interrupted after some cpu adjustments. Required for maintaining the correct icount valus * @return: true if state was restored, false otherwise * * Attempt to restore the state for a fault occurring in translated - * code. If the searched_pc is not in translated code no state is + * code. If @host_pc is not in translated code no state is * restored and the function returns false. */ -bool cpu_restore_state(CPUState *cpu, uintptr_t searched_pc, bool will_exit); +bool cpu_restore_state(CPUState *cpu, uintptr_t host_pc, bool will_exit); G_NORETURN void cpu_loop_exit_noexc(CPUState *cpu); G_NORETURN void cpu_loop_exit(CPUState *cpu); diff --git a/accel/tcg/translate-all.c b/accel/tcg/translate-all.c index XXXXXXX..XXXXXXX 100644 --- a/accel/tcg/translate-all.c +++ b/accel/tcg/translate-all.c @@ -XXX,XX +XXX,XX @@ static int encode_search(TranslationBlock *tb, uint8_t *block) return p - block; } -/* The cpu state corresponding to 'searched_pc' is restored. - * When reset_icount is true, current TB will be interrupted and - * icount should be recalculated. - */ -int cpu_restore_state_from_tb(CPUState *cpu, TranslationBlock *tb, - uintptr_t searched_pc, bool reset_icount) +static int cpu_unwind_data_from_tb(TranslationBlock *tb, uintptr_t host_pc, + uint64_t *data) { - uint64_t data[TARGET_INSN_START_WORDS]; - uintptr_t host_pc = (uintptr_t)tb->tc.ptr; + uintptr_t iter_pc = (uintptr_t)tb->tc.ptr; const uint8_t *p = tb->tc.ptr + tb->tc.size; int i, j, num_insns = tb->icount; -#ifdef CONFIG_PROFILER - TCGProfile *prof = &tcg_ctx->prof; - int64_t ti = profile_getclock(); -#endif - searched_pc -= GETPC_ADJ; + host_pc -= GETPC_ADJ; - if (searched_pc < host_pc) { + if (host_pc < iter_pc) { return -1; } - memset(data, 0, sizeof(data)); + memset(data, 0, sizeof(uint64_t) * TARGET_INSN_START_WORDS); if (!TARGET_TB_PCREL) { data[0] = tb_pc(tb); } - /* Reconstruct the stored insn data while looking for the point at - which the end of the insn exceeds the searched_pc. */ + /* + * Reconstruct the stored insn data while looking for the point + * at which the end of the insn exceeds host_pc. + */ for (i = 0; i < num_insns; ++i) { for (j = 0; j < TARGET_INSN_START_WORDS; ++j) { data[j] += decode_sleb128(&p); } - host_pc += decode_sleb128(&p); - if (host_pc > searched_pc) { - goto found; + iter_pc += decode_sleb128(&p); + if (iter_pc > host_pc) { + return num_insns - i; } } return -1; +} + +/* + * The cpu state corresponding to 'host_pc' is restored. + * When reset_icount is true, current TB will be interrupted and + * icount should be recalculated. + */ +void cpu_restore_state_from_tb(CPUState *cpu, TranslationBlock *tb, + uintptr_t host_pc, bool reset_icount) +{ + uint64_t data[TARGET_INSN_START_WORDS]; +#ifdef CONFIG_PROFILER + TCGProfile *prof = &tcg_ctx->prof; + int64_t ti = profile_getclock(); +#endif + int insns_left = cpu_unwind_data_from_tb(tb, host_pc, data); + + if (insns_left < 0) { + return; + } - found: if (reset_icount && (tb_cflags(tb) & CF_USE_ICOUNT)) { assert(icount_enabled()); - /* Reset the cycle counter to the start of the block - and shift if to the number of actually executed instructions */ - cpu_neg(cpu)->icount_decr.u16.low += num_insns - i; + /* + * Reset the cycle counter to the start of the block and + * shift if to the number of actually executed instructions. + */ + cpu_neg(cpu)->icount_decr.u16.low += insns_left; } cpu->cc->tcg_ops->restore_state_to_opc(cpu, tb, data); @@ -XXX,XX +XXX,XX @@ int cpu_restore_state_from_tb(CPUState *cpu, TranslationBlock *tb, prof->restore_time + profile_getclock() - ti); qatomic_set(&prof->restore_count, prof->restore_count + 1); #endif - return 0; } bool cpu_restore_state(CPUState *cpu, uintptr_t host_pc, bool will_exit) @@ -XXX,XX +XXX,XX @@ bool cpu_restore_state(CPUState *cpu, uintptr_t host_pc, bool will_exit) return false; } +bool cpu_unwind_state_data(CPUState *cpu, uintptr_t host_pc, uint64_t *data) +{ + if (in_code_gen_buffer((const void *)(host_pc - tcg_splitwx_diff))) { + TranslationBlock *tb = tcg_tb_lookup(host_pc); + if (tb) { + return cpu_unwind_data_from_tb(tb, host_pc, data) >= 0; + } + } + return false; +} + void page_init(void) { page_size_init(); -- 2.34.1 Avoid cpu_restore_state, and modifying env->eip out from underneath the translator with TARGET_TB_PCREL. There is some slight duplication from x86_restore_state_to_opc, but it's just a few lines. Resolves: https://gitlab.com/qemu-project/qemu/-/issues/1269 Reviewed-by: Claudio Fontana <cfontana@suse.de> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- target/i386/helper.c | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/target/i386/helper.c b/target/i386/helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/i386/helper.c +++ b/target/i386/helper.c @@ -XXX,XX +XXX,XX @@ void cpu_x86_inject_mce(Monitor *mon, X86CPU *cpu, int bank, } } +static target_ulong get_memio_eip(CPUX86State *env) +{ + uint64_t data[TARGET_INSN_START_WORDS]; + CPUState *cs = env_cpu(env); + + if (!cpu_unwind_state_data(cs, cs->mem_io_pc, data)) { + return env->eip; + } + + /* Per x86_restore_state_to_opc. */ + if (TARGET_TB_PCREL) { + return (env->eip & TARGET_PAGE_MASK) | data[0]; + } else { + return data[0] - env->segs[R_CS].base; + } +} + void cpu_report_tpr_access(CPUX86State *env, TPRAccess access) { X86CPU *cpu = env_archcpu(env); @@ -XXX,XX +XXX,XX @@ void cpu_report_tpr_access(CPUX86State *env, TPRAccess access) cpu_interrupt(cs, CPU_INTERRUPT_TPR); } else if (tcg_enabled()) { - cpu_restore_state(cs, cs->mem_io_pc, false); + target_ulong eip = get_memio_eip(env); - apic_handle_tpr_access_report(cpu->apic_state, env->eip, access); + apic_handle_tpr_access_report(cpu->apic_state, eip, access); } } #endif /* !CONFIG_USER_ONLY */ -- 2.34.1 We have called cpu_restore_state asserting will_exit. Do not go back on that promise. This affects icount. Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- target/openrisc/sys_helper.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/target/openrisc/sys_helper.c b/target/openrisc/sys_helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/openrisc/sys_helper.c +++ b/target/openrisc/sys_helper.c @@ -XXX,XX +XXX,XX @@ void HELPER(mtspr)(CPUOpenRISCState *env, target_ulong spr, target_ulong rb) if (env->pc != rb) { env->pc = rb; env->dflag = 0; - cpu_loop_exit(cs); } + cpu_loop_exit(cs); break; case TO_SPR(0, 17): /* SR */ -- 2.34.1 Since we do not plan to exit, use cpu_unwind_state_data and extract exactly the data requested. This is a bug fix, in that we no longer clobber dflag. Consider: l.j L2 // branch l.mfspr r1, ppc // delay L1: boom L2: l.lwa r3, (r4) Here, dflag would be set by cpu_restore_state (because that is the current state of the cpu), but but not cleared by tb_stop on exiting the TB (because DisasContext has recorded the current value as zero). The next TB begins at L2 with dflag incorrectly set. If the load has a tlb miss, then the exception will be delivered as per a delay slot: with DSX set in the status register and PC decremented (delay slots restart by re-executing the branch). This will cause the return from interrupt to go to L1, and boom! Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- target/openrisc/sys_helper.c | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/target/openrisc/sys_helper.c b/target/openrisc/sys_helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/openrisc/sys_helper.c +++ b/target/openrisc/sys_helper.c @@ -XXX,XX +XXX,XX @@ target_ulong HELPER(mfspr)(CPUOpenRISCState *env, target_ulong rd, target_ulong spr) { #ifndef CONFIG_USER_ONLY + uint64_t data[TARGET_INSN_START_WORDS]; MachineState *ms = MACHINE(qdev_get_machine()); OpenRISCCPU *cpu = env_archcpu(env); CPUState *cs = env_cpu(env); @@ -XXX,XX +XXX,XX @@ target_ulong HELPER(mfspr)(CPUOpenRISCState *env, target_ulong rd, return env->evbar; case TO_SPR(0, 16): /* NPC (equals PC) */ - cpu_restore_state(cs, GETPC(), false); + if (cpu_unwind_state_data(cs, GETPC(), data)) { + return data[0]; + } return env->pc; case TO_SPR(0, 17): /* SR */ return cpu_get_sr(env); case TO_SPR(0, 18): /* PPC */ - cpu_restore_state(cs, GETPC(), false); + if (cpu_unwind_state_data(cs, GETPC(), data)) { + if (data[1] & 2) { + return data[0] - 4; + } + } return env->ppc; case TO_SPR(0, 32): /* EPCR */ -- 2.34.1 The value passed is always true, and if the target's synchronize_from_tb hook is non-trivial, not exiting may be erroneous. Reviewed-by: Claudio Fontana <cfontana@suse.de> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- include/exec/exec-all.h | 5 +---- accel/tcg/cpu-exec-common.c | 2 +- accel/tcg/translate-all.c | 12 ++---------- target/alpha/helper.c | 2 +- target/alpha/mem_helper.c | 2 +- target/arm/op_helper.c | 2 +- target/arm/tlb_helper.c | 8 ++++---- target/cris/helper.c | 2 +- target/i386/tcg/sysemu/svm_helper.c | 2 +- target/m68k/op_helper.c | 4 ++-- target/microblaze/helper.c | 2 +- target/nios2/op_helper.c | 2 +- target/openrisc/sys_helper.c | 4 ++-- target/ppc/excp_helper.c | 2 +- target/s390x/tcg/excp_helper.c | 2 +- target/tricore/op_helper.c | 2 +- target/xtensa/helper.c | 6 +++--- 17 files changed, 25 insertions(+), 36 deletions(-) diff --git a/include/exec/exec-all.h b/include/exec/exec-all.h index XXXXXXX..XXXXXXX 100644 --- a/include/exec/exec-all.h +++ b/include/exec/exec-all.h @@ -XXX,XX +XXX,XX @@ bool cpu_unwind_state_data(CPUState *cpu, uintptr_t host_pc, uint64_t *data); * cpu_restore_state: * @cpu: the cpu context * @host_pc: the host pc within the translation - * @will_exit: true if the TB executed will be interrupted after some - cpu adjustments. Required for maintaining the correct - icount valus * @return: true if state was restored, false otherwise * * Attempt to restore the state for a fault occurring in translated * code. If @host_pc is not in translated code no state is * restored and the function returns false. */ -bool cpu_restore_state(CPUState *cpu, uintptr_t host_pc, bool will_exit); +bool cpu_restore_state(CPUState *cpu, uintptr_t host_pc); G_NORETURN void cpu_loop_exit_noexc(CPUState *cpu); G_NORETURN void cpu_loop_exit(CPUState *cpu); diff --git a/accel/tcg/cpu-exec-common.c b/accel/tcg/cpu-exec-common.c index XXXXXXX..XXXXXXX 100644 --- a/accel/tcg/cpu-exec-common.c +++ b/accel/tcg/cpu-exec-common.c @@ -XXX,XX +XXX,XX @@ void cpu_loop_exit(CPUState *cpu) void cpu_loop_exit_restore(CPUState *cpu, uintptr_t pc) { if (pc) { - cpu_restore_state(cpu, pc, true); + cpu_restore_state(cpu, pc); } cpu_loop_exit(cpu); } diff --git a/accel/tcg/translate-all.c b/accel/tcg/translate-all.c index XXXXXXX..XXXXXXX 100644 --- a/accel/tcg/translate-all.c +++ b/accel/tcg/translate-all.c @@ -XXX,XX +XXX,XX @@ void cpu_restore_state_from_tb(CPUState *cpu, TranslationBlock *tb, #endif } -bool cpu_restore_state(CPUState *cpu, uintptr_t host_pc, bool will_exit) +bool cpu_restore_state(CPUState *cpu, uintptr_t host_pc) { - /* - * The pc update associated with restore without exit will - * break the relative pc adjustments performed by TARGET_TB_PCREL. - */ - if (TARGET_TB_PCREL) { - assert(will_exit); - } - /* * The host_pc has to be in the rx region of the code buffer. * If it is not we will not be able to resolve it here. @@ -XXX,XX +XXX,XX @@ bool cpu_restore_state(CPUState *cpu, uintptr_t host_pc, bool will_exit) if (in_code_gen_buffer((const void *)(host_pc - tcg_splitwx_diff))) { TranslationBlock *tb = tcg_tb_lookup(host_pc); if (tb) { - cpu_restore_state_from_tb(cpu, tb, host_pc, will_exit); + cpu_restore_state_from_tb(cpu, tb, host_pc, true); return true; } } diff --git a/target/alpha/helper.c b/target/alpha/helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/alpha/helper.c +++ b/target/alpha/helper.c @@ -XXX,XX +XXX,XX @@ G_NORETURN void dynamic_excp(CPUAlphaState *env, uintptr_t retaddr, cs->exception_index = excp; env->error_code = error; if (retaddr) { - cpu_restore_state(cs, retaddr, true); + cpu_restore_state(cs, retaddr); /* Floating-point exceptions (our only users) point to the next PC. */ env->pc += 4; } diff --git a/target/alpha/mem_helper.c b/target/alpha/mem_helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/alpha/mem_helper.c +++ b/target/alpha/mem_helper.c @@ -XXX,XX +XXX,XX @@ static void do_unaligned_access(CPUAlphaState *env, vaddr addr, uintptr_t retadd uint64_t pc; uint32_t insn; - cpu_restore_state(env_cpu(env), retaddr, true); + cpu_restore_state(env_cpu(env), retaddr); pc = env->pc; insn = cpu_ldl_code(env, pc); diff --git a/target/arm/op_helper.c b/target/arm/op_helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/op_helper.c +++ b/target/arm/op_helper.c @@ -XXX,XX +XXX,XX @@ void raise_exception_ra(CPUARMState *env, uint32_t excp, uint32_t syndrome, * we must restore CPU state here before setting the syndrome * the caller passed us, and cannot use cpu_loop_exit_restore(). */ - cpu_restore_state(cs, ra, true); + cpu_restore_state(cs, ra); raise_exception(env, excp, syndrome, target_el); } diff --git a/target/arm/tlb_helper.c b/target/arm/tlb_helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/arm/tlb_helper.c +++ b/target/arm/tlb_helper.c @@ -XXX,XX +XXX,XX @@ void arm_cpu_do_unaligned_access(CPUState *cs, vaddr vaddr, ARMMMUFaultInfo fi = {}; /* now we have a real cpu fault */ - cpu_restore_state(cs, retaddr, true); + cpu_restore_state(cs, retaddr); fi.type = ARMFault_Alignment; arm_deliver_fault(cpu, vaddr, access_type, mmu_idx, &fi); @@ -XXX,XX +XXX,XX @@ void arm_cpu_do_transaction_failed(CPUState *cs, hwaddr physaddr, ARMMMUFaultInfo fi = {}; /* now we have a real cpu fault */ - cpu_restore_state(cs, retaddr, true); + cpu_restore_state(cs, retaddr); fi.ea = arm_extabort_type(response); fi.type = ARMFault_SyncExternal; @@ -XXX,XX +XXX,XX @@ bool arm_cpu_tlb_fill(CPUState *cs, vaddr address, int size, return false; } else { /* now we have a real cpu fault */ - cpu_restore_state(cs, retaddr, true); + cpu_restore_state(cs, retaddr); arm_deliver_fault(cpu, address, access_type, mmu_idx, fi); } } @@ -XXX,XX +XXX,XX @@ void arm_cpu_record_sigsegv(CPUState *cs, vaddr addr, * We report both ESR and FAR to signal handlers. * For now, it's easiest to deliver the fault normally. */ - cpu_restore_state(cs, ra, true); + cpu_restore_state(cs, ra); arm_deliver_fault(cpu, addr, access_type, MMU_USER_IDX, &fi); } diff --git a/target/cris/helper.c b/target/cris/helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/cris/helper.c +++ b/target/cris/helper.c @@ -XXX,XX +XXX,XX @@ bool cris_cpu_tlb_fill(CPUState *cs, vaddr address, int size, cs->exception_index = EXCP_BUSFAULT; env->fault_vector = res.bf_vec; if (retaddr) { - if (cpu_restore_state(cs, retaddr, true)) { + if (cpu_restore_state(cs, retaddr)) { /* Evaluate flags after retranslation. */ helper_top_evaluate_flags(env); } diff --git a/target/i386/tcg/sysemu/svm_helper.c b/target/i386/tcg/sysemu/svm_helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/i386/tcg/sysemu/svm_helper.c +++ b/target/i386/tcg/sysemu/svm_helper.c @@ -XXX,XX +XXX,XX @@ void cpu_vmexit(CPUX86State *env, uint32_t exit_code, uint64_t exit_info_1, { CPUState *cs = env_cpu(env); - cpu_restore_state(cs, retaddr, true); + cpu_restore_state(cs, retaddr); qemu_log_mask(CPU_LOG_TB_IN_ASM, "vmexit(%08x, %016" PRIx64 ", %016" PRIx64 ", " TARGET_FMT_lx ")!\n", diff --git a/target/m68k/op_helper.c b/target/m68k/op_helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/m68k/op_helper.c +++ b/target/m68k/op_helper.c @@ -XXX,XX +XXX,XX @@ void m68k_cpu_transaction_failed(CPUState *cs, hwaddr physaddr, vaddr addr, M68kCPU *cpu = M68K_CPU(cs); CPUM68KState *env = &cpu->env; - cpu_restore_state(cs, retaddr, true); + cpu_restore_state(cs, retaddr); if (m68k_feature(env, M68K_FEATURE_M68040)) { env->mmu.mmusr = 0; @@ -XXX,XX +XXX,XX @@ raise_exception_format2(CPUM68KState *env, int tt, int ilen, uintptr_t raddr) cs->exception_index = tt; /* Recover PC and CC_OP for the beginning of the insn. */ - cpu_restore_state(cs, raddr, true); + cpu_restore_state(cs, raddr); /* Flags are current in env->cc_*, or are undefined. */ env->cc_op = CC_OP_FLAGS; diff --git a/target/microblaze/helper.c b/target/microblaze/helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/microblaze/helper.c +++ b/target/microblaze/helper.c @@ -XXX,XX +XXX,XX @@ void mb_cpu_do_unaligned_access(CPUState *cs, vaddr addr, uint32_t esr, iflags; /* Recover the pc and iflags from the corresponding insn_start. */ - cpu_restore_state(cs, retaddr, true); + cpu_restore_state(cs, retaddr); iflags = cpu->env.iflags; qemu_log_mask(CPU_LOG_INT, diff --git a/target/nios2/op_helper.c b/target/nios2/op_helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/nios2/op_helper.c +++ b/target/nios2/op_helper.c @@ -XXX,XX +XXX,XX @@ void nios2_cpu_loop_exit_advance(CPUNios2State *env, uintptr_t retaddr) * Do this here, rather than in restore_state_to_opc(), * lest we affect QEMU internal exceptions, like EXCP_DEBUG. */ - cpu_restore_state(cs, retaddr, true); + cpu_restore_state(cs, retaddr); env->pc += 4; cpu_loop_exit(cs); } diff --git a/target/openrisc/sys_helper.c b/target/openrisc/sys_helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/openrisc/sys_helper.c +++ b/target/openrisc/sys_helper.c @@ -XXX,XX +XXX,XX @@ void HELPER(mtspr)(CPUOpenRISCState *env, target_ulong spr, target_ulong rb) break; case TO_SPR(0, 16): /* NPC */ - cpu_restore_state(cs, GETPC(), true); + cpu_restore_state(cs, GETPC()); /* ??? Mirror or1ksim in not trashing delayed branch state when "jumping" to the current instruction. */ if (env->pc != rb) { @@ -XXX,XX +XXX,XX @@ void HELPER(mtspr)(CPUOpenRISCState *env, target_ulong spr, target_ulong rb) case TO_SPR(8, 0): /* PMR */ env->pmr = rb; if (env->pmr & PMR_DME || env->pmr & PMR_SME) { - cpu_restore_state(cs, GETPC(), true); + cpu_restore_state(cs, GETPC()); env->pc += 4; cs->halted = 1; raise_exception(cpu, EXCP_HALTED); diff --git a/target/ppc/excp_helper.c b/target/ppc/excp_helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/ppc/excp_helper.c +++ b/target/ppc/excp_helper.c @@ -XXX,XX +XXX,XX @@ void ppc_cpu_do_unaligned_access(CPUState *cs, vaddr vaddr, uint32_t insn; /* Restore state and reload the insn we executed, for filling in DSISR. */ - cpu_restore_state(cs, retaddr, true); + cpu_restore_state(cs, retaddr); insn = cpu_ldl_code(env, env->nip); switch (env->mmu_model) { diff --git a/target/s390x/tcg/excp_helper.c b/target/s390x/tcg/excp_helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/s390x/tcg/excp_helper.c +++ b/target/s390x/tcg/excp_helper.c @@ -XXX,XX +XXX,XX @@ G_NORETURN void tcg_s390_program_interrupt(CPUS390XState *env, { CPUState *cs = env_cpu(env); - cpu_restore_state(cs, ra, true); + cpu_restore_state(cs, ra); qemu_log_mask(CPU_LOG_INT, "program interrupt at %#" PRIx64 "\n", env->psw.addr); trigger_pgm_exception(env, code); diff --git a/target/tricore/op_helper.c b/target/tricore/op_helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/tricore/op_helper.c +++ b/target/tricore/op_helper.c @@ -XXX,XX +XXX,XX @@ void raise_exception_sync_internal(CPUTriCoreState *env, uint32_t class, int tin { CPUState *cs = env_cpu(env); /* in case we come from a helper-call we need to restore the PC */ - cpu_restore_state(cs, pc, true); + cpu_restore_state(cs, pc); /* Tin is loaded into d[15] */ env->gpr_d[15] = tin; diff --git a/target/xtensa/helper.c b/target/xtensa/helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/xtensa/helper.c +++ b/target/xtensa/helper.c @@ -XXX,XX +XXX,XX @@ void xtensa_cpu_do_unaligned_access(CPUState *cs, assert(xtensa_option_enabled(env->config, XTENSA_OPTION_UNALIGNED_EXCEPTION)); - cpu_restore_state(CPU(cpu), retaddr, true); + cpu_restore_state(CPU(cpu), retaddr); HELPER(exception_cause_vaddr)(env, env->pc, LOAD_STORE_ALIGNMENT_CAUSE, addr); @@ -XXX,XX +XXX,XX @@ bool xtensa_cpu_tlb_fill(CPUState *cs, vaddr address, int size, } else if (probe) { return false; } else { - cpu_restore_state(cs, retaddr, true); + cpu_restore_state(cs, retaddr); HELPER(exception_cause_vaddr)(env, env->pc, ret, address); } } @@ -XXX,XX +XXX,XX @@ void xtensa_cpu_do_transaction_failed(CPUState *cs, hwaddr physaddr, vaddr addr, XtensaCPU *cpu = XTENSA_CPU(cs); CPUXtensaState *env = &cpu->env; - cpu_restore_state(cs, retaddr, true); + cpu_restore_state(cs, retaddr); HELPER(exception_cause_vaddr)(env, env->pc, access_type == MMU_INST_FETCH ? INSTR_PIF_ADDR_ERROR_CAUSE : -- 2.34.1 The value passed is always true. Reviewed-by: Claudio Fontana <cfontana@suse.de> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- accel/tcg/internal.h | 2 +- accel/tcg/tb-maint.c | 4 ++-- accel/tcg/translate-all.c | 15 +++++++-------- 3 files changed, 10 insertions(+), 11 deletions(-) diff --git a/accel/tcg/internal.h b/accel/tcg/internal.h index XXXXXXX..XXXXXXX 100644 --- a/accel/tcg/internal.h +++ b/accel/tcg/internal.h @@ -XXX,XX +XXX,XX @@ TranslationBlock *tb_link_page(TranslationBlock *tb, tb_page_addr_t phys_pc, tb_page_addr_t phys_page2); bool tb_invalidate_phys_page_unwind(tb_page_addr_t addr, uintptr_t pc); void cpu_restore_state_from_tb(CPUState *cpu, TranslationBlock *tb, - uintptr_t host_pc, bool reset_icount); + uintptr_t host_pc); /* Return the current PC from CPU, which may be cached in TB. */ static inline target_ulong log_pc(CPUState *cpu, const TranslationBlock *tb) diff --git a/accel/tcg/tb-maint.c b/accel/tcg/tb-maint.c index XXXXXXX..XXXXXXX 100644 --- a/accel/tcg/tb-maint.c +++ b/accel/tcg/tb-maint.c @@ -XXX,XX +XXX,XX @@ tb_invalidate_phys_page_range__locked(struct page_collection *pages, * restore the CPU state. */ current_tb_modified = true; - cpu_restore_state_from_tb(cpu, current_tb, retaddr, true); + cpu_restore_state_from_tb(cpu, current_tb, retaddr); } #endif /* TARGET_HAS_PRECISE_SMC */ tb_phys_invalidate__locked(tb); @@ -XXX,XX +XXX,XX @@ bool tb_invalidate_phys_page_unwind(tb_page_addr_t addr, uintptr_t pc) * function to partially restore the CPU state. */ current_tb_modified = true; - cpu_restore_state_from_tb(cpu, current_tb, pc, true); + cpu_restore_state_from_tb(cpu, current_tb, pc); } #endif /* TARGET_HAS_PRECISE_SMC */ tb_phys_invalidate(tb, addr); diff --git a/accel/tcg/translate-all.c b/accel/tcg/translate-all.c index XXXXXXX..XXXXXXX 100644 --- a/accel/tcg/translate-all.c +++ b/accel/tcg/translate-all.c @@ -XXX,XX +XXX,XX @@ static int cpu_unwind_data_from_tb(TranslationBlock *tb, uintptr_t host_pc, } /* - * The cpu state corresponding to 'host_pc' is restored. - * When reset_icount is true, current TB will be interrupted and - * icount should be recalculated. + * The cpu state corresponding to 'host_pc' is restored in + * preparation for exiting the TB. */ void cpu_restore_state_from_tb(CPUState *cpu, TranslationBlock *tb, - uintptr_t host_pc, bool reset_icount) + uintptr_t host_pc) { uint64_t data[TARGET_INSN_START_WORDS]; #ifdef CONFIG_PROFILER @@ -XXX,XX +XXX,XX @@ void cpu_restore_state_from_tb(CPUState *cpu, TranslationBlock *tb, return; } - if (reset_icount && (tb_cflags(tb) & CF_USE_ICOUNT)) { + if (tb_cflags(tb) & CF_USE_ICOUNT) { assert(icount_enabled()); /* * Reset the cycle counter to the start of the block and @@ -XXX,XX +XXX,XX @@ bool cpu_restore_state(CPUState *cpu, uintptr_t host_pc) if (in_code_gen_buffer((const void *)(host_pc - tcg_splitwx_diff))) { TranslationBlock *tb = tcg_tb_lookup(host_pc); if (tb) { - cpu_restore_state_from_tb(cpu, tb, host_pc, true); + cpu_restore_state_from_tb(cpu, tb, host_pc); return true; } } @@ -XXX,XX +XXX,XX @@ void tb_check_watchpoint(CPUState *cpu, uintptr_t retaddr) tb = tcg_tb_lookup(retaddr); if (tb) { /* We can use retranslation to find the PC. */ - cpu_restore_state_from_tb(cpu, tb, retaddr, true); + cpu_restore_state_from_tb(cpu, tb, retaddr); tb_phys_invalidate(tb, -1); } else { /* The exception probably happened in a helper. The CPU state should @@ -XXX,XX +XXX,XX @@ void cpu_io_recompile(CPUState *cpu, uintptr_t retaddr) cpu_abort(cpu, "cpu_io_recompile: could not find TB for pc=%p", (void *)retaddr); } - cpu_restore_state_from_tb(cpu, tb, retaddr, true); + cpu_restore_state_from_tb(cpu, tb, retaddr); /* * Some guests must re-execute the branch when re-executing a delay -- 2.34.1 The helpers for reset_rf, cli, sti, clac, stac are completely trivial; implement them inline. Drop some nearby #if 0 code. Reviewed-by: Paolo Bonzini <pbonzini@redhat.com> Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- target/i386/helper.h | 5 ----- target/i386/tcg/cc_helper.c | 41 ------------------------------------- target/i386/tcg/translate.c | 30 ++++++++++++++++++++++----- 3 files changed, 25 insertions(+), 51 deletions(-) diff --git a/target/i386/helper.h b/target/i386/helper.h index XXXXXXX..XXXXXXX 100644 --- a/target/i386/helper.h +++ b/target/i386/helper.h @@ -XXX,XX +XXX,XX @@ DEF_HELPER_2(syscall, void, env, int) DEF_HELPER_2(sysret, void, env, int) #endif DEF_HELPER_FLAGS_2(pause, TCG_CALL_NO_WG, noreturn, env, int) -DEF_HELPER_1(reset_rf, void, env) DEF_HELPER_FLAGS_3(raise_interrupt, TCG_CALL_NO_WG, noreturn, env, int, int) DEF_HELPER_FLAGS_2(raise_exception, TCG_CALL_NO_WG, noreturn, env, int) -DEF_HELPER_1(cli, void, env) -DEF_HELPER_1(sti, void, env) -DEF_HELPER_1(clac, void, env) -DEF_HELPER_1(stac, void, env) DEF_HELPER_3(boundw, void, env, tl, int) DEF_HELPER_3(boundl, void, env, tl, int) diff --git a/target/i386/tcg/cc_helper.c b/target/i386/tcg/cc_helper.c index XXXXXXX..XXXXXXX 100644 --- a/target/i386/tcg/cc_helper.c +++ b/target/i386/tcg/cc_helper.c @@ -XXX,XX +XXX,XX @@ void helper_clts(CPUX86State *env) env->cr[0] &= ~CR0_TS_MASK; env->hflags &= ~HF_TS_MASK; } - -void helper_reset_rf(CPUX86State *env) -{ - env->eflags &= ~RF_MASK; -} - -void helper_cli(CPUX86State *env) -{ - env->eflags &= ~IF_MASK; -} - -void helper_sti(CPUX86State *env) -{ - env->eflags |= IF_MASK; -} - -void helper_clac(CPUX86State *env) -{ - env->eflags &= ~AC_MASK; -} - -void helper_stac(CPUX86State *env) -{ - env->eflags |= AC_MASK; -} - -#if 0 -/* vm86plus instructions */ -void helper_cli_vm(CPUX86State *env) -{ - env->eflags &= ~VIF_MASK; -} - -void helper_sti_vm(CPUX86State *env) -{ - env->eflags |= VIF_MASK; - if (env->eflags & VIP_MASK) { - raise_exception_ra(env, EXCP0D_GPF, GETPC()); - } -} -#endif diff --git a/target/i386/tcg/translate.c b/target/i386/tcg/translate.c index XXXXXXX..XXXXXXX 100644 --- a/target/i386/tcg/translate.c +++ b/target/i386/tcg/translate.c @@ -XXX,XX +XXX,XX @@ static void gen_reset_hflag(DisasContext *s, uint32_t mask) } } +static void gen_set_eflags(DisasContext *s, target_ulong mask) +{ + TCGv t = tcg_temp_new(); + + tcg_gen_ld_tl(t, cpu_env, offsetof(CPUX86State, eflags)); + tcg_gen_ori_tl(t, t, mask); + tcg_gen_st_tl(t, cpu_env, offsetof(CPUX86State, eflags)); + tcg_temp_free(t); +} + +static void gen_reset_eflags(DisasContext *s, target_ulong mask) +{ + TCGv t = tcg_temp_new(); + + tcg_gen_ld_tl(t, cpu_env, offsetof(CPUX86State, eflags)); + tcg_gen_andi_tl(t, t, ~mask); + tcg_gen_st_tl(t, cpu_env, offsetof(CPUX86State, eflags)); + tcg_temp_free(t); +} + /* Clear BND registers during legacy branches. */ static void gen_bnd_jmp(DisasContext *s) { @@ -XXX,XX +XXX,XX @@ do_gen_eob_worker(DisasContext *s, bool inhibit, bool recheck_tf, bool jr) } if (s->base.tb->flags & HF_RF_MASK) { - gen_helper_reset_rf(cpu_env); + gen_reset_eflags(s, RF_MASK); } if (recheck_tf) { gen_helper_rechecking_single_step(cpu_env); @@ -XXX,XX +XXX,XX @@ static bool disas_insn(DisasContext *s, CPUState *cpu) #endif case 0xfa: /* cli */ if (check_iopl(s)) { - gen_helper_cli(cpu_env); + gen_reset_eflags(s, IF_MASK); } break; case 0xfb: /* sti */ if (check_iopl(s)) { - gen_helper_sti(cpu_env); + gen_set_eflags(s, IF_MASK); /* interruptions are enabled only the first insn after sti */ gen_update_eip_next(s); gen_eob_inhibit_irq(s, true); @@ -XXX,XX +XXX,XX @@ static bool disas_insn(DisasContext *s, CPUState *cpu) || CPL(s) != 0) { goto illegal_op; } - gen_helper_clac(cpu_env); + gen_reset_eflags(s, AC_MASK); s->base.is_jmp = DISAS_EOB_NEXT; break; @@ -XXX,XX +XXX,XX @@ static bool disas_insn(DisasContext *s, CPUState *cpu) || CPL(s) != 0) { goto illegal_op; } - gen_helper_stac(cpu_env); + gen_set_eflags(s, AC_MASK); s->base.is_jmp = DISAS_EOB_NEXT; break; -- 2.34.1