1 | The following changes since commit 3e08b2b9cb64bff2b73fa9128c0e49bfcde0dd40: | 1 | Second pull for this week, since this set is large enough by itself. |
---|---|---|---|
2 | 2 | ||
3 | Merge remote-tracking branch 'remotes/philmd-gitlab/tags/edk2-next-20200121' into staging (2020-01-21 15:29:25 +0000) | 3 | |
4 | r~ | ||
5 | |||
6 | |||
7 | The following changes since commit 7c9236d6d61f30583d5d860097d88dbf0fe487bf: | ||
8 | |||
9 | Merge tag 'pull-tcg-20230116' of https://gitlab.com/rth7680/qemu into staging (2023-01-17 10:24:16 +0000) | ||
4 | 10 | ||
5 | are available in the Git repository at: | 11 | are available in the Git repository at: |
6 | 12 | ||
7 | https://github.com/rth7680/qemu.git tags/pull-tcg-20200121 | 13 | https://gitlab.com/rth7680/qemu.git tags/pull-tcg-20230117 |
8 | 14 | ||
9 | for you to fetch changes up to 75fa376cdab5e5db2c7fdd107358e16f95503ac6: | 15 | for you to fetch changes up to 493c9b19a7fb7f387c4fcf57d3836504d5242bf5: |
10 | 16 | ||
11 | scripts/git.orderfile: Display decodetree before C source (2020-01-21 15:26:09 -1000) | 17 | tcg/riscv: Implement direct branch for goto_tb (2023-01-17 22:36:17 +0000) |
12 | 18 | ||
13 | ---------------------------------------------------------------- | 19 | ---------------------------------------------------------------- |
14 | Remove another limit to NB_MMU_MODES. | 20 | tcg: Fix race conditions in (most) goto_tb implementations |
15 | Fix compilation using uclibc. | ||
16 | Fix defaulting of -accel parameters. | ||
17 | Tidy cputlb basic routines. | ||
18 | Adjust git.orderfile for decodetree. | ||
19 | 21 | ||
20 | ---------------------------------------------------------------- | 22 | ---------------------------------------------------------------- |
21 | Carlos Santos (1): | 23 | Richard Henderson (22): |
22 | util/cacheinfo: fix crash when compiling with uClibc | 24 | tcg: Split out tcg_out_exit_tb |
25 | tcg/i386: Remove unused goto_tb code for indirect jump | ||
26 | tcg/ppc: Remove unused goto_tb code for indirect jump | ||
27 | tcg/sparc64: Remove unused goto_tb code for indirect jump | ||
28 | tcg: Replace asserts on tcg_jmp_insn_offset | ||
29 | tcg: Introduce set_jmp_insn_offset | ||
30 | tcg: Introduce get_jmp_target_addr | ||
31 | tcg: Split out tcg_out_goto_tb | ||
32 | tcg: Rename TB_JMP_RESET_OFFSET_INVALID to TB_JMP_OFFSET_INVALID | ||
33 | tcg: Add gen_tb to TCGContext | ||
34 | tcg: Add TranslationBlock.jmp_insn_offset | ||
35 | tcg: Change tb_target_set_jmp_target arguments | ||
36 | tcg: Move tb_target_set_jmp_target declaration to tcg.h | ||
37 | tcg: Always define tb_target_set_jmp_target | ||
38 | tcg: Remove TCG_TARGET_HAS_direct_jump | ||
39 | tcg/aarch64: Reorg goto_tb implementation | ||
40 | tcg/ppc: Reorg goto_tb implementation | ||
41 | tcg/sparc64: Remove USE_REG_TB | ||
42 | tcg/sparc64: Reorg goto_tb implementation | ||
43 | tcg/arm: Implement direct branch for goto_tb | ||
44 | tcg/riscv: Introduce OPC_NOP | ||
45 | tcg/riscv: Implement direct branch for goto_tb | ||
23 | 46 | ||
24 | Philippe Mathieu-Daudé (1): | 47 | include/exec/exec-all.h | 5 +- |
25 | scripts/git.orderfile: Display decodetree before C source | 48 | include/tcg/tcg.h | 14 ++- |
26 | 49 | tcg/aarch64/tcg-target.h | 6 +- | |
27 | Richard Henderson (14): | 50 | tcg/arm/tcg-target.h | 5 - |
28 | cputlb: Handle NB_MMU_MODES > TARGET_PAGE_BITS_MIN | 51 | tcg/i386/tcg-target.h | 9 -- |
29 | vl: Remove unused variable in configure_accelerators | 52 | tcg/loongarch64/tcg-target.h | 3 - |
30 | vl: Reduce scope of variables in configure_accelerators | 53 | tcg/mips/tcg-target.h | 5 - |
31 | vl: Remove useless test in configure_accelerators | 54 | tcg/ppc/tcg-target.h | 7 +- |
32 | vl: Only choose enabled accelerators in configure_accelerators | 55 | tcg/riscv/tcg-target.h | 4 - |
33 | cputlb: Merge tlb_table_flush_by_mmuidx into tlb_flush_one_mmuidx_locked | 56 | tcg/s390x/tcg-target.h | 11 --- |
34 | cputlb: Make tlb_n_entries private to cputlb.c | 57 | tcg/sparc64/tcg-target.h | 4 - |
35 | cputlb: Pass CPUTLBDescFast to tlb_n_entries and sizeof_tlb | 58 | tcg/tci/tcg-target.h | 4 - |
36 | cputlb: Hoist tlb portions in tlb_mmu_resize_locked | 59 | accel/tcg/cpu-exec.c | 21 ++-- |
37 | cputlb: Hoist tlb portions in tlb_flush_one_mmuidx_locked | 60 | accel/tcg/translate-all.c | 10 +- |
38 | cputlb: Split out tlb_mmu_flush_locked | 61 | tcg/tcg-op.c | 14 +-- |
39 | cputlb: Partially merge tlb_dyn_init into tlb_init | 62 | tcg/tcg.c | 42 +++++--- |
40 | cputlb: Initialize tlbs as flushed | 63 | tcg/aarch64/tcg-target.c.inc | 106 ++++++++++----------- |
41 | cputlb: Hoist timestamp outside of loops over tlbs | 64 | tcg/arm/tcg-target.c.inc | 89 +++++++++++------ |
42 | 65 | tcg/i386/tcg-target.c.inc | 68 +++++++------ | |
43 | include/exec/cpu_ldst.h | 5 - | 66 | tcg/loongarch64/tcg-target.c.inc | 66 +++++++------ |
44 | accel/tcg/cputlb.c | 287 +++++++++++++++++++++++++++++++++--------------- | 67 | tcg/mips/tcg-target.c.inc | 59 +++++++----- |
45 | util/cacheinfo.c | 10 +- | 68 | tcg/ppc/tcg-target.c.inc | 193 ++++++++++++------------------------- |
46 | vl.c | 27 +++-- | 69 | tcg/riscv/tcg-target.c.inc | 65 +++++++++---- |
47 | scripts/git.orderfile | 3 + | 70 | tcg/s390x/tcg-target.c.inc | 67 ++++++++----- |
48 | 5 files changed, 223 insertions(+), 109 deletions(-) | 71 | tcg/sparc64/tcg-target.c.inc | 201 +++++++++++++++------------------------ |
49 | 72 | tcg/tci/tcg-target.c.inc | 31 +++--- | |
73 | 26 files changed, 528 insertions(+), 581 deletions(-) | diff view generated by jsdifflib |
New patch | |||
---|---|---|---|
1 | The INDEX_op_exit_tb opcode needs no register allocation. | ||
2 | Split out a dedicated helper function for it. | ||
1 | 3 | ||
4 | Reviewed-by: Alex Bennée <alex.bennee@linaro.org> | ||
5 | Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org> | ||
6 | Signed-off-by: Richard Henderson <richard.henderson@linaro.org> | ||
7 | --- | ||
8 | tcg/tcg.c | 4 ++++ | ||
9 | tcg/aarch64/tcg-target.c.inc | 22 ++++++++++-------- | ||
10 | tcg/arm/tcg-target.c.inc | 11 +++++---- | ||
11 | tcg/i386/tcg-target.c.inc | 21 +++++++++-------- | ||
12 | tcg/loongarch64/tcg-target.c.inc | 22 ++++++++++-------- | ||
13 | tcg/mips/tcg-target.c.inc | 33 +++++++++++++-------------- | ||
14 | tcg/ppc/tcg-target.c.inc | 11 +++++---- | ||
15 | tcg/riscv/tcg-target.c.inc | 22 ++++++++++-------- | ||
16 | tcg/s390x/tcg-target.c.inc | 23 ++++++++++--------- | ||
17 | tcg/sparc64/tcg-target.c.inc | 39 +++++++++++++++++--------------- | ||
18 | tcg/tci/tcg-target.c.inc | 10 ++++---- | ||
19 | 11 files changed, 121 insertions(+), 97 deletions(-) | ||
20 | |||
21 | diff --git a/tcg/tcg.c b/tcg/tcg.c | ||
22 | index XXXXXXX..XXXXXXX 100644 | ||
23 | --- a/tcg/tcg.c | ||
24 | +++ b/tcg/tcg.c | ||
25 | @@ -XXX,XX +XXX,XX @@ static void tcg_out_ld(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg1, | ||
26 | static bool tcg_out_mov(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg); | ||
27 | static void tcg_out_movi(TCGContext *s, TCGType type, | ||
28 | TCGReg ret, tcg_target_long arg); | ||
29 | +static void tcg_out_exit_tb(TCGContext *s, uintptr_t arg); | ||
30 | static void tcg_out_op(TCGContext *s, TCGOpcode opc, | ||
31 | const TCGArg args[TCG_MAX_OP_ARGS], | ||
32 | const int const_args[TCG_MAX_OP_ARGS]); | ||
33 | @@ -XXX,XX +XXX,XX @@ int tcg_gen_code(TCGContext *s, TranslationBlock *tb, target_ulong pc_start) | ||
34 | case INDEX_op_call: | ||
35 | tcg_reg_alloc_call(s, op); | ||
36 | break; | ||
37 | + case INDEX_op_exit_tb: | ||
38 | + tcg_out_exit_tb(s, op->args[0]); | ||
39 | + break; | ||
40 | case INDEX_op_dup2_vec: | ||
41 | if (tcg_reg_alloc_dup2(s, op)) { | ||
42 | break; | ||
43 | diff --git a/tcg/aarch64/tcg-target.c.inc b/tcg/aarch64/tcg-target.c.inc | ||
44 | index XXXXXXX..XXXXXXX 100644 | ||
45 | --- a/tcg/aarch64/tcg-target.c.inc | ||
46 | +++ b/tcg/aarch64/tcg-target.c.inc | ||
47 | @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_st(TCGContext *s, TCGReg data_reg, TCGReg addr_reg, | ||
48 | |||
49 | static const tcg_insn_unit *tb_ret_addr; | ||
50 | |||
51 | +static void tcg_out_exit_tb(TCGContext *s, uintptr_t a0) | ||
52 | +{ | ||
53 | + /* Reuse the zeroing that exists for goto_ptr. */ | ||
54 | + if (a0 == 0) { | ||
55 | + tcg_out_goto_long(s, tcg_code_gen_epilogue); | ||
56 | + } else { | ||
57 | + tcg_out_movi(s, TCG_TYPE_I64, TCG_REG_X0, a0); | ||
58 | + tcg_out_goto_long(s, tb_ret_addr); | ||
59 | + } | ||
60 | +} | ||
61 | + | ||
62 | static void tcg_out_op(TCGContext *s, TCGOpcode opc, | ||
63 | const TCGArg args[TCG_MAX_OP_ARGS], | ||
64 | const int const_args[TCG_MAX_OP_ARGS]) | ||
65 | @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, | ||
66 | #define REG0(I) (const_args[I] ? TCG_REG_XZR : (TCGReg)args[I]) | ||
67 | |||
68 | switch (opc) { | ||
69 | - case INDEX_op_exit_tb: | ||
70 | - /* Reuse the zeroing that exists for goto_ptr. */ | ||
71 | - if (a0 == 0) { | ||
72 | - tcg_out_goto_long(s, tcg_code_gen_epilogue); | ||
73 | - } else { | ||
74 | - tcg_out_movi(s, TCG_TYPE_I64, TCG_REG_X0, a0); | ||
75 | - tcg_out_goto_long(s, tb_ret_addr); | ||
76 | - } | ||
77 | - break; | ||
78 | - | ||
79 | case INDEX_op_goto_tb: | ||
80 | tcg_debug_assert(s->tb_jmp_insn_offset != NULL); | ||
81 | /* | ||
82 | @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, | ||
83 | case INDEX_op_mov_i32: /* Always emitted via tcg_out_mov. */ | ||
84 | case INDEX_op_mov_i64: | ||
85 | case INDEX_op_call: /* Always emitted via tcg_out_call. */ | ||
86 | + case INDEX_op_exit_tb: /* Always emitted via tcg_out_exit_tb. */ | ||
87 | default: | ||
88 | g_assert_not_reached(); | ||
89 | } | ||
90 | diff --git a/tcg/arm/tcg-target.c.inc b/tcg/arm/tcg-target.c.inc | ||
91 | index XXXXXXX..XXXXXXX 100644 | ||
92 | --- a/tcg/arm/tcg-target.c.inc | ||
93 | +++ b/tcg/arm/tcg-target.c.inc | ||
94 | @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, bool is64) | ||
95 | |||
96 | static void tcg_out_epilogue(TCGContext *s); | ||
97 | |||
98 | +static void tcg_out_exit_tb(TCGContext *s, uintptr_t arg) | ||
99 | +{ | ||
100 | + tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_R0, arg); | ||
101 | + tcg_out_epilogue(s); | ||
102 | +} | ||
103 | + | ||
104 | static void tcg_out_op(TCGContext *s, TCGOpcode opc, | ||
105 | const TCGArg args[TCG_MAX_OP_ARGS], | ||
106 | const int const_args[TCG_MAX_OP_ARGS]) | ||
107 | @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, | ||
108 | int c; | ||
109 | |||
110 | switch (opc) { | ||
111 | - case INDEX_op_exit_tb: | ||
112 | - tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_R0, args[0]); | ||
113 | - tcg_out_epilogue(s); | ||
114 | - break; | ||
115 | case INDEX_op_goto_tb: | ||
116 | { | ||
117 | /* Indirect jump method */ | ||
118 | @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, | ||
119 | |||
120 | case INDEX_op_mov_i32: /* Always emitted via tcg_out_mov. */ | ||
121 | case INDEX_op_call: /* Always emitted via tcg_out_call. */ | ||
122 | + case INDEX_op_exit_tb: /* Always emitted via tcg_out_exit_tb. */ | ||
123 | default: | ||
124 | tcg_abort(); | ||
125 | } | ||
126 | diff --git a/tcg/i386/tcg-target.c.inc b/tcg/i386/tcg-target.c.inc | ||
127 | index XXXXXXX..XXXXXXX 100644 | ||
128 | --- a/tcg/i386/tcg-target.c.inc | ||
129 | +++ b/tcg/i386/tcg-target.c.inc | ||
130 | @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, bool is64) | ||
131 | #endif | ||
132 | } | ||
133 | |||
134 | +static void tcg_out_exit_tb(TCGContext *s, uintptr_t a0) | ||
135 | +{ | ||
136 | + /* Reuse the zeroing that exists for goto_ptr. */ | ||
137 | + if (a0 == 0) { | ||
138 | + tcg_out_jmp(s, tcg_code_gen_epilogue); | ||
139 | + } else { | ||
140 | + tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_EAX, a0); | ||
141 | + tcg_out_jmp(s, tb_ret_addr); | ||
142 | + } | ||
143 | +} | ||
144 | + | ||
145 | static inline void tcg_out_op(TCGContext *s, TCGOpcode opc, | ||
146 | const TCGArg args[TCG_MAX_OP_ARGS], | ||
147 | const int const_args[TCG_MAX_OP_ARGS]) | ||
148 | @@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc, | ||
149 | const_a2 = const_args[2]; | ||
150 | |||
151 | switch (opc) { | ||
152 | - case INDEX_op_exit_tb: | ||
153 | - /* Reuse the zeroing that exists for goto_ptr. */ | ||
154 | - if (a0 == 0) { | ||
155 | - tcg_out_jmp(s, tcg_code_gen_epilogue); | ||
156 | - } else { | ||
157 | - tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_EAX, a0); | ||
158 | - tcg_out_jmp(s, tb_ret_addr); | ||
159 | - } | ||
160 | - break; | ||
161 | case INDEX_op_goto_tb: | ||
162 | if (s->tb_jmp_insn_offset) { | ||
163 | /* direct jump method */ | ||
164 | @@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc, | ||
165 | case INDEX_op_mov_i32: /* Always emitted via tcg_out_mov. */ | ||
166 | case INDEX_op_mov_i64: | ||
167 | case INDEX_op_call: /* Always emitted via tcg_out_call. */ | ||
168 | + case INDEX_op_exit_tb: /* Always emitted via tcg_out_exit_tb. */ | ||
169 | default: | ||
170 | tcg_abort(); | ||
171 | } | ||
172 | diff --git a/tcg/loongarch64/tcg-target.c.inc b/tcg/loongarch64/tcg-target.c.inc | ||
173 | index XXXXXXX..XXXXXXX 100644 | ||
174 | --- a/tcg/loongarch64/tcg-target.c.inc | ||
175 | +++ b/tcg/loongarch64/tcg-target.c.inc | ||
176 | @@ -XXX,XX +XXX,XX @@ void tb_target_set_jmp_target(uintptr_t tc_ptr, uintptr_t jmp_rx, | ||
177 | |||
178 | static const tcg_insn_unit *tb_ret_addr; | ||
179 | |||
180 | +static void tcg_out_exit_tb(TCGContext *s, uintptr_t a0) | ||
181 | +{ | ||
182 | + /* Reuse the zeroing that exists for goto_ptr. */ | ||
183 | + if (a0 == 0) { | ||
184 | + tcg_out_call_int(s, tcg_code_gen_epilogue, true); | ||
185 | + } else { | ||
186 | + tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_A0, a0); | ||
187 | + tcg_out_call_int(s, tb_ret_addr, true); | ||
188 | + } | ||
189 | +} | ||
190 | + | ||
191 | static void tcg_out_op(TCGContext *s, TCGOpcode opc, | ||
192 | const TCGArg args[TCG_MAX_OP_ARGS], | ||
193 | const int const_args[TCG_MAX_OP_ARGS]) | ||
194 | @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, | ||
195 | int c2 = const_args[2]; | ||
196 | |||
197 | switch (opc) { | ||
198 | - case INDEX_op_exit_tb: | ||
199 | - /* Reuse the zeroing that exists for goto_ptr. */ | ||
200 | - if (a0 == 0) { | ||
201 | - tcg_out_call_int(s, tcg_code_gen_epilogue, true); | ||
202 | - } else { | ||
203 | - tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_A0, a0); | ||
204 | - tcg_out_call_int(s, tb_ret_addr, true); | ||
205 | - } | ||
206 | - break; | ||
207 | - | ||
208 | case INDEX_op_goto_tb: | ||
209 | tcg_debug_assert(s->tb_jmp_insn_offset != NULL); | ||
210 | /* | ||
211 | @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, | ||
212 | case INDEX_op_mov_i32: /* Always emitted via tcg_out_mov. */ | ||
213 | case INDEX_op_mov_i64: | ||
214 | case INDEX_op_call: /* Always emitted via tcg_out_call. */ | ||
215 | + case INDEX_op_exit_tb: /* Always emitted via tcg_out_exit_tb. */ | ||
216 | default: | ||
217 | g_assert_not_reached(); | ||
218 | } | ||
219 | diff --git a/tcg/mips/tcg-target.c.inc b/tcg/mips/tcg-target.c.inc | ||
220 | index XXXXXXX..XXXXXXX 100644 | ||
221 | --- a/tcg/mips/tcg-target.c.inc | ||
222 | +++ b/tcg/mips/tcg-target.c.inc | ||
223 | @@ -XXX,XX +XXX,XX @@ static void tcg_out_clz(TCGContext *s, MIPSInsn opcv2, MIPSInsn opcv6, | ||
224 | } | ||
225 | } | ||
226 | |||
227 | +static void tcg_out_exit_tb(TCGContext *s, uintptr_t a0) | ||
228 | +{ | ||
229 | + TCGReg b0 = TCG_REG_ZERO; | ||
230 | + | ||
231 | + if (a0 & ~0xffff) { | ||
232 | + tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_V0, a0 & ~0xffff); | ||
233 | + b0 = TCG_REG_V0; | ||
234 | + } | ||
235 | + if (!tcg_out_opc_jmp(s, OPC_J, tb_ret_addr)) { | ||
236 | + tcg_out_movi(s, TCG_TYPE_PTR, TCG_TMP0, (uintptr_t)tb_ret_addr); | ||
237 | + tcg_out_opc_reg(s, OPC_JR, 0, TCG_TMP0, 0); | ||
238 | + } | ||
239 | + tcg_out_opc_imm(s, OPC_ORI, TCG_REG_V0, b0, a0 & 0xffff); | ||
240 | +} | ||
241 | + | ||
242 | static void tcg_out_op(TCGContext *s, TCGOpcode opc, | ||
243 | const TCGArg args[TCG_MAX_OP_ARGS], | ||
244 | const int const_args[TCG_MAX_OP_ARGS]) | ||
245 | @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, | ||
246 | c2 = const_args[2]; | ||
247 | |||
248 | switch (opc) { | ||
249 | - case INDEX_op_exit_tb: | ||
250 | - { | ||
251 | - TCGReg b0 = TCG_REG_ZERO; | ||
252 | - | ||
253 | - a0 = (intptr_t)a0; | ||
254 | - if (a0 & ~0xffff) { | ||
255 | - tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_V0, a0 & ~0xffff); | ||
256 | - b0 = TCG_REG_V0; | ||
257 | - } | ||
258 | - if (!tcg_out_opc_jmp(s, OPC_J, tb_ret_addr)) { | ||
259 | - tcg_out_movi(s, TCG_TYPE_PTR, TCG_TMP0, | ||
260 | - (uintptr_t)tb_ret_addr); | ||
261 | - tcg_out_opc_reg(s, OPC_JR, 0, TCG_TMP0, 0); | ||
262 | - } | ||
263 | - tcg_out_opc_imm(s, OPC_ORI, TCG_REG_V0, b0, a0 & 0xffff); | ||
264 | - } | ||
265 | - break; | ||
266 | case INDEX_op_goto_tb: | ||
267 | /* indirect jump method */ | ||
268 | tcg_debug_assert(s->tb_jmp_insn_offset == 0); | ||
269 | @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, | ||
270 | case INDEX_op_mov_i32: /* Always emitted via tcg_out_mov. */ | ||
271 | case INDEX_op_mov_i64: | ||
272 | case INDEX_op_call: /* Always emitted via tcg_out_call. */ | ||
273 | + case INDEX_op_exit_tb: /* Always emitted via tcg_out_exit_tb. */ | ||
274 | default: | ||
275 | tcg_abort(); | ||
276 | } | ||
277 | diff --git a/tcg/ppc/tcg-target.c.inc b/tcg/ppc/tcg-target.c.inc | ||
278 | index XXXXXXX..XXXXXXX 100644 | ||
279 | --- a/tcg/ppc/tcg-target.c.inc | ||
280 | +++ b/tcg/ppc/tcg-target.c.inc | ||
281 | @@ -XXX,XX +XXX,XX @@ static void tcg_target_qemu_prologue(TCGContext *s) | ||
282 | tcg_out32(s, BCLR | BO_ALWAYS); | ||
283 | } | ||
284 | |||
285 | +static void tcg_out_exit_tb(TCGContext *s, uintptr_t arg) | ||
286 | +{ | ||
287 | + tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_R3, arg); | ||
288 | + tcg_out_b(s, 0, tcg_code_gen_epilogue); | ||
289 | +} | ||
290 | + | ||
291 | static void tcg_out_op(TCGContext *s, TCGOpcode opc, | ||
292 | const TCGArg args[TCG_MAX_OP_ARGS], | ||
293 | const int const_args[TCG_MAX_OP_ARGS]) | ||
294 | @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, | ||
295 | TCGArg a0, a1, a2; | ||
296 | |||
297 | switch (opc) { | ||
298 | - case INDEX_op_exit_tb: | ||
299 | - tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_R3, args[0]); | ||
300 | - tcg_out_b(s, 0, tcg_code_gen_epilogue); | ||
301 | - break; | ||
302 | case INDEX_op_goto_tb: | ||
303 | if (s->tb_jmp_insn_offset) { | ||
304 | /* Direct jump. */ | ||
305 | @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, | ||
306 | case INDEX_op_mov_i32: /* Always emitted via tcg_out_mov. */ | ||
307 | case INDEX_op_mov_i64: | ||
308 | case INDEX_op_call: /* Always emitted via tcg_out_call. */ | ||
309 | + case INDEX_op_exit_tb: /* Always emitted via tcg_out_exit_tb. */ | ||
310 | default: | ||
311 | tcg_abort(); | ||
312 | } | ||
313 | diff --git a/tcg/riscv/tcg-target.c.inc b/tcg/riscv/tcg-target.c.inc | ||
314 | index XXXXXXX..XXXXXXX 100644 | ||
315 | --- a/tcg/riscv/tcg-target.c.inc | ||
316 | +++ b/tcg/riscv/tcg-target.c.inc | ||
317 | @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, bool is_64) | ||
318 | |||
319 | static const tcg_insn_unit *tb_ret_addr; | ||
320 | |||
321 | +static void tcg_out_exit_tb(TCGContext *s, uintptr_t a0) | ||
322 | +{ | ||
323 | + /* Reuse the zeroing that exists for goto_ptr. */ | ||
324 | + if (a0 == 0) { | ||
325 | + tcg_out_call_int(s, tcg_code_gen_epilogue, true); | ||
326 | + } else { | ||
327 | + tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_A0, a0); | ||
328 | + tcg_out_call_int(s, tb_ret_addr, true); | ||
329 | + } | ||
330 | +} | ||
331 | + | ||
332 | static void tcg_out_op(TCGContext *s, TCGOpcode opc, | ||
333 | const TCGArg args[TCG_MAX_OP_ARGS], | ||
334 | const int const_args[TCG_MAX_OP_ARGS]) | ||
335 | @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, | ||
336 | int c2 = const_args[2]; | ||
337 | |||
338 | switch (opc) { | ||
339 | - case INDEX_op_exit_tb: | ||
340 | - /* Reuse the zeroing that exists for goto_ptr. */ | ||
341 | - if (a0 == 0) { | ||
342 | - tcg_out_call_int(s, tcg_code_gen_epilogue, true); | ||
343 | - } else { | ||
344 | - tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_A0, a0); | ||
345 | - tcg_out_call_int(s, tb_ret_addr, true); | ||
346 | - } | ||
347 | - break; | ||
348 | - | ||
349 | case INDEX_op_goto_tb: | ||
350 | assert(s->tb_jmp_insn_offset == 0); | ||
351 | /* indirect jump method */ | ||
352 | @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, | ||
353 | case INDEX_op_mov_i32: /* Always emitted via tcg_out_mov. */ | ||
354 | case INDEX_op_mov_i64: | ||
355 | case INDEX_op_call: /* Always emitted via tcg_out_call. */ | ||
356 | + case INDEX_op_exit_tb: /* Always emitted via tcg_out_exit_tb. */ | ||
357 | default: | ||
358 | g_assert_not_reached(); | ||
359 | } | ||
360 | diff --git a/tcg/s390x/tcg-target.c.inc b/tcg/s390x/tcg-target.c.inc | ||
361 | index XXXXXXX..XXXXXXX 100644 | ||
362 | --- a/tcg/s390x/tcg-target.c.inc | ||
363 | +++ b/tcg/s390x/tcg-target.c.inc | ||
364 | @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_st(TCGContext* s, TCGReg data_reg, TCGReg addr_reg, | ||
365 | #endif | ||
366 | } | ||
367 | |||
368 | +static void tcg_out_exit_tb(TCGContext *s, uintptr_t a0) | ||
369 | +{ | ||
370 | + /* Reuse the zeroing that exists for goto_ptr. */ | ||
371 | + if (a0 == 0) { | ||
372 | + tgen_gotoi(s, S390_CC_ALWAYS, tcg_code_gen_epilogue); | ||
373 | + } else { | ||
374 | + tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_R2, a0); | ||
375 | + tgen_gotoi(s, S390_CC_ALWAYS, tb_ret_addr); | ||
376 | + } | ||
377 | +} | ||
378 | + | ||
379 | # define OP_32_64(x) \ | ||
380 | case glue(glue(INDEX_op_,x),_i32): \ | ||
381 | case glue(glue(INDEX_op_,x),_i64) | ||
382 | @@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc, | ||
383 | TCGArg a0, a1, a2; | ||
384 | |||
385 | switch (opc) { | ||
386 | - case INDEX_op_exit_tb: | ||
387 | - /* Reuse the zeroing that exists for goto_ptr. */ | ||
388 | - a0 = args[0]; | ||
389 | - if (a0 == 0) { | ||
390 | - tgen_gotoi(s, S390_CC_ALWAYS, tcg_code_gen_epilogue); | ||
391 | - } else { | ||
392 | - tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_R2, a0); | ||
393 | - tgen_gotoi(s, S390_CC_ALWAYS, tb_ret_addr); | ||
394 | - } | ||
395 | - break; | ||
396 | - | ||
397 | case INDEX_op_goto_tb: | ||
398 | a0 = args[0]; | ||
399 | /* | ||
400 | @@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc, | ||
401 | case INDEX_op_mov_i32: /* Always emitted via tcg_out_mov. */ | ||
402 | case INDEX_op_mov_i64: | ||
403 | case INDEX_op_call: /* Always emitted via tcg_out_call. */ | ||
404 | + case INDEX_op_exit_tb: /* Always emitted via tcg_out_exit_tb. */ | ||
405 | default: | ||
406 | tcg_abort(); | ||
407 | } | ||
408 | diff --git a/tcg/sparc64/tcg-target.c.inc b/tcg/sparc64/tcg-target.c.inc | ||
409 | index XXXXXXX..XXXXXXX 100644 | ||
410 | --- a/tcg/sparc64/tcg-target.c.inc | ||
411 | +++ b/tcg/sparc64/tcg-target.c.inc | ||
412 | @@ -XXX,XX +XXX,XX @@ static void tcg_out_qemu_st(TCGContext *s, TCGReg data, TCGReg addr, | ||
413 | #endif /* CONFIG_SOFTMMU */ | ||
414 | } | ||
415 | |||
416 | +static void tcg_out_exit_tb(TCGContext *s, uintptr_t a0) | ||
417 | +{ | ||
418 | + if (check_fit_ptr(a0, 13)) { | ||
419 | + tcg_out_arithi(s, TCG_REG_G0, TCG_REG_I7, 8, RETURN); | ||
420 | + tcg_out_movi_imm13(s, TCG_REG_O0, a0); | ||
421 | + return; | ||
422 | + } else if (USE_REG_TB) { | ||
423 | + intptr_t tb_diff = tcg_tbrel_diff(s, (void *)a0); | ||
424 | + if (check_fit_ptr(tb_diff, 13)) { | ||
425 | + tcg_out_arithi(s, TCG_REG_G0, TCG_REG_I7, 8, RETURN); | ||
426 | + /* Note that TCG_REG_TB has been unwound to O1. */ | ||
427 | + tcg_out_arithi(s, TCG_REG_O0, TCG_REG_O1, tb_diff, ARITH_ADD); | ||
428 | + return; | ||
429 | + } | ||
430 | + } | ||
431 | + tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_I0, a0 & ~0x3ff); | ||
432 | + tcg_out_arithi(s, TCG_REG_G0, TCG_REG_I7, 8, RETURN); | ||
433 | + tcg_out_arithi(s, TCG_REG_O0, TCG_REG_O0, a0 & 0x3ff, ARITH_OR); | ||
434 | +} | ||
435 | + | ||
436 | static void tcg_out_op(TCGContext *s, TCGOpcode opc, | ||
437 | const TCGArg args[TCG_MAX_OP_ARGS], | ||
438 | const int const_args[TCG_MAX_OP_ARGS]) | ||
439 | @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, | ||
440 | c2 = const_args[2]; | ||
441 | |||
442 | switch (opc) { | ||
443 | - case INDEX_op_exit_tb: | ||
444 | - if (check_fit_ptr(a0, 13)) { | ||
445 | - tcg_out_arithi(s, TCG_REG_G0, TCG_REG_I7, 8, RETURN); | ||
446 | - tcg_out_movi_imm13(s, TCG_REG_O0, a0); | ||
447 | - break; | ||
448 | - } else if (USE_REG_TB) { | ||
449 | - intptr_t tb_diff = tcg_tbrel_diff(s, (void *)a0); | ||
450 | - if (check_fit_ptr(tb_diff, 13)) { | ||
451 | - tcg_out_arithi(s, TCG_REG_G0, TCG_REG_I7, 8, RETURN); | ||
452 | - /* Note that TCG_REG_TB has been unwound to O1. */ | ||
453 | - tcg_out_arithi(s, TCG_REG_O0, TCG_REG_O1, tb_diff, ARITH_ADD); | ||
454 | - break; | ||
455 | - } | ||
456 | - } | ||
457 | - tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_I0, a0 & ~0x3ff); | ||
458 | - tcg_out_arithi(s, TCG_REG_G0, TCG_REG_I7, 8, RETURN); | ||
459 | - tcg_out_arithi(s, TCG_REG_O0, TCG_REG_O0, a0 & 0x3ff, ARITH_OR); | ||
460 | - break; | ||
461 | case INDEX_op_goto_tb: | ||
462 | if (s->tb_jmp_insn_offset) { | ||
463 | /* direct jump method */ | ||
464 | @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, | ||
465 | case INDEX_op_mov_i32: /* Always emitted via tcg_out_mov. */ | ||
466 | case INDEX_op_mov_i64: | ||
467 | case INDEX_op_call: /* Always emitted via tcg_out_call. */ | ||
468 | + case INDEX_op_exit_tb: /* Always emitted via tcg_out_exit_tb. */ | ||
469 | default: | ||
470 | tcg_abort(); | ||
471 | } | ||
472 | diff --git a/tcg/tci/tcg-target.c.inc b/tcg/tci/tcg-target.c.inc | ||
473 | index XXXXXXX..XXXXXXX 100644 | ||
474 | --- a/tcg/tci/tcg-target.c.inc | ||
475 | +++ b/tcg/tci/tcg-target.c.inc | ||
476 | @@ -XXX,XX +XXX,XX @@ static void tcg_out_call(TCGContext *s, const tcg_insn_unit *func, | ||
477 | # define CASE_64(x) | ||
478 | #endif | ||
479 | |||
480 | +static void tcg_out_exit_tb(TCGContext *s, uintptr_t arg) | ||
481 | +{ | ||
482 | + tcg_out_op_p(s, INDEX_op_exit_tb, (void *)arg); | ||
483 | +} | ||
484 | + | ||
485 | static void tcg_out_op(TCGContext *s, TCGOpcode opc, | ||
486 | const TCGArg args[TCG_MAX_OP_ARGS], | ||
487 | const int const_args[TCG_MAX_OP_ARGS]) | ||
488 | @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, | ||
489 | TCGOpcode exts; | ||
490 | |||
491 | switch (opc) { | ||
492 | - case INDEX_op_exit_tb: | ||
493 | - tcg_out_op_p(s, opc, (void *)args[0]); | ||
494 | - break; | ||
495 | - | ||
496 | case INDEX_op_goto_tb: | ||
497 | tcg_debug_assert(s->tb_jmp_insn_offset == 0); | ||
498 | /* indirect jump method. */ | ||
499 | @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, | ||
500 | case INDEX_op_mov_i32: /* Always emitted via tcg_out_mov. */ | ||
501 | case INDEX_op_mov_i64: | ||
502 | case INDEX_op_call: /* Always emitted via tcg_out_call. */ | ||
503 | + case INDEX_op_exit_tb: /* Always emitted via tcg_out_exit_tb. */ | ||
504 | default: | ||
505 | tcg_abort(); | ||
506 | } | ||
507 | -- | ||
508 | 2.34.1 | ||
509 | |||
510 | diff view generated by jsdifflib |
New patch | |||
---|---|---|---|
1 | Reviewed-by: Alex Bennée <alex.bennee@linaro.org> | ||
2 | Signed-off-by: Richard Henderson <richard.henderson@linaro.org> | ||
3 | --- | ||
4 | tcg/i386/tcg-target.c.inc | 14 +++++--------- | ||
5 | 1 file changed, 5 insertions(+), 9 deletions(-) | ||
1 | 6 | ||
7 | diff --git a/tcg/i386/tcg-target.c.inc b/tcg/i386/tcg-target.c.inc | ||
8 | index XXXXXXX..XXXXXXX 100644 | ||
9 | --- a/tcg/i386/tcg-target.c.inc | ||
10 | +++ b/tcg/i386/tcg-target.c.inc | ||
11 | @@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc, | ||
12 | |||
13 | switch (opc) { | ||
14 | case INDEX_op_goto_tb: | ||
15 | - if (s->tb_jmp_insn_offset) { | ||
16 | - /* direct jump method */ | ||
17 | - int gap; | ||
18 | - /* jump displacement must be aligned for atomic patching; | ||
19 | + qemu_build_assert(TCG_TARGET_HAS_direct_jump); | ||
20 | + { | ||
21 | + /* | ||
22 | + * Jump displacement must be aligned for atomic patching; | ||
23 | * see if we need to add extra nops before jump | ||
24 | */ | ||
25 | - gap = QEMU_ALIGN_PTR_UP(s->code_ptr + 1, 4) - s->code_ptr; | ||
26 | + int gap = QEMU_ALIGN_PTR_UP(s->code_ptr + 1, 4) - s->code_ptr; | ||
27 | if (gap != 1) { | ||
28 | tcg_out_nopn(s, gap - 1); | ||
29 | } | ||
30 | tcg_out8(s, OPC_JMP_long); /* jmp im */ | ||
31 | s->tb_jmp_insn_offset[a0] = tcg_current_code_size(s); | ||
32 | tcg_out32(s, 0); | ||
33 | - } else { | ||
34 | - /* indirect jump method */ | ||
35 | - tcg_out_modrm_offset(s, OPC_GRP5, EXT5_JMPN_Ev, -1, | ||
36 | - (intptr_t)(s->tb_jmp_target_addr + a0)); | ||
37 | } | ||
38 | set_jmp_reset_offset(s, a0); | ||
39 | break; | ||
40 | -- | ||
41 | 2.34.1 | ||
42 | |||
43 | diff view generated by jsdifflib |
New patch | |||
---|---|---|---|
1 | Reviewed-by: Alex Bennée <alex.bennee@linaro.org> | ||
2 | Signed-off-by: Richard Henderson <richard.henderson@linaro.org> | ||
3 | --- | ||
4 | tcg/ppc/tcg-target.c.inc | 32 +++++++++++++------------------- | ||
5 | 1 file changed, 13 insertions(+), 19 deletions(-) | ||
1 | 6 | ||
7 | diff --git a/tcg/ppc/tcg-target.c.inc b/tcg/ppc/tcg-target.c.inc | ||
8 | index XXXXXXX..XXXXXXX 100644 | ||
9 | --- a/tcg/ppc/tcg-target.c.inc | ||
10 | +++ b/tcg/ppc/tcg-target.c.inc | ||
11 | @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, | ||
12 | |||
13 | switch (opc) { | ||
14 | case INDEX_op_goto_tb: | ||
15 | - if (s->tb_jmp_insn_offset) { | ||
16 | - /* Direct jump. */ | ||
17 | - if (TCG_TARGET_REG_BITS == 64) { | ||
18 | - /* Ensure the next insns are 8 or 16-byte aligned. */ | ||
19 | - while ((uintptr_t)s->code_ptr & (have_isa_2_07 ? 15 : 7)) { | ||
20 | - tcg_out32(s, NOP); | ||
21 | - } | ||
22 | - s->tb_jmp_insn_offset[args[0]] = tcg_current_code_size(s); | ||
23 | - tcg_out32(s, ADDIS | TAI(TCG_REG_TB, TCG_REG_TB, 0)); | ||
24 | - tcg_out32(s, ADDI | TAI(TCG_REG_TB, TCG_REG_TB, 0)); | ||
25 | - } else { | ||
26 | - s->tb_jmp_insn_offset[args[0]] = tcg_current_code_size(s); | ||
27 | - tcg_out32(s, B); | ||
28 | - s->tb_jmp_reset_offset[args[0]] = tcg_current_code_size(s); | ||
29 | - break; | ||
30 | + qemu_build_assert(TCG_TARGET_HAS_direct_jump); | ||
31 | + /* Direct jump. */ | ||
32 | + if (TCG_TARGET_REG_BITS == 64) { | ||
33 | + /* Ensure the next insns are 8 or 16-byte aligned. */ | ||
34 | + while ((uintptr_t)s->code_ptr & (have_isa_2_07 ? 15 : 7)) { | ||
35 | + tcg_out32(s, NOP); | ||
36 | } | ||
37 | + s->tb_jmp_insn_offset[args[0]] = tcg_current_code_size(s); | ||
38 | + tcg_out32(s, ADDIS | TAI(TCG_REG_TB, TCG_REG_TB, 0)); | ||
39 | + tcg_out32(s, ADDI | TAI(TCG_REG_TB, TCG_REG_TB, 0)); | ||
40 | } else { | ||
41 | - /* Indirect jump. */ | ||
42 | - tcg_debug_assert(s->tb_jmp_insn_offset == NULL); | ||
43 | - tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_TB, 0, | ||
44 | - (intptr_t)(s->tb_jmp_insn_offset + args[0])); | ||
45 | + s->tb_jmp_insn_offset[args[0]] = tcg_current_code_size(s); | ||
46 | + tcg_out32(s, B); | ||
47 | + s->tb_jmp_reset_offset[args[0]] = tcg_current_code_size(s); | ||
48 | + break; | ||
49 | } | ||
50 | tcg_out32(s, MTSPR | RS(TCG_REG_TB) | CTR); | ||
51 | tcg_out32(s, BCCTR | BO_ALWAYS); | ||
52 | -- | ||
53 | 2.34.1 | ||
54 | |||
55 | diff view generated by jsdifflib |
New patch | |||
---|---|---|---|
1 | Reviewed-by: Alex Bennée <alex.bennee@linaro.org> | ||
2 | Signed-off-by: Richard Henderson <richard.henderson@linaro.org> | ||
3 | --- | ||
4 | tcg/sparc64/tcg-target.c.inc | 41 +++++++++++------------------------- | ||
5 | 1 file changed, 12 insertions(+), 29 deletions(-) | ||
1 | 6 | ||
7 | diff --git a/tcg/sparc64/tcg-target.c.inc b/tcg/sparc64/tcg-target.c.inc | ||
8 | index XXXXXXX..XXXXXXX 100644 | ||
9 | --- a/tcg/sparc64/tcg-target.c.inc | ||
10 | +++ b/tcg/sparc64/tcg-target.c.inc | ||
11 | @@ -XXX,XX +XXX,XX @@ static bool tcg_out_sti(TCGContext *s, TCGType type, TCGArg val, | ||
12 | return false; | ||
13 | } | ||
14 | |||
15 | -static void tcg_out_ld_ptr(TCGContext *s, TCGReg ret, const void *arg) | ||
16 | -{ | ||
17 | - intptr_t diff = tcg_tbrel_diff(s, arg); | ||
18 | - if (USE_REG_TB && check_fit_ptr(diff, 13)) { | ||
19 | - tcg_out_ld(s, TCG_TYPE_PTR, ret, TCG_REG_TB, diff); | ||
20 | - return; | ||
21 | - } | ||
22 | - tcg_out_movi(s, TCG_TYPE_PTR, ret, (uintptr_t)arg & ~0x3ff); | ||
23 | - tcg_out_ld(s, TCG_TYPE_PTR, ret, ret, (uintptr_t)arg & 0x3ff); | ||
24 | -} | ||
25 | - | ||
26 | static void tcg_out_sety(TCGContext *s, TCGReg rs) | ||
27 | { | ||
28 | tcg_out32(s, WRY | INSN_RS1(TCG_REG_G0) | INSN_RS2(rs)); | ||
29 | @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, | ||
30 | |||
31 | switch (opc) { | ||
32 | case INDEX_op_goto_tb: | ||
33 | - if (s->tb_jmp_insn_offset) { | ||
34 | - /* direct jump method */ | ||
35 | - if (USE_REG_TB) { | ||
36 | - /* make sure the patch is 8-byte aligned. */ | ||
37 | - if ((intptr_t)s->code_ptr & 4) { | ||
38 | - tcg_out_nop(s); | ||
39 | - } | ||
40 | - s->tb_jmp_insn_offset[a0] = tcg_current_code_size(s); | ||
41 | - tcg_out_sethi(s, TCG_REG_T1, 0); | ||
42 | - tcg_out_arithi(s, TCG_REG_T1, TCG_REG_T1, 0, ARITH_OR); | ||
43 | - tcg_out_arith(s, TCG_REG_G0, TCG_REG_TB, TCG_REG_T1, JMPL); | ||
44 | - tcg_out_arith(s, TCG_REG_TB, TCG_REG_TB, TCG_REG_T1, ARITH_ADD); | ||
45 | - } else { | ||
46 | - s->tb_jmp_insn_offset[a0] = tcg_current_code_size(s); | ||
47 | - tcg_out32(s, CALL); | ||
48 | + qemu_build_assert(TCG_TARGET_HAS_direct_jump); | ||
49 | + /* Direct jump. */ | ||
50 | + if (USE_REG_TB) { | ||
51 | + /* make sure the patch is 8-byte aligned. */ | ||
52 | + if ((intptr_t)s->code_ptr & 4) { | ||
53 | tcg_out_nop(s); | ||
54 | } | ||
55 | + s->tb_jmp_insn_offset[a0] = tcg_current_code_size(s); | ||
56 | + tcg_out_sethi(s, TCG_REG_T1, 0); | ||
57 | + tcg_out_arithi(s, TCG_REG_T1, TCG_REG_T1, 0, ARITH_OR); | ||
58 | + tcg_out_arith(s, TCG_REG_G0, TCG_REG_TB, TCG_REG_T1, JMPL); | ||
59 | + tcg_out_arith(s, TCG_REG_TB, TCG_REG_TB, TCG_REG_T1, ARITH_ADD); | ||
60 | } else { | ||
61 | - /* indirect jump method */ | ||
62 | - tcg_out_ld_ptr(s, TCG_REG_TB, s->tb_jmp_target_addr + a0); | ||
63 | - tcg_out_arithi(s, TCG_REG_G0, TCG_REG_TB, 0, JMPL); | ||
64 | + s->tb_jmp_insn_offset[a0] = tcg_current_code_size(s); | ||
65 | + tcg_out32(s, CALL); | ||
66 | tcg_out_nop(s); | ||
67 | } | ||
68 | set_jmp_reset_offset(s, a0); | ||
69 | -- | ||
70 | 2.34.1 | ||
71 | |||
72 | diff view generated by jsdifflib |
1 | From: Philippe Mathieu-Daudé <philmd@redhat.com> | 1 | Test TCG_TARGET_HAS_direct_jump instead of testing an |
---|---|---|---|
2 | implementation pointer. | ||
2 | 3 | ||
3 | To avoid scrolling each instruction when reviewing tcg | ||
4 | helpers written for the decodetree script, display the | ||
5 | .decode files (similar to header declarations) before | ||
6 | the C source (implementation of previous declarations). | ||
7 | |||
8 | Signed-off-by: Philippe Mathieu-Daudé <philmd@redhat.com> | ||
9 | Reviewed-by: Stefano Garzarella <sgarzare@redhat.com> | ||
10 | Reviewed-by: Alex Bennée <alex.bennee@linaro.org> | 4 | Reviewed-by: Alex Bennée <alex.bennee@linaro.org> |
11 | Message-Id: <20191230082856.30556-1-philmd@redhat.com> | 5 | Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org> |
12 | Signed-off-by: Richard Henderson <richard.henderson@linaro.org> | 6 | Signed-off-by: Richard Henderson <richard.henderson@linaro.org> |
13 | --- | 7 | --- |
14 | scripts/git.orderfile | 3 +++ | 8 | tcg/aarch64/tcg-target.c.inc | 2 +- |
15 | 1 file changed, 3 insertions(+) | 9 | tcg/arm/tcg-target.c.inc | 2 +- |
10 | tcg/loongarch64/tcg-target.c.inc | 2 +- | ||
11 | tcg/mips/tcg-target.c.inc | 2 +- | ||
12 | tcg/riscv/tcg-target.c.inc | 2 +- | ||
13 | tcg/tci/tcg-target.c.inc | 2 +- | ||
14 | 6 files changed, 6 insertions(+), 6 deletions(-) | ||
16 | 15 | ||
17 | diff --git a/scripts/git.orderfile b/scripts/git.orderfile | 16 | diff --git a/tcg/aarch64/tcg-target.c.inc b/tcg/aarch64/tcg-target.c.inc |
18 | index XXXXXXX..XXXXXXX 100644 | 17 | index XXXXXXX..XXXXXXX 100644 |
19 | --- a/scripts/git.orderfile | 18 | --- a/tcg/aarch64/tcg-target.c.inc |
20 | +++ b/scripts/git.orderfile | 19 | +++ b/tcg/aarch64/tcg-target.c.inc |
21 | @@ -XXX,XX +XXX,XX @@ qga/*.json | 20 | @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, |
22 | # headers | 21 | |
23 | *.h | 22 | switch (opc) { |
24 | 23 | case INDEX_op_goto_tb: | |
25 | +# decoding tree specification | 24 | - tcg_debug_assert(s->tb_jmp_insn_offset != NULL); |
26 | +*.decode | 25 | + qemu_build_assert(TCG_TARGET_HAS_direct_jump); |
27 | + | 26 | /* |
28 | # code | 27 | * Ensure that ADRP+ADD are 8-byte aligned so that an atomic |
29 | *.c | 28 | * write can be used to patch the target address. |
29 | diff --git a/tcg/arm/tcg-target.c.inc b/tcg/arm/tcg-target.c.inc | ||
30 | index XXXXXXX..XXXXXXX 100644 | ||
31 | --- a/tcg/arm/tcg-target.c.inc | ||
32 | +++ b/tcg/arm/tcg-target.c.inc | ||
33 | @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, | ||
34 | intptr_t ptr, dif, dil; | ||
35 | TCGReg base = TCG_REG_PC; | ||
36 | |||
37 | - tcg_debug_assert(s->tb_jmp_insn_offset == 0); | ||
38 | + qemu_build_assert(!TCG_TARGET_HAS_direct_jump); | ||
39 | ptr = (intptr_t)tcg_splitwx_to_rx(s->tb_jmp_target_addr + args[0]); | ||
40 | dif = tcg_pcrel_diff(s, (void *)ptr) - 8; | ||
41 | dil = sextract32(dif, 0, 12); | ||
42 | diff --git a/tcg/loongarch64/tcg-target.c.inc b/tcg/loongarch64/tcg-target.c.inc | ||
43 | index XXXXXXX..XXXXXXX 100644 | ||
44 | --- a/tcg/loongarch64/tcg-target.c.inc | ||
45 | +++ b/tcg/loongarch64/tcg-target.c.inc | ||
46 | @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, | ||
47 | |||
48 | switch (opc) { | ||
49 | case INDEX_op_goto_tb: | ||
50 | - tcg_debug_assert(s->tb_jmp_insn_offset != NULL); | ||
51 | + qemu_build_assert(TCG_TARGET_HAS_direct_jump); | ||
52 | /* | ||
53 | * Ensure that patch area is 8-byte aligned so that an | ||
54 | * atomic write can be used to patch the target address. | ||
55 | diff --git a/tcg/mips/tcg-target.c.inc b/tcg/mips/tcg-target.c.inc | ||
56 | index XXXXXXX..XXXXXXX 100644 | ||
57 | --- a/tcg/mips/tcg-target.c.inc | ||
58 | +++ b/tcg/mips/tcg-target.c.inc | ||
59 | @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, | ||
60 | switch (opc) { | ||
61 | case INDEX_op_goto_tb: | ||
62 | /* indirect jump method */ | ||
63 | - tcg_debug_assert(s->tb_jmp_insn_offset == 0); | ||
64 | + qemu_build_assert(!TCG_TARGET_HAS_direct_jump); | ||
65 | tcg_out_ld(s, TCG_TYPE_PTR, TCG_TMP0, TCG_REG_ZERO, | ||
66 | (uintptr_t)(s->tb_jmp_target_addr + a0)); | ||
67 | tcg_out_opc_reg(s, OPC_JR, 0, TCG_TMP0, 0); | ||
68 | diff --git a/tcg/riscv/tcg-target.c.inc b/tcg/riscv/tcg-target.c.inc | ||
69 | index XXXXXXX..XXXXXXX 100644 | ||
70 | --- a/tcg/riscv/tcg-target.c.inc | ||
71 | +++ b/tcg/riscv/tcg-target.c.inc | ||
72 | @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, | ||
73 | |||
74 | switch (opc) { | ||
75 | case INDEX_op_goto_tb: | ||
76 | - assert(s->tb_jmp_insn_offset == 0); | ||
77 | + qemu_build_assert(!TCG_TARGET_HAS_direct_jump); | ||
78 | /* indirect jump method */ | ||
79 | tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_TMP0, TCG_REG_ZERO, | ||
80 | (uintptr_t)(s->tb_jmp_target_addr + a0)); | ||
81 | diff --git a/tcg/tci/tcg-target.c.inc b/tcg/tci/tcg-target.c.inc | ||
82 | index XXXXXXX..XXXXXXX 100644 | ||
83 | --- a/tcg/tci/tcg-target.c.inc | ||
84 | +++ b/tcg/tci/tcg-target.c.inc | ||
85 | @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, | ||
86 | |||
87 | switch (opc) { | ||
88 | case INDEX_op_goto_tb: | ||
89 | - tcg_debug_assert(s->tb_jmp_insn_offset == 0); | ||
90 | + qemu_build_assert(!TCG_TARGET_HAS_direct_jump); | ||
91 | /* indirect jump method. */ | ||
92 | tcg_out_op_p(s, opc, s->tb_jmp_target_addr + args[0]); | ||
93 | set_jmp_reset_offset(s, args[0]); | ||
30 | -- | 94 | -- |
31 | 2.20.1 | 95 | 2.34.1 |
32 | 96 | ||
33 | 97 | diff view generated by jsdifflib |
1 | By choosing "tcg:kvm" when kvm is not enabled, we generate | 1 | Similar to the existing set_jmp_reset_offset. Move any assert for |
---|---|---|---|
2 | an incorrect warning: "invalid accelerator kvm". | 2 | TCG_TARGET_HAS_direct_jump into the new function (which now cannot |
3 | be build-time). Will be unused if TCG_TARGET_HAS_direct_jump is | ||
4 | constant 0, but we can't test for constant in the preprocessor, | ||
5 | so just mark it G_GNUC_UNUSED. | ||
3 | 6 | ||
4 | At the same time, use g_str_has_suffix rather than open-coding | ||
5 | the same operation. | ||
6 | |||
7 | Presumably the inverse is also true with --disable-tcg. | ||
8 | |||
9 | Fixes: 28a0961757fc | ||
10 | Acked-by: Paolo Bonzini <pbonzini@redhat.com> | ||
11 | Reviewed-by: Alex Bennée <alex.bennee@linaro.org> | 7 | Reviewed-by: Alex Bennée <alex.bennee@linaro.org> |
12 | Reviewed by: Aleksandar Markovic <amarkovic@wavecomp.com> | ||
13 | Signed-off-by: Richard Henderson <richard.henderson@linaro.org> | 8 | Signed-off-by: Richard Henderson <richard.henderson@linaro.org> |
14 | --- | 9 | --- |
15 | vl.c | 21 +++++++++++++-------- | 10 | tcg/tcg.c | 10 ++++++++++ |
16 | 1 file changed, 13 insertions(+), 8 deletions(-) | 11 | tcg/aarch64/tcg-target.c.inc | 3 +-- |
12 | tcg/i386/tcg-target.c.inc | 3 +-- | ||
13 | tcg/loongarch64/tcg-target.c.inc | 3 +-- | ||
14 | tcg/ppc/tcg-target.c.inc | 7 +++---- | ||
15 | tcg/s390x/tcg-target.c.inc | 2 +- | ||
16 | tcg/sparc64/tcg-target.c.inc | 5 ++--- | ||
17 | 7 files changed, 19 insertions(+), 14 deletions(-) | ||
17 | 18 | ||
18 | diff --git a/vl.c b/vl.c | 19 | diff --git a/tcg/tcg.c b/tcg/tcg.c |
19 | index XXXXXXX..XXXXXXX 100644 | 20 | index XXXXXXX..XXXXXXX 100644 |
20 | --- a/vl.c | 21 | --- a/tcg/tcg.c |
21 | +++ b/vl.c | 22 | +++ b/tcg/tcg.c |
22 | @@ -XXX,XX +XXX,XX @@ static void configure_accelerators(const char *progname) | 23 | @@ -XXX,XX +XXX,XX @@ static void set_jmp_reset_offset(TCGContext *s, int which) |
23 | 24 | s->tb_jmp_reset_offset[which] = tcg_current_code_size(s); | |
24 | if (accel == NULL) { | 25 | } |
25 | /* Select the default accelerator */ | 26 | |
26 | - if (!accel_find("tcg") && !accel_find("kvm")) { | 27 | +static void G_GNUC_UNUSED set_jmp_insn_offset(TCGContext *s, int which) |
27 | - error_report("No accelerator selected and" | 28 | +{ |
28 | - " no default accelerator available"); | 29 | + /* |
29 | - exit(1); | 30 | + * We will check for overflow at the end of the opcode loop in |
30 | - } else { | 31 | + * tcg_gen_code, where we bound tcg_current_code_size to UINT16_MAX. |
31 | - int pnlen = strlen(progname); | 32 | + */ |
32 | - if (pnlen >= 3 && g_str_equal(&progname[pnlen - 3], "kvm")) { | 33 | + tcg_debug_assert(TCG_TARGET_HAS_direct_jump); |
33 | + bool have_tcg = accel_find("tcg"); | 34 | + s->tb_jmp_insn_offset[which] = tcg_current_code_size(s); |
34 | + bool have_kvm = accel_find("kvm"); | 35 | +} |
35 | + | 36 | + |
36 | + if (have_tcg && have_kvm) { | 37 | /* Signal overflow, starting over with fewer guest insns. */ |
37 | + if (g_str_has_suffix(progname, "kvm")) { | 38 | static G_NORETURN |
38 | /* If the program name ends with "kvm", we prefer KVM */ | 39 | void tcg_raise_tb_overflow(TCGContext *s) |
39 | accel = "kvm:tcg"; | 40 | diff --git a/tcg/aarch64/tcg-target.c.inc b/tcg/aarch64/tcg-target.c.inc |
40 | } else { | 41 | index XXXXXXX..XXXXXXX 100644 |
41 | accel = "tcg:kvm"; | 42 | --- a/tcg/aarch64/tcg-target.c.inc |
42 | } | 43 | +++ b/tcg/aarch64/tcg-target.c.inc |
43 | + } else if (have_kvm) { | 44 | @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, |
44 | + accel = "kvm"; | 45 | |
45 | + } else if (have_tcg) { | 46 | switch (opc) { |
46 | + accel = "tcg"; | 47 | case INDEX_op_goto_tb: |
47 | + } else { | 48 | - qemu_build_assert(TCG_TARGET_HAS_direct_jump); |
48 | + error_report("No accelerator selected and" | 49 | /* |
49 | + " no default accelerator available"); | 50 | * Ensure that ADRP+ADD are 8-byte aligned so that an atomic |
50 | + exit(1); | 51 | * write can be used to patch the target address. |
52 | @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, | ||
53 | if ((uintptr_t)s->code_ptr & 7) { | ||
54 | tcg_out32(s, NOP); | ||
55 | } | ||
56 | - s->tb_jmp_insn_offset[a0] = tcg_current_code_size(s); | ||
57 | + set_jmp_insn_offset(s, a0); | ||
58 | /* | ||
59 | * actual branch destination will be patched by | ||
60 | * tb_target_set_jmp_target later | ||
61 | diff --git a/tcg/i386/tcg-target.c.inc b/tcg/i386/tcg-target.c.inc | ||
62 | index XXXXXXX..XXXXXXX 100644 | ||
63 | --- a/tcg/i386/tcg-target.c.inc | ||
64 | +++ b/tcg/i386/tcg-target.c.inc | ||
65 | @@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc, | ||
66 | |||
67 | switch (opc) { | ||
68 | case INDEX_op_goto_tb: | ||
69 | - qemu_build_assert(TCG_TARGET_HAS_direct_jump); | ||
70 | { | ||
71 | /* | ||
72 | * Jump displacement must be aligned for atomic patching; | ||
73 | @@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc, | ||
74 | tcg_out_nopn(s, gap - 1); | ||
51 | } | 75 | } |
76 | tcg_out8(s, OPC_JMP_long); /* jmp im */ | ||
77 | - s->tb_jmp_insn_offset[a0] = tcg_current_code_size(s); | ||
78 | + set_jmp_insn_offset(s, a0); | ||
79 | tcg_out32(s, 0); | ||
52 | } | 80 | } |
53 | - | 81 | set_jmp_reset_offset(s, a0); |
54 | accel_list = g_strsplit(accel, ":", 0); | 82 | diff --git a/tcg/loongarch64/tcg-target.c.inc b/tcg/loongarch64/tcg-target.c.inc |
55 | 83 | index XXXXXXX..XXXXXXX 100644 | |
56 | for (tmp = accel_list; *tmp; tmp++) { | 84 | --- a/tcg/loongarch64/tcg-target.c.inc |
85 | +++ b/tcg/loongarch64/tcg-target.c.inc | ||
86 | @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, | ||
87 | |||
88 | switch (opc) { | ||
89 | case INDEX_op_goto_tb: | ||
90 | - qemu_build_assert(TCG_TARGET_HAS_direct_jump); | ||
91 | /* | ||
92 | * Ensure that patch area is 8-byte aligned so that an | ||
93 | * atomic write can be used to patch the target address. | ||
94 | @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, | ||
95 | if ((uintptr_t)s->code_ptr & 7) { | ||
96 | tcg_out_nop(s); | ||
97 | } | ||
98 | - s->tb_jmp_insn_offset[a0] = tcg_current_code_size(s); | ||
99 | + set_jmp_insn_offset(s, a0); | ||
100 | /* | ||
101 | * actual branch destination will be patched by | ||
102 | * tb_target_set_jmp_target later | ||
103 | diff --git a/tcg/ppc/tcg-target.c.inc b/tcg/ppc/tcg-target.c.inc | ||
104 | index XXXXXXX..XXXXXXX 100644 | ||
105 | --- a/tcg/ppc/tcg-target.c.inc | ||
106 | +++ b/tcg/ppc/tcg-target.c.inc | ||
107 | @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, | ||
108 | |||
109 | switch (opc) { | ||
110 | case INDEX_op_goto_tb: | ||
111 | - qemu_build_assert(TCG_TARGET_HAS_direct_jump); | ||
112 | /* Direct jump. */ | ||
113 | if (TCG_TARGET_REG_BITS == 64) { | ||
114 | /* Ensure the next insns are 8 or 16-byte aligned. */ | ||
115 | while ((uintptr_t)s->code_ptr & (have_isa_2_07 ? 15 : 7)) { | ||
116 | tcg_out32(s, NOP); | ||
117 | } | ||
118 | - s->tb_jmp_insn_offset[args[0]] = tcg_current_code_size(s); | ||
119 | + set_jmp_insn_offset(s, args[0]); | ||
120 | tcg_out32(s, ADDIS | TAI(TCG_REG_TB, TCG_REG_TB, 0)); | ||
121 | tcg_out32(s, ADDI | TAI(TCG_REG_TB, TCG_REG_TB, 0)); | ||
122 | } else { | ||
123 | - s->tb_jmp_insn_offset[args[0]] = tcg_current_code_size(s); | ||
124 | + set_jmp_insn_offset(s, args[0]); | ||
125 | tcg_out32(s, B); | ||
126 | - s->tb_jmp_reset_offset[args[0]] = tcg_current_code_size(s); | ||
127 | + set_jmp_reset_offset(s, args[0]); | ||
128 | break; | ||
129 | } | ||
130 | tcg_out32(s, MTSPR | RS(TCG_REG_TB) | CTR); | ||
131 | diff --git a/tcg/s390x/tcg-target.c.inc b/tcg/s390x/tcg-target.c.inc | ||
132 | index XXXXXXX..XXXXXXX 100644 | ||
133 | --- a/tcg/s390x/tcg-target.c.inc | ||
134 | +++ b/tcg/s390x/tcg-target.c.inc | ||
135 | @@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc, | ||
136 | tcg_out16(s, NOP); | ||
137 | } | ||
138 | tcg_out16(s, RIL_BRCL | (S390_CC_ALWAYS << 4)); | ||
139 | - s->tb_jmp_insn_offset[a0] = tcg_current_code_size(s); | ||
140 | + set_jmp_insn_offset(s, a0); | ||
141 | s->code_ptr += 2; | ||
142 | set_jmp_reset_offset(s, a0); | ||
143 | break; | ||
144 | diff --git a/tcg/sparc64/tcg-target.c.inc b/tcg/sparc64/tcg-target.c.inc | ||
145 | index XXXXXXX..XXXXXXX 100644 | ||
146 | --- a/tcg/sparc64/tcg-target.c.inc | ||
147 | +++ b/tcg/sparc64/tcg-target.c.inc | ||
148 | @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, | ||
149 | |||
150 | switch (opc) { | ||
151 | case INDEX_op_goto_tb: | ||
152 | - qemu_build_assert(TCG_TARGET_HAS_direct_jump); | ||
153 | /* Direct jump. */ | ||
154 | if (USE_REG_TB) { | ||
155 | /* make sure the patch is 8-byte aligned. */ | ||
156 | if ((intptr_t)s->code_ptr & 4) { | ||
157 | tcg_out_nop(s); | ||
158 | } | ||
159 | - s->tb_jmp_insn_offset[a0] = tcg_current_code_size(s); | ||
160 | + set_jmp_insn_offset(s, a0); | ||
161 | tcg_out_sethi(s, TCG_REG_T1, 0); | ||
162 | tcg_out_arithi(s, TCG_REG_T1, TCG_REG_T1, 0, ARITH_OR); | ||
163 | tcg_out_arith(s, TCG_REG_G0, TCG_REG_TB, TCG_REG_T1, JMPL); | ||
164 | tcg_out_arith(s, TCG_REG_TB, TCG_REG_TB, TCG_REG_T1, ARITH_ADD); | ||
165 | } else { | ||
166 | - s->tb_jmp_insn_offset[a0] = tcg_current_code_size(s); | ||
167 | + set_jmp_insn_offset(s, a0); | ||
168 | tcg_out32(s, CALL); | ||
169 | tcg_out_nop(s); | ||
170 | } | ||
57 | -- | 171 | -- |
58 | 2.20.1 | 172 | 2.34.1 |
59 | 173 | ||
60 | 174 | diff view generated by jsdifflib |
1 | There's little point in leaving these data structures half initialized, | 1 | Similar to the existing set_jmp_reset_offset. Include the |
---|---|---|---|
2 | and relying on a flush to be done during reset. | 2 | rw->rx address space conversion done by arm and s390x, and |
3 | forgotten by mips and riscv. | ||
3 | 4 | ||
4 | Reviewed-by: Alex Bennée <alex.bennee@linaro.org> | 5 | Reviewed-by: Alex Bennée <alex.bennee@linaro.org> |
5 | Reviewed-by: Alistair Francis <alistair.francis@wdc.com> | 6 | Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org> |
6 | Signed-off-by: Richard Henderson <richard.henderson@linaro.org> | 7 | Signed-off-by: Richard Henderson <richard.henderson@linaro.org> |
7 | --- | 8 | --- |
8 | accel/tcg/cputlb.c | 5 +++-- | 9 | tcg/tcg.c | 9 +++++++++ |
9 | 1 file changed, 3 insertions(+), 2 deletions(-) | 10 | tcg/arm/tcg-target.c.inc | 2 +- |
11 | tcg/mips/tcg-target.c.inc | 2 +- | ||
12 | tcg/riscv/tcg-target.c.inc | 2 +- | ||
13 | tcg/tci/tcg-target.c.inc | 2 +- | ||
14 | 5 files changed, 13 insertions(+), 4 deletions(-) | ||
10 | 15 | ||
11 | diff --git a/accel/tcg/cputlb.c b/accel/tcg/cputlb.c | 16 | diff --git a/tcg/tcg.c b/tcg/tcg.c |
12 | index XXXXXXX..XXXXXXX 100644 | 17 | index XXXXXXX..XXXXXXX 100644 |
13 | --- a/accel/tcg/cputlb.c | 18 | --- a/tcg/tcg.c |
14 | +++ b/accel/tcg/cputlb.c | 19 | +++ b/tcg/tcg.c |
15 | @@ -XXX,XX +XXX,XX @@ static void tlb_mmu_init(CPUTLBDesc *desc, CPUTLBDescFast *fast, int64_t now) | 20 | @@ -XXX,XX +XXX,XX @@ static void G_GNUC_UNUSED set_jmp_insn_offset(TCGContext *s, int which) |
16 | fast->mask = (n_entries - 1) << CPU_TLB_ENTRY_BITS; | 21 | s->tb_jmp_insn_offset[which] = tcg_current_code_size(s); |
17 | fast->table = g_new(CPUTLBEntry, n_entries); | ||
18 | desc->iotlb = g_new(CPUIOTLBEntry, n_entries); | ||
19 | + tlb_mmu_flush_locked(desc, fast); | ||
20 | } | 22 | } |
21 | 23 | ||
22 | static inline void tlb_n_used_entries_inc(CPUArchState *env, uintptr_t mmu_idx) | 24 | +static uintptr_t G_GNUC_UNUSED get_jmp_target_addr(TCGContext *s, int which) |
23 | @@ -XXX,XX +XXX,XX @@ void tlb_init(CPUState *cpu) | 25 | +{ |
24 | 26 | + /* | |
25 | qemu_spin_init(&env_tlb(env)->c.lock); | 27 | + * Return the read-execute version of the pointer, for the benefit |
26 | 28 | + * of any pc-relative addressing mode. | |
27 | - /* Ensure that cpu_reset performs a full flush. */ | 29 | + */ |
28 | - env_tlb(env)->c.dirty = ALL_MMUIDX_BITS; | 30 | + return (uintptr_t)tcg_splitwx_to_rx(&s->tb_jmp_target_addr[which]); |
29 | + /* All tlbs are initialized flushed. */ | 31 | +} |
30 | + env_tlb(env)->c.dirty = 0; | 32 | + |
31 | 33 | /* Signal overflow, starting over with fewer guest insns. */ | |
32 | for (i = 0; i < NB_MMU_MODES; i++) { | 34 | static G_NORETURN |
33 | tlb_mmu_init(&env_tlb(env)->d[i], &env_tlb(env)->f[i], now); | 35 | void tcg_raise_tb_overflow(TCGContext *s) |
36 | diff --git a/tcg/arm/tcg-target.c.inc b/tcg/arm/tcg-target.c.inc | ||
37 | index XXXXXXX..XXXXXXX 100644 | ||
38 | --- a/tcg/arm/tcg-target.c.inc | ||
39 | +++ b/tcg/arm/tcg-target.c.inc | ||
40 | @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, | ||
41 | TCGReg base = TCG_REG_PC; | ||
42 | |||
43 | qemu_build_assert(!TCG_TARGET_HAS_direct_jump); | ||
44 | - ptr = (intptr_t)tcg_splitwx_to_rx(s->tb_jmp_target_addr + args[0]); | ||
45 | + ptr = get_jmp_target_addr(s, args[0]); | ||
46 | dif = tcg_pcrel_diff(s, (void *)ptr) - 8; | ||
47 | dil = sextract32(dif, 0, 12); | ||
48 | if (dif != dil) { | ||
49 | diff --git a/tcg/mips/tcg-target.c.inc b/tcg/mips/tcg-target.c.inc | ||
50 | index XXXXXXX..XXXXXXX 100644 | ||
51 | --- a/tcg/mips/tcg-target.c.inc | ||
52 | +++ b/tcg/mips/tcg-target.c.inc | ||
53 | @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, | ||
54 | /* indirect jump method */ | ||
55 | qemu_build_assert(!TCG_TARGET_HAS_direct_jump); | ||
56 | tcg_out_ld(s, TCG_TYPE_PTR, TCG_TMP0, TCG_REG_ZERO, | ||
57 | - (uintptr_t)(s->tb_jmp_target_addr + a0)); | ||
58 | + get_jmp_target_addr(s, a0)); | ||
59 | tcg_out_opc_reg(s, OPC_JR, 0, TCG_TMP0, 0); | ||
60 | tcg_out_nop(s); | ||
61 | set_jmp_reset_offset(s, a0); | ||
62 | diff --git a/tcg/riscv/tcg-target.c.inc b/tcg/riscv/tcg-target.c.inc | ||
63 | index XXXXXXX..XXXXXXX 100644 | ||
64 | --- a/tcg/riscv/tcg-target.c.inc | ||
65 | +++ b/tcg/riscv/tcg-target.c.inc | ||
66 | @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, | ||
67 | qemu_build_assert(!TCG_TARGET_HAS_direct_jump); | ||
68 | /* indirect jump method */ | ||
69 | tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_TMP0, TCG_REG_ZERO, | ||
70 | - (uintptr_t)(s->tb_jmp_target_addr + a0)); | ||
71 | + get_jmp_target_addr(s, a0)); | ||
72 | tcg_out_opc_imm(s, OPC_JALR, TCG_REG_ZERO, TCG_REG_TMP0, 0); | ||
73 | set_jmp_reset_offset(s, a0); | ||
74 | break; | ||
75 | diff --git a/tcg/tci/tcg-target.c.inc b/tcg/tci/tcg-target.c.inc | ||
76 | index XXXXXXX..XXXXXXX 100644 | ||
77 | --- a/tcg/tci/tcg-target.c.inc | ||
78 | +++ b/tcg/tci/tcg-target.c.inc | ||
79 | @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, | ||
80 | case INDEX_op_goto_tb: | ||
81 | qemu_build_assert(!TCG_TARGET_HAS_direct_jump); | ||
82 | /* indirect jump method. */ | ||
83 | - tcg_out_op_p(s, opc, s->tb_jmp_target_addr + args[0]); | ||
84 | + tcg_out_op_p(s, opc, (void *)get_jmp_target_addr(s, args[0])); | ||
85 | set_jmp_reset_offset(s, args[0]); | ||
86 | break; | ||
87 | |||
34 | -- | 88 | -- |
35 | 2.20.1 | 89 | 2.34.1 |
36 | 90 | ||
37 | 91 | diff view generated by jsdifflib |
1 | The result of g_strsplit is never NULL. | 1 | The INDEX_op_goto_tb opcode needs no register allocation. |
---|---|---|---|
2 | Split out a dedicated helper function for it. | ||
2 | 3 | ||
3 | Acked-by: Paolo Bonzini <pbonzini@redhat.com> | ||
4 | Reviewed-by: Alex Bennée <alex.bennee@linaro.org> | 4 | Reviewed-by: Alex Bennée <alex.bennee@linaro.org> |
5 | Reviewed-by: Philippe Mathieu-Daudé <philmd@redhat.com> | 5 | Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org> |
6 | Reviewed by: Aleksandar Markovic <amarkovic@wavecomp.com> | ||
7 | Signed-off-by: Richard Henderson <richard.henderson@linaro.org> | 6 | Signed-off-by: Richard Henderson <richard.henderson@linaro.org> |
8 | --- | 7 | --- |
9 | vl.c | 2 +- | 8 | tcg/tcg.c | 4 ++ |
10 | 1 file changed, 1 insertion(+), 1 deletion(-) | 9 | tcg/aarch64/tcg-target.c.inc | 40 ++++++++++--------- |
10 | tcg/arm/tcg-target.c.inc | 49 ++++++++++++----------- | ||
11 | tcg/i386/tcg-target.c.inc | 33 ++++++++-------- | ||
12 | tcg/loongarch64/tcg-target.c.inc | 38 +++++++++--------- | ||
13 | tcg/mips/tcg-target.c.inc | 21 +++++----- | ||
14 | tcg/ppc/tcg-target.c.inc | 52 ++++++++++++------------ | ||
15 | tcg/riscv/tcg-target.c.inc | 20 +++++----- | ||
16 | tcg/s390x/tcg-target.c.inc | 31 ++++++++------- | ||
17 | tcg/sparc64/tcg-target.c.inc | 68 +++++++++++++++++--------------- | ||
18 | tcg/tci/tcg-target.c.inc | 16 ++++---- | ||
19 | 11 files changed, 199 insertions(+), 173 deletions(-) | ||
11 | 20 | ||
12 | diff --git a/vl.c b/vl.c | 21 | diff --git a/tcg/tcg.c b/tcg/tcg.c |
13 | index XXXXXXX..XXXXXXX 100644 | 22 | index XXXXXXX..XXXXXXX 100644 |
14 | --- a/vl.c | 23 | --- a/tcg/tcg.c |
15 | +++ b/vl.c | 24 | +++ b/tcg/tcg.c |
16 | @@ -XXX,XX +XXX,XX @@ static void configure_accelerators(const char *progname) | 25 | @@ -XXX,XX +XXX,XX @@ static bool tcg_out_mov(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg); |
17 | 26 | static void tcg_out_movi(TCGContext *s, TCGType type, | |
18 | accel_list = g_strsplit(accel, ":", 0); | 27 | TCGReg ret, tcg_target_long arg); |
19 | 28 | static void tcg_out_exit_tb(TCGContext *s, uintptr_t arg); | |
20 | - for (tmp = accel_list; tmp && *tmp; tmp++) { | 29 | +static void tcg_out_goto_tb(TCGContext *s, int which); |
21 | + for (tmp = accel_list; *tmp; tmp++) { | 30 | static void tcg_out_op(TCGContext *s, TCGOpcode opc, |
22 | /* | 31 | const TCGArg args[TCG_MAX_OP_ARGS], |
23 | * Filter invalid accelerators here, to prevent obscenities | 32 | const int const_args[TCG_MAX_OP_ARGS]); |
24 | * such as "-machine accel=tcg,,thread=single". | 33 | @@ -XXX,XX +XXX,XX @@ int tcg_gen_code(TCGContext *s, TranslationBlock *tb, target_ulong pc_start) |
34 | case INDEX_op_exit_tb: | ||
35 | tcg_out_exit_tb(s, op->args[0]); | ||
36 | break; | ||
37 | + case INDEX_op_goto_tb: | ||
38 | + tcg_out_goto_tb(s, op->args[0]); | ||
39 | + break; | ||
40 | case INDEX_op_dup2_vec: | ||
41 | if (tcg_reg_alloc_dup2(s, op)) { | ||
42 | break; | ||
43 | diff --git a/tcg/aarch64/tcg-target.c.inc b/tcg/aarch64/tcg-target.c.inc | ||
44 | index XXXXXXX..XXXXXXX 100644 | ||
45 | --- a/tcg/aarch64/tcg-target.c.inc | ||
46 | +++ b/tcg/aarch64/tcg-target.c.inc | ||
47 | @@ -XXX,XX +XXX,XX @@ static void tcg_out_exit_tb(TCGContext *s, uintptr_t a0) | ||
48 | } | ||
49 | } | ||
50 | |||
51 | +static void tcg_out_goto_tb(TCGContext *s, int which) | ||
52 | +{ | ||
53 | + /* | ||
54 | + * Ensure that ADRP+ADD are 8-byte aligned so that an atomic | ||
55 | + * write can be used to patch the target address. | ||
56 | + */ | ||
57 | + if ((uintptr_t)s->code_ptr & 7) { | ||
58 | + tcg_out32(s, NOP); | ||
59 | + } | ||
60 | + set_jmp_insn_offset(s, which); | ||
61 | + /* | ||
62 | + * actual branch destination will be patched by | ||
63 | + * tb_target_set_jmp_target later | ||
64 | + */ | ||
65 | + tcg_out_insn(s, 3406, ADRP, TCG_REG_TMP, 0); | ||
66 | + tcg_out_insn(s, 3401, ADDI, TCG_TYPE_I64, TCG_REG_TMP, TCG_REG_TMP, 0); | ||
67 | + tcg_out_insn(s, 3207, BR, TCG_REG_TMP); | ||
68 | + set_jmp_reset_offset(s, which); | ||
69 | +} | ||
70 | + | ||
71 | static void tcg_out_op(TCGContext *s, TCGOpcode opc, | ||
72 | const TCGArg args[TCG_MAX_OP_ARGS], | ||
73 | const int const_args[TCG_MAX_OP_ARGS]) | ||
74 | @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, | ||
75 | #define REG0(I) (const_args[I] ? TCG_REG_XZR : (TCGReg)args[I]) | ||
76 | |||
77 | switch (opc) { | ||
78 | - case INDEX_op_goto_tb: | ||
79 | - /* | ||
80 | - * Ensure that ADRP+ADD are 8-byte aligned so that an atomic | ||
81 | - * write can be used to patch the target address. | ||
82 | - */ | ||
83 | - if ((uintptr_t)s->code_ptr & 7) { | ||
84 | - tcg_out32(s, NOP); | ||
85 | - } | ||
86 | - set_jmp_insn_offset(s, a0); | ||
87 | - /* | ||
88 | - * actual branch destination will be patched by | ||
89 | - * tb_target_set_jmp_target later | ||
90 | - */ | ||
91 | - tcg_out_insn(s, 3406, ADRP, TCG_REG_TMP, 0); | ||
92 | - tcg_out_insn(s, 3401, ADDI, TCG_TYPE_I64, TCG_REG_TMP, TCG_REG_TMP, 0); | ||
93 | - tcg_out_insn(s, 3207, BR, TCG_REG_TMP); | ||
94 | - set_jmp_reset_offset(s, a0); | ||
95 | - break; | ||
96 | - | ||
97 | case INDEX_op_goto_ptr: | ||
98 | tcg_out_insn(s, 3207, BR, a0); | ||
99 | break; | ||
100 | @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, | ||
101 | case INDEX_op_mov_i64: | ||
102 | case INDEX_op_call: /* Always emitted via tcg_out_call. */ | ||
103 | case INDEX_op_exit_tb: /* Always emitted via tcg_out_exit_tb. */ | ||
104 | + case INDEX_op_goto_tb: /* Always emitted via tcg_out_goto_tb. */ | ||
105 | default: | ||
106 | g_assert_not_reached(); | ||
107 | } | ||
108 | diff --git a/tcg/arm/tcg-target.c.inc b/tcg/arm/tcg-target.c.inc | ||
109 | index XXXXXXX..XXXXXXX 100644 | ||
110 | --- a/tcg/arm/tcg-target.c.inc | ||
111 | +++ b/tcg/arm/tcg-target.c.inc | ||
112 | @@ -XXX,XX +XXX,XX @@ static void tcg_out_exit_tb(TCGContext *s, uintptr_t arg) | ||
113 | tcg_out_epilogue(s); | ||
114 | } | ||
115 | |||
116 | +static void tcg_out_goto_tb(TCGContext *s, int which) | ||
117 | +{ | ||
118 | + /* Indirect jump method */ | ||
119 | + intptr_t ptr, dif, dil; | ||
120 | + TCGReg base = TCG_REG_PC; | ||
121 | + | ||
122 | + qemu_build_assert(!TCG_TARGET_HAS_direct_jump); | ||
123 | + ptr = get_jmp_target_addr(s, which); | ||
124 | + dif = tcg_pcrel_diff(s, (void *)ptr) - 8; | ||
125 | + dil = sextract32(dif, 0, 12); | ||
126 | + if (dif != dil) { | ||
127 | + /* | ||
128 | + * The TB is close, but outside the 12 bits addressable by | ||
129 | + * the load. We can extend this to 20 bits with a sub of a | ||
130 | + * shifted immediate from pc. In the vastly unlikely event | ||
131 | + * the code requires more than 1MB, we'll use 2 insns and | ||
132 | + * be no worse off. | ||
133 | + */ | ||
134 | + base = TCG_REG_R0; | ||
135 | + tcg_out_movi32(s, COND_AL, base, ptr - dil); | ||
136 | + } | ||
137 | + tcg_out_ld32_12(s, COND_AL, TCG_REG_PC, base, dil); | ||
138 | + set_jmp_reset_offset(s, which); | ||
139 | +} | ||
140 | + | ||
141 | static void tcg_out_op(TCGContext *s, TCGOpcode opc, | ||
142 | const TCGArg args[TCG_MAX_OP_ARGS], | ||
143 | const int const_args[TCG_MAX_OP_ARGS]) | ||
144 | @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, | ||
145 | int c; | ||
146 | |||
147 | switch (opc) { | ||
148 | - case INDEX_op_goto_tb: | ||
149 | - { | ||
150 | - /* Indirect jump method */ | ||
151 | - intptr_t ptr, dif, dil; | ||
152 | - TCGReg base = TCG_REG_PC; | ||
153 | - | ||
154 | - qemu_build_assert(!TCG_TARGET_HAS_direct_jump); | ||
155 | - ptr = get_jmp_target_addr(s, args[0]); | ||
156 | - dif = tcg_pcrel_diff(s, (void *)ptr) - 8; | ||
157 | - dil = sextract32(dif, 0, 12); | ||
158 | - if (dif != dil) { | ||
159 | - /* The TB is close, but outside the 12 bits addressable by | ||
160 | - the load. We can extend this to 20 bits with a sub of a | ||
161 | - shifted immediate from pc. In the vastly unlikely event | ||
162 | - the code requires more than 1MB, we'll use 2 insns and | ||
163 | - be no worse off. */ | ||
164 | - base = TCG_REG_R0; | ||
165 | - tcg_out_movi32(s, COND_AL, base, ptr - dil); | ||
166 | - } | ||
167 | - tcg_out_ld32_12(s, COND_AL, TCG_REG_PC, base, dil); | ||
168 | - set_jmp_reset_offset(s, args[0]); | ||
169 | - } | ||
170 | - break; | ||
171 | case INDEX_op_goto_ptr: | ||
172 | tcg_out_b_reg(s, COND_AL, args[0]); | ||
173 | break; | ||
174 | @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, | ||
175 | case INDEX_op_mov_i32: /* Always emitted via tcg_out_mov. */ | ||
176 | case INDEX_op_call: /* Always emitted via tcg_out_call. */ | ||
177 | case INDEX_op_exit_tb: /* Always emitted via tcg_out_exit_tb. */ | ||
178 | + case INDEX_op_goto_tb: /* Always emitted via tcg_out_goto_tb. */ | ||
179 | default: | ||
180 | tcg_abort(); | ||
181 | } | ||
182 | diff --git a/tcg/i386/tcg-target.c.inc b/tcg/i386/tcg-target.c.inc | ||
183 | index XXXXXXX..XXXXXXX 100644 | ||
184 | --- a/tcg/i386/tcg-target.c.inc | ||
185 | +++ b/tcg/i386/tcg-target.c.inc | ||
186 | @@ -XXX,XX +XXX,XX @@ static void tcg_out_exit_tb(TCGContext *s, uintptr_t a0) | ||
187 | } | ||
188 | } | ||
189 | |||
190 | +static void tcg_out_goto_tb(TCGContext *s, int which) | ||
191 | +{ | ||
192 | + /* | ||
193 | + * Jump displacement must be aligned for atomic patching; | ||
194 | + * see if we need to add extra nops before jump | ||
195 | + */ | ||
196 | + int gap = QEMU_ALIGN_PTR_UP(s->code_ptr + 1, 4) - s->code_ptr; | ||
197 | + if (gap != 1) { | ||
198 | + tcg_out_nopn(s, gap - 1); | ||
199 | + } | ||
200 | + tcg_out8(s, OPC_JMP_long); /* jmp im */ | ||
201 | + set_jmp_insn_offset(s, which); | ||
202 | + tcg_out32(s, 0); | ||
203 | + set_jmp_reset_offset(s, which); | ||
204 | +} | ||
205 | + | ||
206 | static inline void tcg_out_op(TCGContext *s, TCGOpcode opc, | ||
207 | const TCGArg args[TCG_MAX_OP_ARGS], | ||
208 | const int const_args[TCG_MAX_OP_ARGS]) | ||
209 | @@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc, | ||
210 | const_a2 = const_args[2]; | ||
211 | |||
212 | switch (opc) { | ||
213 | - case INDEX_op_goto_tb: | ||
214 | - { | ||
215 | - /* | ||
216 | - * Jump displacement must be aligned for atomic patching; | ||
217 | - * see if we need to add extra nops before jump | ||
218 | - */ | ||
219 | - int gap = QEMU_ALIGN_PTR_UP(s->code_ptr + 1, 4) - s->code_ptr; | ||
220 | - if (gap != 1) { | ||
221 | - tcg_out_nopn(s, gap - 1); | ||
222 | - } | ||
223 | - tcg_out8(s, OPC_JMP_long); /* jmp im */ | ||
224 | - set_jmp_insn_offset(s, a0); | ||
225 | - tcg_out32(s, 0); | ||
226 | - } | ||
227 | - set_jmp_reset_offset(s, a0); | ||
228 | - break; | ||
229 | case INDEX_op_goto_ptr: | ||
230 | /* jmp to the given host address (could be epilogue) */ | ||
231 | tcg_out_modrm(s, OPC_GRP5, EXT5_JMPN_Ev, a0); | ||
232 | @@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc, | ||
233 | case INDEX_op_mov_i64: | ||
234 | case INDEX_op_call: /* Always emitted via tcg_out_call. */ | ||
235 | case INDEX_op_exit_tb: /* Always emitted via tcg_out_exit_tb. */ | ||
236 | + case INDEX_op_goto_tb: /* Always emitted via tcg_out_goto_tb. */ | ||
237 | default: | ||
238 | tcg_abort(); | ||
239 | } | ||
240 | diff --git a/tcg/loongarch64/tcg-target.c.inc b/tcg/loongarch64/tcg-target.c.inc | ||
241 | index XXXXXXX..XXXXXXX 100644 | ||
242 | --- a/tcg/loongarch64/tcg-target.c.inc | ||
243 | +++ b/tcg/loongarch64/tcg-target.c.inc | ||
244 | @@ -XXX,XX +XXX,XX @@ static void tcg_out_exit_tb(TCGContext *s, uintptr_t a0) | ||
245 | } | ||
246 | } | ||
247 | |||
248 | +static void tcg_out_goto_tb(TCGContext *s, int which) | ||
249 | +{ | ||
250 | + /* | ||
251 | + * Ensure that patch area is 8-byte aligned so that an | ||
252 | + * atomic write can be used to patch the target address. | ||
253 | + */ | ||
254 | + if ((uintptr_t)s->code_ptr & 7) { | ||
255 | + tcg_out_nop(s); | ||
256 | + } | ||
257 | + set_jmp_insn_offset(s, which); | ||
258 | + /* | ||
259 | + * actual branch destination will be patched by | ||
260 | + * tb_target_set_jmp_target later | ||
261 | + */ | ||
262 | + tcg_out_opc_pcaddu18i(s, TCG_REG_TMP0, 0); | ||
263 | + tcg_out_opc_jirl(s, TCG_REG_ZERO, TCG_REG_TMP0, 0); | ||
264 | + set_jmp_reset_offset(s, which); | ||
265 | +} | ||
266 | + | ||
267 | static void tcg_out_op(TCGContext *s, TCGOpcode opc, | ||
268 | const TCGArg args[TCG_MAX_OP_ARGS], | ||
269 | const int const_args[TCG_MAX_OP_ARGS]) | ||
270 | @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, | ||
271 | int c2 = const_args[2]; | ||
272 | |||
273 | switch (opc) { | ||
274 | - case INDEX_op_goto_tb: | ||
275 | - /* | ||
276 | - * Ensure that patch area is 8-byte aligned so that an | ||
277 | - * atomic write can be used to patch the target address. | ||
278 | - */ | ||
279 | - if ((uintptr_t)s->code_ptr & 7) { | ||
280 | - tcg_out_nop(s); | ||
281 | - } | ||
282 | - set_jmp_insn_offset(s, a0); | ||
283 | - /* | ||
284 | - * actual branch destination will be patched by | ||
285 | - * tb_target_set_jmp_target later | ||
286 | - */ | ||
287 | - tcg_out_opc_pcaddu18i(s, TCG_REG_TMP0, 0); | ||
288 | - tcg_out_opc_jirl(s, TCG_REG_ZERO, TCG_REG_TMP0, 0); | ||
289 | - set_jmp_reset_offset(s, a0); | ||
290 | - break; | ||
291 | - | ||
292 | case INDEX_op_mb: | ||
293 | tcg_out_mb(s, a0); | ||
294 | break; | ||
295 | @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, | ||
296 | case INDEX_op_mov_i64: | ||
297 | case INDEX_op_call: /* Always emitted via tcg_out_call. */ | ||
298 | case INDEX_op_exit_tb: /* Always emitted via tcg_out_exit_tb. */ | ||
299 | + case INDEX_op_goto_tb: /* Always emitted via tcg_out_goto_tb. */ | ||
300 | default: | ||
301 | g_assert_not_reached(); | ||
302 | } | ||
303 | diff --git a/tcg/mips/tcg-target.c.inc b/tcg/mips/tcg-target.c.inc | ||
304 | index XXXXXXX..XXXXXXX 100644 | ||
305 | --- a/tcg/mips/tcg-target.c.inc | ||
306 | +++ b/tcg/mips/tcg-target.c.inc | ||
307 | @@ -XXX,XX +XXX,XX @@ static void tcg_out_exit_tb(TCGContext *s, uintptr_t a0) | ||
308 | tcg_out_opc_imm(s, OPC_ORI, TCG_REG_V0, b0, a0 & 0xffff); | ||
309 | } | ||
310 | |||
311 | +static void tcg_out_goto_tb(TCGContext *s, int which) | ||
312 | +{ | ||
313 | + /* indirect jump method */ | ||
314 | + qemu_build_assert(!TCG_TARGET_HAS_direct_jump); | ||
315 | + tcg_out_ld(s, TCG_TYPE_PTR, TCG_TMP0, TCG_REG_ZERO, | ||
316 | + get_jmp_target_addr(s, which)); | ||
317 | + tcg_out_opc_reg(s, OPC_JR, 0, TCG_TMP0, 0); | ||
318 | + tcg_out_nop(s); | ||
319 | + set_jmp_reset_offset(s, which); | ||
320 | +} | ||
321 | + | ||
322 | static void tcg_out_op(TCGContext *s, TCGOpcode opc, | ||
323 | const TCGArg args[TCG_MAX_OP_ARGS], | ||
324 | const int const_args[TCG_MAX_OP_ARGS]) | ||
325 | @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, | ||
326 | c2 = const_args[2]; | ||
327 | |||
328 | switch (opc) { | ||
329 | - case INDEX_op_goto_tb: | ||
330 | - /* indirect jump method */ | ||
331 | - qemu_build_assert(!TCG_TARGET_HAS_direct_jump); | ||
332 | - tcg_out_ld(s, TCG_TYPE_PTR, TCG_TMP0, TCG_REG_ZERO, | ||
333 | - get_jmp_target_addr(s, a0)); | ||
334 | - tcg_out_opc_reg(s, OPC_JR, 0, TCG_TMP0, 0); | ||
335 | - tcg_out_nop(s); | ||
336 | - set_jmp_reset_offset(s, a0); | ||
337 | - break; | ||
338 | case INDEX_op_goto_ptr: | ||
339 | /* jmp to the given host address (could be epilogue) */ | ||
340 | tcg_out_opc_reg(s, OPC_JR, 0, a0, 0); | ||
341 | @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, | ||
342 | case INDEX_op_mov_i64: | ||
343 | case INDEX_op_call: /* Always emitted via tcg_out_call. */ | ||
344 | case INDEX_op_exit_tb: /* Always emitted via tcg_out_exit_tb. */ | ||
345 | + case INDEX_op_goto_tb: /* Always emitted via tcg_out_goto_tb. */ | ||
346 | default: | ||
347 | tcg_abort(); | ||
348 | } | ||
349 | diff --git a/tcg/ppc/tcg-target.c.inc b/tcg/ppc/tcg-target.c.inc | ||
350 | index XXXXXXX..XXXXXXX 100644 | ||
351 | --- a/tcg/ppc/tcg-target.c.inc | ||
352 | +++ b/tcg/ppc/tcg-target.c.inc | ||
353 | @@ -XXX,XX +XXX,XX @@ static void tcg_out_exit_tb(TCGContext *s, uintptr_t arg) | ||
354 | tcg_out_b(s, 0, tcg_code_gen_epilogue); | ||
355 | } | ||
356 | |||
357 | +static void tcg_out_goto_tb(TCGContext *s, int which) | ||
358 | +{ | ||
359 | + /* Direct jump. */ | ||
360 | + if (TCG_TARGET_REG_BITS == 64) { | ||
361 | + /* Ensure the next insns are 8 or 16-byte aligned. */ | ||
362 | + while ((uintptr_t)s->code_ptr & (have_isa_2_07 ? 15 : 7)) { | ||
363 | + tcg_out32(s, NOP); | ||
364 | + } | ||
365 | + set_jmp_insn_offset(s, which); | ||
366 | + tcg_out32(s, ADDIS | TAI(TCG_REG_TB, TCG_REG_TB, 0)); | ||
367 | + tcg_out32(s, ADDI | TAI(TCG_REG_TB, TCG_REG_TB, 0)); | ||
368 | + tcg_out32(s, MTSPR | RS(TCG_REG_TB) | CTR); | ||
369 | + tcg_out32(s, BCCTR | BO_ALWAYS); | ||
370 | + set_jmp_reset_offset(s, which); | ||
371 | + if (USE_REG_TB) { | ||
372 | + /* For the unlinked case, need to reset TCG_REG_TB. */ | ||
373 | + tcg_out_mem_long(s, ADDI, ADD, TCG_REG_TB, TCG_REG_TB, | ||
374 | + -tcg_current_code_size(s)); | ||
375 | + } | ||
376 | + } else { | ||
377 | + set_jmp_insn_offset(s, which); | ||
378 | + tcg_out32(s, B); | ||
379 | + set_jmp_reset_offset(s, which); | ||
380 | + } | ||
381 | +} | ||
382 | + | ||
383 | static void tcg_out_op(TCGContext *s, TCGOpcode opc, | ||
384 | const TCGArg args[TCG_MAX_OP_ARGS], | ||
385 | const int const_args[TCG_MAX_OP_ARGS]) | ||
386 | @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, | ||
387 | TCGArg a0, a1, a2; | ||
388 | |||
389 | switch (opc) { | ||
390 | - case INDEX_op_goto_tb: | ||
391 | - /* Direct jump. */ | ||
392 | - if (TCG_TARGET_REG_BITS == 64) { | ||
393 | - /* Ensure the next insns are 8 or 16-byte aligned. */ | ||
394 | - while ((uintptr_t)s->code_ptr & (have_isa_2_07 ? 15 : 7)) { | ||
395 | - tcg_out32(s, NOP); | ||
396 | - } | ||
397 | - set_jmp_insn_offset(s, args[0]); | ||
398 | - tcg_out32(s, ADDIS | TAI(TCG_REG_TB, TCG_REG_TB, 0)); | ||
399 | - tcg_out32(s, ADDI | TAI(TCG_REG_TB, TCG_REG_TB, 0)); | ||
400 | - } else { | ||
401 | - set_jmp_insn_offset(s, args[0]); | ||
402 | - tcg_out32(s, B); | ||
403 | - set_jmp_reset_offset(s, args[0]); | ||
404 | - break; | ||
405 | - } | ||
406 | - tcg_out32(s, MTSPR | RS(TCG_REG_TB) | CTR); | ||
407 | - tcg_out32(s, BCCTR | BO_ALWAYS); | ||
408 | - set_jmp_reset_offset(s, args[0]); | ||
409 | - if (USE_REG_TB) { | ||
410 | - /* For the unlinked case, need to reset TCG_REG_TB. */ | ||
411 | - tcg_out_mem_long(s, ADDI, ADD, TCG_REG_TB, TCG_REG_TB, | ||
412 | - -tcg_current_code_size(s)); | ||
413 | - } | ||
414 | - break; | ||
415 | case INDEX_op_goto_ptr: | ||
416 | tcg_out32(s, MTSPR | RS(args[0]) | CTR); | ||
417 | if (USE_REG_TB) { | ||
418 | @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, | ||
419 | case INDEX_op_mov_i64: | ||
420 | case INDEX_op_call: /* Always emitted via tcg_out_call. */ | ||
421 | case INDEX_op_exit_tb: /* Always emitted via tcg_out_exit_tb. */ | ||
422 | + case INDEX_op_goto_tb: /* Always emitted via tcg_out_goto_tb. */ | ||
423 | default: | ||
424 | tcg_abort(); | ||
425 | } | ||
426 | diff --git a/tcg/riscv/tcg-target.c.inc b/tcg/riscv/tcg-target.c.inc | ||
427 | index XXXXXXX..XXXXXXX 100644 | ||
428 | --- a/tcg/riscv/tcg-target.c.inc | ||
429 | +++ b/tcg/riscv/tcg-target.c.inc | ||
430 | @@ -XXX,XX +XXX,XX @@ static void tcg_out_exit_tb(TCGContext *s, uintptr_t a0) | ||
431 | } | ||
432 | } | ||
433 | |||
434 | +static void tcg_out_goto_tb(TCGContext *s, int which) | ||
435 | +{ | ||
436 | + qemu_build_assert(!TCG_TARGET_HAS_direct_jump); | ||
437 | + /* indirect jump method */ | ||
438 | + tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_TMP0, TCG_REG_ZERO, | ||
439 | + get_jmp_target_addr(s, which)); | ||
440 | + tcg_out_opc_imm(s, OPC_JALR, TCG_REG_ZERO, TCG_REG_TMP0, 0); | ||
441 | + set_jmp_reset_offset(s, which); | ||
442 | +} | ||
443 | + | ||
444 | static void tcg_out_op(TCGContext *s, TCGOpcode opc, | ||
445 | const TCGArg args[TCG_MAX_OP_ARGS], | ||
446 | const int const_args[TCG_MAX_OP_ARGS]) | ||
447 | @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, | ||
448 | int c2 = const_args[2]; | ||
449 | |||
450 | switch (opc) { | ||
451 | - case INDEX_op_goto_tb: | ||
452 | - qemu_build_assert(!TCG_TARGET_HAS_direct_jump); | ||
453 | - /* indirect jump method */ | ||
454 | - tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_TMP0, TCG_REG_ZERO, | ||
455 | - get_jmp_target_addr(s, a0)); | ||
456 | - tcg_out_opc_imm(s, OPC_JALR, TCG_REG_ZERO, TCG_REG_TMP0, 0); | ||
457 | - set_jmp_reset_offset(s, a0); | ||
458 | - break; | ||
459 | - | ||
460 | case INDEX_op_goto_ptr: | ||
461 | tcg_out_opc_imm(s, OPC_JALR, TCG_REG_ZERO, a0, 0); | ||
462 | break; | ||
463 | @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, | ||
464 | case INDEX_op_mov_i64: | ||
465 | case INDEX_op_call: /* Always emitted via tcg_out_call. */ | ||
466 | case INDEX_op_exit_tb: /* Always emitted via tcg_out_exit_tb. */ | ||
467 | + case INDEX_op_goto_tb: /* Always emitted via tcg_out_goto_tb. */ | ||
468 | default: | ||
469 | g_assert_not_reached(); | ||
470 | } | ||
471 | diff --git a/tcg/s390x/tcg-target.c.inc b/tcg/s390x/tcg-target.c.inc | ||
472 | index XXXXXXX..XXXXXXX 100644 | ||
473 | --- a/tcg/s390x/tcg-target.c.inc | ||
474 | +++ b/tcg/s390x/tcg-target.c.inc | ||
475 | @@ -XXX,XX +XXX,XX @@ static void tcg_out_exit_tb(TCGContext *s, uintptr_t a0) | ||
476 | } | ||
477 | } | ||
478 | |||
479 | +static void tcg_out_goto_tb(TCGContext *s, int which) | ||
480 | +{ | ||
481 | + /* | ||
482 | + * Branch displacement must be aligned for atomic patching; | ||
483 | + * see if we need to add extra nop before branch | ||
484 | + */ | ||
485 | + if (!QEMU_PTR_IS_ALIGNED(s->code_ptr + 1, 4)) { | ||
486 | + tcg_out16(s, NOP); | ||
487 | + } | ||
488 | + tcg_out16(s, RIL_BRCL | (S390_CC_ALWAYS << 4)); | ||
489 | + set_jmp_insn_offset(s, which); | ||
490 | + s->code_ptr += 2; | ||
491 | + set_jmp_reset_offset(s, which); | ||
492 | +} | ||
493 | + | ||
494 | # define OP_32_64(x) \ | ||
495 | case glue(glue(INDEX_op_,x),_i32): \ | ||
496 | case glue(glue(INDEX_op_,x),_i64) | ||
497 | @@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc, | ||
498 | TCGArg a0, a1, a2; | ||
499 | |||
500 | switch (opc) { | ||
501 | - case INDEX_op_goto_tb: | ||
502 | - a0 = args[0]; | ||
503 | - /* | ||
504 | - * branch displacement must be aligned for atomic patching; | ||
505 | - * see if we need to add extra nop before branch | ||
506 | - */ | ||
507 | - if (!QEMU_PTR_IS_ALIGNED(s->code_ptr + 1, 4)) { | ||
508 | - tcg_out16(s, NOP); | ||
509 | - } | ||
510 | - tcg_out16(s, RIL_BRCL | (S390_CC_ALWAYS << 4)); | ||
511 | - set_jmp_insn_offset(s, a0); | ||
512 | - s->code_ptr += 2; | ||
513 | - set_jmp_reset_offset(s, a0); | ||
514 | - break; | ||
515 | - | ||
516 | case INDEX_op_goto_ptr: | ||
517 | a0 = args[0]; | ||
518 | tcg_out_insn(s, RR, BCR, S390_CC_ALWAYS, a0); | ||
519 | @@ -XXX,XX +XXX,XX @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc, | ||
520 | case INDEX_op_mov_i64: | ||
521 | case INDEX_op_call: /* Always emitted via tcg_out_call. */ | ||
522 | case INDEX_op_exit_tb: /* Always emitted via tcg_out_exit_tb. */ | ||
523 | + case INDEX_op_goto_tb: /* Always emitted via tcg_out_goto_tb. */ | ||
524 | default: | ||
525 | tcg_abort(); | ||
526 | } | ||
527 | diff --git a/tcg/sparc64/tcg-target.c.inc b/tcg/sparc64/tcg-target.c.inc | ||
528 | index XXXXXXX..XXXXXXX 100644 | ||
529 | --- a/tcg/sparc64/tcg-target.c.inc | ||
530 | +++ b/tcg/sparc64/tcg-target.c.inc | ||
531 | @@ -XXX,XX +XXX,XX @@ static void tcg_out_exit_tb(TCGContext *s, uintptr_t a0) | ||
532 | tcg_out_arithi(s, TCG_REG_O0, TCG_REG_O0, a0 & 0x3ff, ARITH_OR); | ||
533 | } | ||
534 | |||
535 | +static void tcg_out_goto_tb(TCGContext *s, int which) | ||
536 | +{ | ||
537 | + /* Direct jump. */ | ||
538 | + if (USE_REG_TB) { | ||
539 | + /* make sure the patch is 8-byte aligned. */ | ||
540 | + if ((intptr_t)s->code_ptr & 4) { | ||
541 | + tcg_out_nop(s); | ||
542 | + } | ||
543 | + set_jmp_insn_offset(s, which); | ||
544 | + tcg_out_sethi(s, TCG_REG_T1, 0); | ||
545 | + tcg_out_arithi(s, TCG_REG_T1, TCG_REG_T1, 0, ARITH_OR); | ||
546 | + tcg_out_arith(s, TCG_REG_G0, TCG_REG_TB, TCG_REG_T1, JMPL); | ||
547 | + tcg_out_arith(s, TCG_REG_TB, TCG_REG_TB, TCG_REG_T1, ARITH_ADD); | ||
548 | + } else { | ||
549 | + set_jmp_insn_offset(s, which); | ||
550 | + tcg_out32(s, CALL); | ||
551 | + tcg_out_nop(s); | ||
552 | + } | ||
553 | + set_jmp_reset_offset(s, which); | ||
554 | + | ||
555 | + /* | ||
556 | + * For the unlinked path of goto_tb, we need to reset TCG_REG_TB | ||
557 | + * to the beginning of this TB. | ||
558 | + */ | ||
559 | + if (USE_REG_TB) { | ||
560 | + int c = -tcg_current_code_size(s); | ||
561 | + if (check_fit_i32(c, 13)) { | ||
562 | + tcg_out_arithi(s, TCG_REG_TB, TCG_REG_TB, c, ARITH_ADD); | ||
563 | + } else { | ||
564 | + tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_T1, c); | ||
565 | + tcg_out_arith(s, TCG_REG_TB, TCG_REG_TB, TCG_REG_T1, ARITH_ADD); | ||
566 | + } | ||
567 | + } | ||
568 | +} | ||
569 | + | ||
570 | static void tcg_out_op(TCGContext *s, TCGOpcode opc, | ||
571 | const TCGArg args[TCG_MAX_OP_ARGS], | ||
572 | const int const_args[TCG_MAX_OP_ARGS]) | ||
573 | @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, | ||
574 | c2 = const_args[2]; | ||
575 | |||
576 | switch (opc) { | ||
577 | - case INDEX_op_goto_tb: | ||
578 | - /* Direct jump. */ | ||
579 | - if (USE_REG_TB) { | ||
580 | - /* make sure the patch is 8-byte aligned. */ | ||
581 | - if ((intptr_t)s->code_ptr & 4) { | ||
582 | - tcg_out_nop(s); | ||
583 | - } | ||
584 | - set_jmp_insn_offset(s, a0); | ||
585 | - tcg_out_sethi(s, TCG_REG_T1, 0); | ||
586 | - tcg_out_arithi(s, TCG_REG_T1, TCG_REG_T1, 0, ARITH_OR); | ||
587 | - tcg_out_arith(s, TCG_REG_G0, TCG_REG_TB, TCG_REG_T1, JMPL); | ||
588 | - tcg_out_arith(s, TCG_REG_TB, TCG_REG_TB, TCG_REG_T1, ARITH_ADD); | ||
589 | - } else { | ||
590 | - set_jmp_insn_offset(s, a0); | ||
591 | - tcg_out32(s, CALL); | ||
592 | - tcg_out_nop(s); | ||
593 | - } | ||
594 | - set_jmp_reset_offset(s, a0); | ||
595 | - | ||
596 | - /* For the unlinked path of goto_tb, we need to reset | ||
597 | - TCG_REG_TB to the beginning of this TB. */ | ||
598 | - if (USE_REG_TB) { | ||
599 | - c = -tcg_current_code_size(s); | ||
600 | - if (check_fit_i32(c, 13)) { | ||
601 | - tcg_out_arithi(s, TCG_REG_TB, TCG_REG_TB, c, ARITH_ADD); | ||
602 | - } else { | ||
603 | - tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_T1, c); | ||
604 | - tcg_out_arith(s, TCG_REG_TB, TCG_REG_TB, | ||
605 | - TCG_REG_T1, ARITH_ADD); | ||
606 | - } | ||
607 | - } | ||
608 | - break; | ||
609 | case INDEX_op_goto_ptr: | ||
610 | tcg_out_arithi(s, TCG_REG_G0, a0, 0, JMPL); | ||
611 | if (USE_REG_TB) { | ||
612 | @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, | ||
613 | case INDEX_op_mov_i64: | ||
614 | case INDEX_op_call: /* Always emitted via tcg_out_call. */ | ||
615 | case INDEX_op_exit_tb: /* Always emitted via tcg_out_exit_tb. */ | ||
616 | + case INDEX_op_goto_tb: /* Always emitted via tcg_out_goto_tb. */ | ||
617 | default: | ||
618 | tcg_abort(); | ||
619 | } | ||
620 | diff --git a/tcg/tci/tcg-target.c.inc b/tcg/tci/tcg-target.c.inc | ||
621 | index XXXXXXX..XXXXXXX 100644 | ||
622 | --- a/tcg/tci/tcg-target.c.inc | ||
623 | +++ b/tcg/tci/tcg-target.c.inc | ||
624 | @@ -XXX,XX +XXX,XX @@ static void tcg_out_exit_tb(TCGContext *s, uintptr_t arg) | ||
625 | tcg_out_op_p(s, INDEX_op_exit_tb, (void *)arg); | ||
626 | } | ||
627 | |||
628 | +static void tcg_out_goto_tb(TCGContext *s, int which) | ||
629 | +{ | ||
630 | + qemu_build_assert(!TCG_TARGET_HAS_direct_jump); | ||
631 | + /* indirect jump method. */ | ||
632 | + tcg_out_op_p(s, INDEX_op_goto_tb, (void *)get_jmp_target_addr(s, which)); | ||
633 | + set_jmp_reset_offset(s, which); | ||
634 | +} | ||
635 | + | ||
636 | static void tcg_out_op(TCGContext *s, TCGOpcode opc, | ||
637 | const TCGArg args[TCG_MAX_OP_ARGS], | ||
638 | const int const_args[TCG_MAX_OP_ARGS]) | ||
639 | @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, | ||
640 | TCGOpcode exts; | ||
641 | |||
642 | switch (opc) { | ||
643 | - case INDEX_op_goto_tb: | ||
644 | - qemu_build_assert(!TCG_TARGET_HAS_direct_jump); | ||
645 | - /* indirect jump method. */ | ||
646 | - tcg_out_op_p(s, opc, (void *)get_jmp_target_addr(s, args[0])); | ||
647 | - set_jmp_reset_offset(s, args[0]); | ||
648 | - break; | ||
649 | - | ||
650 | case INDEX_op_goto_ptr: | ||
651 | tcg_out_op_r(s, opc, args[0]); | ||
652 | break; | ||
653 | @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, | ||
654 | case INDEX_op_mov_i64: | ||
655 | case INDEX_op_call: /* Always emitted via tcg_out_call. */ | ||
656 | case INDEX_op_exit_tb: /* Always emitted via tcg_out_exit_tb. */ | ||
657 | + case INDEX_op_goto_tb: /* Always emitted via tcg_out_goto_tb. */ | ||
658 | default: | ||
659 | tcg_abort(); | ||
660 | } | ||
25 | -- | 661 | -- |
26 | 2.20.1 | 662 | 2.34.1 |
27 | 663 | ||
28 | 664 | diff view generated by jsdifflib |
1 | The accel_initialised variable no longer has any setters. | 1 | This will shortly be used for more than reset. |
---|---|---|---|
2 | 2 | ||
3 | Fixes: 6f6e1698a68c | ||
4 | Acked-by: Paolo Bonzini <pbonzini@redhat.com> | ||
5 | Reviewed-by: Alex Bennée <alex.bennee@linaro.org> | 3 | Reviewed-by: Alex Bennée <alex.bennee@linaro.org> |
6 | Reviewed-by: Philippe Mathieu-Daudé <philmd@redhat.com> | 4 | Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org> |
7 | Reviewed by: Aleksandar Markovic <amarkovic@wavecomp.com> | ||
8 | Signed-off-by: Richard Henderson <richard.henderson@linaro.org> | 5 | Signed-off-by: Richard Henderson <richard.henderson@linaro.org> |
9 | --- | 6 | --- |
10 | vl.c | 3 +-- | 7 | include/exec/exec-all.h | 2 +- |
11 | 1 file changed, 1 insertion(+), 2 deletions(-) | 8 | accel/tcg/translate-all.c | 8 ++++---- |
9 | tcg/tcg.c | 4 ++-- | ||
10 | 3 files changed, 7 insertions(+), 7 deletions(-) | ||
12 | 11 | ||
13 | diff --git a/vl.c b/vl.c | 12 | diff --git a/include/exec/exec-all.h b/include/exec/exec-all.h |
14 | index XXXXXXX..XXXXXXX 100644 | 13 | index XXXXXXX..XXXXXXX 100644 |
15 | --- a/vl.c | 14 | --- a/include/exec/exec-all.h |
16 | +++ b/vl.c | 15 | +++ b/include/exec/exec-all.h |
17 | @@ -XXX,XX +XXX,XX @@ static void configure_accelerators(const char *progname) | 16 | @@ -XXX,XX +XXX,XX @@ struct TranslationBlock { |
18 | { | 17 | * setting one of the jump targets (or patching the jump instruction). Only |
19 | const char *accel; | 18 | * two of such jumps are supported. |
20 | char **accel_list, **tmp; | 19 | */ |
21 | - bool accel_initialised = false; | 20 | +#define TB_JMP_OFFSET_INVALID 0xffff /* indicates no jump generated */ |
22 | bool init_failed = false; | 21 | uint16_t jmp_reset_offset[2]; /* offset of original jump target */ |
23 | 22 | -#define TB_JMP_RESET_OFFSET_INVALID 0xffff /* indicates no jump generated */ | |
24 | qemu_opts_foreach(qemu_find_opts("icount"), | 23 | uintptr_t jmp_target_arg[2]; /* target address or offset */ |
25 | @@ -XXX,XX +XXX,XX @@ static void configure_accelerators(const char *progname) | 24 | |
26 | 25 | /* | |
27 | accel_list = g_strsplit(accel, ":", 0); | 26 | diff --git a/accel/tcg/translate-all.c b/accel/tcg/translate-all.c |
28 | 27 | index XXXXXXX..XXXXXXX 100644 | |
29 | - for (tmp = accel_list; !accel_initialised && tmp && *tmp; tmp++) { | 28 | --- a/accel/tcg/translate-all.c |
30 | + for (tmp = accel_list; tmp && *tmp; tmp++) { | 29 | +++ b/accel/tcg/translate-all.c |
31 | /* | 30 | @@ -XXX,XX +XXX,XX @@ TranslationBlock *tb_gen_code(CPUState *cpu, |
32 | * Filter invalid accelerators here, to prevent obscenities | 31 | tb->jmp_dest[1] = (uintptr_t)NULL; |
33 | * such as "-machine accel=tcg,,thread=single". | 32 | |
33 | /* init original jump addresses which have been set during tcg_gen_code() */ | ||
34 | - if (tb->jmp_reset_offset[0] != TB_JMP_RESET_OFFSET_INVALID) { | ||
35 | + if (tb->jmp_reset_offset[0] != TB_JMP_OFFSET_INVALID) { | ||
36 | tb_reset_jump(tb, 0); | ||
37 | } | ||
38 | - if (tb->jmp_reset_offset[1] != TB_JMP_RESET_OFFSET_INVALID) { | ||
39 | + if (tb->jmp_reset_offset[1] != TB_JMP_OFFSET_INVALID) { | ||
40 | tb_reset_jump(tb, 1); | ||
41 | } | ||
42 | |||
43 | @@ -XXX,XX +XXX,XX @@ static gboolean tb_tree_stats_iter(gpointer key, gpointer value, gpointer data) | ||
44 | if (tb_page_addr1(tb) != -1) { | ||
45 | tst->cross_page++; | ||
46 | } | ||
47 | - if (tb->jmp_reset_offset[0] != TB_JMP_RESET_OFFSET_INVALID) { | ||
48 | + if (tb->jmp_reset_offset[0] != TB_JMP_OFFSET_INVALID) { | ||
49 | tst->direct_jmp_count++; | ||
50 | - if (tb->jmp_reset_offset[1] != TB_JMP_RESET_OFFSET_INVALID) { | ||
51 | + if (tb->jmp_reset_offset[1] != TB_JMP_OFFSET_INVALID) { | ||
52 | tst->direct_jmp2_count++; | ||
53 | } | ||
54 | } | ||
55 | diff --git a/tcg/tcg.c b/tcg/tcg.c | ||
56 | index XXXXXXX..XXXXXXX 100644 | ||
57 | --- a/tcg/tcg.c | ||
58 | +++ b/tcg/tcg.c | ||
59 | @@ -XXX,XX +XXX,XX @@ int tcg_gen_code(TCGContext *s, TranslationBlock *tb, target_ulong pc_start) | ||
60 | #endif | ||
61 | |||
62 | /* Initialize goto_tb jump offsets. */ | ||
63 | - tb->jmp_reset_offset[0] = TB_JMP_RESET_OFFSET_INVALID; | ||
64 | - tb->jmp_reset_offset[1] = TB_JMP_RESET_OFFSET_INVALID; | ||
65 | + tb->jmp_reset_offset[0] = TB_JMP_OFFSET_INVALID; | ||
66 | + tb->jmp_reset_offset[1] = TB_JMP_OFFSET_INVALID; | ||
67 | tcg_ctx->tb_jmp_reset_offset = tb->jmp_reset_offset; | ||
68 | if (TCG_TARGET_HAS_direct_jump) { | ||
69 | tcg_ctx->tb_jmp_insn_offset = tb->jmp_target_arg; | ||
34 | -- | 70 | -- |
35 | 2.20.1 | 71 | 2.34.1 |
36 | 72 | ||
37 | 73 | diff view generated by jsdifflib |
1 | In target/arm we will shortly have "too many" mmu_idx. | 1 | This can replace four other variables that are references |
---|---|---|---|
2 | The current minimum barrier is caused by the way in which | 2 | into the TranslationBlock structure. |
3 | tlb_flush_page_by_mmuidx is coded. | ||
4 | 3 | ||
5 | We can remove this limitation by allocating memory for | 4 | Reviewed-by: Alex Bennée <alex.bennee@linaro.org> |
6 | consumption by the worker. Let us assume that this is | ||
7 | the unlikely case, as will be the case for the majority | ||
8 | of targets which have so far satisfied the BUILD_BUG_ON, | ||
9 | and only allocate memory when necessary. | ||
10 | |||
11 | Reviewed-by: Peter Maydell <peter.maydell@linaro.org> | ||
12 | Signed-off-by: Richard Henderson <richard.henderson@linaro.org> | 5 | Signed-off-by: Richard Henderson <richard.henderson@linaro.org> |
13 | --- | 6 | --- |
14 | accel/tcg/cputlb.c | 167 +++++++++++++++++++++++++++++++++++---------- | 7 | include/tcg/tcg.h | 11 +++-------- |
15 | 1 file changed, 132 insertions(+), 35 deletions(-) | 8 | accel/tcg/translate-all.c | 2 +- |
9 | tcg/tcg-op.c | 14 +++++++------- | ||
10 | tcg/tcg.c | 14 +++----------- | ||
11 | 4 files changed, 14 insertions(+), 27 deletions(-) | ||
16 | 12 | ||
17 | diff --git a/accel/tcg/cputlb.c b/accel/tcg/cputlb.c | 13 | diff --git a/include/tcg/tcg.h b/include/tcg/tcg.h |
18 | index XXXXXXX..XXXXXXX 100644 | 14 | index XXXXXXX..XXXXXXX 100644 |
19 | --- a/accel/tcg/cputlb.c | 15 | --- a/include/tcg/tcg.h |
20 | +++ b/accel/tcg/cputlb.c | 16 | +++ b/include/tcg/tcg.h |
21 | @@ -XXX,XX +XXX,XX @@ static void tlb_flush_page_locked(CPUArchState *env, int midx, | 17 | @@ -XXX,XX +XXX,XX @@ struct TCGContext { |
18 | int nb_indirects; | ||
19 | int nb_ops; | ||
20 | |||
21 | - /* goto_tb support */ | ||
22 | - tcg_insn_unit *code_buf; | ||
23 | - uint16_t *tb_jmp_reset_offset; /* tb->jmp_reset_offset */ | ||
24 | - uintptr_t *tb_jmp_insn_offset; /* tb->jmp_target_arg if direct_jump */ | ||
25 | - uintptr_t *tb_jmp_target_addr; /* tb->jmp_target_arg if !direct_jump */ | ||
26 | - | ||
27 | TCGRegSet reserved_regs; | ||
28 | - uint32_t tb_cflags; /* cflags of the current TB */ | ||
29 | intptr_t current_frame_offset; | ||
30 | intptr_t frame_start; | ||
31 | intptr_t frame_end; | ||
32 | TCGTemp *frame_temp; | ||
33 | |||
34 | - tcg_insn_unit *code_ptr; | ||
35 | + TranslationBlock *gen_tb; /* tb for which code is being generated */ | ||
36 | + tcg_insn_unit *code_buf; /* pointer for start of tb */ | ||
37 | + tcg_insn_unit *code_ptr; /* pointer for running end of tb */ | ||
38 | |||
39 | #ifdef CONFIG_PROFILER | ||
40 | TCGProfile prof; | ||
41 | diff --git a/accel/tcg/translate-all.c b/accel/tcg/translate-all.c | ||
42 | index XXXXXXX..XXXXXXX 100644 | ||
43 | --- a/accel/tcg/translate-all.c | ||
44 | +++ b/accel/tcg/translate-all.c | ||
45 | @@ -XXX,XX +XXX,XX @@ TranslationBlock *tb_gen_code(CPUState *cpu, | ||
46 | tb->trace_vcpu_dstate = *cpu->trace_dstate; | ||
47 | tb_set_page_addr0(tb, phys_pc); | ||
48 | tb_set_page_addr1(tb, -1); | ||
49 | - tcg_ctx->tb_cflags = cflags; | ||
50 | + tcg_ctx->gen_tb = tb; | ||
51 | tb_overflow: | ||
52 | |||
53 | #ifdef CONFIG_PROFILER | ||
54 | diff --git a/tcg/tcg-op.c b/tcg/tcg-op.c | ||
55 | index XXXXXXX..XXXXXXX 100644 | ||
56 | --- a/tcg/tcg-op.c | ||
57 | +++ b/tcg/tcg-op.c | ||
58 | @@ -XXX,XX +XXX,XX @@ void tcg_gen_op6(TCGOpcode opc, TCGArg a1, TCGArg a2, TCGArg a3, | ||
59 | |||
60 | void tcg_gen_mb(TCGBar mb_type) | ||
61 | { | ||
62 | - if (tcg_ctx->tb_cflags & CF_PARALLEL) { | ||
63 | + if (tcg_ctx->gen_tb->cflags & CF_PARALLEL) { | ||
64 | tcg_gen_op1(INDEX_op_mb, mb_type); | ||
22 | } | 65 | } |
23 | } | 66 | } |
24 | 67 | @@ -XXX,XX +XXX,XX @@ void tcg_gen_exit_tb(const TranslationBlock *tb, unsigned idx) | |
25 | -/* As we are going to hijack the bottom bits of the page address for a | 68 | void tcg_gen_goto_tb(unsigned idx) |
26 | - * mmuidx bit mask we need to fail to build if we can't do that | ||
27 | +/** | ||
28 | + * tlb_flush_page_by_mmuidx_async_0: | ||
29 | + * @cpu: cpu on which to flush | ||
30 | + * @addr: page of virtual address to flush | ||
31 | + * @idxmap: set of mmu_idx to flush | ||
32 | + * | ||
33 | + * Helper for tlb_flush_page_by_mmuidx and friends, flush one page | ||
34 | + * at @addr from the tlbs indicated by @idxmap from @cpu. | ||
35 | */ | ||
36 | -QEMU_BUILD_BUG_ON(NB_MMU_MODES > TARGET_PAGE_BITS_MIN); | ||
37 | - | ||
38 | -static void tlb_flush_page_by_mmuidx_async_work(CPUState *cpu, | ||
39 | - run_on_cpu_data data) | ||
40 | +static void tlb_flush_page_by_mmuidx_async_0(CPUState *cpu, | ||
41 | + target_ulong addr, | ||
42 | + uint16_t idxmap) | ||
43 | { | 69 | { |
44 | CPUArchState *env = cpu->env_ptr; | 70 | /* We tested CF_NO_GOTO_TB in translator_use_goto_tb. */ |
45 | - target_ulong addr_and_mmuidx = (target_ulong) data.target_ptr; | 71 | - tcg_debug_assert(!(tcg_ctx->tb_cflags & CF_NO_GOTO_TB)); |
46 | - target_ulong addr = addr_and_mmuidx & TARGET_PAGE_MASK; | 72 | + tcg_debug_assert(!(tcg_ctx->gen_tb->cflags & CF_NO_GOTO_TB)); |
47 | - unsigned long mmu_idx_bitmap = addr_and_mmuidx & ALL_MMUIDX_BITS; | 73 | /* We only support two chained exits. */ |
48 | int mmu_idx; | 74 | tcg_debug_assert(idx <= TB_EXIT_IDXMAX); |
49 | 75 | #ifdef CONFIG_DEBUG_TCG | |
50 | assert_cpu_is_self(cpu); | 76 | @@ -XXX,XX +XXX,XX @@ void tcg_gen_lookup_and_goto_ptr(void) |
51 | 77 | { | |
52 | - tlb_debug("page addr:" TARGET_FMT_lx " mmu_map:0x%lx\n", | 78 | TCGv_ptr ptr; |
53 | - addr, mmu_idx_bitmap); | 79 | |
54 | + tlb_debug("page addr:" TARGET_FMT_lx " mmu_map:0x%x\n", addr, idxmap); | 80 | - if (tcg_ctx->tb_cflags & CF_NO_GOTO_PTR) { |
55 | 81 | + if (tcg_ctx->gen_tb->cflags & CF_NO_GOTO_PTR) { | |
56 | qemu_spin_lock(&env_tlb(env)->c.lock); | 82 | tcg_gen_exit_tb(NULL, 0); |
57 | for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) { | 83 | return; |
58 | - if (test_bit(mmu_idx, &mmu_idx_bitmap)) { | ||
59 | + if ((idxmap >> mmu_idx) & 1) { | ||
60 | tlb_flush_page_locked(env, mmu_idx, addr); | ||
61 | } | ||
62 | } | 84 | } |
63 | @@ -XXX,XX +XXX,XX @@ static void tlb_flush_page_by_mmuidx_async_work(CPUState *cpu, | 85 | @@ -XXX,XX +XXX,XX @@ void tcg_gen_atomic_cmpxchg_i32(TCGv_i32 retv, TCGv addr, TCGv_i32 cmpv, |
64 | tb_flush_jmp_cache(cpu, addr); | 86 | { |
87 | memop = tcg_canonicalize_memop(memop, 0, 0); | ||
88 | |||
89 | - if (!(tcg_ctx->tb_cflags & CF_PARALLEL)) { | ||
90 | + if (!(tcg_ctx->gen_tb->cflags & CF_PARALLEL)) { | ||
91 | TCGv_i32 t1 = tcg_temp_new_i32(); | ||
92 | TCGv_i32 t2 = tcg_temp_new_i32(); | ||
93 | |||
94 | @@ -XXX,XX +XXX,XX @@ void tcg_gen_atomic_cmpxchg_i64(TCGv_i64 retv, TCGv addr, TCGv_i64 cmpv, | ||
95 | { | ||
96 | memop = tcg_canonicalize_memop(memop, 1, 0); | ||
97 | |||
98 | - if (!(tcg_ctx->tb_cflags & CF_PARALLEL)) { | ||
99 | + if (!(tcg_ctx->gen_tb->cflags & CF_PARALLEL)) { | ||
100 | TCGv_i64 t1 = tcg_temp_new_i64(); | ||
101 | TCGv_i64 t2 = tcg_temp_new_i64(); | ||
102 | |||
103 | @@ -XXX,XX +XXX,XX @@ static void * const table_##NAME[(MO_SIZE | MO_BSWAP) + 1] = { \ | ||
104 | void tcg_gen_atomic_##NAME##_i32 \ | ||
105 | (TCGv_i32 ret, TCGv addr, TCGv_i32 val, TCGArg idx, MemOp memop) \ | ||
106 | { \ | ||
107 | - if (tcg_ctx->tb_cflags & CF_PARALLEL) { \ | ||
108 | + if (tcg_ctx->gen_tb->cflags & CF_PARALLEL) { \ | ||
109 | do_atomic_op_i32(ret, addr, val, idx, memop, table_##NAME); \ | ||
110 | } else { \ | ||
111 | do_nonatomic_op_i32(ret, addr, val, idx, memop, NEW, \ | ||
112 | @@ -XXX,XX +XXX,XX @@ void tcg_gen_atomic_##NAME##_i32 \ | ||
113 | void tcg_gen_atomic_##NAME##_i64 \ | ||
114 | (TCGv_i64 ret, TCGv addr, TCGv_i64 val, TCGArg idx, MemOp memop) \ | ||
115 | { \ | ||
116 | - if (tcg_ctx->tb_cflags & CF_PARALLEL) { \ | ||
117 | + if (tcg_ctx->gen_tb->cflags & CF_PARALLEL) { \ | ||
118 | do_atomic_op_i64(ret, addr, val, idx, memop, table_##NAME); \ | ||
119 | } else { \ | ||
120 | do_nonatomic_op_i64(ret, addr, val, idx, memop, NEW, \ | ||
121 | diff --git a/tcg/tcg.c b/tcg/tcg.c | ||
122 | index XXXXXXX..XXXXXXX 100644 | ||
123 | --- a/tcg/tcg.c | ||
124 | +++ b/tcg/tcg.c | ||
125 | @@ -XXX,XX +XXX,XX @@ static void set_jmp_reset_offset(TCGContext *s, int which) | ||
126 | * We will check for overflow at the end of the opcode loop in | ||
127 | * tcg_gen_code, where we bound tcg_current_code_size to UINT16_MAX. | ||
128 | */ | ||
129 | - s->tb_jmp_reset_offset[which] = tcg_current_code_size(s); | ||
130 | + s->gen_tb->jmp_reset_offset[which] = tcg_current_code_size(s); | ||
65 | } | 131 | } |
66 | 132 | ||
67 | +/** | 133 | static void G_GNUC_UNUSED set_jmp_insn_offset(TCGContext *s, int which) |
68 | + * tlb_flush_page_by_mmuidx_async_1: | 134 | @@ -XXX,XX +XXX,XX @@ static void G_GNUC_UNUSED set_jmp_insn_offset(TCGContext *s, int which) |
69 | + * @cpu: cpu on which to flush | 135 | * tcg_gen_code, where we bound tcg_current_code_size to UINT16_MAX. |
70 | + * @data: encoded addr + idxmap | 136 | */ |
71 | + * | 137 | tcg_debug_assert(TCG_TARGET_HAS_direct_jump); |
72 | + * Helper for tlb_flush_page_by_mmuidx and friends, called through | 138 | - s->tb_jmp_insn_offset[which] = tcg_current_code_size(s); |
73 | + * async_run_on_cpu. The idxmap parameter is encoded in the page | 139 | + s->gen_tb->jmp_target_arg[which] = tcg_current_code_size(s); |
74 | + * offset of the target_ptr field. This limits the set of mmu_idx | ||
75 | + * that can be passed via this method. | ||
76 | + */ | ||
77 | +static void tlb_flush_page_by_mmuidx_async_1(CPUState *cpu, | ||
78 | + run_on_cpu_data data) | ||
79 | +{ | ||
80 | + target_ulong addr_and_idxmap = (target_ulong) data.target_ptr; | ||
81 | + target_ulong addr = addr_and_idxmap & TARGET_PAGE_MASK; | ||
82 | + uint16_t idxmap = addr_and_idxmap & ~TARGET_PAGE_MASK; | ||
83 | + | ||
84 | + tlb_flush_page_by_mmuidx_async_0(cpu, addr, idxmap); | ||
85 | +} | ||
86 | + | ||
87 | +typedef struct { | ||
88 | + target_ulong addr; | ||
89 | + uint16_t idxmap; | ||
90 | +} TLBFlushPageByMMUIdxData; | ||
91 | + | ||
92 | +/** | ||
93 | + * tlb_flush_page_by_mmuidx_async_2: | ||
94 | + * @cpu: cpu on which to flush | ||
95 | + * @data: allocated addr + idxmap | ||
96 | + * | ||
97 | + * Helper for tlb_flush_page_by_mmuidx and friends, called through | ||
98 | + * async_run_on_cpu. The addr+idxmap parameters are stored in a | ||
99 | + * TLBFlushPageByMMUIdxData structure that has been allocated | ||
100 | + * specifically for this helper. Free the structure when done. | ||
101 | + */ | ||
102 | +static void tlb_flush_page_by_mmuidx_async_2(CPUState *cpu, | ||
103 | + run_on_cpu_data data) | ||
104 | +{ | ||
105 | + TLBFlushPageByMMUIdxData *d = data.host_ptr; | ||
106 | + | ||
107 | + tlb_flush_page_by_mmuidx_async_0(cpu, d->addr, d->idxmap); | ||
108 | + g_free(d); | ||
109 | +} | ||
110 | + | ||
111 | void tlb_flush_page_by_mmuidx(CPUState *cpu, target_ulong addr, uint16_t idxmap) | ||
112 | { | ||
113 | - target_ulong addr_and_mmu_idx; | ||
114 | - | ||
115 | tlb_debug("addr: "TARGET_FMT_lx" mmu_idx:%" PRIx16 "\n", addr, idxmap); | ||
116 | |||
117 | /* This should already be page aligned */ | ||
118 | - addr_and_mmu_idx = addr & TARGET_PAGE_MASK; | ||
119 | - addr_and_mmu_idx |= idxmap; | ||
120 | + addr &= TARGET_PAGE_MASK; | ||
121 | |||
122 | - if (!qemu_cpu_is_self(cpu)) { | ||
123 | - async_run_on_cpu(cpu, tlb_flush_page_by_mmuidx_async_work, | ||
124 | - RUN_ON_CPU_TARGET_PTR(addr_and_mmu_idx)); | ||
125 | + if (qemu_cpu_is_self(cpu)) { | ||
126 | + tlb_flush_page_by_mmuidx_async_0(cpu, addr, idxmap); | ||
127 | + } else if (idxmap < TARGET_PAGE_SIZE) { | ||
128 | + /* | ||
129 | + * Most targets have only a few mmu_idx. In the case where | ||
130 | + * we can stuff idxmap into the low TARGET_PAGE_BITS, avoid | ||
131 | + * allocating memory for this operation. | ||
132 | + */ | ||
133 | + async_run_on_cpu(cpu, tlb_flush_page_by_mmuidx_async_1, | ||
134 | + RUN_ON_CPU_TARGET_PTR(addr | idxmap)); | ||
135 | } else { | ||
136 | - tlb_flush_page_by_mmuidx_async_work( | ||
137 | - cpu, RUN_ON_CPU_TARGET_PTR(addr_and_mmu_idx)); | ||
138 | + TLBFlushPageByMMUIdxData *d = g_new(TLBFlushPageByMMUIdxData, 1); | ||
139 | + | ||
140 | + /* Otherwise allocate a structure, freed by the worker. */ | ||
141 | + d->addr = addr; | ||
142 | + d->idxmap = idxmap; | ||
143 | + async_run_on_cpu(cpu, tlb_flush_page_by_mmuidx_async_2, | ||
144 | + RUN_ON_CPU_HOST_PTR(d)); | ||
145 | } | ||
146 | } | 140 | } |
147 | 141 | ||
148 | @@ -XXX,XX +XXX,XX @@ void tlb_flush_page(CPUState *cpu, target_ulong addr) | 142 | static uintptr_t G_GNUC_UNUSED get_jmp_target_addr(TCGContext *s, int which) |
149 | void tlb_flush_page_by_mmuidx_all_cpus(CPUState *src_cpu, target_ulong addr, | 143 | @@ -XXX,XX +XXX,XX @@ static uintptr_t G_GNUC_UNUSED get_jmp_target_addr(TCGContext *s, int which) |
150 | uint16_t idxmap) | 144 | * Return the read-execute version of the pointer, for the benefit |
151 | { | 145 | * of any pc-relative addressing mode. |
152 | - const run_on_cpu_func fn = tlb_flush_page_by_mmuidx_async_work; | 146 | */ |
153 | - target_ulong addr_and_mmu_idx; | 147 | - return (uintptr_t)tcg_splitwx_to_rx(&s->tb_jmp_target_addr[which]); |
154 | - | 148 | + return (uintptr_t)tcg_splitwx_to_rx(s->gen_tb->jmp_target_arg + which); |
155 | tlb_debug("addr: "TARGET_FMT_lx" mmu_idx:%"PRIx16"\n", addr, idxmap); | ||
156 | |||
157 | /* This should already be page aligned */ | ||
158 | - addr_and_mmu_idx = addr & TARGET_PAGE_MASK; | ||
159 | - addr_and_mmu_idx |= idxmap; | ||
160 | + addr &= TARGET_PAGE_MASK; | ||
161 | |||
162 | - flush_all_helper(src_cpu, fn, RUN_ON_CPU_TARGET_PTR(addr_and_mmu_idx)); | ||
163 | - fn(src_cpu, RUN_ON_CPU_TARGET_PTR(addr_and_mmu_idx)); | ||
164 | + /* | ||
165 | + * Allocate memory to hold addr+idxmap only when needed. | ||
166 | + * See tlb_flush_page_by_mmuidx for details. | ||
167 | + */ | ||
168 | + if (idxmap < TARGET_PAGE_SIZE) { | ||
169 | + flush_all_helper(src_cpu, tlb_flush_page_by_mmuidx_async_1, | ||
170 | + RUN_ON_CPU_TARGET_PTR(addr | idxmap)); | ||
171 | + } else { | ||
172 | + CPUState *dst_cpu; | ||
173 | + | ||
174 | + /* Allocate a separate data block for each destination cpu. */ | ||
175 | + CPU_FOREACH(dst_cpu) { | ||
176 | + if (dst_cpu != src_cpu) { | ||
177 | + TLBFlushPageByMMUIdxData *d | ||
178 | + = g_new(TLBFlushPageByMMUIdxData, 1); | ||
179 | + | ||
180 | + d->addr = addr; | ||
181 | + d->idxmap = idxmap; | ||
182 | + async_run_on_cpu(dst_cpu, tlb_flush_page_by_mmuidx_async_2, | ||
183 | + RUN_ON_CPU_HOST_PTR(d)); | ||
184 | + } | ||
185 | + } | ||
186 | + } | ||
187 | + | ||
188 | + tlb_flush_page_by_mmuidx_async_0(src_cpu, addr, idxmap); | ||
189 | } | 149 | } |
190 | 150 | ||
191 | void tlb_flush_page_all_cpus(CPUState *src, target_ulong addr) | 151 | /* Signal overflow, starting over with fewer guest insns. */ |
192 | @@ -XXX,XX +XXX,XX @@ void tlb_flush_page_by_mmuidx_all_cpus_synced(CPUState *src_cpu, | 152 | @@ -XXX,XX +XXX,XX @@ int tcg_gen_code(TCGContext *s, TranslationBlock *tb, target_ulong pc_start) |
193 | target_ulong addr, | 153 | /* Initialize goto_tb jump offsets. */ |
194 | uint16_t idxmap) | 154 | tb->jmp_reset_offset[0] = TB_JMP_OFFSET_INVALID; |
195 | { | 155 | tb->jmp_reset_offset[1] = TB_JMP_OFFSET_INVALID; |
196 | - const run_on_cpu_func fn = tlb_flush_page_by_mmuidx_async_work; | 156 | - tcg_ctx->tb_jmp_reset_offset = tb->jmp_reset_offset; |
197 | - target_ulong addr_and_mmu_idx; | 157 | - if (TCG_TARGET_HAS_direct_jump) { |
198 | - | 158 | - tcg_ctx->tb_jmp_insn_offset = tb->jmp_target_arg; |
199 | tlb_debug("addr: "TARGET_FMT_lx" mmu_idx:%"PRIx16"\n", addr, idxmap); | 159 | - tcg_ctx->tb_jmp_target_addr = NULL; |
200 | 160 | - } else { | |
201 | /* This should already be page aligned */ | 161 | - tcg_ctx->tb_jmp_insn_offset = NULL; |
202 | - addr_and_mmu_idx = addr & TARGET_PAGE_MASK; | 162 | - tcg_ctx->tb_jmp_target_addr = tb->jmp_target_arg; |
203 | - addr_and_mmu_idx |= idxmap; | 163 | - } |
204 | + addr &= TARGET_PAGE_MASK; | 164 | |
205 | 165 | tcg_reg_alloc_start(s); | |
206 | - flush_all_helper(src_cpu, fn, RUN_ON_CPU_TARGET_PTR(addr_and_mmu_idx)); | 166 | |
207 | - async_safe_run_on_cpu(src_cpu, fn, RUN_ON_CPU_TARGET_PTR(addr_and_mmu_idx)); | ||
208 | + /* | ||
209 | + * Allocate memory to hold addr+idxmap only when needed. | ||
210 | + * See tlb_flush_page_by_mmuidx for details. | ||
211 | + */ | ||
212 | + if (idxmap < TARGET_PAGE_SIZE) { | ||
213 | + flush_all_helper(src_cpu, tlb_flush_page_by_mmuidx_async_1, | ||
214 | + RUN_ON_CPU_TARGET_PTR(addr | idxmap)); | ||
215 | + async_safe_run_on_cpu(src_cpu, tlb_flush_page_by_mmuidx_async_1, | ||
216 | + RUN_ON_CPU_TARGET_PTR(addr | idxmap)); | ||
217 | + } else { | ||
218 | + CPUState *dst_cpu; | ||
219 | + TLBFlushPageByMMUIdxData *d; | ||
220 | + | ||
221 | + /* Allocate a separate data block for each destination cpu. */ | ||
222 | + CPU_FOREACH(dst_cpu) { | ||
223 | + if (dst_cpu != src_cpu) { | ||
224 | + d = g_new(TLBFlushPageByMMUIdxData, 1); | ||
225 | + d->addr = addr; | ||
226 | + d->idxmap = idxmap; | ||
227 | + async_run_on_cpu(dst_cpu, tlb_flush_page_by_mmuidx_async_2, | ||
228 | + RUN_ON_CPU_HOST_PTR(d)); | ||
229 | + } | ||
230 | + } | ||
231 | + | ||
232 | + d = g_new(TLBFlushPageByMMUIdxData, 1); | ||
233 | + d->addr = addr; | ||
234 | + d->idxmap = idxmap; | ||
235 | + async_safe_run_on_cpu(src_cpu, tlb_flush_page_by_mmuidx_async_2, | ||
236 | + RUN_ON_CPU_HOST_PTR(d)); | ||
237 | + } | ||
238 | } | ||
239 | |||
240 | void tlb_flush_page_all_cpus_synced(CPUState *src, target_ulong addr) | ||
241 | -- | 167 | -- |
242 | 2.20.1 | 168 | 2.34.1 |
243 | 169 | ||
244 | 170 | diff view generated by jsdifflib |
1 | There is only one caller for tlb_table_flush_by_mmuidx. Place | 1 | Stop overloading jmp_target_arg for both offset and address, |
---|---|---|---|
2 | the result at the earlier line number, due to an expected user | 2 | depending on TCG_TARGET_HAS_direct_jump. Instead, add a new |
3 | in the near future. | 3 | field to hold the jump insn offset and always set the target |
4 | address in jmp_target_addr[]. This will allow a tcg backend | ||
5 | to use either direct or indirect depending on displacement. | ||
4 | 6 | ||
5 | Reviewed-by: Alex Bennée <alex.bennee@linaro.org> | 7 | Reviewed-by: Alex Bennée <alex.bennee@linaro.org> |
6 | Reviewed-by: Alistair Francis <alistair.francis@wdc.com> | ||
7 | Signed-off-by: Richard Henderson <richard.henderson@linaro.org> | 8 | Signed-off-by: Richard Henderson <richard.henderson@linaro.org> |
8 | --- | 9 | --- |
9 | accel/tcg/cputlb.c | 19 +++++++------------ | 10 | include/exec/exec-all.h | 3 ++- |
10 | 1 file changed, 7 insertions(+), 12 deletions(-) | 11 | accel/tcg/cpu-exec.c | 5 ++--- |
12 | tcg/tcg.c | 6 ++++-- | ||
13 | 3 files changed, 8 insertions(+), 6 deletions(-) | ||
11 | 14 | ||
12 | diff --git a/accel/tcg/cputlb.c b/accel/tcg/cputlb.c | 15 | diff --git a/include/exec/exec-all.h b/include/exec/exec-all.h |
13 | index XXXXXXX..XXXXXXX 100644 | 16 | index XXXXXXX..XXXXXXX 100644 |
14 | --- a/accel/tcg/cputlb.c | 17 | --- a/include/exec/exec-all.h |
15 | +++ b/accel/tcg/cputlb.c | 18 | +++ b/include/exec/exec-all.h |
16 | @@ -XXX,XX +XXX,XX @@ static void tlb_mmu_resize_locked(CPUArchState *env, int mmu_idx) | 19 | @@ -XXX,XX +XXX,XX @@ struct TranslationBlock { |
20 | */ | ||
21 | #define TB_JMP_OFFSET_INVALID 0xffff /* indicates no jump generated */ | ||
22 | uint16_t jmp_reset_offset[2]; /* offset of original jump target */ | ||
23 | - uintptr_t jmp_target_arg[2]; /* target address or offset */ | ||
24 | + uint16_t jmp_insn_offset[2]; /* offset of direct jump insn */ | ||
25 | + uintptr_t jmp_target_addr[2]; /* target address */ | ||
26 | |||
27 | /* | ||
28 | * Each TB has a NULL-terminated list (jmp_list_head) of incoming jumps. | ||
29 | diff --git a/accel/tcg/cpu-exec.c b/accel/tcg/cpu-exec.c | ||
30 | index XXXXXXX..XXXXXXX 100644 | ||
31 | --- a/accel/tcg/cpu-exec.c | ||
32 | +++ b/accel/tcg/cpu-exec.c | ||
33 | @@ -XXX,XX +XXX,XX @@ void cpu_exec_step_atomic(CPUState *cpu) | ||
34 | |||
35 | void tb_set_jmp_target(TranslationBlock *tb, int n, uintptr_t addr) | ||
36 | { | ||
37 | + tb->jmp_target_addr[n] = addr; | ||
38 | if (TCG_TARGET_HAS_direct_jump) { | ||
39 | - uintptr_t offset = tb->jmp_target_arg[n]; | ||
40 | + uintptr_t offset = tb->jmp_insn_offset[n]; | ||
41 | uintptr_t tc_ptr = (uintptr_t)tb->tc.ptr; | ||
42 | uintptr_t jmp_rx = tc_ptr + offset; | ||
43 | uintptr_t jmp_rw = jmp_rx - tcg_splitwx_diff; | ||
44 | tb_target_set_jmp_target(tc_ptr, jmp_rx, jmp_rw, addr); | ||
45 | - } else { | ||
46 | - tb->jmp_target_arg[n] = addr; | ||
17 | } | 47 | } |
18 | } | 48 | } |
19 | 49 | ||
20 | -static inline void tlb_table_flush_by_mmuidx(CPUArchState *env, int mmu_idx) | 50 | diff --git a/tcg/tcg.c b/tcg/tcg.c |
21 | +static void tlb_flush_one_mmuidx_locked(CPUArchState *env, int mmu_idx) | 51 | index XXXXXXX..XXXXXXX 100644 |
22 | { | 52 | --- a/tcg/tcg.c |
23 | tlb_mmu_resize_locked(env, mmu_idx); | 53 | +++ b/tcg/tcg.c |
24 | - memset(env_tlb(env)->f[mmu_idx].table, -1, sizeof_tlb(env, mmu_idx)); | 54 | @@ -XXX,XX +XXX,XX @@ static void G_GNUC_UNUSED set_jmp_insn_offset(TCGContext *s, int which) |
25 | env_tlb(env)->d[mmu_idx].n_used_entries = 0; | 55 | * tcg_gen_code, where we bound tcg_current_code_size to UINT16_MAX. |
26 | + env_tlb(env)->d[mmu_idx].large_page_addr = -1; | 56 | */ |
27 | + env_tlb(env)->d[mmu_idx].large_page_mask = -1; | 57 | tcg_debug_assert(TCG_TARGET_HAS_direct_jump); |
28 | + env_tlb(env)->d[mmu_idx].vindex = 0; | 58 | - s->gen_tb->jmp_target_arg[which] = tcg_current_code_size(s); |
29 | + memset(env_tlb(env)->f[mmu_idx].table, -1, sizeof_tlb(env, mmu_idx)); | 59 | + s->gen_tb->jmp_insn_offset[which] = tcg_current_code_size(s); |
30 | + memset(env_tlb(env)->d[mmu_idx].vtable, -1, | ||
31 | + sizeof(env_tlb(env)->d[0].vtable)); | ||
32 | } | 60 | } |
33 | 61 | ||
34 | static inline void tlb_n_used_entries_inc(CPUArchState *env, uintptr_t mmu_idx) | 62 | static uintptr_t G_GNUC_UNUSED get_jmp_target_addr(TCGContext *s, int which) |
35 | @@ -XXX,XX +XXX,XX @@ void tlb_flush_counts(size_t *pfull, size_t *ppart, size_t *pelide) | 63 | @@ -XXX,XX +XXX,XX @@ static uintptr_t G_GNUC_UNUSED get_jmp_target_addr(TCGContext *s, int which) |
36 | *pelide = elide; | 64 | * Return the read-execute version of the pointer, for the benefit |
65 | * of any pc-relative addressing mode. | ||
66 | */ | ||
67 | - return (uintptr_t)tcg_splitwx_to_rx(s->gen_tb->jmp_target_arg + which); | ||
68 | + return (uintptr_t)tcg_splitwx_to_rx(&s->gen_tb->jmp_target_addr[which]); | ||
37 | } | 69 | } |
38 | 70 | ||
39 | -static void tlb_flush_one_mmuidx_locked(CPUArchState *env, int mmu_idx) | 71 | /* Signal overflow, starting over with fewer guest insns. */ |
40 | -{ | 72 | @@ -XXX,XX +XXX,XX @@ int tcg_gen_code(TCGContext *s, TranslationBlock *tb, target_ulong pc_start) |
41 | - tlb_table_flush_by_mmuidx(env, mmu_idx); | 73 | /* Initialize goto_tb jump offsets. */ |
42 | - env_tlb(env)->d[mmu_idx].large_page_addr = -1; | 74 | tb->jmp_reset_offset[0] = TB_JMP_OFFSET_INVALID; |
43 | - env_tlb(env)->d[mmu_idx].large_page_mask = -1; | 75 | tb->jmp_reset_offset[1] = TB_JMP_OFFSET_INVALID; |
44 | - env_tlb(env)->d[mmu_idx].vindex = 0; | 76 | + tb->jmp_insn_offset[0] = TB_JMP_OFFSET_INVALID; |
45 | - memset(env_tlb(env)->d[mmu_idx].vtable, -1, | 77 | + tb->jmp_insn_offset[1] = TB_JMP_OFFSET_INVALID; |
46 | - sizeof(env_tlb(env)->d[0].vtable)); | 78 | |
47 | -} | 79 | tcg_reg_alloc_start(s); |
48 | - | 80 | |
49 | static void tlb_flush_by_mmuidx_async_work(CPUState *cpu, run_on_cpu_data data) | ||
50 | { | ||
51 | CPUArchState *env = cpu->env_ptr; | ||
52 | -- | 81 | -- |
53 | 2.20.1 | 82 | 2.34.1 |
54 | 83 | ||
55 | 84 | diff view generated by jsdifflib |
1 | We will want to be able to flush a tlb without resizing. | 1 | Replace 'tc_ptr' and 'addr' with 'tb' and 'n'. |
---|---|---|---|
2 | 2 | ||
3 | Reviewed-by: Alex Bennée <alex.bennee@linaro.org> | 3 | Reviewed-by: Alex Bennée <alex.bennee@linaro.org> |
4 | Reviewed-by: Alistair Francis <alistair.francis@wdc.com> | ||
5 | Reviewed-by: Philippe Mathieu-Daudé <philmd@redhat.com> | ||
6 | Signed-off-by: Richard Henderson <richard.henderson@linaro.org> | 4 | Signed-off-by: Richard Henderson <richard.henderson@linaro.org> |
7 | --- | 5 | --- |
8 | accel/tcg/cputlb.c | 15 ++++++++++----- | 6 | tcg/aarch64/tcg-target.h | 3 ++- |
9 | 1 file changed, 10 insertions(+), 5 deletions(-) | 7 | tcg/arm/tcg-target.h | 3 ++- |
8 | tcg/i386/tcg-target.h | 9 ++------- | ||
9 | tcg/loongarch64/tcg-target.h | 3 ++- | ||
10 | tcg/mips/tcg-target.h | 3 ++- | ||
11 | tcg/ppc/tcg-target.h | 3 ++- | ||
12 | tcg/riscv/tcg-target.h | 3 ++- | ||
13 | tcg/s390x/tcg-target.h | 10 ++-------- | ||
14 | tcg/sparc64/tcg-target.h | 3 ++- | ||
15 | tcg/tci/tcg-target.h | 3 ++- | ||
16 | accel/tcg/cpu-exec.c | 11 ++++++++--- | ||
17 | tcg/aarch64/tcg-target.c.inc | 5 +++-- | ||
18 | tcg/i386/tcg-target.c.inc | 9 +++++++++ | ||
19 | tcg/loongarch64/tcg-target.c.inc | 5 +++-- | ||
20 | tcg/ppc/tcg-target.c.inc | 7 ++++--- | ||
21 | tcg/s390x/tcg-target.c.inc | 10 ++++++++++ | ||
22 | tcg/sparc64/tcg-target.c.inc | 7 ++++--- | ||
23 | 17 files changed, 61 insertions(+), 36 deletions(-) | ||
10 | 24 | ||
11 | diff --git a/accel/tcg/cputlb.c b/accel/tcg/cputlb.c | 25 | diff --git a/tcg/aarch64/tcg-target.h b/tcg/aarch64/tcg-target.h |
12 | index XXXXXXX..XXXXXXX 100644 | 26 | index XXXXXXX..XXXXXXX 100644 |
13 | --- a/accel/tcg/cputlb.c | 27 | --- a/tcg/aarch64/tcg-target.h |
14 | +++ b/accel/tcg/cputlb.c | 28 | +++ b/tcg/aarch64/tcg-target.h |
15 | @@ -XXX,XX +XXX,XX @@ static void tlb_mmu_resize_locked(CPUTLBDesc *desc, CPUTLBDescFast *fast) | 29 | @@ -XXX,XX +XXX,XX @@ typedef enum { |
30 | #define TCG_TARGET_DEFAULT_MO (0) | ||
31 | #define TCG_TARGET_HAS_MEMORY_BSWAP 0 | ||
32 | |||
33 | -void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t, uintptr_t); | ||
34 | +void tb_target_set_jmp_target(const TranslationBlock *, int, | ||
35 | + uintptr_t, uintptr_t); | ||
36 | |||
37 | #define TCG_TARGET_NEED_LDST_LABELS | ||
38 | #define TCG_TARGET_NEED_POOL_LABELS | ||
39 | diff --git a/tcg/arm/tcg-target.h b/tcg/arm/tcg-target.h | ||
40 | index XXXXXXX..XXXXXXX 100644 | ||
41 | --- a/tcg/arm/tcg-target.h | ||
42 | +++ b/tcg/arm/tcg-target.h | ||
43 | @@ -XXX,XX +XXX,XX @@ extern bool use_neon_instructions; | ||
44 | #define TCG_TARGET_HAS_MEMORY_BSWAP 0 | ||
45 | |||
46 | /* not defined -- call should be eliminated at compile time */ | ||
47 | -void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t, uintptr_t); | ||
48 | +void tb_target_set_jmp_target(const TranslationBlock *tb, int n, | ||
49 | + uintptr_t, uintptr_t); | ||
50 | |||
51 | #define TCG_TARGET_NEED_LDST_LABELS | ||
52 | #define TCG_TARGET_NEED_POOL_LABELS | ||
53 | diff --git a/tcg/i386/tcg-target.h b/tcg/i386/tcg-target.h | ||
54 | index XXXXXXX..XXXXXXX 100644 | ||
55 | --- a/tcg/i386/tcg-target.h | ||
56 | +++ b/tcg/i386/tcg-target.h | ||
57 | @@ -XXX,XX +XXX,XX @@ extern bool have_movbe; | ||
58 | #define TCG_TARGET_extract_i64_valid(ofs, len) \ | ||
59 | (((ofs) == 8 && (len) == 8) || ((ofs) + (len)) == 32) | ||
60 | |||
61 | -static inline void tb_target_set_jmp_target(uintptr_t tc_ptr, uintptr_t jmp_rx, | ||
62 | - uintptr_t jmp_rw, uintptr_t addr) | ||
63 | -{ | ||
64 | - /* patch the branch destination */ | ||
65 | - qatomic_set((int32_t *)jmp_rw, addr - (jmp_rx + 4)); | ||
66 | - /* no need to flush icache explicitly */ | ||
67 | -} | ||
68 | +void tb_target_set_jmp_target(const TranslationBlock *, int, | ||
69 | + uintptr_t, uintptr_t); | ||
70 | |||
71 | /* This defines the natural memory order supported by this | ||
72 | * architecture before guarantees made by various barrier | ||
73 | diff --git a/tcg/loongarch64/tcg-target.h b/tcg/loongarch64/tcg-target.h | ||
74 | index XXXXXXX..XXXXXXX 100644 | ||
75 | --- a/tcg/loongarch64/tcg-target.h | ||
76 | +++ b/tcg/loongarch64/tcg-target.h | ||
77 | @@ -XXX,XX +XXX,XX @@ typedef enum { | ||
78 | #define TCG_TARGET_HAS_muluh_i64 1 | ||
79 | #define TCG_TARGET_HAS_mulsh_i64 1 | ||
80 | |||
81 | -void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t, uintptr_t); | ||
82 | +void tb_target_set_jmp_target(const TranslationBlock *tb, int n, | ||
83 | + uintptr_t, uintptr_t); | ||
84 | |||
85 | #define TCG_TARGET_DEFAULT_MO (0) | ||
86 | |||
87 | diff --git a/tcg/mips/tcg-target.h b/tcg/mips/tcg-target.h | ||
88 | index XXXXXXX..XXXXXXX 100644 | ||
89 | --- a/tcg/mips/tcg-target.h | ||
90 | +++ b/tcg/mips/tcg-target.h | ||
91 | @@ -XXX,XX +XXX,XX @@ extern bool use_mips32r2_instructions; | ||
92 | #define TCG_TARGET_HAS_MEMORY_BSWAP 1 | ||
93 | |||
94 | /* not defined -- call should be eliminated at compile time */ | ||
95 | -void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t, uintptr_t) | ||
96 | +void tb_target_set_jmp_target(const TranslationBlock *tb, int n, | ||
97 | + uintptr_t, uintptr_t) | ||
98 | QEMU_ERROR("code path is reachable"); | ||
99 | |||
100 | #define TCG_TARGET_NEED_LDST_LABELS | ||
101 | diff --git a/tcg/ppc/tcg-target.h b/tcg/ppc/tcg-target.h | ||
102 | index XXXXXXX..XXXXXXX 100644 | ||
103 | --- a/tcg/ppc/tcg-target.h | ||
104 | +++ b/tcg/ppc/tcg-target.h | ||
105 | @@ -XXX,XX +XXX,XX @@ extern bool have_vsx; | ||
106 | #define TCG_TARGET_HAS_bitsel_vec have_vsx | ||
107 | #define TCG_TARGET_HAS_cmpsel_vec 0 | ||
108 | |||
109 | -void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t, uintptr_t); | ||
110 | +void tb_target_set_jmp_target(const TranslationBlock *tb, int n, | ||
111 | + uintptr_t, uintptr_t); | ||
112 | |||
113 | #define TCG_TARGET_DEFAULT_MO (0) | ||
114 | #define TCG_TARGET_HAS_MEMORY_BSWAP 1 | ||
115 | diff --git a/tcg/riscv/tcg-target.h b/tcg/riscv/tcg-target.h | ||
116 | index XXXXXXX..XXXXXXX 100644 | ||
117 | --- a/tcg/riscv/tcg-target.h | ||
118 | +++ b/tcg/riscv/tcg-target.h | ||
119 | @@ -XXX,XX +XXX,XX @@ typedef enum { | ||
120 | #endif | ||
121 | |||
122 | /* not defined -- call should be eliminated at compile time */ | ||
123 | -void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t, uintptr_t); | ||
124 | +void tb_target_set_jmp_target(const TranslationBlock *tb, int n, | ||
125 | + uintptr_t, uintptr_t); | ||
126 | |||
127 | #define TCG_TARGET_DEFAULT_MO (0) | ||
128 | |||
129 | diff --git a/tcg/s390x/tcg-target.h b/tcg/s390x/tcg-target.h | ||
130 | index XXXXXXX..XXXXXXX 100644 | ||
131 | --- a/tcg/s390x/tcg-target.h | ||
132 | +++ b/tcg/s390x/tcg-target.h | ||
133 | @@ -XXX,XX +XXX,XX @@ extern uint64_t s390_facilities[3]; | ||
134 | |||
135 | #define TCG_TARGET_DEFAULT_MO (TCG_MO_ALL & ~TCG_MO_ST_LD) | ||
136 | |||
137 | -static inline void tb_target_set_jmp_target(uintptr_t tc_ptr, uintptr_t jmp_rx, | ||
138 | - uintptr_t jmp_rw, uintptr_t addr) | ||
139 | -{ | ||
140 | - /* patch the branch destination */ | ||
141 | - intptr_t disp = addr - (jmp_rx - 2); | ||
142 | - qatomic_set((int32_t *)jmp_rw, disp / 2); | ||
143 | - /* no need to flush icache explicitly */ | ||
144 | -} | ||
145 | +void tb_target_set_jmp_target(const TranslationBlock *tb, int n, | ||
146 | + uintptr_t jmp_rx, uintptr_t jmp_rw); | ||
147 | |||
148 | #define TCG_TARGET_NEED_LDST_LABELS | ||
149 | #define TCG_TARGET_NEED_POOL_LABELS | ||
150 | diff --git a/tcg/sparc64/tcg-target.h b/tcg/sparc64/tcg-target.h | ||
151 | index XXXXXXX..XXXXXXX 100644 | ||
152 | --- a/tcg/sparc64/tcg-target.h | ||
153 | +++ b/tcg/sparc64/tcg-target.h | ||
154 | @@ -XXX,XX +XXX,XX @@ extern bool use_vis3_instructions; | ||
155 | #define TCG_TARGET_DEFAULT_MO (0) | ||
156 | #define TCG_TARGET_HAS_MEMORY_BSWAP 1 | ||
157 | |||
158 | -void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t, uintptr_t); | ||
159 | +void tb_target_set_jmp_target(const TranslationBlock *tb, int n, | ||
160 | + uintptr_t, uintptr_t); | ||
161 | |||
162 | #define TCG_TARGET_NEED_POOL_LABELS | ||
163 | |||
164 | diff --git a/tcg/tci/tcg-target.h b/tcg/tci/tcg-target.h | ||
165 | index XXXXXXX..XXXXXXX 100644 | ||
166 | --- a/tcg/tci/tcg-target.h | ||
167 | +++ b/tcg/tci/tcg-target.h | ||
168 | @@ -XXX,XX +XXX,XX @@ typedef enum { | ||
169 | #define TCG_TARGET_HAS_MEMORY_BSWAP 1 | ||
170 | |||
171 | /* not defined -- call should be eliminated at compile time */ | ||
172 | -void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t, uintptr_t); | ||
173 | +void tb_target_set_jmp_target(const TranslationBlock *tb, int n, | ||
174 | + uintptr_t, uintptr_t); | ||
175 | |||
176 | #endif /* TCG_TARGET_H */ | ||
177 | diff --git a/accel/tcg/cpu-exec.c b/accel/tcg/cpu-exec.c | ||
178 | index XXXXXXX..XXXXXXX 100644 | ||
179 | --- a/accel/tcg/cpu-exec.c | ||
180 | +++ b/accel/tcg/cpu-exec.c | ||
181 | @@ -XXX,XX +XXX,XX @@ void tb_set_jmp_target(TranslationBlock *tb, int n, uintptr_t addr) | ||
182 | { | ||
183 | tb->jmp_target_addr[n] = addr; | ||
184 | if (TCG_TARGET_HAS_direct_jump) { | ||
185 | + /* | ||
186 | + * Get the rx view of the structure, from which we find the | ||
187 | + * executable code address, and tb_target_set_jmp_target can | ||
188 | + * produce a pc-relative displacement to jmp_target_addr[n]. | ||
189 | + */ | ||
190 | + const TranslationBlock *c_tb = tcg_splitwx_to_rx(tb); | ||
191 | uintptr_t offset = tb->jmp_insn_offset[n]; | ||
192 | - uintptr_t tc_ptr = (uintptr_t)tb->tc.ptr; | ||
193 | - uintptr_t jmp_rx = tc_ptr + offset; | ||
194 | + uintptr_t jmp_rx = (uintptr_t)tb->tc.ptr + offset; | ||
195 | uintptr_t jmp_rw = jmp_rx - tcg_splitwx_diff; | ||
196 | - tb_target_set_jmp_target(tc_ptr, jmp_rx, jmp_rw, addr); | ||
197 | + tb_target_set_jmp_target(c_tb, n, jmp_rx, jmp_rw); | ||
16 | } | 198 | } |
17 | } | 199 | } |
18 | 200 | ||
19 | -static void tlb_flush_one_mmuidx_locked(CPUArchState *env, int mmu_idx) | 201 | diff --git a/tcg/aarch64/tcg-target.c.inc b/tcg/aarch64/tcg-target.c.inc |
20 | +static void tlb_mmu_flush_locked(CPUTLBDesc *desc, CPUTLBDescFast *fast) | 202 | index XXXXXXX..XXXXXXX 100644 |
21 | { | 203 | --- a/tcg/aarch64/tcg-target.c.inc |
22 | - CPUTLBDesc *desc = &env_tlb(env)->d[mmu_idx]; | 204 | +++ b/tcg/aarch64/tcg-target.c.inc |
23 | - CPUTLBDescFast *fast = &env_tlb(env)->f[mmu_idx]; | 205 | @@ -XXX,XX +XXX,XX @@ static void tcg_out_call(TCGContext *s, const tcg_insn_unit *target, |
24 | - | 206 | tcg_out_call_int(s, target); |
25 | - tlb_mmu_resize_locked(desc, fast); | 207 | } |
26 | desc->n_used_entries = 0; | 208 | |
27 | desc->large_page_addr = -1; | 209 | -void tb_target_set_jmp_target(uintptr_t tc_ptr, uintptr_t jmp_rx, |
28 | desc->large_page_mask = -1; | 210 | - uintptr_t jmp_rw, uintptr_t addr) |
29 | @@ -XXX,XX +XXX,XX @@ static void tlb_flush_one_mmuidx_locked(CPUArchState *env, int mmu_idx) | 211 | +void tb_target_set_jmp_target(const TranslationBlock *tb, int n, |
30 | memset(desc->vtable, -1, sizeof(desc->vtable)); | 212 | + uintptr_t jmp_rx, uintptr_t jmp_rw) |
31 | } | 213 | { |
32 | 214 | + uintptr_t addr = tb->jmp_target_addr[n]; | |
33 | +static void tlb_flush_one_mmuidx_locked(CPUArchState *env, int mmu_idx) | 215 | tcg_insn_unit i1, i2; |
216 | TCGType rt = TCG_TYPE_I64; | ||
217 | TCGReg rd = TCG_REG_TMP; | ||
218 | diff --git a/tcg/i386/tcg-target.c.inc b/tcg/i386/tcg-target.c.inc | ||
219 | index XXXXXXX..XXXXXXX 100644 | ||
220 | --- a/tcg/i386/tcg-target.c.inc | ||
221 | +++ b/tcg/i386/tcg-target.c.inc | ||
222 | @@ -XXX,XX +XXX,XX @@ static void tcg_out_goto_tb(TCGContext *s, int which) | ||
223 | set_jmp_reset_offset(s, which); | ||
224 | } | ||
225 | |||
226 | +void tb_target_set_jmp_target(const TranslationBlock *tb, int n, | ||
227 | + uintptr_t jmp_rx, uintptr_t jmp_rw) | ||
34 | +{ | 228 | +{ |
35 | + CPUTLBDesc *desc = &env_tlb(env)->d[mmu_idx]; | 229 | + /* patch the branch destination */ |
36 | + CPUTLBDescFast *fast = &env_tlb(env)->f[mmu_idx]; | 230 | + uintptr_t addr = tb->jmp_target_addr[n]; |
37 | + | 231 | + qatomic_set((int32_t *)jmp_rw, addr - (jmp_rx + 4)); |
38 | + tlb_mmu_resize_locked(desc, fast); | 232 | + /* no need to flush icache explicitly */ |
39 | + tlb_mmu_flush_locked(desc, fast); | ||
40 | +} | 233 | +} |
41 | + | 234 | + |
42 | static inline void tlb_n_used_entries_inc(CPUArchState *env, uintptr_t mmu_idx) | 235 | static inline void tcg_out_op(TCGContext *s, TCGOpcode opc, |
43 | { | 236 | const TCGArg args[TCG_MAX_OP_ARGS], |
44 | env_tlb(env)->d[mmu_idx].n_used_entries++; | 237 | const int const_args[TCG_MAX_OP_ARGS]) |
238 | diff --git a/tcg/loongarch64/tcg-target.c.inc b/tcg/loongarch64/tcg-target.c.inc | ||
239 | index XXXXXXX..XXXXXXX 100644 | ||
240 | --- a/tcg/loongarch64/tcg-target.c.inc | ||
241 | +++ b/tcg/loongarch64/tcg-target.c.inc | ||
242 | @@ -XXX,XX +XXX,XX @@ static void tcg_out_nop(TCGContext *s) | ||
243 | tcg_out32(s, NOP); | ||
244 | } | ||
245 | |||
246 | -void tb_target_set_jmp_target(uintptr_t tc_ptr, uintptr_t jmp_rx, | ||
247 | - uintptr_t jmp_rw, uintptr_t addr) | ||
248 | +void tb_target_set_jmp_target(const TranslationBlock *tb, int n, | ||
249 | + uintptr_t jmp_rx, uintptr_t jmp_rw) | ||
250 | { | ||
251 | tcg_insn_unit i1, i2; | ||
252 | ptrdiff_t upper, lower; | ||
253 | + uintptr_t addr = tb->jmp_target_addr[n]; | ||
254 | ptrdiff_t offset = (ptrdiff_t)(addr - jmp_rx) >> 2; | ||
255 | |||
256 | if (offset == sextreg(offset, 0, 26)) { | ||
257 | diff --git a/tcg/ppc/tcg-target.c.inc b/tcg/ppc/tcg-target.c.inc | ||
258 | index XXXXXXX..XXXXXXX 100644 | ||
259 | --- a/tcg/ppc/tcg-target.c.inc | ||
260 | +++ b/tcg/ppc/tcg-target.c.inc | ||
261 | @@ -XXX,XX +XXX,XX @@ static inline void ppc64_replace4(uintptr_t rx, uintptr_t rw, | ||
262 | flush_idcache_range(rx, rw, 16); | ||
263 | } | ||
264 | |||
265 | -void tb_target_set_jmp_target(uintptr_t tc_ptr, uintptr_t jmp_rx, | ||
266 | - uintptr_t jmp_rw, uintptr_t addr) | ||
267 | +void tb_target_set_jmp_target(const TranslationBlock *tb, int n, | ||
268 | + uintptr_t jmp_rx, uintptr_t jmp_rw) | ||
269 | { | ||
270 | tcg_insn_unit i0, i1, i2, i3; | ||
271 | - intptr_t tb_diff = addr - tc_ptr; | ||
272 | + uintptr_t addr = tb->jmp_target_addr[n]; | ||
273 | + intptr_t tb_diff = addr - (uintptr_t)tb->tc.ptr; | ||
274 | intptr_t br_diff = addr - (jmp_rx + 4); | ||
275 | intptr_t lo, hi; | ||
276 | |||
277 | diff --git a/tcg/s390x/tcg-target.c.inc b/tcg/s390x/tcg-target.c.inc | ||
278 | index XXXXXXX..XXXXXXX 100644 | ||
279 | --- a/tcg/s390x/tcg-target.c.inc | ||
280 | +++ b/tcg/s390x/tcg-target.c.inc | ||
281 | @@ -XXX,XX +XXX,XX @@ static void tcg_out_goto_tb(TCGContext *s, int which) | ||
282 | set_jmp_reset_offset(s, which); | ||
283 | } | ||
284 | |||
285 | +void tb_target_set_jmp_target(const TranslationBlock *tb, int n, | ||
286 | + uintptr_t jmp_rx, uintptr_t jmp_rw) | ||
287 | +{ | ||
288 | + /* patch the branch destination */ | ||
289 | + uintptr_t addr = tb->jmp_target_addr[n]; | ||
290 | + intptr_t disp = addr - (jmp_rx - 2); | ||
291 | + qatomic_set((int32_t *)jmp_rw, disp / 2); | ||
292 | + /* no need to flush icache explicitly */ | ||
293 | +} | ||
294 | + | ||
295 | # define OP_32_64(x) \ | ||
296 | case glue(glue(INDEX_op_,x),_i32): \ | ||
297 | case glue(glue(INDEX_op_,x),_i64) | ||
298 | diff --git a/tcg/sparc64/tcg-target.c.inc b/tcg/sparc64/tcg-target.c.inc | ||
299 | index XXXXXXX..XXXXXXX 100644 | ||
300 | --- a/tcg/sparc64/tcg-target.c.inc | ||
301 | +++ b/tcg/sparc64/tcg-target.c.inc | ||
302 | @@ -XXX,XX +XXX,XX @@ void tcg_register_jit(const void *buf, size_t buf_size) | ||
303 | tcg_register_jit_int(buf, buf_size, &debug_frame, sizeof(debug_frame)); | ||
304 | } | ||
305 | |||
306 | -void tb_target_set_jmp_target(uintptr_t tc_ptr, uintptr_t jmp_rx, | ||
307 | - uintptr_t jmp_rw, uintptr_t addr) | ||
308 | +void tb_target_set_jmp_target(const TranslationBlock *tb, int n, | ||
309 | + uintptr_t jmp_rx, uintptr_t jmp_rw) | ||
310 | { | ||
311 | - intptr_t tb_disp = addr - tc_ptr; | ||
312 | + uintptr_t addr = tb->jmp_target_addr[n]; | ||
313 | + intptr_t tb_disp = addr - (uintptr_t)tb->tc.ptr; | ||
314 | intptr_t br_disp = addr - jmp_rx; | ||
315 | tcg_insn_unit i1, i2; | ||
316 | |||
45 | -- | 317 | -- |
46 | 2.20.1 | 318 | 2.34.1 |
47 | 319 | ||
48 | 320 | diff view generated by jsdifflib |
1 | From: Carlos Santos <casantos@redhat.com> | 1 | Reviewed-by: Alex Bennée <alex.bennee@linaro.org> |
---|---|---|---|
2 | 2 | Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org> | |
3 | uClibc defines _SC_LEVEL1_ICACHE_LINESIZE and _SC_LEVEL1_DCACHE_LINESIZE | ||
4 | but the corresponding sysconf calls returns -1, which is a valid result, | ||
5 | meaning that the limit is indeterminate. | ||
6 | |||
7 | Handle this situation using the fallback values instead of crashing due | ||
8 | to an assertion failure. | ||
9 | |||
10 | Signed-off-by: Carlos Santos <casantos@redhat.com> | ||
11 | Message-Id: <20191017123713.30192-1-casantos@redhat.com> | ||
12 | Signed-off-by: Richard Henderson <richard.henderson@linaro.org> | 3 | Signed-off-by: Richard Henderson <richard.henderson@linaro.org> |
13 | --- | 4 | --- |
14 | util/cacheinfo.c | 10 ++++++++-- | 5 | include/tcg/tcg.h | 3 +++ |
15 | 1 file changed, 8 insertions(+), 2 deletions(-) | 6 | tcg/aarch64/tcg-target.h | 4 ---- |
7 | tcg/arm/tcg-target.h | 5 ----- | ||
8 | tcg/i386/tcg-target.h | 3 --- | ||
9 | tcg/loongarch64/tcg-target.h | 3 --- | ||
10 | tcg/mips/tcg-target.h | 5 ----- | ||
11 | tcg/ppc/tcg-target.h | 4 ---- | ||
12 | tcg/riscv/tcg-target.h | 4 ---- | ||
13 | tcg/s390x/tcg-target.h | 4 ---- | ||
14 | tcg/sparc64/tcg-target.h | 4 ---- | ||
15 | tcg/tci/tcg-target.h | 4 ---- | ||
16 | 11 files changed, 3 insertions(+), 40 deletions(-) | ||
16 | 17 | ||
17 | diff --git a/util/cacheinfo.c b/util/cacheinfo.c | 18 | diff --git a/include/tcg/tcg.h b/include/tcg/tcg.h |
18 | index XXXXXXX..XXXXXXX 100644 | 19 | index XXXXXXX..XXXXXXX 100644 |
19 | --- a/util/cacheinfo.c | 20 | --- a/include/tcg/tcg.h |
20 | +++ b/util/cacheinfo.c | 21 | +++ b/include/tcg/tcg.h |
21 | @@ -XXX,XX +XXX,XX @@ static void sys_cache_info(int *isize, int *dsize) | 22 | @@ -XXX,XX +XXX,XX @@ void tcg_func_start(TCGContext *s); |
22 | static void sys_cache_info(int *isize, int *dsize) | 23 | |
23 | { | 24 | int tcg_gen_code(TCGContext *s, TranslationBlock *tb, target_ulong pc_start); |
24 | # ifdef _SC_LEVEL1_ICACHE_LINESIZE | 25 | |
25 | - *isize = sysconf(_SC_LEVEL1_ICACHE_LINESIZE); | 26 | +void tb_target_set_jmp_target(const TranslationBlock *, int, |
26 | + int tmp_isize = (int) sysconf(_SC_LEVEL1_ICACHE_LINESIZE); | 27 | + uintptr_t, uintptr_t); |
27 | + if (tmp_isize > 0) { | 28 | + |
28 | + *isize = tmp_isize; | 29 | void tcg_set_frame(TCGContext *s, TCGReg reg, intptr_t start, intptr_t size); |
29 | + } | 30 | |
30 | # endif | 31 | TCGTemp *tcg_global_mem_new_internal(TCGType, TCGv_ptr, |
31 | # ifdef _SC_LEVEL1_DCACHE_LINESIZE | 32 | diff --git a/tcg/aarch64/tcg-target.h b/tcg/aarch64/tcg-target.h |
32 | - *dsize = sysconf(_SC_LEVEL1_DCACHE_LINESIZE); | 33 | index XXXXXXX..XXXXXXX 100644 |
33 | + int tmp_dsize = (int) sysconf(_SC_LEVEL1_DCACHE_LINESIZE); | 34 | --- a/tcg/aarch64/tcg-target.h |
34 | + if (tmp_dsize > 0) { | 35 | +++ b/tcg/aarch64/tcg-target.h |
35 | + *dsize = tmp_dsize; | 36 | @@ -XXX,XX +XXX,XX @@ typedef enum { |
36 | + } | 37 | |
37 | # endif | 38 | #define TCG_TARGET_DEFAULT_MO (0) |
38 | } | 39 | #define TCG_TARGET_HAS_MEMORY_BSWAP 0 |
39 | #endif /* sys_cache_info */ | 40 | - |
41 | -void tb_target_set_jmp_target(const TranslationBlock *, int, | ||
42 | - uintptr_t, uintptr_t); | ||
43 | - | ||
44 | #define TCG_TARGET_NEED_LDST_LABELS | ||
45 | #define TCG_TARGET_NEED_POOL_LABELS | ||
46 | |||
47 | diff --git a/tcg/arm/tcg-target.h b/tcg/arm/tcg-target.h | ||
48 | index XXXXXXX..XXXXXXX 100644 | ||
49 | --- a/tcg/arm/tcg-target.h | ||
50 | +++ b/tcg/arm/tcg-target.h | ||
51 | @@ -XXX,XX +XXX,XX @@ extern bool use_neon_instructions; | ||
52 | |||
53 | #define TCG_TARGET_DEFAULT_MO (0) | ||
54 | #define TCG_TARGET_HAS_MEMORY_BSWAP 0 | ||
55 | - | ||
56 | -/* not defined -- call should be eliminated at compile time */ | ||
57 | -void tb_target_set_jmp_target(const TranslationBlock *tb, int n, | ||
58 | - uintptr_t, uintptr_t); | ||
59 | - | ||
60 | #define TCG_TARGET_NEED_LDST_LABELS | ||
61 | #define TCG_TARGET_NEED_POOL_LABELS | ||
62 | |||
63 | diff --git a/tcg/i386/tcg-target.h b/tcg/i386/tcg-target.h | ||
64 | index XXXXXXX..XXXXXXX 100644 | ||
65 | --- a/tcg/i386/tcg-target.h | ||
66 | +++ b/tcg/i386/tcg-target.h | ||
67 | @@ -XXX,XX +XXX,XX @@ extern bool have_movbe; | ||
68 | #define TCG_TARGET_extract_i64_valid(ofs, len) \ | ||
69 | (((ofs) == 8 && (len) == 8) || ((ofs) + (len)) == 32) | ||
70 | |||
71 | -void tb_target_set_jmp_target(const TranslationBlock *, int, | ||
72 | - uintptr_t, uintptr_t); | ||
73 | - | ||
74 | /* This defines the natural memory order supported by this | ||
75 | * architecture before guarantees made by various barrier | ||
76 | * instructions. | ||
77 | diff --git a/tcg/loongarch64/tcg-target.h b/tcg/loongarch64/tcg-target.h | ||
78 | index XXXXXXX..XXXXXXX 100644 | ||
79 | --- a/tcg/loongarch64/tcg-target.h | ||
80 | +++ b/tcg/loongarch64/tcg-target.h | ||
81 | @@ -XXX,XX +XXX,XX @@ typedef enum { | ||
82 | #define TCG_TARGET_HAS_muluh_i64 1 | ||
83 | #define TCG_TARGET_HAS_mulsh_i64 1 | ||
84 | |||
85 | -void tb_target_set_jmp_target(const TranslationBlock *tb, int n, | ||
86 | - uintptr_t, uintptr_t); | ||
87 | - | ||
88 | #define TCG_TARGET_DEFAULT_MO (0) | ||
89 | |||
90 | #define TCG_TARGET_NEED_LDST_LABELS | ||
91 | diff --git a/tcg/mips/tcg-target.h b/tcg/mips/tcg-target.h | ||
92 | index XXXXXXX..XXXXXXX 100644 | ||
93 | --- a/tcg/mips/tcg-target.h | ||
94 | +++ b/tcg/mips/tcg-target.h | ||
95 | @@ -XXX,XX +XXX,XX @@ extern bool use_mips32r2_instructions; | ||
96 | #define TCG_TARGET_DEFAULT_MO (0) | ||
97 | #define TCG_TARGET_HAS_MEMORY_BSWAP 1 | ||
98 | |||
99 | -/* not defined -- call should be eliminated at compile time */ | ||
100 | -void tb_target_set_jmp_target(const TranslationBlock *tb, int n, | ||
101 | - uintptr_t, uintptr_t) | ||
102 | - QEMU_ERROR("code path is reachable"); | ||
103 | - | ||
104 | #define TCG_TARGET_NEED_LDST_LABELS | ||
105 | |||
106 | #endif | ||
107 | diff --git a/tcg/ppc/tcg-target.h b/tcg/ppc/tcg-target.h | ||
108 | index XXXXXXX..XXXXXXX 100644 | ||
109 | --- a/tcg/ppc/tcg-target.h | ||
110 | +++ b/tcg/ppc/tcg-target.h | ||
111 | @@ -XXX,XX +XXX,XX @@ extern bool have_vsx; | ||
112 | #define TCG_TARGET_HAS_bitsel_vec have_vsx | ||
113 | #define TCG_TARGET_HAS_cmpsel_vec 0 | ||
114 | |||
115 | -void tb_target_set_jmp_target(const TranslationBlock *tb, int n, | ||
116 | - uintptr_t, uintptr_t); | ||
117 | - | ||
118 | #define TCG_TARGET_DEFAULT_MO (0) | ||
119 | #define TCG_TARGET_HAS_MEMORY_BSWAP 1 | ||
120 | - | ||
121 | #define TCG_TARGET_NEED_LDST_LABELS | ||
122 | #define TCG_TARGET_NEED_POOL_LABELS | ||
123 | |||
124 | diff --git a/tcg/riscv/tcg-target.h b/tcg/riscv/tcg-target.h | ||
125 | index XXXXXXX..XXXXXXX 100644 | ||
126 | --- a/tcg/riscv/tcg-target.h | ||
127 | +++ b/tcg/riscv/tcg-target.h | ||
128 | @@ -XXX,XX +XXX,XX @@ typedef enum { | ||
129 | #define TCG_TARGET_HAS_mulsh_i64 1 | ||
130 | #endif | ||
131 | |||
132 | -/* not defined -- call should be eliminated at compile time */ | ||
133 | -void tb_target_set_jmp_target(const TranslationBlock *tb, int n, | ||
134 | - uintptr_t, uintptr_t); | ||
135 | - | ||
136 | #define TCG_TARGET_DEFAULT_MO (0) | ||
137 | |||
138 | #define TCG_TARGET_NEED_LDST_LABELS | ||
139 | diff --git a/tcg/s390x/tcg-target.h b/tcg/s390x/tcg-target.h | ||
140 | index XXXXXXX..XXXXXXX 100644 | ||
141 | --- a/tcg/s390x/tcg-target.h | ||
142 | +++ b/tcg/s390x/tcg-target.h | ||
143 | @@ -XXX,XX +XXX,XX @@ extern uint64_t s390_facilities[3]; | ||
144 | #define TCG_TARGET_HAS_MEMORY_BSWAP 1 | ||
145 | |||
146 | #define TCG_TARGET_DEFAULT_MO (TCG_MO_ALL & ~TCG_MO_ST_LD) | ||
147 | - | ||
148 | -void tb_target_set_jmp_target(const TranslationBlock *tb, int n, | ||
149 | - uintptr_t jmp_rx, uintptr_t jmp_rw); | ||
150 | - | ||
151 | #define TCG_TARGET_NEED_LDST_LABELS | ||
152 | #define TCG_TARGET_NEED_POOL_LABELS | ||
153 | |||
154 | diff --git a/tcg/sparc64/tcg-target.h b/tcg/sparc64/tcg-target.h | ||
155 | index XXXXXXX..XXXXXXX 100644 | ||
156 | --- a/tcg/sparc64/tcg-target.h | ||
157 | +++ b/tcg/sparc64/tcg-target.h | ||
158 | @@ -XXX,XX +XXX,XX @@ extern bool use_vis3_instructions; | ||
159 | |||
160 | #define TCG_TARGET_DEFAULT_MO (0) | ||
161 | #define TCG_TARGET_HAS_MEMORY_BSWAP 1 | ||
162 | - | ||
163 | -void tb_target_set_jmp_target(const TranslationBlock *tb, int n, | ||
164 | - uintptr_t, uintptr_t); | ||
165 | - | ||
166 | #define TCG_TARGET_NEED_POOL_LABELS | ||
167 | |||
168 | #endif | ||
169 | diff --git a/tcg/tci/tcg-target.h b/tcg/tci/tcg-target.h | ||
170 | index XXXXXXX..XXXXXXX 100644 | ||
171 | --- a/tcg/tci/tcg-target.h | ||
172 | +++ b/tcg/tci/tcg-target.h | ||
173 | @@ -XXX,XX +XXX,XX @@ typedef enum { | ||
174 | |||
175 | #define TCG_TARGET_HAS_MEMORY_BSWAP 1 | ||
176 | |||
177 | -/* not defined -- call should be eliminated at compile time */ | ||
178 | -void tb_target_set_jmp_target(const TranslationBlock *tb, int n, | ||
179 | - uintptr_t, uintptr_t); | ||
180 | - | ||
181 | #endif /* TCG_TARGET_H */ | ||
40 | -- | 182 | -- |
41 | 2.20.1 | 183 | 2.34.1 |
42 | 184 | ||
43 | 185 | diff view generated by jsdifflib |
1 | Do not call get_clock_realtime() in tlb_mmu_resize_locked, | 1 | Install empty versions for !TCG_TARGET_HAS_direct_jump hosts. |
---|---|---|---|
2 | but hoist outside of any loop over a set of tlbs. This is | ||
3 | only two (indirect) callers, tlb_flush_by_mmuidx_async_work | ||
4 | and tlb_flush_page_locked, so not onerous. | ||
5 | 2 | ||
6 | Reviewed-by: Alex Bennée <alex.bennee@linaro.org> | 3 | Reviewed-by: Alex Bennée <alex.bennee@linaro.org> |
7 | Reviewed-by: Alistair Francis <alistair.francis@wdc.com> | 4 | Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org> |
8 | Reviewed-by: Philippe Mathieu-Daudé <philmd@redhat.com> | ||
9 | Signed-off-by: Richard Henderson <richard.henderson@linaro.org> | 5 | Signed-off-by: Richard Henderson <richard.henderson@linaro.org> |
10 | --- | 6 | --- |
11 | accel/tcg/cputlb.c | 14 ++++++++------ | 7 | tcg/arm/tcg-target.c.inc | 6 ++++++ |
12 | 1 file changed, 8 insertions(+), 6 deletions(-) | 8 | tcg/mips/tcg-target.c.inc | 6 ++++++ |
9 | tcg/riscv/tcg-target.c.inc | 6 ++++++ | ||
10 | tcg/tci/tcg-target.c.inc | 6 ++++++ | ||
11 | 4 files changed, 24 insertions(+) | ||
13 | 12 | ||
14 | diff --git a/accel/tcg/cputlb.c b/accel/tcg/cputlb.c | 13 | diff --git a/tcg/arm/tcg-target.c.inc b/tcg/arm/tcg-target.c.inc |
15 | index XXXXXXX..XXXXXXX 100644 | 14 | index XXXXXXX..XXXXXXX 100644 |
16 | --- a/accel/tcg/cputlb.c | 15 | --- a/tcg/arm/tcg-target.c.inc |
17 | +++ b/accel/tcg/cputlb.c | 16 | +++ b/tcg/arm/tcg-target.c.inc |
18 | @@ -XXX,XX +XXX,XX @@ static void tlb_window_reset(CPUTLBDesc *desc, int64_t ns, | 17 | @@ -XXX,XX +XXX,XX @@ static void tcg_out_goto_tb(TCGContext *s, int which) |
19 | * high), since otherwise we are likely to have a significant amount of | 18 | set_jmp_reset_offset(s, which); |
20 | * conflict misses. | ||
21 | */ | ||
22 | -static void tlb_mmu_resize_locked(CPUTLBDesc *desc, CPUTLBDescFast *fast) | ||
23 | +static void tlb_mmu_resize_locked(CPUTLBDesc *desc, CPUTLBDescFast *fast, | ||
24 | + int64_t now) | ||
25 | { | ||
26 | size_t old_size = tlb_n_entries(fast); | ||
27 | size_t rate; | ||
28 | size_t new_size = old_size; | ||
29 | - int64_t now = get_clock_realtime(); | ||
30 | int64_t window_len_ms = 100; | ||
31 | int64_t window_len_ns = window_len_ms * 1000 * 1000; | ||
32 | bool window_expired = now > desc->window_begin_ns + window_len_ns; | ||
33 | @@ -XXX,XX +XXX,XX @@ static void tlb_mmu_flush_locked(CPUTLBDesc *desc, CPUTLBDescFast *fast) | ||
34 | memset(desc->vtable, -1, sizeof(desc->vtable)); | ||
35 | } | 19 | } |
36 | 20 | ||
37 | -static void tlb_flush_one_mmuidx_locked(CPUArchState *env, int mmu_idx) | 21 | +void tb_target_set_jmp_target(const TranslationBlock *tb, int n, |
38 | +static void tlb_flush_one_mmuidx_locked(CPUArchState *env, int mmu_idx, | 22 | + uintptr_t jmp_rx, uintptr_t jmp_rw) |
39 | + int64_t now) | 23 | +{ |
40 | { | 24 | + /* Always indirect, nothing to do */ |
41 | CPUTLBDesc *desc = &env_tlb(env)->d[mmu_idx]; | 25 | +} |
42 | CPUTLBDescFast *fast = &env_tlb(env)->f[mmu_idx]; | 26 | + |
43 | 27 | static void tcg_out_op(TCGContext *s, TCGOpcode opc, | |
44 | - tlb_mmu_resize_locked(desc, fast); | 28 | const TCGArg args[TCG_MAX_OP_ARGS], |
45 | + tlb_mmu_resize_locked(desc, fast, now); | 29 | const int const_args[TCG_MAX_OP_ARGS]) |
46 | tlb_mmu_flush_locked(desc, fast); | 30 | diff --git a/tcg/mips/tcg-target.c.inc b/tcg/mips/tcg-target.c.inc |
31 | index XXXXXXX..XXXXXXX 100644 | ||
32 | --- a/tcg/mips/tcg-target.c.inc | ||
33 | +++ b/tcg/mips/tcg-target.c.inc | ||
34 | @@ -XXX,XX +XXX,XX @@ static void tcg_out_goto_tb(TCGContext *s, int which) | ||
35 | set_jmp_reset_offset(s, which); | ||
47 | } | 36 | } |
48 | 37 | ||
49 | @@ -XXX,XX +XXX,XX @@ static void tlb_flush_by_mmuidx_async_work(CPUState *cpu, run_on_cpu_data data) | 38 | +void tb_target_set_jmp_target(const TranslationBlock *tb, int n, |
50 | CPUArchState *env = cpu->env_ptr; | 39 | + uintptr_t jmp_rx, uintptr_t jmp_rw) |
51 | uint16_t asked = data.host_int; | 40 | +{ |
52 | uint16_t all_dirty, work, to_clean; | 41 | + /* Always indirect, nothing to do */ |
53 | + int64_t now = get_clock_realtime(); | 42 | +} |
54 | 43 | + | |
55 | assert_cpu_is_self(cpu); | 44 | static void tcg_out_op(TCGContext *s, TCGOpcode opc, |
56 | 45 | const TCGArg args[TCG_MAX_OP_ARGS], | |
57 | @@ -XXX,XX +XXX,XX @@ static void tlb_flush_by_mmuidx_async_work(CPUState *cpu, run_on_cpu_data data) | 46 | const int const_args[TCG_MAX_OP_ARGS]) |
58 | 47 | diff --git a/tcg/riscv/tcg-target.c.inc b/tcg/riscv/tcg-target.c.inc | |
59 | for (work = to_clean; work != 0; work &= work - 1) { | 48 | index XXXXXXX..XXXXXXX 100644 |
60 | int mmu_idx = ctz32(work); | 49 | --- a/tcg/riscv/tcg-target.c.inc |
61 | - tlb_flush_one_mmuidx_locked(env, mmu_idx); | 50 | +++ b/tcg/riscv/tcg-target.c.inc |
62 | + tlb_flush_one_mmuidx_locked(env, mmu_idx, now); | 51 | @@ -XXX,XX +XXX,XX @@ static void tcg_out_goto_tb(TCGContext *s, int which) |
63 | } | 52 | set_jmp_reset_offset(s, which); |
64 | 53 | } | |
65 | qemu_spin_unlock(&env_tlb(env)->c.lock); | 54 | |
66 | @@ -XXX,XX +XXX,XX @@ static void tlb_flush_page_locked(CPUArchState *env, int midx, | 55 | +void tb_target_set_jmp_target(const TranslationBlock *tb, int n, |
67 | tlb_debug("forcing full flush midx %d (" | 56 | + uintptr_t jmp_rx, uintptr_t jmp_rw) |
68 | TARGET_FMT_lx "/" TARGET_FMT_lx ")\n", | 57 | +{ |
69 | midx, lp_addr, lp_mask); | 58 | + /* Always indirect, nothing to do */ |
70 | - tlb_flush_one_mmuidx_locked(env, midx); | 59 | +} |
71 | + tlb_flush_one_mmuidx_locked(env, midx, get_clock_realtime()); | 60 | + |
72 | } else { | 61 | static void tcg_out_op(TCGContext *s, TCGOpcode opc, |
73 | if (tlb_flush_entry_locked(tlb_entry(env, midx, page), page)) { | 62 | const TCGArg args[TCG_MAX_OP_ARGS], |
74 | tlb_n_used_entries_dec(env, midx); | 63 | const int const_args[TCG_MAX_OP_ARGS]) |
64 | diff --git a/tcg/tci/tcg-target.c.inc b/tcg/tci/tcg-target.c.inc | ||
65 | index XXXXXXX..XXXXXXX 100644 | ||
66 | --- a/tcg/tci/tcg-target.c.inc | ||
67 | +++ b/tcg/tci/tcg-target.c.inc | ||
68 | @@ -XXX,XX +XXX,XX @@ static void tcg_out_goto_tb(TCGContext *s, int which) | ||
69 | set_jmp_reset_offset(s, which); | ||
70 | } | ||
71 | |||
72 | +void tb_target_set_jmp_target(const TranslationBlock *tb, int n, | ||
73 | + uintptr_t jmp_rx, uintptr_t jmp_rw) | ||
74 | +{ | ||
75 | + /* Always indirect, nothing to do */ | ||
76 | +} | ||
77 | + | ||
78 | static void tcg_out_op(TCGContext *s, TCGOpcode opc, | ||
79 | const TCGArg args[TCG_MAX_OP_ARGS], | ||
80 | const int const_args[TCG_MAX_OP_ARGS]) | ||
75 | -- | 81 | -- |
76 | 2.20.1 | 82 | 2.34.1 |
77 | 83 | ||
78 | 84 | diff view generated by jsdifflib |
New patch | |||
---|---|---|---|
1 | 1 | We now have the option to generate direct or indirect | |
2 | goto_tb depending on the dynamic displacement, thus | ||
3 | the define is no longer necessary or completely accurate. | ||
4 | |||
5 | Reviewed-by: Alex Bennée <alex.bennee@linaro.org> | ||
6 | Signed-off-by: Richard Henderson <richard.henderson@linaro.org> | ||
7 | --- | ||
8 | tcg/aarch64/tcg-target.h | 1 - | ||
9 | tcg/arm/tcg-target.h | 1 - | ||
10 | tcg/i386/tcg-target.h | 1 - | ||
11 | tcg/loongarch64/tcg-target.h | 1 - | ||
12 | tcg/mips/tcg-target.h | 1 - | ||
13 | tcg/ppc/tcg-target.h | 1 - | ||
14 | tcg/riscv/tcg-target.h | 1 - | ||
15 | tcg/s390x/tcg-target.h | 1 - | ||
16 | tcg/sparc64/tcg-target.h | 1 - | ||
17 | tcg/tci/tcg-target.h | 1 - | ||
18 | accel/tcg/cpu-exec.c | 23 +++++++++++------------ | ||
19 | tcg/tcg.c | 1 - | ||
20 | tcg/arm/tcg-target.c.inc | 1 - | ||
21 | tcg/mips/tcg-target.c.inc | 1 - | ||
22 | tcg/riscv/tcg-target.c.inc | 1 - | ||
23 | tcg/s390x/tcg-target.c.inc | 3 +++ | ||
24 | tcg/tci/tcg-target.c.inc | 1 - | ||
25 | 17 files changed, 14 insertions(+), 27 deletions(-) | ||
26 | |||
27 | diff --git a/tcg/aarch64/tcg-target.h b/tcg/aarch64/tcg-target.h | ||
28 | index XXXXXXX..XXXXXXX 100644 | ||
29 | --- a/tcg/aarch64/tcg-target.h | ||
30 | +++ b/tcg/aarch64/tcg-target.h | ||
31 | @@ -XXX,XX +XXX,XX @@ typedef enum { | ||
32 | #define TCG_TARGET_HAS_muls2_i64 0 | ||
33 | #define TCG_TARGET_HAS_muluh_i64 1 | ||
34 | #define TCG_TARGET_HAS_mulsh_i64 1 | ||
35 | -#define TCG_TARGET_HAS_direct_jump 1 | ||
36 | |||
37 | #define TCG_TARGET_HAS_v64 1 | ||
38 | #define TCG_TARGET_HAS_v128 1 | ||
39 | diff --git a/tcg/arm/tcg-target.h b/tcg/arm/tcg-target.h | ||
40 | index XXXXXXX..XXXXXXX 100644 | ||
41 | --- a/tcg/arm/tcg-target.h | ||
42 | +++ b/tcg/arm/tcg-target.h | ||
43 | @@ -XXX,XX +XXX,XX @@ extern bool use_neon_instructions; | ||
44 | #define TCG_TARGET_HAS_mulsh_i32 0 | ||
45 | #define TCG_TARGET_HAS_div_i32 use_idiv_instructions | ||
46 | #define TCG_TARGET_HAS_rem_i32 0 | ||
47 | -#define TCG_TARGET_HAS_direct_jump 0 | ||
48 | #define TCG_TARGET_HAS_qemu_st8_i32 0 | ||
49 | |||
50 | #define TCG_TARGET_HAS_v64 use_neon_instructions | ||
51 | diff --git a/tcg/i386/tcg-target.h b/tcg/i386/tcg-target.h | ||
52 | index XXXXXXX..XXXXXXX 100644 | ||
53 | --- a/tcg/i386/tcg-target.h | ||
54 | +++ b/tcg/i386/tcg-target.h | ||
55 | @@ -XXX,XX +XXX,XX @@ extern bool have_movbe; | ||
56 | #define TCG_TARGET_HAS_muls2_i32 1 | ||
57 | #define TCG_TARGET_HAS_muluh_i32 0 | ||
58 | #define TCG_TARGET_HAS_mulsh_i32 0 | ||
59 | -#define TCG_TARGET_HAS_direct_jump 1 | ||
60 | |||
61 | #if TCG_TARGET_REG_BITS == 64 | ||
62 | /* Keep target addresses zero-extended in a register. */ | ||
63 | diff --git a/tcg/loongarch64/tcg-target.h b/tcg/loongarch64/tcg-target.h | ||
64 | index XXXXXXX..XXXXXXX 100644 | ||
65 | --- a/tcg/loongarch64/tcg-target.h | ||
66 | +++ b/tcg/loongarch64/tcg-target.h | ||
67 | @@ -XXX,XX +XXX,XX @@ typedef enum { | ||
68 | #define TCG_TARGET_HAS_clz_i32 1 | ||
69 | #define TCG_TARGET_HAS_ctz_i32 1 | ||
70 | #define TCG_TARGET_HAS_ctpop_i32 0 | ||
71 | -#define TCG_TARGET_HAS_direct_jump 1 | ||
72 | #define TCG_TARGET_HAS_brcond2 0 | ||
73 | #define TCG_TARGET_HAS_setcond2 0 | ||
74 | #define TCG_TARGET_HAS_qemu_st8_i32 0 | ||
75 | diff --git a/tcg/mips/tcg-target.h b/tcg/mips/tcg-target.h | ||
76 | index XXXXXXX..XXXXXXX 100644 | ||
77 | --- a/tcg/mips/tcg-target.h | ||
78 | +++ b/tcg/mips/tcg-target.h | ||
79 | @@ -XXX,XX +XXX,XX @@ extern bool use_mips32r2_instructions; | ||
80 | #define TCG_TARGET_HAS_muluh_i32 1 | ||
81 | #define TCG_TARGET_HAS_mulsh_i32 1 | ||
82 | #define TCG_TARGET_HAS_bswap32_i32 1 | ||
83 | -#define TCG_TARGET_HAS_direct_jump 0 | ||
84 | |||
85 | #if TCG_TARGET_REG_BITS == 64 | ||
86 | #define TCG_TARGET_HAS_add2_i32 0 | ||
87 | diff --git a/tcg/ppc/tcg-target.h b/tcg/ppc/tcg-target.h | ||
88 | index XXXXXXX..XXXXXXX 100644 | ||
89 | --- a/tcg/ppc/tcg-target.h | ||
90 | +++ b/tcg/ppc/tcg-target.h | ||
91 | @@ -XXX,XX +XXX,XX @@ extern bool have_vsx; | ||
92 | #define TCG_TARGET_HAS_muls2_i32 0 | ||
93 | #define TCG_TARGET_HAS_muluh_i32 1 | ||
94 | #define TCG_TARGET_HAS_mulsh_i32 1 | ||
95 | -#define TCG_TARGET_HAS_direct_jump 1 | ||
96 | #define TCG_TARGET_HAS_qemu_st8_i32 0 | ||
97 | |||
98 | #if TCG_TARGET_REG_BITS == 64 | ||
99 | diff --git a/tcg/riscv/tcg-target.h b/tcg/riscv/tcg-target.h | ||
100 | index XXXXXXX..XXXXXXX 100644 | ||
101 | --- a/tcg/riscv/tcg-target.h | ||
102 | +++ b/tcg/riscv/tcg-target.h | ||
103 | @@ -XXX,XX +XXX,XX @@ typedef enum { | ||
104 | #define TCG_TARGET_HAS_clz_i32 0 | ||
105 | #define TCG_TARGET_HAS_ctz_i32 0 | ||
106 | #define TCG_TARGET_HAS_ctpop_i32 0 | ||
107 | -#define TCG_TARGET_HAS_direct_jump 0 | ||
108 | #define TCG_TARGET_HAS_brcond2 1 | ||
109 | #define TCG_TARGET_HAS_setcond2 1 | ||
110 | #define TCG_TARGET_HAS_qemu_st8_i32 0 | ||
111 | diff --git a/tcg/s390x/tcg-target.h b/tcg/s390x/tcg-target.h | ||
112 | index XXXXXXX..XXXXXXX 100644 | ||
113 | --- a/tcg/s390x/tcg-target.h | ||
114 | +++ b/tcg/s390x/tcg-target.h | ||
115 | @@ -XXX,XX +XXX,XX @@ extern uint64_t s390_facilities[3]; | ||
116 | #define TCG_TARGET_HAS_mulsh_i32 0 | ||
117 | #define TCG_TARGET_HAS_extrl_i64_i32 0 | ||
118 | #define TCG_TARGET_HAS_extrh_i64_i32 0 | ||
119 | -#define TCG_TARGET_HAS_direct_jump 1 | ||
120 | #define TCG_TARGET_HAS_qemu_st8_i32 0 | ||
121 | |||
122 | #define TCG_TARGET_HAS_div2_i64 1 | ||
123 | diff --git a/tcg/sparc64/tcg-target.h b/tcg/sparc64/tcg-target.h | ||
124 | index XXXXXXX..XXXXXXX 100644 | ||
125 | --- a/tcg/sparc64/tcg-target.h | ||
126 | +++ b/tcg/sparc64/tcg-target.h | ||
127 | @@ -XXX,XX +XXX,XX @@ extern bool use_vis3_instructions; | ||
128 | #define TCG_TARGET_HAS_muls2_i32 1 | ||
129 | #define TCG_TARGET_HAS_muluh_i32 0 | ||
130 | #define TCG_TARGET_HAS_mulsh_i32 0 | ||
131 | -#define TCG_TARGET_HAS_direct_jump 1 | ||
132 | #define TCG_TARGET_HAS_qemu_st8_i32 0 | ||
133 | |||
134 | #define TCG_TARGET_HAS_extrl_i64_i32 1 | ||
135 | diff --git a/tcg/tci/tcg-target.h b/tcg/tci/tcg-target.h | ||
136 | index XXXXXXX..XXXXXXX 100644 | ||
137 | --- a/tcg/tci/tcg-target.h | ||
138 | +++ b/tcg/tci/tcg-target.h | ||
139 | @@ -XXX,XX +XXX,XX @@ | ||
140 | #define TCG_TARGET_HAS_muls2_i32 1 | ||
141 | #define TCG_TARGET_HAS_muluh_i32 0 | ||
142 | #define TCG_TARGET_HAS_mulsh_i32 0 | ||
143 | -#define TCG_TARGET_HAS_direct_jump 0 | ||
144 | #define TCG_TARGET_HAS_qemu_st8_i32 0 | ||
145 | |||
146 | #if TCG_TARGET_REG_BITS == 64 | ||
147 | diff --git a/accel/tcg/cpu-exec.c b/accel/tcg/cpu-exec.c | ||
148 | index XXXXXXX..XXXXXXX 100644 | ||
149 | --- a/accel/tcg/cpu-exec.c | ||
150 | +++ b/accel/tcg/cpu-exec.c | ||
151 | @@ -XXX,XX +XXX,XX @@ void cpu_exec_step_atomic(CPUState *cpu) | ||
152 | |||
153 | void tb_set_jmp_target(TranslationBlock *tb, int n, uintptr_t addr) | ||
154 | { | ||
155 | + /* | ||
156 | + * Get the rx view of the structure, from which we find the | ||
157 | + * executable code address, and tb_target_set_jmp_target can | ||
158 | + * produce a pc-relative displacement to jmp_target_addr[n]. | ||
159 | + */ | ||
160 | + const TranslationBlock *c_tb = tcg_splitwx_to_rx(tb); | ||
161 | + uintptr_t offset = tb->jmp_insn_offset[n]; | ||
162 | + uintptr_t jmp_rx = (uintptr_t)tb->tc.ptr + offset; | ||
163 | + uintptr_t jmp_rw = jmp_rx - tcg_splitwx_diff; | ||
164 | + | ||
165 | tb->jmp_target_addr[n] = addr; | ||
166 | - if (TCG_TARGET_HAS_direct_jump) { | ||
167 | - /* | ||
168 | - * Get the rx view of the structure, from which we find the | ||
169 | - * executable code address, and tb_target_set_jmp_target can | ||
170 | - * produce a pc-relative displacement to jmp_target_addr[n]. | ||
171 | - */ | ||
172 | - const TranslationBlock *c_tb = tcg_splitwx_to_rx(tb); | ||
173 | - uintptr_t offset = tb->jmp_insn_offset[n]; | ||
174 | - uintptr_t jmp_rx = (uintptr_t)tb->tc.ptr + offset; | ||
175 | - uintptr_t jmp_rw = jmp_rx - tcg_splitwx_diff; | ||
176 | - tb_target_set_jmp_target(c_tb, n, jmp_rx, jmp_rw); | ||
177 | - } | ||
178 | + tb_target_set_jmp_target(c_tb, n, jmp_rx, jmp_rw); | ||
179 | } | ||
180 | |||
181 | static inline void tb_add_jump(TranslationBlock *tb, int n, | ||
182 | diff --git a/tcg/tcg.c b/tcg/tcg.c | ||
183 | index XXXXXXX..XXXXXXX 100644 | ||
184 | --- a/tcg/tcg.c | ||
185 | +++ b/tcg/tcg.c | ||
186 | @@ -XXX,XX +XXX,XX @@ static void G_GNUC_UNUSED set_jmp_insn_offset(TCGContext *s, int which) | ||
187 | * We will check for overflow at the end of the opcode loop in | ||
188 | * tcg_gen_code, where we bound tcg_current_code_size to UINT16_MAX. | ||
189 | */ | ||
190 | - tcg_debug_assert(TCG_TARGET_HAS_direct_jump); | ||
191 | s->gen_tb->jmp_insn_offset[which] = tcg_current_code_size(s); | ||
192 | } | ||
193 | |||
194 | diff --git a/tcg/arm/tcg-target.c.inc b/tcg/arm/tcg-target.c.inc | ||
195 | index XXXXXXX..XXXXXXX 100644 | ||
196 | --- a/tcg/arm/tcg-target.c.inc | ||
197 | +++ b/tcg/arm/tcg-target.c.inc | ||
198 | @@ -XXX,XX +XXX,XX @@ static void tcg_out_goto_tb(TCGContext *s, int which) | ||
199 | intptr_t ptr, dif, dil; | ||
200 | TCGReg base = TCG_REG_PC; | ||
201 | |||
202 | - qemu_build_assert(!TCG_TARGET_HAS_direct_jump); | ||
203 | ptr = get_jmp_target_addr(s, which); | ||
204 | dif = tcg_pcrel_diff(s, (void *)ptr) - 8; | ||
205 | dil = sextract32(dif, 0, 12); | ||
206 | diff --git a/tcg/mips/tcg-target.c.inc b/tcg/mips/tcg-target.c.inc | ||
207 | index XXXXXXX..XXXXXXX 100644 | ||
208 | --- a/tcg/mips/tcg-target.c.inc | ||
209 | +++ b/tcg/mips/tcg-target.c.inc | ||
210 | @@ -XXX,XX +XXX,XX @@ static void tcg_out_exit_tb(TCGContext *s, uintptr_t a0) | ||
211 | static void tcg_out_goto_tb(TCGContext *s, int which) | ||
212 | { | ||
213 | /* indirect jump method */ | ||
214 | - qemu_build_assert(!TCG_TARGET_HAS_direct_jump); | ||
215 | tcg_out_ld(s, TCG_TYPE_PTR, TCG_TMP0, TCG_REG_ZERO, | ||
216 | get_jmp_target_addr(s, which)); | ||
217 | tcg_out_opc_reg(s, OPC_JR, 0, TCG_TMP0, 0); | ||
218 | diff --git a/tcg/riscv/tcg-target.c.inc b/tcg/riscv/tcg-target.c.inc | ||
219 | index XXXXXXX..XXXXXXX 100644 | ||
220 | --- a/tcg/riscv/tcg-target.c.inc | ||
221 | +++ b/tcg/riscv/tcg-target.c.inc | ||
222 | @@ -XXX,XX +XXX,XX @@ static void tcg_out_exit_tb(TCGContext *s, uintptr_t a0) | ||
223 | |||
224 | static void tcg_out_goto_tb(TCGContext *s, int which) | ||
225 | { | ||
226 | - qemu_build_assert(!TCG_TARGET_HAS_direct_jump); | ||
227 | /* indirect jump method */ | ||
228 | tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_TMP0, TCG_REG_ZERO, | ||
229 | get_jmp_target_addr(s, which)); | ||
230 | diff --git a/tcg/s390x/tcg-target.c.inc b/tcg/s390x/tcg-target.c.inc | ||
231 | index XXXXXXX..XXXXXXX 100644 | ||
232 | --- a/tcg/s390x/tcg-target.c.inc | ||
233 | +++ b/tcg/s390x/tcg-target.c.inc | ||
234 | @@ -XXX,XX +XXX,XX @@ static void tcg_out_goto_tb(TCGContext *s, int which) | ||
235 | void tb_target_set_jmp_target(const TranslationBlock *tb, int n, | ||
236 | uintptr_t jmp_rx, uintptr_t jmp_rw) | ||
237 | { | ||
238 | + if (!HAVE_FACILITY(GEN_INST_EXT)) { | ||
239 | + return; | ||
240 | + } | ||
241 | /* patch the branch destination */ | ||
242 | uintptr_t addr = tb->jmp_target_addr[n]; | ||
243 | intptr_t disp = addr - (jmp_rx - 2); | ||
244 | diff --git a/tcg/tci/tcg-target.c.inc b/tcg/tci/tcg-target.c.inc | ||
245 | index XXXXXXX..XXXXXXX 100644 | ||
246 | --- a/tcg/tci/tcg-target.c.inc | ||
247 | +++ b/tcg/tci/tcg-target.c.inc | ||
248 | @@ -XXX,XX +XXX,XX @@ static void tcg_out_exit_tb(TCGContext *s, uintptr_t arg) | ||
249 | |||
250 | static void tcg_out_goto_tb(TCGContext *s, int which) | ||
251 | { | ||
252 | - qemu_build_assert(!TCG_TARGET_HAS_direct_jump); | ||
253 | /* indirect jump method. */ | ||
254 | tcg_out_op_p(s, INDEX_op_goto_tb, (void *)get_jmp_target_addr(s, which)); | ||
255 | set_jmp_reset_offset(s, which); | ||
256 | -- | ||
257 | 2.34.1 | ||
258 | |||
259 | diff view generated by jsdifflib |
1 | Merge into the only caller, but at the same time split | 1 | The old implementation replaces two insns, swapping between |
---|---|---|---|
2 | out tlb_mmu_init to initialize a single tlb entry. | ||
3 | 2 | ||
3 | b <dest> | ||
4 | nop | ||
5 | br x30 | ||
6 | and | ||
7 | adrp x30, <dest> | ||
8 | addi x30, x30, lo12:<dest> | ||
9 | br x30 | ||
10 | |||
11 | There is a race condition in which a thread could be stopped at | ||
12 | the PC of the second insn, and when restarted does not see the | ||
13 | complete address computation and branches to nowhere. | ||
14 | |||
15 | The new implemetation replaces only one insn, swapping between | ||
16 | |||
17 | b <dest> | ||
18 | br tmp | ||
19 | and | ||
20 | ldr tmp, <jmp_addr> | ||
21 | br tmp | ||
22 | |||
23 | Reported-by: hev <r@hev.cc> | ||
4 | Reviewed-by: Alex Bennée <alex.bennee@linaro.org> | 24 | Reviewed-by: Alex Bennée <alex.bennee@linaro.org> |
5 | Reviewed-by: Alistair Francis <alistair.francis@wdc.com> | ||
6 | Reviewed-by: Philippe Mathieu-Daudé <philmd@redhat.com> | ||
7 | Signed-off-by: Richard Henderson <richard.henderson@linaro.org> | 25 | Signed-off-by: Richard Henderson <richard.henderson@linaro.org> |
8 | --- | 26 | --- |
9 | accel/tcg/cputlb.c | 33 ++++++++++++++++----------------- | 27 | tcg/aarch64/tcg-target.h | 2 +- |
10 | 1 file changed, 16 insertions(+), 17 deletions(-) | 28 | tcg/aarch64/tcg-target.c.inc | 66 +++++++++++++++--------------------- |
29 | 2 files changed, 29 insertions(+), 39 deletions(-) | ||
11 | 30 | ||
12 | diff --git a/accel/tcg/cputlb.c b/accel/tcg/cputlb.c | 31 | diff --git a/tcg/aarch64/tcg-target.h b/tcg/aarch64/tcg-target.h |
13 | index XXXXXXX..XXXXXXX 100644 | 32 | index XXXXXXX..XXXXXXX 100644 |
14 | --- a/accel/tcg/cputlb.c | 33 | --- a/tcg/aarch64/tcg-target.h |
15 | +++ b/accel/tcg/cputlb.c | 34 | +++ b/tcg/aarch64/tcg-target.h |
16 | @@ -XXX,XX +XXX,XX @@ static void tlb_window_reset(CPUTLBDesc *desc, int64_t ns, | 35 | @@ -XXX,XX +XXX,XX @@ |
17 | desc->window_max_entries = max_entries; | 36 | |
37 | #define TCG_TARGET_INSN_UNIT_SIZE 4 | ||
38 | #define TCG_TARGET_TLB_DISPLACEMENT_BITS 24 | ||
39 | -#define MAX_CODE_GEN_BUFFER_SIZE (2 * GiB) | ||
40 | +#define MAX_CODE_GEN_BUFFER_SIZE ((size_t)-1) | ||
41 | |||
42 | typedef enum { | ||
43 | TCG_REG_X0, TCG_REG_X1, TCG_REG_X2, TCG_REG_X3, | ||
44 | diff --git a/tcg/aarch64/tcg-target.c.inc b/tcg/aarch64/tcg-target.c.inc | ||
45 | index XXXXXXX..XXXXXXX 100644 | ||
46 | --- a/tcg/aarch64/tcg-target.c.inc | ||
47 | +++ b/tcg/aarch64/tcg-target.c.inc | ||
48 | @@ -XXX,XX +XXX,XX @@ static void tcg_out_call(TCGContext *s, const tcg_insn_unit *target, | ||
49 | tcg_out_call_int(s, target); | ||
18 | } | 50 | } |
19 | 51 | ||
20 | -static void tlb_dyn_init(CPUArchState *env) | 52 | -void tb_target_set_jmp_target(const TranslationBlock *tb, int n, |
53 | - uintptr_t jmp_rx, uintptr_t jmp_rw) | ||
21 | -{ | 54 | -{ |
22 | - int i; | 55 | - uintptr_t addr = tb->jmp_target_addr[n]; |
56 | - tcg_insn_unit i1, i2; | ||
57 | - TCGType rt = TCG_TYPE_I64; | ||
58 | - TCGReg rd = TCG_REG_TMP; | ||
59 | - uint64_t pair; | ||
23 | - | 60 | - |
24 | - for (i = 0; i < NB_MMU_MODES; i++) { | 61 | - ptrdiff_t offset = addr - jmp_rx; |
25 | - CPUTLBDesc *desc = &env_tlb(env)->d[i]; | ||
26 | - size_t n_entries = 1 << CPU_TLB_DYN_DEFAULT_BITS; | ||
27 | - | 62 | - |
28 | - tlb_window_reset(desc, get_clock_realtime(), 0); | 63 | - if (offset == sextract64(offset, 0, 26)) { |
29 | - desc->n_used_entries = 0; | 64 | - i1 = I3206_B | ((offset >> 2) & 0x3ffffff); |
30 | - env_tlb(env)->f[i].mask = (n_entries - 1) << CPU_TLB_ENTRY_BITS; | 65 | - i2 = NOP; |
31 | - env_tlb(env)->f[i].table = g_new(CPUTLBEntry, n_entries); | 66 | - } else { |
32 | - env_tlb(env)->d[i].iotlb = g_new(CPUIOTLBEntry, n_entries); | 67 | - offset = (addr >> 12) - (jmp_rx >> 12); |
68 | - | ||
69 | - /* patch ADRP */ | ||
70 | - i1 = I3406_ADRP | (offset & 3) << 29 | (offset & 0x1ffffc) << (5 - 2) | rd; | ||
71 | - /* patch ADDI */ | ||
72 | - i2 = I3401_ADDI | rt << 31 | (addr & 0xfff) << 10 | rd << 5 | rd; | ||
33 | - } | 73 | - } |
74 | - pair = (uint64_t)i2 << 32 | i1; | ||
75 | - qatomic_set((uint64_t *)jmp_rw, pair); | ||
76 | - flush_idcache_range(jmp_rx, jmp_rw, 8); | ||
34 | -} | 77 | -} |
35 | - | 78 | - |
36 | /** | 79 | static inline void tcg_out_goto_label(TCGContext *s, TCGLabel *l) |
37 | * tlb_mmu_resize_locked() - perform TLB resize bookkeeping; resize if necessary | 80 | { |
38 | * @desc: The CPUTLBDesc portion of the TLB | 81 | if (!l->has_value) { |
39 | @@ -XXX,XX +XXX,XX @@ static void tlb_flush_one_mmuidx_locked(CPUArchState *env, int mmu_idx) | 82 | @@ -XXX,XX +XXX,XX @@ static void tcg_out_exit_tb(TCGContext *s, uintptr_t a0) |
40 | tlb_mmu_flush_locked(desc, fast); | 83 | static void tcg_out_goto_tb(TCGContext *s, int which) |
84 | { | ||
85 | /* | ||
86 | - * Ensure that ADRP+ADD are 8-byte aligned so that an atomic | ||
87 | - * write can be used to patch the target address. | ||
88 | + * Direct branch, or indirect address load, will be patched | ||
89 | + * by tb_target_set_jmp_target. Assert indirect load offset | ||
90 | + * in range early, regardless of direct branch distance. | ||
91 | */ | ||
92 | - if ((uintptr_t)s->code_ptr & 7) { | ||
93 | - tcg_out32(s, NOP); | ||
94 | - } | ||
95 | + intptr_t i_off = tcg_pcrel_diff(s, (void *)get_jmp_target_addr(s, which)); | ||
96 | + tcg_debug_assert(i_off == sextract64(i_off, 0, 21)); | ||
97 | + | ||
98 | set_jmp_insn_offset(s, which); | ||
99 | - /* | ||
100 | - * actual branch destination will be patched by | ||
101 | - * tb_target_set_jmp_target later | ||
102 | - */ | ||
103 | - tcg_out_insn(s, 3406, ADRP, TCG_REG_TMP, 0); | ||
104 | - tcg_out_insn(s, 3401, ADDI, TCG_TYPE_I64, TCG_REG_TMP, TCG_REG_TMP, 0); | ||
105 | + tcg_out32(s, I3206_B); | ||
106 | tcg_out_insn(s, 3207, BR, TCG_REG_TMP); | ||
107 | set_jmp_reset_offset(s, which); | ||
41 | } | 108 | } |
42 | 109 | ||
43 | +static void tlb_mmu_init(CPUTLBDesc *desc, CPUTLBDescFast *fast, int64_t now) | 110 | +void tb_target_set_jmp_target(const TranslationBlock *tb, int n, |
111 | + uintptr_t jmp_rx, uintptr_t jmp_rw) | ||
44 | +{ | 112 | +{ |
45 | + size_t n_entries = 1 << CPU_TLB_DYN_DEFAULT_BITS; | 113 | + uintptr_t d_addr = tb->jmp_target_addr[n]; |
114 | + ptrdiff_t d_offset = d_addr - jmp_rx; | ||
115 | + tcg_insn_unit insn; | ||
46 | + | 116 | + |
47 | + tlb_window_reset(desc, now, 0); | 117 | + /* Either directly branch, or indirect branch load. */ |
48 | + desc->n_used_entries = 0; | 118 | + if (d_offset == sextract64(d_offset, 0, 28)) { |
49 | + fast->mask = (n_entries - 1) << CPU_TLB_ENTRY_BITS; | 119 | + insn = deposit32(I3206_B, 0, 26, d_offset >> 2); |
50 | + fast->table = g_new(CPUTLBEntry, n_entries); | 120 | + } else { |
51 | + desc->iotlb = g_new(CPUIOTLBEntry, n_entries); | 121 | + uintptr_t i_addr = (uintptr_t)&tb->jmp_target_addr[n]; |
122 | + ptrdiff_t i_offset = i_addr - jmp_rx; | ||
123 | + | ||
124 | + /* Note that we asserted this in range in tcg_out_goto_tb. */ | ||
125 | + insn = deposit32(I3305_LDR | TCG_REG_TMP, 0, 5, i_offset >> 2); | ||
126 | + } | ||
127 | + qatomic_set((uint32_t *)jmp_rw, insn); | ||
128 | + flush_idcache_range(jmp_rx, jmp_rw, 4); | ||
52 | +} | 129 | +} |
53 | + | 130 | + |
54 | static inline void tlb_n_used_entries_inc(CPUArchState *env, uintptr_t mmu_idx) | 131 | static void tcg_out_op(TCGContext *s, TCGOpcode opc, |
55 | { | 132 | const TCGArg args[TCG_MAX_OP_ARGS], |
56 | env_tlb(env)->d[mmu_idx].n_used_entries++; | 133 | const int const_args[TCG_MAX_OP_ARGS]) |
57 | @@ -XXX,XX +XXX,XX @@ static inline void tlb_n_used_entries_dec(CPUArchState *env, uintptr_t mmu_idx) | ||
58 | void tlb_init(CPUState *cpu) | ||
59 | { | ||
60 | CPUArchState *env = cpu->env_ptr; | ||
61 | + int64_t now = get_clock_realtime(); | ||
62 | + int i; | ||
63 | |||
64 | qemu_spin_init(&env_tlb(env)->c.lock); | ||
65 | |||
66 | /* Ensure that cpu_reset performs a full flush. */ | ||
67 | env_tlb(env)->c.dirty = ALL_MMUIDX_BITS; | ||
68 | |||
69 | - tlb_dyn_init(env); | ||
70 | + for (i = 0; i < NB_MMU_MODES; i++) { | ||
71 | + tlb_mmu_init(&env_tlb(env)->d[i], &env_tlb(env)->f[i], now); | ||
72 | + } | ||
73 | } | ||
74 | |||
75 | /* flush_all_helper: run fn across all cpus | ||
76 | -- | 134 | -- |
77 | 2.20.1 | 135 | 2.34.1 |
78 | 136 | ||
79 | 137 | diff view generated by jsdifflib |
New patch | |||
---|---|---|---|
1 | 1 | The old ppc64 implementation replaces 2 or 4 insns, which leaves a race | |
2 | condition in which a thread could be stopped at a PC in the middle of | ||
3 | the sequence, and when restarted does not see the complete address | ||
4 | computation and branches to nowhere. | ||
5 | |||
6 | The new implemetation replaces only one insn, swapping between | ||
7 | |||
8 | b <dest> | ||
9 | and | ||
10 | mtctr r31 | ||
11 | |||
12 | falling through to a general-case indirect branch. | ||
13 | |||
14 | Reviewed-by: Alex Bennée <alex.bennee@linaro.org> | ||
15 | Signed-off-by: Richard Henderson <richard.henderson@linaro.org> | ||
16 | --- | ||
17 | tcg/ppc/tcg-target.h | 3 +- | ||
18 | tcg/ppc/tcg-target.c.inc | 158 +++++++++++---------------------------- | ||
19 | 2 files changed, 44 insertions(+), 117 deletions(-) | ||
20 | |||
21 | diff --git a/tcg/ppc/tcg-target.h b/tcg/ppc/tcg-target.h | ||
22 | index XXXXXXX..XXXXXXX 100644 | ||
23 | --- a/tcg/ppc/tcg-target.h | ||
24 | +++ b/tcg/ppc/tcg-target.h | ||
25 | @@ -XXX,XX +XXX,XX @@ | ||
26 | |||
27 | #ifdef _ARCH_PPC64 | ||
28 | # define TCG_TARGET_REG_BITS 64 | ||
29 | -# define MAX_CODE_GEN_BUFFER_SIZE (2 * GiB) | ||
30 | #else | ||
31 | # define TCG_TARGET_REG_BITS 32 | ||
32 | -# define MAX_CODE_GEN_BUFFER_SIZE (32 * MiB) | ||
33 | #endif | ||
34 | +#define MAX_CODE_GEN_BUFFER_SIZE ((size_t)-1) | ||
35 | |||
36 | #define TCG_TARGET_NB_REGS 64 | ||
37 | #define TCG_TARGET_INSN_UNIT_SIZE 4 | ||
38 | diff --git a/tcg/ppc/tcg-target.c.inc b/tcg/ppc/tcg-target.c.inc | ||
39 | index XXXXXXX..XXXXXXX 100644 | ||
40 | --- a/tcg/ppc/tcg-target.c.inc | ||
41 | +++ b/tcg/ppc/tcg-target.c.inc | ||
42 | @@ -XXX,XX +XXX,XX @@ static void tcg_out_mb(TCGContext *s, TCGArg a0) | ||
43 | tcg_out32(s, insn); | ||
44 | } | ||
45 | |||
46 | -static inline uint64_t make_pair(tcg_insn_unit i1, tcg_insn_unit i2) | ||
47 | -{ | ||
48 | - if (HOST_BIG_ENDIAN) { | ||
49 | - return (uint64_t)i1 << 32 | i2; | ||
50 | - } | ||
51 | - return (uint64_t)i2 << 32 | i1; | ||
52 | -} | ||
53 | - | ||
54 | -static inline void ppc64_replace2(uintptr_t rx, uintptr_t rw, | ||
55 | - tcg_insn_unit i0, tcg_insn_unit i1) | ||
56 | -{ | ||
57 | -#if TCG_TARGET_REG_BITS == 64 | ||
58 | - qatomic_set((uint64_t *)rw, make_pair(i0, i1)); | ||
59 | - flush_idcache_range(rx, rw, 8); | ||
60 | -#else | ||
61 | - qemu_build_not_reached(); | ||
62 | -#endif | ||
63 | -} | ||
64 | - | ||
65 | -static inline void ppc64_replace4(uintptr_t rx, uintptr_t rw, | ||
66 | - tcg_insn_unit i0, tcg_insn_unit i1, | ||
67 | - tcg_insn_unit i2, tcg_insn_unit i3) | ||
68 | -{ | ||
69 | - uint64_t p[2]; | ||
70 | - | ||
71 | - p[!HOST_BIG_ENDIAN] = make_pair(i0, i1); | ||
72 | - p[HOST_BIG_ENDIAN] = make_pair(i2, i3); | ||
73 | - | ||
74 | - /* | ||
75 | - * There's no convenient way to get the compiler to allocate a pair | ||
76 | - * of registers at an even index, so copy into r6/r7 and clobber. | ||
77 | - */ | ||
78 | - asm("mr %%r6, %1\n\t" | ||
79 | - "mr %%r7, %2\n\t" | ||
80 | - "stq %%r6, %0" | ||
81 | - : "=Q"(*(__int128 *)rw) : "r"(p[0]), "r"(p[1]) : "r6", "r7"); | ||
82 | - flush_idcache_range(rx, rw, 16); | ||
83 | -} | ||
84 | - | ||
85 | -void tb_target_set_jmp_target(const TranslationBlock *tb, int n, | ||
86 | - uintptr_t jmp_rx, uintptr_t jmp_rw) | ||
87 | -{ | ||
88 | - tcg_insn_unit i0, i1, i2, i3; | ||
89 | - uintptr_t addr = tb->jmp_target_addr[n]; | ||
90 | - intptr_t tb_diff = addr - (uintptr_t)tb->tc.ptr; | ||
91 | - intptr_t br_diff = addr - (jmp_rx + 4); | ||
92 | - intptr_t lo, hi; | ||
93 | - | ||
94 | - if (TCG_TARGET_REG_BITS == 32) { | ||
95 | - intptr_t diff = addr - jmp_rx; | ||
96 | - tcg_debug_assert(in_range_b(diff)); | ||
97 | - qatomic_set((uint32_t *)jmp_rw, B | (diff & 0x3fffffc)); | ||
98 | - flush_idcache_range(jmp_rx, jmp_rw, 4); | ||
99 | - return; | ||
100 | - } | ||
101 | - | ||
102 | - /* | ||
103 | - * For 16-bit displacements, we can use a single add + branch. | ||
104 | - * This happens quite often. | ||
105 | - */ | ||
106 | - if (tb_diff == (int16_t)tb_diff) { | ||
107 | - i0 = ADDI | TAI(TCG_REG_TB, TCG_REG_TB, tb_diff); | ||
108 | - i1 = B | (br_diff & 0x3fffffc); | ||
109 | - ppc64_replace2(jmp_rx, jmp_rw, i0, i1); | ||
110 | - return; | ||
111 | - } | ||
112 | - | ||
113 | - lo = (int16_t)tb_diff; | ||
114 | - hi = (int32_t)(tb_diff - lo); | ||
115 | - assert(tb_diff == hi + lo); | ||
116 | - i0 = ADDIS | TAI(TCG_REG_TB, TCG_REG_TB, hi >> 16); | ||
117 | - i1 = ADDI | TAI(TCG_REG_TB, TCG_REG_TB, lo); | ||
118 | - | ||
119 | - /* | ||
120 | - * Without stq from 2.07, we can only update two insns, | ||
121 | - * and those must be the ones that load the target address. | ||
122 | - */ | ||
123 | - if (!have_isa_2_07) { | ||
124 | - ppc64_replace2(jmp_rx, jmp_rw, i0, i1); | ||
125 | - return; | ||
126 | - } | ||
127 | - | ||
128 | - /* | ||
129 | - * For 26-bit displacements, we can use a direct branch. | ||
130 | - * Otherwise we still need the indirect branch, which we | ||
131 | - * must restore after a potential direct branch write. | ||
132 | - */ | ||
133 | - br_diff -= 4; | ||
134 | - if (in_range_b(br_diff)) { | ||
135 | - i2 = B | (br_diff & 0x3fffffc); | ||
136 | - i3 = NOP; | ||
137 | - } else { | ||
138 | - i2 = MTSPR | RS(TCG_REG_TB) | CTR; | ||
139 | - i3 = BCCTR | BO_ALWAYS; | ||
140 | - } | ||
141 | - ppc64_replace4(jmp_rx, jmp_rw, i0, i1, i2, i3); | ||
142 | -} | ||
143 | - | ||
144 | static void tcg_out_call_int(TCGContext *s, int lk, | ||
145 | const tcg_insn_unit *target) | ||
146 | { | ||
147 | @@ -XXX,XX +XXX,XX @@ static void tcg_out_exit_tb(TCGContext *s, uintptr_t arg) | ||
148 | |||
149 | static void tcg_out_goto_tb(TCGContext *s, int which) | ||
150 | { | ||
151 | - /* Direct jump. */ | ||
152 | - if (TCG_TARGET_REG_BITS == 64) { | ||
153 | - /* Ensure the next insns are 8 or 16-byte aligned. */ | ||
154 | - while ((uintptr_t)s->code_ptr & (have_isa_2_07 ? 15 : 7)) { | ||
155 | - tcg_out32(s, NOP); | ||
156 | - } | ||
157 | + uintptr_t ptr = get_jmp_target_addr(s, which); | ||
158 | + | ||
159 | + if (USE_REG_TB) { | ||
160 | + ptrdiff_t offset = tcg_tbrel_diff(s, (void *)ptr); | ||
161 | + tcg_out_mem_long(s, LD, LDX, TCG_REG_TB, TCG_REG_TB, offset); | ||
162 | + | ||
163 | + /* Direct branch will be patched by tb_target_set_jmp_target. */ | ||
164 | set_jmp_insn_offset(s, which); | ||
165 | - tcg_out32(s, ADDIS | TAI(TCG_REG_TB, TCG_REG_TB, 0)); | ||
166 | - tcg_out32(s, ADDI | TAI(TCG_REG_TB, TCG_REG_TB, 0)); | ||
167 | tcg_out32(s, MTSPR | RS(TCG_REG_TB) | CTR); | ||
168 | + | ||
169 | + /* When branch is out of range, fall through to indirect. */ | ||
170 | + tcg_out32(s, BCCTR | BO_ALWAYS); | ||
171 | + | ||
172 | + /* For the unlinked case, need to reset TCG_REG_TB. */ | ||
173 | + set_jmp_reset_offset(s, which); | ||
174 | + tcg_out_mem_long(s, ADDI, ADD, TCG_REG_TB, TCG_REG_TB, | ||
175 | + -tcg_current_code_size(s)); | ||
176 | + } else { | ||
177 | + /* Direct branch will be patched by tb_target_set_jmp_target. */ | ||
178 | + set_jmp_insn_offset(s, which); | ||
179 | + tcg_out32(s, NOP); | ||
180 | + | ||
181 | + /* When branch is out of range, fall through to indirect. */ | ||
182 | + tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_TMP1, ptr - (int16_t)ptr); | ||
183 | + tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_TMP1, TCG_REG_TMP1, (int16_t)ptr); | ||
184 | + tcg_out32(s, MTSPR | RS(TCG_REG_TMP1) | CTR); | ||
185 | tcg_out32(s, BCCTR | BO_ALWAYS); | ||
186 | set_jmp_reset_offset(s, which); | ||
187 | - if (USE_REG_TB) { | ||
188 | - /* For the unlinked case, need to reset TCG_REG_TB. */ | ||
189 | - tcg_out_mem_long(s, ADDI, ADD, TCG_REG_TB, TCG_REG_TB, | ||
190 | - -tcg_current_code_size(s)); | ||
191 | - } | ||
192 | - } else { | ||
193 | - set_jmp_insn_offset(s, which); | ||
194 | - tcg_out32(s, B); | ||
195 | - set_jmp_reset_offset(s, which); | ||
196 | } | ||
197 | } | ||
198 | |||
199 | +void tb_target_set_jmp_target(const TranslationBlock *tb, int n, | ||
200 | + uintptr_t jmp_rx, uintptr_t jmp_rw) | ||
201 | +{ | ||
202 | + uintptr_t addr = tb->jmp_target_addr[n]; | ||
203 | + intptr_t diff = addr - jmp_rx; | ||
204 | + tcg_insn_unit insn; | ||
205 | + | ||
206 | + if (in_range_b(diff)) { | ||
207 | + insn = B | (diff & 0x3fffffc); | ||
208 | + } else if (USE_REG_TB) { | ||
209 | + insn = MTSPR | RS(TCG_REG_TB) | CTR; | ||
210 | + } else { | ||
211 | + insn = NOP; | ||
212 | + } | ||
213 | + | ||
214 | + qatomic_set((uint32_t *)jmp_rw, insn); | ||
215 | + flush_idcache_range(jmp_rx, jmp_rw, 4); | ||
216 | +} | ||
217 | + | ||
218 | static void tcg_out_op(TCGContext *s, TCGOpcode opc, | ||
219 | const TCGArg args[TCG_MAX_OP_ARGS], | ||
220 | const int const_args[TCG_MAX_OP_ARGS]) | ||
221 | -- | ||
222 | 2.34.1 | ||
223 | |||
224 | diff view generated by jsdifflib |
1 | No functional change, but the smaller expressions make | 1 | This is always true for sparc64, so this is dead since 3a5f6805c7ca. |
---|---|---|---|
2 | the code easier to read. | ||
3 | 2 | ||
4 | Reviewed-by: Alex Bennée <alex.bennee@linaro.org> | 3 | Reviewed-by: Alex Bennée <alex.bennee@linaro.org> |
5 | Reviewed-by: Alistair Francis <alistair.francis@wdc.com> | 4 | Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org> |
6 | Reviewed-by: Philippe Mathieu-Daudé <philmd@redhat.com> | ||
7 | Signed-off-by: Richard Henderson <richard.henderson@linaro.org> | 5 | Signed-off-by: Richard Henderson <richard.henderson@linaro.org> |
8 | --- | 6 | --- |
9 | accel/tcg/cputlb.c | 35 +++++++++++++++++------------------ | 7 | tcg/sparc64/tcg-target.c.inc | 62 ++++++++++++------------------------ |
10 | 1 file changed, 17 insertions(+), 18 deletions(-) | 8 | 1 file changed, 21 insertions(+), 41 deletions(-) |
11 | 9 | ||
12 | diff --git a/accel/tcg/cputlb.c b/accel/tcg/cputlb.c | 10 | diff --git a/tcg/sparc64/tcg-target.c.inc b/tcg/sparc64/tcg-target.c.inc |
13 | index XXXXXXX..XXXXXXX 100644 | 11 | index XXXXXXX..XXXXXXX 100644 |
14 | --- a/accel/tcg/cputlb.c | 12 | --- a/tcg/sparc64/tcg-target.c.inc |
15 | +++ b/accel/tcg/cputlb.c | 13 | +++ b/tcg/sparc64/tcg-target.c.inc |
16 | @@ -XXX,XX +XXX,XX @@ static void tlb_dyn_init(CPUArchState *env) | 14 | @@ -XXX,XX +XXX,XX @@ static const char * const tcg_target_reg_names[TCG_TARGET_NB_REGS] = { |
17 | 15 | #endif | |
18 | /** | 16 | |
19 | * tlb_mmu_resize_locked() - perform TLB resize bookkeeping; resize if necessary | 17 | #define TCG_REG_TB TCG_REG_I1 |
20 | - * @env: CPU that owns the TLB | 18 | -#define USE_REG_TB (sizeof(void *) > 4) |
21 | - * @mmu_idx: MMU index of the TLB | 19 | |
22 | + * @desc: The CPUTLBDesc portion of the TLB | 20 | static const int tcg_target_reg_alloc_order[] = { |
23 | + * @fast: The CPUTLBDescFast portion of the same TLB | 21 | TCG_REG_L0, |
24 | * | 22 | @@ -XXX,XX +XXX,XX @@ static void tcg_out_movi_int(TCGContext *s, TCGType type, TCGReg ret, |
25 | * Called with tlb_lock_held. | 23 | } |
26 | * | 24 | |
27 | @@ -XXX,XX +XXX,XX @@ static void tlb_dyn_init(CPUArchState *env) | 25 | /* A 13-bit constant relative to the TB. */ |
28 | * high), since otherwise we are likely to have a significant amount of | 26 | - if (!in_prologue && USE_REG_TB) { |
29 | * conflict misses. | 27 | + if (!in_prologue) { |
30 | */ | 28 | test = tcg_tbrel_diff(s, (void *)arg); |
31 | -static void tlb_mmu_resize_locked(CPUArchState *env, int mmu_idx) | 29 | if (check_fit_ptr(test, 13)) { |
32 | +static void tlb_mmu_resize_locked(CPUTLBDesc *desc, CPUTLBDescFast *fast) | 30 | tcg_out_arithi(s, ret, TCG_REG_TB, test, ARITH_ADD); |
31 | @@ -XXX,XX +XXX,XX @@ static void tcg_out_movi_int(TCGContext *s, TCGType type, TCGReg ret, | ||
32 | } | ||
33 | |||
34 | /* Use the constant pool, if possible. */ | ||
35 | - if (!in_prologue && USE_REG_TB) { | ||
36 | + if (!in_prologue) { | ||
37 | new_pool_label(s, arg, R_SPARC_13, s->code_ptr, | ||
38 | tcg_tbrel_diff(s, NULL)); | ||
39 | tcg_out32(s, LDX | INSN_RD(ret) | INSN_RS1(TCG_REG_TB)); | ||
40 | @@ -XXX,XX +XXX,XX @@ static void tcg_target_qemu_prologue(TCGContext *s) | ||
41 | #endif | ||
42 | |||
43 | /* We choose TCG_REG_TB such that no move is required. */ | ||
44 | - if (USE_REG_TB) { | ||
45 | - QEMU_BUILD_BUG_ON(TCG_REG_TB != TCG_REG_I1); | ||
46 | - tcg_regset_set_reg(s->reserved_regs, TCG_REG_TB); | ||
47 | - } | ||
48 | + QEMU_BUILD_BUG_ON(TCG_REG_TB != TCG_REG_I1); | ||
49 | + tcg_regset_set_reg(s->reserved_regs, TCG_REG_TB); | ||
50 | |||
51 | tcg_out_arithi(s, TCG_REG_G0, TCG_REG_I1, 0, JMPL); | ||
52 | /* delay slot */ | ||
53 | @@ -XXX,XX +XXX,XX @@ static void tcg_out_exit_tb(TCGContext *s, uintptr_t a0) | ||
54 | tcg_out_arithi(s, TCG_REG_G0, TCG_REG_I7, 8, RETURN); | ||
55 | tcg_out_movi_imm13(s, TCG_REG_O0, a0); | ||
56 | return; | ||
57 | - } else if (USE_REG_TB) { | ||
58 | + } else { | ||
59 | intptr_t tb_diff = tcg_tbrel_diff(s, (void *)a0); | ||
60 | if (check_fit_ptr(tb_diff, 13)) { | ||
61 | tcg_out_arithi(s, TCG_REG_G0, TCG_REG_I7, 8, RETURN); | ||
62 | @@ -XXX,XX +XXX,XX @@ static void tcg_out_exit_tb(TCGContext *s, uintptr_t a0) | ||
63 | |||
64 | static void tcg_out_goto_tb(TCGContext *s, int which) | ||
33 | { | 65 | { |
34 | - CPUTLBDesc *desc = &env_tlb(env)->d[mmu_idx]; | 66 | + int c; |
35 | - size_t old_size = tlb_n_entries(&env_tlb(env)->f[mmu_idx]); | 67 | + |
36 | + size_t old_size = tlb_n_entries(fast); | 68 | /* Direct jump. */ |
37 | size_t rate; | 69 | - if (USE_REG_TB) { |
38 | size_t new_size = old_size; | 70 | - /* make sure the patch is 8-byte aligned. */ |
39 | int64_t now = get_clock_realtime(); | 71 | - if ((intptr_t)s->code_ptr & 4) { |
40 | @@ -XXX,XX +XXX,XX @@ static void tlb_mmu_resize_locked(CPUArchState *env, int mmu_idx) | 72 | - tcg_out_nop(s); |
41 | return; | 73 | - } |
74 | - set_jmp_insn_offset(s, which); | ||
75 | - tcg_out_sethi(s, TCG_REG_T1, 0); | ||
76 | - tcg_out_arithi(s, TCG_REG_T1, TCG_REG_T1, 0, ARITH_OR); | ||
77 | - tcg_out_arith(s, TCG_REG_G0, TCG_REG_TB, TCG_REG_T1, JMPL); | ||
78 | - tcg_out_arith(s, TCG_REG_TB, TCG_REG_TB, TCG_REG_T1, ARITH_ADD); | ||
79 | - } else { | ||
80 | - set_jmp_insn_offset(s, which); | ||
81 | - tcg_out32(s, CALL); | ||
82 | + /* make sure the patch is 8-byte aligned. */ | ||
83 | + if ((intptr_t)s->code_ptr & 4) { | ||
84 | tcg_out_nop(s); | ||
42 | } | 85 | } |
43 | 86 | + set_jmp_insn_offset(s, which); | |
44 | - g_free(env_tlb(env)->f[mmu_idx].table); | 87 | + tcg_out_sethi(s, TCG_REG_T1, 0); |
45 | - g_free(env_tlb(env)->d[mmu_idx].iotlb); | 88 | + tcg_out_arithi(s, TCG_REG_T1, TCG_REG_T1, 0, ARITH_OR); |
46 | + g_free(fast->table); | 89 | + tcg_out_arith(s, TCG_REG_G0, TCG_REG_TB, TCG_REG_T1, JMPL); |
47 | + g_free(desc->iotlb); | 90 | + tcg_out_arith(s, TCG_REG_TB, TCG_REG_TB, TCG_REG_T1, ARITH_ADD); |
48 | 91 | set_jmp_reset_offset(s, which); | |
49 | tlb_window_reset(desc, now, 0); | 92 | |
50 | /* desc->n_used_entries is cleared by the caller */ | ||
51 | - env_tlb(env)->f[mmu_idx].mask = (new_size - 1) << CPU_TLB_ENTRY_BITS; | ||
52 | - env_tlb(env)->f[mmu_idx].table = g_try_new(CPUTLBEntry, new_size); | ||
53 | - env_tlb(env)->d[mmu_idx].iotlb = g_try_new(CPUIOTLBEntry, new_size); | ||
54 | + fast->mask = (new_size - 1) << CPU_TLB_ENTRY_BITS; | ||
55 | + fast->table = g_try_new(CPUTLBEntry, new_size); | ||
56 | + desc->iotlb = g_try_new(CPUIOTLBEntry, new_size); | ||
57 | + | ||
58 | /* | 93 | /* |
59 | * If the allocations fail, try smaller sizes. We just freed some | 94 | * For the unlinked path of goto_tb, we need to reset TCG_REG_TB |
60 | * memory, so going back to half of new_size has a good chance of working. | 95 | * to the beginning of this TB. |
61 | @@ -XXX,XX +XXX,XX @@ static void tlb_mmu_resize_locked(CPUArchState *env, int mmu_idx) | ||
62 | * allocations to fail though, so we progressively reduce the allocation | ||
63 | * size, aborting if we cannot even allocate the smallest TLB we support. | ||
64 | */ | 96 | */ |
65 | - while (env_tlb(env)->f[mmu_idx].table == NULL || | 97 | - if (USE_REG_TB) { |
66 | - env_tlb(env)->d[mmu_idx].iotlb == NULL) { | 98 | - int c = -tcg_current_code_size(s); |
67 | + while (fast->table == NULL || desc->iotlb == NULL) { | 99 | - if (check_fit_i32(c, 13)) { |
68 | if (new_size == (1 << CPU_TLB_DYN_MIN_BITS)) { | 100 | - tcg_out_arithi(s, TCG_REG_TB, TCG_REG_TB, c, ARITH_ADD); |
69 | error_report("%s: %s", __func__, strerror(errno)); | 101 | - } else { |
70 | abort(); | 102 | - tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_T1, c); |
71 | } | 103 | - tcg_out_arith(s, TCG_REG_TB, TCG_REG_TB, TCG_REG_T1, ARITH_ADD); |
72 | new_size = MAX(new_size >> 1, 1 << CPU_TLB_DYN_MIN_BITS); | 104 | - } |
73 | - env_tlb(env)->f[mmu_idx].mask = (new_size - 1) << CPU_TLB_ENTRY_BITS; | 105 | + c = -tcg_current_code_size(s); |
74 | + fast->mask = (new_size - 1) << CPU_TLB_ENTRY_BITS; | 106 | + if (check_fit_i32(c, 13)) { |
75 | 107 | + tcg_out_arithi(s, TCG_REG_TB, TCG_REG_TB, c, ARITH_ADD); | |
76 | - g_free(env_tlb(env)->f[mmu_idx].table); | 108 | + } else { |
77 | - g_free(env_tlb(env)->d[mmu_idx].iotlb); | 109 | + tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_T1, c); |
78 | - env_tlb(env)->f[mmu_idx].table = g_try_new(CPUTLBEntry, new_size); | 110 | + tcg_out_arith(s, TCG_REG_TB, TCG_REG_TB, TCG_REG_T1, ARITH_ADD); |
79 | - env_tlb(env)->d[mmu_idx].iotlb = g_try_new(CPUIOTLBEntry, new_size); | ||
80 | + g_free(fast->table); | ||
81 | + g_free(desc->iotlb); | ||
82 | + fast->table = g_try_new(CPUTLBEntry, new_size); | ||
83 | + desc->iotlb = g_try_new(CPUIOTLBEntry, new_size); | ||
84 | } | 111 | } |
85 | } | 112 | } |
86 | 113 | ||
87 | static void tlb_flush_one_mmuidx_locked(CPUArchState *env, int mmu_idx) | 114 | @@ -XXX,XX +XXX,XX @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, |
88 | { | 115 | switch (opc) { |
89 | - tlb_mmu_resize_locked(env, mmu_idx); | 116 | case INDEX_op_goto_ptr: |
90 | + tlb_mmu_resize_locked(&env_tlb(env)->d[mmu_idx], &env_tlb(env)->f[mmu_idx]); | 117 | tcg_out_arithi(s, TCG_REG_G0, a0, 0, JMPL); |
91 | env_tlb(env)->d[mmu_idx].n_used_entries = 0; | 118 | - if (USE_REG_TB) { |
92 | env_tlb(env)->d[mmu_idx].large_page_addr = -1; | 119 | - tcg_out_mov_delay(s, TCG_REG_TB, a0); |
93 | env_tlb(env)->d[mmu_idx].large_page_mask = -1; | 120 | - } else { |
121 | - tcg_out_nop(s); | ||
122 | - } | ||
123 | + tcg_out_mov_delay(s, TCG_REG_TB, a0); | ||
124 | break; | ||
125 | case INDEX_op_br: | ||
126 | tcg_out_bpcc(s, COND_A, BPCC_PT, arg_label(a0)); | ||
127 | @@ -XXX,XX +XXX,XX @@ void tb_target_set_jmp_target(const TranslationBlock *tb, int n, | ||
128 | tcg_debug_assert(tb_disp == (int32_t)tb_disp); | ||
129 | tcg_debug_assert(br_disp == (int32_t)br_disp); | ||
130 | |||
131 | - if (!USE_REG_TB) { | ||
132 | - qatomic_set((uint32_t *)jmp_rw, | ||
133 | - deposit32(CALL, 0, 30, br_disp >> 2)); | ||
134 | - flush_idcache_range(jmp_rx, jmp_rw, 4); | ||
135 | - return; | ||
136 | - } | ||
137 | - | ||
138 | /* This does not exercise the range of the branch, but we do | ||
139 | still need to be able to load the new value of TCG_REG_TB. | ||
140 | But this does still happen quite often. */ | ||
94 | -- | 141 | -- |
95 | 2.20.1 | 142 | 2.34.1 |
96 | 143 | ||
97 | 144 | diff view generated by jsdifflib |
1 | There are no users of this function outside cputlb.c, | 1 | The old sparc64 implementation may replace two insns, which leaves |
---|---|---|---|
2 | and its interface will change in the next patch. | 2 | a race condition in which a thread could be stopped at a PC in the |
3 | middle of the sequence, and when restarted does not see the complete | ||
4 | address computation and branches to nowhere. | ||
5 | |||
6 | The new implemetation replaces only one insn, swapping between a | ||
7 | direct branch and a direct call. The TCG_REG_TB register is loaded | ||
8 | from tb->jmp_target_addr[] in the delay slot. | ||
3 | 9 | ||
4 | Reviewed-by: Alex Bennée <alex.bennee@linaro.org> | 10 | Reviewed-by: Alex Bennée <alex.bennee@linaro.org> |
5 | Reviewed-by: Alistair Francis <alistair.francis@wdc.com> | ||
6 | Reviewed-by: Philippe Mathieu-Daudé <philmd@redhat.com> | ||
7 | Signed-off-by: Richard Henderson <richard.henderson@linaro.org> | 11 | Signed-off-by: Richard Henderson <richard.henderson@linaro.org> |
8 | --- | 12 | --- |
9 | include/exec/cpu_ldst.h | 5 ----- | 13 | tcg/sparc64/tcg-target.c.inc | 87 +++++++++++++++--------------------- |
10 | accel/tcg/cputlb.c | 5 +++++ | 14 | 1 file changed, 37 insertions(+), 50 deletions(-) |
11 | 2 files changed, 5 insertions(+), 5 deletions(-) | ||
12 | 15 | ||
13 | diff --git a/include/exec/cpu_ldst.h b/include/exec/cpu_ldst.h | 16 | diff --git a/tcg/sparc64/tcg-target.c.inc b/tcg/sparc64/tcg-target.c.inc |
14 | index XXXXXXX..XXXXXXX 100644 | 17 | index XXXXXXX..XXXXXXX 100644 |
15 | --- a/include/exec/cpu_ldst.h | 18 | --- a/tcg/sparc64/tcg-target.c.inc |
16 | +++ b/include/exec/cpu_ldst.h | 19 | +++ b/tcg/sparc64/tcg-target.c.inc |
17 | @@ -XXX,XX +XXX,XX @@ static inline uintptr_t tlb_index(CPUArchState *env, uintptr_t mmu_idx, | 20 | @@ -XXX,XX +XXX,XX @@ static void tcg_out_exit_tb(TCGContext *s, uintptr_t a0) |
18 | return (addr >> TARGET_PAGE_BITS) & size_mask; | 21 | |
22 | static void tcg_out_goto_tb(TCGContext *s, int which) | ||
23 | { | ||
24 | - int c; | ||
25 | + ptrdiff_t off = tcg_tbrel_diff(s, (void *)get_jmp_target_addr(s, which)); | ||
26 | |||
27 | - /* Direct jump. */ | ||
28 | - /* make sure the patch is 8-byte aligned. */ | ||
29 | - if ((intptr_t)s->code_ptr & 4) { | ||
30 | - tcg_out_nop(s); | ||
31 | - } | ||
32 | + /* Direct branch will be patched by tb_target_set_jmp_target. */ | ||
33 | set_jmp_insn_offset(s, which); | ||
34 | - tcg_out_sethi(s, TCG_REG_T1, 0); | ||
35 | - tcg_out_arithi(s, TCG_REG_T1, TCG_REG_T1, 0, ARITH_OR); | ||
36 | - tcg_out_arith(s, TCG_REG_G0, TCG_REG_TB, TCG_REG_T1, JMPL); | ||
37 | - tcg_out_arith(s, TCG_REG_TB, TCG_REG_TB, TCG_REG_T1, ARITH_ADD); | ||
38 | + tcg_out32(s, CALL); | ||
39 | + /* delay slot */ | ||
40 | + tcg_debug_assert(check_fit_ptr(off, 13)); | ||
41 | + tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_TB, TCG_REG_TB, off); | ||
42 | set_jmp_reset_offset(s, which); | ||
43 | |||
44 | /* | ||
45 | * For the unlinked path of goto_tb, we need to reset TCG_REG_TB | ||
46 | * to the beginning of this TB. | ||
47 | */ | ||
48 | - c = -tcg_current_code_size(s); | ||
49 | - if (check_fit_i32(c, 13)) { | ||
50 | - tcg_out_arithi(s, TCG_REG_TB, TCG_REG_TB, c, ARITH_ADD); | ||
51 | + off = -tcg_current_code_size(s); | ||
52 | + if (check_fit_i32(off, 13)) { | ||
53 | + tcg_out_arithi(s, TCG_REG_TB, TCG_REG_TB, off, ARITH_ADD); | ||
54 | } else { | ||
55 | - tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_T1, c); | ||
56 | + tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_T1, off); | ||
57 | tcg_out_arith(s, TCG_REG_TB, TCG_REG_TB, TCG_REG_T1, ARITH_ADD); | ||
58 | } | ||
19 | } | 59 | } |
20 | 60 | ||
21 | -static inline size_t tlb_n_entries(CPUArchState *env, uintptr_t mmu_idx) | 61 | +void tb_target_set_jmp_target(const TranslationBlock *tb, int n, |
22 | -{ | 62 | + uintptr_t jmp_rx, uintptr_t jmp_rw) |
23 | - return (env_tlb(env)->f[mmu_idx].mask >> CPU_TLB_ENTRY_BITS) + 1; | ||
24 | -} | ||
25 | - | ||
26 | /* Find the TLB entry corresponding to the mmu_idx + address pair. */ | ||
27 | static inline CPUTLBEntry *tlb_entry(CPUArchState *env, uintptr_t mmu_idx, | ||
28 | target_ulong addr) | ||
29 | diff --git a/accel/tcg/cputlb.c b/accel/tcg/cputlb.c | ||
30 | index XXXXXXX..XXXXXXX 100644 | ||
31 | --- a/accel/tcg/cputlb.c | ||
32 | +++ b/accel/tcg/cputlb.c | ||
33 | @@ -XXX,XX +XXX,XX @@ QEMU_BUILD_BUG_ON(sizeof(target_ulong) > sizeof(run_on_cpu_data)); | ||
34 | QEMU_BUILD_BUG_ON(NB_MMU_MODES > 16); | ||
35 | #define ALL_MMUIDX_BITS ((1 << NB_MMU_MODES) - 1) | ||
36 | |||
37 | +static inline size_t tlb_n_entries(CPUArchState *env, uintptr_t mmu_idx) | ||
38 | +{ | 63 | +{ |
39 | + return (env_tlb(env)->f[mmu_idx].mask >> CPU_TLB_ENTRY_BITS) + 1; | 64 | + uintptr_t addr = tb->jmp_target_addr[n]; |
65 | + intptr_t br_disp = (intptr_t)(addr - jmp_rx) >> 2; | ||
66 | + tcg_insn_unit insn; | ||
67 | + | ||
68 | + br_disp >>= 2; | ||
69 | + if (check_fit_ptr(br_disp, 19)) { | ||
70 | + /* ba,pt %icc, addr */ | ||
71 | + insn = deposit32(INSN_OP(0) | INSN_OP2(1) | INSN_COND(COND_A) | ||
72 | + | BPCC_ICC | BPCC_PT, 0, 19, br_disp); | ||
73 | + } else if (check_fit_ptr(br_disp, 22)) { | ||
74 | + /* ba addr */ | ||
75 | + insn = deposit32(INSN_OP(0) | INSN_OP2(2) | INSN_COND(COND_A), | ||
76 | + 0, 22, br_disp); | ||
77 | + } else { | ||
78 | + /* The code_gen_buffer can't be larger than 2GB. */ | ||
79 | + tcg_debug_assert(check_fit_ptr(br_disp, 30)); | ||
80 | + /* call addr */ | ||
81 | + insn = deposit32(CALL, 0, 30, br_disp); | ||
82 | + } | ||
83 | + | ||
84 | + qatomic_set((uint32_t *)jmp_rw, insn); | ||
85 | + flush_idcache_range(jmp_rx, jmp_rw, 4); | ||
40 | +} | 86 | +} |
41 | + | 87 | + |
42 | static inline size_t sizeof_tlb(CPUArchState *env, uintptr_t mmu_idx) | 88 | static void tcg_out_op(TCGContext *s, TCGOpcode opc, |
89 | const TCGArg args[TCG_MAX_OP_ARGS], | ||
90 | const int const_args[TCG_MAX_OP_ARGS]) | ||
91 | @@ -XXX,XX +XXX,XX @@ void tcg_register_jit(const void *buf, size_t buf_size) | ||
43 | { | 92 | { |
44 | return env_tlb(env)->f[mmu_idx].mask + (1 << CPU_TLB_ENTRY_BITS); | 93 | tcg_register_jit_int(buf, buf_size, &debug_frame, sizeof(debug_frame)); |
94 | } | ||
95 | - | ||
96 | -void tb_target_set_jmp_target(const TranslationBlock *tb, int n, | ||
97 | - uintptr_t jmp_rx, uintptr_t jmp_rw) | ||
98 | -{ | ||
99 | - uintptr_t addr = tb->jmp_target_addr[n]; | ||
100 | - intptr_t tb_disp = addr - (uintptr_t)tb->tc.ptr; | ||
101 | - intptr_t br_disp = addr - jmp_rx; | ||
102 | - tcg_insn_unit i1, i2; | ||
103 | - | ||
104 | - /* We can reach the entire address space for ILP32. | ||
105 | - For LP64, the code_gen_buffer can't be larger than 2GB. */ | ||
106 | - tcg_debug_assert(tb_disp == (int32_t)tb_disp); | ||
107 | - tcg_debug_assert(br_disp == (int32_t)br_disp); | ||
108 | - | ||
109 | - /* This does not exercise the range of the branch, but we do | ||
110 | - still need to be able to load the new value of TCG_REG_TB. | ||
111 | - But this does still happen quite often. */ | ||
112 | - if (check_fit_ptr(tb_disp, 13)) { | ||
113 | - /* ba,pt %icc, addr */ | ||
114 | - i1 = (INSN_OP(0) | INSN_OP2(1) | INSN_COND(COND_A) | ||
115 | - | BPCC_ICC | BPCC_PT | INSN_OFF19(br_disp)); | ||
116 | - i2 = (ARITH_ADD | INSN_RD(TCG_REG_TB) | INSN_RS1(TCG_REG_TB) | ||
117 | - | INSN_IMM13(tb_disp)); | ||
118 | - } else if (tb_disp >= 0) { | ||
119 | - i1 = SETHI | INSN_RD(TCG_REG_T1) | ((tb_disp & 0xfffffc00) >> 10); | ||
120 | - i2 = (ARITH_OR | INSN_RD(TCG_REG_T1) | INSN_RS1(TCG_REG_T1) | ||
121 | - | INSN_IMM13(tb_disp & 0x3ff)); | ||
122 | - } else { | ||
123 | - i1 = SETHI | INSN_RD(TCG_REG_T1) | ((~tb_disp & 0xfffffc00) >> 10); | ||
124 | - i2 = (ARITH_XOR | INSN_RD(TCG_REG_T1) | INSN_RS1(TCG_REG_T1) | ||
125 | - | INSN_IMM13((tb_disp & 0x3ff) | -0x400)); | ||
126 | - } | ||
127 | - | ||
128 | - qatomic_set((uint64_t *)jmp_rw, deposit64(i2, 32, 32, i1)); | ||
129 | - flush_idcache_range(jmp_rx, jmp_rw, 8); | ||
130 | -} | ||
45 | -- | 131 | -- |
46 | 2.20.1 | 132 | 2.34.1 |
47 | 133 | ||
48 | 134 | diff view generated by jsdifflib |
1 | We do not need the entire CPUArchState to compute these values. | 1 | Now that tcg can handle direct and indirect goto_tb |
---|---|---|---|
2 | simultaneously, we can optimistically leave space for | ||
3 | a direct branch and fall back to loading the pointer | ||
4 | from the TB for an indirect branch. | ||
2 | 5 | ||
3 | Reviewed-by: Alex Bennée <alex.bennee@linaro.org> | 6 | Reviewed-by: Alex Bennée <alex.bennee@linaro.org> |
4 | Reviewed-by: Alistair Francis <alistair.francis@wdc.com> | ||
5 | Reviewed-by: Philippe Mathieu-Daudé <philmd@redhat.com> | ||
6 | Signed-off-by: Richard Henderson <richard.henderson@linaro.org> | 7 | Signed-off-by: Richard Henderson <richard.henderson@linaro.org> |
7 | --- | 8 | --- |
8 | accel/tcg/cputlb.c | 15 ++++++++------- | 9 | tcg/arm/tcg-target.c.inc | 52 ++++++++++++++++++++++++++++------------ |
9 | 1 file changed, 8 insertions(+), 7 deletions(-) | 10 | 1 file changed, 37 insertions(+), 15 deletions(-) |
10 | 11 | ||
11 | diff --git a/accel/tcg/cputlb.c b/accel/tcg/cputlb.c | 12 | diff --git a/tcg/arm/tcg-target.c.inc b/tcg/arm/tcg-target.c.inc |
12 | index XXXXXXX..XXXXXXX 100644 | 13 | index XXXXXXX..XXXXXXX 100644 |
13 | --- a/accel/tcg/cputlb.c | 14 | --- a/tcg/arm/tcg-target.c.inc |
14 | +++ b/accel/tcg/cputlb.c | 15 | +++ b/tcg/arm/tcg-target.c.inc |
15 | @@ -XXX,XX +XXX,XX @@ QEMU_BUILD_BUG_ON(sizeof(target_ulong) > sizeof(run_on_cpu_data)); | 16 | @@ -XXX,XX +XXX,XX @@ typedef enum { |
16 | QEMU_BUILD_BUG_ON(NB_MMU_MODES > 16); | 17 | ARITH_BIC = 0xe << 21, |
17 | #define ALL_MMUIDX_BITS ((1 << NB_MMU_MODES) - 1) | 18 | ARITH_MVN = 0xf << 21, |
18 | 19 | ||
19 | -static inline size_t tlb_n_entries(CPUArchState *env, uintptr_t mmu_idx) | 20 | + INSN_B = 0x0a000000, |
20 | +static inline size_t tlb_n_entries(CPUTLBDescFast *fast) | 21 | + |
22 | INSN_CLZ = 0x016f0f10, | ||
23 | INSN_RBIT = 0x06ff0f30, | ||
24 | |||
25 | @@ -XXX,XX +XXX,XX @@ static bool tcg_target_const_match(int64_t val, TCGType type, int ct) | ||
26 | |||
27 | static void tcg_out_b_imm(TCGContext *s, ARMCond cond, int32_t offset) | ||
21 | { | 28 | { |
22 | - return (env_tlb(env)->f[mmu_idx].mask >> CPU_TLB_ENTRY_BITS) + 1; | 29 | - tcg_out32(s, (cond << 28) | 0x0a000000 | |
23 | + return (fast->mask >> CPU_TLB_ENTRY_BITS) + 1; | 30 | + tcg_out32(s, (cond << 28) | INSN_B | |
31 | (((offset - 8) >> 2) & 0x00ffffff)); | ||
24 | } | 32 | } |
25 | 33 | ||
26 | -static inline size_t sizeof_tlb(CPUArchState *env, uintptr_t mmu_idx) | 34 | @@ -XXX,XX +XXX,XX @@ static void tcg_out_exit_tb(TCGContext *s, uintptr_t arg) |
27 | +static inline size_t sizeof_tlb(CPUTLBDescFast *fast) | 35 | |
36 | static void tcg_out_goto_tb(TCGContext *s, int which) | ||
28 | { | 37 | { |
29 | - return env_tlb(env)->f[mmu_idx].mask + (1 << CPU_TLB_ENTRY_BITS); | 38 | - /* Indirect jump method */ |
30 | + return fast->mask + (1 << CPU_TLB_ENTRY_BITS); | 39 | - intptr_t ptr, dif, dil; |
40 | - TCGReg base = TCG_REG_PC; | ||
41 | + uintptr_t i_addr; | ||
42 | + intptr_t i_disp; | ||
43 | |||
44 | - ptr = get_jmp_target_addr(s, which); | ||
45 | - dif = tcg_pcrel_diff(s, (void *)ptr) - 8; | ||
46 | - dil = sextract32(dif, 0, 12); | ||
47 | - if (dif != dil) { | ||
48 | + /* Direct branch will be patched by tb_target_set_jmp_target. */ | ||
49 | + set_jmp_insn_offset(s, which); | ||
50 | + tcg_out32(s, INSN_NOP); | ||
51 | + | ||
52 | + /* When branch is out of range, fall through to indirect. */ | ||
53 | + i_addr = get_jmp_target_addr(s, which); | ||
54 | + i_disp = tcg_pcrel_diff(s, (void *)i_addr) - 8; | ||
55 | + tcg_debug_assert(i_disp < 0); | ||
56 | + if (i_disp >= -0xfff) { | ||
57 | + tcg_out_ld32_12(s, COND_AL, TCG_REG_PC, TCG_REG_PC, i_disp); | ||
58 | + } else { | ||
59 | /* | ||
60 | * The TB is close, but outside the 12 bits addressable by | ||
61 | * the load. We can extend this to 20 bits with a sub of a | ||
62 | - * shifted immediate from pc. In the vastly unlikely event | ||
63 | - * the code requires more than 1MB, we'll use 2 insns and | ||
64 | - * be no worse off. | ||
65 | + * shifted immediate from pc. | ||
66 | */ | ||
67 | - base = TCG_REG_R0; | ||
68 | - tcg_out_movi32(s, COND_AL, base, ptr - dil); | ||
69 | + int h = -i_disp; | ||
70 | + int l = h & 0xfff; | ||
71 | + | ||
72 | + h = encode_imm_nofail(h - l); | ||
73 | + tcg_out_dat_imm(s, COND_AL, ARITH_SUB, TCG_REG_R0, TCG_REG_PC, h); | ||
74 | + tcg_out_ld32_12(s, COND_AL, TCG_REG_PC, TCG_REG_R0, l); | ||
75 | } | ||
76 | - tcg_out_ld32_12(s, COND_AL, TCG_REG_PC, base, dil); | ||
77 | set_jmp_reset_offset(s, which); | ||
31 | } | 78 | } |
32 | 79 | ||
33 | static void tlb_window_reset(CPUTLBDesc *desc, int64_t ns, | 80 | void tb_target_set_jmp_target(const TranslationBlock *tb, int n, |
34 | @@ -XXX,XX +XXX,XX @@ static void tlb_dyn_init(CPUArchState *env) | 81 | uintptr_t jmp_rx, uintptr_t jmp_rw) |
35 | static void tlb_mmu_resize_locked(CPUArchState *env, int mmu_idx) | ||
36 | { | 82 | { |
37 | CPUTLBDesc *desc = &env_tlb(env)->d[mmu_idx]; | 83 | - /* Always indirect, nothing to do */ |
38 | - size_t old_size = tlb_n_entries(env, mmu_idx); | 84 | + uintptr_t addr = tb->jmp_target_addr[n]; |
39 | + size_t old_size = tlb_n_entries(&env_tlb(env)->f[mmu_idx]); | 85 | + ptrdiff_t offset = addr - (jmp_rx + 8); |
40 | size_t rate; | 86 | + tcg_insn_unit insn; |
41 | size_t new_size = old_size; | 87 | + |
42 | int64_t now = get_clock_realtime(); | 88 | + /* Either directly branch, or fall through to indirect branch. */ |
43 | @@ -XXX,XX +XXX,XX @@ static void tlb_flush_one_mmuidx_locked(CPUArchState *env, int mmu_idx) | 89 | + if (offset == sextract64(offset, 0, 26)) { |
44 | env_tlb(env)->d[mmu_idx].large_page_addr = -1; | 90 | + /* B <addr> */ |
45 | env_tlb(env)->d[mmu_idx].large_page_mask = -1; | 91 | + insn = deposit32((COND_AL << 28) | INSN_B, 0, 24, offset >> 2); |
46 | env_tlb(env)->d[mmu_idx].vindex = 0; | 92 | + } else { |
47 | - memset(env_tlb(env)->f[mmu_idx].table, -1, sizeof_tlb(env, mmu_idx)); | 93 | + insn = INSN_NOP; |
48 | + memset(env_tlb(env)->f[mmu_idx].table, -1, | 94 | + } |
49 | + sizeof_tlb(&env_tlb(env)->f[mmu_idx])); | 95 | + |
50 | memset(env_tlb(env)->d[mmu_idx].vtable, -1, | 96 | + qatomic_set((uint32_t *)jmp_rw, insn); |
51 | sizeof(env_tlb(env)->d[0].vtable)); | 97 | + flush_idcache_range(jmp_rx, jmp_rw, 4); |
52 | } | 98 | } |
53 | @@ -XXX,XX +XXX,XX @@ void tlb_reset_dirty(CPUState *cpu, ram_addr_t start1, ram_addr_t length) | 99 | |
54 | qemu_spin_lock(&env_tlb(env)->c.lock); | 100 | static void tcg_out_op(TCGContext *s, TCGOpcode opc, |
55 | for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) { | ||
56 | unsigned int i; | ||
57 | - unsigned int n = tlb_n_entries(env, mmu_idx); | ||
58 | + unsigned int n = tlb_n_entries(&env_tlb(env)->f[mmu_idx]); | ||
59 | |||
60 | for (i = 0; i < n; i++) { | ||
61 | tlb_reset_dirty_range_locked(&env_tlb(env)->f[mmu_idx].table[i], | ||
62 | -- | 101 | -- |
63 | 2.20.1 | 102 | 2.34.1 |
64 | 103 | ||
65 | 104 | diff view generated by jsdifflib |
1 | The accel_list and tmp variables are only used when manufacturing | ||
---|---|---|---|
2 | -machine accel, options based on -accel. | ||
3 | |||
4 | Acked-by: Paolo Bonzini <pbonzini@redhat.com> | ||
5 | Reviewed-by: Alex Bennée <alex.bennee@linaro.org> | 1 | Reviewed-by: Alex Bennée <alex.bennee@linaro.org> |
6 | Reviewed by: Aleksandar Markovic <amarkovic@wavecomp.com> | 2 | Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org> |
7 | Signed-off-by: Richard Henderson <richard.henderson@linaro.org> | 3 | Signed-off-by: Richard Henderson <richard.henderson@linaro.org> |
8 | --- | 4 | --- |
9 | vl.c | 3 ++- | 5 | tcg/riscv/tcg-target.c.inc | 3 ++- |
10 | 1 file changed, 2 insertions(+), 1 deletion(-) | 6 | 1 file changed, 2 insertions(+), 1 deletion(-) |
11 | 7 | ||
12 | diff --git a/vl.c b/vl.c | 8 | diff --git a/tcg/riscv/tcg-target.c.inc b/tcg/riscv/tcg-target.c.inc |
13 | index XXXXXXX..XXXXXXX 100644 | 9 | index XXXXXXX..XXXXXXX 100644 |
14 | --- a/vl.c | 10 | --- a/tcg/riscv/tcg-target.c.inc |
15 | +++ b/vl.c | 11 | +++ b/tcg/riscv/tcg-target.c.inc |
16 | @@ -XXX,XX +XXX,XX @@ static int do_configure_accelerator(void *opaque, QemuOpts *opts, Error **errp) | 12 | @@ -XXX,XX +XXX,XX @@ typedef enum { |
17 | static void configure_accelerators(const char *progname) | 13 | #endif |
14 | |||
15 | OPC_FENCE = 0x0000000f, | ||
16 | + OPC_NOP = OPC_ADDI, /* nop = addi r0,r0,0 */ | ||
17 | } RISCVInsn; | ||
18 | |||
19 | /* | ||
20 | @@ -XXX,XX +XXX,XX @@ static void tcg_out_nop_fill(tcg_insn_unit *p, int count) | ||
18 | { | 21 | { |
19 | const char *accel; | 22 | int i; |
20 | - char **accel_list, **tmp; | 23 | for (i = 0; i < count; ++i) { |
21 | bool init_failed = false; | 24 | - p[i] = encode_i(OPC_ADDI, TCG_REG_ZERO, TCG_REG_ZERO, 0); |
22 | 25 | + p[i] = OPC_NOP; | |
23 | qemu_opts_foreach(qemu_find_opts("icount"), | 26 | } |
24 | @@ -XXX,XX +XXX,XX @@ static void configure_accelerators(const char *progname) | 27 | } |
25 | 28 | ||
26 | accel = qemu_opt_get(qemu_get_machine_opts(), "accel"); | ||
27 | if (QTAILQ_EMPTY(&qemu_accel_opts.head)) { | ||
28 | + char **accel_list, **tmp; | ||
29 | + | ||
30 | if (accel == NULL) { | ||
31 | /* Select the default accelerator */ | ||
32 | if (!accel_find("tcg") && !accel_find("kvm")) { | ||
33 | -- | 29 | -- |
34 | 2.20.1 | 30 | 2.34.1 |
35 | 31 | ||
36 | 32 | diff view generated by jsdifflib |
1 | No functional change, but the smaller expressions make | 1 | Now that tcg can handle direct and indirect goto_tb simultaneously, |
---|---|---|---|
2 | the code easier to read. | 2 | we can optimistically leave space for a direct branch and fall back |
3 | to loading the pointer from the TB for an indirect branch. | ||
3 | 4 | ||
4 | Reviewed-by: Alex Bennée <alex.bennee@linaro.org> | 5 | Reviewed-by: Alex Bennée <alex.bennee@linaro.org> |
5 | Reviewed-by: Alistair Francis <alistair.francis@wdc.com> | ||
6 | Reviewed-by: Philippe Mathieu-Daudé <philmd@redhat.com> | ||
7 | Signed-off-by: Richard Henderson <richard.henderson@linaro.org> | 6 | Signed-off-by: Richard Henderson <richard.henderson@linaro.org> |
8 | --- | 7 | --- |
9 | accel/tcg/cputlb.c | 19 ++++++++++--------- | 8 | tcg/riscv/tcg-target.c.inc | 19 +++++++++++++++++-- |
10 | 1 file changed, 10 insertions(+), 9 deletions(-) | 9 | 1 file changed, 17 insertions(+), 2 deletions(-) |
11 | 10 | ||
12 | diff --git a/accel/tcg/cputlb.c b/accel/tcg/cputlb.c | 11 | diff --git a/tcg/riscv/tcg-target.c.inc b/tcg/riscv/tcg-target.c.inc |
13 | index XXXXXXX..XXXXXXX 100644 | 12 | index XXXXXXX..XXXXXXX 100644 |
14 | --- a/accel/tcg/cputlb.c | 13 | --- a/tcg/riscv/tcg-target.c.inc |
15 | +++ b/accel/tcg/cputlb.c | 14 | +++ b/tcg/riscv/tcg-target.c.inc |
16 | @@ -XXX,XX +XXX,XX @@ static void tlb_mmu_resize_locked(CPUTLBDesc *desc, CPUTLBDescFast *fast) | 15 | @@ -XXX,XX +XXX,XX @@ static void tcg_out_exit_tb(TCGContext *s, uintptr_t a0) |
17 | 16 | ||
18 | static void tlb_flush_one_mmuidx_locked(CPUArchState *env, int mmu_idx) | 17 | static void tcg_out_goto_tb(TCGContext *s, int which) |
19 | { | 18 | { |
20 | - tlb_mmu_resize_locked(&env_tlb(env)->d[mmu_idx], &env_tlb(env)->f[mmu_idx]); | 19 | - /* indirect jump method */ |
21 | - env_tlb(env)->d[mmu_idx].n_used_entries = 0; | 20 | + /* Direct branch will be patched by tb_target_set_jmp_target. */ |
22 | - env_tlb(env)->d[mmu_idx].large_page_addr = -1; | 21 | + set_jmp_insn_offset(s, which); |
23 | - env_tlb(env)->d[mmu_idx].large_page_mask = -1; | 22 | + tcg_out32(s, OPC_JAL); |
24 | - env_tlb(env)->d[mmu_idx].vindex = 0; | ||
25 | - memset(env_tlb(env)->f[mmu_idx].table, -1, | ||
26 | - sizeof_tlb(&env_tlb(env)->f[mmu_idx])); | ||
27 | - memset(env_tlb(env)->d[mmu_idx].vtable, -1, | ||
28 | - sizeof(env_tlb(env)->d[0].vtable)); | ||
29 | + CPUTLBDesc *desc = &env_tlb(env)->d[mmu_idx]; | ||
30 | + CPUTLBDescFast *fast = &env_tlb(env)->f[mmu_idx]; | ||
31 | + | 23 | + |
32 | + tlb_mmu_resize_locked(desc, fast); | 24 | + /* When branch is out of range, fall through to indirect. */ |
33 | + desc->n_used_entries = 0; | 25 | tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_TMP0, TCG_REG_ZERO, |
34 | + desc->large_page_addr = -1; | 26 | get_jmp_target_addr(s, which)); |
35 | + desc->large_page_mask = -1; | 27 | tcg_out_opc_imm(s, OPC_JALR, TCG_REG_ZERO, TCG_REG_TMP0, 0); |
36 | + desc->vindex = 0; | 28 | @@ -XXX,XX +XXX,XX @@ static void tcg_out_goto_tb(TCGContext *s, int which) |
37 | + memset(fast->table, -1, sizeof_tlb(fast)); | 29 | void tb_target_set_jmp_target(const TranslationBlock *tb, int n, |
38 | + memset(desc->vtable, -1, sizeof(desc->vtable)); | 30 | uintptr_t jmp_rx, uintptr_t jmp_rw) |
31 | { | ||
32 | - /* Always indirect, nothing to do */ | ||
33 | + uintptr_t addr = tb->jmp_target_addr[n]; | ||
34 | + ptrdiff_t offset = addr - jmp_rx; | ||
35 | + tcg_insn_unit insn; | ||
36 | + | ||
37 | + /* Either directly branch, or fall through to indirect branch. */ | ||
38 | + if (offset == sextreg(offset, 0, 20)) { | ||
39 | + insn = encode_uj(OPC_JAL, TCG_REG_ZERO, offset); | ||
40 | + } else { | ||
41 | + insn = OPC_NOP; | ||
42 | + } | ||
43 | + qatomic_set((uint32_t *)jmp_rw, insn); | ||
44 | + flush_idcache_range(jmp_rx, jmp_rw, 4); | ||
39 | } | 45 | } |
40 | 46 | ||
41 | static inline void tlb_n_used_entries_inc(CPUArchState *env, uintptr_t mmu_idx) | 47 | static void tcg_out_op(TCGContext *s, TCGOpcode opc, |
42 | -- | 48 | -- |
43 | 2.20.1 | 49 | 2.34.1 |
44 | 50 | ||
45 | 51 | diff view generated by jsdifflib |