Compilers estimate the size of an asm() block for inlining purposes.
Constructs with embedded metadata (BUG_FRAME, ALTERNATIVE, EXTABLE, etc)
appear large, depsite often only being a handful of instructions. asm
inline() overrides the estimation to identify the block as being small.
This has a substantial impact on inlining decisions, expected to be for the
better given that the compiler has a more accurate picture to work with.
No functional change.
Signed-off-by: Andrew Cooper <andrew.cooper3@citrix.com>
---
CC: Jan Beulich <JBeulich@suse.com>
CC: Roger Pau Monné <roger.pau@citrix.com>
CC: Stefano Stabellini <sstabellini@kernel.org>
CC: Julien Grall <julien@xen.org>
CC: Volodymyr Babchuk <Volodymyr_Babchuk@epam.com>
CC: Bertrand Marquis <bertrand.marquis@arm.com>
CC: Michal Orzel <michal.orzel@amd.com>
v2:
* Split into multiple patches
* Start with BUG().
The full bloat-o-meter for this on x86 is https://termbin.com/27we although
the saving is better than reported.
Note the pairs such as:
vmx_update_secondary_exec_control.part 2 - -2
vmx_update_secondary_exec_control 60 57 -3
This is becuse the UD2 was out-of-lined, and was CALL'd. When inlined, the 5
byte CALL instruction in is replace with the 2 byte UD2. Further than
reported, we save another 14 bytes due to the 16 byte function alignment.
This undoes an unanticipated side effect of starting to use asm goto().
---
xen/Kconfig | 4 ++++
xen/arch/arm/include/asm/bug.h | 6 ++++--
xen/include/xen/bug.h | 11 ++++++-----
xen/include/xen/compiler.h | 15 +++++++++++++++
4 files changed, 29 insertions(+), 7 deletions(-)
diff --git a/xen/Kconfig b/xen/Kconfig
index 1b24e8f3c0cd..07c4accf881c 100644
--- a/xen/Kconfig
+++ b/xen/Kconfig
@@ -29,6 +29,10 @@ config LD_IS_GNU
config LD_IS_LLVM
def_bool $(success,$(LD) --version | head -n 1 | grep -q "^LLD")
+config CC_HAS_ASM_INLINE
+ # GCC >= 9, Clang >= 11
+ def_bool $(success,echo 'void foo(void) { asm inline (""); }' | $(CC) -x c - -c -o /dev/null)
+
# Use -f{function,data}-sections compiler parameters
config CC_SPLIT_SECTIONS
bool
diff --git a/xen/arch/arm/include/asm/bug.h b/xen/arch/arm/include/asm/bug.h
index 8bf71587bea1..0f436df63f26 100644
--- a/xen/arch/arm/include/asm/bug.h
+++ b/xen/arch/arm/include/asm/bug.h
@@ -34,7 +34,8 @@ struct bug_frame {
#define BUG_FRAME(type, line, file, has_msg, msg) do { \
BUILD_BUG_ON((line) >> 16); \
BUILD_BUG_ON((type) >= BUGFRAME_NR); \
- asm ("1:"BUG_INSTR"\n" \
+ asm_inline ( \
+ "1:"BUG_INSTR"\n" \
".pushsection .rodata.str, \"aMS\", %progbits, 1\n" \
"2:\t.asciz " __stringify(file) "\n" \
"3:\n" \
@@ -60,7 +61,8 @@ struct bug_frame {
*/
#define run_in_exception_handler(fn) do { \
register unsigned long _fn asm (STR(BUG_FN_REG)) = (unsigned long)(fn); \
- asm ("1:"BUG_INSTR"\n" \
+ asm_inline ( \
+ "1:"BUG_INSTR"\n" \
".pushsection .bug_frames." __stringify(BUGFRAME_run_fn) "," \
" \"a\", %%progbits\n" \
"2:\n" \
diff --git a/xen/include/xen/bug.h b/xen/include/xen/bug.h
index 99814c4bef36..0cabdba37992 100644
--- a/xen/include/xen/bug.h
+++ b/xen/include/xen/bug.h
@@ -89,11 +89,12 @@ struct bug_frame {
#ifndef BUG_FRAME
#define BUG_FRAME(type, line, ptr, second_frame, msg) do { \
- BUG_CHECK_LINE_WIDTH(line); \
- BUILD_BUG_ON((type) >= BUGFRAME_NR); \
- asm volatile ( _ASM_BUGFRAME_TEXT(second_frame) \
- :: _ASM_BUGFRAME_INFO(type, line, ptr, msg) ); \
-} while ( false )
+ BUG_CHECK_LINE_WIDTH(line); \
+ BUILD_BUG_ON((type) >= BUGFRAME_NR); \
+ asm_inline volatile ( \
+ _ASM_BUGFRAME_TEXT(second_frame) \
+ :: _ASM_BUGFRAME_INFO(type, line, ptr, msg) ); \
+ } while ( false )
#endif
diff --git a/xen/include/xen/compiler.h b/xen/include/xen/compiler.h
index c68fab189154..735c844d2d15 100644
--- a/xen/include/xen/compiler.h
+++ b/xen/include/xen/compiler.h
@@ -53,6 +53,21 @@
#define unreachable() __builtin_unreachable()
#endif
+/*
+ * Compilers estimate the size of an asm() block for inlining purposes.
+ *
+ * Constructs with embedded metadata (BUG_FRAME, ALTERNATIVE, EXTABLE, etc)
+ * appear large, depsite typically only being a handful of instructions. asm
+ * inline() overrides the estimation to identify the block as being small.
+ *
+ * Note: __inline is needed to avoid getting caught up in INIT_SECTIONS_ONLY.
+ */
+#if CONFIG_CC_HAS_ASM_INLINE
+# define asm_inline asm __inline
+#else
+# define asm_inline asm
+#endif
+
/*
* Add the pseudo keyword 'fallthrough' so case statement blocks
* must end with any of these keywords:
--
2.39.5
On 15.05.2025 21:55, Andrew Cooper wrote: > Compilers estimate the size of an asm() block for inlining purposes. > > Constructs with embedded metadata (BUG_FRAME, ALTERNATIVE, EXTABLE, etc) > appear large, depsite often only being a handful of instructions. asm > inline() overrides the estimation to identify the block as being small. > > This has a substantial impact on inlining decisions, expected to be for the > better given that the compiler has a more accurate picture to work with. > > No functional change. > > Signed-off-by: Andrew Cooper <andrew.cooper3@citrix.com> Reviewed-by: Jan Beulich <jbeulich@suse.com>
On Fri, 16 May 2025, Jan Beulich wrote: > On 15.05.2025 21:55, Andrew Cooper wrote: > > Compilers estimate the size of an asm() block for inlining purposes. > > > > Constructs with embedded metadata (BUG_FRAME, ALTERNATIVE, EXTABLE, etc) > > appear large, depsite often only being a handful of instructions. asm > > inline() overrides the estimation to identify the block as being small. > > > > This has a substantial impact on inlining decisions, expected to be for the > > better given that the compiler has a more accurate picture to work with. > > > > No functional change. > > > > Signed-off-by: Andrew Cooper <andrew.cooper3@citrix.com> > > Reviewed-by: Jan Beulich <jbeulich@suse.com> arm: Acked-by: Stefano Stabellini <sstabellini@kernel.org>
© 2016 - 2025 Red Hat, Inc.