Enable the machine function split optimization for AutoFDO in Clang.
Machine function split (MFS) is a pass in the Clang compiler that
splits a function into hot and cold parts. The linker groups all
cold blocks across functions together. This decreases hot code
fragmentation and improves iCache and iTLB utilization.
MFS requires a profile so this is enabled only for the AutoFDO builds.
Co-developed-by: Han Shen <shenhan@google.com>
Signed-off-by: Han Shen <shenhan@google.com>
Signed-off-by: Rong Xu <xur@google.com>
Suggested-by: Sriraman Tallam <tmsriram@google.com>
Suggested-by: Krzysztof Pszeniczny <kpszeniczny@google.com>
---
include/asm-generic/vmlinux.lds.h | 6 ++++++
scripts/Makefile.autofdo | 2 ++
2 files changed, 8 insertions(+)
diff --git a/include/asm-generic/vmlinux.lds.h b/include/asm-generic/vmlinux.lds.h
index ace617d1af9b..20e46c0917db 100644
--- a/include/asm-generic/vmlinux.lds.h
+++ b/include/asm-generic/vmlinux.lds.h
@@ -565,9 +565,14 @@ defined(CONFIG_AUTOFDO_CLANG)
__unlikely_text_start = .; \
*(.text.unlikely .text.unlikely.*) \
__unlikely_text_end = .;
+#define TEXT_SPLIT \
+ __split_text_start = .; \
+ *(.text.split .text.split.[0-9a-zA-Z_]*) \
+ __split_text_end = .;
#else
#define TEXT_HOT *(.text.hot .text.hot.*)
#define TEXT_UNLIKELY *(.text.unlikely .text.unlikely.*)
+#define TEXT_SPLIT
#endif
/*
@@ -584,6 +589,7 @@ defined(CONFIG_AUTOFDO_CLANG)
ALIGN_FUNCTION(); \
*(.text.asan.* .text.tsan.*) \
*(.text.unknown .text.unknown.*) \
+ TEXT_SPLIT \
TEXT_UNLIKELY \
. = ALIGN(PAGE_SIZE); \
TEXT_HOT \
diff --git a/scripts/Makefile.autofdo b/scripts/Makefile.autofdo
index 9c9a530ef090..380042a301cc 100644
--- a/scripts/Makefile.autofdo
+++ b/scripts/Makefile.autofdo
@@ -11,6 +11,7 @@ endif
ifdef CLANG_AUTOFDO_PROFILE
CFLAGS_AUTOFDO_CLANG += -fprofile-sample-use=$(CLANG_AUTOFDO_PROFILE) -ffunction-sections
+ CFLAGS_AUTOFDO_CLANG += -fsplit-machine-functions
endif
ifdef CONFIG_LTO_CLANG_THIN
@@ -18,6 +19,7 @@ ifdef CONFIG_LTO_CLANG_THIN
KBUILD_LDFLAGS += --lto-sample-profile=$(CLANG_AUTOFDO_PROFILE)
endif
KBUILD_LDFLAGS += --mllvm=-enable-fs-discriminator=true --mllvm=-improved-fs-discriminator=true -plugin-opt=thinlto
+ KBUILD_LDFLAGS += -plugin-opt=-split-machine-functions
endif
export CFLAGS_AUTOFDO_CLANG
--
2.47.0.rc1.288.g06298d1525-googOn Tue, Oct 15, 2024 at 6:33 AM Rong Xu <xur@google.com> wrote: > > Enable the machine function split optimization for AutoFDO in Clang. > > Machine function split (MFS) is a pass in the Clang compiler that > splits a function into hot and cold parts. The linker groups all > cold blocks across functions together. This decreases hot code > fragmentation and improves iCache and iTLB utilization. > > MFS requires a profile so this is enabled only for the AutoFDO builds. > > Co-developed-by: Han Shen <shenhan@google.com> > Signed-off-by: Han Shen <shenhan@google.com> > Signed-off-by: Rong Xu <xur@google.com> > Suggested-by: Sriraman Tallam <tmsriram@google.com> > Suggested-by: Krzysztof Pszeniczny <kpszeniczny@google.com> > --- > include/asm-generic/vmlinux.lds.h | 6 ++++++ > scripts/Makefile.autofdo | 2 ++ > 2 files changed, 8 insertions(+) > > diff --git a/include/asm-generic/vmlinux.lds.h b/include/asm-generic/vmlinux.lds.h > index ace617d1af9b..20e46c0917db 100644 > --- a/include/asm-generic/vmlinux.lds.h > +++ b/include/asm-generic/vmlinux.lds.h > @@ -565,9 +565,14 @@ defined(CONFIG_AUTOFDO_CLANG) > __unlikely_text_start = .; \ > *(.text.unlikely .text.unlikely.*) \ > __unlikely_text_end = .; > +#define TEXT_SPLIT \ > + __split_text_start = .; \ > + *(.text.split .text.split.[0-9a-zA-Z_]*) \ > + __split_text_end = .; > #else > #define TEXT_HOT *(.text.hot .text.hot.*) > #define TEXT_UNLIKELY *(.text.unlikely .text.unlikely.*) > +#define TEXT_SPLIT > #endif Why conditional? Where are __unlikely_text_start and __unlikely_text_end used? -- Best Regards Masahiro Yamada
On Sun, Oct 20, 2024 at 8:18 PM Masahiro Yamada <masahiroy@kernel.org> wrote: > > On Tue, Oct 15, 2024 at 6:33 AM Rong Xu <xur@google.com> wrote: > > > > Enable the machine function split optimization for AutoFDO in Clang. > > > > Machine function split (MFS) is a pass in the Clang compiler that > > splits a function into hot and cold parts. The linker groups all > > cold blocks across functions together. This decreases hot code > > fragmentation and improves iCache and iTLB utilization. > > > > MFS requires a profile so this is enabled only for the AutoFDO builds. > > > > Co-developed-by: Han Shen <shenhan@google.com> > > Signed-off-by: Han Shen <shenhan@google.com> > > Signed-off-by: Rong Xu <xur@google.com> > > Suggested-by: Sriraman Tallam <tmsriram@google.com> > > Suggested-by: Krzysztof Pszeniczny <kpszeniczny@google.com> > > --- > > include/asm-generic/vmlinux.lds.h | 6 ++++++ > > scripts/Makefile.autofdo | 2 ++ > > 2 files changed, 8 insertions(+) > > > > diff --git a/include/asm-generic/vmlinux.lds.h b/include/asm-generic/vmlinux.lds.h > > index ace617d1af9b..20e46c0917db 100644 > > --- a/include/asm-generic/vmlinux.lds.h > > +++ b/include/asm-generic/vmlinux.lds.h > > @@ -565,9 +565,14 @@ defined(CONFIG_AUTOFDO_CLANG) > > __unlikely_text_start = .; \ > > *(.text.unlikely .text.unlikely.*) \ > > __unlikely_text_end = .; > > +#define TEXT_SPLIT \ > > + __split_text_start = .; \ > > + *(.text.split .text.split.[0-9a-zA-Z_]*) \ > > + __split_text_end = .; > > #else > > #define TEXT_HOT *(.text.hot .text.hot.*) > > #define TEXT_UNLIKELY *(.text.unlikely .text.unlikely.*) > > +#define TEXT_SPLIT > > #endif > > > Why conditional? The condition is to ensure that we don't change the default kernel build by any means. The new code will introduce a few new symbols. > > > Where are __unlikely_text_start and __unlikely_text_end used? These new symbols are currently unreferenced within the kernel source tree. However, they provide a valuable means of identifying hot and cold sections of text, and how large they are. I think they are useful information. > > > > > > > > -- > Best Regards > Masahiro Yamada
On Tue, Oct 22, 2024 at 8:28 AM Rong Xu <xur@google.com> wrote: > > On Sun, Oct 20, 2024 at 8:18 PM Masahiro Yamada <masahiroy@kernel.org> wrote: > > > > On Tue, Oct 15, 2024 at 6:33 AM Rong Xu <xur@google.com> wrote: > > > > > > Enable the machine function split optimization for AutoFDO in Clang. > > > > > > Machine function split (MFS) is a pass in the Clang compiler that > > > splits a function into hot and cold parts. The linker groups all > > > cold blocks across functions together. This decreases hot code > > > fragmentation and improves iCache and iTLB utilization. > > > > > > MFS requires a profile so this is enabled only for the AutoFDO builds. > > > > > > Co-developed-by: Han Shen <shenhan@google.com> > > > Signed-off-by: Han Shen <shenhan@google.com> > > > Signed-off-by: Rong Xu <xur@google.com> > > > Suggested-by: Sriraman Tallam <tmsriram@google.com> > > > Suggested-by: Krzysztof Pszeniczny <kpszeniczny@google.com> > > > --- > > > include/asm-generic/vmlinux.lds.h | 6 ++++++ > > > scripts/Makefile.autofdo | 2 ++ > > > 2 files changed, 8 insertions(+) > > > > > > diff --git a/include/asm-generic/vmlinux.lds.h b/include/asm-generic/vmlinux.lds.h > > > index ace617d1af9b..20e46c0917db 100644 > > > --- a/include/asm-generic/vmlinux.lds.h > > > +++ b/include/asm-generic/vmlinux.lds.h > > > @@ -565,9 +565,14 @@ defined(CONFIG_AUTOFDO_CLANG) > > > __unlikely_text_start = .; \ > > > *(.text.unlikely .text.unlikely.*) \ > > > __unlikely_text_end = .; > > > +#define TEXT_SPLIT \ > > > + __split_text_start = .; \ > > > + *(.text.split .text.split.[0-9a-zA-Z_]*) \ > > > + __split_text_end = .; > > > #else > > > #define TEXT_HOT *(.text.hot .text.hot.*) > > > #define TEXT_UNLIKELY *(.text.unlikely .text.unlikely.*) > > > +#define TEXT_SPLIT > > > #endif > > > > > > Why conditional? > > The condition is to ensure that we don't change the default kernel > build by any means. > The new code will introduce a few new symbols. Same. Adding two __split_text_start and __split_text_end markers do not affect anything. It just increases the kallsyms table slightly. You can do it unconditionally. > > > > > > > Where are __unlikely_text_start and __unlikely_text_end used? > > These new symbols are currently unreferenced within the kernel source tree. > However, they provide a valuable means of identifying hot and cold > sections of text, > and how large they are. I think they are useful information. Should be explained in the commit description. -- Best Regards Masahiro Yamada
On Tue, Oct 22, 2024 at 11:50 PM Masahiro Yamada <masahiroy@kernel.org> wrote: > > On Tue, Oct 22, 2024 at 8:28 AM Rong Xu <xur@google.com> wrote: > > > > On Sun, Oct 20, 2024 at 8:18 PM Masahiro Yamada <masahiroy@kernel.org> wrote: > > > > > > On Tue, Oct 15, 2024 at 6:33 AM Rong Xu <xur@google.com> wrote: > > > > > > > > Enable the machine function split optimization for AutoFDO in Clang. > > > > > > > > Machine function split (MFS) is a pass in the Clang compiler that > > > > splits a function into hot and cold parts. The linker groups all > > > > cold blocks across functions together. This decreases hot code > > > > fragmentation and improves iCache and iTLB utilization. > > > > > > > > MFS requires a profile so this is enabled only for the AutoFDO builds. > > > > > > > > Co-developed-by: Han Shen <shenhan@google.com> > > > > Signed-off-by: Han Shen <shenhan@google.com> > > > > Signed-off-by: Rong Xu <xur@google.com> > > > > Suggested-by: Sriraman Tallam <tmsriram@google.com> > > > > Suggested-by: Krzysztof Pszeniczny <kpszeniczny@google.com> > > > > --- > > > > include/asm-generic/vmlinux.lds.h | 6 ++++++ > > > > scripts/Makefile.autofdo | 2 ++ > > > > 2 files changed, 8 insertions(+) > > > > > > > > diff --git a/include/asm-generic/vmlinux.lds.h b/include/asm-generic/vmlinux.lds.h > > > > index ace617d1af9b..20e46c0917db 100644 > > > > --- a/include/asm-generic/vmlinux.lds.h > > > > +++ b/include/asm-generic/vmlinux.lds.h > > > > @@ -565,9 +565,14 @@ defined(CONFIG_AUTOFDO_CLANG) > > > > __unlikely_text_start = .; \ > > > > *(.text.unlikely .text.unlikely.*) \ > > > > __unlikely_text_end = .; > > > > +#define TEXT_SPLIT \ > > > > + __split_text_start = .; \ > > > > + *(.text.split .text.split.[0-9a-zA-Z_]*) \ > > > > + __split_text_end = .; > > > > #else > > > > #define TEXT_HOT *(.text.hot .text.hot.*) > > > > #define TEXT_UNLIKELY *(.text.unlikely .text.unlikely.*) > > > > +#define TEXT_SPLIT > > > > #endif > > > > > > > > > Why conditional? > > > > The condition is to ensure that we don't change the default kernel > > build by any means. > > The new code will introduce a few new symbols. > > > Same. > > Adding two __split_text_start and __split_text_end markers > do not affect anything. It just increases the kallsyms table slightly. > > You can do it unconditionally. Got it. > > > > > > > > > > > > > > Where are __unlikely_text_start and __unlikely_text_end used? > > > > These new symbols are currently unreferenced within the kernel source tree. > > However, they provide a valuable means of identifying hot and cold > > sections of text, > > and how large they are. I think they are useful information. > > > Should be explained in the commit description. Will explain the commit message. > > > > -- > Best Regards > Masahiro Yamada
© 2016 - 2024 Red Hat, Inc.