The percpu section is currently linked at virtual address 0, because
older compilers hardcoded the stack protector canary value at a fixed
offset from the start of the GS segment. Now that the canary is a
normal percpu variable, the percpu section does not need to be linked
at a specific virtual address. This means that x86-64 will calculate
percpu offsets like most other architectures, as the delta between the
initial percpu address and the dynamically allocated memory.
Signed-off-by: Brian Gerst <brgerst@gmail.com>
---
arch/x86/include/asm/processor.h | 6 +++++-
arch/x86/kernel/head_64.S | 19 +++++++++----------
arch/x86/kernel/setup_percpu.c | 12 ++----------
arch/x86/kernel/vmlinux.lds.S | 29 +----------------------------
arch/x86/platform/pvh/head.S | 5 ++---
arch/x86/tools/relocs.c | 10 +++-------
arch/x86/xen/xen-head.S | 9 ++++-----
init/Kconfig | 2 +-
8 files changed, 27 insertions(+), 65 deletions(-)
diff --git a/arch/x86/include/asm/processor.h b/arch/x86/include/asm/processor.h
index 946bebce396f..40d6add8ff31 100644
--- a/arch/x86/include/asm/processor.h
+++ b/arch/x86/include/asm/processor.h
@@ -396,7 +396,11 @@ DECLARE_INIT_PER_CPU(fixed_percpu_data);
static inline unsigned long cpu_kernelmode_gs_base(int cpu)
{
- return (unsigned long)per_cpu(fixed_percpu_data.gs_base, cpu);
+#ifdef CONFIG_SMP
+ return per_cpu_offset(cpu);
+#else
+ return 0;
+#endif
}
extern asmlinkage void entry_SYSCALL32_ignore(void);
diff --git a/arch/x86/kernel/head_64.S b/arch/x86/kernel/head_64.S
index cfbf0486d424..5b2cc711feec 100644
--- a/arch/x86/kernel/head_64.S
+++ b/arch/x86/kernel/head_64.S
@@ -68,11 +68,14 @@ SYM_CODE_START_NOALIGN(startup_64)
/* Set up the stack for verify_cpu() */
leaq __top_init_kernel_stack(%rip), %rsp
- /* Setup GSBASE to allow stack canary access for C code */
+ /*
+ * Set up GSBASE.
+ * Note that, on SMP, the boot cpu uses init data section until
+ * the per cpu areas are set up.
+ */
movl $MSR_GS_BASE, %ecx
- leaq INIT_PER_CPU_VAR(fixed_percpu_data)(%rip), %rdx
- movl %edx, %eax
- shrq $32, %rdx
+ xorl %eax, %eax
+ xorl %edx, %edx
wrmsr
call startup_64_setup_gdt_idt
@@ -359,16 +362,12 @@ SYM_INNER_LABEL(common_startup_64, SYM_L_LOCAL)
movl %eax,%fs
movl %eax,%gs
- /* Set up %gs.
- *
- * The base of %gs always points to fixed_percpu_data.
+ /*
+ * Set up GSBASE.
* Note that, on SMP, the boot cpu uses init data section until
* the per cpu areas are set up.
*/
movl $MSR_GS_BASE,%ecx
-#ifndef CONFIG_SMP
- leaq INIT_PER_CPU_VAR(fixed_percpu_data)(%rip), %rdx
-#endif
movl %edx, %eax
shrq $32, %rdx
wrmsr
diff --git a/arch/x86/kernel/setup_percpu.c b/arch/x86/kernel/setup_percpu.c
index b30d6e180df7..1e7be9409aa2 100644
--- a/arch/x86/kernel/setup_percpu.c
+++ b/arch/x86/kernel/setup_percpu.c
@@ -23,18 +23,10 @@
#include <asm/cpumask.h>
#include <asm/cpu.h>
-#ifdef CONFIG_X86_64
-#define BOOT_PERCPU_OFFSET ((unsigned long)__per_cpu_load)
-#else
-#define BOOT_PERCPU_OFFSET 0
-#endif
-
-DEFINE_PER_CPU_READ_MOSTLY(unsigned long, this_cpu_off) = BOOT_PERCPU_OFFSET;
+DEFINE_PER_CPU_READ_MOSTLY(unsigned long, this_cpu_off);
EXPORT_PER_CPU_SYMBOL(this_cpu_off);
-unsigned long __per_cpu_offset[NR_CPUS] __ro_after_init = {
- [0 ... NR_CPUS-1] = BOOT_PERCPU_OFFSET,
-};
+unsigned long __per_cpu_offset[NR_CPUS] __ro_after_init;
EXPORT_SYMBOL(__per_cpu_offset);
/*
diff --git a/arch/x86/kernel/vmlinux.lds.S b/arch/x86/kernel/vmlinux.lds.S
index 3509afc6a672..0b152f96c24e 100644
--- a/arch/x86/kernel/vmlinux.lds.S
+++ b/arch/x86/kernel/vmlinux.lds.S
@@ -99,12 +99,6 @@ const_pcpu_hot = pcpu_hot;
PHDRS {
text PT_LOAD FLAGS(5); /* R_E */
data PT_LOAD FLAGS(6); /* RW_ */
-#ifdef CONFIG_X86_64
-#ifdef CONFIG_SMP
- percpu PT_LOAD FLAGS(6); /* RW_ */
-#endif
- init PT_LOAD FLAGS(7); /* RWE */
-#endif
note PT_NOTE FLAGS(0); /* ___ */
}
@@ -222,21 +216,7 @@ SECTIONS
__init_begin = .; /* paired with __init_end */
}
-#if defined(CONFIG_X86_64) && defined(CONFIG_SMP)
- /*
- * percpu offsets are zero-based on SMP. PERCPU_VADDR() changes the
- * output PHDR, so the next output section - .init.text - should
- * start another segment - init.
- */
- PERCPU_VADDR(INTERNODE_CACHE_BYTES, 0, :percpu)
- ASSERT(SIZEOF(.data..percpu) < CONFIG_PHYSICAL_START,
- "per-CPU data too large - increase CONFIG_PHYSICAL_START")
-#endif
-
INIT_TEXT_SECTION(PAGE_SIZE)
-#ifdef CONFIG_X86_64
- :init
-#endif
/*
* Section for code used exclusively before alternatives are run. All
@@ -353,9 +333,7 @@ SECTIONS
EXIT_DATA
}
-#if !defined(CONFIG_X86_64) || !defined(CONFIG_SMP)
PERCPU_SECTION(INTERNODE_CACHE_BYTES)
-#endif
. = ALIGN(PAGE_SIZE);
@@ -493,16 +471,11 @@ SECTIONS
* Per-cpu symbols which need to be offset from __per_cpu_load
* for the boot processor.
*/
-#define INIT_PER_CPU(x) init_per_cpu__##x = ABSOLUTE(x) + __per_cpu_load
+#define INIT_PER_CPU(x) init_per_cpu__##x = ABSOLUTE(x)
INIT_PER_CPU(gdt_page);
INIT_PER_CPU(fixed_percpu_data);
INIT_PER_CPU(irq_stack_backing_store);
-#ifdef CONFIG_SMP
-. = ASSERT((fixed_percpu_data == 0),
- "fixed_percpu_data is not at start of per-cpu area");
-#endif
-
#ifdef CONFIG_MITIGATION_UNRET_ENTRY
. = ASSERT((retbleed_return_thunk & 0x3f) == 0, "retbleed_return_thunk not cacheline-aligned");
#endif
diff --git a/arch/x86/platform/pvh/head.S b/arch/x86/platform/pvh/head.S
index 1f1c3230b27b..7e3e07c6170f 100644
--- a/arch/x86/platform/pvh/head.S
+++ b/arch/x86/platform/pvh/head.S
@@ -101,9 +101,8 @@ SYM_CODE_START_LOCAL(pvh_start_xen)
* the per cpu areas are set up.
*/
mov $MSR_GS_BASE,%ecx
- lea INIT_PER_CPU_VAR(fixed_percpu_data)(%rip), %rdx
- mov %edx, %eax
- shr $32, %rdx
+ xor %eax, %eax
+ xor %edx, %edx
wrmsr
call xen_prepare_pvh
diff --git a/arch/x86/tools/relocs.c b/arch/x86/tools/relocs.c
index adf11a48ec70..016650ddaf7f 100644
--- a/arch/x86/tools/relocs.c
+++ b/arch/x86/tools/relocs.c
@@ -839,12 +839,7 @@ static void percpu_init(void)
*/
static int is_percpu_sym(ElfW(Sym) *sym, const char *symname)
{
- int shndx = sym_index(sym);
-
- return (shndx == per_cpu_shndx) &&
- strcmp(symname, "__init_begin") &&
- strcmp(symname, "__per_cpu_load") &&
- strncmp(symname, "init_per_cpu_", 13);
+ return 0;
}
@@ -1068,7 +1063,8 @@ static int cmp_relocs(const void *va, const void *vb)
static void sort_relocs(struct relocs *r)
{
- qsort(r->offset, r->count, sizeof(r->offset[0]), cmp_relocs);
+ if (r->count)
+ qsort(r->offset, r->count, sizeof(r->offset[0]), cmp_relocs);
}
static int write32(uint32_t v, FILE *f)
diff --git a/arch/x86/xen/xen-head.S b/arch/x86/xen/xen-head.S
index ae4672ea00bb..1796884b727d 100644
--- a/arch/x86/xen/xen-head.S
+++ b/arch/x86/xen/xen-head.S
@@ -51,15 +51,14 @@ SYM_CODE_START(startup_xen)
leaq __top_init_kernel_stack(%rip), %rsp
- /* Set up %gs.
- *
- * The base of %gs always points to fixed_percpu_data.
+ /*
+ * Set up GSBASE.
* Note that, on SMP, the boot cpu uses init data section until
* the per cpu areas are set up.
*/
movl $MSR_GS_BASE,%ecx
- movq $INIT_PER_CPU_VAR(fixed_percpu_data),%rax
- cdq
+ xorl %eax, %eax
+ xorl %edx, %edx
wrmsr
mov %rsi, %rdi
diff --git a/init/Kconfig b/init/Kconfig
index f3ea5dea9c85..0f928f82dc7a 100644
--- a/init/Kconfig
+++ b/init/Kconfig
@@ -1784,7 +1784,7 @@ config KALLSYMS_ALL
config KALLSYMS_ABSOLUTE_PERCPU
bool
depends on KALLSYMS
- default X86_64 && SMP
+ default n
config KALLSYMS_BASE_RELATIVE
bool
--
2.44.0On Fri, Mar 22, 2024 at 5:52 PM Brian Gerst <brgerst@gmail.com> wrote:
>
> The percpu section is currently linked at virtual address 0, because
> older compilers hardcoded the stack protector canary value at a fixed
> offset from the start of the GS segment. Now that the canary is a
> normal percpu variable, the percpu section does not need to be linked
> at a specific virtual address. This means that x86-64 will calculate
> percpu offsets like most other architectures, as the delta between the
> initial percpu address and the dynamically allocated memory.
>
> Signed-off-by: Brian Gerst <brgerst@gmail.com>
Reviewed-by: Uros Bizjak <ubizjak@gmail.com>
> ---
> arch/x86/include/asm/processor.h | 6 +++++-
> arch/x86/kernel/head_64.S | 19 +++++++++----------
> arch/x86/kernel/setup_percpu.c | 12 ++----------
> arch/x86/kernel/vmlinux.lds.S | 29 +----------------------------
> arch/x86/platform/pvh/head.S | 5 ++---
> arch/x86/tools/relocs.c | 10 +++-------
> arch/x86/xen/xen-head.S | 9 ++++-----
> init/Kconfig | 2 +-
> 8 files changed, 27 insertions(+), 65 deletions(-)
>
> diff --git a/arch/x86/include/asm/processor.h b/arch/x86/include/asm/processor.h
> index 946bebce396f..40d6add8ff31 100644
> --- a/arch/x86/include/asm/processor.h
> +++ b/arch/x86/include/asm/processor.h
> @@ -396,7 +396,11 @@ DECLARE_INIT_PER_CPU(fixed_percpu_data);
>
> static inline unsigned long cpu_kernelmode_gs_base(int cpu)
> {
> - return (unsigned long)per_cpu(fixed_percpu_data.gs_base, cpu);
> +#ifdef CONFIG_SMP
> + return per_cpu_offset(cpu);
> +#else
> + return 0;
> +#endif
> }
>
> extern asmlinkage void entry_SYSCALL32_ignore(void);
> diff --git a/arch/x86/kernel/head_64.S b/arch/x86/kernel/head_64.S
> index cfbf0486d424..5b2cc711feec 100644
> --- a/arch/x86/kernel/head_64.S
> +++ b/arch/x86/kernel/head_64.S
> @@ -68,11 +68,14 @@ SYM_CODE_START_NOALIGN(startup_64)
> /* Set up the stack for verify_cpu() */
> leaq __top_init_kernel_stack(%rip), %rsp
>
> - /* Setup GSBASE to allow stack canary access for C code */
> + /*
> + * Set up GSBASE.
> + * Note that, on SMP, the boot cpu uses init data section until
> + * the per cpu areas are set up.
> + */
> movl $MSR_GS_BASE, %ecx
> - leaq INIT_PER_CPU_VAR(fixed_percpu_data)(%rip), %rdx
> - movl %edx, %eax
> - shrq $32, %rdx
> + xorl %eax, %eax
> + xorl %edx, %edx
> wrmsr
>
> call startup_64_setup_gdt_idt
> @@ -359,16 +362,12 @@ SYM_INNER_LABEL(common_startup_64, SYM_L_LOCAL)
> movl %eax,%fs
> movl %eax,%gs
>
> - /* Set up %gs.
> - *
> - * The base of %gs always points to fixed_percpu_data.
> + /*
> + * Set up GSBASE.
> * Note that, on SMP, the boot cpu uses init data section until
> * the per cpu areas are set up.
> */
> movl $MSR_GS_BASE,%ecx
> -#ifndef CONFIG_SMP
> - leaq INIT_PER_CPU_VAR(fixed_percpu_data)(%rip), %rdx
> -#endif
> movl %edx, %eax
> shrq $32, %rdx
> wrmsr
> diff --git a/arch/x86/kernel/setup_percpu.c b/arch/x86/kernel/setup_percpu.c
> index b30d6e180df7..1e7be9409aa2 100644
> --- a/arch/x86/kernel/setup_percpu.c
> +++ b/arch/x86/kernel/setup_percpu.c
> @@ -23,18 +23,10 @@
> #include <asm/cpumask.h>
> #include <asm/cpu.h>
>
> -#ifdef CONFIG_X86_64
> -#define BOOT_PERCPU_OFFSET ((unsigned long)__per_cpu_load)
> -#else
> -#define BOOT_PERCPU_OFFSET 0
> -#endif
> -
> -DEFINE_PER_CPU_READ_MOSTLY(unsigned long, this_cpu_off) = BOOT_PERCPU_OFFSET;
> +DEFINE_PER_CPU_READ_MOSTLY(unsigned long, this_cpu_off);
> EXPORT_PER_CPU_SYMBOL(this_cpu_off);
>
> -unsigned long __per_cpu_offset[NR_CPUS] __ro_after_init = {
> - [0 ... NR_CPUS-1] = BOOT_PERCPU_OFFSET,
> -};
> +unsigned long __per_cpu_offset[NR_CPUS] __ro_after_init;
> EXPORT_SYMBOL(__per_cpu_offset);
>
> /*
> diff --git a/arch/x86/kernel/vmlinux.lds.S b/arch/x86/kernel/vmlinux.lds.S
> index 3509afc6a672..0b152f96c24e 100644
> --- a/arch/x86/kernel/vmlinux.lds.S
> +++ b/arch/x86/kernel/vmlinux.lds.S
> @@ -99,12 +99,6 @@ const_pcpu_hot = pcpu_hot;
> PHDRS {
> text PT_LOAD FLAGS(5); /* R_E */
> data PT_LOAD FLAGS(6); /* RW_ */
> -#ifdef CONFIG_X86_64
> -#ifdef CONFIG_SMP
> - percpu PT_LOAD FLAGS(6); /* RW_ */
> -#endif
> - init PT_LOAD FLAGS(7); /* RWE */
> -#endif
> note PT_NOTE FLAGS(0); /* ___ */
> }
>
> @@ -222,21 +216,7 @@ SECTIONS
> __init_begin = .; /* paired with __init_end */
> }
>
> -#if defined(CONFIG_X86_64) && defined(CONFIG_SMP)
> - /*
> - * percpu offsets are zero-based on SMP. PERCPU_VADDR() changes the
> - * output PHDR, so the next output section - .init.text - should
> - * start another segment - init.
> - */
> - PERCPU_VADDR(INTERNODE_CACHE_BYTES, 0, :percpu)
> - ASSERT(SIZEOF(.data..percpu) < CONFIG_PHYSICAL_START,
> - "per-CPU data too large - increase CONFIG_PHYSICAL_START")
> -#endif
> -
> INIT_TEXT_SECTION(PAGE_SIZE)
> -#ifdef CONFIG_X86_64
> - :init
> -#endif
>
> /*
> * Section for code used exclusively before alternatives are run. All
> @@ -353,9 +333,7 @@ SECTIONS
> EXIT_DATA
> }
>
> -#if !defined(CONFIG_X86_64) || !defined(CONFIG_SMP)
> PERCPU_SECTION(INTERNODE_CACHE_BYTES)
> -#endif
>
> . = ALIGN(PAGE_SIZE);
>
> @@ -493,16 +471,11 @@ SECTIONS
> * Per-cpu symbols which need to be offset from __per_cpu_load
> * for the boot processor.
> */
> -#define INIT_PER_CPU(x) init_per_cpu__##x = ABSOLUTE(x) + __per_cpu_load
> +#define INIT_PER_CPU(x) init_per_cpu__##x = ABSOLUTE(x)
> INIT_PER_CPU(gdt_page);
> INIT_PER_CPU(fixed_percpu_data);
> INIT_PER_CPU(irq_stack_backing_store);
>
> -#ifdef CONFIG_SMP
> -. = ASSERT((fixed_percpu_data == 0),
> - "fixed_percpu_data is not at start of per-cpu area");
> -#endif
> -
> #ifdef CONFIG_MITIGATION_UNRET_ENTRY
> . = ASSERT((retbleed_return_thunk & 0x3f) == 0, "retbleed_return_thunk not cacheline-aligned");
> #endif
> diff --git a/arch/x86/platform/pvh/head.S b/arch/x86/platform/pvh/head.S
> index 1f1c3230b27b..7e3e07c6170f 100644
> --- a/arch/x86/platform/pvh/head.S
> +++ b/arch/x86/platform/pvh/head.S
> @@ -101,9 +101,8 @@ SYM_CODE_START_LOCAL(pvh_start_xen)
> * the per cpu areas are set up.
> */
> mov $MSR_GS_BASE,%ecx
> - lea INIT_PER_CPU_VAR(fixed_percpu_data)(%rip), %rdx
> - mov %edx, %eax
> - shr $32, %rdx
> + xor %eax, %eax
> + xor %edx, %edx
> wrmsr
>
> call xen_prepare_pvh
> diff --git a/arch/x86/tools/relocs.c b/arch/x86/tools/relocs.c
> index adf11a48ec70..016650ddaf7f 100644
> --- a/arch/x86/tools/relocs.c
> +++ b/arch/x86/tools/relocs.c
> @@ -839,12 +839,7 @@ static void percpu_init(void)
> */
> static int is_percpu_sym(ElfW(Sym) *sym, const char *symname)
> {
> - int shndx = sym_index(sym);
> -
> - return (shndx == per_cpu_shndx) &&
> - strcmp(symname, "__init_begin") &&
> - strcmp(symname, "__per_cpu_load") &&
> - strncmp(symname, "init_per_cpu_", 13);
> + return 0;
> }
>
>
> @@ -1068,7 +1063,8 @@ static int cmp_relocs(const void *va, const void *vb)
>
> static void sort_relocs(struct relocs *r)
> {
> - qsort(r->offset, r->count, sizeof(r->offset[0]), cmp_relocs);
> + if (r->count)
> + qsort(r->offset, r->count, sizeof(r->offset[0]), cmp_relocs);
> }
>
> static int write32(uint32_t v, FILE *f)
> diff --git a/arch/x86/xen/xen-head.S b/arch/x86/xen/xen-head.S
> index ae4672ea00bb..1796884b727d 100644
> --- a/arch/x86/xen/xen-head.S
> +++ b/arch/x86/xen/xen-head.S
> @@ -51,15 +51,14 @@ SYM_CODE_START(startup_xen)
>
> leaq __top_init_kernel_stack(%rip), %rsp
>
> - /* Set up %gs.
> - *
> - * The base of %gs always points to fixed_percpu_data.
> + /*
> + * Set up GSBASE.
> * Note that, on SMP, the boot cpu uses init data section until
> * the per cpu areas are set up.
> */
> movl $MSR_GS_BASE,%ecx
> - movq $INIT_PER_CPU_VAR(fixed_percpu_data),%rax
> - cdq
> + xorl %eax, %eax
> + xorl %edx, %edx
> wrmsr
>
> mov %rsi, %rdi
> diff --git a/init/Kconfig b/init/Kconfig
> index f3ea5dea9c85..0f928f82dc7a 100644
> --- a/init/Kconfig
> +++ b/init/Kconfig
> @@ -1784,7 +1784,7 @@ config KALLSYMS_ALL
> config KALLSYMS_ABSOLUTE_PERCPU
> bool
> depends on KALLSYMS
> - default X86_64 && SMP
> + default n
>
> config KALLSYMS_BASE_RELATIVE
> bool
> --
> 2.44.0
>
© 2016 - 2026 Red Hat, Inc.