From: Ard Biesheuvel <ardb@kernel.org>
Some of the early x86_64 startup code is written in C, and executes in
the early 1:1 mapping of the kernel, which is not the address it was
linked at, and this requires special care when accessing global
variables. This is currently being dealt with on an ad-hoc basis,
primarily in head64.c, using explicit pointer fixups, but it would be
better to rely on the compiler for this, by using -fPIE to generate code
that can run at any address, and uses RIP-relative accesses to refer to
global variables.
While it is possible to avoid most GOT based symbol references that the
compiler typically emits when running in -fPIE mode, by using 'hidden'
visibility, there are cases where the compiler will always rely on the
GOT, for instance, for weak external references (which may remain
unsatisfied at link time).
This means the build may produce a small number of GOT entries
nonetheless. So update the reloc processing host tool to add support for
this, and place the GOT in the .text section rather than discard it.
Note that multiple GOT based references to the same symbol will share a
single GOT entry, and so naively emitting a relocation for the GOT entry
each time a reference to it is encountered could result in duplicates.
Work around this by relying on the fact that the relocation lists are
sorted, and deduplicate 64-bit relocations as they are emitted by
comparing each entry with the previous one.
Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
---
arch/x86/Makefile | 4 +++
arch/x86/kernel/vmlinux.lds.S | 5 +++
arch/x86/tools/relocs.c | 33 ++++++++++++++++++--
include/asm-generic/vmlinux.lds.h | 7 +++++
4 files changed, 47 insertions(+), 2 deletions(-)
diff --git a/arch/x86/Makefile b/arch/x86/Makefile
index 801fd85c3ef6..6b3fe6e2aadd 100644
--- a/arch/x86/Makefile
+++ b/arch/x86/Makefile
@@ -192,6 +192,10 @@ else
KBUILD_CFLAGS += -mcmodel=kernel
KBUILD_RUSTFLAGS += -Cno-redzone=y
KBUILD_RUSTFLAGS += -Ccode-model=kernel
+
+ # Don't emit relaxable GOTPCREL relocations
+ KBUILD_AFLAGS_KERNEL += -Wa,-mrelax-relocations=no
+ KBUILD_CFLAGS_KERNEL += -Wa,-mrelax-relocations=no
endif
#
diff --git a/arch/x86/kernel/vmlinux.lds.S b/arch/x86/kernel/vmlinux.lds.S
index 6e73403e874f..7f060d873f75 100644
--- a/arch/x86/kernel/vmlinux.lds.S
+++ b/arch/x86/kernel/vmlinux.lds.S
@@ -20,6 +20,9 @@
#define RUNTIME_DISCARD_EXIT
#define EMITS_PT_NOTE
#define RO_EXCEPTION_TABLE_ALIGN 16
+#ifdef CONFIG_X86_64
+#define GOT_IN_RODATA
+#endif
#include <asm-generic/vmlinux.lds.h>
#include <asm/asm-offsets.h>
@@ -464,10 +467,12 @@ SECTIONS
* Sections that should stay zero sized, which is safer to
* explicitly check instead of blindly discarding.
*/
+#ifdef CONFIG_X86_32
.got : {
*(.got) *(.igot.*)
}
ASSERT(SIZEOF(.got) == 0, "Unexpected GOT entries detected!")
+#endif
.plt : {
*(.plt) *(.plt.*) *(.iplt)
diff --git a/arch/x86/tools/relocs.c b/arch/x86/tools/relocs.c
index 35a73e4aa74d..880f0f2e465e 100644
--- a/arch/x86/tools/relocs.c
+++ b/arch/x86/tools/relocs.c
@@ -223,6 +223,8 @@ static const char *rel_type(unsigned type)
REL_TYPE(R_X86_64_JUMP_SLOT),
REL_TYPE(R_X86_64_RELATIVE),
REL_TYPE(R_X86_64_GOTPCREL),
+ REL_TYPE(R_X86_64_GOTPCRELX),
+ REL_TYPE(R_X86_64_REX_GOTPCRELX),
REL_TYPE(R_X86_64_32),
REL_TYPE(R_X86_64_32S),
REL_TYPE(R_X86_64_16),
@@ -843,6 +845,7 @@ static int do_reloc64(struct section *sec, Elf_Rel *rel, ElfW(Sym) *sym,
case R_X86_64_32:
case R_X86_64_32S:
case R_X86_64_64:
+ case R_X86_64_GOTPCREL:
/*
* References to the percpu area don't need to be adjusted.
*/
@@ -861,6 +864,31 @@ static int do_reloc64(struct section *sec, Elf_Rel *rel, ElfW(Sym) *sym,
break;
}
+ if (r_type == R_X86_64_GOTPCREL) {
+ Elf_Shdr *s = &secs[sec->shdr.sh_info].shdr;
+ unsigned file_off = offset - s->sh_addr + s->sh_offset;
+
+ /*
+ * GOTPCREL relocations refer to instructions that load
+ * a 64-bit address via a 32-bit relative reference to
+ * the GOT. In this case, it is the GOT entry that
+ * needs to be fixed up, not the immediate offset in
+ * the opcode. Note that the linker will have applied an
+ * addend of -4 to compensate for the delta between the
+ * relocation offset and the value of RIP when the
+ * instruction executes, and this needs to be backed out
+ * again. (Addends other than -4 are permitted in
+ * principle, but make no sense in practice so they are
+ * not supported.)
+ */
+ if (rel->r_addend != -4) {
+ die("invalid addend (%ld) for %s relocation: %s\n",
+ rel->r_addend, rel_type(r_type), symname);
+ break;
+ }
+ offset += 4 + (int32_t)get_unaligned_le32(elf_image + file_off);
+ }
+
/*
* Relocation offsets for 64 bit kernels are output
* as 32 bits and sign extended back to 64 bits when
@@ -870,7 +898,7 @@ static int do_reloc64(struct section *sec, Elf_Rel *rel, ElfW(Sym) *sym,
if ((int32_t)offset != (int64_t)offset)
die("Relocation offset doesn't fit in 32 bits\n");
- if (r_type == R_X86_64_64)
+ if (r_type == R_X86_64_64 || r_type == R_X86_64_GOTPCREL)
add_reloc(&relocs64, offset);
else
add_reloc(&relocs32, offset);
@@ -1085,7 +1113,8 @@ static void emit_relocs(int as_text, int use_real_mode)
/* Now print each relocation */
for (i = 0; i < relocs64.count; i++)
- write_reloc(relocs64.offset[i], stdout);
+ if (!i || relocs64.offset[i] != relocs64.offset[i - 1])
+ write_reloc(relocs64.offset[i], stdout);
/* Print a stop */
write_reloc(0, stdout);
diff --git a/include/asm-generic/vmlinux.lds.h b/include/asm-generic/vmlinux.lds.h
index 19ec49a9179b..cc14d780c70d 100644
--- a/include/asm-generic/vmlinux.lds.h
+++ b/include/asm-generic/vmlinux.lds.h
@@ -443,6 +443,12 @@
#endif
#endif
+#ifdef GOT_IN_RODATA
+#define GOT_RODATA *(.got .igot*)
+#else
+#define GOT_RODATA
+#endif
+
/*
* Read only Data
*/
@@ -454,6 +460,7 @@
SCHED_DATA \
RO_AFTER_INIT_DATA /* Read only after init */ \
. = ALIGN(8); \
+ GOT_RODATA \
BOUNDED_SECTION_BY(__tracepoints_ptrs, ___tracepoints_ptrs) \
*(__tracepoints_strings)/* Tracepoints: strings */ \
} \
--
2.46.0.792.g87dc391469-goog