From nobody Sat Feb 7 18:15:09 2026 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id AEC70EB64DD for ; Wed, 12 Jul 2023 03:31:28 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S231497AbjGLDb0 (ORCPT ); Tue, 11 Jul 2023 23:31:26 -0400 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:37082 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S229843AbjGLDbU (ORCPT ); Tue, 11 Jul 2023 23:31:20 -0400 Received: from out0-201.mail.aliyun.com (out0-201.mail.aliyun.com [140.205.0.201]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id 97704E5F for ; Tue, 11 Jul 2023 20:31:19 -0700 (PDT) X-Alimail-AntiSpam: AC=PASS;BC=-1|-1;BR=01201311R201e4;CH=green;DM=||false|;DS=||;FP=0|-1|-1|-1|0|-1|-1|-1;HT=ay29a033018047206;MF=houwenlong.hwl@antgroup.com;NM=1;PH=DS;RN=13;SR=0;TI=SMTPD_---.TrdHwby_1689132672; Received: from localhost(mailfrom:houwenlong.hwl@antgroup.com fp:SMTPD_---.TrdHwby_1689132672) by smtp.aliyun-inc.com; Wed, 12 Jul 2023 11:31:13 +0800 From: "Hou Wenlong" To: linux-kernel@vger.kernel.org Cc: "Lai Jiangshan" , "Hou Wenlong" , "Thomas Gleixner" , "Ingo Molnar" , "Borislav Petkov" , "Dave Hansen" , "=?UTF-8?B?bWFpbnRhaW5lcjpYODYgQVJDSElURUNUVVJFIDMyLUJJVCBBTkQgNjQtQklU?=" , "H. Peter Anvin" , "Josh Poimboeuf" , "Anshuman Khandual" , "Mike Rapoport" , "Pasha Tatashin" Subject: [PATCH RFC 1/7] x86/head/64: Mark startup_gdt and startup_gdt_descr as __initdata Date: Wed, 12 Jul 2023 11:30:05 +0800 Message-Id: X-Mailer: git-send-email 2.31.1 In-Reply-To: References: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Precedence: bulk List-ID: X-Mailing-List: linux-kernel@vger.kernel.org Content-Type: text/plain; charset="utf-8" As startup_gdt and startup_gdt_descr are only used in booting, make them as __initdata to allow them to be freed after boot. Signed-off-by: Hou Wenlong --- arch/x86/kernel/head64.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/x86/kernel/head64.c b/arch/x86/kernel/head64.c index 49f7629b17f7..dd357807ffb5 100644 --- a/arch/x86/kernel/head64.c +++ b/arch/x86/kernel/head64.c @@ -69,7 +69,7 @@ EXPORT_SYMBOL(vmemmap_base); /* * GDT used on the boot CPU before switching to virtual addresses. */ -static struct desc_struct startup_gdt[GDT_ENTRIES] =3D { +static struct desc_struct startup_gdt[GDT_ENTRIES] __initdata =3D { [GDT_ENTRY_KERNEL32_CS] =3D GDT_ENTRY_INIT(0xc09b, 0, 0xfffff), [GDT_ENTRY_KERNEL_CS] =3D GDT_ENTRY_INIT(0xa09b, 0, 0xfffff), [GDT_ENTRY_KERNEL_DS] =3D GDT_ENTRY_INIT(0xc093, 0, 0xfffff), @@ -79,7 +79,7 @@ static struct desc_struct startup_gdt[GDT_ENTRIES] =3D { * Address needs to be set at runtime because it references the startup_gdt * while the kernel still uses a direct mapping. */ -static struct desc_ptr startup_gdt_descr =3D { +static struct desc_ptr startup_gdt_descr __initdata =3D { .size =3D sizeof(startup_gdt), .address =3D 0, }; --=20 2.31.1 From nobody Sat Feb 7 18:15:09 2026 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id DCC4CEB64DC for ; Wed, 12 Jul 2023 03:31:33 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S231742AbjGLDbc (ORCPT ); Tue, 11 Jul 2023 23:31:32 -0400 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:37096 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S231494AbjGLDbZ (ORCPT ); Tue, 11 Jul 2023 23:31:25 -0400 Received: from out0-195.mail.aliyun.com (out0-195.mail.aliyun.com [140.205.0.195]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id 67663E5F for ; Tue, 11 Jul 2023 20:31:24 -0700 (PDT) X-Alimail-AntiSpam: AC=PASS;BC=-1|-1;BR=01201311R431e4;CH=green;DM=||false|;DS=||;FP=0|-1|-1|-1|0|-1|-1|-1;HT=ay29a033018047201;MF=houwenlong.hwl@antgroup.com;NM=1;PH=DS;RN=13;SR=0;TI=SMTPD_---.TrdQGkX_1689132675; Received: from localhost(mailfrom:houwenlong.hwl@antgroup.com fp:SMTPD_---.TrdQGkX_1689132675) by smtp.aliyun-inc.com; Wed, 12 Jul 2023 11:31:16 +0800 From: "Hou Wenlong" To: linux-kernel@vger.kernel.org Cc: "Lai Jiangshan" , "Hou Wenlong" , "Thomas Gleixner" , "Ingo Molnar" , "Borislav Petkov" , "Dave Hansen" , "=?UTF-8?B?bWFpbnRhaW5lcjpYODYgQVJDSElURUNUVVJFIDMyLUJJVCBBTkQgNjQtQklU?=" , "H. Peter Anvin" , "Josh Poimboeuf" , "Anshuman Khandual" , "Mike Rapoport" , "Pasha Tatashin" Subject: [PATCH RFC 2/7] x86/head/64: Add missing __head annotation to startup_64_load_idt() Date: Wed, 12 Jul 2023 11:30:06 +0800 Message-Id: X-Mailer: git-send-email 2.31.1 In-Reply-To: References: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Precedence: bulk List-ID: X-Mailing-List: linux-kernel@vger.kernel.org Content-Type: text/plain; charset="utf-8" This function is currently only used in the head code and is only called from startup_64_setup_env(). Although it would be inlined by the compiler, it would be better to mark it as __head too. Signed-off-by: Hou Wenlong --- arch/x86/kernel/head64.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/x86/kernel/head64.c b/arch/x86/kernel/head64.c index dd357807ffb5..5066189b00ac 100644 --- a/arch/x86/kernel/head64.c +++ b/arch/x86/kernel/head64.c @@ -588,7 +588,7 @@ static void set_bringup_idt_handler(gate_desc *idt, int= n, void *handler) } =20 /* This runs while still in the direct mapping */ -static void startup_64_load_idt(unsigned long physbase) +static void __head startup_64_load_idt(unsigned long physbase) { struct desc_ptr *desc =3D fixup_pointer(&bringup_idt_descr, physbase); gate_desc *idt =3D fixup_pointer(bringup_idt_table, physbase); --=20 2.31.1 From nobody Sat Feb 7 18:15:09 2026 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id 31FEAEB64DD for ; Wed, 12 Jul 2023 03:31:58 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S231210AbjGLDb4 (ORCPT ); Tue, 11 Jul 2023 23:31:56 -0400 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:37294 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S231428AbjGLDbp (ORCPT ); Tue, 11 Jul 2023 23:31:45 -0400 Received: from out0-197.mail.aliyun.com (out0-197.mail.aliyun.com [140.205.0.197]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id 422311BD6 for ; Tue, 11 Jul 2023 20:31:38 -0700 (PDT) X-Alimail-AntiSpam: AC=PASS;BC=-1|-1;BR=01201311R181e4;CH=green;DM=||false|;DS=||;FP=0|-1|-1|-1|0|-1|-1|-1;HT=ay29a033018047208;MF=houwenlong.hwl@antgroup.com;NM=1;PH=DS;RN=23;SR=0;TI=SMTPD_---.TrdETZJ_1689132690; Received: from localhost(mailfrom:houwenlong.hwl@antgroup.com fp:SMTPD_---.TrdETZJ_1689132690) by smtp.aliyun-inc.com; Wed, 12 Jul 2023 11:31:31 +0800 From: "Hou Wenlong" To: linux-kernel@vger.kernel.org Cc: "Lai Jiangshan" , "Hou Wenlong" , "Thomas Gleixner" , "Ingo Molnar" , "Borislav Petkov" , "Dave Hansen" , "=?UTF-8?B?bWFpbnRhaW5lcjpYODYgQVJDSElURUNUVVJFIDMyLUJJVCBBTkQgNjQtQklU?=" , "H. Peter Anvin" , "Nathan Chancellor" , "Nick Desaulniers" , "Tom Rix" , "Brian Gerst" , "Eric W. Biederman" , "=?UTF-8?B?UGV0ZXIgWmlqbHN0cmEgKEludGVsKQ==?=" , "=?UTF-8?B?TWFzYW1pIEhpcmFtYXRzdSAoR29vZ2xlKQ==?=" , "Masahiro Yamada" , "Sami Tolvanen" , "Alexander Potapenko" , "Mike Rapoport" , "Pasha Tatashin" , "Josh Poimboeuf" , "=?UTF-8?B?b3BlbiBsaXN0OkNMQU5HL0xMVk0gQlVJTEQgU1VQUE9SVA==?=" Subject: [PATCH RFC 3/7] x86/head/64: Move all head code from head64.c into another file Date: Wed, 12 Jul 2023 11:30:07 +0800 Message-Id: <7c026a8167cd90a3b3605a0f617dee5ebee16a36.1689130310.git.houwenlong.hwl@antgroup.com> X-Mailer: git-send-email 2.31.1 In-Reply-To: References: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Precedence: bulk List-ID: X-Mailing-List: linux-kernel@vger.kernel.org Content-Type: text/plain; charset="utf-8" This is preparation for building the head code as PIE and getting rid of all fixup_pointer() calls. Suggested-by: Lai Jiangshan Signed-off-by: Hou Wenlong --- arch/x86/include/asm/desc.h | 12 ++ arch/x86/kernel/Makefile | 5 +- arch/x86/kernel/head64.c | 307 +---------------------------- arch/x86/kernel/head64_identity.c | 315 ++++++++++++++++++++++++++++++ 4 files changed, 334 insertions(+), 305 deletions(-) create mode 100644 arch/x86/kernel/head64_identity.c diff --git a/arch/x86/include/asm/desc.h b/arch/x86/include/asm/desc.h index ab97b22ac04a..b568331b0caa 100644 --- a/arch/x86/include/asm/desc.h +++ b/arch/x86/include/asm/desc.h @@ -431,6 +431,18 @@ static inline void idt_init_desc(gate_desc *gate, cons= t struct idt_data *d) #endif } =20 +static inline void set_bringup_idt_handler(gate_desc *idt, int n, void *ha= ndler) +{ +#ifdef CONFIG_AMD_MEM_ENCRYPT + struct idt_data data; + gate_desc desc; + + init_idt_data(&data, n, handler); + idt_init_desc(&desc, &data); + native_write_idt_entry(idt, n, &desc); +#endif +} + extern unsigned long system_vectors[]; =20 extern void load_current_idt(void); diff --git a/arch/x86/kernel/Makefile b/arch/x86/kernel/Makefile index 4070a01c11b7..2fd9a4fe27b1 100644 --- a/arch/x86/kernel/Makefile +++ b/arch/x86/kernel/Makefile @@ -16,11 +16,13 @@ CFLAGS_REMOVE_kvmclock.o =3D -pg CFLAGS_REMOVE_ftrace.o =3D -pg CFLAGS_REMOVE_early_printk.o =3D -pg CFLAGS_REMOVE_head64.o =3D -pg +CFLAGS_REMOVE_head64_identity.o =3D -pg CFLAGS_REMOVE_sev.o =3D -pg CFLAGS_REMOVE_rethook.o =3D -pg endif =20 KASAN_SANITIZE_head$(BITS).o :=3D n +KASAN_SANITIZE_head64_identity.o :=3D n KASAN_SANITIZE_dumpstack.o :=3D n KASAN_SANITIZE_dumpstack_$(BITS).o :=3D n KASAN_SANITIZE_stacktrace.o :=3D n @@ -31,6 +33,7 @@ KASAN_SANITIZE_sev.o :=3D n # by several compilation units. To be safe, disable all instrumentation. KCSAN_SANITIZE :=3D n KMSAN_SANITIZE_head$(BITS).o :=3D n +KMSAN_SANITIZE_head64_identity.o :=3D n KMSAN_SANITIZE_nmi.o :=3D n =20 # If instrumentation of this dir is enabled, boot hangs during first secon= d. @@ -153,5 +156,5 @@ ifeq ($(CONFIG_X86_64),y) obj-$(CONFIG_GART_IOMMU) +=3D amd_gart_64.o aperture_64.o =20 obj-$(CONFIG_MMCONF_FAM10H) +=3D mmconf-fam10h_64.o - obj-y +=3D vsmp_64.o + obj-y +=3D vsmp_64.o head64_identity.o endif diff --git a/arch/x86/kernel/head64.c b/arch/x86/kernel/head64.c index 5066189b00ac..6eca5fe97c1f 100644 --- a/arch/x86/kernel/head64.c +++ b/arch/x86/kernel/head64.c @@ -46,7 +46,7 @@ * Manage page tables very early on. */ extern pmd_t early_dynamic_pgts[EARLY_DYNAMIC_PAGE_TABLES][PTRS_PER_PMD]; -static unsigned int __initdata next_early_pgt; +unsigned int __initdata next_early_pgt; pmdval_t early_pmd_flags =3D __PAGE_KERNEL_LARGE & ~(_PAGE_GLOBAL | _PAGE_= NX); =20 #ifdef CONFIG_X86_5LEVEL @@ -66,259 +66,6 @@ unsigned long vmemmap_base __ro_after_init =3D __VMEMMA= P_BASE_L4; EXPORT_SYMBOL(vmemmap_base); #endif =20 -/* - * GDT used on the boot CPU before switching to virtual addresses. - */ -static struct desc_struct startup_gdt[GDT_ENTRIES] __initdata =3D { - [GDT_ENTRY_KERNEL32_CS] =3D GDT_ENTRY_INIT(0xc09b, 0, 0xfffff), - [GDT_ENTRY_KERNEL_CS] =3D GDT_ENTRY_INIT(0xa09b, 0, 0xfffff), - [GDT_ENTRY_KERNEL_DS] =3D GDT_ENTRY_INIT(0xc093, 0, 0xfffff), -}; - -/* - * Address needs to be set at runtime because it references the startup_gdt - * while the kernel still uses a direct mapping. - */ -static struct desc_ptr startup_gdt_descr __initdata =3D { - .size =3D sizeof(startup_gdt), - .address =3D 0, -}; - -#define __head __section(".head.text") - -static void __head *fixup_pointer(void *ptr, unsigned long physaddr) -{ - return ptr - (void *)_text + (void *)physaddr; -} - -static unsigned long __head *fixup_long(void *ptr, unsigned long physaddr) -{ - return fixup_pointer(ptr, physaddr); -} - -#ifdef CONFIG_X86_5LEVEL -static unsigned int __head *fixup_int(void *ptr, unsigned long physaddr) -{ - return fixup_pointer(ptr, physaddr); -} - -static bool __head check_la57_support(unsigned long physaddr) -{ - /* - * 5-level paging is detected and enabled at kernel decompression - * stage. Only check if it has been enabled there. - */ - if (!(native_read_cr4() & X86_CR4_LA57)) - return false; - - *fixup_int(&__pgtable_l5_enabled, physaddr) =3D 1; - *fixup_int(&pgdir_shift, physaddr) =3D 48; - *fixup_int(&ptrs_per_p4d, physaddr) =3D 512; - *fixup_long(&page_offset_base, physaddr) =3D __PAGE_OFFSET_BASE_L5; - *fixup_long(&vmalloc_base, physaddr) =3D __VMALLOC_BASE_L5; - *fixup_long(&vmemmap_base, physaddr) =3D __VMEMMAP_BASE_L5; - - return true; -} -#else -static bool __head check_la57_support(unsigned long physaddr) -{ - return false; -} -#endif - -static unsigned long __head sme_postprocess_startup(struct boot_params *bp= , pmdval_t *pmd) -{ - unsigned long vaddr, vaddr_end; - int i; - - /* Encrypt the kernel and related (if SME is active) */ - sme_encrypt_kernel(bp); - - /* - * Clear the memory encryption mask from the .bss..decrypted section. - * The bss section will be memset to zero later in the initialization so - * there is no need to zero it after changing the memory encryption - * attribute. - */ - if (sme_get_me_mask()) { - vaddr =3D (unsigned long)__start_bss_decrypted; - vaddr_end =3D (unsigned long)__end_bss_decrypted; - - for (; vaddr < vaddr_end; vaddr +=3D PMD_SIZE) { - /* - * On SNP, transition the page to shared in the RMP table so that - * it is consistent with the page table attribute change. - * - * __start_bss_decrypted has a virtual address in the high range - * mapping (kernel .text). PVALIDATE, by way of - * early_snp_set_memory_shared(), requires a valid virtual - * address but the kernel is currently running off of the identity - * mapping so use __pa() to get a *currently* valid virtual address. - */ - early_snp_set_memory_shared(__pa(vaddr), __pa(vaddr), PTRS_PER_PMD); - - i =3D pmd_index(vaddr); - pmd[i] -=3D sme_get_me_mask(); - } - } - - /* - * Return the SME encryption mask (if SME is active) to be used as a - * modifier for the initial pgdir entry programmed into CR3. - */ - return sme_get_me_mask(); -} - -/* Code in __startup_64() can be relocated during execution, but the compi= ler - * doesn't have to generate PC-relative relocations when accessing globals= from - * that function. Clang actually does not generate them, which leads to - * boot-time crashes. To work around this problem, every global pointer mu= st - * be adjusted using fixup_pointer(). - */ -unsigned long __head __startup_64(unsigned long physaddr, - struct boot_params *bp) -{ - unsigned long load_delta, *p; - unsigned long pgtable_flags; - pgdval_t *pgd; - p4dval_t *p4d; - pudval_t *pud; - pmdval_t *pmd, pmd_entry; - pteval_t *mask_ptr; - bool la57; - int i; - unsigned int *next_pgt_ptr; - - la57 =3D check_la57_support(physaddr); - - /* Is the address too large? */ - if (physaddr >> MAX_PHYSMEM_BITS) - for (;;); - - /* - * Compute the delta between the address I am compiled to run at - * and the address I am actually running at. - */ - load_delta =3D physaddr - (unsigned long)(_text - __START_KERNEL_map); - - /* Is the address not 2M aligned? */ - if (load_delta & ~PMD_MASK) - for (;;); - - /* Include the SME encryption mask in the fixup value */ - load_delta +=3D sme_get_me_mask(); - - /* Fixup the physical addresses in the page table */ - - pgd =3D fixup_pointer(&early_top_pgt, physaddr); - p =3D pgd + pgd_index(__START_KERNEL_map); - if (la57) - *p =3D (unsigned long)level4_kernel_pgt; - else - *p =3D (unsigned long)level3_kernel_pgt; - *p +=3D _PAGE_TABLE_NOENC - __START_KERNEL_map + load_delta; - - if (la57) { - p4d =3D fixup_pointer(&level4_kernel_pgt, physaddr); - p4d[511] +=3D load_delta; - } - - pud =3D fixup_pointer(&level3_kernel_pgt, physaddr); - pud[510] +=3D load_delta; - pud[511] +=3D load_delta; - - pmd =3D fixup_pointer(level2_fixmap_pgt, physaddr); - for (i =3D FIXMAP_PMD_TOP; i > FIXMAP_PMD_TOP - FIXMAP_PMD_NUM; i--) - pmd[i] +=3D load_delta; - - /* - * Set up the identity mapping for the switchover. These - * entries should *NOT* have the global bit set! This also - * creates a bunch of nonsense entries but that is fine -- - * it avoids problems around wraparound. - */ - - next_pgt_ptr =3D fixup_pointer(&next_early_pgt, physaddr); - pud =3D fixup_pointer(early_dynamic_pgts[(*next_pgt_ptr)++], physaddr); - pmd =3D fixup_pointer(early_dynamic_pgts[(*next_pgt_ptr)++], physaddr); - - pgtable_flags =3D _KERNPG_TABLE_NOENC + sme_get_me_mask(); - - if (la57) { - p4d =3D fixup_pointer(early_dynamic_pgts[(*next_pgt_ptr)++], - physaddr); - - i =3D (physaddr >> PGDIR_SHIFT) % PTRS_PER_PGD; - pgd[i + 0] =3D (pgdval_t)p4d + pgtable_flags; - pgd[i + 1] =3D (pgdval_t)p4d + pgtable_flags; - - i =3D physaddr >> P4D_SHIFT; - p4d[(i + 0) % PTRS_PER_P4D] =3D (pgdval_t)pud + pgtable_flags; - p4d[(i + 1) % PTRS_PER_P4D] =3D (pgdval_t)pud + pgtable_flags; - } else { - i =3D (physaddr >> PGDIR_SHIFT) % PTRS_PER_PGD; - pgd[i + 0] =3D (pgdval_t)pud + pgtable_flags; - pgd[i + 1] =3D (pgdval_t)pud + pgtable_flags; - } - - i =3D physaddr >> PUD_SHIFT; - pud[(i + 0) % PTRS_PER_PUD] =3D (pudval_t)pmd + pgtable_flags; - pud[(i + 1) % PTRS_PER_PUD] =3D (pudval_t)pmd + pgtable_flags; - - pmd_entry =3D __PAGE_KERNEL_LARGE_EXEC & ~_PAGE_GLOBAL; - /* Filter out unsupported __PAGE_KERNEL_* bits: */ - mask_ptr =3D fixup_pointer(&__supported_pte_mask, physaddr); - pmd_entry &=3D *mask_ptr; - pmd_entry +=3D sme_get_me_mask(); - pmd_entry +=3D physaddr; - - for (i =3D 0; i < DIV_ROUND_UP(_end - _text, PMD_SIZE); i++) { - int idx =3D i + (physaddr >> PMD_SHIFT); - - pmd[idx % PTRS_PER_PMD] =3D pmd_entry + i * PMD_SIZE; - } - - /* - * Fixup the kernel text+data virtual addresses. Note that - * we might write invalid pmds, when the kernel is relocated - * cleanup_highmap() fixes this up along with the mappings - * beyond _end. - * - * Only the region occupied by the kernel image has so far - * been checked against the table of usable memory regions - * provided by the firmware, so invalidate pages outside that - * region. A page table entry that maps to a reserved area of - * memory would allow processor speculation into that area, - * and on some hardware (particularly the UV platform) even - * speculative access to some reserved areas is caught as an - * error, causing the BIOS to halt the system. - */ - - pmd =3D fixup_pointer(level2_kernel_pgt, physaddr); - - /* invalidate pages before the kernel image */ - for (i =3D 0; i < pmd_index((unsigned long)_text); i++) - pmd[i] &=3D ~_PAGE_PRESENT; - - /* fixup pages that are part of the kernel image */ - for (; i <=3D pmd_index((unsigned long)_end); i++) - if (pmd[i] & _PAGE_PRESENT) - pmd[i] +=3D load_delta; - - /* invalidate pages after the kernel image */ - for (; i < PTRS_PER_PMD; i++) - pmd[i] &=3D ~_PAGE_PRESENT; - - /* - * Fixup phys_base - remove the memory encryption mask to obtain - * the true physical address. - */ - *fixup_long(&phys_base, physaddr) +=3D load_delta - sme_get_me_mask(); - - return sme_postprocess_startup(bp, pmd); -} - /* Wipe all early page tables except for the kernel symbol map */ static void __init reset_early_page_tables(void) { @@ -568,44 +315,13 @@ void __init __noreturn x86_64_start_reservations(char= *real_mode_data) * configured which require certain CPU state to be setup already (like TS= S), * which also hasn't happened yet in early CPU bringup. */ -static gate_desc bringup_idt_table[NUM_EXCEPTION_VECTORS] __page_aligned_d= ata; +gate_desc bringup_idt_table[NUM_EXCEPTION_VECTORS] __page_aligned_data; =20 -static struct desc_ptr bringup_idt_descr =3D { +struct desc_ptr bringup_idt_descr =3D { .size =3D (NUM_EXCEPTION_VECTORS * sizeof(gate_desc)) - 1, .address =3D 0, /* Set at runtime */ }; =20 -static void set_bringup_idt_handler(gate_desc *idt, int n, void *handler) -{ -#ifdef CONFIG_AMD_MEM_ENCRYPT - struct idt_data data; - gate_desc desc; - - init_idt_data(&data, n, handler); - idt_init_desc(&desc, &data); - native_write_idt_entry(idt, n, &desc); -#endif -} - -/* This runs while still in the direct mapping */ -static void __head startup_64_load_idt(unsigned long physbase) -{ - struct desc_ptr *desc =3D fixup_pointer(&bringup_idt_descr, physbase); - gate_desc *idt =3D fixup_pointer(bringup_idt_table, physbase); - - - if (IS_ENABLED(CONFIG_AMD_MEM_ENCRYPT)) { - void *handler; - - /* VMM Communication Exception */ - handler =3D fixup_pointer(vc_no_ghcb, physbase); - set_bringup_idt_handler(idt, X86_TRAP_VC, handler); - } - - desc->address =3D (unsigned long)idt; - native_load_idt(desc); -} - /* This is used when running on kernel addresses */ void early_setup_idt(void) { @@ -618,20 +334,3 @@ void early_setup_idt(void) bringup_idt_descr.address =3D (unsigned long)bringup_idt_table; native_load_idt(&bringup_idt_descr); } - -/* - * Setup boot CPU state needed before kernel switches to virtual addresses. - */ -void __head startup_64_setup_env(unsigned long physbase) -{ - /* Load GDT */ - startup_gdt_descr.address =3D (unsigned long)fixup_pointer(startup_gdt, p= hysbase); - native_load_gdt(&startup_gdt_descr); - - /* New GDT is live - reload data segment registers */ - asm volatile("movl %%eax, %%ds\n" - "movl %%eax, %%ss\n" - "movl %%eax, %%es\n" : : "a"(__KERNEL_DS) : "memory"); - - startup_64_load_idt(physbase); -} diff --git a/arch/x86/kernel/head64_identity.c b/arch/x86/kernel/head64_ide= ntity.c new file mode 100644 index 000000000000..a10acbe00fe9 --- /dev/null +++ b/arch/x86/kernel/head64_identity.c @@ -0,0 +1,315 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * prepare to run common code + * + * Copyright (C) 2000 Andrea Arcangeli SuSE + */ + +#define DISABLE_BRANCH_PROFILING + +/* cpu_feature_enabled() cannot be used this early */ +#define USE_EARLY_PGTABLE_L5 + +#include +#include +#include +#include + +#include +#include +#include +#include + +extern pmd_t early_dynamic_pgts[EARLY_DYNAMIC_PAGE_TABLES][PTRS_PER_PMD]; +extern unsigned int next_early_pgt; +extern gate_desc bringup_idt_table[NUM_EXCEPTION_VECTORS]; +extern struct desc_ptr bringup_idt_descr; + +/* + * GDT used on the boot CPU before switching to virtual addresses. + */ +static struct desc_struct startup_gdt[GDT_ENTRIES] __initdata =3D { + [GDT_ENTRY_KERNEL32_CS] =3D GDT_ENTRY_INIT(0xc09b, 0, 0xfffff), + [GDT_ENTRY_KERNEL_CS] =3D GDT_ENTRY_INIT(0xa09b, 0, 0xfffff), + [GDT_ENTRY_KERNEL_DS] =3D GDT_ENTRY_INIT(0xc093, 0, 0xfffff), +}; + +/* + * Address needs to be set at runtime because it references the startup_gdt + * while the kernel still uses a direct mapping. + */ +static struct desc_ptr startup_gdt_descr __initdata =3D { + .size =3D sizeof(startup_gdt), + .address =3D 0, +}; + +#define __head __section(".head.text") + +static void __head *fixup_pointer(void *ptr, unsigned long physaddr) +{ + return ptr - (void *)_text + (void *)physaddr; +} + +static unsigned long __head *fixup_long(void *ptr, unsigned long physaddr) +{ + return fixup_pointer(ptr, physaddr); +} + +#ifdef CONFIG_X86_5LEVEL +static unsigned int __head *fixup_int(void *ptr, unsigned long physaddr) +{ + return fixup_pointer(ptr, physaddr); +} + +static bool __head check_la57_support(unsigned long physaddr) +{ + /* + * 5-level paging is detected and enabled at kernel decompression + * stage. Only check if it has been enabled there. + */ + if (!(native_read_cr4() & X86_CR4_LA57)) + return false; + + *fixup_int(&__pgtable_l5_enabled, physaddr) =3D 1; + *fixup_int(&pgdir_shift, physaddr) =3D 48; + *fixup_int(&ptrs_per_p4d, physaddr) =3D 512; + *fixup_long(&page_offset_base, physaddr) =3D __PAGE_OFFSET_BASE_L5; + *fixup_long(&vmalloc_base, physaddr) =3D __VMALLOC_BASE_L5; + *fixup_long(&vmemmap_base, physaddr) =3D __VMEMMAP_BASE_L5; + + return true; +} +#else +static bool __head check_la57_support(unsigned long physaddr) +{ + return false; +} +#endif + +static unsigned long __head sme_postprocess_startup(struct boot_params *bp= , pmdval_t *pmd) +{ + unsigned long vaddr, vaddr_end; + int i; + + /* Encrypt the kernel and related (if SME is active) */ + sme_encrypt_kernel(bp); + + /* + * Clear the memory encryption mask from the .bss..decrypted section. + * The bss section will be memset to zero later in the initialization so + * there is no need to zero it after changing the memory encryption + * attribute. + */ + if (sme_get_me_mask()) { + vaddr =3D (unsigned long)__start_bss_decrypted; + vaddr_end =3D (unsigned long)__end_bss_decrypted; + + for (; vaddr < vaddr_end; vaddr +=3D PMD_SIZE) { + /* + * On SNP, transition the page to shared in the RMP table so that + * it is consistent with the page table attribute change. + * + * __start_bss_decrypted has a virtual address in the high range + * mapping (kernel .text). PVALIDATE, by way of + * early_snp_set_memory_shared(), requires a valid virtual + * address but the kernel is currently running off of the identity + * mapping so use __pa() to get a *currently* valid virtual address. + */ + early_snp_set_memory_shared(__pa(vaddr), __pa(vaddr), PTRS_PER_PMD); + + i =3D pmd_index(vaddr); + pmd[i] -=3D sme_get_me_mask(); + } + } + + /* + * Return the SME encryption mask (if SME is active) to be used as a + * modifier for the initial pgdir entry programmed into CR3. + */ + return sme_get_me_mask(); +} + +/* Code in __startup_64() can be relocated during execution, but the compi= ler + * doesn't have to generate PC-relative relocations when accessing globals= from + * that function. Clang actually does not generate them, which leads to + * boot-time crashes. To work around this problem, every global pointer mu= st + * be adjusted using fixup_pointer(). + */ +unsigned long __head __startup_64(unsigned long physaddr, + struct boot_params *bp) +{ + unsigned long load_delta, *p; + unsigned long pgtable_flags; + pgdval_t *pgd; + p4dval_t *p4d; + pudval_t *pud; + pmdval_t *pmd, pmd_entry; + pteval_t *mask_ptr; + bool la57; + int i; + unsigned int *next_pgt_ptr; + + la57 =3D check_la57_support(physaddr); + + /* Is the address too large? */ + if (physaddr >> MAX_PHYSMEM_BITS) + for (;;); + + /* + * Compute the delta between the address I am compiled to run at + * and the address I am actually running at. + */ + load_delta =3D physaddr - (unsigned long)(_text - __START_KERNEL_map); + + /* Is the address not 2M aligned? */ + if (load_delta & ~PMD_MASK) + for (;;); + + /* Include the SME encryption mask in the fixup value */ + load_delta +=3D sme_get_me_mask(); + + /* Fixup the physical addresses in the page table */ + + pgd =3D fixup_pointer(&early_top_pgt, physaddr); + p =3D pgd + pgd_index(__START_KERNEL_map); + if (la57) + *p =3D (unsigned long)level4_kernel_pgt; + else + *p =3D (unsigned long)level3_kernel_pgt; + *p +=3D _PAGE_TABLE_NOENC - __START_KERNEL_map + load_delta; + + if (la57) { + p4d =3D fixup_pointer(&level4_kernel_pgt, physaddr); + p4d[511] +=3D load_delta; + } + + pud =3D fixup_pointer(&level3_kernel_pgt, physaddr); + pud[510] +=3D load_delta; + pud[511] +=3D load_delta; + + pmd =3D fixup_pointer(level2_fixmap_pgt, physaddr); + for (i =3D FIXMAP_PMD_TOP; i > FIXMAP_PMD_TOP - FIXMAP_PMD_NUM; i--) + pmd[i] +=3D load_delta; + + /* + * Set up the identity mapping for the switchover. These + * entries should *NOT* have the global bit set! This also + * creates a bunch of nonsense entries but that is fine -- + * it avoids problems around wraparound. + */ + + next_pgt_ptr =3D fixup_pointer(&next_early_pgt, physaddr); + pud =3D fixup_pointer(early_dynamic_pgts[(*next_pgt_ptr)++], physaddr); + pmd =3D fixup_pointer(early_dynamic_pgts[(*next_pgt_ptr)++], physaddr); + + pgtable_flags =3D _KERNPG_TABLE_NOENC + sme_get_me_mask(); + + if (la57) { + p4d =3D fixup_pointer(early_dynamic_pgts[(*next_pgt_ptr)++], + physaddr); + + i =3D (physaddr >> PGDIR_SHIFT) % PTRS_PER_PGD; + pgd[i + 0] =3D (pgdval_t)p4d + pgtable_flags; + pgd[i + 1] =3D (pgdval_t)p4d + pgtable_flags; + + i =3D physaddr >> P4D_SHIFT; + p4d[(i + 0) % PTRS_PER_P4D] =3D (pgdval_t)pud + pgtable_flags; + p4d[(i + 1) % PTRS_PER_P4D] =3D (pgdval_t)pud + pgtable_flags; + } else { + i =3D (physaddr >> PGDIR_SHIFT) % PTRS_PER_PGD; + pgd[i + 0] =3D (pgdval_t)pud + pgtable_flags; + pgd[i + 1] =3D (pgdval_t)pud + pgtable_flags; + } + + i =3D physaddr >> PUD_SHIFT; + pud[(i + 0) % PTRS_PER_PUD] =3D (pudval_t)pmd + pgtable_flags; + pud[(i + 1) % PTRS_PER_PUD] =3D (pudval_t)pmd + pgtable_flags; + + pmd_entry =3D __PAGE_KERNEL_LARGE_EXEC & ~_PAGE_GLOBAL; + /* Filter out unsupported __PAGE_KERNEL_* bits: */ + mask_ptr =3D fixup_pointer(&__supported_pte_mask, physaddr); + pmd_entry &=3D *mask_ptr; + pmd_entry +=3D sme_get_me_mask(); + pmd_entry +=3D physaddr; + + for (i =3D 0; i < DIV_ROUND_UP(_end - _text, PMD_SIZE); i++) { + int idx =3D i + (physaddr >> PMD_SHIFT); + + pmd[idx % PTRS_PER_PMD] =3D pmd_entry + i * PMD_SIZE; + } + + /* + * Fixup the kernel text+data virtual addresses. Note that + * we might write invalid pmds, when the kernel is relocated + * cleanup_highmap() fixes this up along with the mappings + * beyond _end. + * + * Only the region occupied by the kernel image has so far + * been checked against the table of usable memory regions + * provided by the firmware, so invalidate pages outside that + * region. A page table entry that maps to a reserved area of + * memory would allow processor speculation into that area, + * and on some hardware (particularly the UV platform) even + * speculative access to some reserved areas is caught as an + * error, causing the BIOS to halt the system. + */ + + pmd =3D fixup_pointer(level2_kernel_pgt, physaddr); + + /* invalidate pages before the kernel image */ + for (i =3D 0; i < pmd_index((unsigned long)_text); i++) + pmd[i] &=3D ~_PAGE_PRESENT; + + /* fixup pages that are part of the kernel image */ + for (; i <=3D pmd_index((unsigned long)_end); i++) + if (pmd[i] & _PAGE_PRESENT) + pmd[i] +=3D load_delta; + + /* invalidate pages after the kernel image */ + for (; i < PTRS_PER_PMD; i++) + pmd[i] &=3D ~_PAGE_PRESENT; + + /* + * Fixup phys_base - remove the memory encryption mask to obtain + * the true physical address. + */ + *fixup_long(&phys_base, physaddr) +=3D load_delta - sme_get_me_mask(); + + return sme_postprocess_startup(bp, pmd); +} + +/* This runs while still in the direct mapping */ +static void __head startup_64_load_idt(unsigned long physbase) +{ + struct desc_ptr *desc =3D fixup_pointer(&bringup_idt_descr, physbase); + gate_desc *idt =3D fixup_pointer(bringup_idt_table, physbase); + + + if (IS_ENABLED(CONFIG_AMD_MEM_ENCRYPT)) { + void *handler; + + /* VMM Communication Exception */ + handler =3D fixup_pointer(vc_no_ghcb, physbase); + set_bringup_idt_handler(idt, X86_TRAP_VC, handler); + } + + desc->address =3D (unsigned long)idt; + native_load_idt(desc); +} + +/* + * Setup boot CPU state needed before kernel switches to virtual addresses. + */ +void __head startup_64_setup_env(unsigned long physbase) +{ + /* Load GDT */ + startup_gdt_descr.address =3D (unsigned long)fixup_pointer(startup_gdt, p= hysbase); + native_load_gdt(&startup_gdt_descr); + + /* New GDT is live - reload data segment registers */ + asm volatile("movl %%eax, %%ds\n" + "movl %%eax, %%ss\n" + "movl %%eax, %%es\n" : : "a"(__KERNEL_DS) : "memory"); + + startup_64_load_idt(physbase); +} --=20 2.31.1 From nobody Sat Feb 7 18:15:09 2026 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id 89039EB64DD for ; Wed, 12 Jul 2023 03:32:02 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S231881AbjGLDcA (ORCPT ); Tue, 11 Jul 2023 23:32:00 -0400 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:37250 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S231716AbjGLDbs (ORCPT ); Tue, 11 Jul 2023 23:31:48 -0400 Received: from out0-194.mail.aliyun.com (out0-194.mail.aliyun.com [140.205.0.194]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id 599F610EF for ; Tue, 11 Jul 2023 20:31:41 -0700 (PDT) X-Alimail-AntiSpam: AC=PASS;BC=-1|-1;BR=01201311R741e4;CH=green;DM=||false|;DS=||;FP=0|-1|-1|-1|0|-1|-1|-1;HT=ay29a033018047208;MF=houwenlong.hwl@antgroup.com;NM=1;PH=DS;RN=15;SR=0;TI=SMTPD_---.TrdHwxy_1689132693; Received: from localhost(mailfrom:houwenlong.hwl@antgroup.com fp:SMTPD_---.TrdHwxy_1689132693) by smtp.aliyun-inc.com; Wed, 12 Jul 2023 11:31:34 +0800 From: "Hou Wenlong" To: linux-kernel@vger.kernel.org Cc: "Lai Jiangshan" , "Hou Wenlong" , "Thomas Gleixner" , "Ingo Molnar" , "Borislav Petkov" , "Dave Hansen" , "=?UTF-8?B?bWFpbnRhaW5lcjpYODYgQVJDSElURUNUVVJFIDMyLUJJVCBBTkQgNjQtQklU?=" , "H. Peter Anvin" , "Nathan Chancellor" , "Ard Biesheuvel" , "Kirill A. Shutemov" , "Nick Desaulniers" , "Petr Pavlu" , "Xin Li" Subject: [PATCH RFC 4/7] x86/boot/compressed: Adapt sed command if head code is built as PIE Date: Wed, 12 Jul 2023 11:30:08 +0800 Message-Id: <0856eaae3439581fdcb7143389ee4cc00859cc9d.1689130310.git.houwenlong.hwl@antgroup.com> X-Mailer: git-send-email 2.31.1 In-Reply-To: References: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Precedence: bulk List-ID: X-Mailing-List: linux-kernel@vger.kernel.org Content-Type: text/plain; charset="utf-8" When x86/head64_identity.c is built as PIE, all symbols would be set as hidden to omit GOT references. According to the generic ABI, a hidden symbol contained in a relocatable object must be either removed or converted to STB_LOCAL binding by the link-editor when the relocatable object is included in an executable file or shared object. Both gold and ld.lld change the binding of a STV_HIDDEND symbol to STB_LOCAL. However, for GNU ld, it will keep the global hidden. The sed command to generate voffset.h only captures global symbols, so an empty voffset.h would be generated with ld.lld. Therefore, local symbols should also be captured in the sed command. Signed-off-by: Hou Wenlong --- arch/x86/boot/compressed/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/x86/boot/compressed/Makefile b/arch/x86/boot/compressed/M= akefile index 40d2ff503079..ca57ad0d2d22 100644 --- a/arch/x86/boot/compressed/Makefile +++ b/arch/x86/boot/compressed/Makefile @@ -79,7 +79,7 @@ LDFLAGS_vmlinux +=3D -T hostprogs :=3D mkpiggy HOST_EXTRACFLAGS +=3D -I$(srctree)/tools/include =20 -sed-voffset :=3D -e 's/^\([0-9a-fA-F]*\) [ABCDGRSTVW] \(_text\|__bss_start= \|_end\)$$/\#define VO_\2 _AC(0x\1,UL)/p' +sed-voffset :=3D -e 's/^\([0-9a-fA-F]*\) [ABCDGRSTVWabcdgrstvw] \(_text\|_= _bss_start\|_end\)$$/\#define VO_\2 _AC(0x\1,UL)/p' =20 quiet_cmd_voffset =3D VOFFSET $@ cmd_voffset =3D $(NM) $< | sed -n $(sed-voffset) > $@ --=20 2.31.1 From nobody Sat Feb 7 18:15:09 2026 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id 0029FC001DC for ; Wed, 12 Jul 2023 03:32:20 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S231784AbjGLDcS (ORCPT ); Tue, 11 Jul 2023 23:32:18 -0400 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:37630 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S231765AbjGLDb7 (ORCPT ); Tue, 11 Jul 2023 23:31:59 -0400 Received: from out0-216.mail.aliyun.com (out0-216.mail.aliyun.com [140.205.0.216]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id 42F27A1 for ; Tue, 11 Jul 2023 20:31:54 -0700 (PDT) X-Alimail-AntiSpam: AC=PASS;BC=-1|-1;BR=01201311R841e4;CH=green;DM=||false|;DS=||;FP=0|-1|-1|-1|0|-1|-1|-1;HT=ay29a033018047193;MF=houwenlong.hwl@antgroup.com;NM=1;PH=DS;RN=26;SR=0;TI=SMTPD_---.TrdHx1m_1689132706; Received: from localhost(mailfrom:houwenlong.hwl@antgroup.com fp:SMTPD_---.TrdHx1m_1689132706) by smtp.aliyun-inc.com; Wed, 12 Jul 2023 11:31:47 +0800 From: "Hou Wenlong" To: linux-kernel@vger.kernel.org Cc: "Lai Jiangshan" , "Hou Wenlong" , "Thomas Gleixner" , "Ingo Molnar" , "Borislav Petkov" , "Dave Hansen" , "=?UTF-8?B?bWFpbnRhaW5lcjpYODYgQVJDSElURUNUVVJFIDMyLUJJVCBBTkQgNjQtQklU?=" , "H. Peter Anvin" , "Nathan Chancellor" , "Nick Desaulniers" , "Tom Rix" , "Josh Poimboeuf" , "=?UTF-8?B?UGV0ZXIgWmlqbHN0cmEgKEludGVsKQ==?=" , "Brian Gerst" , "Eric W. Biederman" , "=?UTF-8?B?TWFzYW1pIEhpcmFtYXRzdSAoR29vZ2xlKQ==?=" , "Masahiro Yamada" , "Sami Tolvanen" , "Alexander Potapenko" , "Mike Rapoport" , "Pasha Tatashin" , "David Woodhouse" , "Usama Arif" , "Tom Lendacky" , "=?UTF-8?B?b3BlbiBsaXN0OkNMQU5HL0xMVk0gQlVJTEQgU1VQUE9SVA==?=" Subject: [PATCH RFC 5/7] x86/head/64: Build the head code as PIE Date: Wed, 12 Jul 2023 11:30:09 +0800 Message-Id: <12bb41e38979f68760a98be3b25cc66cb955e559.1689130310.git.houwenlong.hwl@antgroup.com> X-Mailer: git-send-email 2.31.1 In-Reply-To: References: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Precedence: bulk List-ID: X-Mailing-List: linux-kernel@vger.kernel.org Content-Type: text/plain; charset="utf-8" During the early boot stage, the head code runs in a low identity address, so all absolute references would be incorrect. However, the compiler doesn't have to generate PC-relative references when accessing globals. To work around this problem, every global variable access must be adjusted using fixup_pointer(). Nevertheless, the compiler could generate absolute references for some global variable accesses, and the behavior differs between GCC and CLANG. For example, GCC generates PC-relative references for 'next_early_pgt', while CLANG generates absolute references. Moreover, the rule is not always clear, e.g., 'pgdir_shift' is a non-static global variable similar to 'next_early_pgt', but the compiler could generate the right PC-relative reference, so fixup_pointer() is not applied when using the PGDIR_SHIFT macro. To avoid such cases, build the head code as PIE to force the generation of PC-relative references and eliminate the need for fixup_pointer(). However, it may be necessary to obtain an absolute virtual address for some symbols in the head code. In this case, use 'movabsq'. Additionally, the 'mcmodel=3Dkernel' option is not compatible with the 'fPIE' option and needs to be removed. This will result in using the '%fs' register for stack canary access if the stack protector is enabled. The 'mstack-protector-guard-reg' compiler option could be used to fix this, but it is only used in 32-bit now and 64-bit is still using the fixed location version. Since the head code is in the early booting stage, it is safe to disable the stack protector for the head code. Suggested-by: Lai Jiangshan Signed-off-by: Hou Wenlong --- arch/x86/include/asm/setup.h | 2 +- arch/x86/kernel/Makefile | 11 +++ arch/x86/kernel/head64_identity.c | 112 +++++++++++------------------- arch/x86/kernel/head_64.S | 2 - 4 files changed, 52 insertions(+), 75 deletions(-) diff --git a/arch/x86/include/asm/setup.h b/arch/x86/include/asm/setup.h index f3495623ac99..b893b0cdddac 100644 --- a/arch/x86/include/asm/setup.h +++ b/arch/x86/include/asm/setup.h @@ -50,7 +50,7 @@ extern unsigned long saved_video_mode; extern void reserve_standard_io_resources(void); extern void i386_reserve_resources(void); extern unsigned long __startup_64(unsigned long physaddr, struct boot_para= ms *bp); -extern void startup_64_setup_env(unsigned long physbase); +extern void startup_64_setup_env(void); extern void early_setup_idt(void); extern void __init do_early_exception(struct pt_regs *regs, int trapnr); =20 diff --git a/arch/x86/kernel/Makefile b/arch/x86/kernel/Makefile index 2fd9a4fe27b1..6564113f5298 100644 --- a/arch/x86/kernel/Makefile +++ b/arch/x86/kernel/Makefile @@ -44,6 +44,17 @@ KCOV_INSTRUMENT :=3D n =20 CFLAGS_irq.o :=3D -I $(srctree)/$(src)/../include/asm/trace =20 +# The 'mcmodel=3Dkernel' option is not compatible with the 'fPIE' option a= nd +# needs to be removed. This will result in using the '%fs' register for st= ack +# canary access if the stack protector is enabled. The +# 'mstack-protector-guard-reg' compiler option could be used to fix this, = but +# it is only used in 32-bit now and 64-bit is still using the fixed locati= on +# version. Since the head code is in the early booting stage, it is safe to +# disable the stack protector for the head code. +CFLAGS_REMOVE_head64_identity.o +=3D -mcmodel=3Dkernel +CFLAGS_head64_identity.o +=3D -fPIE -include $(srctree)/include/linux/hidd= en.h +CFLAGS_head64_identity.o +=3D -fno-stack-protector + obj-y +=3D head_$(BITS).o obj-y +=3D head$(BITS).o obj-y +=3D ebda.o diff --git a/arch/x86/kernel/head64_identity.c b/arch/x86/kernel/head64_ide= ntity.c index a10acbe00fe9..93f5831917bc 100644 --- a/arch/x86/kernel/head64_identity.c +++ b/arch/x86/kernel/head64_identity.c @@ -45,23 +45,8 @@ static struct desc_ptr startup_gdt_descr __initdata =3D { =20 #define __head __section(".head.text") =20 -static void __head *fixup_pointer(void *ptr, unsigned long physaddr) -{ - return ptr - (void *)_text + (void *)physaddr; -} - -static unsigned long __head *fixup_long(void *ptr, unsigned long physaddr) -{ - return fixup_pointer(ptr, physaddr); -} - #ifdef CONFIG_X86_5LEVEL -static unsigned int __head *fixup_int(void *ptr, unsigned long physaddr) -{ - return fixup_pointer(ptr, physaddr); -} - -static bool __head check_la57_support(unsigned long physaddr) +static bool __head check_la57_support(void) { /* * 5-level paging is detected and enabled at kernel decompression @@ -70,22 +55,27 @@ static bool __head check_la57_support(unsigned long phy= saddr) if (!(native_read_cr4() & X86_CR4_LA57)) return false; =20 - *fixup_int(&__pgtable_l5_enabled, physaddr) =3D 1; - *fixup_int(&pgdir_shift, physaddr) =3D 48; - *fixup_int(&ptrs_per_p4d, physaddr) =3D 512; - *fixup_long(&page_offset_base, physaddr) =3D __PAGE_OFFSET_BASE_L5; - *fixup_long(&vmalloc_base, physaddr) =3D __VMALLOC_BASE_L5; - *fixup_long(&vmemmap_base, physaddr) =3D __VMEMMAP_BASE_L5; + __pgtable_l5_enabled =3D 1; + pgdir_shift =3D 48; + ptrs_per_p4d =3D 512; + page_offset_base =3D __PAGE_OFFSET_BASE_L5; + vmalloc_base =3D __VMALLOC_BASE_L5; + vmemmap_base =3D __VMEMMAP_BASE_L5; =20 return true; } #else -static bool __head check_la57_support(unsigned long physaddr) +static bool __head check_la57_support(void) { return false; } #endif =20 +#define SYM_ABS_VA(sym) ({ \ + unsigned long __v; \ + asm("movabsq $" __stringify(sym) ", %0":"=3Dr"(__v)); \ + __v; }) + static unsigned long __head sme_postprocess_startup(struct boot_params *bp= , pmdval_t *pmd) { unsigned long vaddr, vaddr_end; @@ -101,8 +91,8 @@ static unsigned long __head sme_postprocess_startup(stru= ct boot_params *bp, pmdv * attribute. */ if (sme_get_me_mask()) { - vaddr =3D (unsigned long)__start_bss_decrypted; - vaddr_end =3D (unsigned long)__end_bss_decrypted; + vaddr =3D SYM_ABS_VA(__start_bss_decrypted); + vaddr_end =3D SYM_ABS_VA(__end_bss_decrypted); =20 for (; vaddr < vaddr_end; vaddr +=3D PMD_SIZE) { /* @@ -129,12 +119,6 @@ static unsigned long __head sme_postprocess_startup(st= ruct boot_params *bp, pmdv return sme_get_me_mask(); } =20 -/* Code in __startup_64() can be relocated during execution, but the compi= ler - * doesn't have to generate PC-relative relocations when accessing globals= from - * that function. Clang actually does not generate them, which leads to - * boot-time crashes. To work around this problem, every global pointer mu= st - * be adjusted using fixup_pointer(). - */ unsigned long __head __startup_64(unsigned long physaddr, struct boot_params *bp) { @@ -144,12 +128,10 @@ unsigned long __head __startup_64(unsigned long physa= ddr, p4dval_t *p4d; pudval_t *pud; pmdval_t *pmd, pmd_entry; - pteval_t *mask_ptr; bool la57; int i; - unsigned int *next_pgt_ptr; =20 - la57 =3D check_la57_support(physaddr); + la57 =3D check_la57_support(); =20 /* Is the address too large? */ if (physaddr >> MAX_PHYSMEM_BITS) @@ -159,7 +141,7 @@ unsigned long __head __startup_64(unsigned long physadd= r, * Compute the delta between the address I am compiled to run at * and the address I am actually running at. */ - load_delta =3D physaddr - (unsigned long)(_text - __START_KERNEL_map); + load_delta =3D physaddr - (SYM_ABS_VA(_text) - __START_KERNEL_map); =20 /* Is the address not 2M aligned? */ if (load_delta & ~PMD_MASK) @@ -170,26 +152,24 @@ unsigned long __head __startup_64(unsigned long physa= ddr, =20 /* Fixup the physical addresses in the page table */ =20 - pgd =3D fixup_pointer(&early_top_pgt, physaddr); + pgd =3D (pgdval_t *)early_top_pgt; p =3D pgd + pgd_index(__START_KERNEL_map); if (la57) *p =3D (unsigned long)level4_kernel_pgt; else *p =3D (unsigned long)level3_kernel_pgt; - *p +=3D _PAGE_TABLE_NOENC - __START_KERNEL_map + load_delta; + *p +=3D _PAGE_TABLE_NOENC + sme_get_me_mask(); =20 if (la57) { - p4d =3D fixup_pointer(&level4_kernel_pgt, physaddr); + p4d =3D (p4dval_t *)level4_kernel_pgt; p4d[511] +=3D load_delta; } =20 - pud =3D fixup_pointer(&level3_kernel_pgt, physaddr); - pud[510] +=3D load_delta; - pud[511] +=3D load_delta; + level3_kernel_pgt[510].pud +=3D load_delta; + level3_kernel_pgt[511].pud +=3D load_delta; =20 - pmd =3D fixup_pointer(level2_fixmap_pgt, physaddr); for (i =3D FIXMAP_PMD_TOP; i > FIXMAP_PMD_TOP - FIXMAP_PMD_NUM; i--) - pmd[i] +=3D load_delta; + level2_fixmap_pgt[i].pmd +=3D load_delta; =20 /* * Set up the identity mapping for the switchover. These @@ -198,15 +178,13 @@ unsigned long __head __startup_64(unsigned long physa= ddr, * it avoids problems around wraparound. */ =20 - next_pgt_ptr =3D fixup_pointer(&next_early_pgt, physaddr); - pud =3D fixup_pointer(early_dynamic_pgts[(*next_pgt_ptr)++], physaddr); - pmd =3D fixup_pointer(early_dynamic_pgts[(*next_pgt_ptr)++], physaddr); + pud =3D (pudval_t *)early_dynamic_pgts[next_early_pgt++]; + pmd =3D (pmdval_t *)early_dynamic_pgts[next_early_pgt++]; =20 pgtable_flags =3D _KERNPG_TABLE_NOENC + sme_get_me_mask(); =20 if (la57) { - p4d =3D fixup_pointer(early_dynamic_pgts[(*next_pgt_ptr)++], - physaddr); + p4d =3D (p4dval_t *)early_dynamic_pgts[next_early_pgt++]; =20 i =3D (physaddr >> PGDIR_SHIFT) % PTRS_PER_PGD; pgd[i + 0] =3D (pgdval_t)p4d + pgtable_flags; @@ -227,8 +205,7 @@ unsigned long __head __startup_64(unsigned long physadd= r, =20 pmd_entry =3D __PAGE_KERNEL_LARGE_EXEC & ~_PAGE_GLOBAL; /* Filter out unsupported __PAGE_KERNEL_* bits: */ - mask_ptr =3D fixup_pointer(&__supported_pte_mask, physaddr); - pmd_entry &=3D *mask_ptr; + pmd_entry &=3D __supported_pte_mask; pmd_entry +=3D sme_get_me_mask(); pmd_entry +=3D physaddr; =20 @@ -253,15 +230,14 @@ unsigned long __head __startup_64(unsigned long physa= ddr, * speculative access to some reserved areas is caught as an * error, causing the BIOS to halt the system. */ - - pmd =3D fixup_pointer(level2_kernel_pgt, physaddr); + pmd =3D (pmdval_t *)level2_kernel_pgt; =20 /* invalidate pages before the kernel image */ - for (i =3D 0; i < pmd_index((unsigned long)_text); i++) + for (i =3D 0; i < pmd_index(SYM_ABS_VA(_text)); i++) pmd[i] &=3D ~_PAGE_PRESENT; =20 /* fixup pages that are part of the kernel image */ - for (; i <=3D pmd_index((unsigned long)_end); i++) + for (; i <=3D pmd_index(SYM_ABS_VA(_end)); i++) if (pmd[i] & _PAGE_PRESENT) pmd[i] +=3D load_delta; =20 @@ -273,37 +249,29 @@ unsigned long __head __startup_64(unsigned long physa= ddr, * Fixup phys_base - remove the memory encryption mask to obtain * the true physical address. */ - *fixup_long(&phys_base, physaddr) +=3D load_delta - sme_get_me_mask(); + phys_base +=3D load_delta - sme_get_me_mask(); =20 return sme_postprocess_startup(bp, pmd); } =20 /* This runs while still in the direct mapping */ -static void __head startup_64_load_idt(unsigned long physbase) +static void __head startup_64_load_idt(void) { - struct desc_ptr *desc =3D fixup_pointer(&bringup_idt_descr, physbase); - gate_desc *idt =3D fixup_pointer(bringup_idt_table, physbase); - - - if (IS_ENABLED(CONFIG_AMD_MEM_ENCRYPT)) { - void *handler; - - /* VMM Communication Exception */ - handler =3D fixup_pointer(vc_no_ghcb, physbase); - set_bringup_idt_handler(idt, X86_TRAP_VC, handler); - } + /* VMM Communication Exception */ + if (IS_ENABLED(CONFIG_AMD_MEM_ENCRYPT)) + set_bringup_idt_handler(bringup_idt_table, X86_TRAP_VC, vc_no_ghcb); =20 - desc->address =3D (unsigned long)idt; - native_load_idt(desc); + bringup_idt_descr.address =3D (unsigned long)bringup_idt_table; + native_load_idt(&bringup_idt_descr); } =20 /* * Setup boot CPU state needed before kernel switches to virtual addresses. */ -void __head startup_64_setup_env(unsigned long physbase) +void __head startup_64_setup_env(void) { /* Load GDT */ - startup_gdt_descr.address =3D (unsigned long)fixup_pointer(startup_gdt, p= hysbase); + startup_gdt_descr.address =3D (unsigned long)startup_gdt; native_load_gdt(&startup_gdt_descr); =20 /* New GDT is live - reload data segment registers */ @@ -311,5 +279,5 @@ void __head startup_64_setup_env(unsigned long physbase) "movl %%eax, %%ss\n" "movl %%eax, %%es\n" : : "a"(__KERNEL_DS) : "memory"); =20 - startup_64_load_idt(physbase); + startup_64_load_idt(); } diff --git a/arch/x86/kernel/head_64.S b/arch/x86/kernel/head_64.S index c5b9289837dc..5b46da66d6c8 100644 --- a/arch/x86/kernel/head_64.S +++ b/arch/x86/kernel/head_64.S @@ -66,8 +66,6 @@ SYM_CODE_START_NOALIGN(startup_64) /* Set up the stack for verify_cpu() */ leaq (__end_init_task - PTREGS_SIZE)(%rip), %rsp =20 - leaq _text(%rip), %rdi - /* Setup GSBASE to allow stack canary access for C code */ movl $MSR_GS_BASE, %ecx leaq INIT_PER_CPU_VAR(fixed_percpu_data)(%rip), %rdx --=20 2.31.1 From nobody Sat Feb 7 18:15:09 2026 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id 38F15EB64DD for ; Wed, 12 Jul 2023 03:32:24 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S231452AbjGLDcW (ORCPT ); Tue, 11 Jul 2023 23:32:22 -0400 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:37200 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S231986AbjGLDcK (ORCPT ); Tue, 11 Jul 2023 23:32:10 -0400 Received: from out0-222.mail.aliyun.com (out0-222.mail.aliyun.com [140.205.0.222]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id 924B91BDC for ; Tue, 11 Jul 2023 20:32:04 -0700 (PDT) X-Alimail-AntiSpam: AC=PASS;BC=-1|-1;BR=01201311R221e4;CH=green;DM=||false|;DS=||;FP=0|-1|-1|-1|0|-1|-1|-1;HT=ay29a033018047211;MF=houwenlong.hwl@antgroup.com;NM=1;PH=DS;RN=19;SR=0;TI=SMTPD_---.TrdETkS_1689132717; Received: from localhost(mailfrom:houwenlong.hwl@antgroup.com fp:SMTPD_---.TrdETkS_1689132717) by smtp.aliyun-inc.com; Wed, 12 Jul 2023 11:31:58 +0800 From: "Hou Wenlong" To: linux-kernel@vger.kernel.org Cc: "Lai Jiangshan" , "Hou Wenlong" , "Thomas Gleixner" , "Ingo Molnar" , "Borislav Petkov" , "Dave Hansen" , "=?UTF-8?B?bWFpbnRhaW5lcjpYODYgQVJDSElURUNUVVJFIDMyLUJJVCBBTkQgNjQtQklU?=" , "H. Peter Anvin" , "Andy Lutomirski" , "Peter Zijlstra" , "Andrew Morton" , "Anshuman Khandual" , "Stephen Rothwell" , "Michael Kelley" , "Arnd Bergmann" , "Mike Rapoport" , "Josh Poimboeuf" , "Pasha Tatashin" Subject: [PATCH RFC 6/7] x86/sme: Mark code as __head in mem_encrypt_identity.c Date: Wed, 12 Jul 2023 11:30:10 +0800 Message-Id: <65bc4576158690453a23699f119d073b227acede.1689130310.git.houwenlong.hwl@antgroup.com> X-Mailer: git-send-email 2.31.1 In-Reply-To: References: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Precedence: bulk List-ID: X-Mailing-List: linux-kernel@vger.kernel.org Content-Type: text/plain; charset="utf-8" The functions sme_enable() and sme_encrypt_kernel() are only called by the head code which runs in identity virtual address. Therefore, it's better to mark them as __head as well. Signed-off-by: Hou Wenlong --- arch/x86/include/asm/init.h | 2 ++ arch/x86/include/asm/mem_encrypt.h | 8 ++++---- arch/x86/kernel/head64_identity.c | 3 +-- arch/x86/mm/mem_encrypt_identity.c | 27 ++++++++++++++------------- 4 files changed, 21 insertions(+), 19 deletions(-) diff --git a/arch/x86/include/asm/init.h b/arch/x86/include/asm/init.h index 5f1d3c421f68..cc9ccf61b6bd 100644 --- a/arch/x86/include/asm/init.h +++ b/arch/x86/include/asm/init.h @@ -2,6 +2,8 @@ #ifndef _ASM_X86_INIT_H #define _ASM_X86_INIT_H =20 +#define __head __section(".head.text") + struct x86_mapping_info { void *(*alloc_pgt_page)(void *); /* allocate buf for page table */ void *context; /* context for alloc_pgt_page */ diff --git a/arch/x86/include/asm/mem_encrypt.h b/arch/x86/include/asm/mem_= encrypt.h index 7f97a8a97e24..7b6616f67398 100644 --- a/arch/x86/include/asm/mem_encrypt.h +++ b/arch/x86/include/asm/mem_encrypt.h @@ -45,8 +45,8 @@ void __init sme_unmap_bootdata(char *real_mode_data); void __init sme_early_init(void); void __init sev_setup_arch(void); =20 -void __init sme_encrypt_kernel(struct boot_params *bp); -void __init sme_enable(struct boot_params *bp); +void sme_encrypt_kernel(struct boot_params *bp); +void sme_enable(struct boot_params *bp); =20 int __init early_set_memory_decrypted(unsigned long vaddr, unsigned long s= ize); int __init early_set_memory_encrypted(unsigned long vaddr, unsigned long s= ize); @@ -75,8 +75,8 @@ static inline void __init sme_unmap_bootdata(char *real_m= ode_data) { } static inline void __init sme_early_init(void) { } static inline void __init sev_setup_arch(void) { } =20 -static inline void __init sme_encrypt_kernel(struct boot_params *bp) { } -static inline void __init sme_enable(struct boot_params *bp) { } +static inline void sme_encrypt_kernel(struct boot_params *bp) { } +static inline void sme_enable(struct boot_params *bp) { } =20 static inline void sev_es_init_vc_handling(void) { } =20 diff --git a/arch/x86/kernel/head64_identity.c b/arch/x86/kernel/head64_ide= ntity.c index 93f5831917bc..521fef98cccd 100644 --- a/arch/x86/kernel/head64_identity.c +++ b/arch/x86/kernel/head64_identity.c @@ -19,6 +19,7 @@ #include #include #include +#include =20 extern pmd_t early_dynamic_pgts[EARLY_DYNAMIC_PAGE_TABLES][PTRS_PER_PMD]; extern unsigned int next_early_pgt; @@ -43,8 +44,6 @@ static struct desc_ptr startup_gdt_descr __initdata =3D { .address =3D 0, }; =20 -#define __head __section(".head.text") - #ifdef CONFIG_X86_5LEVEL static bool __head check_la57_support(void) { diff --git a/arch/x86/mm/mem_encrypt_identity.c b/arch/x86/mm/mem_encrypt_i= dentity.c index d73aeb16417f..72aeb0f3dec6 100644 --- a/arch/x86/mm/mem_encrypt_identity.c +++ b/arch/x86/mm/mem_encrypt_identity.c @@ -46,6 +46,7 @@ #include #include #include +#include =20 #include "mm_internal.h" =20 @@ -99,7 +100,7 @@ static char sme_cmdline_arg[] __initdata =3D "mem_encryp= t"; static char sme_cmdline_on[] __initdata =3D "on"; static char sme_cmdline_off[] __initdata =3D "off"; =20 -static void __init sme_clear_pgd(struct sme_populate_pgd_data *ppd) +static void __head sme_clear_pgd(struct sme_populate_pgd_data *ppd) { unsigned long pgd_start, pgd_end, pgd_size; pgd_t *pgd_p; @@ -114,7 +115,7 @@ static void __init sme_clear_pgd(struct sme_populate_pg= d_data *ppd) memset(pgd_p, 0, pgd_size); } =20 -static pud_t __init *sme_prepare_pgd(struct sme_populate_pgd_data *ppd) +static pud_t __head *sme_prepare_pgd(struct sme_populate_pgd_data *ppd) { pgd_t *pgd; p4d_t *p4d; @@ -151,7 +152,7 @@ static pud_t __init *sme_prepare_pgd(struct sme_populat= e_pgd_data *ppd) return pud; } =20 -static void __init sme_populate_pgd_large(struct sme_populate_pgd_data *pp= d) +static void __head sme_populate_pgd_large(struct sme_populate_pgd_data *pp= d) { pud_t *pud; pmd_t *pmd; @@ -167,7 +168,7 @@ static void __init sme_populate_pgd_large(struct sme_po= pulate_pgd_data *ppd) set_pmd(pmd, __pmd(ppd->paddr | ppd->pmd_flags)); } =20 -static void __init sme_populate_pgd(struct sme_populate_pgd_data *ppd) +static void __head sme_populate_pgd(struct sme_populate_pgd_data *ppd) { pud_t *pud; pmd_t *pmd; @@ -193,7 +194,7 @@ static void __init sme_populate_pgd(struct sme_populate= _pgd_data *ppd) set_pte(pte, __pte(ppd->paddr | ppd->pte_flags)); } =20 -static void __init __sme_map_range_pmd(struct sme_populate_pgd_data *ppd) +static void __head __sme_map_range_pmd(struct sme_populate_pgd_data *ppd) { while (ppd->vaddr < ppd->vaddr_end) { sme_populate_pgd_large(ppd); @@ -203,7 +204,7 @@ static void __init __sme_map_range_pmd(struct sme_popul= ate_pgd_data *ppd) } } =20 -static void __init __sme_map_range_pte(struct sme_populate_pgd_data *ppd) +static void __head __sme_map_range_pte(struct sme_populate_pgd_data *ppd) { while (ppd->vaddr < ppd->vaddr_end) { sme_populate_pgd(ppd); @@ -213,7 +214,7 @@ static void __init __sme_map_range_pte(struct sme_popul= ate_pgd_data *ppd) } } =20 -static void __init __sme_map_range(struct sme_populate_pgd_data *ppd, +static void __head __sme_map_range(struct sme_populate_pgd_data *ppd, pmdval_t pmd_flags, pteval_t pte_flags) { unsigned long vaddr_end; @@ -237,22 +238,22 @@ static void __init __sme_map_range(struct sme_populat= e_pgd_data *ppd, __sme_map_range_pte(ppd); } =20 -static void __init sme_map_range_encrypted(struct sme_populate_pgd_data *p= pd) +static void __head sme_map_range_encrypted(struct sme_populate_pgd_data *p= pd) { __sme_map_range(ppd, PMD_FLAGS_ENC, PTE_FLAGS_ENC); } =20 -static void __init sme_map_range_decrypted(struct sme_populate_pgd_data *p= pd) +static void __head sme_map_range_decrypted(struct sme_populate_pgd_data *p= pd) { __sme_map_range(ppd, PMD_FLAGS_DEC, PTE_FLAGS_DEC); } =20 -static void __init sme_map_range_decrypted_wp(struct sme_populate_pgd_data= *ppd) +static void __head sme_map_range_decrypted_wp(struct sme_populate_pgd_data= *ppd) { __sme_map_range(ppd, PMD_FLAGS_DEC_WP, PTE_FLAGS_DEC_WP); } =20 -static unsigned long __init sme_pgtable_calc(unsigned long len) +static unsigned long __head sme_pgtable_calc(unsigned long len) { unsigned long entries =3D 0, tables =3D 0; =20 @@ -289,7 +290,7 @@ static unsigned long __init sme_pgtable_calc(unsigned l= ong len) return entries + tables; } =20 -void __init sme_encrypt_kernel(struct boot_params *bp) +void __head sme_encrypt_kernel(struct boot_params *bp) { unsigned long workarea_start, workarea_end, workarea_len; unsigned long execute_start, execute_end, execute_len; @@ -502,7 +503,7 @@ void __init sme_encrypt_kernel(struct boot_params *bp) native_write_cr3(__native_read_cr3()); } =20 -void __init sme_enable(struct boot_params *bp) +void __head sme_enable(struct boot_params *bp) { const char *cmdline_ptr, *cmdline_arg, *cmdline_on, *cmdline_off; unsigned int eax, ebx, ecx, edx; --=20 2.31.1 From nobody Sat Feb 7 18:15:09 2026 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id 15C11EB64DC for ; Wed, 12 Jul 2023 03:32:44 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S232116AbjGLDcm (ORCPT ); Tue, 11 Jul 2023 23:32:42 -0400 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:37650 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S229537AbjGLDcQ (ORCPT ); Tue, 11 Jul 2023 23:32:16 -0400 Received: from out0-207.mail.aliyun.com (out0-207.mail.aliyun.com [140.205.0.207]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id 053381BF5 for ; Tue, 11 Jul 2023 20:32:07 -0700 (PDT) X-Alimail-AntiSpam: AC=PASS;BC=-1|-1;BR=01201311R191e4;CH=green;DM=||false|;DS=||;FP=0|-1|-1|-1|0|-1|-1|-1;HT=ay29a033018047207;MF=houwenlong.hwl@antgroup.com;NM=1;PH=DS;RN=11;SR=0;TI=SMTPD_---.TrdHx8q_1689132719; Received: from localhost(mailfrom:houwenlong.hwl@antgroup.com fp:SMTPD_---.TrdHx8q_1689132719) by smtp.aliyun-inc.com; Wed, 12 Jul 2023 11:32:00 +0800 From: "Hou Wenlong" To: linux-kernel@vger.kernel.org Cc: "Lai Jiangshan" , "Hou Wenlong" , "Dave Hansen" , "Andy Lutomirski" , "Peter Zijlstra" , "Thomas Gleixner" , "Ingo Molnar" , "Borislav Petkov" , "=?UTF-8?B?bWFpbnRhaW5lcjpYODYgQVJDSElURUNUVVJFIDMyLUJJVCBBTkQgNjQtQklU?=" , "H. Peter Anvin" Subject: [PATCH RFC 7/7] x86/sme: Build the code in mem_encrypt_identity.c as PIE Date: Wed, 12 Jul 2023 11:30:11 +0800 Message-Id: <80367ca39ed736b6fc839c5de99a006f54182c45.1689130310.git.houwenlong.hwl@antgroup.com> X-Mailer: git-send-email 2.31.1 In-Reply-To: References: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Precedence: bulk List-ID: X-Mailing-List: linux-kernel@vger.kernel.org Content-Type: text/plain; charset="utf-8" Similar to head64.c, all the code in mem_encrypt_identity.c runs in identity address. However, it uses inline assembly to use RIP-relative reference for some globals. Therefore, build the code as PIE to force the compiler to generate RIP-relative reference, allowing for all inline assembly to be removed. Suggested-by: Lai Jiangshan Signed-off-by: Hou Wenlong --- The changes have not been tested on AMD SME. arch/x86/mm/Makefile | 3 +++ arch/x86/mm/mem_encrypt_identity.c | 31 +++++------------------------- 2 files changed, 8 insertions(+), 26 deletions(-) diff --git a/arch/x86/mm/Makefile b/arch/x86/mm/Makefile index c80febc44cd2..f5d7b22c5f1b 100644 --- a/arch/x86/mm/Makefile +++ b/arch/x86/mm/Makefile @@ -32,6 +32,9 @@ obj-y +=3D pat/ # Make sure __phys_addr has no stackprotector CFLAGS_physaddr.o :=3D -fno-stack-protector CFLAGS_mem_encrypt_identity.o :=3D -fno-stack-protector +CFLAGS_mem_encrypt_identity.o +=3D -fPIE -include $(srctree)/include/linu= x/hidden.h + +CFLAGS_REMOVE_mem_encrypt_identity.o +=3D -mcmodel=3Dkernel CFLAGS_fault.o :=3D -I $(srctree)/$(src)/../include/asm/trace diff --git a/arch/x86/mm/mem_encrypt_identity.c b/arch/x86/mm/mem_encrypt_i= dentity.c index 72aeb0f3dec6..2f292ab4e6a9 100644 --- a/arch/x86/mm/mem_encrypt_identity.c +++ b/arch/x86/mm/mem_encrypt_identity.c @@ -343,13 +343,7 @@ void __head sme_encrypt_kernel(struct boot_params *bp) } #endif - /* - * We're running identity mapped, so we must obtain the address to the - * SME encryption workarea using rip-relative addressing. - */ - asm ("lea sme_workarea(%%rip), %0" - : "=3Dr" (workarea_start) - : "p" (sme_workarea)); + workarea_start =3D (unsigned long)(void *)sme_workarea; /* * Calculate required number of workarea bytes needed: @@ -505,7 +499,7 @@ void __head sme_encrypt_kernel(struct boot_params *bp) void __head sme_enable(struct boot_params *bp) { - const char *cmdline_ptr, *cmdline_arg, *cmdline_on, *cmdline_off; + const char *cmdline_ptr; unsigned int eax, ebx, ecx, edx; unsigned long feature_mask; bool active_by_default; @@ -578,21 +572,6 @@ void __head sme_enable(struct boot_params *bp) goto out; } - /* - * Fixups have not been applied to phys_base yet and we're running - * identity mapped, so we must obtain the address to the SME command - * line argument data using rip-relative addressing. - */ - asm ("lea sme_cmdline_arg(%%rip), %0" - : "=3Dr" (cmdline_arg) - : "p" (sme_cmdline_arg)); - asm ("lea sme_cmdline_on(%%rip), %0" - : "=3Dr" (cmdline_on) - : "p" (sme_cmdline_on)); - asm ("lea sme_cmdline_off(%%rip), %0" - : "=3Dr" (cmdline_off) - : "p" (sme_cmdline_off)); - if (IS_ENABLED(CONFIG_AMD_MEM_ENCRYPT_ACTIVE_BY_DEFAULT)) active_by_default =3D true; else @@ -601,12 +580,12 @@ void __head sme_enable(struct boot_params *bp) cmdline_ptr =3D (const char *)((u64)bp->hdr.cmd_line_ptr | ((u64)bp->ext_cmd_line_ptr << 32)); - if (cmdline_find_option(cmdline_ptr, cmdline_arg, buffer, sizeof(buffer))= < 0) + if (cmdline_find_option(cmdline_ptr, sme_cmdline_arg, buffer, sizeof(buff= er)) < 0) return; - if (!strncmp(buffer, cmdline_on, sizeof(buffer))) + if (!strncmp(buffer, sme_cmdline_on, sizeof(buffer))) sme_me_mask =3D me_mask; - else if (!strncmp(buffer, cmdline_off, sizeof(buffer))) + else if (!strncmp(buffer, sme_cmdline_off, sizeof(buffer))) sme_me_mask =3D 0; else sme_me_mask =3D active_by_default ? me_mask : 0; -- 2.31.1