From nobody Sat Jul 25 15:50:34 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-alma10-1.taild15c8.ts.net [100.103.45.18]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id CA13E414A0F; Thu, 16 Jul 2026 14:09:52 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=100.103.45.18 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1784210994; cv=none; b=NuLlqiiCWmQyLQ6amMyDt8AMWiQih5sw7JTbFr/Kysi1WyRZ92Xxgs1ZQBsAk7ChfB4iOUQ5caZehW2fmE+ekiN1jNc19TTXRNyIC19wpRqZx2nGIl8GKulBkEo7WjsvPUHQYyDVNKt7W5Cp5S8bNlKZzUyyOHNXajYRxcEt6iw= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1784210994; c=relaxed/simple; bh=GctVr1DgD6Jz34y6GPxpYmwKbkU1HBpG+IhUiTfroCM=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=B+RasJy1OYPWJMQ80lqTwd87jGlG9ntRHvjhsMBsLh5WoZdo8/1Mc3Ar7nGJVFk2PxVgCM50iygrt6mXnn9091n9pOBgmrkGRlvYaO9Ai3dPCq2EZmlCEd7ZA7lBIwOjM373fPHk5lJLJ9AGdsovrtYq3k2vEVbujDfLb5rU+pk= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=A67Yizku; arc=none smtp.client-ip=100.103.45.18 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="A67Yizku" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 281F31F00A3A; Thu, 16 Jul 2026 14:09:47 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=kernel.org; s=k20260515; t=1784210992; bh=jCeHexlG946Nk3LJkXBgsYOFeHolftNzDBMOUV6Plgs=; h=From:To:Cc:Subject:Date:In-Reply-To:References; b=A67YizkuSaQzd14Fkiw3HkuWSsg3eRQhzLwXaXFF5VkzQX9jIgfYbaEP9SyjcqqH2 fgPS0zJKtIENwH+hdF2qVgiYMAzEV1r9W8jEE7j5UJTzY6JFhD3cM+RYevXfeaOdct PY2OkmglSBtMPB688v5H6bAwXcDHy0n3QqWL3m4hcluEJTkscsglvBLFny8xAXOBYa uFp+zKpChPg05TRS59JX8H6hWdAnIljnbYwMeOJ6iabeHRf4VyuFqI0YYc06q+eJXh RuFXkKyJLHbAT/Ri3Ys3D1KficczmUQb3KcvljZ5UG1M5vih3ldj0mudYjczpLDPgg Va+/8tKVUv/uQ== From: "Aneesh Kumar K.V (Arm)" To: linux-kernel@vger.kernel.org, linux-arm-kernel@lists.infradead.org, kvmarm@lists.linux.dev Cc: "Aneesh Kumar K.V (Arm)" , Marc Zyngier , Oliver Upton , Joey Gouly , Steffen Eiden , Suzuki K Poulose , Zenghui Yu , Catalin Marinas , Will Deacon Subject: [RFC PATCH 1/5] KVM: arm64: Make hyp symbol address conversion explicit Date: Thu, 16 Jul 2026 19:39:32 +0530 Message-ID: <20260716140936.4003182-2-aneesh.kumar@kernel.org> X-Mailer: git-send-email 2.43.0 In-Reply-To: <20260716140936.4003182-1-aneesh.kumar@kernel.org> References: <20260716140936.4003182-1-aneesh.kumar@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" kvm_ksym_ref() and kvm_ksym_ref_nvhe() have confusing names which do not clearly describe the address conversion they perform. Replace them with kvm_hyp_kimg_kaddr() for selecting the kernel address of a hyp symbol and kern_sym_hyp_va() for obtaining its hyp VA. Use lm_alias() directly where a linear-map address is required. This is a naming cleanup with no functional change. Signed-off-by: Aneesh Kumar K.V (Arm) --- arch/arm64/include/asm/kvm_asm.h | 10 --------- arch/arm64/include/asm/kvm_mmu.h | 33 +++++++++++++++++++++++++++ arch/arm64/kvm/arm.c | 38 ++++++++++++++++---------------- arch/arm64/kvm/va_layout.c | 2 +- 4 files changed, 53 insertions(+), 30 deletions(-) diff --git a/arch/arm64/include/asm/kvm_asm.h b/arch/arm64/include/asm/kvm_= asm.h index 043495f7fc78..6e0cc007bfe1 100644 --- a/arch/arm64/include/asm/kvm_asm.h +++ b/arch/arm64/include/asm/kvm_asm.h @@ -233,16 +233,6 @@ struct kvm_nvhe_stacktrace_info { unsigned long pc; }; =20 -/* Translate a kernel address @ptr into its equivalent linear mapping */ -#define kvm_ksym_ref(ptr) \ - ({ \ - void *val =3D (ptr); \ - if (!is_kernel_in_hyp_mode()) \ - val =3D lm_alias((ptr)); \ - val; \ - }) -#define kvm_ksym_ref_nvhe(sym) kvm_ksym_ref(kvm_nvhe_sym(sym)) - struct kvm; struct kvm_vcpu; struct kvm_s2_mmu; diff --git a/arch/arm64/include/asm/kvm_mmu.h b/arch/arm64/include/asm/kvm_= mmu.h index 6eae7e7e2a68..6b708ff176fd 100644 --- a/arch/arm64/include/asm/kvm_mmu.h +++ b/arch/arm64/include/asm/kvm_mmu.h @@ -107,6 +107,28 @@ u32 kvm_hyp_va_bits(void); void kvm_apply_hyp_relocations(void); =20 #define __hyp_pa(x) (((phys_addr_t)(x)) + hyp_physvirt_offset) +/** + * kvm_hyp_kimg_kaddr - Select the kernel address used for an EL2 mapping + * @ptr: Address within the kernel image + * + * Return @ptr when EL2 uses the kernel-image address directly, as with + * VHE. Otherwise return the corresponding linear-map alias used as input + * to the legacy nVHE VA conversion. + * + * Return: A kernel address suitable as the basis for an EL2 virtual addre= ss. + */ +static __always_inline void *kvm_hyp_kimg_kaddr(void *ptr) +{ + /* + * With VHE, the host runs at EL2 using the kernel mapping, so + * kernel-image VAs need no translation. + * Only legacy nVHE requires a linear-map alias. + */ + if (is_kernel_in_hyp_mode()) + return ptr; + + return lm_alias(ptr); +} =20 /* * Convert a kernel VA into a HYP VA. @@ -140,6 +162,17 @@ static __always_inline unsigned long __kern_hyp_va(uns= igned long v) =20 #define kern_hyp_va(v) ((typeof(v))(__kern_hyp_va((unsigned long)(v)))) =20 +/** + * kern_sym_hyp_va - Convert a kernel image symbol address to its runtime = hyp VA + * @ptr: Address of a symbol in the kernel image + * + * Return: The runtime hyp virtual address corresponding to @ptr. + */ +static __always_inline void *kern_sym_hyp_va(void *ptr) +{ + return kern_hyp_va(kvm_hyp_kimg_kaddr(ptr)); +} + extern u32 __hyp_va_bits; =20 /* diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c index 50adfff75be8..e1613186efb0 100644 --- a/arch/arm64/kvm/arm.c +++ b/arch/arm64/kvm/arm.c @@ -2105,10 +2105,10 @@ static int kvm_init_vector_slots(void) int err; void *base; =20 - base =3D kern_hyp_va(kvm_ksym_ref(__kvm_hyp_vector)); + base =3D kern_sym_hyp_va(__kvm_hyp_vector); kvm_init_vector_slot(base, HYP_VECTOR_DIRECT); =20 - base =3D kern_hyp_va(kvm_ksym_ref(__bp_harden_hyp_vecs)); + base =3D kern_sym_hyp_va(__bp_harden_hyp_vecs); kvm_init_vector_slot(base, HYP_VECTOR_SPECTRE_DIRECT); =20 if (kvm_system_needs_idmapped_vectors() && @@ -2136,7 +2136,7 @@ static void __init cpu_prepare_hyp_mode(int cpu, u32 = hyp_va_bits) * Also drop the KASAN tag which gets in the way... */ params->tpidr_el2 =3D (unsigned long)kasan_reset_tag(per_cpu_ptr_nvhe_sym= (__per_cpu_start, cpu)) - - (unsigned long)kvm_ksym_ref(CHOOSE_NVHE_SYM(__per_cpu_start)); + (unsigned long)kvm_hyp_kimg_kaddr(CHOOSE_NVHE_SYM(__per_cpu_start)); =20 params->mair_el2 =3D read_sysreg(mair_el1); =20 @@ -2532,13 +2532,13 @@ static void __init teardown_hyp_mode(void) =20 static int __init do_pkvm_init(u32 hyp_va_bits) { - void *per_cpu_base =3D kvm_ksym_ref(kvm_nvhe_sym(kvm_arm_hyp_percpu_base)= ); + void *per_cpu_base =3D kvm_nvhe_sym(kvm_arm_hyp_percpu_base); int ret; =20 preempt_disable(); cpu_hyp_init_context(); ret =3D kvm_call_hyp_nvhe(__pkvm_init, hyp_mem_base, hyp_mem_size, - kern_hyp_va(per_cpu_base), + kern_sym_hyp_va(per_cpu_base), hyp_va_bits); cpu_hyp_init_features(); =20 @@ -2611,8 +2611,8 @@ static void kvm_hyp_init_symbols(void) * Flush entire BSS since part of its data containing init symbols is read * while the MMU is off. */ - kvm_flush_dcache_to_poc(kvm_ksym_ref(__hyp_bss_start), - kvm_ksym_ref(__hyp_bss_end) - kvm_ksym_ref(__hyp_bss_start)); + kvm_flush_dcache_to_poc(lm_alias(__hyp_bss_start), + lm_alias(__hyp_bss_end) - lm_alias(__hyp_bss_start)); } =20 static int __init kvm_hyp_init_protection(u32 hyp_va_bits) @@ -2772,29 +2772,29 @@ static int __init init_hyp_mode(void) /* * Map the Hyp-code called directly from the host */ - err =3D create_hyp_mappings(kvm_ksym_ref(__hyp_text_start), - kvm_ksym_ref(__hyp_text_end), PAGE_HYP_EXEC); + err =3D create_hyp_mappings(kvm_hyp_kimg_kaddr(__hyp_text_start), + kvm_hyp_kimg_kaddr(__hyp_text_end), PAGE_HYP_EXEC); if (err) { kvm_err("Cannot map world-switch code\n"); goto out_err; } =20 - err =3D create_hyp_mappings(kvm_ksym_ref(__hyp_data_start), - kvm_ksym_ref(__hyp_data_end), PAGE_HYP); + err =3D create_hyp_mappings(kvm_hyp_kimg_kaddr(__hyp_data_start), + kvm_hyp_kimg_kaddr(__hyp_data_end), PAGE_HYP); if (err) { kvm_err("Cannot map .hyp.data section\n"); goto out_err; } =20 - err =3D create_hyp_mappings(kvm_ksym_ref(__hyp_rodata_start), - kvm_ksym_ref(__hyp_rodata_end), PAGE_HYP_RO); + err =3D create_hyp_mappings(kvm_hyp_kimg_kaddr(__hyp_rodata_start), + kvm_hyp_kimg_kaddr(__hyp_rodata_end), PAGE_HYP_RO); if (err) { kvm_err("Cannot map .hyp.rodata section\n"); goto out_err; } =20 - err =3D create_hyp_mappings(kvm_ksym_ref(__start_rodata), - kvm_ksym_ref(__end_rodata), PAGE_HYP_RO); + err =3D create_hyp_mappings(kvm_hyp_kimg_kaddr(__start_rodata), + kvm_hyp_kimg_kaddr(__end_rodata), PAGE_HYP_RO); if (err) { kvm_err("Cannot map rodata section\n"); goto out_err; @@ -2805,15 +2805,15 @@ static int __init init_hyp_mode(void) * section thanks to an assertion in the linker script. Map it RW and * the rest of .bss RO. */ - err =3D create_hyp_mappings(kvm_ksym_ref(__hyp_bss_start), - kvm_ksym_ref(__hyp_bss_end), PAGE_HYP); + err =3D create_hyp_mappings(kvm_hyp_kimg_kaddr(__hyp_bss_start), + kvm_hyp_kimg_kaddr(__hyp_bss_end), PAGE_HYP); if (err) { kvm_err("Cannot map hyp bss section: %d\n", err); goto out_err; } =20 - err =3D create_hyp_mappings(kvm_ksym_ref(__hyp_bss_end), - kvm_ksym_ref(__bss_stop), PAGE_HYP_RO); + err =3D create_hyp_mappings(kvm_hyp_kimg_kaddr(__hyp_bss_end), + kvm_hyp_kimg_kaddr(__bss_stop), PAGE_HYP_RO); if (err) { kvm_err("Cannot map bss section\n"); goto out_err; diff --git a/arch/arm64/kvm/va_layout.c b/arch/arm64/kvm/va_layout.c index 2346f9435a71..63bb58a7868a 100644 --- a/arch/arm64/kvm/va_layout.c +++ b/arch/arm64/kvm/va_layout.c @@ -223,7 +223,7 @@ void kvm_patch_vector_branch(struct alt_instr *alt, /* * Compute HYP VA by using the same computation as kern_hyp_va() */ - addr =3D __early_kern_hyp_va((u64)kvm_ksym_ref(__kvm_hyp_vector)); + addr =3D __early_kern_hyp_va((u64)kvm_hyp_kimg_kaddr(__kvm_hyp_vector)); =20 /* Use PC[10:7] to branch to the same vector in KVM */ addr |=3D ((u64)origptr & GENMASK_ULL(10, 7)); --=20 2.43.0 From nobody Sat Jul 25 15:50:34 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-alma10-1.taild15c8.ts.net [100.103.45.18]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 0E3BC41F37E; Thu, 16 Jul 2026 14:09:57 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=100.103.45.18 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1784210999; cv=none; b=IBsS0XfuIsOCdN4anLeRxnzTJiRw2/2OeKQ4bE+PvnwqgvHlfAK3DFMmqkjvqhWrs1JLl2SDaumh83yqZtloHuuRG9fDtV350/il3plCXgg0lJUxFTwqX+aYIXoQqrMetZCOjucq6OIECRwtCqe4rTfbwAeBMkglwCMHzppGypI= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1784210999; c=relaxed/simple; bh=w0juLWssNH6OlODoK0hQewuMPxF50K2J8wxRk8w5Q7Y=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=SNWjoLTLFXlTXV4TpG0/HDPW4e+DbvBQfVxvB7cuIuSUIhmKRZeVB373vBTfXPXg7C2Fh/Kn2LZwMDXDeLeGeU7ahBe7yyPqjx007hbqvSahygqVCPCM3xQtI+ZRKRucUAIbc/FVU8NmEetF1JXZrPSIAlP7i7ulw5SD+nI/2Zc= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=TaqlmgNH; arc=none smtp.client-ip=100.103.45.18 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="TaqlmgNH" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 128C81F000E9; Thu, 16 Jul 2026 14:09:52 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=kernel.org; s=k20260515; t=1784210997; bh=pGjkGxlJuV1y6dw2eZzjP+247wJLZyc/NOARJ+i0tsY=; h=From:To:Cc:Subject:Date:In-Reply-To:References; b=TaqlmgNHxf7v6dfxYDKUNK3NIREPzioMN1RM9+U49mVKlpU6dCBOGUbNJkT+lzBZn Bos1N7in0wJCKDbjkJUfejmLzgfF53FUhRJHpkN//s6evtmE+Ewjnk1r0sEbOB9ODR 252P8AfK4VDGqyA+Iu6DX12FWOCZTjCkq5+BABIQCwPQasORC6jXWbOcY16+eJ6kV+ gSVrzZbzJBlO1farHBA5Q6APephp1EC3MVfujeE0lB7i2+ucPhu/9RD44ErHOxin72 U9ishXTogAyFJ/yo9RjaVs1SjOo7jLRElugCYOYnKea0ePZbJfqYcC/n7ifnUp9S9E /XFL1CNqlDlZw== From: "Aneesh Kumar K.V (Arm)" To: linux-kernel@vger.kernel.org, linux-arm-kernel@lists.infradead.org, kvmarm@lists.linux.dev Cc: "Aneesh Kumar K.V (Arm)" , Marc Zyngier , Oliver Upton , Joey Gouly , Steffen Eiden , Suzuki K Poulose , Zenghui Yu , Catalin Marinas , Will Deacon Subject: [RFC PATCH 2/5] KVM: arm64: Split hyp mapping APIs by address type Date: Thu, 16 Jul 2026 19:39:33 +0530 Message-ID: <20260716140936.4003182-3-aneesh.kumar@kernel.org> X-Mailer: git-send-email 2.43.0 In-Reply-To: <20260716140936.4003182-1-aneesh.kumar@kernel.org> References: <20260716140936.4003182-1-aneesh.kumar@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Kernel-image symbols and linear-map addresses may refer to the same physical memory, but they are not interchangeable when constructing the EL2 address space. A kernel-image symbol requires __pa_symbol() for its physical address and kern_sym_hyp_va() for its runtime hyp VA, whereas a linear-map address requires __pa() and kern_hyp_va(). The existing create_hyp_mappings() and pkvm_create_mappings() interfaces do not express this distinction. Host callers prepare symbol addresses before invoking create_hyp_mappings(), while the common implementation attempts to infer the physical address from the resulting pointer. The pKVM implementation similarly assumes that every input can be translated using the linear hyp VA-to-PA offset. This ambiguity works while hyp symbols and linear allocations share the same fixed-offset translation. A later patch introduces an hVHE TTBR1 layout which retains kernel-image VAs for hyp symbols. That layout requires symbol addresses to be distinguished from linear-map addresses. Replace create_hyp_mappings() with explicit create_hyp_symbol_mappings() and create_hyp_linear_mappings() interfaces. Resolve the appropriate hyp VA and PA in each wrapper, then pass those values to a common mapping helper. This also removes the need for the generic kernel-address-to-physical-address classifier. Apply the same separation to the pKVM mapping code. Introduce symbol and linear-map wrappers around an internal helper which receives an already resolved hyp VA range and starting PA. Retain a locked linear variant for callers which already hold the hyp page-table lock. Update all callers to use the interface matching the address type they provide. Signed-off-by: Aneesh Kumar K.V (Arm) --- arch/arm64/include/asm/kvm_mmu.h | 5 +- arch/arm64/kvm/arm.c | 29 +++++------ arch/arm64/kvm/hyp/include/nvhe/mm.h | 8 ++- arch/arm64/kvm/hyp/nvhe/mem_protect.c | 16 ++++-- arch/arm64/kvm/hyp/nvhe/mm.c | 46 ++++++++++++----- arch/arm64/kvm/hyp/nvhe/setup.c | 18 ++++--- arch/arm64/kvm/hyp_trace.c | 2 +- arch/arm64/kvm/mmu.c | 71 +++++++++++++++------------ 8 files changed, 122 insertions(+), 73 deletions(-) diff --git a/arch/arm64/include/asm/kvm_mmu.h b/arch/arm64/include/asm/kvm_= mmu.h index 6b708ff176fd..8c2972aa83b0 100644 --- a/arch/arm64/include/asm/kvm_mmu.h +++ b/arch/arm64/include/asm/kvm_mmu.h @@ -190,7 +190,10 @@ extern u32 __hyp_va_bits; =20 int kvm_share_hyp(void *from, void *to); void kvm_unshare_hyp(void *from, void *to); -int create_hyp_mappings(void *from, void *to, enum kvm_pgtable_prot prot); +int create_hyp_linear_mappings(void *from, void *to, + enum kvm_pgtable_prot prot); +int create_hyp_symbol_mappings(void *from, void *to, + enum kvm_pgtable_prot prot); int __create_hyp_mappings(unsigned long start, unsigned long size, unsigned long phys, enum kvm_pgtable_prot prot); int hyp_alloc_private_va_range(size_t size, unsigned long *haddr); diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c index e1613186efb0..e498bb780382 100644 --- a/arch/arm64/kvm/arm.c +++ b/arch/arm64/kvm/arm.c @@ -2620,7 +2620,7 @@ static int __init kvm_hyp_init_protection(u32 hyp_va_= bits) void *addr =3D phys_to_virt(hyp_mem_base); int ret; =20 - ret =3D create_hyp_mappings(addr, addr + hyp_mem_size, PAGE_HYP); + ret =3D create_hyp_linear_mappings(addr, addr + hyp_mem_size, PAGE_HYP); if (ret) return ret; =20 @@ -2772,29 +2772,29 @@ static int __init init_hyp_mode(void) /* * Map the Hyp-code called directly from the host */ - err =3D create_hyp_mappings(kvm_hyp_kimg_kaddr(__hyp_text_start), - kvm_hyp_kimg_kaddr(__hyp_text_end), PAGE_HYP_EXEC); + err =3D create_hyp_symbol_mappings(__hyp_text_start, __hyp_text_end, + PAGE_HYP_EXEC); if (err) { kvm_err("Cannot map world-switch code\n"); goto out_err; } =20 - err =3D create_hyp_mappings(kvm_hyp_kimg_kaddr(__hyp_data_start), - kvm_hyp_kimg_kaddr(__hyp_data_end), PAGE_HYP); + err =3D create_hyp_symbol_mappings(__hyp_data_start, __hyp_data_end, + PAGE_HYP); if (err) { kvm_err("Cannot map .hyp.data section\n"); goto out_err; } =20 - err =3D create_hyp_mappings(kvm_hyp_kimg_kaddr(__hyp_rodata_start), - kvm_hyp_kimg_kaddr(__hyp_rodata_end), PAGE_HYP_RO); + err =3D create_hyp_symbol_mappings(__hyp_rodata_start, __hyp_rodata_end, + PAGE_HYP_RO); if (err) { kvm_err("Cannot map .hyp.rodata section\n"); goto out_err; } =20 - err =3D create_hyp_mappings(kvm_hyp_kimg_kaddr(__start_rodata), - kvm_hyp_kimg_kaddr(__end_rodata), PAGE_HYP_RO); + err =3D create_hyp_symbol_mappings(__start_rodata, __end_rodata, + PAGE_HYP_RO); if (err) { kvm_err("Cannot map rodata section\n"); goto out_err; @@ -2805,15 +2805,15 @@ static int __init init_hyp_mode(void) * section thanks to an assertion in the linker script. Map it RW and * the rest of .bss RO. */ - err =3D create_hyp_mappings(kvm_hyp_kimg_kaddr(__hyp_bss_start), - kvm_hyp_kimg_kaddr(__hyp_bss_end), PAGE_HYP); + err =3D create_hyp_symbol_mappings(__hyp_bss_start, __hyp_bss_end, + PAGE_HYP); if (err) { kvm_err("Cannot map hyp bss section: %d\n", err); goto out_err; } =20 - err =3D create_hyp_mappings(kvm_hyp_kimg_kaddr(__hyp_bss_end), - kvm_hyp_kimg_kaddr(__bss_stop), PAGE_HYP_RO); + err =3D create_hyp_symbol_mappings(__hyp_bss_end, __bss_stop, + PAGE_HYP_RO); if (err) { kvm_err("Cannot map bss section\n"); goto out_err; @@ -2846,7 +2846,8 @@ static int __init init_hyp_mode(void) char *percpu_end =3D percpu_begin + nvhe_percpu_size(); =20 /* Map Hyp percpu pages */ - err =3D create_hyp_mappings(percpu_begin, percpu_end, PAGE_HYP); + err =3D create_hyp_linear_mappings(percpu_begin, percpu_end, + PAGE_HYP); if (err) { kvm_err("Cannot map hyp percpu region\n"); goto out_err; diff --git a/arch/arm64/kvm/hyp/include/nvhe/mm.h b/arch/arm64/kvm/hyp/incl= ude/nvhe/mm.h index 6e83ce35c2f2..c41152c84705 100644 --- a/arch/arm64/kvm/hyp/include/nvhe/mm.h +++ b/arch/arm64/kvm/hyp/include/nvhe/mm.h @@ -23,8 +23,12 @@ int hyp_create_idmap(u32 hyp_va_bits); int hyp_map_vectors(void); int hyp_back_vmemmap(phys_addr_t back); int pkvm_cpu_set_vector(enum arm64_hyp_spectre_vector slot); -int pkvm_create_mappings(void *from, void *to, enum kvm_pgtable_prot prot); -int pkvm_create_mappings_locked(void *from, void *to, enum kvm_pgtable_pro= t prot); +int pkvm_create_linear_mappings(void *from, void *to, + enum kvm_pgtable_prot prot); +int pkvm_create_symbol_mappings(void *from, void *to, + enum kvm_pgtable_prot prot); +int pkvm_create_linear_mappings_locked(void *from, void *to, + enum kvm_pgtable_prot prot); int __pkvm_create_private_mapping(phys_addr_t phys, size_t size, enum kvm_pgtable_prot prot, unsigned long *haddr); diff --git a/arch/arm64/kvm/hyp/nvhe/mem_protect.c b/arch/arm64/kvm/hyp/nvh= e/mem_protect.c index 4e329e39a695..7f815345d6a5 100644 --- a/arch/arm64/kvm/hyp/nvhe/mem_protect.c +++ b/arch/arm64/kvm/hyp/nvhe/mem_protect.c @@ -1109,7 +1109,7 @@ int __pkvm_host_donate_hyp(u64 pfn, u64 nr_pages) goto unlock; =20 __hyp_set_page_state_range(phys, size, PKVM_PAGE_OWNED); - WARN_ON(pkvm_create_mappings_locked(virt, virt + size, PAGE_HYP)); + WARN_ON(pkvm_create_linear_mappings_locked(virt, virt + size, PAGE_HYP)); WARN_ON(host_stage2_set_owner_locked(phys, size, PKVM_ID_HYP)); =20 unlock: @@ -1175,12 +1175,18 @@ int hyp_pin_shared_mem(void *from, void *to) goto unlock; =20 for (cur =3D start; cur < end; cur +=3D PAGE_SIZE) { + void *addr =3D (void *)cur; + p =3D hyp_virt_to_page(cur); hyp_page_ref_inc(p); - if (p->refcount =3D=3D 1) - WARN_ON(pkvm_create_mappings_locked((void *)cur, - (void *)cur + PAGE_SIZE, - PAGE_HYP)); + if (p->refcount =3D=3D 1) { + int err; + + err =3D pkvm_create_linear_mappings_locked(addr, + addr + PAGE_SIZE, + PAGE_HYP); + WARN_ON(err); + } } =20 unlock: diff --git a/arch/arm64/kvm/hyp/nvhe/mm.c b/arch/arm64/kvm/hyp/nvhe/mm.c index 3b0bee496bff..9150ff94b01a 100644 --- a/arch/arm64/kvm/hyp/nvhe/mm.c +++ b/arch/arm64/kvm/hyp/nvhe/mm.c @@ -110,23 +110,21 @@ int __pkvm_create_private_mapping(phys_addr_t phys, s= ize_t size, return err; } =20 -int pkvm_create_mappings_locked(void *from, void *to, enum kvm_pgtable_pro= t prot) +static int __pkvm_create_mappings_locked(unsigned long start, + unsigned long end, + phys_addr_t phys, + enum kvm_pgtable_prot prot) { - unsigned long start =3D (unsigned long)from; - unsigned long end =3D (unsigned long)to; - unsigned long virt_addr; - phys_addr_t phys; - hyp_assert_lock_held(&pkvm_pgd_lock); =20 start =3D start & PAGE_MASK; end =3D PAGE_ALIGN(end); + phys =3D ALIGN_DOWN(phys, PAGE_SIZE); =20 - for (virt_addr =3D start; virt_addr < end; virt_addr +=3D PAGE_SIZE) { + for (; start < end; start +=3D PAGE_SIZE, phys +=3D PAGE_SIZE) { int err; =20 - phys =3D hyp_virt_to_phys((void *)virt_addr); - err =3D kvm_pgtable_hyp_map(&pkvm_pgtable, virt_addr, PAGE_SIZE, + err =3D kvm_pgtable_hyp_map(&pkvm_pgtable, start, PAGE_SIZE, phys, prot); if (err) return err; @@ -135,12 +133,38 @@ int pkvm_create_mappings_locked(void *from, void *to,= enum kvm_pgtable_prot prot return 0; } =20 -int pkvm_create_mappings(void *from, void *to, enum kvm_pgtable_prot prot) +int pkvm_create_linear_mappings_locked(void *from, void *to, + enum kvm_pgtable_prot prot) +{ + unsigned long start =3D (unsigned long)from & PAGE_MASK; + phys_addr_t phys =3D hyp_virt_to_phys((void *)start); + + return __pkvm_create_mappings_locked(start, (unsigned long)to, + phys, prot); +} + +int pkvm_create_linear_mappings(void *from, void *to, + enum kvm_pgtable_prot prot) +{ + int ret; + + hyp_spin_lock(&pkvm_pgd_lock); + ret =3D pkvm_create_linear_mappings_locked(from, to, prot); + hyp_spin_unlock(&pkvm_pgd_lock); + + return ret; +} + +int pkvm_create_symbol_mappings(void *from, void *to, + enum kvm_pgtable_prot prot) { + unsigned long start =3D (unsigned long)from & PAGE_MASK; + phys_addr_t phys =3D __hyp_pa(start); int ret; =20 hyp_spin_lock(&pkvm_pgd_lock); - ret =3D pkvm_create_mappings_locked(from, to, prot); + ret =3D __pkvm_create_mappings_locked(start, (unsigned long)to, + phys, prot); hyp_spin_unlock(&pkvm_pgd_lock); =20 return ret; diff --git a/arch/arm64/kvm/hyp/nvhe/setup.c b/arch/arm64/kvm/hyp/nvhe/setu= p.c index 75b00c323310..66a3d01cd79e 100644 --- a/arch/arm64/kvm/hyp/nvhe/setup.c +++ b/arch/arm64/kvm/hyp/nvhe/setup.c @@ -86,7 +86,7 @@ static int pkvm_create_host_sve_mappings(void) =20 start =3D kern_hyp_va(sve_regs); end =3D start + PAGE_ALIGN(pkvm_host_sve_state_size()); - ret =3D pkvm_create_mappings(start, end, PAGE_HYP); + ret =3D pkvm_create_linear_mappings(start, end, PAGE_HYP); if (ret) return ret; } @@ -121,23 +121,27 @@ static int recreate_hyp_mappings(phys_addr_t phys, un= signed long size, if (ret) return ret; =20 - ret =3D pkvm_create_mappings(__hyp_text_start, __hyp_text_end, PAGE_HYP_E= XEC); + ret =3D pkvm_create_symbol_mappings(__hyp_text_start, __hyp_text_end, + PAGE_HYP_EXEC); if (ret) return ret; =20 - ret =3D pkvm_create_mappings(__hyp_data_start, __hyp_data_end, PAGE_HYP); + ret =3D pkvm_create_symbol_mappings(__hyp_data_start, __hyp_data_end, + PAGE_HYP); if (ret) return ret; =20 - ret =3D pkvm_create_mappings(__hyp_rodata_start, __hyp_rodata_end, PAGE_H= YP_RO); + ret =3D pkvm_create_symbol_mappings(__hyp_rodata_start, __hyp_rodata_end, + PAGE_HYP_RO); if (ret) return ret; =20 - ret =3D pkvm_create_mappings(__hyp_bss_start, __hyp_bss_end, PAGE_HYP); + ret =3D pkvm_create_symbol_mappings(__hyp_bss_start, __hyp_bss_end, + PAGE_HYP); if (ret) return ret; =20 - ret =3D pkvm_create_mappings(virt, virt + size, PAGE_HYP); + ret =3D pkvm_create_linear_mappings(virt, virt + size, PAGE_HYP); if (ret) return ret; =20 @@ -146,7 +150,7 @@ static int recreate_hyp_mappings(phys_addr_t phys, unsi= gned long size, =20 start =3D (void *)kern_hyp_va(per_cpu_base[i]); end =3D start + PAGE_ALIGN(hyp_percpu_size); - ret =3D pkvm_create_mappings(start, end, PAGE_HYP); + ret =3D pkvm_create_linear_mappings(start, end, PAGE_HYP); if (ret) return ret; =20 diff --git a/arch/arm64/kvm/hyp_trace.c b/arch/arm64/kvm/hyp_trace.c index 2411b4c32932..0cf073682173 100644 --- a/arch/arm64/kvm/hyp_trace.c +++ b/arch/arm64/kvm/hyp_trace.c @@ -142,7 +142,7 @@ static int __map_hyp(void *start, size_t size) if (is_protected_kvm_enabled()) return 0; =20 - return create_hyp_mappings(start, start + size, PAGE_HYP); + return create_hyp_linear_mappings(start, start + size, PAGE_HYP); } =20 static int __share_page(unsigned long va) diff --git a/arch/arm64/kvm/mmu.c b/arch/arm64/kvm/mmu.c index 6c941aaa10c6..154d53dca619 100644 --- a/arch/arm64/kvm/mmu.c +++ b/arch/arm64/kvm/mmu.c @@ -436,17 +436,6 @@ int __create_hyp_mappings(unsigned long start, unsigne= d long size, return err; } =20 -static phys_addr_t kvm_kaddr_to_phys(void *kaddr) -{ - if (!is_vmalloc_addr(kaddr)) { - BUG_ON(!virt_addr_valid(kaddr)); - return __pa(kaddr); - } else { - return page_to_phys(vmalloc_to_page(kaddr)) + - offset_in_page(kaddr); - } -} - struct hyp_shared_pfn { u64 pfn; int count; @@ -559,7 +548,7 @@ int kvm_share_hyp(void *from, void *to) return -EINVAL; =20 if (kvm_host_owns_hyp_mappings()) - return create_hyp_mappings(from, to, PAGE_HYP); + return create_hyp_linear_mappings(from, to, PAGE_HYP); =20 start =3D ALIGN_DOWN(__pa(from), PAGE_SIZE); end =3D PAGE_ALIGN(__pa(to)); @@ -608,23 +597,10 @@ void kvm_unshare_hyp(void *from, void *to) } } =20 -/** - * create_hyp_mappings - duplicate a kernel virtual address range in Hyp m= ode - * @from: The virtual kernel start address of the range - * @to: The virtual kernel end address of the range (exclusive) - * @prot: The protection to be applied to this range - * - * The same virtual address as the kernel virtual address is also used - * in Hyp-mode mapping (modulo HYP_PAGE_OFFSET) to the same underlying - * physical pages. - */ -int create_hyp_mappings(void *from, void *to, enum kvm_pgtable_prot prot) +static int __create_hyp_mapping_range(unsigned long start, unsigned long e= nd, + phys_addr_t phys, + enum kvm_pgtable_prot prot) { - phys_addr_t phys_addr; - unsigned long virt_addr; - unsigned long start =3D kern_hyp_va((unsigned long)from); - unsigned long end =3D kern_hyp_va((unsigned long)to); - if (is_kernel_in_hyp_mode()) return 0; =20 @@ -633,13 +609,12 @@ int create_hyp_mappings(void *from, void *to, enum kv= m_pgtable_prot prot) =20 start =3D start & PAGE_MASK; end =3D PAGE_ALIGN(end); + phys =3D ALIGN_DOWN(phys, PAGE_SIZE); =20 - for (virt_addr =3D start; virt_addr < end; virt_addr +=3D PAGE_SIZE) { + for (; start < end; start +=3D PAGE_SIZE, phys +=3D PAGE_SIZE) { int err; =20 - phys_addr =3D kvm_kaddr_to_phys(from + virt_addr - start); - err =3D __create_hyp_mappings(virt_addr, PAGE_SIZE, phys_addr, - prot); + err =3D __create_hyp_mappings(start, PAGE_SIZE, phys, prot); if (err) return err; } @@ -647,6 +622,38 @@ int create_hyp_mappings(void *from, void *to, enum kvm= _pgtable_prot prot) return 0; } =20 +/** + * create_hyp_linear_mappings - map a host linear-map range at EL2 + * @from: The host linear-map start address + * @to: The host linear-map end address (exclusive) + * @prot: The protection to apply to the mapping + */ +int create_hyp_linear_mappings(void *from, void *to, + enum kvm_pgtable_prot prot) +{ + unsigned long start =3D kern_hyp_va((unsigned long)from); + unsigned long end =3D kern_hyp_va((unsigned long)to); + phys_addr_t phys =3D __pa((unsigned long)from & PAGE_MASK); + + return __create_hyp_mapping_range(start, end, phys, prot); +} + +/** + * create_hyp_symbol_mappings - map a kernel-image symbol range at EL2 + * @from: The kernel-image symbol at the start of the range + * @to: The kernel-image symbol at the end of the range (exclusive) + * @prot: The protection to apply to the mapping + */ +int create_hyp_symbol_mappings(void *from, void *to, + enum kvm_pgtable_prot prot) +{ + unsigned long start =3D (unsigned long)kern_sym_hyp_va(from); + unsigned long end =3D (unsigned long)kern_sym_hyp_va(to); + phys_addr_t phys =3D __pa_symbol((unsigned long)from & PAGE_MASK); + + return __create_hyp_mapping_range(start, end, phys, prot); +} + static int __hyp_alloc_private_va_range(unsigned long base) { lockdep_assert_held(&kvm_hyp_pgd_mutex); --=20 2.43.0 From nobody Sat Jul 25 15:50:34 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-alma10-1.taild15c8.ts.net [100.103.45.18]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id E38FE425884; Thu, 16 Jul 2026 14:10:02 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=100.103.45.18 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1784211004; cv=none; b=j0TALeQjbf4RICX/UxSU4kCogypgg0NolXsTLeALMzuLnXy1wEmCZb0zbLRgtvJRNuTWENO6cUb0zIT7D8Xsb8Zlat8QOvfQWbBF1JqolTc8anolZN/X/yvW4Bodr4qMM4jC7ck+QfR8iaHgIhifjN1KzWEV6nj+Zx85+plPf8Y= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1784211004; c=relaxed/simple; bh=Qkq4WHhNG2sp8SakFb2ZAOUpg/WLOqCsLiXhN2MZPFg=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=IaBusuAF2yn7vSa5aTcpQNQIaR7RLm1CckzTMQpmw5gbmfsFrAegpV1KrwzqYD1QsUeXvFEicWgUQzlKXTcXGEuHLmjKc5GYcVMpUTOrj4sew07c19HFNIEpzxpM9yWJRqB7l4AoKhFt2brbkbTYuz5TUyOR8WwjJHmu/FuIGBM= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=I74tlGvr; arc=none smtp.client-ip=100.103.45.18 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="I74tlGvr" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 65E6D1F00A3A; Thu, 16 Jul 2026 14:09:58 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=kernel.org; s=k20260515; t=1784211002; bh=qVn/qwPXA97H6053Zd5tcPuPNzdGfESKigCqcNfJyJ0=; h=From:To:Cc:Subject:Date:In-Reply-To:References; b=I74tlGvrFWwGVKwTtbJ3bETrDuWXDc1tJqvFSERH63hVCDVaDxplfDOtvlbOB30kb A1QK3LXx4dc0Ezu32LEOgk/pwAtoNhii8H89q76g/n/tjxdrGBSYR+Jst3x6XsF3Oa qNKkjq/AWPRDBESqA0yCiN+PSOyWmKzDGLa9SJGITwhB7/ehrWba/dk3tkRZTpssrn UEmxg7GOqer9ku3wM2X44lZVYJfss2XobeqF69A4njU6SO4a2M1MRzklOk/6hsrljF /nIDVSViR0/EfEcPtmwugZSW142+deMgK8QlQkz4Cd4tTRppm0HT1n+b2/VsnWwIOx f4FMZ6MSDEa1Q== From: "Aneesh Kumar K.V (Arm)" To: linux-kernel@vger.kernel.org, linux-arm-kernel@lists.infradead.org, kvmarm@lists.linux.dev Cc: "Aneesh Kumar K.V (Arm)" , Marc Zyngier , Oliver Upton , Joey Gouly , Steffen Eiden , Suzuki K Poulose , Zenghui Yu , Catalin Marinas , Will Deacon Subject: [RFC PATCH 3/5] KVM: arm64: Split hyp VA-to-PA conversion by address type Date: Thu, 16 Jul 2026 19:39:34 +0530 Message-ID: <20260716140936.4003182-4-aneesh.kumar@kernel.org> X-Mailer: git-send-email 2.43.0 In-Reply-To: <20260716140936.4003182-1-aneesh.kumar@kernel.org> References: <20260716140936.4003182-1-aneesh.kumar@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" The existing hyp VA-to-PA conversion uses one physvirt offset for both kernel-image symbols and linear-map addresses. This only works while both address types use the same EL2 VA transformation. With hVHE TTBR1, hyp symbols retain their kernel-image VAs while pools, per-CPU data and shared memory remain linear-map-derived. Introduce a separate symbol physvirt offset and explicit symbol and linear conversion helpers. Update every caller according to its address type and remove the ambiguous hyp_pa and __hyp_pa helpers. The offsets remain same as hyp_physvirt_offset, making this preparatory for the TTBR1 layout. Signed-off-by: Aneesh Kumar K.V (Arm) --- arch/arm64/include/asm/kvm_host.h | 2 ++ arch/arm64/include/asm/kvm_mmu.h | 17 ++++++++++++----- arch/arm64/kvm/arm.c | 2 +- arch/arm64/kvm/hyp/include/nvhe/memory.h | 13 +++++++------ arch/arm64/kvm/hyp/nvhe/early_alloc.c | 1 + arch/arm64/kvm/hyp/nvhe/events.c | 2 +- arch/arm64/kvm/hyp/nvhe/host.S | 4 ++-- arch/arm64/kvm/hyp/nvhe/mem_protect.c | 14 +++++++------- arch/arm64/kvm/hyp/nvhe/mm.c | 11 ++++++----- arch/arm64/kvm/hyp/nvhe/psci-relay.c | 12 ++++++------ arch/arm64/kvm/hyp/nvhe/setup.c | 4 ++-- arch/arm64/kvm/va_layout.c | 2 ++ 12 files changed, 49 insertions(+), 35 deletions(-) diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm= _host.h index bae2c4f92ef5..d92e0d066de3 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h @@ -827,6 +827,8 @@ extern struct kvm_host_psci_config kvm_nvhe_sym(kvm_hos= t_psci_config); =20 extern s64 kvm_nvhe_sym(hyp_physvirt_offset); #define hyp_physvirt_offset CHOOSE_NVHE_SYM(hyp_physvirt_offset) +extern s64 kvm_nvhe_sym(hyp_symbol_physvirt_offset); +#define hyp_symbol_physvirt_offset CHOOSE_NVHE_SYM(hyp_symbol_physvirt_off= set) =20 extern u64 kvm_nvhe_sym(hyp_cpu_logical_map)[NR_CPUS]; #define hyp_cpu_logical_map CHOOSE_NVHE_SYM(hyp_cpu_logical_map) diff --git a/arch/arm64/include/asm/kvm_mmu.h b/arch/arm64/include/asm/kvm_= mmu.h index 8c2972aa83b0..bcc8e28985f7 100644 --- a/arch/arm64/include/asm/kvm_mmu.h +++ b/arch/arm64/include/asm/kvm_mmu.h @@ -54,15 +54,21 @@ #include =20 /* - * Convert a hypervisor VA to a PA + * Convert an address in the hypervisor linear map to a PA * reg: hypervisor address to be converted in place * tmp: temporary register */ -.macro hyp_pa reg, tmp +.macro hyp_linear_pa reg, tmp ldr_l \tmp, hyp_physvirt_offset add \reg, \reg, \tmp .endm =20 +/* Convert a hypervisor kernel-image VA to a PA. */ +.macro hyp_symbol_pa reg, tmp + ldr_l \tmp, hyp_symbol_physvirt_offset + add \reg, \reg, \tmp +.endm + /* * Convert a hypervisor VA to a kernel image address * reg: hypervisor address to be converted in place @@ -74,8 +80,8 @@ * specific registers encoded in the instructions). */ .macro hyp_kimg_va reg, tmp - /* Convert hyp VA -> PA. */ - hyp_pa \reg, \tmp + /* Convert hyp kernel-image VA -> PA. */ + hyp_symbol_pa \reg, \tmp =20 /* Load kimage_voffset. */ alternative_cb ARM64_ALWAYS_SYSTEM, kvm_get_kimage_voffset @@ -106,7 +112,6 @@ void kvm_compute_layout(void); u32 kvm_hyp_va_bits(void); void kvm_apply_hyp_relocations(void); =20 -#define __hyp_pa(x) (((phys_addr_t)(x)) + hyp_physvirt_offset) /** * kvm_hyp_kimg_kaddr - Select the kernel address used for an EL2 mapping * @ptr: Address within the kernel image @@ -130,6 +135,8 @@ static __always_inline void *kvm_hyp_kimg_kaddr(void *p= tr) return lm_alias(ptr); } =20 +#define __hyp_linear_pa(x) (((phys_addr_t)(x)) + hyp_physvirt_offset) +#define __hyp_symbol_pa(x) (((phys_addr_t)(x)) + hyp_symbol_physvirt_offse= t) /* * Convert a kernel VA into a HYP VA. * diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c index e498bb780382..924f33c1a9d4 100644 --- a/arch/arm64/kvm/arm.c +++ b/arch/arm64/kvm/arm.c @@ -2835,7 +2835,7 @@ static int __init init_hyp_mode(void) /* * Save the stack PA in nvhe_init_params. This will be needed * to recreate the stack mapping in protected nVHE mode. - * __hyp_pa() won't do the right thing there, since the stack + * __hyp_linear_pa() won't do the right thing there, since the stack * has been mapped in the flexible private VA space. */ params->stack_pa =3D __pa(stack_base); diff --git a/arch/arm64/kvm/hyp/include/nvhe/memory.h b/arch/arm64/kvm/hyp/= include/nvhe/memory.h index b50712d47f6d..a7c3e70f7256 100644 --- a/arch/arm64/kvm/hyp/include/nvhe/memory.h +++ b/arch/arm64/kvm/hyp/include/nvhe/memory.h @@ -73,16 +73,17 @@ struct hyp_page { extern u64 __hyp_vmemmap; #define hyp_vmemmap ((struct hyp_page *)__hyp_vmemmap) =20 -#define __hyp_va(phys) ((void *)((phys_addr_t)(phys) - hyp_physvirt_offset= )) +#define __hyp_linear_va(phys) \ + ((void *)((phys_addr_t)(phys) - hyp_physvirt_offset)) =20 static inline void *hyp_phys_to_virt(phys_addr_t phys) { - return __hyp_va(phys); + return __hyp_linear_va(phys); } =20 static inline phys_addr_t hyp_virt_to_phys(void *addr) { - return __hyp_pa(addr); + return __hyp_linear_pa(addr); } =20 #define hyp_phys_to_pfn(phys) ((phys) >> PAGE_SHIFT) @@ -94,12 +95,12 @@ static inline struct hyp_page *hyp_phys_to_page(phys_ad= dr_t phys) return &hyp_vmemmap[hyp_phys_to_pfn(phys)]; } =20 -#define hyp_virt_to_page(virt) hyp_phys_to_page(__hyp_pa(virt)) -#define hyp_virt_to_pfn(virt) hyp_phys_to_pfn(__hyp_pa(virt)) +#define hyp_virt_to_page(virt) hyp_phys_to_page(__hyp_linear_pa(virt)) +#define hyp_virt_to_pfn(virt) hyp_phys_to_pfn(__hyp_linear_pa(virt)) =20 #define hyp_page_to_pfn(page) ((struct hyp_page *)(page) - hyp_vmemmap) #define hyp_page_to_phys(page) hyp_pfn_to_phys((hyp_page_to_pfn(page))) -#define hyp_page_to_virt(page) __hyp_va(hyp_page_to_phys(page)) +#define hyp_page_to_virt(page) __hyp_linear_va(hyp_page_to_phys(page)) #define hyp_page_to_pool(page) (((struct hyp_page *)page)->pool) =20 static inline enum pkvm_page_state get_host_state(struct hyp_page *p) diff --git a/arch/arm64/kvm/hyp/nvhe/early_alloc.c b/arch/arm64/kvm/hyp/nvh= e/early_alloc.c index 00de04153cc6..6002c61ae62c 100644 --- a/arch/arm64/kvm/hyp/nvhe/early_alloc.c +++ b/arch/arm64/kvm/hyp/nvhe/early_alloc.c @@ -11,6 +11,7 @@ =20 struct kvm_pgtable_mm_ops hyp_early_alloc_mm_ops; s64 __ro_after_init hyp_physvirt_offset; +s64 __ro_after_init hyp_symbol_physvirt_offset; =20 static unsigned long base; static unsigned long end; diff --git a/arch/arm64/kvm/hyp/nvhe/events.c b/arch/arm64/kvm/hyp/nvhe/eve= nts.c index add9383aadb5..4204cf81f0e4 100644 --- a/arch/arm64/kvm/hyp/nvhe/events.c +++ b/arch/arm64/kvm/hyp/nvhe/events.c @@ -17,7 +17,7 @@ int __tracing_enable_event(unsigned short id, bool enable) if (event_id >=3D __hyp_event_ids_end) return -EINVAL; =20 - enabled =3D hyp_fixmap_map(__hyp_pa(&event_id->enabled)); + enabled =3D hyp_fixmap_map(__hyp_symbol_pa(&event_id->enabled)); atomic_set(enabled, enable); hyp_fixmap_unmap(); =20 diff --git a/arch/arm64/kvm/hyp/nvhe/host.S b/arch/arm64/kvm/hyp/nvhe/host.S index 9393fe3ea6a1..079788f152d1 100644 --- a/arch/arm64/kvm/hyp/nvhe/host.S +++ b/arch/arm64/kvm/hyp/nvhe/host.S @@ -133,7 +133,7 @@ SYM_FUNC_START(__hyp_do_panic) mrs x0, esr_el2 mov x4, x3 mov x3, x2 - hyp_pa x3, x6 + hyp_symbol_pa x3, x6 get_vcpu_ptr x5, x6 mrs x6, far_el2 mrs x7, hpfar_el2 @@ -163,7 +163,7 @@ alternative_else_nop_endif * Preserve x0-x4, which may contain stub parameters. */ adr_l x5, __kvm_handle_stub_hvc - hyp_pa x5, x6 + hyp_symbol_pa x5, x6 br x5 SYM_FUNC_END(__host_hvc) =20 diff --git a/arch/arm64/kvm/hyp/nvhe/mem_protect.c b/arch/arm64/kvm/hyp/nvh= e/mem_protect.c index 7f815345d6a5..593359d04053 100644 --- a/arch/arm64/kvm/hyp/nvhe/mem_protect.c +++ b/arch/arm64/kvm/hyp/nvhe/mem_protect.c @@ -175,7 +175,7 @@ int kvm_host_prepare_stage2(void *pgt_pool_base) if (ret) return ret; =20 - mmu->pgd_phys =3D __hyp_pa(host_mmu.pgt.pgd); + mmu->pgd_phys =3D __hyp_linear_pa(host_mmu.pgt.pgd); mmu->pgt =3D &host_mmu.pgt; atomic64_set(&mmu->vmid.id, 0); =20 @@ -243,9 +243,9 @@ static void __apply_guest_page(void *va, size_t size, void *map; =20 if (IS_ALIGNED((unsigned long)va, PMD_SIZE) && size >=3D PMD_SIZE) - map =3D hyp_fixblock_map(__hyp_pa(va), &map_size); + map =3D hyp_fixblock_map(__hyp_linear_pa(va), &map_size); else - map =3D hyp_fixmap_map(__hyp_pa(va)); + map =3D hyp_fixmap_map(__hyp_linear_pa(va)); =20 func(map, map_size); =20 @@ -300,7 +300,7 @@ int kvm_guest_prepare_stage2(struct pkvm_hyp_vm *vm, vo= id *pgd) if (ret) return ret; =20 - vm->kvm.arch.mmu.pgd_phys =3D __hyp_pa(vm->pgt.pgd); + vm->kvm.arch.mmu.pgd_phys =3D __hyp_linear_pa(vm->pgt.pgd); =20 return 0; } @@ -1092,7 +1092,7 @@ int __pkvm_host_donate_hyp(u64 pfn, u64 nr_pages) { u64 phys =3D hyp_pfn_to_phys(pfn); u64 size =3D PAGE_SIZE * nr_pages; - void *virt =3D __hyp_va(phys); + void *virt =3D __hyp_linear_va(phys); int ret; =20 if (!pfn_range_is_valid(pfn, nr_pages)) @@ -1123,7 +1123,7 @@ int __pkvm_hyp_donate_host(u64 pfn, u64 nr_pages) { u64 phys =3D hyp_pfn_to_phys(pfn); u64 size =3D PAGE_SIZE * nr_pages; - u64 virt =3D (u64)__hyp_va(phys); + u64 virt =3D (u64)__hyp_linear_va(phys); int ret; =20 if (!pfn_range_is_valid(pfn, nr_pages)) @@ -1158,7 +1158,7 @@ int hyp_pin_shared_mem(void *from, void *to) { u64 cur, start =3D ALIGN_DOWN((u64)from, PAGE_SIZE); u64 end =3D PAGE_ALIGN((u64)to); - u64 phys =3D __hyp_pa(start); + u64 phys =3D __hyp_linear_pa(start); u64 size =3D end - start; struct hyp_page *p; int ret; diff --git a/arch/arm64/kvm/hyp/nvhe/mm.c b/arch/arm64/kvm/hyp/nvhe/mm.c index 9150ff94b01a..061fca95a78a 100644 --- a/arch/arm64/kvm/hyp/nvhe/mm.c +++ b/arch/arm64/kvm/hyp/nvhe/mm.c @@ -159,7 +159,7 @@ int pkvm_create_symbol_mappings(void *from, void *to, enum kvm_pgtable_prot prot) { unsigned long start =3D (unsigned long)from & PAGE_MASK; - phys_addr_t phys =3D __hyp_pa(start); + phys_addr_t phys =3D __hyp_symbol_pa(start); int ret; =20 hyp_spin_lock(&pkvm_pgd_lock); @@ -242,7 +242,7 @@ int hyp_map_vectors(void) return 0; } =20 - phys =3D __hyp_pa(__bp_harden_hyp_vecs); + phys =3D __hyp_symbol_pa(__bp_harden_hyp_vecs); ret =3D __pkvm_create_private_mapping(phys, __BP_HARDEN_HYP_VECS_SZ, PAGE_HYP_EXEC, &bp_base); if (ret) @@ -418,7 +418,8 @@ int hyp_create_fixmap(void) return ret; =20 ret =3D kvm_pgtable_hyp_map(&pkvm_pgtable, addr, PAGE_SIZE, - __hyp_pa(__hyp_bss_start), PAGE_HYP); + __hyp_symbol_pa(__hyp_bss_start), + PAGE_HYP); if (ret) return ret; =20 @@ -434,10 +435,10 @@ int hyp_create_idmap(u32 hyp_va_bits) { unsigned long start, end; =20 - start =3D hyp_virt_to_phys((void *)__hyp_idmap_text_start); + start =3D __hyp_symbol_pa(__hyp_idmap_text_start); start =3D ALIGN_DOWN(start, PAGE_SIZE); =20 - end =3D hyp_virt_to_phys((void *)__hyp_idmap_text_end); + end =3D __hyp_symbol_pa(__hyp_idmap_text_end); end =3D ALIGN(end, PAGE_SIZE); =20 /* diff --git a/arch/arm64/kvm/hyp/nvhe/psci-relay.c b/arch/arm64/kvm/hyp/nvhe= /psci-relay.c index e20db999e328..91b658ac74ee 100644 --- a/arch/arm64/kvm/hyp/nvhe/psci-relay.c +++ b/arch/arm64/kvm/hyp/nvhe/psci-relay.c @@ -139,8 +139,8 @@ static int psci_cpu_on(u64 func_id, struct kvm_cpu_cont= ext *host_ctxt) wmb(); =20 ret =3D psci_call(func_id, mpidr, - __hyp_pa(&kvm_hyp_cpu_entry), - __hyp_pa(init_params)); + __hyp_symbol_pa(&kvm_hyp_cpu_entry), + __hyp_linear_pa(init_params)); =20 /* If successful, the lock will be released by the target CPU. */ if (ret !=3D PSCI_RET_SUCCESS) @@ -173,8 +173,8 @@ static int psci_cpu_suspend(u64 func_id, struct kvm_cpu= _context *host_ctxt) * point if it is a deep sleep state. */ return psci_call(func_id, power_state, - __hyp_pa(&kvm_hyp_cpu_resume), - __hyp_pa(init_params)); + __hyp_symbol_pa(&kvm_hyp_cpu_resume), + __hyp_linear_pa(init_params)); } =20 static int psci_system_suspend(u64 func_id, struct kvm_cpu_context *host_c= txt) @@ -197,8 +197,8 @@ static int psci_system_suspend(u64 func_id, struct kvm_= cpu_context *host_ctxt) =20 /* Will only return on error. */ return psci_call(func_id, - __hyp_pa(&kvm_hyp_cpu_resume), - __hyp_pa(init_params), 0); + __hyp_symbol_pa(&kvm_hyp_cpu_resume), + __hyp_linear_pa(init_params), 0); } =20 static void __noreturn __kvm_host_psci_cpu_entry(unsigned long pc, unsigne= d long r0) diff --git a/arch/arm64/kvm/hyp/nvhe/setup.c b/arch/arm64/kvm/hyp/nvhe/setu= p.c index 66a3d01cd79e..2ef1972cc3dd 100644 --- a/arch/arm64/kvm/hyp/nvhe/setup.c +++ b/arch/arm64/kvm/hyp/nvhe/setup.c @@ -169,7 +169,7 @@ static void update_nvhe_init_params(void) =20 for (i =3D 0; i < hyp_nr_cpus; i++) { params =3D per_cpu_ptr(&kvm_init_params, i); - params->pgd_pa =3D __hyp_pa(pkvm_pgtable.pgd); + params->pgd_pa =3D __hyp_linear_pa(pkvm_pgtable.pgd); dcache_clean_inval_poc((unsigned long)params, (unsigned long)params + sizeof(*params)); } @@ -371,7 +371,7 @@ int __pkvm_init(phys_addr_t phys, unsigned long size, u= nsigned long *per_cpu_bas =20 /* Jump in the idmap page to switch to the new page-tables */ params =3D this_cpu_ptr(&kvm_init_params); - fn =3D (typeof(fn))__hyp_pa(__pkvm_init_switch_pgd); + fn =3D (typeof(fn))__hyp_symbol_pa(__pkvm_init_switch_pgd); fn(params->pgd_pa, params->stack_hyp_va, __pkvm_init_finalise); =20 unreachable(); diff --git a/arch/arm64/kvm/va_layout.c b/arch/arm64/kvm/va_layout.c index 63bb58a7868a..75a89ad8fecf 100644 --- a/arch/arm64/kvm/va_layout.c +++ b/arch/arm64/kvm/va_layout.c @@ -44,6 +44,8 @@ static void init_hyp_physvirt_offset(void) kern_va =3D (u64)lm_alias(__hyp_text_start); hyp_va =3D __early_kern_hyp_va(kern_va); hyp_physvirt_offset =3D (s64)__pa(kern_va) - (s64)hyp_va; + + hyp_symbol_physvirt_offset =3D hyp_physvirt_offset; } =20 /* --=20 2.43.0 From nobody Sat Jul 25 15:50:34 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-alma10-1.taild15c8.ts.net [100.103.45.18]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 1A8DF429810; Thu, 16 Jul 2026 14:10:07 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=100.103.45.18 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1784211008; cv=none; b=RuAQhkyPb6ys5LMJVelmflGT3x1Kacd6lxVogP/J3d6B9Wtq8rDOpN/mu3rwjHpRg6XUC3k3vBnxHAg1FS+x/RrEPfdSkN4Dd+IBnqgogZKLSfKoe1TWMGvO9jxPPpKZ1eedo5pyUEwz50hlTLJ6LFRJamI6sr30bjoLtCVHyJ8= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1784211008; c=relaxed/simple; bh=5Vhn9qPsQ17z4o4OUS4pFlj4NuxUdVv8pbmBvRTuB+E=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=X3+hWBssQYIKd7gafgZJDRTiMjpCrIjGubJkkhYu6FIHSgO+c81SIg7IZFVn/qozygueLJdRv+oxgQdwrg9QOm4/8fIKTZYZuL5HIkB5aaejJf381YUSNuydcNZAhXQIsuIGM4RReKUVxUx52KlmRmCwMv6ekGEc7V4F+enkxc4= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=ZeciVQYK; arc=none smtp.client-ip=100.103.45.18 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="ZeciVQYK" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 353CD1F00A3D; Thu, 16 Jul 2026 14:10:02 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=kernel.org; s=k20260515; t=1784211007; bh=iiBob4u3aYVxL2rvtOvM7lNyulBaXp//qwtdfCmAvSI=; h=From:To:Cc:Subject:Date:In-Reply-To:References; b=ZeciVQYKDBpXeGR5yBpMPBNFWFdnl6kXZ7TXw3NQOEkQepZ3fR3khduZJo2KByfLx qAQYt72eXClmm6VNVzfsCzS5dw9IkLHd+jOBZ8kCzWuTD51lCMR52B5m/Vc9SO+P8X hm+jMRYP+yY5lpXXidXqiQu+Vz5UVUAOuI1UBCa8sahjb0G0gCmRWaHh9XNaR9UeeU pQXg5qYN1cmbLxK9Udj0BmP1yn3IgVHiU/R3+TOsKNDC+QVWTwmEn9Cp78MssVCRQ5 YYr/ALI0T2vaqMRLj/r1jMONRmiSIhykV3mf8G5e703SXOs0BUy7fCL5uwy94gcBZC tt8DvbZ8sx87w== From: "Aneesh Kumar K.V (Arm)" To: linux-kernel@vger.kernel.org, linux-arm-kernel@lists.infradead.org, kvmarm@lists.linux.dev Cc: "Aneesh Kumar K.V (Arm)" , Marc Zyngier , Oliver Upton , Joey Gouly , Steffen Eiden , Suzuki K Poulose , Zenghui Yu , Catalin Marinas , Will Deacon Subject: [RFC PATCH 4/5] KVM: arm64: Rename the hyp private VA allocation base Date: Thu, 16 Jul 2026 19:39:35 +0530 Message-ID: <20260716140936.4003182-5-aneesh.kumar@kernel.org> X-Mailer: git-send-email 2.43.0 In-Reply-To: <20260716140936.4003182-1-aneesh.kumar@kernel.org> References: <20260716140936.4003182-1-aneesh.kumar@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" io_map_base is not used for I/O mappings. It is the allocation base for private hyp VAs used by stacks, fixmaps and others. Rename it to __hyp_private_va_base in the hyp code No functional change in this patch. Signed-off-by: Aneesh Kumar K.V (Arm) --- arch/arm64/kvm/hyp/nvhe/mm.c | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/arch/arm64/kvm/hyp/nvhe/mm.c b/arch/arm64/kvm/hyp/nvhe/mm.c index 061fca95a78a..3233a7c70f7c 100644 --- a/arch/arm64/kvm/hyp/nvhe/mm.c +++ b/arch/arm64/kvm/hyp/nvhe/mm.c @@ -24,7 +24,7 @@ hyp_spinlock_t pkvm_pgd_lock; struct memblock_region hyp_memory[HYP_MEMBLOCK_REGIONS]; unsigned int hyp_memblock_nr; =20 -static u64 __io_map_base; +static u64 __hyp_private_va_base; =20 struct hyp_fixmap_slot { u64 addr; @@ -50,7 +50,7 @@ static int __pkvm_alloc_private_va_range(unsigned long st= art, size_t size) =20 hyp_assert_lock_held(&pkvm_pgd_lock); =20 - if (!start || start < __io_map_base) + if (!start || start < __hyp_private_va_base) return -EINVAL; =20 /* The allocated size is always a multiple of PAGE_SIZE */ @@ -60,7 +60,7 @@ static int __pkvm_alloc_private_va_range(unsigned long st= art, size_t size) if (cur > __hyp_vmemmap) return -ENOMEM; =20 - __io_map_base =3D cur; + __hyp_private_va_base =3D cur; =20 return 0; } @@ -70,8 +70,8 @@ static int __pkvm_alloc_private_va_range(unsigned long st= art, size_t size) * @size: The size of the VA range to reserve. * @haddr: The hypervisor virtual start address of the allocation. * - * The private virtual address (VA) range is allocated above __io_map_base - * and aligned based on the order of @size. + * The private virtual address (VA) range is allocated from + * __hyp_private_va_base and aligned based on the order of @size. * * Return: 0 on success or negative error code on failure. */ @@ -81,7 +81,7 @@ int pkvm_alloc_private_va_range(size_t size, unsigned lon= g *haddr) int ret; =20 hyp_spin_lock(&pkvm_pgd_lock); - addr =3D __io_map_base; + addr =3D __hyp_private_va_base; ret =3D __pkvm_alloc_private_va_range(addr, size); hyp_spin_unlock(&pkvm_pgd_lock); =20 @@ -365,7 +365,7 @@ static int create_fixblock(void) return -EINVAL; =20 hyp_spin_lock(&pkvm_pgd_lock); - addr =3D ALIGN(__io_map_base, PMD_SIZE); + addr =3D ALIGN(__hyp_private_va_base, PMD_SIZE); ret =3D __pkvm_alloc_private_va_range(addr, PMD_SIZE); if (ret) goto unlock; @@ -446,12 +446,13 @@ int hyp_create_idmap(u32 hyp_va_bits) * memory -- see va_layout.c for more details. The other half of the VA * space contains the trampoline page, and needs some care. Split that * second half in two and find the quarter of VA space not conflicting - * with the idmap to place the IOs and the vmemmap. IOs use the lower - * half of the quarter and the vmemmap the upper half. + * with the idmap to place private mappings and the vmemmap. Private + * mappings use the lower half of the quarter and the vmemmap the upper + * half. */ - __io_map_base =3D start & BIT(hyp_va_bits - 2); - __io_map_base ^=3D BIT(hyp_va_bits - 2); - __hyp_vmemmap =3D __io_map_base | BIT(hyp_va_bits - 3); + __hyp_private_va_base =3D start & BIT(hyp_va_bits - 2); + __hyp_private_va_base ^=3D BIT(hyp_va_bits - 2); + __hyp_vmemmap =3D __hyp_private_va_base | BIT(hyp_va_bits - 3); =20 return __pkvm_create_mappings(start, end - start, start, PAGE_HYP_EXEC); } @@ -464,13 +465,13 @@ int pkvm_create_stack(phys_addr_t phys, unsigned long= *haddr) =20 hyp_spin_lock(&pkvm_pgd_lock); =20 - prev_base =3D __io_map_base; + prev_base =3D __hyp_private_va_base; /* * Efficient stack verification using the NVHE_STACK_SHIFT bit implies * an alignment of our allocation on the order of the size. */ size =3D NVHE_STACK_SIZE * 2; - addr =3D ALIGN(__io_map_base, size); + addr =3D ALIGN(__hyp_private_va_base, size); =20 ret =3D __pkvm_alloc_private_va_range(addr, size); if (!ret) { @@ -486,7 +487,7 @@ int pkvm_create_stack(phys_addr_t phys, unsigned long *= haddr) ret =3D kvm_pgtable_hyp_map(&pkvm_pgtable, addr + NVHE_STACK_SIZE, NVHE_STACK_SIZE, phys, PAGE_HYP); if (ret) - __io_map_base =3D prev_base; + __hyp_private_va_base =3D prev_base; } hyp_spin_unlock(&pkvm_pgd_lock); =20 --=20 2.43.0 From nobody Sat Jul 25 15:50:34 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-alma10-1.taild15c8.ts.net [100.103.45.18]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 8757C3FE660; Thu, 16 Jul 2026 14:10:12 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=100.103.45.18 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1784211014; cv=none; b=Ih21oefds3JochX5juTPAhpljrerNansluZeVKmJy5EQrZxPvlKZcM+xaK79QLSFf+wapBt/ngVivAXTEEIyuVtGtCM2pUHxlWBDKiidbBTwk/rnjYYz5Y6JB01SX9iYZInZ7knrGl2pqFHjchnfgAUeeIbw1w6urdcZBmlRY88= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1784211014; c=relaxed/simple; bh=nQgX0gW3D0MxlqQj8hNSakZmQWk5+JlEU7hGmfosL0o=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=jhJlI84sPAX9mlzfvrvdpiKnPP/DD8VEXKB3UF3prgzFK0vnHdOvWzkqbpcnmQD6bC9U2h5G1l6W8Sh7W/Nr18g+NpdaQwtPjvw4SM/BS2wfwQ8HPuRdZs3qfeJhAf/K9ivs9iqChpMVDYHxZJoxoWSbJFHbn7PGb0A6ZHZWHW4= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=E48SCxK9; arc=none smtp.client-ip=100.103.45.18 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="E48SCxK9" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 930CC1F000E9; Thu, 16 Jul 2026 14:10:07 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=kernel.org; s=k20260515; t=1784211012; bh=i+9aCree0QE72tWNWq9iGnJd39ScqLp1k1EqLmo1VjQ=; h=From:To:Cc:Subject:Date:In-Reply-To:References; b=E48SCxK9zTeDNz2D3T0ERHGZStkep9q6e+Qz3wT4CYaGv+98AES7pXM6Emxp08MuB Dba+LALbMFCnGsnMJNy7WRVGvtL/cF9zSteRJwKQSAzrsF1/rzxiBDTmMOD+RCl2gE BgQnE/Ltzeb4jyq5mJ9AV3zlAAU0ZoIva5aVQ3pDnCIy7VMsfpSbU9t4fQA8Dz0zEc wdQXzDQQ9k0EVfRR1vnOScuki5XuJ54ThzaI94BwZ3NWDsVRKlXiV8yYCkCflvxCWU 0hZwmCnfKfGeCLJV8U6OmSmKK3QiAlhifsYNlbH7niFpfbDndGR6qOkdhbHkFPfCft ijZnfrwp8sbKA== From: "Aneesh Kumar K.V (Arm)" To: linux-kernel@vger.kernel.org, linux-arm-kernel@lists.infradead.org, kvmarm@lists.linux.dev Cc: "Aneesh Kumar K.V (Arm)" , Marc Zyngier , Oliver Upton , Joey Gouly , Steffen Eiden , Suzuki K Poulose , Zenghui Yu , Catalin Marinas , Will Deacon Subject: [RFC PATCH 5/5] KVM: arm64: Use TTBR1_EL2 for hVHE runtime mappings Date: Thu, 16 Jul 2026 19:39:36 +0530 Message-ID: <20260716140936.4003182-6-aneesh.kumar@kernel.org> X-Mailer: git-send-email 2.43.0 In-Reply-To: <20260716140936.4003182-1-aneesh.kumar@kernel.org> References: <20260716140936.4003182-1-aneesh.kumar@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" hVHE currently disables TTBR1_EL2 and places both the identity mapping and runtime hyp mappings in TTBR0_EL2. This unnecessarily retains the legacy nVHE VA layout even though hVHE can use canonical kernel-image addresses through the EL2&0 translation regime. Place hVHE runtime mappings in TTBR1_EL2 and reserve TTBR0_EL2 for the identity mapping used during MMU and page-table transitions. Select this layout based on hVHE support independently of protected mode, while retaining the existing TTBR0-only layout for nVHE and protected nVHE. Keep hyp symbols at their linked kernel-image VAs, disable the legacy VA relocations for hVHE, and configure TCR_EL2 for both translation ranges. Allocate separate runtime and idmap page tables during initial setup and pKVM finalization, pass both roots through the per-CPU initialization parameters, and install both during page-table transitions. Add support for walking canonical TTBR1 hyp page tables, including handling the wrap at the top of the address space. Use this when fixing page-table reference counts and destroying hyp page tables. Signed-off-by: Aneesh Kumar K.V (Arm) --- arch/arm64/include/asm/kvm_asm.h | 3 +- arch/arm64/include/asm/kvm_hyp.h | 4 +- arch/arm64/include/asm/kvm_mmu.h | 46 ++++++++++++++++++--- arch/arm64/include/asm/kvm_pgtable.h | 27 ++++++++++++ arch/arm64/include/asm/kvm_pkvm.h | 3 ++ arch/arm64/kernel/asm-offsets.c | 5 ++- arch/arm64/kvm/arm.c | 47 +++++++++++++-------- arch/arm64/kvm/hyp/include/nvhe/mm.h | 1 + arch/arm64/kvm/hyp/nvhe/hyp-init.S | 29 ++++++++++--- arch/arm64/kvm/hyp/nvhe/mm.c | 24 +++++++++++ arch/arm64/kvm/hyp/nvhe/setup.c | 35 ++++++++++++++-- arch/arm64/kvm/hyp/pgtable.c | 62 +++++++++++++++++++++++++--- arch/arm64/kvm/mmu.c | 57 +++++++++++++++++++++++-- arch/arm64/kvm/pkvm.c | 3 ++ arch/arm64/kvm/va_layout.c | 32 +++++++++++--- 15 files changed, 328 insertions(+), 50 deletions(-) diff --git a/arch/arm64/include/asm/kvm_asm.h b/arch/arm64/include/asm/kvm_= asm.h index 6e0cc007bfe1..e42baadf7990 100644 --- a/arch/arm64/include/asm/kvm_asm.h +++ b/arch/arm64/include/asm/kvm_asm.h @@ -210,7 +210,8 @@ struct kvm_nvhe_init_params { unsigned long tpidr_el2; unsigned long stack_hyp_va; unsigned long stack_pa; - phys_addr_t pgd_pa; + phys_addr_t ttbr0_pgd_pa; + phys_addr_t ttbr1_pgd_pa; unsigned long hcr_el2; unsigned long vttbr; unsigned long vtcr; diff --git a/arch/arm64/include/asm/kvm_hyp.h b/arch/arm64/include/asm/kvm_= hyp.h index 4974492744cc..5cc196f84958 100644 --- a/arch/arm64/include/asm/kvm_hyp.h +++ b/arch/arm64/include/asm/kvm_hyp.h @@ -131,8 +131,8 @@ void __noreturn __hyp_do_panic(struct kvm_cpu_context *= host_ctxt, u64 spsr, #endif =20 #ifdef __KVM_NVHE_HYPERVISOR__ -void __pkvm_init_switch_pgd(phys_addr_t pgd, unsigned long sp, - void (*fn)(void)); +void __pkvm_init_switch_pgd(phys_addr_t ttbr0, phys_addr_t ttbr1, + unsigned long sp, void (*fn)(void)); int __pkvm_init(phys_addr_t phys, unsigned long size, unsigned long *per_c= pu_base, u32 hyp_va_bits); void __noreturn __host_enter(struct kvm_cpu_context *host_ctxt); #endif diff --git a/arch/arm64/include/asm/kvm_mmu.h b/arch/arm64/include/asm/kvm_= mmu.h index bcc8e28985f7..30ac5cecaa93 100644 --- a/arch/arm64/include/asm/kvm_mmu.h +++ b/arch/arm64/include/asm/kvm_mmu.h @@ -105,6 +105,7 @@ alternative_cb_end #include #include #include +#include =20 void kvm_update_va_mask(struct alt_instr *alt, __le32 *origptr, __le32 *updptr, int nr_inst); @@ -112,13 +113,28 @@ void kvm_compute_layout(void); u32 kvm_hyp_va_bits(void); void kvm_apply_hyp_relocations(void); =20 +static __always_inline bool kvm_hyp_init_uses_ttbr1(void) +{ + BUILD_BUG_ON(__is_defined(__KVM_NVHE_HYPERVISOR__)); + + return arm64_test_sw_feature_override(ARM64_SW_FEATURE_OVERRIDE_HVHE); +} + +static __always_inline bool kvm_hyp_uses_ttbr1(void) +{ + return cpus_have_final_cap(ARM64_KVM_HVHE); +} + /** * kvm_hyp_kimg_kaddr - Select the kernel address used for an EL2 mapping * @ptr: Address within the kernel image * - * Return @ptr when EL2 uses the kernel-image address directly, as with - * VHE. Otherwise return the corresponding linear-map alias used as input - * to the legacy nVHE VA conversion. + * Return @ptr when EL2 uses the kernel-image address directly, as with VHE + * or hVHE TTBR1. Otherwise return the corresponding linear-map alias used + * as input to the legacy nVHE VA conversion. + * + * This helper relies on finalized CPU capabilities and must not be used f= rom + * earlier boot stages. * * Return: A kernel address suitable as the basis for an EL2 virtual addre= ss. */ @@ -126,10 +142,11 @@ static __always_inline void *kvm_hyp_kimg_kaddr(void = *ptr) { /* * With VHE, the host runs at EL2 using the kernel mapping, so - * kernel-image VAs need no translation. - * Only legacy nVHE requires a linear-map alias. + * kernel-image VAs need no translation. hVHE similarly retains + * kernel-image VAs in its TTBR1 EL2 mapping. Only legacy nVHE + * requires a linear-map alias. */ - if (is_kernel_in_hyp_mode()) + if (is_kernel_in_hyp_mode() || kvm_hyp_uses_ttbr1()) return ptr; =20 return lm_alias(ptr); @@ -137,6 +154,22 @@ static __always_inline void *kvm_hyp_kimg_kaddr(void *= ptr) =20 #define __hyp_linear_pa(x) (((phys_addr_t)(x)) + hyp_physvirt_offset) #define __hyp_symbol_pa(x) (((phys_addr_t)(x)) + hyp_symbol_physvirt_offse= t) + +static __always_inline unsigned long kvm_hyp_ttbr1_private_start(void) +{ + return PAGE_ALIGN((unsigned long)__hyp_bss_end); +} + +static __always_inline unsigned long kvm_hyp_ttbr1_private_end(void) +{ + return VMALLOC_END; +} + +static __always_inline unsigned long kvm_hyp_ttbr1_vmemmap_base(void) +{ + return kvm_hyp_ttbr1_private_start() + SZ_1G; +} + /* * Convert a kernel VA into a HYP VA. * @@ -228,6 +261,7 @@ int kvm_handle_guest_sea(struct kvm_vcpu *vcpu); int kvm_handle_guest_abort(struct kvm_vcpu *vcpu); =20 phys_addr_t kvm_mmu_get_httbr(void); +phys_addr_t kvm_mmu_get_hyp_ttbr1(void); phys_addr_t kvm_get_idmap_vector(void); int __init kvm_mmu_init(u32 hyp_va_bits); =20 diff --git a/arch/arm64/include/asm/kvm_pgtable.h b/arch/arm64/include/asm/= kvm_pgtable.h index 41a8687938eb..1d51dcaaac0d 100644 --- a/arch/arm64/include/asm/kvm_pgtable.h +++ b/arch/arm64/include/asm/kvm_pgtable.h @@ -255,6 +255,14 @@ enum kvm_pgtable_stage2_flags { KVM_PGTABLE_S2_AS_S1 =3D BIT(1), }; =20 +/** + * enum kvm_pgtable_hyp_flags - Hypervisor page-table flags. + * @KVM_PGTABLE_HYP_TTBR1: Walk TTBR1 canonical virtual addresses. + */ +enum kvm_pgtable_hyp_flags { + KVM_PGTABLE_HYP_TTBR1 =3D BIT(0), +}; + /** * enum kvm_pgtable_prot - Page-table permissions and attributes. * @KVM_PGTABLE_PROT_UX: Unprivileged execute permission. @@ -443,6 +451,7 @@ static inline bool kvm_pgtable_walk_lock_held(void) * @start_level: Level at which the page-table walk starts. * @pgd: Pointer to the first top-level entry of the page-table. * @mm_ops: Memory management callbacks. + * @hyp_flags: Hypervisor page-table flags. * @mmu: Stage-2 KVM MMU struct. Unused for stage-1 page-tables. * @flags: Stage-2 page-table flags. * @force_pte_cb: Function that returns true if page level mappings must @@ -457,6 +466,9 @@ struct kvm_pgtable { kvm_pteref_t pgd; struct kvm_pgtable_mm_ops *mm_ops; =20 + /* Hyp only */ + enum kvm_pgtable_hyp_flags hyp_flags; + /* Stage-2 only */ enum kvm_pgtable_stage2_flags flags; kvm_pgtable_force_pte_cb_t force_pte_cb; @@ -465,6 +477,11 @@ struct kvm_pgtable { struct kvm_s2_mmu *mmu; }; =20 +static inline void kvm_pgtable_hyp_enable_ttbr1(struct kvm_pgtable *pgt) +{ + pgt->hyp_flags |=3D KVM_PGTABLE_HYP_TTBR1; +} + /** * kvm_pgtable_hyp_init() - Initialise a hypervisor stage-1 page-table. * @pgt: Uninitialised page-table structure to initialise. @@ -851,6 +868,16 @@ int kvm_pgtable_stage2_split(struct kvm_pgtable *pgt, = u64 addr, u64 size, int kvm_pgtable_walk(struct kvm_pgtable *pgt, u64 addr, u64 size, struct kvm_pgtable_walker *walker); =20 +/** + * kvm_pgtable_hyp_walk() - Walk the entire usable VA range of a hyp pgtab= le. + * @pgt: Page-table structure initialised by kvm_pgtable_hyp_init(). + * @walker: Walker callback description. + * + * Return: 0 on success, negative error code on failure. + */ +int kvm_pgtable_hyp_walk(struct kvm_pgtable *pgt, + struct kvm_pgtable_walker *walker); + /** * kvm_pgtable_get_leaf() - Walk a page-table and retrieve the leaf entry * with its level. diff --git a/arch/arm64/include/asm/kvm_pkvm.h b/arch/arm64/include/asm/kvm= _pkvm.h index 74fedd9c5ff0..e632fe4ec43c 100644 --- a/arch/arm64/include/asm/kvm_pkvm.h +++ b/arch/arm64/include/asm/kvm_pkvm.h @@ -139,6 +139,9 @@ static inline unsigned long hyp_s1_pgtable_pages(void) /* Allow 1 GiB for private mappings */ res +=3D __hyp_pgtable_max_pages(SZ_1G >> PAGE_SHIFT); =20 + /* Allow a separate TTBR0 idmap page-table. */ + res +=3D __hyp_pgtable_max_pages(1); + return res; } =20 diff --git a/arch/arm64/kernel/asm-offsets.c b/arch/arm64/kernel/asm-offset= s.c index b6367ff3a49c..f95883771d34 100644 --- a/arch/arm64/kernel/asm-offsets.c +++ b/arch/arm64/kernel/asm-offsets.c @@ -120,7 +120,10 @@ int main(void) DEFINE(NVHE_INIT_TCR_EL2, offsetof(struct kvm_nvhe_init_params, tcr_el2)= ); DEFINE(NVHE_INIT_TPIDR_EL2, offsetof(struct kvm_nvhe_init_params, tpidr_= el2)); DEFINE(NVHE_INIT_STACK_HYP_VA, offsetof(struct kvm_nvhe_init_params, sta= ck_hyp_va)); - DEFINE(NVHE_INIT_PGD_PA, offsetof(struct kvm_nvhe_init_params, pgd_pa)); + DEFINE(NVHE_INIT_TTBR0_PGD_PA, + offsetof(struct kvm_nvhe_init_params, ttbr0_pgd_pa)); + DEFINE(NVHE_INIT_TTBR1_PGD_PA, + offsetof(struct kvm_nvhe_init_params, ttbr1_pgd_pa)); DEFINE(NVHE_INIT_HCR_EL2, offsetof(struct kvm_nvhe_init_params, hcr_el2)= ); DEFINE(NVHE_INIT_VTTBR, offsetof(struct kvm_nvhe_init_params, vttbr)); DEFINE(NVHE_INIT_VTCR, offsetof(struct kvm_nvhe_init_params, vtcr)); diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c index 924f33c1a9d4..f2ea75461c1f 100644 --- a/arch/arm64/kvm/arm.c +++ b/arch/arm64/kvm/arm.c @@ -2124,38 +2124,51 @@ static int kvm_init_vector_slots(void) return 0; } =20 +static unsigned long __init kvm_compute_hyp_tcr(u32 hyp_va_bits) +{ + unsigned long tcr =3D read_sysreg(tcr_el1); + + if (cpus_have_final_cap(ARM64_KVM_HVHE)) { + tcr &=3D ~(TCR_HD | TCR_HA | TCR_A1 | TCR_T0SZ_MASK | + TCR_T1SZ_MASK | + TCR_EPD0_MASK | TCR_EPD1_MASK); + + if (kvm_hyp_uses_ttbr1()) + tcr |=3D TCR_T1SZ(hyp_va_bits); + else + tcr |=3D TCR_EPD1_MASK; + } else { + unsigned long ips =3D FIELD_GET(TCR_IPS_MASK, tcr); + + tcr &=3D TCR_EL2_MASK; + tcr |=3D TCR_EL2_RES1 | FIELD_PREP(TCR_EL2_PS_MASK, ips); + if (lpa2_is_enabled()) + tcr |=3D TCR_EL2_DS; + } + + return tcr | TCR_T0SZ(hyp_va_bits); +} + static void __init cpu_prepare_hyp_mode(int cpu, u32 hyp_va_bits) { struct kvm_nvhe_init_params *params =3D per_cpu_ptr_nvhe_sym(kvm_init_par= ams, cpu); - unsigned long tcr; =20 /* * Calculate the raw per-cpu offset without a translation from the * kernel's mapping to the linear mapping, and store it in tpidr_el2 * so that we can use adr_l to access per-cpu variables in EL2. + * In TTBR1 mode, this is the delta from the canonical hyp symbol to + * the linear-map VA at which the allocated per-cpu copy is mapped. * Also drop the KASAN tag which gets in the way... */ params->tpidr_el2 =3D (unsigned long)kasan_reset_tag(per_cpu_ptr_nvhe_sym= (__per_cpu_start, cpu)) - (unsigned long)kvm_hyp_kimg_kaddr(CHOOSE_NVHE_SYM(__per_cpu_start)); =20 params->mair_el2 =3D read_sysreg(mair_el1); + params->tcr_el2 =3D kvm_compute_hyp_tcr(hyp_va_bits); =20 - tcr =3D read_sysreg(tcr_el1); - if (cpus_have_final_cap(ARM64_KVM_HVHE)) { - tcr &=3D ~(TCR_HD | TCR_HA | TCR_A1 | TCR_T0SZ_MASK); - tcr |=3D TCR_EPD1_MASK; - } else { - unsigned long ips =3D FIELD_GET(TCR_IPS_MASK, tcr); - - tcr &=3D TCR_EL2_MASK; - tcr |=3D TCR_EL2_RES1 | FIELD_PREP(TCR_EL2_PS_MASK, ips); - if (lpa2_is_enabled()) - tcr |=3D TCR_EL2_DS; - } - tcr |=3D TCR_T0SZ(hyp_va_bits); - params->tcr_el2 =3D tcr; - - params->pgd_pa =3D kvm_mmu_get_httbr(); + params->ttbr0_pgd_pa =3D kvm_mmu_get_httbr(); + params->ttbr1_pgd_pa =3D kvm_mmu_get_hyp_ttbr1(); if (is_protected_kvm_enabled()) params->hcr_el2 =3D HCR_HOST_NVHE_PROTECTED_FLAGS; else diff --git a/arch/arm64/kvm/hyp/include/nvhe/mm.h b/arch/arm64/kvm/hyp/incl= ude/nvhe/mm.h index c41152c84705..46f008ee6916 100644 --- a/arch/arm64/kvm/hyp/include/nvhe/mm.h +++ b/arch/arm64/kvm/hyp/include/nvhe/mm.h @@ -11,6 +11,7 @@ #include =20 extern struct kvm_pgtable pkvm_pgtable; +extern struct kvm_pgtable pkvm_idmap_pgtable; extern hyp_spinlock_t pkvm_pgd_lock; =20 int hyp_create_fixmap(void); diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-init.S b/arch/arm64/kvm/hyp/nvhe/h= yp-init.S index 89cb553be1e5..07fc29c34fb3 100644 --- a/arch/arm64/kvm/hyp/nvhe/hyp-init.S +++ b/arch/arm64/kvm/hyp/nvhe/hyp-init.S @@ -127,13 +127,22 @@ SYM_CODE_START_LOCAL(___kvm_hyp_init) ldr x1, [x0, #NVHE_INIT_VTCR] msr vtcr_el2, x1 =20 - ldr x1, [x0, #NVHE_INIT_PGD_PA] + ldr x1, [x0, #NVHE_INIT_TTBR0_PGD_PA] phys_to_ttbr x2, x1 alternative_if ARM64_HAS_CNP orr x2, x2, #TTBRx_EL1_CnP alternative_else_nop_endif msr ttbr0_el2, x2 =20 + ldr x1, [x0, #NVHE_INIT_TTBR1_PGD_PA] + cbz x1, 2f + phys_to_ttbr x2, x1 +alternative_if ARM64_HAS_CNP + orr x2, x2, #TTBRx_EL1_CnP +alternative_else_nop_endif + msr_s SYS_TTBR1_EL2, x2 +2: + ldr x0, [x0, #NVHE_INIT_TCR_EL2] msr tcr_el2, x0 =20 @@ -261,13 +270,15 @@ reset: SYM_CODE_END(__kvm_handle_stub_hvc) =20 /* - * void __pkvm_init_switch_pgd(phys_addr_t pgd, unsigned long sp, - * void (*fn)(void)); + * void __pkvm_init_switch_pgd(phys_addr_t ttbr0, phys_addr_t ttbr1, + * unsigned long sp, void (*fn)(void)); * * SYM_TYPED_FUNC_START() allows C to call this ID-mapped function indirec= tly * using a physical pointer without triggering a kCFI failure. */ SYM_TYPED_FUNC_START(__pkvm_init_switch_pgd) + mov x6, x3 + /* Turn the MMU off */ pre_disable_mmu_workaround mrs x3, sctlr_el2 @@ -284,14 +295,22 @@ alternative_if ARM64_HAS_CNP alternative_else_nop_endif msr ttbr0_el2, x5 =20 + cbz x1, 1f + phys_to_ttbr x5, x1 +alternative_if ARM64_HAS_CNP + orr x5, x5, #TTBRx_EL1_CnP +alternative_else_nop_endif + msr_s SYS_TTBR1_EL2, x5 +1: + /* Set the new stack pointer */ - mov sp, x1 + mov sp, x2 =20 /* And turn the MMU back on! */ dsb nsh isb set_sctlr_el2 x3 - ret x2 + ret x6 SYM_FUNC_END(__pkvm_init_switch_pgd) =20 .popsection diff --git a/arch/arm64/kvm/hyp/nvhe/mm.c b/arch/arm64/kvm/hyp/nvhe/mm.c index 3233a7c70f7c..c856b2399960 100644 --- a/arch/arm64/kvm/hyp/nvhe/mm.c +++ b/arch/arm64/kvm/hyp/nvhe/mm.c @@ -19,6 +19,7 @@ #include =20 struct kvm_pgtable pkvm_pgtable; +struct kvm_pgtable pkvm_idmap_pgtable; hyp_spinlock_t pkvm_pgd_lock; =20 struct memblock_region hyp_memory[HYP_MEMBLOCK_REGIONS]; @@ -431,6 +432,22 @@ int hyp_create_fixmap(void) return create_fixblock(); } =20 +static int pkvm_map_idmap_text(unsigned long start, unsigned long end) +{ + unsigned long idmap_start, idmap_end; + + idmap_start =3D __hyp_symbol_pa(__hyp_idmap_text_start); + idmap_start =3D ALIGN_DOWN(idmap_start, PAGE_SIZE); + idmap_end =3D __hyp_symbol_pa(__hyp_idmap_text_end); + idmap_end =3D ALIGN(idmap_end, PAGE_SIZE); + + if (WARN_ON(start !=3D idmap_start || end !=3D idmap_end)) + return -EINVAL; + + return kvm_pgtable_hyp_map(&pkvm_idmap_pgtable, start, + end - start, start, PAGE_HYP_EXEC); +} + int hyp_create_idmap(u32 hyp_va_bits) { unsigned long start, end; @@ -441,6 +458,13 @@ int hyp_create_idmap(u32 hyp_va_bits) end =3D __hyp_symbol_pa(__hyp_idmap_text_end); end =3D ALIGN(end, PAGE_SIZE); =20 + if (kvm_hyp_uses_ttbr1()) { + __hyp_private_va_base =3D kvm_hyp_ttbr1_private_start(); + __hyp_vmemmap =3D kvm_hyp_ttbr1_vmemmap_base(); + + return pkvm_map_idmap_text(start, end); + } + /* * One half of the VA space is reserved to linearly map portions of * memory -- see va_layout.c for more details. The other half of the VA diff --git a/arch/arm64/kvm/hyp/nvhe/setup.c b/arch/arm64/kvm/hyp/nvhe/setu= p.c index 2ef1972cc3dd..45b8f7213e39 100644 --- a/arch/arm64/kvm/hyp/nvhe/setup.c +++ b/arch/arm64/kvm/hyp/nvhe/setup.c @@ -109,6 +109,15 @@ static int recreate_hyp_mappings(phys_addr_t phys, uns= igned long size, if (ret) return ret; =20 + if (kvm_hyp_uses_ttbr1()) { + kvm_pgtable_hyp_enable_ttbr1(&pkvm_pgtable); + + ret =3D kvm_pgtable_hyp_init(&pkvm_idmap_pgtable, hyp_va_bits, + &hyp_early_alloc_mm_ops); + if (ret) + return ret; + } + ret =3D hyp_create_idmap(hyp_va_bits); if (ret) return ret; @@ -169,12 +178,27 @@ static void update_nvhe_init_params(void) =20 for (i =3D 0; i < hyp_nr_cpus; i++) { params =3D per_cpu_ptr(&kvm_init_params, i); - params->pgd_pa =3D __hyp_linear_pa(pkvm_pgtable.pgd); + params->ttbr0_pgd_pa =3D kvm_hyp_uses_ttbr1() ? + __hyp_linear_pa(pkvm_idmap_pgtable.pgd) : + __hyp_linear_pa(pkvm_pgtable.pgd); + params->ttbr1_pgd_pa =3D kvm_hyp_uses_ttbr1() ? + __hyp_linear_pa(pkvm_pgtable.pgd) : 0; dcache_clean_inval_poc((unsigned long)params, (unsigned long)params + sizeof(*params)); } } =20 +static bool pkvm_init_ttbrs_valid(struct kvm_nvhe_init_params *params) +{ + if (!params->ttbr0_pgd_pa) + return false; + + if (kvm_hyp_uses_ttbr1()) + return params->ttbr1_pgd_pa; + + return !params->ttbr1_pgd_pa; +} + static void *hyp_zalloc_hyp_page(void *arg) { return hyp_alloc_pages(&hpool, 0); @@ -284,8 +308,7 @@ static int fix_hyp_pgtable_refcnt(void) .arg =3D pkvm_pgtable.mm_ops, }; =20 - return kvm_pgtable_walk(&pkvm_pgtable, 0, BIT(pkvm_pgtable.ia_bits), - &walker); + return kvm_pgtable_hyp_walk(&pkvm_pgtable, &walker); } =20 void __noreturn __pkvm_init_finalise(void) @@ -371,8 +394,12 @@ int __pkvm_init(phys_addr_t phys, unsigned long size, = unsigned long *per_cpu_bas =20 /* Jump in the idmap page to switch to the new page-tables */ params =3D this_cpu_ptr(&kvm_init_params); + if (WARN_ON(!pkvm_init_ttbrs_valid(params))) + return -EINVAL; + fn =3D (typeof(fn))__hyp_symbol_pa(__pkvm_init_switch_pgd); - fn(params->pgd_pa, params->stack_hyp_va, __pkvm_init_finalise); + fn(params->ttbr0_pgd_pa, params->ttbr1_pgd_pa, + params->stack_hyp_va, __pkvm_init_finalise); =20 unreachable(); } diff --git a/arch/arm64/kvm/hyp/pgtable.c b/arch/arm64/kvm/hyp/pgtable.c index 91a7dfad6686..ca07fc2a55ec 100644 --- a/arch/arm64/kvm/hyp/pgtable.c +++ b/arch/arm64/kvm/hyp/pgtable.c @@ -10,6 +10,7 @@ #include #include #include +#include =20 struct kvm_pgtable_walk_data { struct kvm_pgtable_walker *walker; @@ -71,6 +72,31 @@ static u32 kvm_pgd_pages(u32 ia_bits, s8 start_level) return kvm_pgd_page_idx(&pgt, -1ULL) + 1; } =20 +static bool kvm_pgtable_walk_range_valid(struct kvm_pgtable *pgt, + struct kvm_pgtable_walk_data *data) +{ + u64 limit =3D BIT(pgt->ia_bits); + u64 mask =3D limit - 1; + u64 last =3D data->end - 1; + + if (data->addr > data->end) + return false; + + if (!(pgt->hyp_flags & KVM_PGTABLE_HYP_TTBR1)) + return data->addr <=3D limit && data->end <=3D limit; + + /* + * TTBR1 hyp walks use canonical virtual addresses supplied by the + * caller. The index helpers extract the relevant bits for each level; + * the masked comparison here only rejects ranges that cross out of the + * single TTBR1 address window described by ia_bits. + */ + if ((data->addr | mask) !=3D ~0ULL || (last | mask) !=3D ~0ULL) + return false; + + return (data->addr & mask) <=3D (last & mask); +} + static bool kvm_pte_table(kvm_pte_t pte, s8 level) { if (level =3D=3D KVM_PGTABLE_LAST_LEVEL) @@ -193,8 +219,13 @@ static inline int __kvm_pgtable_visit(struct kvm_pgtab= le_walk_data *data, goto out; =20 if (!table) { - data->addr =3D ALIGN_DOWN(data->addr, kvm_granule_size(level)); - data->addr +=3D kvm_granule_size(level); + u64 granule =3D kvm_granule_size(level); + u64 addr =3D ALIGN_DOWN(data->addr, granule); + + if (granule > data->end - addr) + data->addr =3D data->end; + else + data->addr =3D addr + granule; goto out; } =20 @@ -241,9 +272,8 @@ static int _kvm_pgtable_walk(struct kvm_pgtable *pgt, s= truct kvm_pgtable_walk_da { u32 idx; int ret =3D 0; - u64 limit =3D BIT(pgt->ia_bits); =20 - if (data->addr > limit || data->end > limit) + if (!kvm_pgtable_walk_range_valid(pgt, data)) return -ERANGE; =20 if (!pgt->pgd) @@ -281,6 +311,26 @@ int kvm_pgtable_walk(struct kvm_pgtable *pgt, u64 addr= , u64 size, return r; } =20 +int kvm_pgtable_hyp_walk(struct kvm_pgtable *pgt, + struct kvm_pgtable_walker *walker) +{ + u64 size =3D BIT(pgt->ia_bits); + u64 addr =3D 0; + + if (pgt->hyp_flags & KVM_PGTABLE_HYP_TTBR1) { + addr =3D -size; + + /* + * The exclusive end of the TTBR1 range cannot be represented in + * a u64. Exclude the last page, which cannot be mapped through + * kvm_pgtable_walk() for the same reason. + */ + size -=3D PAGE_SIZE; + } + + return kvm_pgtable_walk(pgt, addr, size, walker); +} + struct leaf_walk_data { kvm_pte_t pte; s8 level; @@ -539,6 +589,7 @@ int kvm_pgtable_hyp_init(struct kvm_pgtable *pgt, u32 v= a_bits, pgt->ia_bits =3D va_bits; pgt->start_level =3D start_level; pgt->mm_ops =3D mm_ops; + pgt->hyp_flags =3D 0; pgt->mmu =3D NULL; pgt->force_pte_cb =3D NULL; =20 @@ -568,7 +619,7 @@ void kvm_pgtable_hyp_destroy(struct kvm_pgtable *pgt) .flags =3D KVM_PGTABLE_WALK_LEAF | KVM_PGTABLE_WALK_TABLE_POST, }; =20 - WARN_ON(kvm_pgtable_walk(pgt, 0, BIT(pgt->ia_bits), &walker)); + WARN_ON(kvm_pgtable_hyp_walk(pgt, &walker)); pgt->mm_ops->put_page(kvm_dereference_pteref(&walker, pgt->pgd)); pgt->pgd =3D NULL; } @@ -1589,6 +1640,7 @@ int __kvm_pgtable_stage2_init(struct kvm_pgtable *pgt= , struct kvm_s2_mmu *mmu, pgt->start_level =3D start_level; pgt->mm_ops =3D mm_ops; pgt->mmu =3D mmu; + pgt->hyp_flags =3D 0; pgt->flags =3D flags; pgt->force_pte_cb =3D force_pte_cb; =20 diff --git a/arch/arm64/kvm/mmu.c b/arch/arm64/kvm/mmu.c index 154d53dca619..46fd7040bca7 100644 --- a/arch/arm64/kvm/mmu.c +++ b/arch/arm64/kvm/mmu.c @@ -20,11 +20,13 @@ #include #include #include +#include #include =20 #include "trace.h" =20 static struct kvm_pgtable *hyp_pgtable; +static struct kvm_pgtable *hyp_idmap_pgtable; static DEFINE_MUTEX(kvm_hyp_pgd_mutex); =20 static unsigned long __ro_after_init hyp_idmap_start; @@ -392,6 +394,11 @@ static void stage2_flush_vm(struct kvm *kvm) void __init free_hyp_pgds(void) { mutex_lock(&kvm_hyp_pgd_mutex); + if (hyp_idmap_pgtable) { + kvm_pgtable_hyp_destroy(hyp_idmap_pgtable); + kfree(hyp_idmap_pgtable); + hyp_idmap_pgtable =3D NULL; + } if (hyp_pgtable) { kvm_pgtable_hyp_destroy(hyp_pgtable); kfree(hyp_pgtable); @@ -2481,6 +2488,17 @@ bool kvm_test_age_gfn(struct kvm *kvm, struct kvm_gf= n_range *range) =20 phys_addr_t kvm_mmu_get_httbr(void) { + if (hyp_idmap_pgtable) + return __pa(hyp_idmap_pgtable->pgd); + + return __pa(hyp_pgtable->pgd); +} + +phys_addr_t kvm_mmu_get_hyp_ttbr1(void) +{ + if (!kvm_hyp_uses_ttbr1()) + return 0; + return __pa(hyp_pgtable->pgd); } =20 @@ -2491,9 +2509,15 @@ phys_addr_t kvm_get_idmap_vector(void) =20 static int kvm_map_idmap_text(void) { + struct kvm_pgtable *pgt =3D hyp_idmap_pgtable ?: hyp_pgtable; unsigned long size =3D hyp_idmap_end - hyp_idmap_start; - int err =3D __create_hyp_mappings(hyp_idmap_start, size, hyp_idmap_start, - PAGE_HYP_EXEC); + int err; + + if (WARN_ON(kvm_hyp_uses_ttbr1() && !hyp_idmap_pgtable)) + return -EINVAL; + + err =3D kvm_pgtable_hyp_map(pgt, hyp_idmap_start, size, hyp_idmap_start, + PAGE_HYP_EXEC); if (err) kvm_err("Failed to idmap %lx-%lx\n", hyp_idmap_start, hyp_idmap_end); @@ -2559,14 +2583,39 @@ int __init kvm_mmu_init(u32 hyp_va_bits) if (err) goto out_free_pgtable; =20 + if (kvm_hyp_uses_ttbr1()) { + kvm_pgtable_hyp_enable_ttbr1(hyp_pgtable); + + hyp_idmap_pgtable =3D kzalloc_obj(*hyp_idmap_pgtable); + if (!hyp_idmap_pgtable) { + kvm_err("Hyp mode idmap page-table not allocated\n"); + err =3D -ENOMEM; + goto out_destroy_pgtable; + } + + err =3D kvm_pgtable_hyp_init(hyp_idmap_pgtable, hyp_va_bits, + &kvm_hyp_mm_ops); + if (err) + goto out_free_idmap_pgtable; + } + err =3D kvm_map_idmap_text(); if (err) - goto out_destroy_pgtable; + goto out_destroy_idmap_pgtable; =20 - io_map_base =3D hyp_idmap_start; + if (kvm_hyp_uses_ttbr1()) + io_map_base =3D kvm_hyp_ttbr1_private_end(); + else + io_map_base =3D hyp_idmap_start; __hyp_va_bits =3D hyp_va_bits; return 0; =20 +out_destroy_idmap_pgtable: + if (hyp_idmap_pgtable) + kvm_pgtable_hyp_destroy(hyp_idmap_pgtable); +out_free_idmap_pgtable: + kfree(hyp_idmap_pgtable); + hyp_idmap_pgtable =3D NULL; out_destroy_pgtable: kvm_pgtable_hyp_destroy(hyp_pgtable); out_free_pgtable: diff --git a/arch/arm64/kvm/pkvm.c b/arch/arm64/kvm/pkvm.c index 053e4f733e4b..1352c89d2bd8 100644 --- a/arch/arm64/kvm/pkvm.c +++ b/arch/arm64/kvm/pkvm.c @@ -292,6 +292,9 @@ static int __init finalize_pkvm(void) ret =3D pkvm_drop_host_privileges(); if (ret) pr_err("Failed to finalize Hyp protection: %d\n", ret); + else + kvm_info("pKVM finalized in %s mode\n", + kvm_hyp_uses_ttbr1() ? "hVHE" : "nVHE"); =20 return ret; } diff --git a/arch/arm64/kvm/va_layout.c b/arch/arm64/kvm/va_layout.c index 75a89ad8fecf..1e1816995fe4 100644 --- a/arch/arm64/kvm/va_layout.c +++ b/arch/arm64/kvm/va_layout.c @@ -33,19 +33,29 @@ static u64 __early_kern_hyp_va(u64 addr) return addr; } =20 +static u64 __early_kern_sym_hyp_va(void *ptr) +{ + if (is_kernel_in_hyp_mode() || kvm_hyp_init_uses_ttbr1()) + return (u64)ptr; + + return __early_kern_hyp_va((u64)lm_alias(ptr)); +} + /* * Store a hyp VA <-> PA offset into a EL2-owned variable. */ static void init_hyp_physvirt_offset(void) { - u64 kern_va, hyp_va; + u64 kern_va, hyp_va, sym_va; =20 /* Compute the offset from the hyp VA and PA of a random symbol. */ kern_va =3D (u64)lm_alias(__hyp_text_start); hyp_va =3D __early_kern_hyp_va(kern_va); hyp_physvirt_offset =3D (s64)__pa(kern_va) - (s64)hyp_va; =20 - hyp_symbol_physvirt_offset =3D hyp_physvirt_offset; + sym_va =3D __early_kern_sym_hyp_va(__hyp_text_start); + hyp_symbol_physvirt_offset =3D (s64)__pa_symbol(__hyp_text_start) - + (s64)sym_va; } =20 /* @@ -87,6 +97,14 @@ __init void kvm_compute_layout(void) u64 hyp_va_msb; u32 hyp_va_bits =3D kvm_hyp_va_bits(); =20 + if (kvm_hyp_init_uses_ttbr1()) { + tag_lsb =3D 0; + tag_val =3D 0; + va_mask =3D ~0ULL; + init_hyp_physvirt_offset(); + return; + } + /* Where is my RAM region? */ hyp_va_msb =3D idmap_addr & BIT(hyp_va_bits - 1); hyp_va_msb ^=3D BIT(hyp_va_bits - 1); @@ -117,6 +135,9 @@ __init void kvm_apply_hyp_relocations(void) int32_t *begin =3D (int32_t *)__hyp_reloc_begin; int32_t *end =3D (int32_t *)__hyp_reloc_end; =20 + if (kvm_hyp_init_uses_ttbr1()) + return; + for (rel =3D begin; rel < end; ++rel) { uintptr_t *ptr, kimg_va; =20 @@ -187,14 +208,15 @@ void __init kvm_update_va_mask(struct alt_instr *alt, u32 rd, rn, insn, oinsn; =20 /* - * VHE doesn't need any address translation, let's NOP + * VHE/hVHE doesn't need any address translation, let's NOP * everything. * * Alternatively, if the tag is zero (because the layout * dictates it and we don't have any spare bits in the * address), NOP everything after masking the kernel VA. */ - if (cpus_have_cap(ARM64_HAS_VIRT_HOST_EXTN) || (!tag_val && i > 0)) { + if (cpus_have_cap(ARM64_HAS_VIRT_HOST_EXTN) || + kvm_hyp_init_uses_ttbr1() || (!tag_val && i > 0)) { updptr[i] =3D cpu_to_le32(aarch64_insn_gen_nop()); continue; } @@ -225,7 +247,7 @@ void kvm_patch_vector_branch(struct alt_instr *alt, /* * Compute HYP VA by using the same computation as kern_hyp_va() */ - addr =3D __early_kern_hyp_va((u64)kvm_hyp_kimg_kaddr(__kvm_hyp_vector)); + addr =3D __early_kern_sym_hyp_va(__kvm_hyp_vector); =20 /* Use PC[10:7] to branch to the same vector in KVM */ addr |=3D ((u64)origptr & GENMASK_ULL(10, 7)); --=20 2.43.0