AMD SME/SEV uses a bit in the page table entries to indicate that the
page is encrypted and not accessible to the VMM.

TDX uses a similar approach, but the polarity of the mask is opposite to
AMD: if the bit is set the page is accessible to VMM.

Provide vendor-neutral API to deal with the mask:

  - cc_mkenc() and cc_mkdec() modify given address to make it
    encrypted/decrypted. It can be applied to phys_addr_t, pgprotval_t
    or page table entry value.

  - cc_get_mask() returns encryption or decrypthion mask. It is useful
    for set_memory_encrypted() and set_memory_decrypted()
    implementation.

The implementation will be extended to cover TDX.

pgprot_decrypted() is used by drivers (i915, virtio_gpu, vfio).
cc_mkdec() called by pgprot_decrypted(). Export cc_mkdec().

HyperV doesn't use bits in page table entries, so the mask is 0 for both
encrypthion and decrypthion.

Signed-off-by: Kirill A. Shutemov <kirill.shutemov@linux.intel.com>
---
 arch/x86/include/asm/coco.h        | 12 +++++++++-
 arch/x86/include/asm/pgtable.h     | 13 ++++++-----
 arch/x86/kernel/cc_platform.c      | 35 +++++++++++++++++++++++++++++-
 arch/x86/kernel/cpu/mshyperv.c     |  2 +-
 arch/x86/mm/mem_encrypt_identity.c |  2 +-
 arch/x86/mm/pat/set_memory.c       |  4 ++--
 6 files changed, 56 insertions(+), 12 deletions(-)

diff --git a/arch/x86/include/asm/coco.h b/arch/x86/include/asm/coco.h
index 6e770e0dd683..802d87d08e31 100644
--- a/arch/x86/include/asm/coco.h
+++ b/arch/x86/include/asm/coco.h
@@ -11,6 +11,16 @@ enum cc_vendor {
 	CC_VENDOR_INTEL,
 };
 
-void cc_init(enum cc_vendor);
+void cc_init(enum cc_vendor, u64 mask);
+
+#ifdef CONFIG_ARCH_HAS_CC_PLATFORM
+u64 cc_get_mask(bool enc);
+u64 cc_mkenc(u64 val);
+u64 cc_mkdec(u64 val);
+#else
+#define cc_get_mask(enc)	0
+#define cc_mkenc(val)		(val)
+#define cc_mkdec(val)		(val)
+#endif
 
 #endif /* _ASM_X86_COCO_H */
diff --git a/arch/x86/include/asm/pgtable.h b/arch/x86/include/asm/pgtable.h
index 8a9432fb3802..62ab07e24aef 100644
--- a/arch/x86/include/asm/pgtable.h
+++ b/arch/x86/include/asm/pgtable.h
@@ -15,17 +15,12 @@
 		     cachemode2protval(_PAGE_CACHE_MODE_UC_MINUS)))	\
 	 : (prot))
 
-/*
- * Macros to add or remove encryption attribute
- */
-#define pgprot_encrypted(prot)	__pgprot(__sme_set(pgprot_val(prot)))
-#define pgprot_decrypted(prot)	__pgprot(__sme_clr(pgprot_val(prot)))
-
 #ifndef __ASSEMBLY__
 #include <linux/spinlock.h>
 #include <asm/x86_init.h>
 #include <asm/pkru.h>
 #include <asm/fpu/api.h>
+#include <asm/coco.h>
 #include <asm-generic/pgtable_uffd.h>
 #include <linux/page_table_check.h>
 
@@ -38,6 +33,12 @@ void ptdump_walk_pgd_level_debugfs(struct seq_file *m, struct mm_struct *mm,
 void ptdump_walk_pgd_level_checkwx(void);
 void ptdump_walk_user_pgd_level_checkwx(void);
 
+/*
+ * Macros to add or remove encryption attribute
+ */
+#define pgprot_encrypted(prot)	__pgprot(cc_mkenc(pgprot_val(prot)))
+#define pgprot_decrypted(prot)	__pgprot(cc_mkdec(pgprot_val(prot)))
+
 #ifdef CONFIG_DEBUG_WX
 #define debug_checkwx()		ptdump_walk_pgd_level_checkwx()
 #define debug_checkwx_user()	ptdump_walk_user_pgd_level_checkwx()
diff --git a/arch/x86/kernel/cc_platform.c b/arch/x86/kernel/cc_platform.c
index 891d3074a16e..93e6be7b7eca 100644
--- a/arch/x86/kernel/cc_platform.c
+++ b/arch/x86/kernel/cc_platform.c
@@ -13,6 +13,7 @@
 #include <asm/coco.h>
 #include <asm/processor.h>
 
+static u64 cc_mask;
 static enum cc_vendor cc_vendor;
 
 static bool intel_cc_platform_has(enum cc_attr attr)
@@ -84,7 +85,39 @@ bool cc_platform_has(enum cc_attr attr)
 }
 EXPORT_SYMBOL_GPL(cc_platform_has);
 
-__init void cc_init(enum cc_vendor vendor)
+u64 cc_get_mask(bool enc)
+{
+	switch (cc_vendor) {
+	case CC_VENDOR_AMD:
+		return enc ? cc_mask : 0;
+	default:
+		return 0;
+	}
+}
+
+u64 cc_mkenc(u64 val)
+{
+	switch (cc_vendor) {
+	case CC_VENDOR_AMD:
+		return val | cc_mask;
+	default:
+		return val;
+	}
+}
+
+u64 cc_mkdec(u64 val)
+{
+	switch (cc_vendor) {
+	case CC_VENDOR_AMD:
+		return val & ~cc_mask;
+	default:
+		return val;
+	}
+}
+EXPORT_SYMBOL_GPL(cc_mkdec);
+
+__init void cc_init(enum cc_vendor vendor, u64 mask)
 {
 	cc_vendor = vendor;
+	cc_mask = mask;
 }
diff --git a/arch/x86/kernel/cpu/mshyperv.c b/arch/x86/kernel/cpu/mshyperv.c
index d77cf3a31f07..9af6be143998 100644
--- a/arch/x86/kernel/cpu/mshyperv.c
+++ b/arch/x86/kernel/cpu/mshyperv.c
@@ -346,7 +346,7 @@ static void __init ms_hyperv_init_platform(void)
 		swiotlb_force = SWIOTLB_FORCE;
 #endif
 		if (hv_get_isolation_type() != HV_ISOLATION_TYPE_NONE)
-			cc_init(CC_VENDOR_HYPERV);
+			cc_init(CC_VENDOR_HYPERV, 0);
 	}
 
 	if (hv_max_functions_eax >= HYPERV_CPUID_NESTED_FEATURES) {
diff --git a/arch/x86/mm/mem_encrypt_identity.c b/arch/x86/mm/mem_encrypt_identity.c
index eb7fbd85b77e..fa758247ab57 100644
--- a/arch/x86/mm/mem_encrypt_identity.c
+++ b/arch/x86/mm/mem_encrypt_identity.c
@@ -603,5 +603,5 @@ void __init sme_enable(struct boot_params *bp)
 out:
 	physical_mask &= ~sme_me_mask;
 	if (sme_me_mask)
-		cc_init(CC_VENDOR_AMD);
+		cc_init(CC_VENDOR_AMD, sme_me_mask);
 }
diff --git a/arch/x86/mm/pat/set_memory.c b/arch/x86/mm/pat/set_memory.c
index b4072115c8ef..e79366b8a9da 100644
--- a/arch/x86/mm/pat/set_memory.c
+++ b/arch/x86/mm/pat/set_memory.c
@@ -1999,8 +1999,8 @@ static int __set_memory_enc_pgtable(unsigned long addr, int numpages, bool enc)
 	memset(&cpa, 0, sizeof(cpa));
 	cpa.vaddr = &addr;
 	cpa.numpages = numpages;
-	cpa.mask_set = enc ? __pgprot(_PAGE_ENC) : __pgprot(0);
-	cpa.mask_clr = enc ? __pgprot(0) : __pgprot(_PAGE_ENC);
+	cpa.mask_set = __pgprot(cc_get_mask(enc));
+	cpa.mask_clr = __pgprot(cc_get_mask(!enc));
 	cpa.pgd = init_mm.pgd;
 
 	/* Must avoid aliasing mappings in the highmem code */
-- 
2.34.1

On 2/18/22 18:13, Kirill A. Shutemov wrote:
> AMD SME/SEV uses a bit in the page table entries to indicate that the
> page is encrypted and not accessible to the VMM.
> 
> TDX uses a similar approach, but the polarity of the mask is opposite to
> AMD: if the bit is set the page is accessible to VMM.
> 
> Provide vendor-neutral API to deal with the mask:
> 
>    - cc_mkenc() and cc_mkdec() modify given address to make it
>      encrypted/decrypted. It can be applied to phys_addr_t, pgprotval_t
>      or page table entry value.
> 
>    - cc_get_mask() returns encryption or decrypthion mask. It is useful
>      for set_memory_encrypted() and set_memory_decrypted()
>      implementation.
> 
> The implementation will be extended to cover TDX.
> 
> pgprot_decrypted() is used by drivers (i915, virtio_gpu, vfio).
> cc_mkdec() called by pgprot_decrypted(). Export cc_mkdec().
> 
> HyperV doesn't use bits in page table entries, so the mask is 0 for both
> encrypthion and decrypthion.
> 
> Signed-off-by: Kirill A. Shutemov <kirill.shutemov@linux.intel.com>

> @@ -84,7 +85,39 @@ bool cc_platform_has(enum cc_attr attr)
>   }
>   EXPORT_SYMBOL_GPL(cc_platform_has);
>   
> -__init void cc_init(enum cc_vendor vendor)
> +u64 cc_get_mask(bool enc)

I was a bit confused by this name, expecting to always get the mask, but 
it is dependent on the input parameter. How about cc_get_operation_mask() 
or ... ? I'm struggling to come up with a good name, but plain 
cc_get_mask() is very confusing.

> +{
> +	switch (cc_vendor) {
> +	case CC_VENDOR_AMD:
> +		return enc ? cc_mask : 0;
> +	default:
> +		return 0;
> +	}
> +}
> +

> diff --git a/arch/x86/mm/mem_encrypt_identity.c b/arch/x86/mm/mem_encrypt_identity.c
> index eb7fbd85b77e..fa758247ab57 100644
> --- a/arch/x86/mm/mem_encrypt_identity.c
> +++ b/arch/x86/mm/mem_encrypt_identity.c
> @@ -603,5 +603,5 @@ void __init sme_enable(struct boot_params *bp)
>   out:
>   	physical_mask &= ~sme_me_mask;
>   	if (sme_me_mask)
> -		cc_init(CC_VENDOR_AMD);
> +		cc_init(CC_VENDOR_AMD, sme_me_mask);

Since you're adding the if statement, you can probably wrap the adjustment 
to physical_mask within the if, also (I guess in the previous patch). Not 
required, though.

Thanks,
Tom

On Mon, Feb 21, 2022 at 12:31:40PM -0600, Tom Lendacky wrote:
> On 2/18/22 18:13, Kirill A. Shutemov wrote:
> > AMD SME/SEV uses a bit in the page table entries to indicate that the
> > page is encrypted and not accessible to the VMM.
> > 
> > TDX uses a similar approach, but the polarity of the mask is opposite to
> > AMD: if the bit is set the page is accessible to VMM.
> > 
> > Provide vendor-neutral API to deal with the mask:
> > 
> >    - cc_mkenc() and cc_mkdec() modify given address to make it
> >      encrypted/decrypted. It can be applied to phys_addr_t, pgprotval_t
> >      or page table entry value.
> > 
> >    - cc_get_mask() returns encryption or decrypthion mask. It is useful
> >      for set_memory_encrypted() and set_memory_decrypted()
> >      implementation.
> > 
> > The implementation will be extended to cover TDX.
> > 
> > pgprot_decrypted() is used by drivers (i915, virtio_gpu, vfio).
> > cc_mkdec() called by pgprot_decrypted(). Export cc_mkdec().
> > 
> > HyperV doesn't use bits in page table entries, so the mask is 0 for both
> > encrypthion and decrypthion.
> > 
> > Signed-off-by: Kirill A. Shutemov <kirill.shutemov@linux.intel.com>
> 
> > @@ -84,7 +85,39 @@ bool cc_platform_has(enum cc_attr attr)
> >   }
> >   EXPORT_SYMBOL_GPL(cc_platform_has);
> > -__init void cc_init(enum cc_vendor vendor)
> > +u64 cc_get_mask(bool enc)
> 
> I was a bit confused by this name, expecting to always get the mask, but it
> is dependent on the input parameter.

Well, it always get you a mask. But it can be empty :P

> How about cc_get_operation_mask() or ... ? I'm struggling to come up
> with a good name, but plain cc_get_mask() is very confusing.

Yeah. Naming is hard. cc_get_operation_mask() doesn't clarify anything
comparing to cc_get_mask(). To me it is as confusing.
> 
> > +{
> > +	switch (cc_vendor) {
> > +	case CC_VENDOR_AMD:
> > +		return enc ? cc_mask : 0;
> > +	default:
> > +		return 0;
> > +	}
> > +}
> > +
> 
> > diff --git a/arch/x86/mm/mem_encrypt_identity.c b/arch/x86/mm/mem_encrypt_identity.c
> > index eb7fbd85b77e..fa758247ab57 100644
> > --- a/arch/x86/mm/mem_encrypt_identity.c
> > +++ b/arch/x86/mm/mem_encrypt_identity.c
> > @@ -603,5 +603,5 @@ void __init sme_enable(struct boot_params *bp)
> >   out:
> >   	physical_mask &= ~sme_me_mask;
> >   	if (sme_me_mask)
> > -		cc_init(CC_VENDOR_AMD);
> > +		cc_init(CC_VENDOR_AMD, sme_me_mask);
> 
> Since you're adding the if statement, you can probably wrap the adjustment
> to physical_mask within the if, also (I guess in the previous patch). Not
> required, though.

Can do.

-- 
 Kirill A. Shutemov