Although THP-granularity exec COW can reduce the number of faults
and improve iTLB hit rates, but after enabling it, the THP folio
allocating and copying operations may introduce higher latency,
and it consumes more memory compared to page COW handling. These
side effects may be unacceptable in certain scenarios.
Therefore, we add use_exec_cow sysfs knob for THP COW of executable
private file mmap. It's enabled by default, kernel will try to
allocate PMD page and map it. If it's disabled, it will fallback to
split PMD mapping and do pte fault handle.
Signed-off-by: Zhang Qilong <zhangqilong3@huawei.com>
---
Documentation/admin-guide/mm/transhuge.rst | 8 ++++++++
include/linux/huge_mm.h | 4 ++++
mm/huge_memory.c | 18 +++++++++++++++++-
mm/memory.c | 3 ++-
4 files changed, 31 insertions(+), 2 deletions(-)
diff --git a/Documentation/admin-guide/mm/transhuge.rst b/Documentation/admin-guide/mm/transhuge.rst
index 5fbc3d89bb07..c6d7ca045c03 100644
--- a/Documentation/admin-guide/mm/transhuge.rst
+++ b/Documentation/admin-guide/mm/transhuge.rst
@@ -201,10 +201,18 @@ page fault to anonymous mapping. It's possible to disable huge zero
page by writing 0 or enable it back by writing 1::
echo 0 >/sys/kernel/mm/transparent_hugepage/use_zero_page
echo 1 >/sys/kernel/mm/transparent_hugepage/use_zero_page
+By default kernel tries to use huge, PMD-mappable page on private
+executable file THP mmap fault handle. It's possible to disable
+THP COW of private executable mmap by writing 0 or enable it back
+by writing 1::
+
+ echo 0 >/sys/kernel/mm/transparent_hugepage/use_exec_cow
+ echo 1 >/sys/kernel/mm/transparent_hugepage/use_exec_cow
+
Some userspace (such as a test program, or an optimized memory
allocation library) may want to know the size (in bytes) of a
PMD-mappable transparent hugepage::
cat /sys/kernel/mm/transparent_hugepage/hpage_pmd_size
diff --git a/include/linux/huge_mm.h b/include/linux/huge_mm.h
index bae856a53e1f..d86215f06ac9 100644
--- a/include/linux/huge_mm.h
+++ b/include/linux/huge_mm.h
@@ -55,10 +55,11 @@ enum transparent_hugepage_flag {
TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG,
TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG,
TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG,
TRANSPARENT_HUGEPAGE_DEFRAG_KHUGEPAGED_FLAG,
TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG,
+ TRANSPARENT_HUGEPAGE_USE_EXEC_COW_FLAG,
};
struct kobject;
struct kobj_attribute;
@@ -323,10 +324,13 @@ struct thpsize {
#define transparent_hugepage_use_zero_page() \
(transparent_hugepage_flags & \
(1<<TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG))
+#define transparent_hugepage_use_exec_cow() \
+ (transparent_hugepage_flags & \
+ (1<<TRANSPARENT_HUGEPAGE_USE_EXEC_COW_FLAG))
/*
* Check whether THPs are explicitly disabled for this VMA, for example,
* through madvise or prctl.
*/
static inline bool vma_thp_disabled(struct vm_area_struct *vma,
diff --git a/mm/huge_memory.c b/mm/huge_memory.c
index 35ecd62f64c4..430b80318aae 100644
--- a/mm/huge_memory.c
+++ b/mm/huge_memory.c
@@ -63,11 +63,12 @@ unsigned long transparent_hugepage_flags __read_mostly =
#ifdef CONFIG_TRANSPARENT_HUGEPAGE_MADVISE
(1<<TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG)|
#endif
(1<<TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG)|
(1<<TRANSPARENT_HUGEPAGE_DEFRAG_KHUGEPAGED_FLAG)|
- (1<<TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG);
+ (1<<TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG)|
+ (1<<TRANSPARENT_HUGEPAGE_USE_EXEC_COW_FLAG);
static struct shrinker *deferred_split_shrinker;
static unsigned long deferred_split_count(struct shrinker *shrink,
struct shrink_control *sc);
static unsigned long deferred_split_scan(struct shrinker *shrink,
@@ -442,10 +443,24 @@ static ssize_t use_zero_page_store(struct kobject *kobj,
return single_hugepage_flag_store(kobj, attr, buf, count,
TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG);
}
static struct kobj_attribute use_zero_page_attr = __ATTR_RW(use_zero_page);
+static ssize_t use_exec_cow_show(struct kobject *kobj,
+ struct kobj_attribute *attr, char *buf)
+{
+ return single_hugepage_flag_show(kobj, attr, buf,
+ TRANSPARENT_HUGEPAGE_USE_EXEC_COW_FLAG);
+}
+static ssize_t use_exec_cow_store(struct kobject *kobj,
+ struct kobj_attribute *attr, const char *buf, size_t count)
+{
+ return single_hugepage_flag_store(kobj, attr, buf, count,
+ TRANSPARENT_HUGEPAGE_USE_EXEC_COW_FLAG);
+}
+static struct kobj_attribute use_exec_cow_attr = __ATTR_RW(use_exec_cow);
+
static ssize_t hpage_pmd_size_show(struct kobject *kobj,
struct kobj_attribute *attr, char *buf)
{
return sysfs_emit(buf, "%lu\n", HPAGE_PMD_SIZE);
}
@@ -475,10 +490,11 @@ static struct kobj_attribute split_underused_thp_attr = __ATTR(
static struct attribute *hugepage_attr[] = {
&enabled_attr.attr,
&defrag_attr.attr,
&use_zero_page_attr.attr,
+ &use_exec_cow_attr.attr,
&hpage_pmd_size_attr.attr,
#ifdef CONFIG_SHMEM
&shmem_enabled_attr.attr,
#endif
&split_underused_thp_attr.attr,
diff --git a/mm/memory.c b/mm/memory.c
index e282adec9165..5e3354e16b32 100644
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -6134,11 +6134,12 @@ static inline vm_fault_t wp_huge_pmd(struct vm_fault *vmf)
return ret;
}
}
- if (is_exec_mapping(vma->vm_flags) &&
+ if (transparent_hugepage_use_exec_cow() &&
+ is_exec_mapping(vma->vm_flags) &&
is_cow_mapping(vma->vm_flags)) {
/* Skip special and shmem */
if (vma_is_special_huge(vma) || vma_is_shmem(vma))
goto split;
--
2.43.0Hi Zhang,
kernel test robot noticed the following build errors:
[auto build test ERROR on next-20251215]
url: https://github.com/intel-lab-lkp/linux/commits/Zhang-Qilong/mm-huge_memory-Implementation-of-THP-COW-for-executable-file-mmap/20251215-204035
base: next-20251215
patch link: https://lore.kernel.org/r/20251215123407.380813-3-zhangqilong3%40huawei.com
patch subject: [PATCH next 2/2] mm/huge_memory: Add sysfs knob for executable THP COW
config: x86_64-rhel-9.4 (https://download.01.org/0day-ci/archive/20251216/202512161328.lAGmHVt0-lkp@intel.com/config)
compiler: gcc-14 (Debian 14.2.0-19) 14.2.0
reproduce (this is a W=1 build): (https://download.01.org/0day-ci/archive/20251216/202512161328.lAGmHVt0-lkp@intel.com/reproduce)
If you fix the issue in a separate patch/commit (i.e. not just a new version of
the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <lkp@intel.com>
| Closes: https://lore.kernel.org/oe-kbuild-all/202512161328.lAGmHVt0-lkp@intel.com/
All errors (new ones prefixed by >>):
mm/memory.c: In function 'wp_huge_pmd':
>> mm/memory.c:6131:13: error: implicit declaration of function 'transparent_hugepage_use_exec_cow' [-Wimplicit-function-declaration]
6131 | if (transparent_hugepage_use_exec_cow() &&
| ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
vim +/transparent_hugepage_use_exec_cow +6131 mm/memory.c
6104
6105 /* `inline' is required to avoid gcc 4.1.2 build error */
6106 static inline vm_fault_t wp_huge_pmd(struct vm_fault *vmf)
6107 {
6108 struct vm_area_struct *vma = vmf->vma;
6109 const bool unshare = vmf->flags & FAULT_FLAG_UNSHARE;
6110 vm_fault_t ret;
6111
6112 if (vma_is_anonymous(vma)) {
6113 if (likely(!unshare) &&
6114 userfaultfd_huge_pmd_wp(vma, vmf->orig_pmd)) {
6115 if (userfaultfd_wp_async(vmf->vma))
6116 goto split;
6117 return handle_userfault(vmf, VM_UFFD_WP);
6118 }
6119 return do_huge_pmd_wp_page(vmf);
6120 }
6121
6122 if (vma->vm_flags & (VM_SHARED | VM_MAYSHARE)) {
6123 if (vma->vm_ops->huge_fault) {
6124 ret = vma->vm_ops->huge_fault(vmf, PMD_ORDER);
6125 if (!(ret & VM_FAULT_FALLBACK))
6126 return ret;
6127 }
6128 }
6129
6130
> 6131 if (transparent_hugepage_use_exec_cow() &&
6132 is_exec_mapping(vma->vm_flags) &&
6133 is_cow_mapping(vma->vm_flags)) {
6134 /* Skip special and shmem */
6135 if (vma_is_special_huge(vma) || vma_is_shmem(vma))
6136 goto split;
6137
6138 ret = do_huge_pmd_exec_cow(vmf);
6139 if (!(ret & VM_FAULT_FALLBACK))
6140 return ret;
6141 }
6142
6143 split:
6144 /* COW or write-notify handled on pte level: split pmd. */
6145 __split_huge_pmd(vma, vmf->pmd, vmf->address, false);
6146
6147 return VM_FAULT_FALLBACK;
6148 }
6149
--
0-DAY CI Kernel Test Service
https://github.com/intel/lkp-tests/wiki
Hi Zhang,
kernel test robot noticed the following build errors:
[auto build test ERROR on next-20251215]
url: https://github.com/intel-lab-lkp/linux/commits/Zhang-Qilong/mm-huge_memory-Implementation-of-THP-COW-for-executable-file-mmap/20251215-204035
base: next-20251215
patch link: https://lore.kernel.org/r/20251215123407.380813-3-zhangqilong3%40huawei.com
patch subject: [PATCH next 2/2] mm/huge_memory: Add sysfs knob for executable THP COW
config: x86_64-kexec (https://download.01.org/0day-ci/archive/20251216/202512161231.CTAFQzip-lkp@intel.com/config)
compiler: clang version 20.1.8 (https://github.com/llvm/llvm-project 87f0227cb60147a26a1eeb4fb06e3b505e9c7261)
reproduce (this is a W=1 build): (https://download.01.org/0day-ci/archive/20251216/202512161231.CTAFQzip-lkp@intel.com/reproduce)
If you fix the issue in a separate patch/commit (i.e. not just a new version of
the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <lkp@intel.com>
| Closes: https://lore.kernel.org/oe-kbuild-all/202512161231.CTAFQzip-lkp@intel.com/
All errors (new ones prefixed by >>):
>> mm/memory.c:6131:6: error: call to undeclared function 'transparent_hugepage_use_exec_cow'; ISO C99 and later do not support implicit function declarations [-Wimplicit-function-declaration]
6131 | if (transparent_hugepage_use_exec_cow() &&
| ^
mm/memory.c:6135:7: error: call to undeclared function 'vma_is_special_huge'; ISO C99 and later do not support implicit function declarations [-Wimplicit-function-declaration]
6135 | if (vma_is_special_huge(vma) || vma_is_shmem(vma))
| ^
2 errors generated.
vim +/transparent_hugepage_use_exec_cow +6131 mm/memory.c
6104
6105 /* `inline' is required to avoid gcc 4.1.2 build error */
6106 static inline vm_fault_t wp_huge_pmd(struct vm_fault *vmf)
6107 {
6108 struct vm_area_struct *vma = vmf->vma;
6109 const bool unshare = vmf->flags & FAULT_FLAG_UNSHARE;
6110 vm_fault_t ret;
6111
6112 if (vma_is_anonymous(vma)) {
6113 if (likely(!unshare) &&
6114 userfaultfd_huge_pmd_wp(vma, vmf->orig_pmd)) {
6115 if (userfaultfd_wp_async(vmf->vma))
6116 goto split;
6117 return handle_userfault(vmf, VM_UFFD_WP);
6118 }
6119 return do_huge_pmd_wp_page(vmf);
6120 }
6121
6122 if (vma->vm_flags & (VM_SHARED | VM_MAYSHARE)) {
6123 if (vma->vm_ops->huge_fault) {
6124 ret = vma->vm_ops->huge_fault(vmf, PMD_ORDER);
6125 if (!(ret & VM_FAULT_FALLBACK))
6126 return ret;
6127 }
6128 }
6129
6130
> 6131 if (transparent_hugepage_use_exec_cow() &&
6132 is_exec_mapping(vma->vm_flags) &&
6133 is_cow_mapping(vma->vm_flags)) {
6134 /* Skip special and shmem */
6135 if (vma_is_special_huge(vma) || vma_is_shmem(vma))
6136 goto split;
6137
6138 ret = do_huge_pmd_exec_cow(vmf);
6139 if (!(ret & VM_FAULT_FALLBACK))
6140 return ret;
6141 }
6142
6143 split:
6144 /* COW or write-notify handled on pte level: split pmd. */
6145 __split_huge_pmd(vma, vmf->pmd, vmf->address, false);
6146
6147 return VM_FAULT_FALLBACK;
6148 }
6149
--
0-DAY CI Kernel Test Service
https://github.com/intel/lkp-tests/wiki
Hi Zhang,
kernel test robot noticed the following build errors:
[auto build test ERROR on next-20251215]
url: https://github.com/intel-lab-lkp/linux/commits/Zhang-Qilong/mm-huge_memory-Implementation-of-THP-COW-for-executable-file-mmap/20251215-204035
base: next-20251215
patch link: https://lore.kernel.org/r/20251215123407.380813-3-zhangqilong3%40huawei.com
patch subject: [PATCH next 2/2] mm/huge_memory: Add sysfs knob for executable THP COW
config: nios2-allnoconfig (https://download.01.org/0day-ci/archive/20251216/202512161352.7XX7Lgv9-lkp@intel.com/config)
compiler: nios2-linux-gcc (GCC) 11.5.0
reproduce (this is a W=1 build): (https://download.01.org/0day-ci/archive/20251216/202512161352.7XX7Lgv9-lkp@intel.com/reproduce)
If you fix the issue in a separate patch/commit (i.e. not just a new version of
the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <lkp@intel.com>
| Closes: https://lore.kernel.org/oe-kbuild-all/202512161352.7XX7Lgv9-lkp@intel.com/
All errors (new ones prefixed by >>):
mm/memory.c: In function 'wp_huge_pmd':
>> mm/memory.c:6131:13: error: implicit declaration of function 'transparent_hugepage_use_exec_cow' [-Werror=implicit-function-declaration]
6131 | if (transparent_hugepage_use_exec_cow() &&
| ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
mm/memory.c:6135:21: error: implicit declaration of function 'vma_is_special_huge'; did you mean 'vma_is_special_mapping'? [-Werror=implicit-function-declaration]
6135 | if (vma_is_special_huge(vma) || vma_is_shmem(vma))
| ^~~~~~~~~~~~~~~~~~~
| vma_is_special_mapping
cc1: some warnings being treated as errors
vim +/transparent_hugepage_use_exec_cow +6131 mm/memory.c
6104
6105 /* `inline' is required to avoid gcc 4.1.2 build error */
6106 static inline vm_fault_t wp_huge_pmd(struct vm_fault *vmf)
6107 {
6108 struct vm_area_struct *vma = vmf->vma;
6109 const bool unshare = vmf->flags & FAULT_FLAG_UNSHARE;
6110 vm_fault_t ret;
6111
6112 if (vma_is_anonymous(vma)) {
6113 if (likely(!unshare) &&
6114 userfaultfd_huge_pmd_wp(vma, vmf->orig_pmd)) {
6115 if (userfaultfd_wp_async(vmf->vma))
6116 goto split;
6117 return handle_userfault(vmf, VM_UFFD_WP);
6118 }
6119 return do_huge_pmd_wp_page(vmf);
6120 }
6121
6122 if (vma->vm_flags & (VM_SHARED | VM_MAYSHARE)) {
6123 if (vma->vm_ops->huge_fault) {
6124 ret = vma->vm_ops->huge_fault(vmf, PMD_ORDER);
6125 if (!(ret & VM_FAULT_FALLBACK))
6126 return ret;
6127 }
6128 }
6129
6130
> 6131 if (transparent_hugepage_use_exec_cow() &&
6132 is_exec_mapping(vma->vm_flags) &&
6133 is_cow_mapping(vma->vm_flags)) {
6134 /* Skip special and shmem */
6135 if (vma_is_special_huge(vma) || vma_is_shmem(vma))
6136 goto split;
6137
6138 ret = do_huge_pmd_exec_cow(vmf);
6139 if (!(ret & VM_FAULT_FALLBACK))
6140 return ret;
6141 }
6142
6143 split:
6144 /* COW or write-notify handled on pte level: split pmd. */
6145 __split_huge_pmd(vma, vmf->pmd, vmf->address, false);
6146
6147 return VM_FAULT_FALLBACK;
6148 }
6149
--
0-DAY CI Kernel Test Service
https://github.com/intel/lkp-tests/wiki
© 2016 - 2025 Red Hat, Inc.