From nobody Sun Feb 8 05:42:11 2026 Received: from foss.arm.com (foss.arm.com [217.140.110.172]) by smtp.subspace.kernel.org (Postfix) with ESMTP id DF64918B474 for ; Thu, 8 Aug 2024 11:18:59 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=217.140.110.172 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1723115941; cv=none; b=EkgBVcmEjlEbxVgridd0wxFfxbkRIV5vbSN6DXnWTQ6VZBOWmAI7w457+o7X96pF+mOd93l8RYnCOh/nr+PPH1mIuQRLRV75NA83CRj5+lQCU70XQ4OUmvLvheBXJ+2ieB9TTN0DYt2g1CO0gM4rnX/+UibIqFEWMa1Rh48ezLE= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1723115941; c=relaxed/simple; bh=rKWnLje+3XCbY3wPN+vMhOC5+eitLItFmOJp2JKVgHU=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=LbGiFo8N1Yfb95mDCqqOoCAI8hEBJorXwysnKEpdk5K97a+FW0XEEtaq/QtylzaSLh64BLh16jJjGcXvzBI6yJeLPLVZjeCbpocwaw5z2Jiab9ErTASO7qzP4VF3vc0PUitasBp2aiEToLKgrSphapXC4HjnBYxzqmvTO6vkTdg= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com; spf=pass smtp.mailfrom=arm.com; arc=none smtp.client-ip=217.140.110.172 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=arm.com Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id 0255D1042; Thu, 8 Aug 2024 04:19:25 -0700 (PDT) Received: from e125769.cambridge.arm.com (e125769.cambridge.arm.com [10.1.196.27]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPSA id CD3003F71E; Thu, 8 Aug 2024 04:18:57 -0700 (PDT) From: Ryan Roberts To: Andrew Morton , Hugh Dickins , "Matthew Wilcox (Oracle)" , David Hildenbrand , Barry Song , Lance Yang , Baolin Wang , Gavin Shan Cc: Ryan Roberts , linux-kernel@vger.kernel.org, linux-mm@kvack.org Subject: [PATCH v3 1/2] mm: Cleanup count_mthp_stat() definition Date: Thu, 8 Aug 2024 12:18:46 +0100 Message-ID: <20240808111849.651867-2-ryan.roberts@arm.com> X-Mailer: git-send-email 2.43.0 In-Reply-To: <20240808111849.651867-1-ryan.roberts@arm.com> References: <20240808111849.651867-1-ryan.roberts@arm.com> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Let's move count_mthp_stat() so that it's always defined, even when THP is disabled. Previously uses of the function in files such as shmem.c, which are compiled even when THP is disabled, required ugly THP ifdeferry. With this cleanup, we can remove those ifdefs and the function resolves to a nop when THP is disabled. I shortly plan to call count_mthp_stat() from more THP-invariant source files. Signed-off-by: Ryan Roberts Acked-by: Barry Song Reviewed-by: Baolin Wang Reviewed-by: Lance Yang Acked-by: David Hildenbrand --- include/linux/huge_mm.h | 70 ++++++++++++++++++++--------------------- mm/memory.c | 2 -- mm/shmem.c | 6 ---- 3 files changed, 35 insertions(+), 43 deletions(-) diff --git a/include/linux/huge_mm.h b/include/linux/huge_mm.h index 2b5a175196e7a..969f11f360d29 100644 --- a/include/linux/huge_mm.h +++ b/include/linux/huge_mm.h @@ -114,6 +114,41 @@ extern struct kobj_attribute thpsize_shmem_enabled_att= r; #define HPAGE_PUD_MASK (~(HPAGE_PUD_SIZE - 1)) #define HPAGE_PUD_SIZE ((1UL) << HPAGE_PUD_SHIFT) =20 +enum mthp_stat_item { + MTHP_STAT_ANON_FAULT_ALLOC, + MTHP_STAT_ANON_FAULT_FALLBACK, + MTHP_STAT_ANON_FAULT_FALLBACK_CHARGE, + MTHP_STAT_SWPOUT, + MTHP_STAT_SWPOUT_FALLBACK, + MTHP_STAT_SHMEM_ALLOC, + MTHP_STAT_SHMEM_FALLBACK, + MTHP_STAT_SHMEM_FALLBACK_CHARGE, + MTHP_STAT_SPLIT, + MTHP_STAT_SPLIT_FAILED, + MTHP_STAT_SPLIT_DEFERRED, + __MTHP_STAT_COUNT +}; + +#if defined(CONFIG_TRANSPARENT_HUGEPAGE) && defined(CONFIG_SYSFS) +struct mthp_stat { + unsigned long stats[ilog2(MAX_PTRS_PER_PTE) + 1][__MTHP_STAT_COUNT]; +}; + +DECLARE_PER_CPU(struct mthp_stat, mthp_stats); + +static inline void count_mthp_stat(int order, enum mthp_stat_item item) +{ + if (order <=3D 0 || order > PMD_ORDER) + return; + + this_cpu_inc(mthp_stats.stats[order][item]); +} +#else +static inline void count_mthp_stat(int order, enum mthp_stat_item item) +{ +} +#endif + #ifdef CONFIG_TRANSPARENT_HUGEPAGE =20 extern unsigned long transparent_hugepage_flags; @@ -269,41 +304,6 @@ struct thpsize { =20 #define to_thpsize(kobj) container_of(kobj, struct thpsize, kobj) =20 -enum mthp_stat_item { - MTHP_STAT_ANON_FAULT_ALLOC, - MTHP_STAT_ANON_FAULT_FALLBACK, - MTHP_STAT_ANON_FAULT_FALLBACK_CHARGE, - MTHP_STAT_SWPOUT, - MTHP_STAT_SWPOUT_FALLBACK, - MTHP_STAT_SHMEM_ALLOC, - MTHP_STAT_SHMEM_FALLBACK, - MTHP_STAT_SHMEM_FALLBACK_CHARGE, - MTHP_STAT_SPLIT, - MTHP_STAT_SPLIT_FAILED, - MTHP_STAT_SPLIT_DEFERRED, - __MTHP_STAT_COUNT -}; - -struct mthp_stat { - unsigned long stats[ilog2(MAX_PTRS_PER_PTE) + 1][__MTHP_STAT_COUNT]; -}; - -#ifdef CONFIG_SYSFS -DECLARE_PER_CPU(struct mthp_stat, mthp_stats); - -static inline void count_mthp_stat(int order, enum mthp_stat_item item) -{ - if (order <=3D 0 || order > PMD_ORDER) - return; - - this_cpu_inc(mthp_stats.stats[order][item]); -} -#else -static inline void count_mthp_stat(int order, enum mthp_stat_item item) -{ -} -#endif - #define transparent_hugepage_use_zero_page() \ (transparent_hugepage_flags & \ (1<vm_mm, MM_ANONPAGES, nr_pages); -#ifdef CONFIG_TRANSPARENT_HUGEPAGE count_mthp_stat(folio_order(folio), MTHP_STAT_ANON_FAULT_ALLOC); -#endif folio_add_new_anon_rmap(folio, vma, addr, RMAP_EXCLUSIVE); folio_add_lru_vma(folio, vma); setpte: diff --git a/mm/shmem.c b/mm/shmem.c index 68c9a31bc763b..33a146843e4c0 100644 --- a/mm/shmem.c +++ b/mm/shmem.c @@ -1791,9 +1791,7 @@ static struct folio *shmem_alloc_and_add_folio(struct= vm_fault *vmf, =20 if (pages =3D=3D HPAGE_PMD_NR) count_vm_event(THP_FILE_FALLBACK); -#ifdef CONFIG_TRANSPARENT_HUGEPAGE count_mthp_stat(order, MTHP_STAT_SHMEM_FALLBACK); -#endif order =3D next_order(&suitable_orders, order); } } else { @@ -1818,10 +1816,8 @@ static struct folio *shmem_alloc_and_add_folio(struc= t vm_fault *vmf, count_vm_event(THP_FILE_FALLBACK); count_vm_event(THP_FILE_FALLBACK_CHARGE); } -#ifdef CONFIG_TRANSPARENT_HUGEPAGE count_mthp_stat(folio_order(folio), MTHP_STAT_SHMEM_FALLBACK); count_mthp_stat(folio_order(folio), MTHP_STAT_SHMEM_FALLBACK_CHARGE); -#endif } goto unlock; } @@ -2301,9 +2297,7 @@ static int shmem_get_folio_gfp(struct inode *inode, p= goff_t index, if (!IS_ERR(folio)) { if (folio_test_pmd_mappable(folio)) count_vm_event(THP_FILE_ALLOC); -#ifdef CONFIG_TRANSPARENT_HUGEPAGE count_mthp_stat(folio_order(folio), MTHP_STAT_SHMEM_ALLOC); -#endif goto alloced; } if (PTR_ERR(folio) =3D=3D -EEXIST) --=20 2.43.0 From nobody Sun Feb 8 05:42:11 2026 Received: from foss.arm.com (foss.arm.com [217.140.110.172]) by smtp.subspace.kernel.org (Postfix) with ESMTP id 6CDD218C337 for ; Thu, 8 Aug 2024 11:19:01 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=217.140.110.172 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1723115943; cv=none; b=bXfKSCt6UxZthwd5jdE3v0sCKqe/dC//PfdtnN79RHnneoI7ht56w7MaktJq5Y3JNSAMS4QKMjpbLv8ZAiaKZehkusZgip1FfilT+MuEHoFgCo9ZEKHuk2Lo7tit0AHxlUAQ6fGoJKqqm5EN8CcwUWBWpITrBYq+tGujUA+qSx4= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1723115943; c=relaxed/simple; bh=XVvQeBUz27N0FgchdGrG0itzCFqwQD8rnk79VCeiyps=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=rROlc4R9lCr/yjJ+LaTO6x5AibaDB0hoik2sPZmPD0VZ3gJ1hNrzaZsiYegehV9WzbTdexcBs53WKsJ+uoJ48H5hrV5dSlGUyElxYSSp63xMz7Cr37fWS0ngPy8HvU77moiZxPbeAJZBEzxNW93xWp2AC6Qw7I0ImyarvIIz0zs= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com; spf=pass smtp.mailfrom=arm.com; arc=none smtp.client-ip=217.140.110.172 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=arm.com Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id 97797139F; Thu, 8 Aug 2024 04:19:26 -0700 (PDT) Received: from e125769.cambridge.arm.com (e125769.cambridge.arm.com [10.1.196.27]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPSA id 6EACD3F71E; Thu, 8 Aug 2024 04:18:59 -0700 (PDT) From: Ryan Roberts To: Andrew Morton , Hugh Dickins , "Matthew Wilcox (Oracle)" , David Hildenbrand , Barry Song , Lance Yang , Baolin Wang , Gavin Shan Cc: Ryan Roberts , linux-kernel@vger.kernel.org, linux-mm@kvack.org Subject: [PATCH v3 2/2] mm: Tidy up shmem mTHP controls and stats Date: Thu, 8 Aug 2024 12:18:47 +0100 Message-ID: <20240808111849.651867-3-ryan.roberts@arm.com> X-Mailer: git-send-email 2.43.0 In-Reply-To: <20240808111849.651867-1-ryan.roberts@arm.com> References: <20240808111849.651867-1-ryan.roberts@arm.com> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Previously we had a situation where shmem mTHP controls and stats were not exposed for some supported sizes and were exposed for some unsupported sizes. So let's clean that up. Anon mTHP can support all large orders [2, PMD_ORDER]. But shmem can support all large orders [1, MAX_PAGECACHE_ORDER]. However, per-size shmem controls and stats were previously being exposed for all the anon mTHP orders, meaning order-1 was not present, and for arm64 64K base pages, orders 12 and 13 were exposed but were not supported internally. Tidy this all up by defining ctrl and stats attribute groups for anon and file separately. Anon ctrl and stats groups are populated for all orders in THP_ORDERS_ALL_ANON and file ctrl and stats groups are populated for all orders in THP_ORDERS_ALL_FILE_DEFAULT. Additionally, create "any" ctrl and stats attribute groups which are populated for all orders in (THP_ORDERS_ALL_ANON | THP_ORDERS_ALL_FILE_DEFAULT). swpout stats use this since they apply to anon and shmem. The side-effect of all this is that different hugepage-*kB directories contain different sets of controls and stats, depending on which memory types support that size. This approach is preferred over the alternative, which is to populate dummy controls and stats for memory types that do not support a given size. Signed-off-by: Ryan Roberts Reviewed-by: Baolin Wang Tested-by: Barry Song --- mm/huge_memory.c | 144 +++++++++++++++++++++++++++++++++++++---------- 1 file changed, 114 insertions(+), 30 deletions(-) diff --git a/mm/huge_memory.c b/mm/huge_memory.c index 0c3075ee00012..082d86b7c6c2f 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -482,8 +482,8 @@ static void thpsize_release(struct kobject *kobj); static DEFINE_SPINLOCK(huge_anon_orders_lock); static LIST_HEAD(thpsize_list); =20 -static ssize_t thpsize_enabled_show(struct kobject *kobj, - struct kobj_attribute *attr, char *buf) +static ssize_t anon_enabled_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) { int order =3D to_thpsize(kobj)->order; const char *output; @@ -500,9 +500,9 @@ static ssize_t thpsize_enabled_show(struct kobject *kob= j, return sysfs_emit(buf, "%s\n", output); } =20 -static ssize_t thpsize_enabled_store(struct kobject *kobj, - struct kobj_attribute *attr, - const char *buf, size_t count) +static ssize_t anon_enabled_store(struct kobject *kobj, + struct kobj_attribute *attr, + const char *buf, size_t count) { int order =3D to_thpsize(kobj)->order; ssize_t ret =3D count; @@ -544,19 +544,35 @@ static ssize_t thpsize_enabled_store(struct kobject *= kobj, return ret; } =20 -static struct kobj_attribute thpsize_enabled_attr =3D - __ATTR(enabled, 0644, thpsize_enabled_show, thpsize_enabled_store); +static struct kobj_attribute anon_enabled_attr =3D + __ATTR(enabled, 0644, anon_enabled_show, anon_enabled_store); =20 -static struct attribute *thpsize_attrs[] =3D { - &thpsize_enabled_attr.attr, +static struct attribute *anon_ctrl_attrs[] =3D { + &anon_enabled_attr.attr, + NULL, +}; + +static const struct attribute_group anon_ctrl_attr_grp =3D { + .attrs =3D anon_ctrl_attrs, +}; + +static struct attribute *file_ctrl_attrs[] =3D { #ifdef CONFIG_SHMEM &thpsize_shmem_enabled_attr.attr, #endif NULL, }; =20 -static const struct attribute_group thpsize_attr_group =3D { - .attrs =3D thpsize_attrs, +static const struct attribute_group file_ctrl_attr_grp =3D { + .attrs =3D file_ctrl_attrs, +}; + +static struct attribute *any_ctrl_attrs[] =3D { + NULL, +}; + +static const struct attribute_group any_ctrl_attr_grp =3D { + .attrs =3D any_ctrl_attrs, }; =20 static const struct kobj_type thpsize_ktype =3D { @@ -595,64 +611,132 @@ DEFINE_MTHP_STAT_ATTR(anon_fault_fallback, MTHP_STAT= _ANON_FAULT_FALLBACK); DEFINE_MTHP_STAT_ATTR(anon_fault_fallback_charge, MTHP_STAT_ANON_FAULT_FAL= LBACK_CHARGE); DEFINE_MTHP_STAT_ATTR(swpout, MTHP_STAT_SWPOUT); DEFINE_MTHP_STAT_ATTR(swpout_fallback, MTHP_STAT_SWPOUT_FALLBACK); +#ifdef CONFIG_SHMEM DEFINE_MTHP_STAT_ATTR(shmem_alloc, MTHP_STAT_SHMEM_ALLOC); DEFINE_MTHP_STAT_ATTR(shmem_fallback, MTHP_STAT_SHMEM_FALLBACK); DEFINE_MTHP_STAT_ATTR(shmem_fallback_charge, MTHP_STAT_SHMEM_FALLBACK_CHAR= GE); +#endif DEFINE_MTHP_STAT_ATTR(split, MTHP_STAT_SPLIT); DEFINE_MTHP_STAT_ATTR(split_failed, MTHP_STAT_SPLIT_FAILED); DEFINE_MTHP_STAT_ATTR(split_deferred, MTHP_STAT_SPLIT_DEFERRED); =20 -static struct attribute *stats_attrs[] =3D { +static struct attribute *anon_stats_attrs[] =3D { &anon_fault_alloc_attr.attr, &anon_fault_fallback_attr.attr, &anon_fault_fallback_charge_attr.attr, +#ifndef CONFIG_SHMEM &swpout_attr.attr, &swpout_fallback_attr.attr, - &shmem_alloc_attr.attr, - &shmem_fallback_attr.attr, - &shmem_fallback_charge_attr.attr, +#endif &split_attr.attr, &split_failed_attr.attr, &split_deferred_attr.attr, NULL, }; =20 -static struct attribute_group stats_attr_group =3D { +static struct attribute_group anon_stats_attr_grp =3D { + .name =3D "stats", + .attrs =3D anon_stats_attrs, +}; + +static struct attribute *file_stats_attrs[] =3D { +#ifdef CONFIG_SHMEM + &shmem_alloc_attr.attr, + &shmem_fallback_attr.attr, + &shmem_fallback_charge_attr.attr, +#endif + NULL, +}; + +static struct attribute_group file_stats_attr_grp =3D { + .name =3D "stats", + .attrs =3D file_stats_attrs, +}; + +static struct attribute *any_stats_attrs[] =3D { +#ifdef CONFIG_SHMEM + &swpout_attr.attr, + &swpout_fallback_attr.attr, +#endif + NULL, +}; + +static struct attribute_group any_stats_attr_grp =3D { .name =3D "stats", - .attrs =3D stats_attrs, + .attrs =3D any_stats_attrs, }; =20 +static int sysfs_add_group(struct kobject *kobj, + const struct attribute_group *grp) +{ + int ret =3D -ENOENT; + + /* + * If the group is named, try to merge first, assuming the subdirectory + * was already created. This avoids the warning emitted by + * sysfs_create_group() if the directory already exists. + */ + if (grp->name) + ret =3D sysfs_merge_group(kobj, grp); + if (ret) + ret =3D sysfs_create_group(kobj, grp); + + return ret; +} + static struct thpsize *thpsize_create(int order, struct kobject *parent) { unsigned long size =3D (PAGE_SIZE << order) / SZ_1K; struct thpsize *thpsize; - int ret; + int ret =3D -ENOMEM; =20 thpsize =3D kzalloc(sizeof(*thpsize), GFP_KERNEL); if (!thpsize) - return ERR_PTR(-ENOMEM); + goto err; + + thpsize->order =3D order; =20 ret =3D kobject_init_and_add(&thpsize->kobj, &thpsize_ktype, parent, "hugepages-%lukB", size); if (ret) { kfree(thpsize); - return ERR_PTR(ret); + goto err; } =20 - ret =3D sysfs_create_group(&thpsize->kobj, &thpsize_attr_group); - if (ret) { - kobject_put(&thpsize->kobj); - return ERR_PTR(ret); + + ret =3D sysfs_add_group(&thpsize->kobj, &any_ctrl_attr_grp); + if (ret) + goto err_put; + + ret =3D sysfs_add_group(&thpsize->kobj, &any_stats_attr_grp); + if (ret) + goto err_put; + + if (BIT(order) & THP_ORDERS_ALL_ANON) { + ret =3D sysfs_add_group(&thpsize->kobj, &anon_ctrl_attr_grp); + if (ret) + goto err_put; + + ret =3D sysfs_add_group(&thpsize->kobj, &anon_stats_attr_grp); + if (ret) + goto err_put; } =20 - ret =3D sysfs_create_group(&thpsize->kobj, &stats_attr_group); - if (ret) { - kobject_put(&thpsize->kobj); - return ERR_PTR(ret); + if (BIT(order) & THP_ORDERS_ALL_FILE_DEFAULT) { + ret =3D sysfs_add_group(&thpsize->kobj, &file_ctrl_attr_grp); + if (ret) + goto err_put; + + ret =3D sysfs_add_group(&thpsize->kobj, &file_stats_attr_grp); + if (ret) + goto err_put; } =20 - thpsize->order =3D order; return thpsize; +err_put: + kobject_put(&thpsize->kobj); +err: + return ERR_PTR(ret); } =20 static void thpsize_release(struct kobject *kobj) @@ -692,7 +776,7 @@ static int __init hugepage_init_sysfs(struct kobject **= hugepage_kobj) goto remove_hp_group; } =20 - orders =3D THP_ORDERS_ALL_ANON; + orders =3D THP_ORDERS_ALL_ANON | THP_ORDERS_ALL_FILE_DEFAULT; order =3D highest_order(orders); while (orders) { thpsize =3D thpsize_create(order, *hugepage_kobj); --=20 2.43.0