From nobody Wed Feb 11 05:04:54 2026 Received: from out30-130.freemail.mail.aliyun.com (out30-130.freemail.mail.aliyun.com [115.124.30.130]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 88D6C482CA for ; Mon, 22 Apr 2024 07:03:00 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=115.124.30.130 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1713769383; cv=none; b=CYftjywwC/1PmhtAnYDJX0BvMWxyOL48jbk1MKh+sp+k8zlA8Koj0UzUNGBRQvz8g9G/EyEMAfPnFPla9Fpxvl/45YJzIpxdB7W6FX0g1JWso2ZjlG39Ez7ibO73hKkitBDlnYNlRKDhxqTOXNbszvkrZ2n7qYxXk4q9zIiO8Zg= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1713769383; c=relaxed/simple; bh=TBzYTl2S89wFgHKU5IZMBLPsDVAVa/SBQeUl+du9Wzk=; h=From:To:Cc:Subject:Date:Message-Id:In-Reply-To:References: MIME-Version; b=PAtPjmD8Vv/ilVhhYQlfeX//vJO2Bt8x4XTbZMzMdVrcs9DkXx812aF6RC/3sIlNMvfnDqKKV/OqUrJ87gl3UM7IDjrHI9TDCTONRRCaNaSmTAp3u8QyC8bNnrGi70zaS5iFwhSG5K4FWTvh0x74uz5vVD4iIIvjh6bRb2t3ooI= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=linux.alibaba.com; spf=pass smtp.mailfrom=linux.alibaba.com; dkim=pass (1024-bit key) header.d=linux.alibaba.com header.i=@linux.alibaba.com header.b=d9H7dXA3; arc=none smtp.client-ip=115.124.30.130 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=linux.alibaba.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=linux.alibaba.com Authentication-Results: smtp.subspace.kernel.org; dkim=pass (1024-bit key) header.d=linux.alibaba.com header.i=@linux.alibaba.com header.b="d9H7dXA3" DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=linux.alibaba.com; s=default; t=1713769378; h=From:To:Subject:Date:Message-Id:MIME-Version; bh=iA51Bkqzy9zcXZ9S3Nkg6q/BBso18kv2IFiMWpEhGCs=; b=d9H7dXA3OkN5DMCvZyHjmmKv7zMP+IdJ9zqkOuoZ0p+W8tBIIEhqlgSceJJq1t1yms1JHJ2VSlMYTqaz2Au6/rREcdgMlWllo4gONZI28TaeFugKCrksHHLgAiAl2M0XWaegw8q2PG7XtPQTDGlwjijQnre9x+n77yV/drgkE6s= X-Alimail-AntiSpam: AC=PASS;BC=-1|-1;BR=01201311R191e4;CH=green;DM=||false|;DS=||;FP=0|-1|-1|-1|0|-1|-1|-1;HT=ay29a033018045168;MF=baolin.wang@linux.alibaba.com;NM=1;PH=DS;RN=13;SR=0;TI=SMTPD_---0W5.T-Y-_1713769375; Received: from localhost(mailfrom:baolin.wang@linux.alibaba.com fp:SMTPD_---0W5.T-Y-_1713769375) by smtp.aliyun-inc.com; Mon, 22 Apr 2024 15:02:57 +0800 From: Baolin Wang To: akpm@linux-foundation.org, hughd@google.com Cc: willy@infradead.org, david@redhat.com, wangkefeng.wang@huawei.com, 21cnbao@gmail.com, ryan.roberts@arm.com, ying.huang@intel.com, shy828301@gmail.com, ziy@nvidia.com, baolin.wang@linux.alibaba.com, linux-mm@kvack.org, linux-kernel@vger.kernel.org Subject: [RFC PATCH 1/5] mm: memory: extend finish_fault() to support large folio Date: Mon, 22 Apr 2024 15:02:39 +0800 Message-Id: <358aefb1858b63164894d7d8504f3dae0b495366.1713755580.git.baolin.wang@linux.alibaba.com> X-Mailer: git-send-email 2.39.3 In-Reply-To: References: Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Add large folio mapping establishment support for finish_fault() as a prepa= ration, to support multi-size THP allocation of anonymous shared pages in the follo= wing patches. Signed-off-by: Baolin Wang --- mm/memory.c | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/mm/memory.c b/mm/memory.c index b6fa5146b260..094a76730776 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -4766,7 +4766,10 @@ vm_fault_t finish_fault(struct vm_fault *vmf) { struct vm_area_struct *vma =3D vmf->vma; struct page *page; + struct folio *folio; vm_fault_t ret; + int nr_pages, i; + unsigned long addr; =20 /* Did we COW the page? */ if ((vmf->flags & FAULT_FLAG_WRITE) && !(vma->vm_flags & VM_SHARED)) @@ -4797,22 +4800,30 @@ vm_fault_t finish_fault(struct vm_fault *vmf) return VM_FAULT_OOM; } =20 + folio =3D page_folio(page); + nr_pages =3D folio_nr_pages(folio); + addr =3D ALIGN_DOWN(vmf->address, nr_pages * PAGE_SIZE); vmf->pte =3D pte_offset_map_lock(vma->vm_mm, vmf->pmd, - vmf->address, &vmf->ptl); + addr, &vmf->ptl); if (!vmf->pte) return VM_FAULT_NOPAGE; =20 /* Re-check under ptl */ - if (likely(!vmf_pte_changed(vmf))) { - struct folio *folio =3D page_folio(page); - - set_pte_range(vmf, folio, page, 1, vmf->address); - ret =3D 0; - } else { + if (nr_pages =3D=3D 1 && vmf_pte_changed(vmf)) { update_mmu_tlb(vma, vmf->address, vmf->pte); ret =3D VM_FAULT_NOPAGE; + goto unlock; + } else if (nr_pages > 1 && !pte_range_none(vmf->pte, nr_pages)) { + for (i =3D 0; i < nr_pages; i++) + update_mmu_tlb(vma, addr + PAGE_SIZE * i, vmf->pte + i); + ret =3D VM_FAULT_NOPAGE; + goto unlock; } =20 + set_pte_range(vmf, folio, &folio->page, nr_pages, addr); + ret =3D 0; + +unlock: pte_unmap_unlock(vmf->pte, vmf->ptl); return ret; } --=20 2.39.3 From nobody Wed Feb 11 05:04:54 2026 Received: from out30-98.freemail.mail.aliyun.com (out30-98.freemail.mail.aliyun.com [115.124.30.98]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 1F97D4C62A for ; Mon, 22 Apr 2024 07:03:07 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=115.124.30.98 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1713769390; cv=none; b=Ovux13yXhI9GzWSeTwROWK1UhYIW/03baYpC69MjPZkOMEqczf42DyyJCJ6k6A3J+o3wmAY0ogLvPZgmlXkqMiDmEm6+62XhX0QyFqU6vnphGSr51rbRUMyeghRMtogTDbnvXdQOyNrQeLi/kTX+/ngb2ZBdBSFTDreblw/0Zrs= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1713769390; c=relaxed/simple; bh=UVfFjDMATJJc4EWQb5F1toBkG4v8CsSGgcSCyesdW4M=; h=From:To:Cc:Subject:Date:Message-Id:In-Reply-To:References: MIME-Version; b=OLmCEJY+1dEBJubeQrbvzv0Iw3RsUl8Hwd02ARDvdqyXcE0DQSqsDgxmAxjIf80t2I6ve5SEL2RLIVI+aUoM4HGZPUMiC/kiLGY5v4MkPBoSZLh/ZZK2oS5r5i74aGukQ7h+YZJxtSgb8wPUsNCTAax+psAI7DJEBkQCnTQ/fSo= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=linux.alibaba.com; spf=pass smtp.mailfrom=linux.alibaba.com; dkim=pass (1024-bit key) header.d=linux.alibaba.com header.i=@linux.alibaba.com header.b=tDCV8JxE; arc=none smtp.client-ip=115.124.30.98 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=linux.alibaba.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=linux.alibaba.com Authentication-Results: smtp.subspace.kernel.org; dkim=pass (1024-bit key) header.d=linux.alibaba.com header.i=@linux.alibaba.com header.b="tDCV8JxE" DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=linux.alibaba.com; s=default; t=1713769380; h=From:To:Subject:Date:Message-Id:MIME-Version; bh=BV2U/IeVyqp2XSH/3wx9gpceXxijONOM8cRleKyQmTE=; b=tDCV8JxEBCbqsQDyR0vP1FoyKkyzBNksQ1vUo9tSdXVQLC+8aj9bSUOj1fJ8OyL1HC1zu90J4qQ21uq0mOdfF2zWdEUhtdb7Q5z29x2j3J+9g9y9v/iUG9NQL5MtAuuh6jL4rlYLap2RbNR5G7N5jlTe9dTMuZ3xMRCuNqwqEys= X-Alimail-AntiSpam: AC=PASS;BC=-1|-1;BR=01201311R891e4;CH=green;DM=||false|;DS=||;FP=0|-1|-1|-1|0|-1|-1|-1;HT=ay29a033018045176;MF=baolin.wang@linux.alibaba.com;NM=1;PH=DS;RN=13;SR=0;TI=SMTPD_---0W5.T-YL_1713769377; Received: from localhost(mailfrom:baolin.wang@linux.alibaba.com fp:SMTPD_---0W5.T-YL_1713769377) by smtp.aliyun-inc.com; Mon, 22 Apr 2024 15:02:58 +0800 From: Baolin Wang To: akpm@linux-foundation.org, hughd@google.com Cc: willy@infradead.org, david@redhat.com, wangkefeng.wang@huawei.com, 21cnbao@gmail.com, ryan.roberts@arm.com, ying.huang@intel.com, shy828301@gmail.com, ziy@nvidia.com, baolin.wang@linux.alibaba.com, linux-mm@kvack.org, linux-kernel@vger.kernel.org Subject: [RFC PATCH 2/5] mm: shmem: add an 'order' parameter for shmem_alloc_hugefolio() Date: Mon, 22 Apr 2024 15:02:40 +0800 Message-Id: <2014bf7370d78bc1f5600731af5bf8f569e5868b.1713755580.git.baolin.wang@linux.alibaba.com> X-Mailer: git-send-email 2.39.3 In-Reply-To: References: Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Add a new parameter to specify the huge page order for shmem_alloc_hugefoli= o(), as a preparation to supoort mTHP. Signed-off-by: Baolin Wang --- mm/shmem.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/mm/shmem.c b/mm/shmem.c index fa2a0ed97507..893c88efc45f 100644 --- a/mm/shmem.c +++ b/mm/shmem.c @@ -1604,14 +1604,14 @@ static gfp_t limit_gfp_mask(gfp_t huge_gfp, gfp_t l= imit_gfp) } =20 static struct folio *shmem_alloc_hugefolio(gfp_t gfp, - struct shmem_inode_info *info, pgoff_t index) + struct shmem_inode_info *info, pgoff_t index, int order) { struct mempolicy *mpol; pgoff_t ilx; struct page *page; =20 - mpol =3D shmem_get_pgoff_policy(info, index, HPAGE_PMD_ORDER, &ilx); - page =3D alloc_pages_mpol(gfp, HPAGE_PMD_ORDER, mpol, ilx, numa_node_id()= ); + mpol =3D shmem_get_pgoff_policy(info, index, order, &ilx); + page =3D alloc_pages_mpol(gfp, order, mpol, ilx, numa_node_id()); mpol_cond_put(mpol); =20 return page_rmappable_folio(page); @@ -1639,13 +1639,14 @@ static struct folio *shmem_alloc_and_add_folio(gfp_= t gfp, struct shmem_inode_info *info =3D SHMEM_I(inode); struct folio *folio; long pages; - int error; + int error, order; =20 if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) huge =3D false; =20 if (huge) { pages =3D HPAGE_PMD_NR; + order =3D HPAGE_PMD_ORDER; index =3D round_down(index, HPAGE_PMD_NR); =20 /* @@ -1660,7 +1661,7 @@ static struct folio *shmem_alloc_and_add_folio(gfp_t = gfp, index + HPAGE_PMD_NR - 1, XA_PRESENT)) return ERR_PTR(-E2BIG); =20 - folio =3D shmem_alloc_hugefolio(gfp, info, index); + folio =3D shmem_alloc_hugefolio(gfp, info, index, order); if (!folio) count_vm_event(THP_FILE_FALLBACK); } else { --=20 2.39.3 From nobody Wed Feb 11 05:04:54 2026 Received: from out30-119.freemail.mail.aliyun.com (out30-119.freemail.mail.aliyun.com [115.124.30.119]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id A2019495F0 for ; Mon, 22 Apr 2024 07:03:05 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=115.124.30.119 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1713769388; cv=none; b=XNGUCUv/Rgf9EQ3LZu1JjoZZjF9JcN0CxYZpf+tTnbxTyOhP1be/JzPdeE31CUK62HyzQHVtQZAaYybYG1I4iJTfk0sQzU3OwUFCPS6atFackVE8UGdGUtW6UVctbcsZDYIsmRL1jwleYe4BkjVeGTo+8d5L5q5FZkZJqMTyOdo= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1713769388; c=relaxed/simple; bh=lv5PqU5g2qQjCdgqUVI5mkhEGLV73xIpYwqgswJR48g=; h=From:To:Cc:Subject:Date:Message-Id:In-Reply-To:References: MIME-Version; b=PAaWkZpz6bwAgWmBc47wJkI1Td5/La+0OOd99W+rk9ZeF7KL9j41AgmFrJyUfMk6oAds+b0VEncPKFfWcvK4Rdthc/foIb8y9R/UtxG9ekJftYTb/ihAE+2D3QacZFibicOim3CB0Bd/MFwOvAuBOec2w00LYlWvzGx7gQCemlA= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=linux.alibaba.com; spf=pass smtp.mailfrom=linux.alibaba.com; dkim=pass (1024-bit key) header.d=linux.alibaba.com header.i=@linux.alibaba.com header.b=goBv34hw; arc=none smtp.client-ip=115.124.30.119 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=linux.alibaba.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=linux.alibaba.com Authentication-Results: smtp.subspace.kernel.org; dkim=pass (1024-bit key) header.d=linux.alibaba.com header.i=@linux.alibaba.com header.b="goBv34hw" DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=linux.alibaba.com; s=default; t=1713769381; h=From:To:Subject:Date:Message-Id:MIME-Version; bh=0SsdcViQ2lxDH4gZYEPPhvXdWD/avqV+Env2Q3pLz7o=; b=goBv34hww93Jw8e3t4O4G8/ufkQgeZVJPNDOb3rqoX9XQhrlJAT2O4fZRs9/hlu+FvA1+/mfSnm9tSQr/+tOEHwv1iSKQnBjA66Az/Hv1YLNMWKnQJ1nm6mp9L/LikHGKe32xORmnNg/TCsFSFgoV3UYW3hAY0yMJGjNLb/yA0c= X-Alimail-AntiSpam: AC=PASS;BC=-1|-1;BR=01201311R121e4;CH=green;DM=||false|;DS=||;FP=0|-1|-1|-1|0|-1|-1|-1;HT=ay29a033018046051;MF=baolin.wang@linux.alibaba.com;NM=1;PH=DS;RN=13;SR=0;TI=SMTPD_---0W5-amKe_1713769378; Received: from localhost(mailfrom:baolin.wang@linux.alibaba.com fp:SMTPD_---0W5-amKe_1713769378) by smtp.aliyun-inc.com; Mon, 22 Apr 2024 15:02:59 +0800 From: Baolin Wang To: akpm@linux-foundation.org, hughd@google.com Cc: willy@infradead.org, david@redhat.com, wangkefeng.wang@huawei.com, 21cnbao@gmail.com, ryan.roberts@arm.com, ying.huang@intel.com, shy828301@gmail.com, ziy@nvidia.com, baolin.wang@linux.alibaba.com, linux-mm@kvack.org, linux-kernel@vger.kernel.org Subject: [RFC PATCH 3/5] mm: shmem: add THP validation for PMD-mapped THP related statistics Date: Mon, 22 Apr 2024 15:02:41 +0800 Message-Id: <4107dcc957f3b62a37c83e30ca22305c373ef149.1713755580.git.baolin.wang@linux.alibaba.com> X-Mailer: git-send-email 2.39.3 In-Reply-To: References: Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" In order to extend support for mTHP, add THP validation for PMD-mapped THP related statistics to avoid statistical confusion. Signed-off-by: Baolin Wang Reviewed-by: Barry Song --- mm/shmem.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/mm/shmem.c b/mm/shmem.c index 893c88efc45f..b4afda71a3f0 100644 --- a/mm/shmem.c +++ b/mm/shmem.c @@ -1662,7 +1662,7 @@ static struct folio *shmem_alloc_and_add_folio(gfp_t = gfp, return ERR_PTR(-E2BIG); =20 folio =3D shmem_alloc_hugefolio(gfp, info, index, order); - if (!folio) + if (!folio && pages =3D=3D HPAGE_PMD_NR) count_vm_event(THP_FILE_FALLBACK); } else { pages =3D 1; @@ -1680,7 +1680,7 @@ static struct folio *shmem_alloc_and_add_folio(gfp_t = gfp, if (xa_find(&mapping->i_pages, &index, index + pages - 1, XA_PRESENT)) { error =3D -EEXIST; - } else if (huge) { + } else if (pages =3D=3D HPAGE_PMD_NR) { count_vm_event(THP_FILE_FALLBACK); count_vm_event(THP_FILE_FALLBACK_CHARGE); } @@ -2046,7 +2046,8 @@ static int shmem_get_folio_gfp(struct inode *inode, p= goff_t index, folio =3D shmem_alloc_and_add_folio(huge_gfp, inode, index, fault_mm, true); if (!IS_ERR(folio)) { - count_vm_event(THP_FILE_ALLOC); + if (folio_test_pmd_mappable(folio)) + count_vm_event(THP_FILE_ALLOC); goto alloced; } if (PTR_ERR(folio) =3D=3D -EEXIST) --=20 2.39.3 From nobody Wed Feb 11 05:04:54 2026 Received: from out30-133.freemail.mail.aliyun.com (out30-133.freemail.mail.aliyun.com [115.124.30.133]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 249BD4D595 for ; Mon, 22 Apr 2024 07:03:10 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=115.124.30.133 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1713769393; cv=none; b=ufabPKDprOA57qrUagqHWw6tDcDV7gPsEMt2tiOMnUBjkmuyI/WuJFKatF9rT3p9w7lX45J4bSROtG//LBy20EBrOuGo/a5JZ1hyumYbQs2cgGSgfb3AlxHIbBuRW0XXYxzzdLpH1bRQqDjCtQeSXjvjhZLSyVtZbI+N7WprHUk= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1713769393; c=relaxed/simple; bh=yWTurDLcdd+LcVTVcTzcMm/uwWFm+Hh2kWaKCWEaBTI=; h=From:To:Cc:Subject:Date:Message-Id:In-Reply-To:References: MIME-Version; b=glDhGtYcRuVhHRfnMpNOMeCgTen4b//fKVRDhS35YvPtpAlfHQyD8ucBOjr2aAac3XdfyIS6pBEwM5o8vFwVsj0lCQmZpd4C+cwBPCIQ1ZMtqhikvyF4+x/3M68ht1LAYkyuRVABqFIXDMk7N+v90HYYonTProDPZQs/j/Ear0M= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=linux.alibaba.com; spf=pass smtp.mailfrom=linux.alibaba.com; dkim=pass (1024-bit key) header.d=linux.alibaba.com header.i=@linux.alibaba.com header.b=IN0hdimM; arc=none smtp.client-ip=115.124.30.133 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=linux.alibaba.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=linux.alibaba.com Authentication-Results: smtp.subspace.kernel.org; dkim=pass (1024-bit key) header.d=linux.alibaba.com header.i=@linux.alibaba.com header.b="IN0hdimM" DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=linux.alibaba.com; s=default; t=1713769383; h=From:To:Subject:Date:Message-Id:MIME-Version; bh=7TGllQsMjthzKZ5x9FWnodK5JAUiOYpAC2Vw3I8Fy2s=; b=IN0hdimMUqaWZcFSTaJIXjCH+tTFJzxeCmLPJhd2oCkve4kbVOBopXgZi3qJ563KQkqxzzrVybxfgvay497lLxddiSY2kLybvJDna37YJQPjRCbi+WO4iwNqXHlnHfiY+UO/czgukWWmIpvfq3HOh0obcaRMVfYCPqCPDLyjOwM= X-Alimail-AntiSpam: AC=PASS;BC=-1|-1;BR=01201311R171e4;CH=green;DM=||false|;DS=||;FP=0|-1|-1|-1|0|-1|-1|-1;HT=ay29a033018046059;MF=baolin.wang@linux.alibaba.com;NM=1;PH=DS;RN=13;SR=0;TI=SMTPD_---0W5.WnjE_1713769380; Received: from localhost(mailfrom:baolin.wang@linux.alibaba.com fp:SMTPD_---0W5.WnjE_1713769380) by smtp.aliyun-inc.com; Mon, 22 Apr 2024 15:03:01 +0800 From: Baolin Wang To: akpm@linux-foundation.org, hughd@google.com Cc: willy@infradead.org, david@redhat.com, wangkefeng.wang@huawei.com, 21cnbao@gmail.com, ryan.roberts@arm.com, ying.huang@intel.com, shy828301@gmail.com, ziy@nvidia.com, baolin.wang@linux.alibaba.com, linux-mm@kvack.org, linux-kernel@vger.kernel.org Subject: [RFC PATCH 4/5] mm: shmem: add mTHP support for anonymous share pages Date: Mon, 22 Apr 2024 15:02:42 +0800 Message-Id: <8f2725a856dc02c7c89b4139cc3628686c9de524.1713755580.git.baolin.wang@linux.alibaba.com> X-Mailer: git-send-email 2.39.3 In-Reply-To: References: Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Commit 19eaf44954df adds multi-size THP (mTHP) for anonymous pages, that can allow THP to be configured through the sysfs interface located at '/sys/kernel/mm/transparent_hugepage/hugepage-XXkb/enabled'. However, the anonymous share pages will ignore the anonymous mTHP rule configured through the sysfs interface, and can only use the PMD-mapped THP, that is not reasonable. Users expect to apply the mTHP rule for all anonymous pages, including the anonymous share pages, in order to enjoy the benefits of mTHP. For example, lower latency than PMD-mapped THP, smaller memory bloat than PMD-mapped THP, contiguous PTEs on ARM architectu= re to reduce TLB miss etc. The primary strategy is that, the use of huge pages for anonymous shared pa= ges still follows the global control determined by the mount option "huge=3D" p= arameter or the sysfs interface at '/sys/kernel/mm/transparent_hugepage/shmem_enable= d'. The utilization of mTHP is allowed only when the global 'huge' switch is en= abled. Subsequently, the mTHP sysfs interface (/sys/kernel/mm/transparent_hugepage= /hugepage-XXkb/enabled) is checked to determine the mTHP size that can be used for large folio allo= cation for these anonymous shared pages. Signed-off-by: Baolin Wang --- include/linux/huge_mm.h | 2 +- mm/huge_memory.c | 4 +- mm/shmem.c | 92 +++++++++++++++++++++++++++++++---------- 3 files changed, 74 insertions(+), 24 deletions(-) diff --git a/include/linux/huge_mm.h b/include/linux/huge_mm.h index b67294d5814f..26b6fa98d8ac 100644 --- a/include/linux/huge_mm.h +++ b/include/linux/huge_mm.h @@ -246,7 +246,7 @@ unsigned long thp_vma_allowable_orders(struct vm_area_s= truct *vma, unsigned long orders) { /* Optimization to check if required orders are enabled early. */ - if (enforce_sysfs && vma_is_anonymous(vma)) { + if (enforce_sysfs && (vma_is_anonymous(vma) || vma_is_anon_shmem(vma))) { unsigned long mask =3D READ_ONCE(huge_anon_orders_always); =20 if (vm_flags & VM_HUGEPAGE) diff --git a/mm/huge_memory.c b/mm/huge_memory.c index 9a1b57ef9c60..9e52c0db7580 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -86,7 +86,7 @@ unsigned long __thp_vma_allowable_orders(struct vm_area_s= truct *vma, unsigned long orders) { /* Check the intersection of requested and supported orders. */ - orders &=3D vma_is_anonymous(vma) ? + orders &=3D (vma_is_anonymous(vma) || vma_is_anon_shmem(vma)) ? THP_ORDERS_ALL_ANON : THP_ORDERS_ALL_FILE; if (!orders) return 0; @@ -152,7 +152,7 @@ unsigned long __thp_vma_allowable_orders(struct vm_area= _struct *vma, !enforce_sysfs, vma->vm_mm, vm_flags) ? orders : 0; =20 - if (!vma_is_anonymous(vma)) { + if (!vma_is_anonymous(vma) && !vma_is_anon_shmem(vma)) { /* * Enforce sysfs THP requirements as necessary. Anonymous vmas * were already handled in thp_vma_allowable_orders(). diff --git a/mm/shmem.c b/mm/shmem.c index b4afda71a3f0..8b009e7040b2 100644 --- a/mm/shmem.c +++ b/mm/shmem.c @@ -1603,6 +1603,39 @@ static gfp_t limit_gfp_mask(gfp_t huge_gfp, gfp_t li= mit_gfp) return result; } =20 +static unsigned long anon_shmem_suitable_orders(struct vm_fault *vmf, + struct address_space *mapping, pgoff_t index) +{ + struct vm_area_struct *vma =3D vmf ? vmf->vma : NULL; + unsigned long orders, pages; + int order; + + /* + * Get a list of all the (large) orders below PMD_ORDER + 1 that are enab= led + * for this vma. Then filter out the orders that can't be allocated over + * the faulting address and still be fully contained in the vma. + */ + orders =3D thp_vma_allowable_orders(vma, vma->vm_flags, false, true, true, + BIT(PMD_ORDER + 1) - 1); + orders =3D thp_vma_suitable_orders(vma, vmf->address, orders); + + if (!orders) + return orders; + + /* Find the highest order that can add into the page cache */ + order =3D highest_order(orders); + while (orders) { + pages =3D 1UL << order; + index =3D round_down(index, pages); + if (!xa_find(&mapping->i_pages, &index, + index + pages - 1, XA_PRESENT)) + break; + order =3D next_order(&orders, order); + } + + return orders; +} + static struct folio *shmem_alloc_hugefolio(gfp_t gfp, struct shmem_inode_info *info, pgoff_t index, int order) { @@ -1631,39 +1664,55 @@ static struct folio *shmem_alloc_folio(gfp_t gfp, return (struct folio *)page; } =20 -static struct folio *shmem_alloc_and_add_folio(gfp_t gfp, - struct inode *inode, pgoff_t index, +static struct folio *shmem_alloc_and_add_folio(struct vm_fault *vmf, + gfp_t gfp, struct inode *inode, pgoff_t index, struct mm_struct *fault_mm, bool huge) { struct address_space *mapping =3D inode->i_mapping; struct shmem_inode_info *info =3D SHMEM_I(inode); + struct vm_area_struct *vma =3D vmf ? vmf->vma : NULL; struct folio *folio; long pages; int error, order; + unsigned long orders; =20 if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) huge =3D false; =20 if (huge) { - pages =3D HPAGE_PMD_NR; - order =3D HPAGE_PMD_ORDER; - index =3D round_down(index, HPAGE_PMD_NR); + if (vma && vma_is_anon_shmem(vma)) { + orders =3D anon_shmem_suitable_orders(vmf, mapping, index); + WARN_ON_ONCE(!orders); + } else { + pages =3D HPAGE_PMD_NR; + orders =3D BIT(HPAGE_PMD_ORDER); + index =3D round_down(index, HPAGE_PMD_NR); =20 - /* - * Check for conflict before waiting on a huge allocation. - * Conflict might be that a huge page has just been allocated - * and added to page cache by a racing thread, or that there - * is already at least one small page in the huge extent. - * Be careful to retry when appropriate, but not forever! - * Elsewhere -EEXIST would be the right code, but not here. - */ - if (xa_find(&mapping->i_pages, &index, + /* + * Check for conflict before waiting on a huge allocation. + * Conflict might be that a huge page has just been allocated + * and added to page cache by a racing thread, or that there + * is already at least one small page in the huge extent. + * Be careful to retry when appropriate, but not forever! + * Elsewhere -EEXIST would be the right code, but not here. + */ + if (xa_find(&mapping->i_pages, &index, index + HPAGE_PMD_NR - 1, XA_PRESENT)) - return ERR_PTR(-E2BIG); + return ERR_PTR(-E2BIG); + } =20 - folio =3D shmem_alloc_hugefolio(gfp, info, index, order); - if (!folio && pages =3D=3D HPAGE_PMD_NR) - count_vm_event(THP_FILE_FALLBACK); + order =3D highest_order(orders); + while (orders) { + pages =3D 1 << order; + index =3D round_down(index, pages); + folio =3D shmem_alloc_hugefolio(gfp, info, index, order); + if (folio) + goto allocated; + + if (pages =3D=3D HPAGE_PMD_NR) + count_vm_event(THP_FILE_FALLBACK); + order =3D next_order(&orders, order); + } } else { pages =3D 1; folio =3D shmem_alloc_folio(gfp, info, index); @@ -1671,6 +1720,7 @@ static struct folio *shmem_alloc_and_add_folio(gfp_t = gfp, if (!folio) return ERR_PTR(-ENOMEM); =20 +allocated: __folio_set_locked(folio); __folio_set_swapbacked(folio); =20 @@ -2043,7 +2093,7 @@ static int shmem_get_folio_gfp(struct inode *inode, p= goff_t index, =20 huge_gfp =3D vma_thp_gfp_mask(vma); huge_gfp =3D limit_gfp_mask(huge_gfp, gfp); - folio =3D shmem_alloc_and_add_folio(huge_gfp, + folio =3D shmem_alloc_and_add_folio(vmf, huge_gfp, inode, index, fault_mm, true); if (!IS_ERR(folio)) { if (folio_test_pmd_mappable(folio)) @@ -2054,7 +2104,7 @@ static int shmem_get_folio_gfp(struct inode *inode, p= goff_t index, goto repeat; } =20 - folio =3D shmem_alloc_and_add_folio(gfp, inode, index, fault_mm, false); + folio =3D shmem_alloc_and_add_folio(vmf, gfp, inode, index, fault_mm, fal= se); if (IS_ERR(folio)) { error =3D PTR_ERR(folio); if (error =3D=3D -EEXIST) @@ -2065,7 +2115,7 @@ static int shmem_get_folio_gfp(struct inode *inode, p= goff_t index, =20 alloced: alloced =3D true; - if (folio_test_pmd_mappable(folio) && + if (folio_test_large(folio) && DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE) < folio_next_index(folio) - 1) { struct shmem_sb_info *sbinfo =3D SHMEM_SB(inode->i_sb); --=20 2.39.3 From nobody Wed Feb 11 05:04:54 2026 Received: from out199-1.us.a.mail.aliyun.com (out199-1.us.a.mail.aliyun.com [47.90.199.1]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 37EDD4E1D2 for ; Mon, 22 Apr 2024 07:03:17 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=47.90.199.1 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1713769400; cv=none; b=VqXRYSe7XMwMHBjwSb0DyWVYys6B1XFCO7hh7D/CQ9g9D/VnBsCMiLjC3j0aNhX+M8eOG28BrujsvCMaPPIiDFZEgXhfPkz5T7TYXTKE+s/ZBBJo918E3rgm37D1wh8Df3VgpPwnKUhcyd538vNKRDNl9ewU79ozRzrDZOweoh8= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1713769400; c=relaxed/simple; bh=ItrSE0/IiQfbtdgPm+4vgP6ZkRS2znwH+LvZWGKpYXk=; h=From:To:Cc:Subject:Date:Message-Id:In-Reply-To:References: MIME-Version; b=FXmeoi3wE57oTF5MZ/IbwNjRhBhs1Ap6SAIcFJAHTrwdsgB9wowsp18eCwwo36zJwCqDgPMP3Oh/XwooorZZ2c5+wTgGwIdcNfKJR8W2tN3GbzMk0/PmtoCBSVSYFCML37cgrEqjyMN6Eu68flcLCoc0jVoFRgK6PpJAhe1ZyxA= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=linux.alibaba.com; spf=pass smtp.mailfrom=linux.alibaba.com; dkim=pass (1024-bit key) header.d=linux.alibaba.com header.i=@linux.alibaba.com header.b=NY06ytDd; arc=none smtp.client-ip=47.90.199.1 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=linux.alibaba.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=linux.alibaba.com Authentication-Results: smtp.subspace.kernel.org; dkim=pass (1024-bit key) header.d=linux.alibaba.com header.i=@linux.alibaba.com header.b="NY06ytDd" DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=linux.alibaba.com; s=default; t=1713769384; h=From:To:Subject:Date:Message-Id:MIME-Version; bh=wvGbdG/PyCf+R85lbGysoGrudwoqTXGK9KleQ+1X+Xo=; b=NY06ytDd1TwKFWdcj4xVMFXRlMTqSaQnGa4qKsjTBdRmK+SdrU7+03BVecy6yt3e3p3kw2//FrXrAAZxzS90keLr1mMw5L29u4gF8HLLJmx64pKvV+eqaxxFtkvRdlGYGtm9mEBUT6mGF/OkHk7EKbq7VOJmZBIpmIzNgtyJ9K4= X-Alimail-AntiSpam: AC=PASS;BC=-1|-1;BR=01201311R421e4;CH=green;DM=||false|;DS=||;FP=0|-1|-1|-1|0|-1|-1|-1;HT=ay29a033018046056;MF=baolin.wang@linux.alibaba.com;NM=1;PH=DS;RN=13;SR=0;TI=SMTPD_---0W5-amM8_1713769381; Received: from localhost(mailfrom:baolin.wang@linux.alibaba.com fp:SMTPD_---0W5-amM8_1713769381) by smtp.aliyun-inc.com; Mon, 22 Apr 2024 15:03:02 +0800 From: Baolin Wang To: akpm@linux-foundation.org, hughd@google.com Cc: willy@infradead.org, david@redhat.com, wangkefeng.wang@huawei.com, 21cnbao@gmail.com, ryan.roberts@arm.com, ying.huang@intel.com, shy828301@gmail.com, ziy@nvidia.com, baolin.wang@linux.alibaba.com, linux-mm@kvack.org, linux-kernel@vger.kernel.org Subject: [RFC PATCH 5/5] mm: shmem: add anonymous share mTHP counters Date: Mon, 22 Apr 2024 15:02:43 +0800 Message-Id: <05d0096e4ec3e572d1d52d33a31a661321ac1551.1713755580.git.baolin.wang@linux.alibaba.com> X-Mailer: git-send-email 2.39.3 In-Reply-To: References: Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Signed-off-by: Baolin Wang --- include/linux/huge_mm.h | 2 ++ mm/huge_memory.c | 4 ++++ mm/shmem.c | 5 ++++- 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/include/linux/huge_mm.h b/include/linux/huge_mm.h index 26b6fa98d8ac..67b9c1acad31 100644 --- a/include/linux/huge_mm.h +++ b/include/linux/huge_mm.h @@ -270,6 +270,8 @@ enum mthp_stat_item { MTHP_STAT_ANON_SWPOUT, MTHP_STAT_ANON_SWPOUT_FALLBACK, MTHP_STAT_ANON_SWPIN_REFAULT, + MTHP_STAT_SHMEM_ANON_ALLOC, + MTHP_STAT_SHMEM_ANON_ALLOC_FALLBACK, __MTHP_STAT_COUNT }; =20 diff --git a/mm/huge_memory.c b/mm/huge_memory.c index 9e52c0db7580..dc15240c1ab3 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -557,6 +557,8 @@ DEFINE_MTHP_STAT_ATTR(anon_alloc_fallback, MTHP_STAT_AN= ON_ALLOC_FALLBACK); DEFINE_MTHP_STAT_ATTR(anon_swpout, MTHP_STAT_ANON_SWPOUT); DEFINE_MTHP_STAT_ATTR(anon_swpout_fallback, MTHP_STAT_ANON_SWPOUT_FALLBACK= ); DEFINE_MTHP_STAT_ATTR(anon_swpin_refault, MTHP_STAT_ANON_SWPIN_REFAULT); +DEFINE_MTHP_STAT_ATTR(shmem_anon_alloc, MTHP_STAT_SHMEM_ANON_ALLOC); +DEFINE_MTHP_STAT_ATTR(shmem_anon_alloc_fallback, MTHP_STAT_SHMEM_ANON_ALLO= C_FALLBACK); =20 static struct attribute *stats_attrs[] =3D { &anon_alloc_attr.attr, @@ -564,6 +566,8 @@ static struct attribute *stats_attrs[] =3D { &anon_swpout_attr.attr, &anon_swpout_fallback_attr.attr, &anon_swpin_refault_attr.attr, + &shmem_anon_alloc_attr.attr, + &shmem_anon_alloc_fallback_attr.attr, NULL, }; =20 diff --git a/mm/shmem.c b/mm/shmem.c index 8b009e7040b2..4a0aa75ab29c 100644 --- a/mm/shmem.c +++ b/mm/shmem.c @@ -1706,11 +1706,14 @@ static struct folio *shmem_alloc_and_add_folio(stru= ct vm_fault *vmf, pages =3D 1 << order; index =3D round_down(index, pages); folio =3D shmem_alloc_hugefolio(gfp, info, index, order); - if (folio) + if (folio) { + count_mthp_stat(order, MTHP_STAT_SHMEM_ANON_ALLOC); goto allocated; + } =20 if (pages =3D=3D HPAGE_PMD_NR) count_vm_event(THP_FILE_FALLBACK); + count_mthp_stat(order, MTHP_STAT_SHMEM_ANON_ALLOC_FALLBACK); order =3D next_order(&orders, order); } } else { --=20 2.39.3