From nobody Fri Nov 29 18:38:18 2024 Received: from foss.arm.com (foss.arm.com [217.140.110.172]) by smtp.subspace.kernel.org (Postfix) with ESMTP id E832B14F9DD; Tue, 17 Sep 2024 07:32:10 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=217.140.110.172 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1726558334; cv=none; b=NC4g9IU5672JI5A0hFiMeYjuZGzluYrwL5YZOr+M+4hgr/2sCQs/2+BeP+epGsxtil15eL99dDBqbxIhB3rTZx+6yUfo8CfM03VO68kOM7nooynyam4J26xMSrv8dfGvDy5J69z4HGfKV67wK4BiDsGeoNUz9mkqygg6aTXzhzE= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1726558334; c=relaxed/simple; bh=byzUaKMXKIiFFn04+lY2/QmKHmmTfjycNQK95YoobV0=; h=From:To:Cc:Subject:Date:Message-Id:In-Reply-To:References: MIME-Version; b=tGmXhrBdAENy/pWS4yWXszhq6VzPrNNIKuVFsR8QUfTOKzjPB+qjaV52Ofl6cWtJb5/Dt5m+9c8lte38j59P5KxFzJvhGAyEUC9xn17jhl0ZG5gPO++qFM0vEENtuoflILyTrbAlF1FxBhiOus6BX7a8eRXFkp16UD1pI6xEPzA= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com; spf=pass smtp.mailfrom=arm.com; arc=none smtp.client-ip=217.140.110.172 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=arm.com Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id DCE391063; Tue, 17 Sep 2024 00:32:39 -0700 (PDT) Received: from a077893.arm.com (unknown [10.163.61.158]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPA id C5E223F64C; Tue, 17 Sep 2024 00:32:02 -0700 (PDT) From: Anshuman Khandual To: linux-mm@kvack.org Cc: Anshuman Khandual , Andrew Morton , David Hildenbrand , Ryan Roberts , "Mike Rapoport (IBM)" , Arnd Bergmann , x86@kernel.org, linux-m68k@lists.linux-m68k.org, linux-fsdevel@vger.kernel.org, kasan-dev@googlegroups.com, linux-kernel@vger.kernel.org, linux-perf-users@vger.kernel.org, Dimitri Sivanich , Alexander Viro , Muchun Song , Andrey Ryabinin , Miaohe Lin , Dennis Zhou , Tejun Heo , Christoph Lameter , Uladzislau Rezki , Christoph Hellwig Subject: [PATCH V2 6/7] mm: Use p4dp_get() for accessing P4D entries Date: Tue, 17 Sep 2024 13:01:16 +0530 Message-Id: <20240917073117.1531207-7-anshuman.khandual@arm.com> X-Mailer: git-send-email 2.25.1 In-Reply-To: <20240917073117.1531207-1-anshuman.khandual@arm.com> References: <20240917073117.1531207-1-anshuman.khandual@arm.com> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Convert P4D accesses via p4dp_get() helper that defaults as READ_ONCE() but also provides the platform an opportunity to override when required. This stores read page table entry value in a local variable which can be used in multiple instances there after. This helps in avoiding multiple memory load operations as well possible race conditions. Cc: Dimitri Sivanich Cc: Alexander Viro Cc: Muchun Song Cc: Andrey Ryabinin Cc: Miaohe Lin Cc: Dennis Zhou Cc: Tejun Heo cc: Christoph Lameter Cc: Uladzislau Rezki Cc: Christoph Hellwig Cc: linux-kernel@vger.kernel.org Cc: linux-fsdevel@vger.kernel.org Cc: linux-perf-users@vger.kernel.org Cc: linux-mm@kvack.org Cc: kasan-dev@googlegroups.com Signed-off-by: Anshuman Khandual --- drivers/misc/sgi-gru/grufault.c | 2 +- fs/userfaultfd.c | 2 +- include/linux/pgtable.h | 9 ++++++--- kernel/events/core.c | 2 +- mm/gup.c | 6 +++--- mm/hugetlb.c | 2 +- mm/kasan/init.c | 10 +++++----- mm/kasan/shadow.c | 2 +- mm/memory-failure.c | 2 +- mm/memory.c | 16 +++++++++------- mm/page_vma_mapped.c | 2 +- mm/percpu.c | 2 +- mm/pgalloc-track.h | 2 +- mm/pgtable-generic.c | 2 +- mm/ptdump.c | 2 +- mm/rmap.c | 2 +- mm/sparse-vmemmap.c | 2 +- mm/vmalloc.c | 15 ++++++++------- mm/vmscan.c | 2 +- 19 files changed, 45 insertions(+), 39 deletions(-) diff --git a/drivers/misc/sgi-gru/grufault.c b/drivers/misc/sgi-gru/grufaul= t.c index 95d479d5e40f..fcaceac60659 100644 --- a/drivers/misc/sgi-gru/grufault.c +++ b/drivers/misc/sgi-gru/grufault.c @@ -216,7 +216,7 @@ static int atomic_pte_lookup(struct vm_area_struct *vma= , unsigned long vaddr, goto err; =20 p4dp =3D p4d_offset(pgdp, vaddr); - if (unlikely(p4d_none(*p4dp))) + if (unlikely(p4d_none(p4dp_get(p4dp)))) goto err; =20 pudp =3D pud_offset(p4dp, vaddr); diff --git a/fs/userfaultfd.c b/fs/userfaultfd.c index 00719a0f688c..4044e15cdfd9 100644 --- a/fs/userfaultfd.c +++ b/fs/userfaultfd.c @@ -307,7 +307,7 @@ static inline bool userfaultfd_must_wait(struct userfau= ltfd_ctx *ctx, if (!pgd_present(*pgd)) goto out; p4d =3D p4d_offset(pgd, address); - if (!p4d_present(*p4d)) + if (!p4d_present(p4dp_get(p4d))) goto out; pud =3D pud_offset(p4d, address); if (!pud_present(pudp_get(pud))) diff --git a/include/linux/pgtable.h b/include/linux/pgtable.h index eb993ef0946f..689cd5a32157 100644 --- a/include/linux/pgtable.h +++ b/include/linux/pgtable.h @@ -1081,7 +1081,8 @@ static inline int pgd_same(pgd_t pgd_a, pgd_t pgd_b) =20 #define set_p4d_safe(p4dp, p4d) \ ({ \ - WARN_ON_ONCE(p4d_present(*p4dp) && !p4d_same(*p4dp, p4d)); \ + p4d_t __old =3D p4dp_get(p4dp); \ + WARN_ON_ONCE(p4d_present(__old) && !p4d_same(__old, p4d)); \ set_p4d(p4dp, p4d); \ }) =20 @@ -1251,9 +1252,11 @@ static inline int pgd_none_or_clear_bad(pgd_t *pgd) =20 static inline int p4d_none_or_clear_bad(p4d_t *p4d) { - if (p4d_none(*p4d)) + p4d_t old_p4d =3D p4dp_get(p4d); + + if (p4d_none(old_p4d)) return 1; - if (unlikely(p4d_bad(*p4d))) { + if (unlikely(p4d_bad(old_p4d))) { p4d_clear_bad(p4d); return 1; } diff --git a/kernel/events/core.c b/kernel/events/core.c index 35e2f2789246..4e56a276ed25 100644 --- a/kernel/events/core.c +++ b/kernel/events/core.c @@ -7611,7 +7611,7 @@ static u64 perf_get_pgtable_size(struct mm_struct *mm= , unsigned long addr) return pgd_leaf_size(pgd); =20 p4dp =3D p4d_offset_lockless(pgdp, pgd, addr); - p4d =3D READ_ONCE(*p4dp); + p4d =3D p4dp_get(p4dp); if (!p4d_present(p4d)) return 0; =20 diff --git a/mm/gup.c b/mm/gup.c index 300fc7eb306c..3a97d0263052 100644 --- a/mm/gup.c +++ b/mm/gup.c @@ -1014,7 +1014,7 @@ static struct page *follow_p4d_mask(struct vm_area_st= ruct *vma, p4d_t *p4dp, p4d; =20 p4dp =3D p4d_offset(pgdp, address); - p4d =3D READ_ONCE(*p4dp); + p4d =3D p4dp_get(p4dp); BUILD_BUG_ON(p4d_leaf(p4d)); =20 if (!p4d_present(p4d) || p4d_bad(p4d)) @@ -1114,7 +1114,7 @@ static int get_gate_page(struct mm_struct *mm, unsign= ed long address, if (pgd_none(*pgd)) return -EFAULT; p4d =3D p4d_offset(pgd, address); - if (p4d_none(*p4d)) + if (p4d_none(p4dp_get(p4d))) return -EFAULT; pud =3D pud_offset(p4d, address); if (pud_none(pudp_get(pud))) @@ -3245,7 +3245,7 @@ static int gup_fast_p4d_range(pgd_t *pgdp, pgd_t pgd,= unsigned long addr, =20 p4dp =3D p4d_offset_lockless(pgdp, pgd, addr); do { - p4d_t p4d =3D READ_ONCE(*p4dp); + p4d_t p4d =3D p4dp_get(p4dp); =20 next =3D p4d_addr_end(addr, end); if (!p4d_present(p4d)) diff --git a/mm/hugetlb.c b/mm/hugetlb.c index a3820242b01e..4fdb91c8cc2b 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -7454,7 +7454,7 @@ pte_t *huge_pte_offset(struct mm_struct *mm, if (!pgd_present(*pgd)) return NULL; p4d =3D p4d_offset(pgd, addr); - if (!p4d_present(*p4d)) + if (!p4d_present(p4dp_get(p4d))) return NULL; =20 pud =3D pud_offset(p4d, addr); diff --git a/mm/kasan/init.c b/mm/kasan/init.c index f4cf519443e1..02af738fee5e 100644 --- a/mm/kasan/init.c +++ b/mm/kasan/init.c @@ -208,7 +208,7 @@ static int __ref zero_p4d_populate(pgd_t *pgd, unsigned= long addr, continue; } =20 - if (p4d_none(*p4d)) { + if (p4d_none(p4dp_get(p4d))) { pud_t *p; =20 if (slab_is_available()) { @@ -330,7 +330,7 @@ static void kasan_free_pud(pud_t *pud_start, p4d_t *p4d) return; } =20 - pud_free(&init_mm, (pud_t *)page_to_virt(p4d_page(*p4d))); + pud_free(&init_mm, (pud_t *)page_to_virt(p4d_page(p4dp_get(p4d)))); p4d_clear(p4d); } =20 @@ -341,7 +341,7 @@ static void kasan_free_p4d(p4d_t *p4d_start, pgd_t *pgd) =20 for (i =3D 0; i < PTRS_PER_P4D; i++) { p4d =3D p4d_start + i; - if (!p4d_none(*p4d)) + if (!p4d_none(p4dp_get(p4d))) return; } =20 @@ -434,10 +434,10 @@ static void kasan_remove_p4d_table(p4d_t *p4d, unsign= ed long addr, =20 next =3D p4d_addr_end(addr, end); =20 - if (!p4d_present(*p4d)) + if (!p4d_present(p4dp_get(p4d))) continue; =20 - if (kasan_pud_table(*p4d)) { + if (kasan_pud_table(p4dp_get(p4d))) { if (IS_ALIGNED(addr, P4D_SIZE) && IS_ALIGNED(next, P4D_SIZE)) { p4d_clear(p4d); diff --git a/mm/kasan/shadow.c b/mm/kasan/shadow.c index dbd8164c75f1..52150cc5ae5f 100644 --- a/mm/kasan/shadow.c +++ b/mm/kasan/shadow.c @@ -194,7 +194,7 @@ static bool shadow_mapped(unsigned long addr) if (pgd_none(*pgd)) return false; p4d =3D p4d_offset(pgd, addr); - if (p4d_none(*p4d)) + if (p4d_none(p4dp_get(p4d))) return false; pud =3D pud_offset(p4d, addr); if (pud_none(pudp_get(pud))) diff --git a/mm/memory-failure.c b/mm/memory-failure.c index fbb63401fb51..3d900cc039b3 100644 --- a/mm/memory-failure.c +++ b/mm/memory-failure.c @@ -414,7 +414,7 @@ static unsigned long dev_pagemap_mapping_shift(struct v= m_area_struct *vma, if (!pgd_present(*pgd)) return 0; p4d =3D p4d_offset(pgd, address); - if (!p4d_present(*p4d)) + if (!p4d_present(p4dp_get(p4d))) return 0; pud =3D pud_offset(p4d, address); if (!pud_present(pudp_get(pud))) diff --git a/mm/memory.c b/mm/memory.c index 801750e4337c..5056f39f2c3b 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -2906,7 +2906,7 @@ static int apply_to_p4d_range(struct mm_struct *mm, p= gd_t *pgd, pte_fn_t fn, void *data, bool create, pgtbl_mod_mask *mask) { - p4d_t *p4d; + p4d_t *p4d, old_p4d; unsigned long next; int err =3D 0; =20 @@ -2919,11 +2919,12 @@ static int apply_to_p4d_range(struct mm_struct *mm,= pgd_t *pgd, } do { next =3D p4d_addr_end(addr, end); - if (p4d_none(*p4d) && !create) + old_p4d =3D p4dp_get(p4d); + if (p4d_none(old_p4d) && !create) continue; - if (WARN_ON_ONCE(p4d_leaf(*p4d))) + if (WARN_ON_ONCE(p4d_leaf(old_p4d))) return -EINVAL; - if (!p4d_none(*p4d) && WARN_ON_ONCE(p4d_bad(*p4d))) { + if (!p4d_none(old_p4d) && WARN_ON_ONCE(p4d_bad(old_p4d))) { if (!create) continue; p4d_clear_bad(p4d); @@ -6075,7 +6076,7 @@ int __pud_alloc(struct mm_struct *mm, p4d_t *p4d, uns= igned long address) return -ENOMEM; =20 spin_lock(&mm->page_table_lock); - if (!p4d_present(*p4d)) { + if (!p4d_present(p4dp_get(p4d))) { mm_inc_nr_puds(mm); smp_wmb(); /* See comment in pmd_install() */ p4d_populate(mm, p4d, new); @@ -6143,7 +6144,7 @@ int follow_pte(struct vm_area_struct *vma, unsigned l= ong address, { struct mm_struct *mm =3D vma->vm_mm; pgd_t *pgd; - p4d_t *p4d; + p4d_t *p4d, old_p4d; pud_t *pud; pmd_t *pmd; pte_t *ptep; @@ -6160,7 +6161,8 @@ int follow_pte(struct vm_area_struct *vma, unsigned l= ong address, goto out; =20 p4d =3D p4d_offset(pgd, address); - if (p4d_none(*p4d) || unlikely(p4d_bad(*p4d))) + old_p4d =3D p4dp_get(p4d); + if (p4d_none(old_p4d) || unlikely(p4d_bad(old_p4d))) goto out; =20 pud =3D pud_offset(p4d, address); diff --git a/mm/page_vma_mapped.c b/mm/page_vma_mapped.c index 511266307771..a33f92db2666 100644 --- a/mm/page_vma_mapped.c +++ b/mm/page_vma_mapped.c @@ -217,7 +217,7 @@ bool page_vma_mapped_walk(struct page_vma_mapped_walk *= pvmw) continue; } p4d =3D p4d_offset(pgd, pvmw->address); - if (!p4d_present(*p4d)) { + if (!p4d_present(p4dp_get(p4d))) { step_forward(pvmw, P4D_SIZE); continue; } diff --git a/mm/percpu.c b/mm/percpu.c index 5f32164b04a2..58660e8eb892 100644 --- a/mm/percpu.c +++ b/mm/percpu.c @@ -3192,7 +3192,7 @@ void __init __weak pcpu_populate_pte(unsigned long ad= dr) } =20 p4d =3D p4d_offset(pgd, addr); - if (p4d_none(*p4d)) { + if (p4d_none(p4dp_get(p4d))) { pud =3D memblock_alloc(PUD_TABLE_SIZE, PUD_TABLE_SIZE); if (!pud) goto err_alloc; diff --git a/mm/pgalloc-track.h b/mm/pgalloc-track.h index 0f6b809431a3..3db8ccbcb141 100644 --- a/mm/pgalloc-track.h +++ b/mm/pgalloc-track.h @@ -20,7 +20,7 @@ static inline pud_t *pud_alloc_track(struct mm_struct *mm= , p4d_t *p4d, unsigned long address, pgtbl_mod_mask *mod_mask) { - if (unlikely(p4d_none(*p4d))) { + if (unlikely(p4d_none(p4dp_get(p4d)))) { if (__pud_alloc(mm, p4d, address)) return NULL; *mod_mask |=3D PGTBL_P4D_MODIFIED; diff --git a/mm/pgtable-generic.c b/mm/pgtable-generic.c index e09e3f920f7a..f5ab52beb536 100644 --- a/mm/pgtable-generic.c +++ b/mm/pgtable-generic.c @@ -31,7 +31,7 @@ void pgd_clear_bad(pgd_t *pgd) #ifndef __PAGETABLE_P4D_FOLDED void p4d_clear_bad(p4d_t *p4d) { - p4d_ERROR(*p4d); + p4d_ERROR(p4dp_get(p4d)); p4d_clear(p4d); } #endif diff --git a/mm/ptdump.c b/mm/ptdump.c index 32ae8e829329..2c40224b8ad0 100644 --- a/mm/ptdump.c +++ b/mm/ptdump.c @@ -53,7 +53,7 @@ static int ptdump_p4d_entry(p4d_t *p4d, unsigned long add= r, unsigned long next, struct mm_walk *walk) { struct ptdump_state *st =3D walk->private; - p4d_t val =3D READ_ONCE(*p4d); + p4d_t val =3D p4dp_get(p4d); =20 #if CONFIG_PGTABLE_LEVELS > 3 && \ (defined(CONFIG_KASAN_GENERIC) || defined(CONFIG_KASAN_SW_TAGS)) diff --git a/mm/rmap.c b/mm/rmap.c index 81f1946653e0..a0ff325467eb 100644 --- a/mm/rmap.c +++ b/mm/rmap.c @@ -813,7 +813,7 @@ pmd_t *mm_find_pmd(struct mm_struct *mm, unsigned long = address) goto out; =20 p4d =3D p4d_offset(pgd, address); - if (!p4d_present(*p4d)) + if (!p4d_present(p4dp_get(p4d))) goto out; =20 pud =3D pud_offset(p4d, address); diff --git a/mm/sparse-vmemmap.c b/mm/sparse-vmemmap.c index d8ea64ec665f..2bd1c95f107a 100644 --- a/mm/sparse-vmemmap.c +++ b/mm/sparse-vmemmap.c @@ -220,7 +220,7 @@ void __weak __meminit pud_init(void *addr) p4d_t * __meminit vmemmap_p4d_populate(pgd_t *pgd, unsigned long addr, int= node) { p4d_t *p4d =3D p4d_offset(pgd, addr); - if (p4d_none(*p4d)) { + if (p4d_none(p4dp_get(p4d))) { void *p =3D vmemmap_alloc_block_zero(PAGE_SIZE, node); if (!p) return NULL; diff --git a/mm/vmalloc.c b/mm/vmalloc.c index 05292d998122..f27ecac7bd6e 100644 --- a/mm/vmalloc.c +++ b/mm/vmalloc.c @@ -251,7 +251,7 @@ static int vmap_try_huge_p4d(p4d_t *p4d, unsigned long = addr, unsigned long end, if (!IS_ALIGNED(phys_addr, P4D_SIZE)) return 0; =20 - if (p4d_present(*p4d) && !p4d_free_pud_page(p4d, addr)) + if (p4d_present(p4dp_get(p4d)) && !p4d_free_pud_page(p4d, addr)) return 0; =20 return p4d_set_huge(p4d, phys_addr, prot); @@ -418,7 +418,7 @@ static void vunmap_p4d_range(pgd_t *pgd, unsigned long = addr, unsigned long end, next =3D p4d_addr_end(addr, end); =20 p4d_clear_huge(p4d); - if (p4d_bad(*p4d)) + if (p4d_bad(p4dp_get(p4d))) *mask |=3D PGTBL_P4D_MODIFIED; =20 if (p4d_none_or_clear_bad(p4d)) @@ -741,7 +741,7 @@ struct page *vmalloc_to_page(const void *vmalloc_addr) unsigned long addr =3D (unsigned long) vmalloc_addr; struct page *page =3D NULL; pgd_t *pgd =3D pgd_offset_k(addr); - p4d_t *p4d; + p4d_t *p4d, old_p4d; pud_t *pud, old_pud; pmd_t *pmd, old_pmd; pte_t *ptep, pte; @@ -760,11 +760,12 @@ struct page *vmalloc_to_page(const void *vmalloc_addr) return NULL; =20 p4d =3D p4d_offset(pgd, addr); - if (p4d_none(*p4d)) + old_p4d =3D p4dp_get(p4d); + if (p4d_none(old_p4d)) return NULL; - if (p4d_leaf(*p4d)) - return p4d_page(*p4d) + ((addr & ~P4D_MASK) >> PAGE_SHIFT); - if (WARN_ON_ONCE(p4d_bad(*p4d))) + if (p4d_leaf(old_p4d)) + return p4d_page(old_p4d) + ((addr & ~P4D_MASK) >> PAGE_SHIFT); + if (WARN_ON_ONCE(p4d_bad(old_p4d))) return NULL; =20 pud =3D pud_offset(p4d, addr); diff --git a/mm/vmscan.c b/mm/vmscan.c index 04b03e6c3095..b16925b5f072 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -3579,7 +3579,7 @@ static int walk_pud_range(p4d_t *p4d, unsigned long s= tart, unsigned long end, unsigned long next; struct lru_gen_mm_walk *walk =3D args->private; =20 - VM_WARN_ON_ONCE(p4d_leaf(*p4d)); + VM_WARN_ON_ONCE(p4d_leaf(p4dp_get(p4d))); =20 pud =3D pud_offset(p4d, start & P4D_MASK); restart: --=20 2.25.1