From nobody Sun Dec 14 06:20:07 2025 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-1.web.codeaurora.org [10.30.226.201]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id D68F625F7A2; Mon, 28 Apr 2025 09:23:44 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=10.30.226.201 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1745832225; cv=none; b=GVZhNgZeWk40Cm7PdZKeM6nPQNvOBseSakwTSsem8zUwGjpQEiuvh4Da/JVxkntityI5tALqHJk4HlxCvfhhumP6oXRlRadC45ecXpgWKnlu4UKgofO7QgpMLnPTJXG8ho1BMzBoaNHBQsrz+yuN6ztTZA64vpCN/wi5LzRDuko= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1745832225; c=relaxed/simple; bh=BHrpmZ7+n44Wwug7Tkul5xfu4MbMbr93E8OVig3WwUA=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=E1fkCcChrN4CI98/IlY4YHvUloHWQca2JEmla7m6Q0Jv5u0cK2S9RT4Wgn8Ih6KpicGFOnfDPmBFc0N96cm4CiDepDgIgs9aBZhXHwJwN/GIUuXt+lIdyetQ/eenlZZ7Tyf4V9R51r6vlhFcAN3pAacAPn/XIimDsApWjNKwCLE= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=GPO21Dmt; arc=none smtp.client-ip=10.30.226.201 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="GPO21Dmt" Received: by smtp.kernel.org (Postfix) with ESMTPSA id C9187C4CEE4; Mon, 28 Apr 2025 09:23:43 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1745832224; bh=BHrpmZ7+n44Wwug7Tkul5xfu4MbMbr93E8OVig3WwUA=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=GPO21DmtP5aut8YN9Fmpy1EJ5T2SwH0t6xJG37Yv/F+KpLF8RADDddn7mc1YJFzl6 C0785Hq24L/3abOG5FAtwzoMRIbZPhplz6jlqIrPK2Y6PLUuh7GO4ONn3kIBLilWv9 ShacrRkR35u/zf4X8cFKM2Psiy9M5qiFT6c+0ZAuLe9wUDndhx30X1li1TyLMJY2Kw MO9PokUSk437Oet6tu8ESsECMl+qlZgUY7QZW9r2LzVgCkPnQKdjH9pofDx5iSVJcH h6kxN6S/dmXhlLtYFsOJA/2TFHeu6MyK5maSnDmZ8DLakQZv9WAqmr09dIhpa5tPiJ hG+xi/1GGtYxw== From: Leon Romanovsky To: Marek Szyprowski , Jens Axboe , Christoph Hellwig , Keith Busch Cc: Leon Romanovsky , Jake Edge , Jonathan Corbet , Jason Gunthorpe , Zhu Yanjun , Robin Murphy , Joerg Roedel , Will Deacon , Sagi Grimberg , Bjorn Helgaas , Logan Gunthorpe , Yishai Hadas , Shameer Kolothum , Kevin Tian , Alex Williamson , =?UTF-8?q?J=C3=A9r=C3=B4me=20Glisse?= , Andrew Morton , linux-doc@vger.kernel.org, linux-kernel@vger.kernel.org, linux-block@vger.kernel.org, linux-rdma@vger.kernel.org, iommu@lists.linux.dev, linux-nvme@lists.infradead.org, linux-pci@vger.kernel.org, kvm@vger.kernel.org, linux-mm@kvack.org, Niklas Schnelle , Chuck Lever , Luis Chamberlain , Matthew Wilcox , Dan Williams , Kanchan Joshi , Chaitanya Kulkarni , Jason Gunthorpe Subject: [PATCH v10 12/24] RDMA/umem: Store ODP access mask information in PFN Date: Mon, 28 Apr 2025 12:22:18 +0300 Message-ID: <96f840717ffaae3dff1063cb2dcb249365159e72.1745831017.git.leon@kernel.org> X-Mailer: git-send-email 2.49.0 In-Reply-To: References: Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" From: Leon Romanovsky As a preparation to remove dma_list, store access mask in PFN pointer and not in dma_addr_t. Tested-by: Jens Axboe Reviewed-by: Jason Gunthorpe Signed-off-by: Leon Romanovsky --- drivers/infiniband/core/umem_odp.c | 103 +++++++++++---------------- drivers/infiniband/hw/mlx5/mlx5_ib.h | 1 + drivers/infiniband/hw/mlx5/odp.c | 37 +++++----- drivers/infiniband/sw/rxe/rxe_odp.c | 14 ++-- include/rdma/ib_umem_odp.h | 14 +--- 5 files changed, 70 insertions(+), 99 deletions(-) diff --git a/drivers/infiniband/core/umem_odp.c b/drivers/infiniband/core/u= mem_odp.c index c48ef6083020..2a061f377eb1 100644 --- a/drivers/infiniband/core/umem_odp.c +++ b/drivers/infiniband/core/umem_odp.c @@ -298,22 +298,11 @@ EXPORT_SYMBOL(ib_umem_odp_release); static int ib_umem_odp_map_dma_single_page( struct ib_umem_odp *umem_odp, unsigned int dma_index, - struct page *page, - u64 access_mask) + struct page *page) { struct ib_device *dev =3D umem_odp->umem.ibdev; dma_addr_t *dma_addr =3D &umem_odp->dma_list[dma_index]; =20 - if (*dma_addr) { - /* - * If the page is already dma mapped it means it went through - * a non-invalidating trasition, like read-only to writable. - * Resync the flags. - */ - *dma_addr =3D (*dma_addr & ODP_DMA_ADDR_MASK) | access_mask; - return 0; - } - *dma_addr =3D ib_dma_map_page(dev, page, 0, 1 << umem_odp->page_shift, DMA_BIDIRECTIONAL); if (ib_dma_mapping_error(dev, *dma_addr)) { @@ -321,7 +310,6 @@ static int ib_umem_odp_map_dma_single_page( return -EFAULT; } umem_odp->npages++; - *dma_addr |=3D access_mask; return 0; } =20 @@ -357,9 +345,6 @@ int ib_umem_odp_map_dma_and_lock(struct ib_umem_odp *um= em_odp, u64 user_virt, struct hmm_range range =3D {}; unsigned long timeout; =20 - if (access_mask =3D=3D 0) - return -EINVAL; - if (user_virt < ib_umem_start(umem_odp) || user_virt + bcnt > ib_umem_end(umem_odp)) return -EFAULT; @@ -385,7 +370,7 @@ int ib_umem_odp_map_dma_and_lock(struct ib_umem_odp *um= em_odp, u64 user_virt, if (fault) { range.default_flags =3D HMM_PFN_REQ_FAULT; =20 - if (access_mask & ODP_WRITE_ALLOWED_BIT) + if (access_mask & HMM_PFN_WRITE) range.default_flags |=3D HMM_PFN_REQ_WRITE; } =20 @@ -417,22 +402,17 @@ int ib_umem_odp_map_dma_and_lock(struct ib_umem_odp *= umem_odp, u64 user_virt, for (pfn_index =3D 0; pfn_index < num_pfns; pfn_index +=3D 1 << (page_shift - PAGE_SHIFT), dma_index++) { =20 - if (fault) { - /* - * Since we asked for hmm_range_fault() to populate - * pages it shouldn't return an error entry on success. - */ - WARN_ON(range.hmm_pfns[pfn_index] & HMM_PFN_ERROR); - WARN_ON(!(range.hmm_pfns[pfn_index] & HMM_PFN_VALID)); - } else { - if (!(range.hmm_pfns[pfn_index] & HMM_PFN_VALID)) { - WARN_ON(umem_odp->dma_list[dma_index]); - continue; - } - access_mask =3D ODP_READ_ALLOWED_BIT; - if (range.hmm_pfns[pfn_index] & HMM_PFN_WRITE) - access_mask |=3D ODP_WRITE_ALLOWED_BIT; - } + /* + * Since we asked for hmm_range_fault() to populate + * pages it shouldn't return an error entry on success. + */ + WARN_ON(fault && range.hmm_pfns[pfn_index] & HMM_PFN_ERROR); + WARN_ON(fault && !(range.hmm_pfns[pfn_index] & HMM_PFN_VALID)); + if (!(range.hmm_pfns[pfn_index] & HMM_PFN_VALID)) + continue; + + if (range.hmm_pfns[pfn_index] & HMM_PFN_DMA_MAPPED) + continue; =20 hmm_order =3D hmm_pfn_to_map_order(range.hmm_pfns[pfn_index]); /* If a hugepage was detected and ODP wasn't set for, the umem @@ -447,13 +427,14 @@ int ib_umem_odp_map_dma_and_lock(struct ib_umem_odp *= umem_odp, u64 user_virt, } =20 ret =3D ib_umem_odp_map_dma_single_page( - umem_odp, dma_index, hmm_pfn_to_page(range.hmm_pfns[pfn_index]), - access_mask); + umem_odp, dma_index, + hmm_pfn_to_page(range.hmm_pfns[pfn_index])); if (ret < 0) { ibdev_dbg(umem_odp->umem.ibdev, "ib_umem_odp_map_dma_single_page failed with error %d\n", ret); break; } + range.hmm_pfns[pfn_index] |=3D HMM_PFN_DMA_MAPPED; } /* upon success lock should stay on hold for the callee */ if (!ret) @@ -473,7 +454,6 @@ EXPORT_SYMBOL(ib_umem_odp_map_dma_and_lock); void ib_umem_odp_unmap_dma_pages(struct ib_umem_odp *umem_odp, u64 virt, u64 bound) { - dma_addr_t dma_addr; dma_addr_t dma; int idx; u64 addr; @@ -484,34 +464,37 @@ void ib_umem_odp_unmap_dma_pages(struct ib_umem_odp *= umem_odp, u64 virt, virt =3D max_t(u64, virt, ib_umem_start(umem_odp)); bound =3D min_t(u64, bound, ib_umem_end(umem_odp)); for (addr =3D virt; addr < bound; addr +=3D BIT(umem_odp->page_shift)) { + unsigned long pfn_idx =3D (addr - ib_umem_start(umem_odp)) >> + PAGE_SHIFT; + struct page *page =3D + hmm_pfn_to_page(umem_odp->pfn_list[pfn_idx]); + idx =3D (addr - ib_umem_start(umem_odp)) >> umem_odp->page_shift; dma =3D umem_odp->dma_list[idx]; =20 - /* The access flags guaranteed a valid DMA address in case was NULL */ - if (dma) { - unsigned long pfn_idx =3D (addr - ib_umem_start(umem_odp)) >> PAGE_SHIF= T; - struct page *page =3D hmm_pfn_to_page(umem_odp->pfn_list[pfn_idx]); - - dma_addr =3D dma & ODP_DMA_ADDR_MASK; - ib_dma_unmap_page(dev, dma_addr, - BIT(umem_odp->page_shift), - DMA_BIDIRECTIONAL); - if (dma & ODP_WRITE_ALLOWED_BIT) { - struct page *head_page =3D compound_head(page); - /* - * set_page_dirty prefers being called with - * the page lock. However, MMU notifiers are - * called sometimes with and sometimes without - * the lock. We rely on the umem_mutex instead - * to prevent other mmu notifiers from - * continuing and allowing the page mapping to - * be removed. - */ - set_page_dirty(head_page); - } - umem_odp->dma_list[idx] =3D 0; - umem_odp->npages--; + if (!(umem_odp->pfn_list[pfn_idx] & HMM_PFN_VALID)) + goto clear; + if (!(umem_odp->pfn_list[pfn_idx] & HMM_PFN_DMA_MAPPED)) + goto clear; + + ib_dma_unmap_page(dev, dma, BIT(umem_odp->page_shift), + DMA_BIDIRECTIONAL); + if (umem_odp->pfn_list[pfn_idx] & HMM_PFN_WRITE) { + struct page *head_page =3D compound_head(page); + /* + * set_page_dirty prefers being called with + * the page lock. However, MMU notifiers are + * called sometimes with and sometimes without + * the lock. We rely on the umem_mutex instead + * to prevent other mmu notifiers from + * continuing and allowing the page mapping to + * be removed. + */ + set_page_dirty(head_page); } + umem_odp->npages--; +clear: + umem_odp->pfn_list[pfn_idx] &=3D ~HMM_PFN_FLAGS; } } EXPORT_SYMBOL(ib_umem_odp_unmap_dma_pages); diff --git a/drivers/infiniband/hw/mlx5/mlx5_ib.h b/drivers/infiniband/hw/m= lx5/mlx5_ib.h index ace2df3e1d9f..7424b23bb0d9 100644 --- a/drivers/infiniband/hw/mlx5/mlx5_ib.h +++ b/drivers/infiniband/hw/mlx5/mlx5_ib.h @@ -351,6 +351,7 @@ struct mlx5_ib_flow_db { #define MLX5_IB_UPD_XLT_PD BIT(4) #define MLX5_IB_UPD_XLT_ACCESS BIT(5) #define MLX5_IB_UPD_XLT_INDIRECT BIT(6) +#define MLX5_IB_UPD_XLT_DOWNGRADE BIT(7) =20 /* Private QP creation flags to be passed in ib_qp_init_attr.create_flags. * diff --git a/drivers/infiniband/hw/mlx5/odp.c b/drivers/infiniband/hw/mlx5/= odp.c index 86d8fa63bf69..6074c541885c 100644 --- a/drivers/infiniband/hw/mlx5/odp.c +++ b/drivers/infiniband/hw/mlx5/odp.c @@ -34,6 +34,7 @@ #include #include #include +#include =20 #include "mlx5_ib.h" #include "cmd.h" @@ -158,22 +159,12 @@ static void populate_klm(struct mlx5_klm *pklm, size_= t idx, size_t nentries, } } =20 -static u64 umem_dma_to_mtt(dma_addr_t umem_dma) -{ - u64 mtt_entry =3D umem_dma & ODP_DMA_ADDR_MASK; - - if (umem_dma & ODP_READ_ALLOWED_BIT) - mtt_entry |=3D MLX5_IB_MTT_READ; - if (umem_dma & ODP_WRITE_ALLOWED_BIT) - mtt_entry |=3D MLX5_IB_MTT_WRITE; - - return mtt_entry; -} - static void populate_mtt(__be64 *pas, size_t idx, size_t nentries, struct mlx5_ib_mr *mr, int flags) { struct ib_umem_odp *odp =3D to_ib_umem_odp(mr->umem); + bool downgrade =3D flags & MLX5_IB_UPD_XLT_DOWNGRADE; + unsigned long pfn; dma_addr_t pa; size_t i; =20 @@ -181,8 +172,17 @@ static void populate_mtt(__be64 *pas, size_t idx, size= _t nentries, return; =20 for (i =3D 0; i < nentries; i++) { + pfn =3D odp->pfn_list[idx + i]; + if (!(pfn & HMM_PFN_VALID)) + /* ODP initialization */ + continue; + pa =3D odp->dma_list[idx + i]; - pas[i] =3D cpu_to_be64(umem_dma_to_mtt(pa)); + pa |=3D MLX5_IB_MTT_READ; + if ((pfn & HMM_PFN_WRITE) && !downgrade) + pa |=3D MLX5_IB_MTT_WRITE; + + pas[i] =3D cpu_to_be64(pa); } } =20 @@ -303,8 +303,7 @@ static bool mlx5_ib_invalidate_range(struct mmu_interva= l_notifier *mni, * estimate the cost of another UMR vs. the cost of bigger * UMR. */ - if (umem_odp->dma_list[idx] & - (ODP_READ_ALLOWED_BIT | ODP_WRITE_ALLOWED_BIT)) { + if (umem_odp->pfn_list[idx] & HMM_PFN_VALID) { if (!in_block) { blk_start_idx =3D idx; in_block =3D 1; @@ -687,7 +686,7 @@ static int pagefault_real_mr(struct mlx5_ib_mr *mr, str= uct ib_umem_odp *odp, { int page_shift, ret, np; bool downgrade =3D flags & MLX5_PF_FLAGS_DOWNGRADE; - u64 access_mask; + u64 access_mask =3D 0; u64 start_idx; bool fault =3D !(flags & MLX5_PF_FLAGS_SNAPSHOT); u32 xlt_flags =3D MLX5_IB_UPD_XLT_ATOMIC; @@ -695,12 +694,14 @@ static int pagefault_real_mr(struct mlx5_ib_mr *mr, s= truct ib_umem_odp *odp, if (flags & MLX5_PF_FLAGS_ENABLE) xlt_flags |=3D MLX5_IB_UPD_XLT_ENABLE; =20 + if (flags & MLX5_PF_FLAGS_DOWNGRADE) + xlt_flags |=3D MLX5_IB_UPD_XLT_DOWNGRADE; + page_shift =3D odp->page_shift; start_idx =3D (user_va - ib_umem_start(odp)) >> page_shift; - access_mask =3D ODP_READ_ALLOWED_BIT; =20 if (odp->umem.writable && !downgrade) - access_mask |=3D ODP_WRITE_ALLOWED_BIT; + access_mask |=3D HMM_PFN_WRITE; =20 np =3D ib_umem_odp_map_dma_and_lock(odp, user_va, bcnt, access_mask, faul= t); if (np < 0) diff --git a/drivers/infiniband/sw/rxe/rxe_odp.c b/drivers/infiniband/sw/rx= e/rxe_odp.c index 9f6e2bb2a269..caedd9ef2fe3 100644 --- a/drivers/infiniband/sw/rxe/rxe_odp.c +++ b/drivers/infiniband/sw/rxe/rxe_odp.c @@ -26,7 +26,7 @@ static bool rxe_ib_invalidate_range(struct mmu_interval_n= otifier *mni, start =3D max_t(u64, ib_umem_start(umem_odp), range->start); end =3D min_t(u64, ib_umem_end(umem_odp), range->end); =20 - /* update umem_odp->dma_list */ + /* update umem_odp->map.pfn_list */ ib_umem_odp_unmap_dma_pages(umem_odp, start, end); =20 mutex_unlock(&umem_odp->umem_mutex); @@ -44,12 +44,11 @@ static int rxe_odp_do_pagefault_and_lock(struct rxe_mr = *mr, u64 user_va, int bcn { struct ib_umem_odp *umem_odp =3D to_ib_umem_odp(mr->umem); bool fault =3D !(flags & RXE_PAGEFAULT_SNAPSHOT); - u64 access_mask; + u64 access_mask =3D 0; int np; =20 - access_mask =3D ODP_READ_ALLOWED_BIT; if (umem_odp->umem.writable && !(flags & RXE_PAGEFAULT_RDONLY)) - access_mask |=3D ODP_WRITE_ALLOWED_BIT; + access_mask |=3D HMM_PFN_WRITE; =20 /* * ib_umem_odp_map_dma_and_lock() locks umem_mutex on success. @@ -137,7 +136,7 @@ static inline bool rxe_check_pagefault(struct ib_umem_o= dp *umem_odp, while (addr < iova + length) { idx =3D (addr - ib_umem_start(umem_odp)) >> umem_odp->page_shift; =20 - if (!(umem_odp->dma_list[idx] & perm)) { + if (!(umem_odp->map.pfn_list[idx] & perm)) { need_fault =3D true; break; } @@ -151,15 +150,14 @@ static int rxe_odp_map_range_and_lock(struct rxe_mr *= mr, u64 iova, int length, u { struct ib_umem_odp *umem_odp =3D to_ib_umem_odp(mr->umem); bool need_fault; - u64 perm; + u64 perm =3D 0; int err; =20 if (unlikely(length < 1)) return -EINVAL; =20 - perm =3D ODP_READ_ALLOWED_BIT; if (!(flags & RXE_PAGEFAULT_RDONLY)) - perm |=3D ODP_WRITE_ALLOWED_BIT; + perm |=3D HMM_PFN_WRITE; =20 mutex_lock(&umem_odp->umem_mutex); =20 diff --git a/include/rdma/ib_umem_odp.h b/include/rdma/ib_umem_odp.h index 0844c1d05ac6..a345c26a745d 100644 --- a/include/rdma/ib_umem_odp.h +++ b/include/rdma/ib_umem_odp.h @@ -8,6 +8,7 @@ =20 #include #include +#include =20 struct ib_umem_odp { struct ib_umem umem; @@ -67,19 +68,6 @@ static inline size_t ib_umem_odp_num_pages(struct ib_ume= m_odp *umem_odp) umem_odp->page_shift; } =20 -/* - * The lower 2 bits of the DMA address signal the R/W permissions for - * the entry. To upgrade the permissions, provide the appropriate - * bitmask to the map_dma_pages function. - * - * Be aware that upgrading a mapped address might result in change of - * the DMA address for the page. - */ -#define ODP_READ_ALLOWED_BIT (1<<0ULL) -#define ODP_WRITE_ALLOWED_BIT (1<<1ULL) - -#define ODP_DMA_ADDR_MASK (~(ODP_READ_ALLOWED_BIT | ODP_WRITE_ALLOWED_BIT)) - #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING =20 struct ib_umem_odp * --=20 2.49.0