From nobody Fri Dec 19 04:23:14 2025 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-1.web.codeaurora.org [10.30.226.201]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 235CE2BE0EE; Fri, 18 Apr 2025 06:49:21 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=10.30.226.201 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1744958961; cv=none; b=fnuSVHBr+HxnArEmy7uoOhE1GCrvVCRyf3NzcIJlWqZag+bz3Pv8RhTmuTfPy2VZpdouxD9rBZH0JtlxHu1grbpwVoWjJNIPVH1jH/ONVW3NRIasK+BdZuavAMz0bn5WGL4PfUw679hwFxG84YR+pSb+OcvB4Sm8CeKrvL4l1Nk= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1744958961; c=relaxed/simple; bh=ncT22RRno1ysJfodh7c5Z7bhAHTaWTZvj00fBZ8GjdU=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=j/21H1IXJ8pkhySnndcMSpMWcKXngjbJn4a65xIt7bz7KLvZY1GUpL3u6YPsEfCnyaJdwJb0wARGxGL4/Aj51d8U+4B2N5PxCKtLLz5CcJLNbpsPcHj52SNJfPe+qKTkhKhuhu3bVn6/zVFnTwAW4nVPL/rzUGKs9La7RH+f34c= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=VljbOr6C; arc=none smtp.client-ip=10.30.226.201 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="VljbOr6C" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 53C4AC4CEEE; Fri, 18 Apr 2025 06:49:20 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1744958961; bh=ncT22RRno1ysJfodh7c5Z7bhAHTaWTZvj00fBZ8GjdU=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=VljbOr6CNKaLf/Ig+b8CJqcAbbEQc6kVsJCqAh7evwhAyf0qHyoqT4T2sYT9PW+FJ fB2hJor7HTX1Tvahp8PJxr0H6D1WQ85raW4vXZv0OQBOHa1Mo2iZX6mImll5Qj6I+y 59lFSSgTgxJvEIrJVB75uJZwGLnlzsDPiRQGcLXnbcOmd+jyt94xlNfeO/r7dHZ5eO a79DRCPqtk+dJRDOP94Oyrky59EfEm28IAh6jgqZaVkaSpRFL6D32wib+WenPl2wp9 /yXVgb9nqpR0EVFVuRtaxiO8sPNK0HeHh3HWgGpRp9tGIvU4NbIDduyCQuWg68698q BsOrKRZg8pSmQ== From: Leon Romanovsky To: Marek Szyprowski , Jens Axboe , Christoph Hellwig , Keith Busch Cc: Jake Edge , Jonathan Corbet , Jason Gunthorpe , Zhu Yanjun , Robin Murphy , Joerg Roedel , Will Deacon , Sagi Grimberg , Bjorn Helgaas , Logan Gunthorpe , Yishai Hadas , Shameer Kolothum , Kevin Tian , Alex Williamson , =?UTF-8?q?J=C3=A9r=C3=B4me=20Glisse?= , Andrew Morton , linux-doc@vger.kernel.org, linux-kernel@vger.kernel.org, linux-block@vger.kernel.org, linux-rdma@vger.kernel.org, iommu@lists.linux.dev, linux-nvme@lists.infradead.org, linux-pci@vger.kernel.org, kvm@vger.kernel.org, linux-mm@kvack.org, Niklas Schnelle , Chuck Lever , Luis Chamberlain , Matthew Wilcox , Dan Williams , Kanchan Joshi , Chaitanya Kulkarni , Leon Romanovsky Subject: [PATCH v8 22/24] nvme-pci: use a better encoding for small prp pool allocations Date: Fri, 18 Apr 2025 09:47:52 +0300 Message-ID: <0d7998e10dd5e785759599c79c0935ba68eb83f0.1744825142.git.leon@kernel.org> X-Mailer: git-send-email 2.49.0 In-Reply-To: References: Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" From: Christoph Hellwig There is plenty of unused space in the iod next to nr_descriptors. Add a separate bool (which could be condensed to a single bit once we start running out of space) to encode that the transfer is using the full page sized pool, and use a normal 0..n count for the number of descriptors. Signed-off-by: Christoph Hellwig Signed-off-by: Chaitanya Kulkarni Signed-off-by: Leon Romanovsky --- drivers/nvme/host/pci.c | 89 +++++++++++++++++++---------------------- 1 file changed, 42 insertions(+), 47 deletions(-) diff --git a/drivers/nvme/host/pci.c b/drivers/nvme/host/pci.c index 638e759b29ad..5deb492cc18e 100644 --- a/drivers/nvme/host/pci.c +++ b/drivers/nvme/host/pci.c @@ -44,6 +44,7 @@ #define NVME_MAX_SEGS 128 #define NVME_MAX_META_SEGS 15 #define NVME_MAX_NR_DESCRIPTORS 5 +#define NVME_SMALL_DESCRIPTOR_SIZE 256 =20 static int use_threaded_interrupts; module_param(use_threaded_interrupts, int, 0444); @@ -226,8 +227,8 @@ struct nvme_iod { struct nvme_request req; struct nvme_command cmd; bool aborted; - /* # of PRP/SGL descriptors: (0 for small pool) */ - s8 nr_descriptors; + u8 nr_descriptors; /* # of PRP/SGL descriptors */ + bool large_descriptors; /* uses the full page sized descriptor pool */ unsigned int dma_len; /* length of single DMA segment mapping */ dma_addr_t first_dma; dma_addr_t meta_dma; @@ -529,13 +530,27 @@ static inline bool nvme_pci_use_sgls(struct nvme_dev = *dev, struct request *req, return true; } =20 -static void nvme_free_prps(struct nvme_dev *dev, struct request *req) +static inline struct dma_pool *nvme_dma_pool(struct nvme_dev *dev, + struct nvme_iod *iod) +{ + if (iod->large_descriptors) + return dev->prp_page_pool; + return dev->prp_small_pool; +} + +static void nvme_free_descriptors(struct nvme_dev *dev, struct request *re= q) { const int last_prp =3D NVME_CTRL_PAGE_SIZE / sizeof(__le64) - 1; struct nvme_iod *iod =3D blk_mq_rq_to_pdu(req); dma_addr_t dma_addr =3D iod->first_dma; int i; =20 + if (iod->nr_descriptors =3D=3D 1) { + dma_pool_free(nvme_dma_pool(dev, iod), iod->descriptors[0], + dma_addr); + return; + } + for (i =3D 0; i < iod->nr_descriptors; i++) { __le64 *prp_list =3D iod->descriptors[i]; dma_addr_t next_dma_addr =3D le64_to_cpu(prp_list[last_prp]); @@ -558,15 +573,7 @@ static void nvme_unmap_data(struct nvme_dev *dev, stru= ct request *req) WARN_ON_ONCE(!iod->sgt.nents); =20 dma_unmap_sgtable(dev->dev, &iod->sgt, rq_dma_dir(req), 0); - - if (iod->nr_descriptors =3D=3D 0) - dma_pool_free(dev->prp_small_pool, iod->descriptors[0], - iod->first_dma); - else if (iod->nr_descriptors =3D=3D 1) - dma_pool_free(dev->prp_page_pool, iod->descriptors[0], - iod->first_dma); - else - nvme_free_prps(dev, req); + nvme_free_descriptors(dev, req); mempool_free(iod->sgt.sgl, dev->iod_mempool); } =20 @@ -588,7 +595,6 @@ static blk_status_t nvme_pci_setup_prps(struct nvme_dev= *dev, struct request *req, struct nvme_rw_command *cmnd) { struct nvme_iod *iod =3D blk_mq_rq_to_pdu(req); - struct dma_pool *pool; int length =3D blk_rq_payload_bytes(req); struct scatterlist *sg =3D iod->sgt.sgl; int dma_len =3D sg_dma_len(sg); @@ -596,7 +602,7 @@ static blk_status_t nvme_pci_setup_prps(struct nvme_dev= *dev, int offset =3D dma_addr & (NVME_CTRL_PAGE_SIZE - 1); __le64 *prp_list; dma_addr_t prp_dma; - int nprps, i; + int i; =20 length -=3D (NVME_CTRL_PAGE_SIZE - offset); if (length <=3D 0) { @@ -618,27 +624,23 @@ static blk_status_t nvme_pci_setup_prps(struct nvme_d= ev *dev, goto done; } =20 - nprps =3D DIV_ROUND_UP(length, NVME_CTRL_PAGE_SIZE); - if (nprps <=3D (256 / 8)) { - pool =3D dev->prp_small_pool; - iod->nr_descriptors =3D 0; - } else { - pool =3D dev->prp_page_pool; - iod->nr_descriptors =3D 1; - } + if (DIV_ROUND_UP(length, NVME_CTRL_PAGE_SIZE) > + NVME_SMALL_DESCRIPTOR_SIZE / sizeof(__le64)) + iod->large_descriptors =3D true; =20 - prp_list =3D dma_pool_alloc(pool, GFP_ATOMIC, &prp_dma); - if (!prp_list) { - iod->nr_descriptors =3D -1; + prp_list =3D dma_pool_alloc(nvme_dma_pool(dev, iod), GFP_ATOMIC, + &prp_dma); + if (!prp_list) return BLK_STS_RESOURCE; - } - iod->descriptors[0] =3D prp_list; + iod->descriptors[iod->nr_descriptors++] =3D prp_list; iod->first_dma =3D prp_dma; i =3D 0; for (;;) { if (i =3D=3D NVME_CTRL_PAGE_SIZE >> 3) { __le64 *old_prp_list =3D prp_list; - prp_list =3D dma_pool_alloc(pool, GFP_ATOMIC, &prp_dma); + + prp_list =3D dma_pool_alloc(dev->prp_page_pool, + GFP_ATOMIC, &prp_dma); if (!prp_list) goto free_prps; iod->descriptors[iod->nr_descriptors++] =3D prp_list; @@ -665,7 +667,7 @@ static blk_status_t nvme_pci_setup_prps(struct nvme_dev= *dev, cmnd->dptr.prp2 =3D cpu_to_le64(iod->first_dma); return BLK_STS_OK; free_prps: - nvme_free_prps(dev, req); + nvme_free_descriptors(dev, req); return BLK_STS_RESOURCE; bad_sgl: WARN(DO_ONCE(nvme_print_sgl, iod->sgt.sgl, iod->sgt.nents), @@ -694,7 +696,6 @@ static blk_status_t nvme_pci_setup_sgls(struct nvme_dev= *dev, struct request *req, struct nvme_rw_command *cmd) { struct nvme_iod *iod =3D blk_mq_rq_to_pdu(req); - struct dma_pool *pool; struct nvme_sgl_desc *sg_list; struct scatterlist *sg =3D iod->sgt.sgl; unsigned int entries =3D iod->sgt.nents; @@ -709,21 +710,13 @@ static blk_status_t nvme_pci_setup_sgls(struct nvme_d= ev *dev, return BLK_STS_OK; } =20 - if (entries <=3D (256 / sizeof(struct nvme_sgl_desc))) { - pool =3D dev->prp_small_pool; - iod->nr_descriptors =3D 0; - } else { - pool =3D dev->prp_page_pool; - iod->nr_descriptors =3D 1; - } + if (entries > NVME_SMALL_DESCRIPTOR_SIZE / sizeof(*sg_list)) + iod->large_descriptors =3D true; =20 - sg_list =3D dma_pool_alloc(pool, GFP_ATOMIC, &sgl_dma); - if (!sg_list) { - iod->nr_descriptors =3D -1; + sg_list =3D dma_pool_alloc(nvme_dma_pool(dev, iod), GFP_ATOMIC, &sgl_dma); + if (!sg_list) return BLK_STS_RESOURCE; - } - - iod->descriptors[0] =3D sg_list; + iod->descriptors[iod->nr_descriptors++] =3D sg_list; iod->first_dma =3D sgl_dma; =20 nvme_pci_sgl_set_seg(&cmd->dptr.sgl, sgl_dma, entries); @@ -915,7 +908,8 @@ static blk_status_t nvme_prep_rq(struct nvme_dev *dev, = struct request *req) blk_status_t ret; =20 iod->aborted =3D false; - iod->nr_descriptors =3D -1; + iod->nr_descriptors =3D 0; + iod->large_descriptors =3D false; iod->sgt.nents =3D 0; iod->meta_sgt.nents =3D 0; =20 @@ -2833,7 +2827,7 @@ static int nvme_disable_prepare_reset(struct nvme_dev= *dev, bool shutdown) =20 static int nvme_setup_prp_pools(struct nvme_dev *dev) { - size_t small_align =3D 256; + size_t small_align =3D NVME_SMALL_DESCRIPTOR_SIZE; =20 dev->prp_page_pool =3D dma_pool_create("prp list page", dev->dev, NVME_CTRL_PAGE_SIZE, @@ -2841,12 +2835,13 @@ static int nvme_setup_prp_pools(struct nvme_dev *de= v) if (!dev->prp_page_pool) return -ENOMEM; =20 + BUILD_BUG_ON(NVME_SMALL_DESCRIPTOR_SIZE !=3D 256); if (dev->ctrl.quirks & NVME_QUIRK_DMAPOOL_ALIGN_512) - small_align =3D 512; + small_align *=3D 2; =20 /* Optimisation for I/Os between 4k and 128k */ dev->prp_small_pool =3D dma_pool_create("prp list 256", dev->dev, - 256, small_align, 0); + NVME_SMALL_DESCRIPTOR_SIZE, small_align, 0); if (!dev->prp_small_pool) { dma_pool_destroy(dev->prp_page_pool); return -ENOMEM; --=20 2.49.0