From nobody Tue Apr 28 02:28:15 2026 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id 3D94BC433EF for ; Tue, 7 Jun 2022 21:39:33 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1381083AbiFGVjb (ORCPT ); Tue, 7 Jun 2022 17:39:31 -0400 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:51692 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1377161AbiFGUuQ (ORCPT ); Tue, 7 Jun 2022 16:50:16 -0400 X-Greylist: delayed 61 seconds by postgrey-1.37 at lindbergh.monkeyblade.net; Tue, 07 Jun 2022 11:39:57 PDT Received: from mail.cybernetics.com (mail.cybernetics.com [173.71.130.66]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id 099651F77E0 for ; Tue, 7 Jun 2022 11:39:56 -0700 (PDT) X-ASG-Debug-ID: 1654627180-1cf43917f33964e0001-xx1T2L Received: from cybernetics.com ([10.10.4.126]) by mail.cybernetics.com with ESMTP id DY7NxP6F0bNdGTkt; Tue, 07 Jun 2022 14:39:40 -0400 (EDT) X-Barracuda-Envelope-From: tonyb@cybernetics.com X-ASG-Whitelist: Client DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; d=cybernetics.com; s=mail; bh=9dUzxpvwws15FcrE4ulKEcPQJF2dj8RsJsfXaNUjy8Y=; h=Content-Transfer-Encoding:Content-Type:In-Reply-To:References:Cc:To:From: Content-Language:Subject:MIME-Version:Date:Message-ID; b=Ou8kArJXBehA8GyTgqeQ adcuNOIrLO8fMzz0ojoRfoLJz+wpXSuttbZmibRiMkyO4zjv3Cc4jorMTfjLqzNGd4Ra5SNqwUzks CrBOzuZhKxfRzb7xZ87l9rErl5HNWx/eNoZqJ1E4btmP6cEUUNx+veehFLQCr6cAhEojgbXAZs= Received: from [10.157.2.224] (HELO [192.168.200.1]) by cybernetics.com (CommuniGate Pro SMTP 7.1.1) with ESMTPS id 11859391; Tue, 07 Jun 2022 14:39:40 -0400 Message-ID: <7ba0de95-5f5b-e140-c4ee-ff34e70e2546@cybernetics.com> Date: Tue, 7 Jun 2022 14:39:40 -0400 MIME-Version: 1.0 User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:91.0) Gecko/20100101 Thunderbird/91.9.1 Subject: [PATCH v6 01/11] dmapool: remove checks for dev == NULL Content-Language: en-US X-ASG-Orig-Subj: [PATCH v6 01/11] dmapool: remove checks for dev == NULL From: Tony Battersby To: linux-mm@kvack.org, linux-kernel@vger.kernel.org Cc: iommu@lists.linux-foundation.org, kernel-team@fb.com, Matthew Wilcox , Keith Busch , Andy Shevchenko , Robin Murphy , Tony Lindgren References: <340ff8ef-9ff5-7175-c234-4132bbdfc5f7@cybernetics.com> In-Reply-To: <340ff8ef-9ff5-7175-c234-4132bbdfc5f7@cybernetics.com> Content-Transfer-Encoding: quoted-printable X-Barracuda-Connect: UNKNOWN[10.10.4.126] X-Barracuda-Start-Time: 1654627180 X-Barracuda-URL: https://10.10.4.122:443/cgi-mod/mark.cgi X-Barracuda-BRTS-Status: 1 X-Virus-Scanned: by bsmtpd at cybernetics.com X-Barracuda-Scan-Msg-Size: 3594 Precedence: bulk List-ID: X-Mailing-List: linux-kernel@vger.kernel.org Content-Type: text/plain; charset="utf-8" dmapool originally tried to support pools without a device because dma_alloc_coherent() supports allocations without a device. But nobody ended up using dma pools without a device, and trying to do so will result in an oops. So remove the checks for pool->dev =3D=3D NULL since th= ey are unneeded bloat. Signed-off-by: Tony Battersby --- mm/dmapool.c | 42 +++++++++++------------------------------- 1 file changed, 11 insertions(+), 31 deletions(-) diff --git a/mm/dmapool.c b/mm/dmapool.c index a7eb5d0eb2da..0f89de408cbe 100644 --- a/mm/dmapool.c +++ b/mm/dmapool.c @@ -275,7 +275,7 @@ void dma_pool_destroy(struct dma_pool *pool) mutex_lock(&pools_reg_lock); mutex_lock(&pools_lock); list_del(&pool->pools); - if (pool->dev && list_empty(&pool->dev->dma_pools)) + if (list_empty(&pool->dev->dma_pools)) empty =3D true; mutex_unlock(&pools_lock); if (empty) @@ -284,12 +284,8 @@ void dma_pool_destroy(struct dma_pool *pool) =20 list_for_each_entry_safe(page, tmp, &pool->page_list, page_list) { if (is_page_busy(page)) { - if (pool->dev) - dev_err(pool->dev, "%s %s, %p busy\n", __func__, - pool->name, page->vaddr); - else - pr_err("%s %s, %p busy\n", __func__, - pool->name, page->vaddr); + dev_err(pool->dev, "%s %s, %p busy\n", __func__, + pool->name, page->vaddr); /* leak the still-in-use consistent memory */ list_del(&page->page_list); kfree(page); @@ -351,12 +347,8 @@ void *dma_pool_alloc(struct dma_pool *pool, gfp_t mem_= flags, for (i =3D sizeof(page->offset); i < pool->size; i++) { if (data[i] =3D=3D POOL_POISON_FREED) continue; - if (pool->dev) - dev_err(pool->dev, "%s %s, %p (corrupted)\n", - __func__, pool->name, retval); - else - pr_err("%s %s, %p (corrupted)\n", - __func__, pool->name, retval); + dev_err(pool->dev, "%s %s, %p (corrupted)\n", + __func__, pool->name, retval); =20 /* * Dump the first 4 bytes even if they are not @@ -411,12 +403,8 @@ void dma_pool_free(struct dma_pool *pool, void *vaddr,= dma_addr_t dma) page =3D pool_find_page(pool, dma); if (!page) { spin_unlock_irqrestore(&pool->lock, flags); - if (pool->dev) - dev_err(pool->dev, "%s %s, %p/%pad (bad dma)\n", - __func__, pool->name, vaddr, &dma); - else - pr_err("%s %s, %p/%pad (bad dma)\n", - __func__, pool->name, vaddr, &dma); + dev_err(pool->dev, "%s %s, %p/%pad (bad dma)\n", + __func__, pool->name, vaddr, &dma); return; } =20 @@ -426,12 +414,8 @@ void dma_pool_free(struct dma_pool *pool, void *vaddr,= dma_addr_t dma) #ifdef DMAPOOL_DEBUG if ((dma - page->dma) !=3D offset) { spin_unlock_irqrestore(&pool->lock, flags); - if (pool->dev) - dev_err(pool->dev, "%s %s, %p (bad vaddr)/%pad\n", - __func__, pool->name, vaddr, &dma); - else - pr_err("%s %s, %p (bad vaddr)/%pad\n", - __func__, pool->name, vaddr, &dma); + dev_err(pool->dev, "%s %s, %p (bad vaddr)/%pad\n", + __func__, pool->name, vaddr, &dma); return; } { @@ -442,12 +426,8 @@ void dma_pool_free(struct dma_pool *pool, void *vaddr,= dma_addr_t dma) continue; } spin_unlock_irqrestore(&pool->lock, flags); - if (pool->dev) - dev_err(pool->dev, "%s %s, dma %pad already free\n", - __func__, pool->name, &dma); - else - pr_err("%s %s, dma %pad already free\n", - __func__, pool->name, &dma); + dev_err(pool->dev, "%s %s, dma %pad already free\n", + __func__, pool->name, &dma); return; } } --=20 2.25.1 From nobody Tue Apr 28 02:28:15 2026 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id 3A229CCA47F for ; Tue, 7 Jun 2022 21:42:12 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1381353AbiFGVka (ORCPT ); Tue, 7 Jun 2022 17:40:30 -0400 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:51930 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1377683AbiFGUux (ORCPT ); Tue, 7 Jun 2022 16:50:53 -0400 Received: from mail.cybernetics.com (mail.cybernetics.com [173.71.130.66]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id AD7EF18484F for ; Tue, 7 Jun 2022 11:40:29 -0700 (PDT) X-ASG-Debug-ID: 1654627227-1cf43917f3396520001-xx1T2L Received: from cybernetics.com ([10.10.4.126]) by mail.cybernetics.com with ESMTP id 7q8f4i5VB5tulwTq; Tue, 07 Jun 2022 14:40:27 -0400 (EDT) X-Barracuda-Envelope-From: tonyb@cybernetics.com X-ASG-Whitelist: Client DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; d=cybernetics.com; s=mail; bh=cPbWsHAkkEQ/IV9PNOMIz+rmz60KjJDA9BYZCB0e3iI=; h=Content-Transfer-Encoding:Content-Type:In-Reply-To:References:Cc:To:From: Content-Language:Subject:MIME-Version:Date:Message-ID; b=J61v7CvSfK8dXjZ+w6d6 8xsL1vF77oaGreuBYU7E3kJZx8xX3v0D3QVShQw+LlGqoqmzsxMRHMWmX0lcshWXQX6z4NcKT9bI+ q8x408aTjLn4GMLHAA6WESTFy6mRrWzbZH1Xax48DUgHMZxC7lO4zW3Q1Hp8lmRcXny/TJ7c6w= Received: from [10.157.2.224] (HELO [192.168.200.1]) by cybernetics.com (CommuniGate Pro SMTP 7.1.1) with ESMTPS id 11859402; Tue, 07 Jun 2022 14:40:26 -0400 Message-ID: <6c627beb-eb61-5baa-484a-e35567f05e89@cybernetics.com> Date: Tue, 7 Jun 2022 14:40:27 -0400 MIME-Version: 1.0 User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:91.0) Gecko/20100101 Thunderbird/91.9.1 Subject: [PATCH v6 02/11] dmapool: use sysfs_emit() instead of scnprintf() Content-Language: en-US X-ASG-Orig-Subj: [PATCH v6 02/11] dmapool: use sysfs_emit() instead of scnprintf() From: Tony Battersby To: linux-mm@kvack.org, linux-kernel@vger.kernel.org Cc: iommu@lists.linux-foundation.org, kernel-team@fb.com, Matthew Wilcox , Keith Busch , Andy Shevchenko , Robin Murphy , Tony Lindgren References: <340ff8ef-9ff5-7175-c234-4132bbdfc5f7@cybernetics.com> In-Reply-To: <340ff8ef-9ff5-7175-c234-4132bbdfc5f7@cybernetics.com> Content-Transfer-Encoding: quoted-printable X-Barracuda-Connect: UNKNOWN[10.10.4.126] X-Barracuda-Start-Time: 1654627227 X-Barracuda-URL: https://10.10.4.122:443/cgi-mod/mark.cgi X-Barracuda-BRTS-Status: 1 X-Virus-Scanned: by bsmtpd at cybernetics.com X-Barracuda-Scan-Msg-Size: 1630 Precedence: bulk List-ID: X-Mailing-List: linux-kernel@vger.kernel.org Content-Type: text/plain; charset="utf-8" Use sysfs_emit instead of scnprintf, snprintf or sprintf. Signed-off-by: Tony Battersby --- Changes since v5: This patch was not in v5. mm/dmapool.c | 23 +++++++---------------- 1 file changed, 7 insertions(+), 16 deletions(-) diff --git a/mm/dmapool.c b/mm/dmapool.c index 0f89de408cbe..1829291f5d70 100644 --- a/mm/dmapool.c +++ b/mm/dmapool.c @@ -64,18 +64,11 @@ static DEFINE_MUTEX(pools_reg_lock); =20 static ssize_t pools_show(struct device *dev, struct device_attribute *att= r, char *buf) { - unsigned temp; - unsigned size; - char *next; + int size; struct dma_page *page; struct dma_pool *pool; =20 - next =3D buf; - size =3D PAGE_SIZE; - - temp =3D scnprintf(next, size, "poolinfo - 0.1\n"); - size -=3D temp; - next +=3D temp; + size =3D sysfs_emit(buf, "poolinfo - 0.1\n"); =20 mutex_lock(&pools_lock); list_for_each_entry(pool, &dev->dma_pools, pools) { @@ -90,16 +83,14 @@ static ssize_t pools_show(struct device *dev, struct de= vice_attribute *attr, cha spin_unlock_irq(&pool->lock); =20 /* per-pool info, no real statistics yet */ - temp =3D scnprintf(next, size, "%-16s %4u %4zu %4zu %2u\n", - pool->name, blocks, - pages * (pool->allocation / pool->size), - pool->size, pages); - size -=3D temp; - next +=3D temp; + size +=3D sysfs_emit_at(buf, size, "%-16s %4u %4zu %4zu %2u\n", + pool->name, blocks, + pages * (pool->allocation / pool->size), + pool->size, pages); } mutex_unlock(&pools_lock); =20 - return PAGE_SIZE - size; + return size; } =20 static DEVICE_ATTR_RO(pools); --=20 2.25.1 From nobody Tue Apr 28 02:28:15 2026 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id 21988CCA48F for ; Tue, 7 Jun 2022 21:42:13 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1381621AbiFGVlW (ORCPT ); Tue, 7 Jun 2022 17:41:22 -0400 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:49306 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1378036AbiFGUvW (ORCPT ); Tue, 7 Jun 2022 16:51:22 -0400 Received: from mail.cybernetics.com (mail.cybernetics.com [173.71.130.66]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id 410D0184866 for ; Tue, 7 Jun 2022 11:41:27 -0700 (PDT) X-ASG-Debug-ID: 1654627285-1cf43917f3396570001-xx1T2L Received: from cybernetics.com ([10.10.4.126]) by mail.cybernetics.com with ESMTP id DDLvrFGSLFUxedt8; Tue, 07 Jun 2022 14:41:25 -0400 (EDT) X-Barracuda-Envelope-From: tonyb@cybernetics.com X-ASG-Whitelist: Client DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; d=cybernetics.com; s=mail; bh=TzT+rsQP1dU+4gl9FedwnDXSnYdnIujwXRUJ3rWtVjY=; h=Content-Transfer-Encoding:Content-Type:In-Reply-To:References:Cc:To:From: Content-Language:Subject:MIME-Version:Date:Message-ID; b=QobQ5f3Vw7/QICg5Lf52 wSTg0IRpHCrg41KSASEOHyfrsoeCDJHx7Lswmn08jk4qmskWB4B+joh+Vn6isYj0RlrIHUCO8AhM2 vPVcqWRgNXU7foh0YWIo9WPEl49krQojYiEe3yGDcotj+irgOpjXVMKlH3PAJG7mY6kDq1fYBw= Received: from [10.157.2.224] (HELO [192.168.200.1]) by cybernetics.com (CommuniGate Pro SMTP 7.1.1) with ESMTPS id 11859414; Tue, 07 Jun 2022 14:41:25 -0400 Message-ID: Date: Tue, 7 Jun 2022 14:41:25 -0400 MIME-Version: 1.0 User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:91.0) Gecko/20100101 Thunderbird/91.9.1 Subject: [PATCH v6 03/11] dmapool: cleanup integer types Content-Language: en-US X-ASG-Orig-Subj: [PATCH v6 03/11] dmapool: cleanup integer types From: Tony Battersby To: linux-mm@kvack.org, linux-kernel@vger.kernel.org Cc: iommu@lists.linux-foundation.org, kernel-team@fb.com, Matthew Wilcox , Keith Busch , Andy Shevchenko , Robin Murphy , Tony Lindgren References: <340ff8ef-9ff5-7175-c234-4132bbdfc5f7@cybernetics.com> In-Reply-To: <340ff8ef-9ff5-7175-c234-4132bbdfc5f7@cybernetics.com> Content-Transfer-Encoding: quoted-printable X-Barracuda-Connect: UNKNOWN[10.10.4.126] X-Barracuda-Start-Time: 1654627285 X-Barracuda-URL: https://10.10.4.122:443/cgi-mod/mark.cgi X-Barracuda-BRTS-Status: 1 X-Virus-Scanned: by bsmtpd at cybernetics.com X-Barracuda-Scan-Msg-Size: 3538 Precedence: bulk List-ID: X-Mailing-List: linux-kernel@vger.kernel.org Content-Type: text/plain; charset="utf-8" To represent the size of a single allocation, dmapool currently uses 'unsigned int' in some places and 'size_t' in other places. Standardize on 'unsigned int' to reduce overhead, but use 'size_t' when counting all the blocks in the entire pool. Signed-off-by: Tony Battersby --- Changes since v5: moved 'size' down in struct dma_pool to be near the other 32-bit ints. blks_per_alloc will fill out the hole in a later patch. This puts an upper bound on 'size' of INT_MAX to avoid overflowing the following comparison in pool_initialise_page(): unsigned int offset =3D 0; unsigned int next =3D offset + pool->size; if (unlikely((next + pool->size) > ... 'boundary' is passed in as a size_t but gets stored as an unsigned int. 'boundary' values >=3D 'allocation' do not have any effect, so clipping 'boundary' to 'allocation' keeps it within the range of unsigned int without affecting anything else. A few lines above (not in the diff) you can see that if 'boundary' is passed in as 0 then it is set to 'allocation', so it is nothing new. For reference, here is the relevant code after being patched: if (!boundary) boundary =3D allocation; else if ((boundary < size) || (boundary & (boundary - 1))) return NULL; boundary =3D min(boundary, allocation); mm/dmapool.c | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/mm/dmapool.c b/mm/dmapool.c index 1829291f5d70..f85d6bde2205 100644 --- a/mm/dmapool.c +++ b/mm/dmapool.c @@ -43,10 +43,10 @@ struct dma_pool { /* the pool */ struct list_head page_list; spinlock_t lock; - size_t size; struct device *dev; - size_t allocation; - size_t boundary; + unsigned int size; + unsigned int allocation; + unsigned int boundary; char name[32]; struct list_head pools; }; @@ -73,7 +73,7 @@ static ssize_t pools_show(struct device *dev, struct devi= ce_attribute *attr, cha mutex_lock(&pools_lock); list_for_each_entry(pool, &dev->dma_pools, pools) { unsigned pages =3D 0; - unsigned blocks =3D 0; + size_t blocks =3D 0; =20 spin_lock_irq(&pool->lock); list_for_each_entry(page, &pool->page_list, page_list) { @@ -83,9 +83,10 @@ static ssize_t pools_show(struct device *dev, struct dev= ice_attribute *attr, cha spin_unlock_irq(&pool->lock); =20 /* per-pool info, no real statistics yet */ - size +=3D sysfs_emit_at(buf, size, "%-16s %4u %4zu %4zu %2u\n", + size +=3D sysfs_emit_at(buf, size, "%-16s %4zu %4zu %4u %2u\n", pool->name, blocks, - pages * (pool->allocation / pool->size), + (size_t) pages * + (pool->allocation / pool->size), pool->size, pages); } mutex_unlock(&pools_lock); @@ -130,7 +131,7 @@ struct dma_pool *dma_pool_create(const char *name, stru= ct device *dev, else if (align & (align - 1)) return NULL; =20 - if (size =3D=3D 0) + if (size =3D=3D 0 || size > INT_MAX) return NULL; else if (size < 4) size =3D 4; @@ -143,6 +144,8 @@ struct dma_pool *dma_pool_create(const char *name, stru= ct device *dev, else if ((boundary < size) || (boundary & (boundary - 1))) return NULL; =20 + boundary =3D min(boundary, allocation); + retval =3D kmalloc(sizeof(*retval), GFP_KERNEL); if (!retval) return retval; @@ -303,7 +306,7 @@ void *dma_pool_alloc(struct dma_pool *pool, gfp_t mem_f= lags, { unsigned long flags; struct dma_page *page; - size_t offset; + unsigned int offset; void *retval; =20 might_alloc(mem_flags); --=20 2.25.1 From nobody Tue Apr 28 02:28:15 2026 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id 2FC72CCA492 for ; Tue, 7 Jun 2022 21:46:52 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1382175AbiFGVql (ORCPT ); Tue, 7 Jun 2022 17:46:41 -0400 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:51974 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1378379AbiFGUvm (ORCPT ); Tue, 7 Jun 2022 16:51:42 -0400 Received: from mail.cybernetics.com (mail.cybernetics.com [173.71.130.66]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id 06C23115A43 for ; Tue, 7 Jun 2022 11:42:10 -0700 (PDT) X-ASG-Debug-ID: 1654627329-1cf43917f33965c0001-xx1T2L Received: from cybernetics.com ([10.10.4.126]) by mail.cybernetics.com with ESMTP id 79iRHZabdgVhHOFB; Tue, 07 Jun 2022 14:42:09 -0400 (EDT) X-Barracuda-Envelope-From: tonyb@cybernetics.com X-ASG-Whitelist: Client DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; d=cybernetics.com; s=mail; bh=QbGgNEs9DYDdDETge9nU6qZiAN8p+i0+8LJZVlYu570=; h=Content-Transfer-Encoding:Content-Type:In-Reply-To:References:Cc:To:From: Content-Language:Subject:MIME-Version:Date:Message-ID; b=jsWpK2MOULqWUEtEYypu BZF113V20OKS/bUvhZVyqHKTY3exH3pnTa1QbMsrI0H9CbQGv9hAAG8PzWuG8R9hLqsUNEYUi26EP 8sXIBWYag6D+3OMUTNSodUaPd+OaQi/fx5JrYEKxj7ulxkltkon72+19JDoSSGToGumE1b7Fto= Received: from [10.157.2.224] (HELO [192.168.200.1]) by cybernetics.com (CommuniGate Pro SMTP 7.1.1) with ESMTPS id 11859424; Tue, 07 Jun 2022 14:42:09 -0400 Message-ID: <6eaea467-cc50-acbd-6232-892011fc803c@cybernetics.com> Date: Tue, 7 Jun 2022 14:42:09 -0400 MIME-Version: 1.0 User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:91.0) Gecko/20100101 Thunderbird/91.9.1 Subject: [PATCH v6 04/11] dmapool: fix boundary comparison Content-Language: en-US X-ASG-Orig-Subj: [PATCH v6 04/11] dmapool: fix boundary comparison From: Tony Battersby To: linux-mm@kvack.org, linux-kernel@vger.kernel.org Cc: iommu@lists.linux-foundation.org, kernel-team@fb.com, Matthew Wilcox , Keith Busch , Andy Shevchenko , Robin Murphy , Tony Lindgren References: <340ff8ef-9ff5-7175-c234-4132bbdfc5f7@cybernetics.com> In-Reply-To: <340ff8ef-9ff5-7175-c234-4132bbdfc5f7@cybernetics.com> Content-Transfer-Encoding: quoted-printable X-Barracuda-Connect: UNKNOWN[10.10.4.126] X-Barracuda-Start-Time: 1654627329 X-Barracuda-URL: https://10.10.4.122:443/cgi-mod/mark.cgi X-Barracuda-BRTS-Status: 1 X-Virus-Scanned: by bsmtpd at cybernetics.com X-Barracuda-Scan-Msg-Size: 1388 Precedence: bulk List-ID: X-Mailing-List: linux-kernel@vger.kernel.org Content-Type: text/plain; charset="utf-8" Fix the boundary comparison when constructing the list of free blocks for the case that 'size' is a power of two. Since 'boundary' is also a power of two, that would make 'boundary' a multiple of 'size', in which case a single block would never cross the boundary. This bug would cause some of the allocated memory to be wasted (but not leaked). Example: size =3D 512 boundary =3D 2048 allocation =3D 4096 Address range 0 - 511 512 - 1023 1024 - 1535 1536 - 2047 * 2048 - 2559 2560 - 3071 3072 - 3583 3584 - 4095 * Prior to this fix, the address ranges marked with "*" would not have been used even though they didn't cross the given boundary. Fixes: e34f44b3517f ("pool: Improve memory usage for devices which can't cr= oss boundaries") Signed-off-by: Tony Battersby --- mm/dmapool.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mm/dmapool.c b/mm/dmapool.c index f85d6bde2205..122781fe2c03 100644 --- a/mm/dmapool.c +++ b/mm/dmapool.c @@ -201,7 +201,7 @@ static void pool_initialise_page(struct dma_pool *pool,= struct dma_page *page) =20 do { unsigned int next =3D offset + pool->size; - if (unlikely((next + pool->size) >=3D next_boundary)) { + if (unlikely((next + pool->size) > next_boundary)) { next =3D next_boundary; next_boundary +=3D pool->boundary; } --=20 2.25.1 From nobody Tue Apr 28 02:28:15 2026 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id 4D4A9CCA486 for ; Tue, 7 Jun 2022 21:46:51 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1381916AbiFGVpk (ORCPT ); Tue, 7 Jun 2022 17:45:40 -0400 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:51644 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1378749AbiFGUw0 (ORCPT ); Tue, 7 Jun 2022 16:52:26 -0400 Received: from mail.cybernetics.com (mail.cybernetics.com [173.71.130.66]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id 206B91ADBF for ; Tue, 7 Jun 2022 11:42:58 -0700 (PDT) X-ASG-Debug-ID: 1654627376-1cf43917f33965e0001-xx1T2L Received: from cybernetics.com ([10.10.4.126]) by mail.cybernetics.com with ESMTP id 9IJWbVWe0kxl8Q4W; Tue, 07 Jun 2022 14:42:56 -0400 (EDT) X-Barracuda-Envelope-From: tonyb@cybernetics.com X-ASG-Whitelist: Client DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; d=cybernetics.com; s=mail; bh=eQM8tN6t62Fopbgjt8uTQgBSZ1EO2wkhvfxABlFtH9o=; h=Content-Transfer-Encoding:Content-Type:In-Reply-To:References:Cc:To:From: Content-Language:Subject:MIME-Version:Date:Message-ID; b=IixmWpGiaqvQUprEzkz0 RLY8E/16bD+CZb7eDVxwsJJx0WhcGJberWRDEq6vXklSwUFj/Uz/gezadInQpSSObCp8dKbeGiBa7 w3Y7jLuYeY+HtSBYaVfbLDLoltgTK+A7YCT6QfatyfhhlsxJVy8j/AsCZzK72KxdoC1mOT995M= Received: from [10.157.2.224] (HELO [192.168.200.1]) by cybernetics.com (CommuniGate Pro SMTP 7.1.1) with ESMTPS id 11859435; Tue, 07 Jun 2022 14:42:56 -0400 Message-ID: Date: Tue, 7 Jun 2022 14:42:56 -0400 MIME-Version: 1.0 User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:91.0) Gecko/20100101 Thunderbird/91.9.1 Subject: [PATCH v6 05/11] dmapool: improve accuracy of debug statistics Content-Language: en-US X-ASG-Orig-Subj: [PATCH v6 05/11] dmapool: improve accuracy of debug statistics From: Tony Battersby To: linux-mm@kvack.org, linux-kernel@vger.kernel.org Cc: iommu@lists.linux-foundation.org, kernel-team@fb.com, Matthew Wilcox , Keith Busch , Andy Shevchenko , Robin Murphy , Tony Lindgren References: <340ff8ef-9ff5-7175-c234-4132bbdfc5f7@cybernetics.com> In-Reply-To: <340ff8ef-9ff5-7175-c234-4132bbdfc5f7@cybernetics.com> Content-Transfer-Encoding: quoted-printable X-Barracuda-Connect: UNKNOWN[10.10.4.126] X-Barracuda-Start-Time: 1654627376 X-Barracuda-URL: https://10.10.4.122:443/cgi-mod/mark.cgi X-Barracuda-BRTS-Status: 1 X-Virus-Scanned: by bsmtpd at cybernetics.com X-Barracuda-Scan-Msg-Size: 1818 Precedence: bulk List-ID: X-Mailing-List: linux-kernel@vger.kernel.org Content-Type: text/plain; charset="utf-8" The "total number of blocks in pool" debug statistic currently does not take the boundary value into account, so it diverges from the "total number of blocks in use" statistic when a boundary is in effect. Add a calculation for the number of blocks per allocation that takes the boundary into account, and use it to replace the inaccurate calculation. This depends on the patch "dmapool: fix boundary comparison" for the calculated blks_per_alloc value to be correct. Signed-off-by: Tony Battersby --- The added blks_per_alloc value will also be used in the next patch. mm/dmapool.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/mm/dmapool.c b/mm/dmapool.c index 122781fe2c03..d3e5a6151fb4 100644 --- a/mm/dmapool.c +++ b/mm/dmapool.c @@ -47,6 +47,7 @@ struct dma_pool { /* the pool */ unsigned int size; unsigned int allocation; unsigned int boundary; + unsigned int blks_per_alloc; char name[32]; struct list_head pools; }; @@ -85,8 +86,7 @@ static ssize_t pools_show(struct device *dev, struct devi= ce_attribute *attr, cha /* per-pool info, no real statistics yet */ size +=3D sysfs_emit_at(buf, size, "%-16s %4zu %4zu %4u %2u\n", pool->name, blocks, - (size_t) pages * - (pool->allocation / pool->size), + (size_t) pages * pool->blks_per_alloc, pool->size, pages); } mutex_unlock(&pools_lock); @@ -159,6 +159,9 @@ struct dma_pool *dma_pool_create(const char *name, stru= ct device *dev, retval->size =3D size; retval->boundary =3D boundary; retval->allocation =3D allocation; + retval->blks_per_alloc =3D + (allocation / boundary) * (boundary / size) + + (allocation % boundary) / size; =20 INIT_LIST_HEAD(&retval->pools); =20 --=20 2.25.1 From nobody Tue Apr 28 02:28:15 2026 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id 20B7FCCA493 for ; Tue, 7 Jun 2022 21:46:52 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1382161AbiFGVqj (ORCPT ); Tue, 7 Jun 2022 17:46:39 -0400 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:51806 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1379038AbiFGUwt (ORCPT ); Tue, 7 Jun 2022 16:52:49 -0400 Received: from mail.cybernetics.com (mail.cybernetics.com [173.71.130.66]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id 5AADF6C0FD for ; Tue, 7 Jun 2022 11:43:42 -0700 (PDT) X-ASG-Debug-ID: 1654627420-1cf43917f3396640001-xx1T2L Received: from cybernetics.com ([10.10.4.126]) by mail.cybernetics.com with ESMTP id 6i000wcOepKFgepC; Tue, 07 Jun 2022 14:43:40 -0400 (EDT) X-Barracuda-Envelope-From: tonyb@cybernetics.com X-ASG-Whitelist: Client DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; d=cybernetics.com; s=mail; bh=PTdmYDYQtaJrowq+CO1r9NBKL5iWIOZcs9cyzc3ahus=; h=Content-Transfer-Encoding:Content-Type:In-Reply-To:References:Cc:To:From: Content-Language:Subject:MIME-Version:Date:Message-ID; b=eFhH/8oZoQRI6kwNblqq GO5bRSb+r01mmH471kRl9m4M/4VIu+CfaebGobYNsjfg2fnazTi3XAKgI8m7tUKWYNLACdWIB6xn1 VaxxNPgygaNxZbPv24YRGmXC9TWyOMflOFNoBQif05lIe4+grirAT5xh3/y5RRsrnZO8H3xzgs= Received: from [10.157.2.224] (HELO [192.168.200.1]) by cybernetics.com (CommuniGate Pro SMTP 7.1.1) with ESMTPS id 11859449; Tue, 07 Jun 2022 14:43:40 -0400 Message-ID: <568967ea-13a7-4a09-6846-0891032e6cfe@cybernetics.com> Date: Tue, 7 Jun 2022 14:43:39 -0400 MIME-Version: 1.0 User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:91.0) Gecko/20100101 Thunderbird/91.9.1 Subject: [PATCH v6 06/11] dmapool: debug: prevent endless loop in case of corruption Content-Language: en-US X-ASG-Orig-Subj: [PATCH v6 06/11] dmapool: debug: prevent endless loop in case of corruption From: Tony Battersby To: linux-mm@kvack.org, linux-kernel@vger.kernel.org Cc: iommu@lists.linux-foundation.org, kernel-team@fb.com, Matthew Wilcox , Keith Busch , Andy Shevchenko , Robin Murphy , Tony Lindgren References: <340ff8ef-9ff5-7175-c234-4132bbdfc5f7@cybernetics.com> In-Reply-To: <340ff8ef-9ff5-7175-c234-4132bbdfc5f7@cybernetics.com> Content-Transfer-Encoding: quoted-printable X-Barracuda-Connect: UNKNOWN[10.10.4.126] X-Barracuda-Start-Time: 1654627420 X-Barracuda-URL: https://10.10.4.122:443/cgi-mod/mark.cgi X-Barracuda-BRTS-Status: 1 X-Virus-Scanned: by bsmtpd at cybernetics.com X-Barracuda-Scan-Msg-Size: 1849 Precedence: bulk List-ID: X-Mailing-List: linux-kernel@vger.kernel.org Content-Type: text/plain; charset="utf-8" Prevent a possible endless loop with DMAPOOL_DEBUG enabled if a buggy driver corrupts DMA pool memory. Signed-off-by: Tony Battersby --- mm/dmapool.c | 37 ++++++++++++++++++++++++++++++------- 1 file changed, 30 insertions(+), 7 deletions(-) diff --git a/mm/dmapool.c b/mm/dmapool.c index d3e5a6151fb4..facdb3571976 100644 --- a/mm/dmapool.c +++ b/mm/dmapool.c @@ -417,16 +417,39 @@ void dma_pool_free(struct dma_pool *pool, void *vaddr= , dma_addr_t dma) } { unsigned int chain =3D page->offset; + unsigned int free_blks =3D 0; + while (chain < pool->allocation) { - if (chain !=3D offset) { - chain =3D *(int *)(page->vaddr + chain); - continue; + if (unlikely(chain =3D=3D offset)) { + spin_unlock_irqrestore(&pool->lock, flags); + dev_err(pool->dev, + "%s %s, dma %pad already free\n", + __func__, pool->name, &dma); + return; } - spin_unlock_irqrestore(&pool->lock, flags); - dev_err(pool->dev, "%s %s, dma %pad already free\n", - __func__, pool->name, &dma); - return; + + /* + * A buggy driver could corrupt the freelist by + * use-after-free, buffer overflow, etc. Besides + * checking for corruption, this also prevents an + * endless loop in case corruption causes a circular + * loop in the freelist. + */ + if (unlikely(++free_blks + page->in_use > + pool->blks_per_alloc)) { + freelist_corrupt: + spin_unlock_irqrestore(&pool->lock, flags); + dev_err(pool->dev, + "%s %s, freelist corrupted\n", + __func__, pool->name); + return; + } + + chain =3D *(int *)(page->vaddr + chain); } + if (unlikely(free_blks + page->in_use !=3D + pool->blks_per_alloc)) + goto freelist_corrupt; } memset(vaddr, POOL_POISON_FREED, pool->size); #endif --=20 2.25.1 From nobody Tue Apr 28 02:28:15 2026 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id 88661CCA483 for ; Tue, 7 Jun 2022 21:54:49 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1383203AbiFGVwk (ORCPT ); Tue, 7 Jun 2022 17:52:40 -0400 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:33134 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1377130AbiFGU5d (ORCPT ); Tue, 7 Jun 2022 16:57:33 -0400 Received: from mail.cybernetics.com (mail.cybernetics.com [173.71.130.66]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id 9C395201FF3 for ; Tue, 7 Jun 2022 11:44:21 -0700 (PDT) X-ASG-Debug-ID: 1654627459-1cf43917f3396680001-xx1T2L Received: from cybernetics.com ([10.10.4.126]) by mail.cybernetics.com with ESMTP id ytKMQdwzzCTN1huP; Tue, 07 Jun 2022 14:44:19 -0400 (EDT) X-Barracuda-Envelope-From: tonyb@cybernetics.com X-ASG-Whitelist: Client DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; d=cybernetics.com; s=mail; bh=ko90FEQrFc3kr/EXmZEbhsh++NujIvq49Z2hdvGupnI=; h=Content-Transfer-Encoding:Content-Type:In-Reply-To:References:Cc:To:From: Content-Language:Subject:MIME-Version:Date:Message-ID; b=WUGhF+zc7hcw6JoFhMVn sGaBwqMKrtT0LAsXqUcUj76iU06x+W8kn/yVArt8p4dHAKYUv4FvRp/bnqdKNITJ8eZ4h4+G4cdWa MiUopTVPkmkKCknVE11Zis2M2WKnboszBLWCavFSHLU31NMgSYjzMIvvxF7fzNbz4pg+2PoWvM= Received: from [10.157.2.224] (HELO [192.168.200.1]) by cybernetics.com (CommuniGate Pro SMTP 7.1.1) with ESMTPS id 11859456; Tue, 07 Jun 2022 14:44:19 -0400 Message-ID: <06f83492-1eaf-16c4-31b0-1f148995ee59@cybernetics.com> Date: Tue, 7 Jun 2022 14:44:18 -0400 MIME-Version: 1.0 User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:91.0) Gecko/20100101 Thunderbird/91.9.1 Subject: [PATCH v6 07/11] dmapool: ignore init_on_free when DMAPOOL_DEBUG enabled Content-Language: en-US X-ASG-Orig-Subj: [PATCH v6 07/11] dmapool: ignore init_on_free when DMAPOOL_DEBUG enabled From: Tony Battersby To: linux-mm@kvack.org, linux-kernel@vger.kernel.org Cc: iommu@lists.linux-foundation.org, kernel-team@fb.com, Matthew Wilcox , Keith Busch , Andy Shevchenko , Robin Murphy , Tony Lindgren References: <340ff8ef-9ff5-7175-c234-4132bbdfc5f7@cybernetics.com> In-Reply-To: <340ff8ef-9ff5-7175-c234-4132bbdfc5f7@cybernetics.com> Content-Transfer-Encoding: quoted-printable X-Barracuda-Connect: UNKNOWN[10.10.4.126] X-Barracuda-Start-Time: 1654627459 X-Barracuda-URL: https://10.10.4.122:443/cgi-mod/mark.cgi X-Barracuda-BRTS-Status: 1 X-Virus-Scanned: by bsmtpd at cybernetics.com X-Barracuda-Scan-Msg-Size: 1450 Precedence: bulk List-ID: X-Mailing-List: linux-kernel@vger.kernel.org Content-Type: text/plain; charset="utf-8" There are two cases: 1) In the normal case that the memory is being freed correctly, then DMAPOOL_DEBUG will memset the memory anyway, so speed thing up by avoiding a double-memset of the same memory. 2) In the abnormal case that DMAPOOL_DEBUG detects that a driver passes incorrect parameters to dma_pool_free() (e.g. double-free, invalid free, mismatched vaddr/dma, etc.), then that is a kernel bug, and we don't want to clear the passed-in possibly-invalid memory pointer because we can't be sure that the memory is really free. So don't clear it just because init_on_free=3D1. Signed-off-by: Tony Battersby --- mm/dmapool.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/mm/dmapool.c b/mm/dmapool.c index facdb3571976..44038089a41a 100644 --- a/mm/dmapool.c +++ b/mm/dmapool.c @@ -406,8 +406,6 @@ void dma_pool_free(struct dma_pool *pool, void *vaddr, = dma_addr_t dma) } =20 offset =3D vaddr - page->vaddr; - if (want_init_on_free()) - memset(vaddr, 0, pool->size); #ifdef DMAPOOL_DEBUG if ((dma - page->dma) !=3D offset) { spin_unlock_irqrestore(&pool->lock, flags); @@ -452,6 +450,9 @@ void dma_pool_free(struct dma_pool *pool, void *vaddr, = dma_addr_t dma) goto freelist_corrupt; } memset(vaddr, POOL_POISON_FREED, pool->size); +#else + if (want_init_on_free()) + memset(vaddr, 0, pool->size); #endif =20 page->in_use--; --=20 2.25.1 From nobody Tue Apr 28 02:28:15 2026 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id 6CDADCCA48C for ; Tue, 7 Jun 2022 21:54:50 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1383605AbiFGVxV (ORCPT ); Tue, 7 Jun 2022 17:53:21 -0400 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:45976 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1377506AbiFGU7n (ORCPT ); Tue, 7 Jun 2022 16:59:43 -0400 Received: from mail.cybernetics.com (mail.cybernetics.com [173.71.130.66]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id D6BB366C8A for ; Tue, 7 Jun 2022 11:45:00 -0700 (PDT) X-ASG-Debug-ID: 1654627498-1cf43917f33966c0001-xx1T2L Received: from cybernetics.com ([10.10.4.126]) by mail.cybernetics.com with ESMTP id QrVV0e8kIBE4Cwz7; Tue, 07 Jun 2022 14:44:58 -0400 (EDT) X-Barracuda-Envelope-From: tonyb@cybernetics.com X-ASG-Whitelist: Client DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; d=cybernetics.com; s=mail; bh=btLFiUXlrqByYR1DIyy+G6nrdsPdAmYyLow6+1WajEE=; h=Content-Transfer-Encoding:Content-Type:In-Reply-To:References:Cc:To:From: Content-Language:Subject:MIME-Version:Date:Message-ID; b=kMCkFAqaqzd846YJ+lfv P4h/NJYwHpSNVC1LQlF8sCQD9tdRu8eU6Qyj6nA3UeEKS+Q9SlxZQGnGHONgBLNRjIqoLyhLGMtJU PX5uv5AZczFROzSnYLENhV5Gvejwrk80VJPKT5Zp3M2xPaJ0IR+U/zBt+GENsvWIAsbrV1uxdI= Received: from [10.157.2.224] (HELO [192.168.200.1]) by cybernetics.com (CommuniGate Pro SMTP 7.1.1) with ESMTPS id 11859466; Tue, 07 Jun 2022 14:44:58 -0400 Message-ID: Date: Tue, 7 Jun 2022 14:44:58 -0400 MIME-Version: 1.0 User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:91.0) Gecko/20100101 Thunderbird/91.9.1 Subject: [PATCH v6 08/11] dmapool: speedup DMAPOOL_DEBUG with init_on_alloc Content-Language: en-US X-ASG-Orig-Subj: [PATCH v6 08/11] dmapool: speedup DMAPOOL_DEBUG with init_on_alloc From: Tony Battersby To: linux-mm@kvack.org, linux-kernel@vger.kernel.org Cc: iommu@lists.linux-foundation.org, kernel-team@fb.com, Matthew Wilcox , Keith Busch , Andy Shevchenko , Robin Murphy , Tony Lindgren References: <340ff8ef-9ff5-7175-c234-4132bbdfc5f7@cybernetics.com> In-Reply-To: <340ff8ef-9ff5-7175-c234-4132bbdfc5f7@cybernetics.com> Content-Transfer-Encoding: quoted-printable X-Barracuda-Connect: UNKNOWN[10.10.4.126] X-Barracuda-Start-Time: 1654627498 X-Barracuda-URL: https://10.10.4.122:443/cgi-mod/mark.cgi X-Barracuda-BRTS-Status: 1 X-Virus-Scanned: by bsmtpd at cybernetics.com X-Barracuda-Scan-Msg-Size: 688 Precedence: bulk List-ID: X-Mailing-List: linux-kernel@vger.kernel.org Content-Type: text/plain; charset="utf-8" Avoid double-memset of the same allocated memory in dma_pool_alloc() when both DMAPOOL_DEBUG is enabled and init_on_alloc=3D1. Signed-off-by: Tony Battersby --- mm/dmapool.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mm/dmapool.c b/mm/dmapool.c index 44038089a41a..c7ec38cb4631 100644 --- a/mm/dmapool.c +++ b/mm/dmapool.c @@ -356,7 +356,7 @@ void *dma_pool_alloc(struct dma_pool *pool, gfp_t mem_f= lags, break; } } - if (!(mem_flags & __GFP_ZERO)) + if (!want_init_on_alloc(mem_flags)) memset(retval, POOL_POISON_ALLOCATED, pool->size); #endif spin_unlock_irqrestore(&pool->lock, flags); --=20 2.25.1 From nobody Tue Apr 28 02:28:15 2026 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id 53D6ECCA48A for ; Tue, 7 Jun 2022 22:02:26 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1383354AbiFGWBv (ORCPT ); Tue, 7 Jun 2022 18:01:51 -0400 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:50450 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1379058AbiFGVCA (ORCPT ); Tue, 7 Jun 2022 17:02:00 -0400 Received: from mail.cybernetics.com (mail.cybernetics.com [173.71.130.66]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id 44DEA188E66 for ; Tue, 7 Jun 2022 11:46:17 -0700 (PDT) X-ASG-Debug-ID: 1654627544-1cf43917f3396710001-xx1T2L Received: from cybernetics.com ([10.10.4.126]) by mail.cybernetics.com with ESMTP id u3h3PzNcv3GISwVG; Tue, 07 Jun 2022 14:45:44 -0400 (EDT) X-Barracuda-Envelope-From: tonyb@cybernetics.com X-ASG-Whitelist: Client DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; d=cybernetics.com; s=mail; bh=fJQpcCRHMP3bElV7sUkA+QTTFc8Tk97S5Mqvw1QDnTQ=; h=Content-Transfer-Encoding:Content-Type:In-Reply-To:References:Cc:To:From: Content-Language:Subject:MIME-Version:Date:Message-ID; b=ghina6onuUS2ov/P3hH9 L5dEdD+6sGEHGT4vP9jtWoQ5eCxd6dCBxtPoExAZw2K5opBuq2H/Z4oCWx5J1HX8r2nziqL09VDv3 7e7fCgDlngbrr1wSqKCea76X8QbWpcNpautqqHUnpKW3+01/yk5/iaywpvmAI7fEdg/W5S6oPo= Received: from [10.157.2.224] (HELO [192.168.200.1]) by cybernetics.com (CommuniGate Pro SMTP 7.1.1) with ESMTPS id 11859468; Tue, 07 Jun 2022 14:45:44 -0400 Message-ID: <11b7e3cd-c929-654c-1f45-1abe9a6f6a98@cybernetics.com> Date: Tue, 7 Jun 2022 14:45:44 -0400 MIME-Version: 1.0 User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:91.0) Gecko/20100101 Thunderbird/91.9.1 Subject: [PATCH v6 09/11] dmapool: cleanup dma_pool_destroy Content-Language: en-US X-ASG-Orig-Subj: [PATCH v6 09/11] dmapool: cleanup dma_pool_destroy From: Tony Battersby To: linux-mm@kvack.org, linux-kernel@vger.kernel.org Cc: iommu@lists.linux-foundation.org, kernel-team@fb.com, Matthew Wilcox , Keith Busch , Andy Shevchenko , Robin Murphy , Tony Lindgren References: <340ff8ef-9ff5-7175-c234-4132bbdfc5f7@cybernetics.com> In-Reply-To: <340ff8ef-9ff5-7175-c234-4132bbdfc5f7@cybernetics.com> Content-Transfer-Encoding: quoted-printable X-Barracuda-Connect: UNKNOWN[10.10.4.126] X-Barracuda-Start-Time: 1654627544 X-Barracuda-URL: https://10.10.4.122:443/cgi-mod/mark.cgi X-Barracuda-BRTS-Status: 1 X-Virus-Scanned: by bsmtpd at cybernetics.com X-Barracuda-Scan-Msg-Size: 2336 Precedence: bulk List-ID: X-Mailing-List: linux-kernel@vger.kernel.org Content-Type: text/plain; charset="utf-8" pool_free_page() is called only from dma_pool_destroy(), so inline it and make it less generic since we know that the pool is being destroyed. Signed-off-by: Tony Battersby --- Changes since v5: Take the opposite approach and inline pool_free_page() into dma_pool_destroy() instead of moving the is_page_busy() check into pool_free_page(). mm/dmapool.c | 33 ++++++++++++++------------------- 1 file changed, 14 insertions(+), 19 deletions(-) diff --git a/mm/dmapool.c b/mm/dmapool.c index c7ec38cb4631..4e075feb038f 100644 --- a/mm/dmapool.c +++ b/mm/dmapool.c @@ -241,18 +241,6 @@ static inline bool is_page_busy(struct dma_page *page) return page->in_use !=3D 0; } =20 -static void pool_free_page(struct dma_pool *pool, struct dma_page *page) -{ - dma_addr_t dma =3D page->dma; - -#ifdef DMAPOOL_DEBUG - memset(page->vaddr, POOL_POISON_FREED, pool->allocation); -#endif - dma_free_coherent(pool->dev, pool->allocation, page->vaddr, dma); - list_del(&page->page_list); - kfree(page); -} - /** * dma_pool_destroy - destroys a pool of dma memory blocks. * @pool: dma pool that will be destroyed @@ -280,14 +268,22 @@ void dma_pool_destroy(struct dma_pool *pool) mutex_unlock(&pools_reg_lock); =20 list_for_each_entry_safe(page, tmp, &pool->page_list, page_list) { + void *vaddr =3D page->vaddr; + if (is_page_busy(page)) { dev_err(pool->dev, "%s %s, %p busy\n", __func__, - pool->name, page->vaddr); + pool->name, vaddr); /* leak the still-in-use consistent memory */ - list_del(&page->page_list); - kfree(page); - } else - pool_free_page(pool, page); + } else { +#ifdef DMAPOOL_DEBUG + memset(vaddr, POOL_POISON_FREED, pool->allocation); +#endif + dma_free_coherent(pool->dev, + pool->allocation, + vaddr, + page->dma); + } + kfree(page); } =20 kfree(pool); @@ -459,8 +455,7 @@ void dma_pool_free(struct dma_pool *pool, void *vaddr, = dma_addr_t dma) *(int *)vaddr =3D page->offset; page->offset =3D offset; /* - * Resist a temptation to do - * if (!is_page_busy(page)) pool_free_page(pool, page); + * Resist a temptation to free unused pages immediately. * Better have a few empty pages hang around. */ spin_unlock_irqrestore(&pool->lock, flags); --=20 2.25.1 From nobody Tue Apr 28 02:28:15 2026 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id 5904CCCA496 for ; Tue, 7 Jun 2022 21:54:51 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1384205AbiFGVyQ (ORCPT ); Tue, 7 Jun 2022 17:54:16 -0400 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:45732 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1379078AbiFGVCB (ORCPT ); Tue, 7 Jun 2022 17:02:01 -0400 Received: from mail.cybernetics.com (mail.cybernetics.com [173.71.130.66]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id 194E711CB68 for ; Tue, 7 Jun 2022 11:46:25 -0700 (PDT) X-ASG-Debug-ID: 1654627583-1cf43917f3396720001-xx1T2L Received: from cybernetics.com ([10.10.4.126]) by mail.cybernetics.com with ESMTP id OBWPxyFZLrQFHE0J; Tue, 07 Jun 2022 14:46:23 -0400 (EDT) X-Barracuda-Envelope-From: tonyb@cybernetics.com X-ASG-Whitelist: Client DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; d=cybernetics.com; s=mail; bh=vcQ5hy/KPkqucbXNbQAJCyvSrO4lyJAHF32hpadfWSw=; h=Content-Transfer-Encoding:Content-Type:In-Reply-To:References:Cc:To:From: Content-Language:Subject:MIME-Version:Date:Message-ID; b=FibAY+bNXDPJpThVea4/ cKatgfJ/9zkZVzgmJpUaNWz4GlH3d1tP684DpenycawhOTl2oXD9dhnIwNmsEm1uhbqoud6Yi7oNt VP+IFIP5LnFsThvuOOC5tFSnFAO0UGzS6gbIQCs6YkRXa4wDmtwWjguWZhXVGIhkw5eOeUBHVQ= Received: from [10.157.2.224] (HELO [192.168.200.1]) by cybernetics.com (CommuniGate Pro SMTP 7.1.1) with ESMTPS id 11859479; Tue, 07 Jun 2022 14:46:23 -0400 Message-ID: <6a4fb3c3-e627-6266-7c49-322253abefb9@cybernetics.com> Date: Tue, 7 Jun 2022 14:46:23 -0400 MIME-Version: 1.0 User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:91.0) Gecko/20100101 Thunderbird/91.9.1 Subject: [PATCH v6 10/11] dmapool: improve scalability of dma_pool_alloc Content-Language: en-US X-ASG-Orig-Subj: [PATCH v6 10/11] dmapool: improve scalability of dma_pool_alloc From: Tony Battersby To: linux-mm@kvack.org, linux-kernel@vger.kernel.org Cc: iommu@lists.linux-foundation.org, kernel-team@fb.com, Matthew Wilcox , Keith Busch , Andy Shevchenko , Robin Murphy , Tony Lindgren References: <340ff8ef-9ff5-7175-c234-4132bbdfc5f7@cybernetics.com> In-Reply-To: <340ff8ef-9ff5-7175-c234-4132bbdfc5f7@cybernetics.com> Content-Transfer-Encoding: quoted-printable X-Barracuda-Connect: UNKNOWN[10.10.4.126] X-Barracuda-Start-Time: 1654627583 X-Barracuda-URL: https://10.10.4.122:443/cgi-mod/mark.cgi X-Barracuda-BRTS-Status: 1 X-Virus-Scanned: by bsmtpd at cybernetics.com X-Barracuda-Scan-Msg-Size: 3491 Precedence: bulk List-ID: X-Mailing-List: linux-kernel@vger.kernel.org Content-Type: text/plain; charset="utf-8" dma_pool_alloc() scales poorly when allocating a large number of pages because it does a linear scan of all previously-allocated pages before allocating a new one. Improve its scalability by maintaining a separate list of pages that have free blocks ready to (re)allocate. In big O notation, this improves the algorithm from O(n) to O(1). Signed-off-by: Tony Battersby --- Changes since v5: pool_free_page() no longer exists. Updated big O usage in description. mm/dmapool.c | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/mm/dmapool.c b/mm/dmapool.c index 4e075feb038f..fc9ae0683c20 100644 --- a/mm/dmapool.c +++ b/mm/dmapool.c @@ -17,6 +17,10 @@ * least 'size' bytes. Free blocks are tracked in an unsorted singly-link= ed * list of free blocks within the page. Used blocks aren't tracked, but we * keep a count of how many are currently allocated from each page. + * + * The avail_page_list keeps track of pages that have one or more free blo= cks + * available to (re)allocate. Pages are moved in and out of avail_page_li= st + * as their blocks are allocated and freed. */ =20 #include @@ -42,6 +46,7 @@ =20 struct dma_pool { /* the pool */ struct list_head page_list; + struct list_head avail_page_list; spinlock_t lock; struct device *dev; unsigned int size; @@ -54,6 +59,7 @@ struct dma_pool { /* the pool */ =20 struct dma_page { /* cacheable header for 'allocation' bytes */ struct list_head page_list; + struct list_head avail_page_link; void *vaddr; dma_addr_t dma; unsigned int in_use; @@ -155,6 +161,7 @@ struct dma_pool *dma_pool_create(const char *name, stru= ct device *dev, retval->dev =3D dev; =20 INIT_LIST_HEAD(&retval->page_list); + INIT_LIST_HEAD(&retval->avail_page_list); spin_lock_init(&retval->lock); retval->size =3D size; retval->boundary =3D boundary; @@ -311,10 +318,11 @@ void *dma_pool_alloc(struct dma_pool *pool, gfp_t mem= _flags, might_alloc(mem_flags); =20 spin_lock_irqsave(&pool->lock, flags); - list_for_each_entry(page, &pool->page_list, page_list) { - if (page->offset < pool->allocation) - goto ready; - } + page =3D list_first_entry_or_null(&pool->avail_page_list, + struct dma_page, + avail_page_link); + if (page) + goto ready; =20 /* pool_alloc_page() might sleep, so temporarily drop &pool->lock */ spin_unlock_irqrestore(&pool->lock, flags); @@ -326,10 +334,13 @@ void *dma_pool_alloc(struct dma_pool *pool, gfp_t mem= _flags, spin_lock_irqsave(&pool->lock, flags); =20 list_add(&page->page_list, &pool->page_list); + list_add(&page->avail_page_link, &pool->avail_page_list); ready: page->in_use++; offset =3D page->offset; page->offset =3D *(int *)(page->vaddr + offset); + if (page->offset >=3D pool->allocation) + list_del_init(&page->avail_page_link); retval =3D offset + page->vaddr; *handle =3D offset + page->dma; #ifdef DMAPOOL_DEBUG @@ -451,6 +462,13 @@ void dma_pool_free(struct dma_pool *pool, void *vaddr,= dma_addr_t dma) memset(vaddr, 0, pool->size); #endif =20 + /* + * list_empty() on the page tests if the page is already linked into + * avail_page_list to avoid adding it more than once. + */ + if (list_empty(&page->avail_page_link)) + list_add(&page->avail_page_link, &pool->avail_page_list); + page->in_use--; *(int *)vaddr =3D page->offset; page->offset =3D offset; --=20 2.25.1 From nobody Tue Apr 28 02:28:15 2026 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id 14796CCA48F for ; Tue, 7 Jun 2022 21:56:17 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1381864AbiFGVzZ (ORCPT ); Tue, 7 Jun 2022 17:55:25 -0400 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:49304 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1379220AbiFGVCM (ORCPT ); Tue, 7 Jun 2022 17:02:12 -0400 Received: from mail.cybernetics.com (mail.cybernetics.com [173.71.130.66]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id EE0C6CF6 for ; Tue, 7 Jun 2022 11:46:57 -0700 (PDT) X-ASG-Debug-ID: 1654627616-1cf43917f3396750001-xx1T2L Received: from cybernetics.com ([10.10.4.126]) by mail.cybernetics.com with ESMTP id lD8Qa5y0EHhIboul; Tue, 07 Jun 2022 14:46:56 -0400 (EDT) X-Barracuda-Envelope-From: tonyb@cybernetics.com X-ASG-Whitelist: Client DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; d=cybernetics.com; s=mail; bh=pJSQmtU0P/ap6gGJCAtyT1g5eLDr6t020b+qVsQwBhU=; h=Content-Transfer-Encoding:Content-Type:In-Reply-To:References:Cc:To:From: Content-Language:Subject:MIME-Version:Date:Message-ID; b=qCDPKbR6fpfRS/1Vyedw kTg7EJHrmL7z6wkQxrxRv9Y0AuZm1QBKl4pPfefiTru6k4i96wB02QvZHalD5mJ7MJoniBZXXkpXE tFyia+Oz5EZ1sTcIlW10puXby/VlExrNDod72YpRfKjh2Jlf7nPD4cHk4p2PAqwMnqpLgCQYVg= Received: from [10.157.2.224] (HELO [192.168.200.1]) by cybernetics.com (CommuniGate Pro SMTP 7.1.1) with ESMTPS id 11859489; Tue, 07 Jun 2022 14:46:56 -0400 Message-ID: <65b3425b-989e-25ed-9e0d-18122e529a35@cybernetics.com> Date: Tue, 7 Jun 2022 14:46:56 -0400 MIME-Version: 1.0 User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:91.0) Gecko/20100101 Thunderbird/91.9.1 Subject: [PATCH v6 11/11] dmapool: improve scalability of dma_pool_free Content-Language: en-US X-ASG-Orig-Subj: [PATCH v6 11/11] dmapool: improve scalability of dma_pool_free From: Tony Battersby To: linux-mm@kvack.org, linux-kernel@vger.kernel.org Cc: iommu@lists.linux-foundation.org, kernel-team@fb.com, Matthew Wilcox , Keith Busch , Andy Shevchenko , Robin Murphy , Tony Lindgren References: <340ff8ef-9ff5-7175-c234-4132bbdfc5f7@cybernetics.com> In-Reply-To: <340ff8ef-9ff5-7175-c234-4132bbdfc5f7@cybernetics.com> Content-Transfer-Encoding: quoted-printable X-Barracuda-Connect: UNKNOWN[10.10.4.126] X-Barracuda-Start-Time: 1654627616 X-Barracuda-URL: https://10.10.4.122:443/cgi-mod/mark.cgi X-Barracuda-BRTS-Status: 1 X-Virus-Scanned: by bsmtpd at cybernetics.com X-Barracuda-Scan-Msg-Size: 7050 Precedence: bulk List-ID: X-Mailing-List: linux-kernel@vger.kernel.org Content-Type: text/plain; charset="utf-8" dma_pool_free() scales poorly when the pool contains many pages because pool_find_page() does a linear scan of all allocated pages. Improve its scalability by replacing the linear scan with a red-black tree lookup. In big O notation, this improves the algorithm from O(n) to O(log n). Signed-off-by: Tony Battersby --- Changes since v5: pool_free_page() no longer exists. Less churn in dma_pool_destroy(). Updated big O usage in description. mm/dmapool.c | 114 ++++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 90 insertions(+), 24 deletions(-) diff --git a/mm/dmapool.c b/mm/dmapool.c index fc9ae0683c20..31102a00fa7c 100644 --- a/mm/dmapool.c +++ b/mm/dmapool.c @@ -12,11 +12,12 @@ * Many older drivers still have their own code to do this. * * The current design of this allocator is fairly simple. The pool is - * represented by the 'struct dma_pool' which keeps a doubly-linked list of - * allocated pages. Each page in the page_list is split into blocks of at - * least 'size' bytes. Free blocks are tracked in an unsorted singly-link= ed - * list of free blocks within the page. Used blocks aren't tracked, but we - * keep a count of how many are currently allocated from each page. + * represented by the 'struct dma_pool' which keeps a red-black tree of all + * allocated pages, keyed by DMA address for fast lookup when freeing. + * Each page in the page_tree is split into blocks of at least 'size' byte= s. + * Free blocks are tracked in an unsorted singly-linked list of free blocks + * within the page. Used blocks aren't tracked, but we keep a count of how + * many are currently allocated from each page. * * The avail_page_list keeps track of pages that have one or more free blo= cks * available to (re)allocate. Pages are moved in and out of avail_page_li= st @@ -36,6 +37,7 @@ #include #include #include +#include #include #include #include @@ -45,7 +47,7 @@ #endif =20 struct dma_pool { /* the pool */ - struct list_head page_list; + struct rb_root page_tree; struct list_head avail_page_list; spinlock_t lock; struct device *dev; @@ -58,7 +60,7 @@ struct dma_pool { /* the pool */ }; =20 struct dma_page { /* cacheable header for 'allocation' bytes */ - struct list_head page_list; + struct rb_node page_node; struct list_head avail_page_link; void *vaddr; dma_addr_t dma; @@ -69,11 +71,17 @@ struct dma_page { /* cacheable header for 'allocation'= bytes */ static DEFINE_MUTEX(pools_lock); static DEFINE_MUTEX(pools_reg_lock); =20 +static inline struct dma_page *rb_to_dma_page(struct rb_node *node) +{ + return rb_entry(node, struct dma_page, page_node); +} + static ssize_t pools_show(struct device *dev, struct device_attribute *att= r, char *buf) { int size; struct dma_page *page; struct dma_pool *pool; + struct rb_node *node; =20 size =3D sysfs_emit(buf, "poolinfo - 0.1\n"); =20 @@ -83,7 +91,10 @@ static ssize_t pools_show(struct device *dev, struct dev= ice_attribute *attr, cha size_t blocks =3D 0; =20 spin_lock_irq(&pool->lock); - list_for_each_entry(page, &pool->page_list, page_list) { + for (node =3D rb_first(&pool->page_tree); + node; + node =3D rb_next(node)) { + page =3D rb_to_dma_page(node); pages++; blocks +=3D page->in_use; } @@ -160,7 +171,7 @@ struct dma_pool *dma_pool_create(const char *name, stru= ct device *dev, =20 retval->dev =3D dev; =20 - INIT_LIST_HEAD(&retval->page_list); + retval->page_tree =3D RB_ROOT; INIT_LIST_HEAD(&retval->avail_page_list); spin_lock_init(&retval->lock); retval->size =3D size; @@ -204,6 +215,63 @@ struct dma_pool *dma_pool_create(const char *name, str= uct device *dev, } EXPORT_SYMBOL(dma_pool_create); =20 +/* + * Find the dma_page that manages the given DMA address. + */ +static struct dma_page *pool_find_page(struct dma_pool *pool, dma_addr_t d= ma) +{ + struct rb_node *node =3D pool->page_tree.rb_node; + + while (node) { + struct dma_page *page =3D rb_to_dma_page(node); + + if (dma < page->dma) + node =3D node->rb_left; + else if ((dma - page->dma) >=3D pool->allocation) + node =3D node->rb_right; + else + return page; + } + return NULL; +} + +/* + * Insert a dma_page into the page_tree. + */ +static int pool_insert_page(struct dma_pool *pool, struct dma_page *new_pa= ge) +{ + dma_addr_t dma =3D new_page->dma; + struct rb_node **node =3D &(pool->page_tree.rb_node), *parent =3D NULL; + + while (*node) { + struct dma_page *this_page =3D rb_to_dma_page(*node); + + parent =3D *node; + if (dma < this_page->dma) + node =3D &((*node)->rb_left); + else if (likely((dma - this_page->dma) >=3D pool->allocation)) + node =3D &((*node)->rb_right); + else { + /* + * A page that overlaps the new DMA range is already + * present in the tree. This should not happen. + */ + WARN(1, + "%s: %s: DMA address overlap: old %pad new %pad len %u\n", + dev_name(pool->dev), + pool->name, &this_page->dma, &dma, + pool->allocation); + return -1; + } + } + + /* Add new node and rebalance tree. */ + rb_link_node(&new_page->page_node, parent, node); + rb_insert_color(&new_page->page_node, &pool->page_tree); + + return 0; +} + static void pool_initialise_page(struct dma_pool *pool, struct dma_page *p= age) { unsigned int offset =3D 0; @@ -274,7 +342,10 @@ void dma_pool_destroy(struct dma_pool *pool) device_remove_file(pool->dev, &dev_attr_pools); mutex_unlock(&pools_reg_lock); =20 - list_for_each_entry_safe(page, tmp, &pool->page_list, page_list) { + rbtree_postorder_for_each_entry_safe(page, + tmp, + &pool->page_tree, + page_node) { void *vaddr =3D page->vaddr; =20 if (is_page_busy(page)) { @@ -333,7 +404,15 @@ void *dma_pool_alloc(struct dma_pool *pool, gfp_t mem_= flags, =20 spin_lock_irqsave(&pool->lock, flags); =20 - list_add(&page->page_list, &pool->page_list); + if (unlikely(pool_insert_page(pool, page))) { + /* + * This should not happen, so something must have gone horribly + * wrong. Instead of crashing, intentionally leak the memory + * and make for the exit. + */ + spin_unlock_irqrestore(&pool->lock, flags); + return NULL; + } list_add(&page->avail_page_link, &pool->avail_page_list); ready: page->in_use++; @@ -375,19 +454,6 @@ void *dma_pool_alloc(struct dma_pool *pool, gfp_t mem_= flags, } EXPORT_SYMBOL(dma_pool_alloc); =20 -static struct dma_page *pool_find_page(struct dma_pool *pool, dma_addr_t d= ma) -{ - struct dma_page *page; - - list_for_each_entry(page, &pool->page_list, page_list) { - if (dma < page->dma) - continue; - if ((dma - page->dma) < pool->allocation) - return page; - } - return NULL; -} - /** * dma_pool_free - put block back into dma pool * @pool: the dma pool holding the block --=20 2.25.1