From nobody Thu Apr 30 08:12:14 2026 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id E8564C433F5 for ; Tue, 31 May 2022 18:12:39 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1346847AbiEaSMi (ORCPT ); Tue, 31 May 2022 14:12:38 -0400 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:54456 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S234110AbiEaSMf (ORCPT ); Tue, 31 May 2022 14:12:35 -0400 X-Greylist: delayed 69 seconds by postgrey-1.37 at lindbergh.monkeyblade.net; Tue, 31 May 2022 11:12:32 PDT Received: from mail.cybernetics.com (mail.cybernetics.com [173.71.130.66]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id 69D683BBE5 for ; Tue, 31 May 2022 11:12:29 -0700 (PDT) X-ASG-Debug-ID: 1654020747-1cf43917f334afd0001-xx1T2L Received: from cybernetics.com ([10.10.4.126]) by mail.cybernetics.com with ESMTP id KkLxI1f9KEPaUEs3; Tue, 31 May 2022 14:12:27 -0400 (EDT) X-Barracuda-Envelope-From: tonyb@cybernetics.com X-ASG-Whitelist: Client DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; d=cybernetics.com; s=mail; bh=rbu53iA56EgPM9xWDcYjPsiC8+GQnnX8Zrf5JmbXf/U=; h=Content-Transfer-Encoding:Content-Type:In-Reply-To:References:Cc:To:From: Content-Language:Subject:MIME-Version:Date:Message-ID; b=MLxPLW10G4ATiUw5UYmx t78zeovkkrHB3/myd4m5l4OI/fZWQFzn7qyXTKUPDmIdClw0OPbpFdbxl6a1IjoezSDDbHOd4+quj 9nKyBRjVcAeJ7AsvtIMU/2MvrJwgBQe6xiU9thrtItTyCQrp31Wt8yCATFW8TrVFv8J9qgPiug= Received: from [10.157.2.224] (HELO [192.168.200.1]) by cybernetics.com (CommuniGate Pro SMTP 7.1.1) with ESMTPS id 11829186; Tue, 31 May 2022 14:12:27 -0400 Message-ID: <7f6f9ff5-cdb9-e386-988d-fa013538dee7@cybernetics.com> Date: Tue, 31 May 2022 14:12:27 -0400 MIME-Version: 1.0 User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:91.0) Gecko/20100101 Thunderbird/91.9.1 Subject: [PATCH 01/10] dmapool: remove checks for dev == NULL Content-Language: en-US X-ASG-Orig-Subj: [PATCH 01/10] dmapool: remove checks for dev == NULL From: Tony Battersby To: linux-mm@kvack.org, linux-kernel@vger.kernel.org Cc: iommu@lists.linux-foundation.org, kernel-team@fb.com, Matthew Wilcox , Keith Busch , Andy Shevchenko , Robin Murphy , Tony Lindgren References: <9b08ab7c-b80b-527d-9adf-7716b0868fbc@cybernetics.com> In-Reply-To: <9b08ab7c-b80b-527d-9adf-7716b0868fbc@cybernetics.com> Content-Transfer-Encoding: quoted-printable X-Barracuda-Connect: UNKNOWN[10.10.4.126] X-Barracuda-Start-Time: 1654020747 X-Barracuda-URL: https://10.10.4.122:443/cgi-mod/mark.cgi X-Barracuda-BRTS-Status: 1 X-Virus-Scanned: by bsmtpd at cybernetics.com X-Barracuda-Scan-Msg-Size: 3586 Precedence: bulk List-ID: X-Mailing-List: linux-kernel@vger.kernel.org Content-Type: text/plain; charset="utf-8" dmapool originally tried to support pools without a device because dma_alloc_coherent() supports allocations without a device. But nobody ended up using dma pools without a device, so the current checks in dmapool.c for pool->dev =3D=3D NULL are both insufficient and causing bloat. Remove them. Signed-off-by: Tony Battersby Reviewed-by: Robin Murphy --- mm/dmapool.c | 42 +++++++++++------------------------------- 1 file changed, 11 insertions(+), 31 deletions(-) diff --git a/mm/dmapool.c b/mm/dmapool.c index a7eb5d0eb2da..0f89de408cbe 100644 --- a/mm/dmapool.c +++ b/mm/dmapool.c @@ -275,7 +275,7 @@ void dma_pool_destroy(struct dma_pool *pool) mutex_lock(&pools_reg_lock); mutex_lock(&pools_lock); list_del(&pool->pools); - if (pool->dev && list_empty(&pool->dev->dma_pools)) + if (list_empty(&pool->dev->dma_pools)) empty =3D true; mutex_unlock(&pools_lock); if (empty) @@ -284,12 +284,8 @@ void dma_pool_destroy(struct dma_pool *pool) =20 list_for_each_entry_safe(page, tmp, &pool->page_list, page_list) { if (is_page_busy(page)) { - if (pool->dev) - dev_err(pool->dev, "%s %s, %p busy\n", __func__, - pool->name, page->vaddr); - else - pr_err("%s %s, %p busy\n", __func__, - pool->name, page->vaddr); + dev_err(pool->dev, "%s %s, %p busy\n", __func__, + pool->name, page->vaddr); /* leak the still-in-use consistent memory */ list_del(&page->page_list); kfree(page); @@ -351,12 +347,8 @@ void *dma_pool_alloc(struct dma_pool *pool, gfp_t mem_= flags, for (i =3D sizeof(page->offset); i < pool->size; i++) { if (data[i] =3D=3D POOL_POISON_FREED) continue; - if (pool->dev) - dev_err(pool->dev, "%s %s, %p (corrupted)\n", - __func__, pool->name, retval); - else - pr_err("%s %s, %p (corrupted)\n", - __func__, pool->name, retval); + dev_err(pool->dev, "%s %s, %p (corrupted)\n", + __func__, pool->name, retval); =20 /* * Dump the first 4 bytes even if they are not @@ -411,12 +403,8 @@ void dma_pool_free(struct dma_pool *pool, void *vaddr,= dma_addr_t dma) page =3D pool_find_page(pool, dma); if (!page) { spin_unlock_irqrestore(&pool->lock, flags); - if (pool->dev) - dev_err(pool->dev, "%s %s, %p/%pad (bad dma)\n", - __func__, pool->name, vaddr, &dma); - else - pr_err("%s %s, %p/%pad (bad dma)\n", - __func__, pool->name, vaddr, &dma); + dev_err(pool->dev, "%s %s, %p/%pad (bad dma)\n", + __func__, pool->name, vaddr, &dma); return; } =20 @@ -426,12 +414,8 @@ void dma_pool_free(struct dma_pool *pool, void *vaddr,= dma_addr_t dma) #ifdef DMAPOOL_DEBUG if ((dma - page->dma) !=3D offset) { spin_unlock_irqrestore(&pool->lock, flags); - if (pool->dev) - dev_err(pool->dev, "%s %s, %p (bad vaddr)/%pad\n", - __func__, pool->name, vaddr, &dma); - else - pr_err("%s %s, %p (bad vaddr)/%pad\n", - __func__, pool->name, vaddr, &dma); + dev_err(pool->dev, "%s %s, %p (bad vaddr)/%pad\n", + __func__, pool->name, vaddr, &dma); return; } { @@ -442,12 +426,8 @@ void dma_pool_free(struct dma_pool *pool, void *vaddr,= dma_addr_t dma) continue; } spin_unlock_irqrestore(&pool->lock, flags); - if (pool->dev) - dev_err(pool->dev, "%s %s, dma %pad already free\n", - __func__, pool->name, &dma); - else - pr_err("%s %s, dma %pad already free\n", - __func__, pool->name, &dma); + dev_err(pool->dev, "%s %s, dma %pad already free\n", + __func__, pool->name, &dma); return; } } --=20 2.25.1 From nobody Thu Apr 30 08:12:14 2026 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id 68849C433F5 for ; Tue, 31 May 2022 18:14:02 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1346923AbiEaSOA (ORCPT ); Tue, 31 May 2022 14:14:00 -0400 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:59376 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1346882AbiEaSNv (ORCPT ); Tue, 31 May 2022 14:13:51 -0400 Received: from mail.cybernetics.com (mail.cybernetics.com [173.71.130.66]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id 6B1469CC99 for ; Tue, 31 May 2022 11:13:34 -0700 (PDT) X-ASG-Debug-ID: 1654020812-1cf43917f334afe0001-xx1T2L Received: from cybernetics.com ([10.10.4.126]) by mail.cybernetics.com with ESMTP id nHifSkbHIxrhwGDp; Tue, 31 May 2022 14:13:32 -0400 (EDT) X-Barracuda-Envelope-From: tonyb@cybernetics.com X-ASG-Whitelist: Client DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; d=cybernetics.com; s=mail; bh=FrnayQTLRg3bImjySCMn4AK6vTiCgnyfnm+pKLSvhx0=; h=Content-Transfer-Encoding:Content-Type:In-Reply-To:References:Cc:To:From: Content-Language:Subject:MIME-Version:Date:Message-ID; b=G9I1y0Cm5SI5a20B9p45 Ch/vtXr3T2oFr+6ys40VeK3ubfORXd8KFUN5jCE/6JUWAIy1cBsSzzxHDqoXYj3XE0YBtRWEEJxSV VLobdcVMmmCaDWU+U0KnUIuyfrTSUdqRMkNdomm+YVy9APQXHE5/7HR7dEPHZYkExGOv45asZE= Received: from [10.157.2.224] (HELO [192.168.200.1]) by cybernetics.com (CommuniGate Pro SMTP 7.1.1) with ESMTPS id 11829190; Tue, 31 May 2022 14:13:32 -0400 Message-ID: <81004e69-d91f-9fbb-2b94-217b48a064c3@cybernetics.com> Date: Tue, 31 May 2022 14:13:32 -0400 MIME-Version: 1.0 User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:91.0) Gecko/20100101 Thunderbird/91.9.1 Subject: [PATCH 02/10] dmapool: cleanup integer types Content-Language: en-US X-ASG-Orig-Subj: [PATCH 02/10] dmapool: cleanup integer types From: Tony Battersby To: linux-mm@kvack.org, linux-kernel@vger.kernel.org Cc: iommu@lists.linux-foundation.org, kernel-team@fb.com, Matthew Wilcox , Keith Busch , Andy Shevchenko , Robin Murphy , Tony Lindgren References: <9b08ab7c-b80b-527d-9adf-7716b0868fbc@cybernetics.com> In-Reply-To: <9b08ab7c-b80b-527d-9adf-7716b0868fbc@cybernetics.com> Content-Transfer-Encoding: quoted-printable X-Barracuda-Connect: UNKNOWN[10.10.4.126] X-Barracuda-Start-Time: 1654020812 X-Barracuda-URL: https://10.10.4.122:443/cgi-mod/mark.cgi X-Barracuda-BRTS-Status: 1 X-Virus-Scanned: by bsmtpd at cybernetics.com X-Barracuda-Scan-Msg-Size: 3356 Precedence: bulk List-ID: X-Mailing-List: linux-kernel@vger.kernel.org Content-Type: text/plain; charset="utf-8" To represent the size of a single allocation, dmapool currently uses 'unsigned int' in some places and 'size_t' in other places. Standardize on 'unsigned int' to reduce overhead, but use 'size_t' when counting all the blocks in the entire pool. Signed-off-by: Tony Battersby --- This puts an upper bound on 'size' of INT_MAX to avoid overflowing the following comparison in pool_initialise_page(): unsigned int offset =3D 0; unsigned int next =3D offset + pool->size; if (unlikely((next + pool->size) > ... 'boundary' is passed in as a size_t but gets stored as an unsigned int. 'boundary' values >=3D 'allocation' do not have any effect, so clipping 'boundary' to 'allocation' keeps it within the range of unsigned int without affecting anything else. A few lines above (not in the diff) you can see that if 'boundary' is passed in as 0 then it is set to 'allocation', so it is nothing new. For reference, here is the relevant code after being patched: if (!boundary) boundary =3D allocation; else if ((boundary < size) || (boundary & (boundary - 1))) return NULL; boundary =3D min(boundary, allocation); mm/dmapool.c | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/mm/dmapool.c b/mm/dmapool.c index 0f89de408cbe..d7b372248111 100644 --- a/mm/dmapool.c +++ b/mm/dmapool.c @@ -43,10 +43,10 @@ struct dma_pool { /* the pool */ struct list_head page_list; spinlock_t lock; - size_t size; + unsigned int size; struct device *dev; - size_t allocation; - size_t boundary; + unsigned int allocation; + unsigned int boundary; char name[32]; struct list_head pools; }; @@ -80,7 +80,7 @@ static ssize_t pools_show(struct device *dev, struct devi= ce_attribute *attr, cha mutex_lock(&pools_lock); list_for_each_entry(pool, &dev->dma_pools, pools) { unsigned pages =3D 0; - unsigned blocks =3D 0; + size_t blocks =3D 0; =20 spin_lock_irq(&pool->lock); list_for_each_entry(page, &pool->page_list, page_list) { @@ -90,9 +90,10 @@ static ssize_t pools_show(struct device *dev, struct dev= ice_attribute *attr, cha spin_unlock_irq(&pool->lock); =20 /* per-pool info, no real statistics yet */ - temp =3D scnprintf(next, size, "%-16s %4u %4zu %4zu %2u\n", + temp =3D scnprintf(next, size, "%-16s %4zu %4zu %4u %2u\n", pool->name, blocks, - pages * (pool->allocation / pool->size), + (size_t) pages * + (pool->allocation / pool->size), pool->size, pages); size -=3D temp; next +=3D temp; @@ -139,7 +140,7 @@ struct dma_pool *dma_pool_create(const char *name, stru= ct device *dev, else if (align & (align - 1)) return NULL; =20 - if (size =3D=3D 0) + if (size =3D=3D 0 || size > INT_MAX) return NULL; else if (size < 4) size =3D 4; @@ -152,6 +153,8 @@ struct dma_pool *dma_pool_create(const char *name, stru= ct device *dev, else if ((boundary < size) || (boundary & (boundary - 1))) return NULL; =20 + boundary =3D min(boundary, allocation); + retval =3D kmalloc(sizeof(*retval), GFP_KERNEL); if (!retval) return retval; @@ -312,7 +315,7 @@ void *dma_pool_alloc(struct dma_pool *pool, gfp_t mem_f= lags, { unsigned long flags; struct dma_page *page; - size_t offset; + unsigned int offset; void *retval; =20 might_alloc(mem_flags); --=20 2.25.1 From nobody Thu Apr 30 08:12:14 2026 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id 7AFBAC433EF for ; Tue, 31 May 2022 18:14:34 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1344562AbiEaSOd (ORCPT ); Tue, 31 May 2022 14:14:33 -0400 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:33930 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1346879AbiEaSO0 (ORCPT ); Tue, 31 May 2022 14:14:26 -0400 Received: from mail.cybernetics.com (mail.cybernetics.com [173.71.130.66]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id B71FF8CCCE for ; Tue, 31 May 2022 11:14:25 -0700 (PDT) X-ASG-Debug-ID: 1654020863-1cf43917f334b000001-xx1T2L Received: from cybernetics.com ([10.10.4.126]) by mail.cybernetics.com with ESMTP id oOy2yfF6AsqWuHgq; Tue, 31 May 2022 14:14:23 -0400 (EDT) X-Barracuda-Envelope-From: tonyb@cybernetics.com X-ASG-Whitelist: Client DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; d=cybernetics.com; s=mail; bh=RasPjcgW3MbjOYmk499B8lRQ6/y64Pgdc8ibAu0hQ94=; h=Content-Transfer-Encoding:Content-Type:In-Reply-To:References:Cc:To:From: Content-Language:Subject:MIME-Version:Date:Message-ID; b=Bt5vfluu/fRwhcrHUI+Q RP31wdoXrw1Y6q/RDhu8jjMvd8Xv/prWojQ2sBb93UI9yWCDk8+z3hJRHrqRB9Kyl/P33NDljwTGT 6LP6dpwARPHNZ8W9RmxQn7ynBzURXdzx83Lam9kFm87wr8w+LJVJi4f0pmJ+NCuSFMASSmTsmw= Received: from [10.157.2.224] (HELO [192.168.200.1]) by cybernetics.com (CommuniGate Pro SMTP 7.1.1) with ESMTPS id 11829197; Tue, 31 May 2022 14:14:23 -0400 Message-ID: Date: Tue, 31 May 2022 14:14:23 -0400 MIME-Version: 1.0 User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:91.0) Gecko/20100101 Thunderbird/91.9.1 Subject: [PATCH 03/10] dmapool: fix boundary comparison Content-Language: en-US X-ASG-Orig-Subj: [PATCH 03/10] dmapool: fix boundary comparison From: Tony Battersby To: linux-mm@kvack.org, linux-kernel@vger.kernel.org Cc: iommu@lists.linux-foundation.org, kernel-team@fb.com, Matthew Wilcox , Keith Busch , Andy Shevchenko , Robin Murphy , Tony Lindgren References: <9b08ab7c-b80b-527d-9adf-7716b0868fbc@cybernetics.com> In-Reply-To: <9b08ab7c-b80b-527d-9adf-7716b0868fbc@cybernetics.com> Content-Transfer-Encoding: quoted-printable X-Barracuda-Connect: UNKNOWN[10.10.4.126] X-Barracuda-Start-Time: 1654020863 X-Barracuda-URL: https://10.10.4.122:443/cgi-mod/mark.cgi X-Barracuda-BRTS-Status: 1 X-Virus-Scanned: by bsmtpd at cybernetics.com X-Barracuda-Scan-Msg-Size: 1388 Precedence: bulk List-ID: X-Mailing-List: linux-kernel@vger.kernel.org Content-Type: text/plain; charset="utf-8" Fix the boundary comparison when constructing the list of free blocks for the case that 'size' is a power of two. Since 'boundary' is also a power of two, that would make 'boundary' a multiple of 'size', in which case a single block would never cross the boundary. This bug would cause some of the allocated memory to be wasted (but not leaked). Example: size =3D 512 boundary =3D 2048 allocation =3D 4096 Address range 0 - 511 512 - 1023 1024 - 1535 1536 - 2047 * 2048 - 2559 2560 - 3071 3072 - 3583 3584 - 4095 * Prior to this fix, the address ranges marked with "*" would not have been used even though they didn't cross the given boundary. Fixes: e34f44b3517f ("pool: Improve memory usage for devices which can't cr= oss boundaries") Signed-off-by: Tony Battersby --- mm/dmapool.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mm/dmapool.c b/mm/dmapool.c index d7b372248111..782143144a32 100644 --- a/mm/dmapool.c +++ b/mm/dmapool.c @@ -210,7 +210,7 @@ static void pool_initialise_page(struct dma_pool *pool,= struct dma_page *page) =20 do { unsigned int next =3D offset + pool->size; - if (unlikely((next + pool->size) >=3D next_boundary)) { + if (unlikely((next + pool->size) > next_boundary)) { next =3D next_boundary; next_boundary +=3D pool->boundary; } --=20 2.25.1 From nobody Thu Apr 30 08:12:14 2026 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id A7EDCC433FE for ; Tue, 31 May 2022 18:17:56 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1346934AbiEaSRy (ORCPT ); Tue, 31 May 2022 14:17:54 -0400 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:42836 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1346918AbiEaSRs (ORCPT ); Tue, 31 May 2022 14:17:48 -0400 Received: from mail.cybernetics.com (mail.cybernetics.com [173.71.130.66]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id 80E3E8CCFA for ; Tue, 31 May 2022 11:17:46 -0700 (PDT) X-ASG-Debug-ID: 1654021064-1cf43917f334b060001-xx1T2L Received: from cybernetics.com ([10.10.4.126]) by mail.cybernetics.com with ESMTP id FnUZ9qI4R7alRl3A; Tue, 31 May 2022 14:17:44 -0400 (EDT) X-Barracuda-Envelope-From: tonyb@cybernetics.com X-ASG-Whitelist: Client DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; d=cybernetics.com; s=mail; bh=rVEbYmgCF5qNaIoY0xv9QyqFa+k5xItzdRlzexwASwY=; h=Content-Transfer-Encoding:Content-Type:In-Reply-To:References:Cc:To:From: Content-Language:Subject:MIME-Version:Date:Message-ID; b=PXGrGGD2CLPjjbPjqwhg zGiqsmvhXj07rY/Ga/zD8aEmocIXs+Ic868AT/FzjqouS9izAh8h4wg0ckuZd+Ja1jlF7EjM0eYd3 mlXd+sZ5m0ONdtjtAziIP1j0Set+ooCEjaMO/cnHIX+CZIE7D8h6dIEr0gMQUBlyfSMfftPsDw= Received: from [10.157.2.224] (HELO [192.168.200.1]) by cybernetics.com (CommuniGate Pro SMTP 7.1.1) with ESMTPS id 11829214; Tue, 31 May 2022 14:17:44 -0400 Message-ID: Date: Tue, 31 May 2022 14:17:44 -0400 MIME-Version: 1.0 User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:91.0) Gecko/20100101 Thunderbird/91.9.1 Subject: [PATCH 04/10] dmapool: improve accuracy of debug statistics Content-Language: en-US X-ASG-Orig-Subj: [PATCH 04/10] dmapool: improve accuracy of debug statistics From: Tony Battersby To: linux-mm@kvack.org, linux-kernel@vger.kernel.org Cc: iommu@lists.linux-foundation.org, kernel-team@fb.com, Matthew Wilcox , Keith Busch , Andy Shevchenko , Robin Murphy , Tony Lindgren References: <9b08ab7c-b80b-527d-9adf-7716b0868fbc@cybernetics.com> In-Reply-To: <9b08ab7c-b80b-527d-9adf-7716b0868fbc@cybernetics.com> Content-Transfer-Encoding: quoted-printable X-Barracuda-Connect: UNKNOWN[10.10.4.126] X-Barracuda-Start-Time: 1654021064 X-Barracuda-URL: https://10.10.4.122:443/cgi-mod/mark.cgi X-Barracuda-BRTS-Status: 1 X-Virus-Scanned: by bsmtpd at cybernetics.com X-Barracuda-Scan-Msg-Size: 1718 Precedence: bulk List-ID: X-Mailing-List: linux-kernel@vger.kernel.org Content-Type: text/plain; charset="utf-8" The "total number of blocks in pool" debug statistic currently does not take the boundary value into account, so it diverges from the "total number of blocks in use" statistic when a boundary is in effect. Add a calculation for the number of blocks per allocation that takes the boundary into account, and use it to replace the inaccurate calculation. This depends on the patch "dmapool: fix boundary comparison" for the calculated blks_per_alloc value to be correct. Signed-off-by: Tony Battersby --- mm/dmapool.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/mm/dmapool.c b/mm/dmapool.c index 782143144a32..9e30f4425dea 100644 --- a/mm/dmapool.c +++ b/mm/dmapool.c @@ -47,6 +47,7 @@ struct dma_pool { /* the pool */ struct device *dev; unsigned int allocation; unsigned int boundary; + unsigned int blks_per_alloc; char name[32]; struct list_head pools; }; @@ -92,8 +93,7 @@ static ssize_t pools_show(struct device *dev, struct devi= ce_attribute *attr, cha /* per-pool info, no real statistics yet */ temp =3D scnprintf(next, size, "%-16s %4zu %4zu %4u %2u\n", pool->name, blocks, - (size_t) pages * - (pool->allocation / pool->size), + (size_t) pages * pool->blks_per_alloc, pool->size, pages); size -=3D temp; next +=3D temp; @@ -168,6 +168,9 @@ struct dma_pool *dma_pool_create(const char *name, stru= ct device *dev, retval->size =3D size; retval->boundary =3D boundary; retval->allocation =3D allocation; + retval->blks_per_alloc =3D + (allocation / boundary) * (boundary / size) + + (allocation % boundary) / size; =20 INIT_LIST_HEAD(&retval->pools); =20 --=20 2.25.1 From nobody Thu Apr 30 08:12:14 2026 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id 4CE36C433EF for ; Tue, 31 May 2022 18:18:51 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S242951AbiEaSSt (ORCPT ); Tue, 31 May 2022 14:18:49 -0400 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:47798 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1346947AbiEaSSo (ORCPT ); Tue, 31 May 2022 14:18:44 -0400 Received: from mail.cybernetics.com (mail.cybernetics.com [173.71.130.66]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id 5AE0B8DDE8 for ; Tue, 31 May 2022 11:18:42 -0700 (PDT) X-ASG-Debug-ID: 1654021121-1cf43917f334b070001-xx1T2L Received: from cybernetics.com ([10.10.4.126]) by mail.cybernetics.com with ESMTP id yj29GjpIIDRwn4BW; Tue, 31 May 2022 14:18:41 -0400 (EDT) X-Barracuda-Envelope-From: tonyb@cybernetics.com X-ASG-Whitelist: Client DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; d=cybernetics.com; s=mail; bh=Z2Ky3BhmVZCP6nQJsAzLenh1k2tIAiuPv7c6Ywt4xp0=; h=Content-Transfer-Encoding:Content-Type:In-Reply-To:References:Cc:To:From: Content-Language:Subject:MIME-Version:Date:Message-ID; b=l4gmGdtEEc475y+cPS0i b3CXKfRcWn4bzL+nlsGSrpLAqCAgnrQN4vO8XeOkunQPpE6QaJJS41Y/Ya9CMBv8n21a+SHmXaAVC XqS+12dpa3XgoD+yv6qzm0y6xH8gkXnhke2W5TRMUbJSCpDTjZ3EVBmeUajwr+X7dja0fJwqU8= Received: from [10.157.2.224] (HELO [192.168.200.1]) by cybernetics.com (CommuniGate Pro SMTP 7.1.1) with ESMTPS id 11829213; Tue, 31 May 2022 14:18:41 -0400 Message-ID: <0c6c1548-6e3a-0d8d-4bb7-471fdfb403ca@cybernetics.com> Date: Tue, 31 May 2022 14:18:41 -0400 MIME-Version: 1.0 User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:91.0) Gecko/20100101 Thunderbird/91.9.1 Subject: [PATCH 05/10] dmapool: debug: prevent endless loop in case of corruption Content-Language: en-US X-ASG-Orig-Subj: [PATCH 05/10] dmapool: debug: prevent endless loop in case of corruption From: Tony Battersby To: linux-mm@kvack.org, linux-kernel@vger.kernel.org Cc: iommu@lists.linux-foundation.org, kernel-team@fb.com, Matthew Wilcox , Keith Busch , Andy Shevchenko , Robin Murphy , Tony Lindgren References: <9b08ab7c-b80b-527d-9adf-7716b0868fbc@cybernetics.com> In-Reply-To: <9b08ab7c-b80b-527d-9adf-7716b0868fbc@cybernetics.com> Content-Transfer-Encoding: quoted-printable X-Barracuda-Connect: UNKNOWN[10.10.4.126] X-Barracuda-Start-Time: 1654021121 X-Barracuda-URL: https://10.10.4.122:443/cgi-mod/mark.cgi X-Barracuda-BRTS-Status: 1 X-Virus-Scanned: by bsmtpd at cybernetics.com X-Barracuda-Scan-Msg-Size: 1849 Precedence: bulk List-ID: X-Mailing-List: linux-kernel@vger.kernel.org Content-Type: text/plain; charset="utf-8" Prevent a possible endless loop with DMAPOOL_DEBUG enabled if a buggy driver corrupts DMA pool memory. Signed-off-by: Tony Battersby --- mm/dmapool.c | 37 ++++++++++++++++++++++++++++++------- 1 file changed, 30 insertions(+), 7 deletions(-) diff --git a/mm/dmapool.c b/mm/dmapool.c index 9e30f4425dea..7a9161d4f7a6 100644 --- a/mm/dmapool.c +++ b/mm/dmapool.c @@ -426,16 +426,39 @@ void dma_pool_free(struct dma_pool *pool, void *vaddr= , dma_addr_t dma) } { unsigned int chain =3D page->offset; + unsigned int free_blks =3D 0; + while (chain < pool->allocation) { - if (chain !=3D offset) { - chain =3D *(int *)(page->vaddr + chain); - continue; + if (unlikely(chain =3D=3D offset)) { + spin_unlock_irqrestore(&pool->lock, flags); + dev_err(pool->dev, + "%s %s, dma %pad already free\n", + __func__, pool->name, &dma); + return; } - spin_unlock_irqrestore(&pool->lock, flags); - dev_err(pool->dev, "%s %s, dma %pad already free\n", - __func__, pool->name, &dma); - return; + + /* + * A buggy driver could corrupt the freelist by + * use-after-free, buffer overflow, etc. Besides + * checking for corruption, this also prevents an + * endless loop in case corruption causes a circular + * loop in the freelist. + */ + if (unlikely(++free_blks + page->in_use > + pool->blks_per_alloc)) { + freelist_corrupt: + spin_unlock_irqrestore(&pool->lock, flags); + dev_err(pool->dev, + "%s %s, freelist corrupted\n", + __func__, pool->name); + return; + } + + chain =3D *(int *)(page->vaddr + chain); } + if (unlikely(free_blks + page->in_use !=3D + pool->blks_per_alloc)) + goto freelist_corrupt; } memset(vaddr, POOL_POISON_FREED, pool->size); #endif --=20 2.25.1 From nobody Thu Apr 30 08:12:14 2026 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id 24A23C433F5 for ; Tue, 31 May 2022 18:21:05 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1344968AbiEaSVE (ORCPT ); Tue, 31 May 2022 14:21:04 -0400 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:52090 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1346991AbiEaSUy (ORCPT ); Tue, 31 May 2022 14:20:54 -0400 Received: from mail.cybernetics.com (mail.cybernetics.com [173.71.130.66]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id CE2779C2DF for ; Tue, 31 May 2022 11:20:53 -0700 (PDT) X-ASG-Debug-ID: 1654021252-1cf43917f334b0a0001-xx1T2L Received: from cybernetics.com ([10.10.4.126]) by mail.cybernetics.com with ESMTP id ozEi06WftSCDC9Dn; Tue, 31 May 2022 14:20:52 -0400 (EDT) X-Barracuda-Envelope-From: tonyb@cybernetics.com X-ASG-Whitelist: Client DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; d=cybernetics.com; s=mail; bh=UMD3peLOU8xJDRQl6tZsqYH8OxWwTPFkkNKZO4VgqaY=; h=Content-Transfer-Encoding:Content-Type:In-Reply-To:References:Cc:To:From: Content-Language:Subject:MIME-Version:Date:Message-ID; b=rydeBuE75zkiJLNnZRfb /WOveUINnngDKbuBAnssoipJio6AJ8XrL7lA7d2NMt/+12EgQmGf8wBBwWfKyMEadPRL7VtaKsi4T JRDTXjFqiek64nTQq0wNQegdcxjMhWIPs7A+BZAqIBPh7j1EN9i6chqiPkEMvl7Zyb2O2XsC+w= Received: from [10.157.2.224] (HELO [192.168.200.1]) by cybernetics.com (CommuniGate Pro SMTP 7.1.1) with ESMTPS id 11829220; Tue, 31 May 2022 14:20:52 -0400 Message-ID: Date: Tue, 31 May 2022 14:20:51 -0400 MIME-Version: 1.0 User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:91.0) Gecko/20100101 Thunderbird/91.9.1 Subject: [PATCH 06/10] dmapool: ignore init_on_free when DMAPOOL_DEBUG enabled Content-Language: en-US X-ASG-Orig-Subj: [PATCH 06/10] dmapool: ignore init_on_free when DMAPOOL_DEBUG enabled From: Tony Battersby To: linux-mm@kvack.org, linux-kernel@vger.kernel.org Cc: iommu@lists.linux-foundation.org, kernel-team@fb.com, Matthew Wilcox , Keith Busch , Andy Shevchenko , Robin Murphy , Tony Lindgren References: <9b08ab7c-b80b-527d-9adf-7716b0868fbc@cybernetics.com> In-Reply-To: <9b08ab7c-b80b-527d-9adf-7716b0868fbc@cybernetics.com> Content-Transfer-Encoding: quoted-printable X-Barracuda-Connect: UNKNOWN[10.10.4.126] X-Barracuda-Start-Time: 1654021252 X-Barracuda-URL: https://10.10.4.122:443/cgi-mod/mark.cgi X-Barracuda-BRTS-Status: 1 X-Virus-Scanned: by bsmtpd at cybernetics.com X-Barracuda-Scan-Msg-Size: 1451 Precedence: bulk List-ID: X-Mailing-List: linux-kernel@vger.kernel.org Content-Type: text/plain; charset="utf-8" There are two cases: 1) In the normal case that the memory is being freed correctly, then DMAPOOL_DEBUG will memset the memory anyway, so speed things up by avoiding a double-memset of the same memory. 2) In the abnormal case that DMAPOOL_DEBUG detects that a driver passes incorrect parameters to dma_pool_free() (e.g. double-free, invalid free, mismatched vaddr/dma, etc.), then that is a kernel bug, and we don't want to clear the passed-in possibly-invalid memory pointer because we can't be sure that the memory is really free. So don't clear it just because init_on_free=3D1. Signed-off-by: Tony Battersby --- mm/dmapool.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/mm/dmapool.c b/mm/dmapool.c index 7a9161d4f7a6..49019ef6dd83 100644 --- a/mm/dmapool.c +++ b/mm/dmapool.c @@ -415,8 +415,6 @@ void dma_pool_free(struct dma_pool *pool, void *vaddr, = dma_addr_t dma) } =20 offset =3D vaddr - page->vaddr; - if (want_init_on_free()) - memset(vaddr, 0, pool->size); #ifdef DMAPOOL_DEBUG if ((dma - page->dma) !=3D offset) { spin_unlock_irqrestore(&pool->lock, flags); @@ -461,6 +459,9 @@ void dma_pool_free(struct dma_pool *pool, void *vaddr, = dma_addr_t dma) goto freelist_corrupt; } memset(vaddr, POOL_POISON_FREED, pool->size); +#else + if (want_init_on_free()) + memset(vaddr, 0, pool->size); #endif =20 page->in_use--; --=20 2.25.1 From nobody Thu Apr 30 08:12:14 2026 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id A3D0BC433FE for ; Tue, 31 May 2022 18:21:42 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1347004AbiEaSVl (ORCPT ); Tue, 31 May 2022 14:21:41 -0400 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:52882 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1346916AbiEaSVi (ORCPT ); Tue, 31 May 2022 14:21:38 -0400 Received: from mail.cybernetics.com (mail.cybernetics.com [173.71.130.66]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id 9D4059CC9C for ; Tue, 31 May 2022 11:21:37 -0700 (PDT) X-ASG-Debug-ID: 1654021295-1cf43917f334b0b0001-xx1T2L Received: from cybernetics.com ([10.10.4.126]) by mail.cybernetics.com with ESMTP id QlerWlG95wKKsGNK; Tue, 31 May 2022 14:21:35 -0400 (EDT) X-Barracuda-Envelope-From: tonyb@cybernetics.com X-ASG-Whitelist: Client DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; d=cybernetics.com; s=mail; bh=oESBKLtZD0XfE8ixha9kfGkjSenTdPQVKS9D5YrpgSM=; h=Content-Transfer-Encoding:Content-Type:In-Reply-To:References:Cc:To:From: Content-Language:Subject:MIME-Version:Date:Message-ID; b=E06FpEHrfSATsbsioMp8 Ae89/HwwmRl53JAltDNa/4r2t995qyZVnUrG5PTWufTxHMM9HZasP0FlsIdfwNZD3wSKdGWvfSF4w 0Sx3tUnjdQ1AdRBSGzvjqUkh0hQq84PRsu5g+uKPf3Cw9WgT6Itnr8UU18IZBHv8nQ52Q5CCMM= Received: from [10.157.2.224] (HELO [192.168.200.1]) by cybernetics.com (CommuniGate Pro SMTP 7.1.1) with ESMTPS id 11829227; Tue, 31 May 2022 14:21:35 -0400 Message-ID: <35eeaddc-b27c-aee7-8c0f-96afcb2858d5@cybernetics.com> Date: Tue, 31 May 2022 14:21:35 -0400 MIME-Version: 1.0 User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:91.0) Gecko/20100101 Thunderbird/91.9.1 Subject: [PATCH 07/10] dmapool: speedup DMAPOOL_DEBUG with init_on_alloc Content-Language: en-US X-ASG-Orig-Subj: [PATCH 07/10] dmapool: speedup DMAPOOL_DEBUG with init_on_alloc From: Tony Battersby To: linux-mm@kvack.org, linux-kernel@vger.kernel.org Cc: iommu@lists.linux-foundation.org, kernel-team@fb.com, Matthew Wilcox , Keith Busch , Andy Shevchenko , Robin Murphy , Tony Lindgren References: <9b08ab7c-b80b-527d-9adf-7716b0868fbc@cybernetics.com> In-Reply-To: <9b08ab7c-b80b-527d-9adf-7716b0868fbc@cybernetics.com> Content-Transfer-Encoding: quoted-printable X-Barracuda-Connect: UNKNOWN[10.10.4.126] X-Barracuda-Start-Time: 1654021295 X-Barracuda-URL: https://10.10.4.122:443/cgi-mod/mark.cgi X-Barracuda-BRTS-Status: 1 X-Virus-Scanned: by bsmtpd at cybernetics.com X-Barracuda-Scan-Msg-Size: 688 Precedence: bulk List-ID: X-Mailing-List: linux-kernel@vger.kernel.org Content-Type: text/plain; charset="utf-8" Avoid double-memset of the same allocated memory in dma_pool_alloc() when both DMAPOOL_DEBUG is enabled and init_on_alloc=3D1. Signed-off-by: Tony Battersby --- mm/dmapool.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mm/dmapool.c b/mm/dmapool.c index 49019ef6dd83..8749a9d7927e 100644 --- a/mm/dmapool.c +++ b/mm/dmapool.c @@ -365,7 +365,7 @@ void *dma_pool_alloc(struct dma_pool *pool, gfp_t mem_f= lags, break; } } - if (!(mem_flags & __GFP_ZERO)) + if (!want_init_on_alloc(mem_flags)) memset(retval, POOL_POISON_ALLOCATED, pool->size); #endif spin_unlock_irqrestore(&pool->lock, flags); --=20 2.25.1 From nobody Thu Apr 30 08:12:14 2026 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id 37427C433F5 for ; Tue, 31 May 2022 18:22:29 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1347008AbiEaSW0 (ORCPT ); Tue, 31 May 2022 14:22:26 -0400 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:54456 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S244594AbiEaSWY (ORCPT ); Tue, 31 May 2022 14:22:24 -0400 Received: from mail.cybernetics.com (mail.cybernetics.com [173.71.130.66]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id 23BD07A81D for ; Tue, 31 May 2022 11:22:22 -0700 (PDT) X-ASG-Debug-ID: 1654021341-1cf43917f334b0c0001-xx1T2L Received: from cybernetics.com ([10.10.4.126]) by mail.cybernetics.com with ESMTP id ZzsyGX4QF38BVlyE; Tue, 31 May 2022 14:22:21 -0400 (EDT) X-Barracuda-Envelope-From: tonyb@cybernetics.com X-ASG-Whitelist: Client DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; d=cybernetics.com; s=mail; bh=eQIejJQOI/tAoaluQWJF6F3pwzd0w/dyu0/0oFwF/ME=; h=Content-Transfer-Encoding:Content-Type:In-Reply-To:References:Cc:To:From: Content-Language:Subject:MIME-Version:Date:Message-ID; b=F5jnWTUFSe+/KQ5AraCn x2M3sgS5D7KBF0kTbxgh7aTkSikU4O+wLG1/eYe2hQgBrdXZpcZLc+QHWFzhFiJ92GFCKNWUEd5+3 oUhgNCdJEQ4T0fqGo8bOwBceIA6Bh8pHjcMOaKZj21/jCvAUBq7twmACoj8fHfQ7+QWinqXjTs= Received: from [10.157.2.224] (HELO [192.168.200.1]) by cybernetics.com (CommuniGate Pro SMTP 7.1.1) with ESMTPS id 11829234; Tue, 31 May 2022 14:22:21 -0400 Message-ID: <30fd23ae-7035-5ce3-5643-89a5956f1e79@cybernetics.com> Date: Tue, 31 May 2022 14:22:21 -0400 MIME-Version: 1.0 User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:91.0) Gecko/20100101 Thunderbird/91.9.1 Subject: [PATCH 08/10] dmapool: cleanup dma_pool_destroy Content-Language: en-US X-ASG-Orig-Subj: [PATCH 08/10] dmapool: cleanup dma_pool_destroy From: Tony Battersby To: linux-mm@kvack.org, linux-kernel@vger.kernel.org Cc: iommu@lists.linux-foundation.org, kernel-team@fb.com, Matthew Wilcox , Keith Busch , Andy Shevchenko , Robin Murphy , Tony Lindgren References: <9b08ab7c-b80b-527d-9adf-7716b0868fbc@cybernetics.com> In-Reply-To: <9b08ab7c-b80b-527d-9adf-7716b0868fbc@cybernetics.com> Content-Transfer-Encoding: quoted-printable X-Barracuda-Connect: UNKNOWN[10.10.4.126] X-Barracuda-Start-Time: 1654021341 X-Barracuda-URL: https://10.10.4.122:443/cgi-mod/mark.cgi X-Virus-Scanned: by bsmtpd at cybernetics.com X-Barracuda-Scan-Msg-Size: 2711 X-Barracuda-BRTS-Status: 1 Precedence: bulk List-ID: X-Mailing-List: linux-kernel@vger.kernel.org Content-Type: text/plain; charset="utf-8" Remove a small amount of code duplication between dma_pool_destroy() and pool_free_page() in preparation for adding more code without having to duplicate it. No functional changes. Signed-off-by: Tony Battersby --- mm/dmapool.c | 34 ++++++++++++++++++++-------------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/mm/dmapool.c b/mm/dmapool.c index 8749a9d7927e..58c11dcaa4e4 100644 --- a/mm/dmapool.c +++ b/mm/dmapool.c @@ -250,14 +250,25 @@ static inline bool is_page_busy(struct dma_page *page) return page->in_use !=3D 0; } =20 -static void pool_free_page(struct dma_pool *pool, struct dma_page *page) +static void pool_free_page(struct dma_pool *pool, + struct dma_page *page, + bool destroying_pool) { + void *vaddr =3D page->vaddr; dma_addr_t dma =3D page->dma; =20 + if (destroying_pool && is_page_busy(page)) { + dev_err(pool->dev, + "dma_pool_destroy %s, %p busy\n", + pool->name, vaddr); + /* leak the still-in-use consistent memory */ + } else { #ifdef DMAPOOL_DEBUG - memset(page->vaddr, POOL_POISON_FREED, pool->allocation); + memset(vaddr, POOL_POISON_FREED, pool->allocation); #endif - dma_free_coherent(pool->dev, pool->allocation, page->vaddr, dma); + dma_free_coherent(pool->dev, pool->allocation, vaddr, dma); + } + list_del(&page->page_list); kfree(page); } @@ -272,7 +283,7 @@ static void pool_free_page(struct dma_pool *pool, struc= t dma_page *page) */ void dma_pool_destroy(struct dma_pool *pool) { - struct dma_page *page, *tmp; + struct dma_page *page; bool empty =3D false; =20 if (unlikely(!pool)) @@ -288,15 +299,10 @@ void dma_pool_destroy(struct dma_pool *pool) device_remove_file(pool->dev, &dev_attr_pools); mutex_unlock(&pools_reg_lock); =20 - list_for_each_entry_safe(page, tmp, &pool->page_list, page_list) { - if (is_page_busy(page)) { - dev_err(pool->dev, "%s %s, %p busy\n", __func__, - pool->name, page->vaddr); - /* leak the still-in-use consistent memory */ - list_del(&page->page_list); - kfree(page); - } else - pool_free_page(pool, page); + while ((page =3D list_first_entry_or_null(&pool->page_list, + struct dma_page, + page_list))) { + pool_free_page(pool, page, true); } =20 kfree(pool); @@ -469,7 +475,7 @@ void dma_pool_free(struct dma_pool *pool, void *vaddr, = dma_addr_t dma) page->offset =3D offset; /* * Resist a temptation to do - * if (!is_page_busy(page)) pool_free_page(pool, page); + * if (!is_page_busy(page)) pool_free_page(pool, page, false); * Better have a few empty pages hang around. */ spin_unlock_irqrestore(&pool->lock, flags); --=20 2.25.1 From nobody Thu Apr 30 08:12:14 2026 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id DBCF6C433FE for ; Tue, 31 May 2022 18:23:08 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1346984AbiEaSXH (ORCPT ); Tue, 31 May 2022 14:23:07 -0400 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:54872 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S234905AbiEaSXG (ORCPT ); Tue, 31 May 2022 14:23:06 -0400 Received: from mail.cybernetics.com (mail.cybernetics.com [173.71.130.66]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id 8808D2D8 for ; Tue, 31 May 2022 11:23:04 -0700 (PDT) X-ASG-Debug-ID: 1654021382-1cf43917f334b0e0001-xx1T2L Received: from cybernetics.com ([10.10.4.126]) by mail.cybernetics.com with ESMTP id rbOwRDxhxvyi8U8j; Tue, 31 May 2022 14:23:02 -0400 (EDT) X-Barracuda-Envelope-From: tonyb@cybernetics.com X-ASG-Whitelist: Client DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; d=cybernetics.com; s=mail; bh=DOnD3hVWfzzlQtOH7c2ljAgtJzaUt05vjMFxkgGY0NM=; h=Content-Transfer-Encoding:Content-Type:In-Reply-To:References:Cc:To:From: Content-Language:Subject:MIME-Version:Date:Message-ID; b=q/GVT243covpjwcmjMW5 z4a6gNMYZabMfljvc8LnFzU6VBDffzO74DsNJcFVzgVVWgtszVkbPOdj7dQVe8BH9hQSlkbXhFiKm x/uNZmiZQcoW7bKGHC3BqfdQw5ypKnuPYBbR9W0R5xk2kzXdVLhFDQXi2vNknYGNjdMVKjqrCY= Received: from [10.157.2.224] (HELO [192.168.200.1]) by cybernetics.com (CommuniGate Pro SMTP 7.1.1) with ESMTPS id 11829238; Tue, 31 May 2022 14:23:02 -0400 Message-ID: Date: Tue, 31 May 2022 14:23:02 -0400 MIME-Version: 1.0 User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:91.0) Gecko/20100101 Thunderbird/91.9.1 Subject: [PATCH 09/10] dmapool: improve scalability of dma_pool_alloc Content-Language: en-US X-ASG-Orig-Subj: [PATCH 09/10] dmapool: improve scalability of dma_pool_alloc From: Tony Battersby To: linux-mm@kvack.org, linux-kernel@vger.kernel.org Cc: iommu@lists.linux-foundation.org, kernel-team@fb.com, Matthew Wilcox , Keith Busch , Andy Shevchenko , Robin Murphy , Tony Lindgren References: <9b08ab7c-b80b-527d-9adf-7716b0868fbc@cybernetics.com> In-Reply-To: <9b08ab7c-b80b-527d-9adf-7716b0868fbc@cybernetics.com> Content-Transfer-Encoding: quoted-printable X-Barracuda-Connect: UNKNOWN[10.10.4.126] X-Barracuda-Start-Time: 1654021382 X-Barracuda-URL: https://10.10.4.122:443/cgi-mod/mark.cgi X-Barracuda-BRTS-Status: 1 X-Virus-Scanned: by bsmtpd at cybernetics.com X-Barracuda-Scan-Msg-Size: 3568 Precedence: bulk List-ID: X-Mailing-List: linux-kernel@vger.kernel.org Content-Type: text/plain; charset="utf-8" dma_pool_alloc() scales poorly when allocating a large number of pages because it does a linear scan of all previously-allocated pages before allocating a new one. Improve its scalability by maintaining a separate list of pages that have free blocks ready to (re)allocate. In big O notation, this improves the algorithm from O(n^2) to O(n). Signed-off-by: Tony Battersby --- mm/dmapool.c | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/mm/dmapool.c b/mm/dmapool.c index 58c11dcaa4e4..b3dd2ace0d2a 100644 --- a/mm/dmapool.c +++ b/mm/dmapool.c @@ -17,6 +17,10 @@ * least 'size' bytes. Free blocks are tracked in an unsorted singly-link= ed * list of free blocks within the page. Used blocks aren't tracked, but we * keep a count of how many are currently allocated from each page. + * + * The avail_page_list keeps track of pages that have one or more free blo= cks + * available to (re)allocate. Pages are moved in and out of avail_page_li= st + * as their blocks are allocated and freed. */ =20 #include @@ -42,6 +46,7 @@ =20 struct dma_pool { /* the pool */ struct list_head page_list; + struct list_head avail_page_list; spinlock_t lock; unsigned int size; struct device *dev; @@ -54,6 +59,7 @@ struct dma_pool { /* the pool */ =20 struct dma_page { /* cacheable header for 'allocation' bytes */ struct list_head page_list; + struct list_head avail_page_link; void *vaddr; dma_addr_t dma; unsigned int in_use; @@ -164,6 +170,7 @@ struct dma_pool *dma_pool_create(const char *name, stru= ct device *dev, retval->dev =3D dev; =20 INIT_LIST_HEAD(&retval->page_list); + INIT_LIST_HEAD(&retval->avail_page_list); spin_lock_init(&retval->lock); retval->size =3D size; retval->boundary =3D boundary; @@ -270,6 +277,7 @@ static void pool_free_page(struct dma_pool *pool, } =20 list_del(&page->page_list); + list_del(&page->avail_page_link); kfree(page); } =20 @@ -330,10 +338,11 @@ void *dma_pool_alloc(struct dma_pool *pool, gfp_t mem= _flags, might_alloc(mem_flags); =20 spin_lock_irqsave(&pool->lock, flags); - list_for_each_entry(page, &pool->page_list, page_list) { - if (page->offset < pool->allocation) - goto ready; - } + page =3D list_first_entry_or_null(&pool->avail_page_list, + struct dma_page, + avail_page_link); + if (page) + goto ready; =20 /* pool_alloc_page() might sleep, so temporarily drop &pool->lock */ spin_unlock_irqrestore(&pool->lock, flags); @@ -345,10 +354,13 @@ void *dma_pool_alloc(struct dma_pool *pool, gfp_t mem= _flags, spin_lock_irqsave(&pool->lock, flags); =20 list_add(&page->page_list, &pool->page_list); + list_add(&page->avail_page_link, &pool->avail_page_list); ready: page->in_use++; offset =3D page->offset; page->offset =3D *(int *)(page->vaddr + offset); + if (page->offset >=3D pool->allocation) + list_del_init(&page->avail_page_link); retval =3D offset + page->vaddr; *handle =3D offset + page->dma; #ifdef DMAPOOL_DEBUG @@ -470,6 +482,13 @@ void dma_pool_free(struct dma_pool *pool, void *vaddr,= dma_addr_t dma) memset(vaddr, 0, pool->size); #endif =20 + /* + * list_empty() on the page tests if the page is already linked into + * avail_page_list to avoid adding it more than once. + */ + if (list_empty(&page->avail_page_link)) + list_add(&page->avail_page_link, &pool->avail_page_list); + page->in_use--; *(int *)vaddr =3D page->offset; page->offset =3D offset; --=20 2.25.1 From nobody Thu Apr 30 08:12:14 2026 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id 392BEC433EF for ; Tue, 31 May 2022 18:23:51 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1346994AbiEaSXu (ORCPT ); Tue, 31 May 2022 14:23:50 -0400 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:55280 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S234905AbiEaSXr (ORCPT ); Tue, 31 May 2022 14:23:47 -0400 Received: from mail.cybernetics.com (mail.cybernetics.com [173.71.130.66]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id 9F0327A80B for ; Tue, 31 May 2022 11:23:46 -0700 (PDT) X-ASG-Debug-ID: 1654021424-1cf43917f334b100001-xx1T2L Received: from cybernetics.com ([10.10.4.126]) by mail.cybernetics.com with ESMTP id DaFkvBiBCRTezthM; Tue, 31 May 2022 14:23:44 -0400 (EDT) X-Barracuda-Envelope-From: tonyb@cybernetics.com X-ASG-Whitelist: Client DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; d=cybernetics.com; s=mail; bh=R86aZY51t2u2IQYQVHsV2xLBx7C5x16DOE0BI7kIaD0=; h=Content-Transfer-Encoding:Content-Type:In-Reply-To:References:Cc:To:From: Content-Language:Subject:MIME-Version:Date:Message-ID; b=PIl/mfpVeGR7pK26lLIL gnhce3+ZeKiFpemaNrrs1l5X4DngDsIEVNLKfAhTglIjzl+cvY33UqCrmlUzwPOmaRCet3xUvt5OC PVWsBnrBRmW5aGCHNFC18P+06bzg2BtKSMlsWVXFKrE3OktjOIg9MRYf3K1G/daJJx45TKOB7w= Received: from [10.157.2.224] (HELO [192.168.200.1]) by cybernetics.com (CommuniGate Pro SMTP 7.1.1) with ESMTPS id 11829239; Tue, 31 May 2022 14:23:44 -0400 Message-ID: <801335ba-00f3-12ae-59e0-119d7d8fd8cd@cybernetics.com> Date: Tue, 31 May 2022 14:23:44 -0400 MIME-Version: 1.0 User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:91.0) Gecko/20100101 Thunderbird/91.9.1 Subject: [PATCH 10/10] dmapool: improve scalability of dma_pool_free Content-Language: en-US X-ASG-Orig-Subj: [PATCH 10/10] dmapool: improve scalability of dma_pool_free From: Tony Battersby To: linux-mm@kvack.org, linux-kernel@vger.kernel.org Cc: iommu@lists.linux-foundation.org, kernel-team@fb.com, Matthew Wilcox , Keith Busch , Andy Shevchenko , Robin Murphy , Tony Lindgren References: <9b08ab7c-b80b-527d-9adf-7716b0868fbc@cybernetics.com> In-Reply-To: <9b08ab7c-b80b-527d-9adf-7716b0868fbc@cybernetics.com> Content-Transfer-Encoding: quoted-printable X-Barracuda-Connect: UNKNOWN[10.10.4.126] X-Barracuda-Start-Time: 1654021424 X-Barracuda-URL: https://10.10.4.122:443/cgi-mod/mark.cgi X-Barracuda-BRTS-Status: 1 X-Virus-Scanned: by bsmtpd at cybernetics.com X-Barracuda-Scan-Msg-Size: 7804 Precedence: bulk List-ID: X-Mailing-List: linux-kernel@vger.kernel.org Content-Type: text/plain; charset="utf-8" dma_pool_free() scales poorly when the pool contains many pages because pool_find_page() does a linear scan of all allocated pages. Improve its scalability by replacing the linear scan with a red-black tree lookup. In big O notation, this improves the algorithm from O(n^2) to O(n * log n). Signed-off-by: Tony Battersby --- mm/dmapool.c | 128 ++++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 100 insertions(+), 28 deletions(-) diff --git a/mm/dmapool.c b/mm/dmapool.c index b3dd2ace0d2a..24535483f781 100644 --- a/mm/dmapool.c +++ b/mm/dmapool.c @@ -12,11 +12,12 @@ * Many older drivers still have their own code to do this. * * The current design of this allocator is fairly simple. The pool is - * represented by the 'struct dma_pool' which keeps a doubly-linked list of - * allocated pages. Each page in the page_list is split into blocks of at - * least 'size' bytes. Free blocks are tracked in an unsorted singly-link= ed - * list of free blocks within the page. Used blocks aren't tracked, but we - * keep a count of how many are currently allocated from each page. + * represented by the 'struct dma_pool' which keeps a red-black tree of all + * allocated pages, keyed by DMA address for fast lookup when freeing. + * Each page in the page_tree is split into blocks of at least 'size' byte= s. + * Free blocks are tracked in an unsorted singly-linked list of free blocks + * within the page. Used blocks aren't tracked, but we keep a count of how + * many are currently allocated from each page. * * The avail_page_list keeps track of pages that have one or more free blo= cks * available to (re)allocate. Pages are moved in and out of avail_page_li= st @@ -36,6 +37,7 @@ #include #include #include +#include #include #include #include @@ -45,7 +47,7 @@ #endif =20 struct dma_pool { /* the pool */ - struct list_head page_list; + struct rb_root page_tree; struct list_head avail_page_list; spinlock_t lock; unsigned int size; @@ -58,7 +60,7 @@ struct dma_pool { /* the pool */ }; =20 struct dma_page { /* cacheable header for 'allocation' bytes */ - struct list_head page_list; + struct rb_node page_node; struct list_head avail_page_link; void *vaddr; dma_addr_t dma; @@ -69,6 +71,11 @@ struct dma_page { /* cacheable header for 'allocation' = bytes */ static DEFINE_MUTEX(pools_lock); static DEFINE_MUTEX(pools_reg_lock); =20 +static inline struct dma_page *rb_to_dma_page(struct rb_node *node) +{ + return rb_entry(node, struct dma_page, page_node); +} + static ssize_t pools_show(struct device *dev, struct device_attribute *att= r, char *buf) { unsigned temp; @@ -76,6 +83,7 @@ static ssize_t pools_show(struct device *dev, struct devi= ce_attribute *attr, cha char *next; struct dma_page *page; struct dma_pool *pool; + struct rb_node *node; =20 next =3D buf; size =3D PAGE_SIZE; @@ -90,7 +98,10 @@ static ssize_t pools_show(struct device *dev, struct dev= ice_attribute *attr, cha size_t blocks =3D 0; =20 spin_lock_irq(&pool->lock); - list_for_each_entry(page, &pool->page_list, page_list) { + for (node =3D rb_first(&pool->page_tree); + node; + node =3D rb_next(node)) { + page =3D rb_to_dma_page(node); pages++; blocks +=3D page->in_use; } @@ -169,7 +180,7 @@ struct dma_pool *dma_pool_create(const char *name, stru= ct device *dev, =20 retval->dev =3D dev; =20 - INIT_LIST_HEAD(&retval->page_list); + retval->page_tree =3D RB_ROOT; INIT_LIST_HEAD(&retval->avail_page_list); spin_lock_init(&retval->lock); retval->size =3D size; @@ -213,6 +224,63 @@ struct dma_pool *dma_pool_create(const char *name, str= uct device *dev, } EXPORT_SYMBOL(dma_pool_create); =20 +/* + * Find the dma_page that manages the given DMA address. + */ +static struct dma_page *pool_find_page(struct dma_pool *pool, dma_addr_t d= ma) +{ + struct rb_node *node =3D pool->page_tree.rb_node; + + while (node) { + struct dma_page *page =3D rb_to_dma_page(node); + + if (dma < page->dma) + node =3D node->rb_left; + else if ((dma - page->dma) >=3D pool->allocation) + node =3D node->rb_right; + else + return page; + } + return NULL; +} + +/* + * Insert a dma_page into the page_tree. + */ +static int pool_insert_page(struct dma_pool *pool, struct dma_page *new_pa= ge) +{ + dma_addr_t dma =3D new_page->dma; + struct rb_node **node =3D &(pool->page_tree.rb_node), *parent =3D NULL; + + while (*node) { + struct dma_page *this_page =3D rb_to_dma_page(*node); + + parent =3D *node; + if (dma < this_page->dma) + node =3D &((*node)->rb_left); + else if (likely((dma - this_page->dma) >=3D pool->allocation)) + node =3D &((*node)->rb_right); + else { + /* + * A page that overlaps the new DMA range is already + * present in the tree. This should not happen. + */ + WARN(1, + "%s: %s: DMA address overlap: old %pad new %pad len %u\n", + dev_name(pool->dev), + pool->name, &this_page->dma, &dma, + pool->allocation); + return -1; + } + } + + /* Add new node and rebalance tree. */ + rb_link_node(&new_page->page_node, parent, node); + rb_insert_color(&new_page->page_node, &pool->page_tree); + + return 0; +} + static void pool_initialise_page(struct dma_pool *pool, struct dma_page *p= age) { unsigned int offset =3D 0; @@ -276,8 +344,16 @@ static void pool_free_page(struct dma_pool *pool, dma_free_coherent(pool->dev, pool->allocation, vaddr, dma); } =20 - list_del(&page->page_list); list_del(&page->avail_page_link); + + /* + * If the pool is being destroyed, it is not safe to modify the + * page_tree while iterating over it, and it is also unnecessary since + * the whole tree will be discarded anyway. + */ + if (!destroying_pool) + rb_erase(&page->page_node, &pool->page_tree); + kfree(page); } =20 @@ -291,7 +367,7 @@ static void pool_free_page(struct dma_pool *pool, */ void dma_pool_destroy(struct dma_pool *pool) { - struct dma_page *page; + struct dma_page *page, *tmp; bool empty =3D false; =20 if (unlikely(!pool)) @@ -307,9 +383,10 @@ void dma_pool_destroy(struct dma_pool *pool) device_remove_file(pool->dev, &dev_attr_pools); mutex_unlock(&pools_reg_lock); =20 - while ((page =3D list_first_entry_or_null(&pool->page_list, - struct dma_page, - page_list))) { + rbtree_postorder_for_each_entry_safe(page, + tmp, + &pool->page_tree, + page_node) { pool_free_page(pool, page, true); } =20 @@ -353,7 +430,15 @@ void *dma_pool_alloc(struct dma_pool *pool, gfp_t mem_= flags, =20 spin_lock_irqsave(&pool->lock, flags); =20 - list_add(&page->page_list, &pool->page_list); + if (unlikely(pool_insert_page(pool, page))) { + /* + * This should not happen, so something must have gone horribly + * wrong. Instead of crashing, intentionally leak the memory + * and make for the exit. + */ + spin_unlock_irqrestore(&pool->lock, flags); + return NULL; + } list_add(&page->avail_page_link, &pool->avail_page_list); ready: page->in_use++; @@ -395,19 +480,6 @@ void *dma_pool_alloc(struct dma_pool *pool, gfp_t mem_= flags, } EXPORT_SYMBOL(dma_pool_alloc); =20 -static struct dma_page *pool_find_page(struct dma_pool *pool, dma_addr_t d= ma) -{ - struct dma_page *page; - - list_for_each_entry(page, &pool->page_list, page_list) { - if (dma < page->dma) - continue; - if ((dma - page->dma) < pool->allocation) - return page; - } - return NULL; -} - /** * dma_pool_free - put block back into dma pool * @pool: the dma pool holding the block --=20 2.25.1