From nobody Sun Feb 8 05:27:04 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-1.web.codeaurora.org [10.30.226.201]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 423662D028A; Sun, 11 Jan 2026 21:39:07 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=10.30.226.201 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1768167547; cv=none; b=b/Re+xrbxau44VtD1BtZUxgpTwTLduRhsja7nvnaMq4hJf/rndJ0BM003gtjNs0l696hQ9Txqi9NEAih465LhyNGF1xomUkUVenY5UdWTIdovR2gmZ6m4kHKjeIx14TWVrE5H8mx9VJRzYsNzarfFRNeU1QJY3pXnH0tT3t99Gk= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1768167547; c=relaxed/simple; bh=HnHWABtvCkKcsqbtikkBLOHjLvBbhWd2x5g9UCb6u80=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=oopoitc4xE8IUeag/42j+MsnFOgKdT/b31FFPGyk6yiCHQ7Ckz+j+2Fjn465Pxk2jwK0J2dfbnLAe4AssMztxNZGzSDNNA8zvWGgN7U7Fiqug4e7a+T5TvlzZ0oz7IHyBE01Z7aU5C/Bfu64X1v2PSV+Sv/nsfKrza6U0keDE1A= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=kBXlRssN; arc=none smtp.client-ip=10.30.226.201 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="kBXlRssN" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 7D96FC19422; Sun, 11 Jan 2026 21:39:06 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1768167546; bh=HnHWABtvCkKcsqbtikkBLOHjLvBbhWd2x5g9UCb6u80=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=kBXlRssNNAZ1GjHGLKSp9zD+//rp3xufor/E3G47Q3hTAnAZEIwgjLpdGlZ1u8YDd ak7isCIKcB61R/uwTffImDHRNDbUOWrgyaeI++bRsITZCOgv4Xi6apZu2owYrUyRvl hmmmp0JX+Q6leiVORuBpSYBSrNS5gy7xLCFzhctCSG8ewpx97J7riHHa98wOh0xrDs UzsOMZT/YsUnmM3poQFC5/Z3yYSk1zdO3TqXU1I6x8bvJKhy6lGvycX/OTQHM7NLni T7lF0gscrgJnGk+BOkILRbiEWjyr9HLPnS9GM4dzw5jNACHWLsKlP7lgznBfc9iT1U VVuy2O3y5H18g== From: Eric Biggers To: dm-devel@lists.linux.dev, Alasdair Kergon , Mike Snitzer , Mikulas Patocka , Benjamin Marzinski Cc: Sami Tolvanen , linux-kernel@vger.kernel.org, Eric Biggers Subject: [PATCH 1/2] dm-bufio: merge cache_put() into cache_put_and_wake() Date: Sun, 11 Jan 2026 13:38:20 -0800 Message-ID: <20260111213821.94895-2-ebiggers@kernel.org> X-Mailer: git-send-email 2.52.0 In-Reply-To: <20260111213821.94895-1-ebiggers@kernel.org> References: <20260111213821.94895-1-ebiggers@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Merge cache_put() into its only caller, cache_put_and_wake(). Signed-off-by: Eric Biggers --- drivers/md/dm-bufio.c | 32 ++++++++++---------------------- 1 file changed, 10 insertions(+), 22 deletions(-) diff --git a/drivers/md/dm-bufio.c b/drivers/md/dm-bufio.c index 5235f3e4924b..6e61982c9b92 100644 --- a/drivers/md/dm-bufio.c +++ b/drivers/md/dm-bufio.c @@ -367,12 +367,12 @@ struct dm_buffer { * It does *not* handle: * - allocation/freeing of buffers. * - IO * - Eviction or cache sizing. * - * cache_get() and cache_put() are threadsafe, you do not need to - * protect these calls with a surrounding mutex. All the other + * cache_get() and cache_put_and_wake() are threadsafe, you do not need + * to protect these calls with a surrounding mutex. All the other * methods are not threadsafe; they do use locking primitives, but * only enough to ensure get/put are threadsafe. */ =20 struct buffer_tree { @@ -617,28 +617,10 @@ static struct dm_buffer *cache_get(struct dm_buffer_c= ache *bc, sector_t block) return b; } =20 /*--------------*/ =20 -/* - * Returns true if the hold count hits zero. - * threadsafe - */ -static bool cache_put(struct dm_buffer_cache *bc, struct dm_buffer *b) -{ - bool r; - - cache_read_lock(bc, b->block); - BUG_ON(!atomic_read(&b->hold_count)); - r =3D atomic_dec_and_test(&b->hold_count); - cache_read_unlock(bc, b->block); - - return r; -} - -/*--------------*/ - typedef enum evict_result (*b_predicate)(struct dm_buffer *, void *); =20 /* * Evicts a buffer based on a predicate. The oldest buffer that * matches the predicate will be selected. In addition to the @@ -1743,16 +1725,22 @@ static void __check_watermark(struct dm_bufio_clien= t *c, *-------------------------------------------------------------- */ =20 static void cache_put_and_wake(struct dm_bufio_client *c, struct dm_buffer= *b) { + bool wake; + + cache_read_lock(&c->cache, b->block); + BUG_ON(!atomic_read(&b->hold_count)); + wake =3D atomic_dec_and_test(&b->hold_count); + cache_read_unlock(&c->cache, b->block); + /* * Relying on waitqueue_active() is racey, but we sleep * with schedule_timeout anyway. */ - if (cache_put(&c->cache, b) && - unlikely(waitqueue_active(&c->free_buffer_wait))) + if (wake && unlikely(waitqueue_active(&c->free_buffer_wait))) wake_up(&c->free_buffer_wait); } =20 /* * This assumes you have already checked the cache to see if the buffer --=20 2.52.0 From nobody Sun Feb 8 05:27:04 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-1.web.codeaurora.org [10.30.226.201]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 903ED2D0C8B; Sun, 11 Jan 2026 21:39:07 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=10.30.226.201 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1768167547; cv=none; b=PY0iI76PwO7B9VCD/E8GaVvuxjydvlOeU8d3PGd7IRL5ed20XCNynUxZigif4kL7mnh9Y/9woE3TE31PK639zC5lx147/9Okg5mFS8pelRFWF5ICcDk/HgKrSOZdAcv2bGYEl/Zh8Yj/cD0/DprMEhiYnQ/na6i5+t0Gctbn/zM= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1768167547; c=relaxed/simple; bh=Af8W62bv0jW+bAoYRq25YAcvSaktAJ/vliRDwFM+uF0=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=r1pkLMRVTHkYqQ/6rrYAhaq/syQu/dBcbZ/ptgr5gKbDqWy/g/OfZemvQF5T51/i14ycLlQPqV/FcTZL9Ga4RtwZyGz7oUwiw0Ku/KexCSeI1V/sbancIDlDWDgJwdp0bG0MWQhRkwZNZAbUJ6CckjDJH/T+qDP5bR1wuva8gdM= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=t3mJ5Tqx; arc=none smtp.client-ip=10.30.226.201 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="t3mJ5Tqx" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 1117DC4CEF7; Sun, 11 Jan 2026 21:39:07 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1768167547; bh=Af8W62bv0jW+bAoYRq25YAcvSaktAJ/vliRDwFM+uF0=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=t3mJ5TqxwFctbY4ftZwt30Etwdj+1P1yvEOYEK7JRqCAdtBP0y3lHJl0ZjRSR6ktI WKkgvzhJWbh9PIggCq9VKaTPegvm8Lf/p7/NziWsR1tSHceWnq56iu98Q/PzCVsWwS BKYN5iUJ9aTelomNjj7foBjybQu+9IM+nOHMX7OErvwzjBiDjOZAtx4iu7mQgtYXxv My3cY5LVs5MBZwI8NaXDYosdtjNjLh5c5VXnbGPzeQzGt85DCpIMt2A2alFMNvLufO /rfNDlo/h0mvdUWP4bkQ1U3b58A7VlyBI6PYHlpLz1QLkFG/f7kROdrKAd4e1qZa9e A0FSg5GXwVHdQ== From: Eric Biggers To: dm-devel@lists.linux.dev, Alasdair Kergon , Mike Snitzer , Mikulas Patocka , Benjamin Marzinski Cc: Sami Tolvanen , linux-kernel@vger.kernel.org, Eric Biggers Subject: [PATCH 2/2] dm-bufio: avoid redundant buffer_tree lookups Date: Sun, 11 Jan 2026 13:38:21 -0800 Message-ID: <20260111213821.94895-3-ebiggers@kernel.org> X-Mailer: git-send-email 2.52.0 In-Reply-To: <20260111213821.94895-1-ebiggers@kernel.org> References: <20260111213821.94895-1-ebiggers@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" dm-bufio's map from block number to buffer is organized as a hash table of red-black trees. It does far more lookups in this hash table than necessary: typically one lookup to lock the tree, one lookup to search the tree, and one lookup to unlock the tree. Only one of those lookups is needed. Optimize it to do only the minimum number of lookups. This improves performance. It also reduces the object code size, considering that the redundant hash table lookups were being inlined. For example, the size of the text section of dm-bufio.o decreases from 15599 to 15070 bytes with gcc 15 and x86_64, or from 20652 to 20244 bytes with clang 21 and arm64. Signed-off-by: Eric Biggers --- drivers/md/dm-bufio.c | 142 ++++++++++++++++++++++++++---------------- 1 file changed, 89 insertions(+), 53 deletions(-) diff --git a/drivers/md/dm-bufio.c b/drivers/md/dm-bufio.c index 6e61982c9b92..ae169f3091e3 100644 --- a/drivers/md/dm-bufio.c +++ b/drivers/md/dm-bufio.c @@ -399,40 +399,55 @@ static DEFINE_STATIC_KEY_FALSE(no_sleep_enabled); static inline unsigned int cache_index(sector_t block, unsigned int num_lo= cks) { return dm_hash_locks_index(block, num_locks); } =20 -static inline void cache_read_lock(struct dm_buffer_cache *bc, sector_t bl= ock) +/* Get the buffer tree in the cache for the given block. Doesn't lock it.= */ +static inline struct buffer_tree *cache_get_tree(struct dm_buffer_cache *b= c, + sector_t block) +{ + return &bc->trees[cache_index(block, bc->num_locks)]; +} + +/* Lock the given buffer tree in the cache for reading. */ +static inline void cache_read_lock(struct dm_buffer_cache *bc, + struct buffer_tree *tree) { if (static_branch_unlikely(&no_sleep_enabled) && bc->no_sleep) - read_lock_bh(&bc->trees[cache_index(block, bc->num_locks)].u.spinlock); + read_lock_bh(&tree->u.spinlock); else - down_read(&bc->trees[cache_index(block, bc->num_locks)].u.lock); + down_read(&tree->u.lock); } =20 -static inline void cache_read_unlock(struct dm_buffer_cache *bc, sector_t = block) +/* Unlock the given buffer tree in the cache for reading. */ +static inline void cache_read_unlock(struct dm_buffer_cache *bc, + struct buffer_tree *tree) { if (static_branch_unlikely(&no_sleep_enabled) && bc->no_sleep) - read_unlock_bh(&bc->trees[cache_index(block, bc->num_locks)].u.spinlock); + read_unlock_bh(&tree->u.spinlock); else - up_read(&bc->trees[cache_index(block, bc->num_locks)].u.lock); + up_read(&tree->u.lock); } =20 -static inline void cache_write_lock(struct dm_buffer_cache *bc, sector_t b= lock) +/* Lock the given buffer tree in the cache for writing. */ +static inline void cache_write_lock(struct dm_buffer_cache *bc, + struct buffer_tree *tree) { if (static_branch_unlikely(&no_sleep_enabled) && bc->no_sleep) - write_lock_bh(&bc->trees[cache_index(block, bc->num_locks)].u.spinlock); + write_lock_bh(&tree->u.spinlock); else - down_write(&bc->trees[cache_index(block, bc->num_locks)].u.lock); + down_write(&tree->u.lock); } =20 -static inline void cache_write_unlock(struct dm_buffer_cache *bc, sector_t= block) +/* Unlock the given buffer tree in the cache for writing. */ +static inline void cache_write_unlock(struct dm_buffer_cache *bc, + struct buffer_tree *tree) { if (static_branch_unlikely(&no_sleep_enabled) && bc->no_sleep) - write_unlock_bh(&bc->trees[cache_index(block, bc->num_locks)].u.spinlock= ); + write_unlock_bh(&tree->u.spinlock); else - up_write(&bc->trees[cache_index(block, bc->num_locks)].u.lock); + up_write(&tree->u.lock); } =20 /* * Sometimes we want to repeatedly get and drop locks as part of an iterat= ion. * This struct helps avoid redundant drop and gets of the same lock. @@ -600,21 +615,23 @@ static void __cache_inc_buffer(struct dm_buffer *b) { atomic_inc(&b->hold_count); WRITE_ONCE(b->last_accessed, jiffies); } =20 -static struct dm_buffer *cache_get(struct dm_buffer_cache *bc, sector_t bl= ock) +static struct dm_buffer *cache_get(struct dm_buffer_cache *bc, + struct buffer_tree *tree, sector_t block) { struct dm_buffer *b; =20 - cache_read_lock(bc, block); - b =3D __cache_get(&bc->trees[cache_index(block, bc->num_locks)].root, blo= ck); + /* Assuming tree =3D=3D cache_get_tree(bc, block) */ + cache_read_lock(bc, tree); + b =3D __cache_get(&tree->root, block); if (b) { lru_reference(&b->lru); __cache_inc_buffer(b); } - cache_read_unlock(bc, block); + cache_read_unlock(bc, tree); =20 return b; } =20 /*--------------*/ @@ -661,11 +678,11 @@ static struct dm_buffer *__cache_evict(struct dm_buff= er_cache *bc, int list_mode if (!le) return NULL; =20 b =3D le_to_buffer(le); /* __evict_pred will have locked the appropriate tree. */ - rb_erase(&b->node, &bc->trees[cache_index(b->block, bc->num_locks)].root); + rb_erase(&b->node, &cache_get_tree(bc, b->block)->root); =20 return b; } =20 static struct dm_buffer *cache_evict(struct dm_buffer_cache *bc, int list_= mode, @@ -684,19 +701,21 @@ static struct dm_buffer *cache_evict(struct dm_buffer= _cache *bc, int list_mode, /*--------------*/ =20 /* * Mark a buffer as clean or dirty. Not threadsafe. */ -static void cache_mark(struct dm_buffer_cache *bc, struct dm_buffer *b, in= t list_mode) +static void cache_mark(struct dm_buffer_cache *bc, struct buffer_tree *tre= e, + struct dm_buffer *b, int list_mode) { - cache_write_lock(bc, b->block); + /* Assuming tree =3D=3D cache_get_tree(bc, b->block) */ + cache_write_lock(bc, tree); if (list_mode !=3D b->list_mode) { lru_remove(&bc->lru[b->list_mode], &b->lru); b->list_mode =3D list_mode; lru_insert(&bc->lru[b->list_mode], &b->lru); } - cache_write_unlock(bc, b->block); + cache_write_unlock(bc, tree); } =20 /*--------------*/ =20 /* @@ -818,23 +837,25 @@ static bool __cache_insert(struct rb_root *root, stru= ct dm_buffer *b) rb_insert_color(&b->node, root); =20 return true; } =20 -static bool cache_insert(struct dm_buffer_cache *bc, struct dm_buffer *b) +static bool cache_insert(struct dm_buffer_cache *bc, struct buffer_tree *t= ree, + struct dm_buffer *b) { bool r; =20 if (WARN_ON_ONCE(b->list_mode >=3D LIST_SIZE)) return false; =20 - cache_write_lock(bc, b->block); + /* Assuming tree =3D=3D cache_get_tree(bc, b->block) */ + cache_write_lock(bc, tree); BUG_ON(atomic_read(&b->hold_count) !=3D 1); - r =3D __cache_insert(&bc->trees[cache_index(b->block, bc->num_locks)].roo= t, b); + r =3D __cache_insert(&tree->root, b); if (r) lru_insert(&bc->lru[b->list_mode], &b->lru); - cache_write_unlock(bc, b->block); + cache_write_unlock(bc, tree); =20 return r; } =20 /*--------------*/ @@ -843,25 +864,27 @@ static bool cache_insert(struct dm_buffer_cache *bc, = struct dm_buffer *b) * Removes buffer from cache, ownership of the buffer passes back to the c= aller. * Fails if the hold_count is not one (ie. the caller holds the only refer= ence). * * Not threadsafe. */ -static bool cache_remove(struct dm_buffer_cache *bc, struct dm_buffer *b) +static bool cache_remove(struct dm_buffer_cache *bc, struct buffer_tree *t= ree, + struct dm_buffer *b) { bool r; =20 - cache_write_lock(bc, b->block); + /* Assuming tree =3D=3D cache_get_tree(bc, b->block) */ + cache_write_lock(bc, tree); =20 if (atomic_read(&b->hold_count) !=3D 1) { r =3D false; } else { r =3D true; - rb_erase(&b->node, &bc->trees[cache_index(b->block, bc->num_locks)].root= ); + rb_erase(&b->node, &tree->root); lru_remove(&bc->lru[b->list_mode], &b->lru); } =20 - cache_write_unlock(bc, b->block); + cache_write_unlock(bc, tree); =20 return r; } =20 /*--------------*/ @@ -1723,18 +1746,20 @@ static void __check_watermark(struct dm_bufio_clien= t *c, *-------------------------------------------------------------- * Getting a buffer *-------------------------------------------------------------- */ =20 -static void cache_put_and_wake(struct dm_bufio_client *c, struct dm_buffer= *b) +static void cache_put_and_wake(struct dm_bufio_client *c, + struct buffer_tree *tree, struct dm_buffer *b) { bool wake; =20 - cache_read_lock(&c->cache, b->block); + /* Assuming tree =3D=3D cache_get_tree(&c->cache, b->block) */ + cache_read_lock(&c->cache, tree); BUG_ON(!atomic_read(&b->hold_count)); wake =3D atomic_dec_and_test(&b->hold_count); - cache_read_unlock(&c->cache, b->block); + cache_read_unlock(&c->cache, tree); =20 /* * Relying on waitqueue_active() is racey, but we sleep * with schedule_timeout anyway. */ @@ -1744,11 +1769,12 @@ static void cache_put_and_wake(struct dm_bufio_clie= nt *c, struct dm_buffer *b) =20 /* * This assumes you have already checked the cache to see if the buffer * is already present (it will recheck after dropping the lock for allocat= ion). */ -static struct dm_buffer *__bufio_new(struct dm_bufio_client *c, sector_t b= lock, +static struct dm_buffer *__bufio_new(struct dm_bufio_client *c, + struct buffer_tree *tree, sector_t block, enum new_flag nf, int *need_submit, struct list_head *write_list) { struct dm_buffer *b, *new_b =3D NULL; =20 @@ -1764,11 +1790,11 @@ static struct dm_buffer *__bufio_new(struct dm_bufi= o_client *c, sector_t block, =20 /* * We've had a period where the mutex was unlocked, so need to * recheck the buffer tree. */ - b =3D cache_get(&c->cache, block); + b =3D cache_get(&c->cache, tree, block); if (b) { __free_buffer_wake(new_b); goto found_buffer; } =20 @@ -1792,17 +1818,17 @@ static struct dm_buffer *__bufio_new(struct dm_bufi= o_client *c, sector_t block, /* * We mustn't insert into the cache until the B_READING state * is set. Otherwise another thread could get it and use * it before it had been read. */ - cache_insert(&c->cache, b); + cache_insert(&c->cache, tree, b); =20 return b; =20 found_buffer: if (nf =3D=3D NF_PREFETCH) { - cache_put_and_wake(c, b); + cache_put_and_wake(c, tree, b); return NULL; } =20 /* * Note: it is essential that we don't wait for the buffer to be @@ -1810,11 +1836,11 @@ static struct dm_buffer *__bufio_new(struct dm_bufi= o_client *c, sector_t block, * dm_bufio_prefetch can be used in the driver request routine. * If the user called both dm_bufio_prefetch and dm_bufio_get on * the same buffer, it would deadlock if we waited. */ if (nf =3D=3D NF_GET && unlikely(test_bit_acquire(B_READING, &b->state)))= { - cache_put_and_wake(c, b); + cache_put_and_wake(c, tree, b); return NULL; } =20 return b; } @@ -1844,10 +1870,11 @@ static void read_endio(struct dm_buffer *b, blk_sta= tus_t status) */ static void *new_read(struct dm_bufio_client *c, sector_t block, enum new_flag nf, struct dm_buffer **bp, unsigned short ioprio) { + struct buffer_tree *tree; int need_submit =3D 0; struct dm_buffer *b; =20 LIST_HEAD(write_list); =20 @@ -1855,14 +1882,15 @@ static void *new_read(struct dm_bufio_client *c, se= ctor_t block, =20 /* * Fast path, hopefully the block is already in the cache. No need * to get the client lock for this. */ - b =3D cache_get(&c->cache, block); + tree =3D cache_get_tree(&c->cache, block); + b =3D cache_get(&c->cache, tree, block); if (b) { if (nf =3D=3D NF_PREFETCH) { - cache_put_and_wake(c, b); + cache_put_and_wake(c, tree, b); return NULL; } =20 /* * Note: it is essential that we don't wait for the buffer to be @@ -1870,21 +1898,21 @@ static void *new_read(struct dm_bufio_client *c, se= ctor_t block, * dm_bufio_prefetch can be used in the driver request routine. * If the user called both dm_bufio_prefetch and dm_bufio_get on * the same buffer, it would deadlock if we waited. */ if (nf =3D=3D NF_GET && unlikely(test_bit_acquire(B_READING, &b->state))= ) { - cache_put_and_wake(c, b); + cache_put_and_wake(c, tree, b); return NULL; } } =20 if (!b) { if (nf =3D=3D NF_GET) return NULL; =20 dm_bufio_lock(c); - b =3D __bufio_new(c, block, nf, &need_submit, &write_list); + b =3D __bufio_new(c, tree, block, nf, &need_submit, &write_list); dm_bufio_unlock(c); } =20 #ifdef CONFIG_DM_DEBUG_BLOCK_STACK_TRACING if (b && (atomic_read(&b->hold_count) =3D=3D 1)) @@ -1967,22 +1995,24 @@ static void __dm_bufio_prefetch(struct dm_bufio_cli= ent *c, return; /* should never happen */ =20 blk_start_plug(&plug); =20 for (; n_blocks--; block++) { - int need_submit; + struct buffer_tree *tree; struct dm_buffer *b; + int need_submit; =20 - b =3D cache_get(&c->cache, block); + tree =3D cache_get_tree(&c->cache, block); + b =3D cache_get(&c->cache, tree, block); if (b) { /* already in cache */ - cache_put_and_wake(c, b); + cache_put_and_wake(c, tree, b); continue; } =20 dm_bufio_lock(c); - b =3D __bufio_new(c, block, NF_PREFETCH, &need_submit, + b =3D __bufio_new(c, tree, block, NF_PREFETCH, &need_submit, &write_list); if (unlikely(!list_empty(&write_list))) { dm_bufio_unlock(c); blk_finish_plug(&plug); __flush_write_list(&write_list); @@ -2023,10 +2053,11 @@ void dm_bufio_prefetch_with_ioprio(struct dm_bufio_= client *c, sector_t block, EXPORT_SYMBOL_GPL(dm_bufio_prefetch_with_ioprio); =20 void dm_bufio_release(struct dm_buffer *b) { struct dm_bufio_client *c =3D b->c; + struct buffer_tree *tree =3D cache_get_tree(&c->cache, b->block); =20 /* * If there were errors on the buffer, and the buffer is not * to be written, free the buffer. There is no point in caching * invalid buffer. @@ -2036,20 +2067,20 @@ void dm_bufio_release(struct dm_buffer *b) !test_bit(B_WRITING, &b->state) && !test_bit(B_DIRTY, &b->state)) { dm_bufio_lock(c); =20 /* cache remove can fail if there are other holders */ - if (cache_remove(&c->cache, b)) { + if (cache_remove(&c->cache, tree, b)) { __free_buffer_wake(b); dm_bufio_unlock(c); return; } =20 dm_bufio_unlock(c); } =20 - cache_put_and_wake(c, b); + cache_put_and_wake(c, tree, b); } EXPORT_SYMBOL_GPL(dm_bufio_release); =20 void dm_bufio_mark_partial_buffer_dirty(struct dm_buffer *b, unsigned int start, unsigned int end) @@ -2064,11 +2095,12 @@ void dm_bufio_mark_partial_buffer_dirty(struct dm_b= uffer *b, BUG_ON(test_bit(B_READING, &b->state)); =20 if (!test_and_set_bit(B_DIRTY, &b->state)) { b->dirty_start =3D start; b->dirty_end =3D end; - cache_mark(&c->cache, b, LIST_DIRTY); + cache_mark(&c->cache, cache_get_tree(&c->cache, b->block), b, + LIST_DIRTY); } else { if (start < b->dirty_start) b->dirty_start =3D start; if (end > b->dirty_end) b->dirty_end =3D end; @@ -2129,10 +2161,11 @@ int dm_bufio_write_dirty_buffers(struct dm_bufio_cl= ient *c) =20 nr_buffers =3D cache_count(&c->cache, LIST_DIRTY); lru_iter_begin(&c->cache.lru[LIST_DIRTY], &it); while ((e =3D lru_iter_next(&it, is_writing, c))) { struct dm_buffer *b =3D le_to_buffer(e); + struct buffer_tree *tree; __cache_inc_buffer(b); =20 BUG_ON(test_bit(B_READING, &b->state)); =20 if (nr_buffers) { @@ -2142,14 +2175,16 @@ int dm_bufio_write_dirty_buffers(struct dm_bufio_cl= ient *c) dm_bufio_lock(c); } else { wait_on_bit_io(&b->state, B_WRITING, TASK_UNINTERRUPTIBLE); } =20 + tree =3D cache_get_tree(&c->cache, b->block); + if (!test_bit(B_DIRTY, &b->state) && !test_bit(B_WRITING, &b->state)) - cache_mark(&c->cache, b, LIST_CLEAN); + cache_mark(&c->cache, tree, b, LIST_CLEAN); =20 - cache_put_and_wake(c, b); + cache_put_and_wake(c, tree, b); =20 cond_resched(); } lru_iter_end(&it); =20 @@ -2213,21 +2248,22 @@ int dm_bufio_issue_discard(struct dm_bufio_client *= c, sector_t block, sector_t c } EXPORT_SYMBOL_GPL(dm_bufio_issue_discard); =20 static void forget_buffer(struct dm_bufio_client *c, sector_t block) { + struct buffer_tree *tree =3D cache_get_tree(&c->cache, block); struct dm_buffer *b; =20 - b =3D cache_get(&c->cache, block); + b =3D cache_get(&c->cache, tree, block); if (b) { if (likely(!smp_load_acquire(&b->state))) { - if (cache_remove(&c->cache, b)) + if (cache_remove(&c->cache, tree, b)) __free_buffer_wake(b); else - cache_put_and_wake(c, b); + cache_put_and_wake(c, tree, b); } else { - cache_put_and_wake(c, b); + cache_put_and_wake(c, tree, b); } } } =20 /* --=20 2.52.0