From nobody Sun Dec 14 06:40:51 2025 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id E3613C83F1B for ; Tue, 29 Aug 2023 17:13:45 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S237751AbjH2RNV (ORCPT ); Tue, 29 Aug 2023 13:13:21 -0400 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:35802 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S237736AbjH2RM5 (ORCPT ); Tue, 29 Aug 2023 13:12:57 -0400 Received: from out-248.mta1.migadu.com (out-248.mta1.migadu.com [IPv6:2001:41d0:203:375::f8]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id 5F3A3CE7 for ; Tue, 29 Aug 2023 10:12:43 -0700 (PDT) X-Report-Abuse: Please report any abuse attempt to abuse@migadu.com and include these headers. DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=linux.dev; s=key1; t=1693329161; h=from:from:reply-to:subject:subject:date:date:message-id:message-id: to:to:cc:cc:mime-version:mime-version: content-transfer-encoding:content-transfer-encoding: in-reply-to:in-reply-to:references:references; bh=78m2o0At4EqoxYvPO8GgUXqZmV/z9BTR2roI196o/ig=; b=aVdkqRkgoIFSKNeh42TLnbqZmPMoSkLTIn6T1JEhVD3Qqk6MuAku7bMKEsC8wSyrSX5enC FPGjSthHCdAdFwFBUcruYbc6A6xXWGBHMtC2szN93Qqo9L4dPl6NYeHve9SBanQLnxvT1U U8zvI/sIeAjU3YhEOjSFZSV+LFNUxSw= From: andrey.konovalov@linux.dev To: Marco Elver , Alexander Potapenko Cc: Andrey Konovalov , Dmitry Vyukov , Vlastimil Babka , kasan-dev@googlegroups.com, Evgenii Stepanov , Andrew Morton , linux-mm@kvack.org, linux-kernel@vger.kernel.org, Andrey Konovalov Subject: [PATCH 10/15] stackdepot: store free stack records in a freelist Date: Tue, 29 Aug 2023 19:11:20 +0200 Message-Id: <0853a38f849f75a428a76fe9bcd093c0502d26f4.1693328501.git.andreyknvl@google.com> In-Reply-To: References: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable X-Migadu-Flow: FLOW_OUT Precedence: bulk List-ID: X-Mailing-List: linux-kernel@vger.kernel.org Content-Type: text/plain; charset="utf-8" From: Andrey Konovalov Instead of using the global pool_offset variable to find a free slot when storing a new stack record, mainlain a freelist of free slots within the allocated stack pools. A global next_stack variable is used as the head of the freelist, and the next field in the stack_record struct is reused as freelist link (when the record is not in the freelist, this field is used as a link in the hash table). This is preparatory patch for implementing the eviction of stack records from the stack depot. Signed-off-by: Andrey Konovalov --- lib/stackdepot.c | 130 +++++++++++++++++++++++++++++------------------ 1 file changed, 81 insertions(+), 49 deletions(-) diff --git a/lib/stackdepot.c b/lib/stackdepot.c index 5982ea79939d..9011f4adcf20 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -55,8 +55,8 @@ union handle_parts { }; =20 struct stack_record { - struct stack_record *next; /* Link in the hash table */ - u32 hash; /* Hash in the hash table */ + struct stack_record *next; /* Link in hash table or freelist */ + u32 hash; /* Hash in hash table */ u32 size; /* Number of stored frames */ union handle_parts handle; unsigned long entries[DEPOT_STACK_MAX_FRAMES]; /* Frames */ @@ -88,10 +88,10 @@ static unsigned int stack_hash_mask; static void *stack_pools[DEPOT_MAX_POOLS]; /* Newly allocated pool that is not yet added to stack_pools. */ static void *new_pool; -/* Currently used pool in stack_pools. */ -static int pool_index; -/* Offset to the unused space in the currently used pool. */ -static size_t pool_offset; +/* Number of pools in stack_pools. */ +static int pools_num; +/* Next stack in the freelist of stack records within stack_pools. */ +static struct stack_record *next_stack; /* Lock that protects the variables above. */ static DEFINE_RAW_SPINLOCK(pool_lock); /* @@ -221,6 +221,41 @@ int stack_depot_init(void) } EXPORT_SYMBOL_GPL(stack_depot_init); =20 +/* Initializes a stack depol pool. */ +static void depot_init_pool(void *pool) +{ + const int records_in_pool =3D DEPOT_POOL_SIZE / DEPOT_STACK_RECORD_SIZE; + int i, offset; + + /* Initialize handles and link stack records to each other. */ + for (i =3D 0, offset =3D 0; offset < DEPOT_POOL_SIZE; + i++, offset +=3D DEPOT_STACK_RECORD_SIZE) { + struct stack_record *stack =3D pool + offset; + + stack->handle.pool_index =3D pools_num; + stack->handle.offset =3D offset >> DEPOT_STACK_ALIGN; + stack->handle.extra =3D 0; + + if (i < records_in_pool - 1) + stack->next =3D (void *)stack + DEPOT_STACK_RECORD_SIZE; + else + stack->next =3D NULL; + } + + /* Link stack records into the freelist. */ + WARN_ON(next_stack); + next_stack =3D pool; + + /* Save reference to the pool to be used by depot_fetch_stack. */ + stack_pools[pools_num] =3D pool; + + /* + * WRITE_ONCE pairs with potential concurrent read in + * depot_fetch_stack. + */ + WRITE_ONCE(pools_num, pools_num + 1); +} + /* Keeps the preallocated memory to be used for a new stack depot pool. */ static void depot_keep_new_pool(void **prealloc) { @@ -237,7 +272,7 @@ static void depot_keep_new_pool(void **prealloc) * Use the preallocated memory for the new pool * as long as we do not exceed the maximum number of pools. */ - if (pool_index + 1 < DEPOT_MAX_POOLS) { + if (pools_num < DEPOT_MAX_POOLS) { new_pool =3D *prealloc; *prealloc =3D NULL; } @@ -252,45 +287,42 @@ static void depot_keep_new_pool(void **prealloc) } =20 /* Updates refences to the current and the next stack depot pools. */ -static bool depot_update_pools(size_t required_size, void **prealloc) +static bool depot_update_pools(void **prealloc) { - /* Check if there is not enough space in the current pool. */ - if (unlikely(pool_offset + required_size > DEPOT_POOL_SIZE)) { - /* Bail out if we reached the pool limit. */ - if (unlikely(pool_index + 1 >=3D DEPOT_MAX_POOLS)) { - WARN_ONCE(1, "Stack depot reached limit capacity"); - return false; - } + /* Check if we still have objects in the freelist. */ + if (next_stack) + goto out_keep_prealloc; =20 - /* - * Move on to the new pool. - * WRITE_ONCE pairs with potential concurrent read in - * stack_depot_fetch. - */ - WRITE_ONCE(pool_index, pool_index + 1); - stack_pools[pool_index] =3D new_pool; + /* Check if we have a new pool saved and use it. */ + if (new_pool) { + depot_init_pool(new_pool); new_pool =3D NULL; - pool_offset =3D 0; =20 - /* - * If the maximum number of pools is not reached, take note - * that yet another new pool needs to be allocated. - * smp_store_release pairs with smp_load_acquire in - * stack_depot_save. - */ - if (pool_index + 1 < DEPOT_MAX_POOLS) + /* Take note that we might need a new new_pool. */ + if (pools_num < DEPOT_MAX_POOLS) smp_store_release(&new_pool_required, 1); + + /* Try keeping the preallocated memory for new_pool. */ + goto out_keep_prealloc; + } + + /* Bail out if we reached the pool limit. */ + if (unlikely(pools_num >=3D DEPOT_MAX_POOLS)) { + WARN_ONCE(1, "Stack depot reached limit capacity"); + return false; } =20 - /* Check if the current pool is not yet allocated. */ - if (*prealloc && stack_pools[pool_index] =3D=3D NULL) { - /* Use the preallocated memory for the current pool. */ - stack_pools[pool_index] =3D *prealloc; + /* Check if we have preallocated memory and use it. */ + if (*prealloc) { + depot_init_pool(*prealloc); *prealloc =3D NULL; return true; } =20 - /* Otherwise, try using the preallocated memory for a new pool. */ + return false; + +out_keep_prealloc: + /* Keep the preallocated memory for a new pool if required. */ if (*prealloc) depot_keep_new_pool(prealloc); return true; @@ -301,35 +333,35 @@ static struct stack_record * depot_alloc_stack(unsigned long *entries, int size, u32 hash, void **preal= loc) { struct stack_record *stack; - size_t required_size =3D DEPOT_STACK_RECORD_SIZE; =20 /* Update current and new pools if required and possible. */ - if (!depot_update_pools(required_size, prealloc)) + if (!depot_update_pools(prealloc)) return NULL; =20 - /* Check if we have a pool to save the stack trace. */ - if (stack_pools[pool_index] =3D=3D NULL) + /* Check if we have a stack record to save the stack trace. */ + stack =3D next_stack; + if (!stack) return NULL; =20 + /* Advance the freelist. */ + next_stack =3D stack->next; + /* Limit number of saved frames to DEPOT_STACK_MAX_FRAMES. */ if (size > DEPOT_STACK_MAX_FRAMES) size =3D DEPOT_STACK_MAX_FRAMES; =20 /* Save the stack trace. */ - stack =3D stack_pools[pool_index] + pool_offset; + stack->next =3D NULL; stack->hash =3D hash; stack->size =3D size; - stack->handle.pool_index =3D pool_index; - stack->handle.offset =3D pool_offset >> DEPOT_STACK_ALIGN; - stack->handle.extra =3D 0; + /* stack->handle is already filled in by depot_init_pool. */ memcpy(stack->entries, entries, flex_array_size(stack, entries, size)); - pool_offset +=3D required_size; =20 /* * Let KMSAN know the stored stack record is initialized. This shall * prevent false positive reports if instrumented code accesses it. */ - kmsan_unpoison_memory(stack, required_size); + kmsan_unpoison_memory(stack, DEPOT_STACK_RECORD_SIZE); =20 return stack; } @@ -339,16 +371,16 @@ static struct stack_record *depot_fetch_stack(depot_s= tack_handle_t handle) union handle_parts parts =3D { .handle =3D handle }; /* * READ_ONCE pairs with potential concurrent write in - * depot_update_pools. + * depot_init_pool. */ - int pool_index_cached =3D READ_ONCE(pool_index); + int pools_num_cached =3D READ_ONCE(pools_num); void *pool; size_t offset =3D parts.offset << DEPOT_STACK_ALIGN; struct stack_record *stack; =20 - if (parts.pool_index > pool_index_cached) { + if (parts.pool_index > pools_num_cached) { WARN(1, "pool index %d out of bounds (%d) for stack id %08x\n", - parts.pool_index, pool_index_cached, handle); + parts.pool_index, pools_num_cached, handle); return NULL; } =20 --=20 2.25.1