From nobody Sun May 24 20:33:10 2026 Received: from out-188.mta0.migadu.com (out-188.mta0.migadu.com [91.218.175.188]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 52436288C2C for ; Fri, 22 May 2026 01:19:34 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=91.218.175.188 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1779412776; cv=none; b=as/7J04z6+Ts5w2YOiQ8nGqEfiofbO//rBjg/27NNj5Oqm1tG8lP/yHxiIYMTbab1T+u7N4Ht+B8w0iep6TDvlSbENAerFjNc6vYbjZv+eH52VQZccGpdQhzeq0difZhlwFHPsM2CjJVc9U/clevmbQ26Pe0PnLFvLJHpyiLDj0= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1779412776; c=relaxed/simple; bh=xLxOYPKAwtDEDPSKWxvp6uLbahfTn6YNz/WtelkG08Y=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=fUFdz3rV5jGcWnDLHiAu8wqFQAu8ZiAPZgu+QfcvBJVSP7xEWRvD1O3mFvbbXs/SpFc6YTRKbg6fi0bd7RKkT+X6JbGIEIeoUYs4LnW6PdxN2V9YbUEAGAruvr58yt8Q2kVTMFATCsebITw3KNosQ/70GcDVtPLMnBtb9Y4bUJo= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=linux.dev; spf=pass smtp.mailfrom=linux.dev; dkim=pass (1024-bit key) header.d=linux.dev header.i=@linux.dev header.b=ayV2q67y; arc=none smtp.client-ip=91.218.175.188 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=linux.dev Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=linux.dev Authentication-Results: smtp.subspace.kernel.org; dkim=pass (1024-bit key) header.d=linux.dev header.i=@linux.dev header.b="ayV2q67y" X-Report-Abuse: Please report any abuse attempt to abuse@migadu.com and include these headers. DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=linux.dev; s=key1; t=1779412772; h=from:from:reply-to:subject:subject:date:date:message-id:message-id: to:to:cc:cc:mime-version:mime-version: content-transfer-encoding:content-transfer-encoding: in-reply-to:in-reply-to:references:references; bh=ChUKgi1DGYIv7W9zbcNT51//Ts23bQgwEMP6yFTU2u0=; b=ayV2q67yTiwGs/a603OC4YKOX7ifFKLcTHFP4dounJlMM00mLp0D2lEtrR8oZ1oxq+qCp2 3IKLqEv7Ej1Sgqt8ZlrhBv4lPATprFDc995gW7GGPjg6MX50TQZE2wFTBtFyvASwUegbZQ VENs7FskY+etg3nnOQ0bHixam6Mvn74= From: Shakeel Butt To: Andrew Morton Cc: Johannes Weiner , Michal Hocko , Roman Gushchin , Muchun Song , Qi Zheng , Alexandre Ghiti , Joshua Hahn , Harry Yoo , Meta kernel team , linux-mm@kvack.org, cgroups@vger.kernel.org, linux-kernel@vger.kernel.org, kernel test robot Subject: [PATCH v2 1/4] memcg: store node_id instead of pglist_data pointer Date: Thu, 21 May 2026 18:19:05 -0700 Message-ID: <20260522011908.1669332-2-shakeel.butt@linux.dev> In-Reply-To: <20260522011908.1669332-1-shakeel.butt@linux.dev> References: <20260522011908.1669332-1-shakeel.butt@linux.dev> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable X-Migadu-Flow: FLOW_OUT Content-Type: text/plain; charset="utf-8" The struct obj_stock_pcp stores a pointer to pglist_data for the slab stats cached on the cpu. On 64-bit machines, this costs 8 bytes. The pointer is not strictly required: NODE_DATA() can recover it from the node id. Replace cached_pgdat with int16_t node_id and use NUMA_NO_NODE as the "no stats cached" sentinel. At the moment all the archs limit MAX_NUMNODES to 1024 so int16_t is plenty; a BUILD_BUG_ON() makes sure we notice if that ever changes. Fixes: 01b9da291c49 ("mm: memcontrol: convert objcg to be per-memcg per-nod= e type") Tested-by: kernel test robot Signed-off-by: Shakeel Butt Acked-by: Muchun Song Reviewed-by: Harry Yoo (Oracle) Acked-by: Qi Zheng Reported-by: kernel test robot --- Changes since v1: - Added tags in the commit message mm/memcontrol.c | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/mm/memcontrol.c b/mm/memcontrol.c index b8caeb7ccaa3..d7c162946719 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -2021,7 +2021,7 @@ struct obj_stock_pcp { local_trylock_t lock; unsigned int nr_bytes; struct obj_cgroup *cached_objcg; - struct pglist_data *cached_pgdat; + int16_t node_id; int nr_slab_reclaimable_b; int nr_slab_unreclaimable_b; =20 @@ -2031,6 +2031,7 @@ struct obj_stock_pcp { =20 static DEFINE_PER_CPU_ALIGNED(struct obj_stock_pcp, obj_stock) =3D { .lock =3D INIT_LOCAL_TRYLOCK(lock), + .node_id =3D NUMA_NO_NODE, }; =20 static DEFINE_MUTEX(percpu_charge_mutex); @@ -3159,6 +3160,13 @@ static void __account_obj_stock(struct obj_cgroup *o= bjcg, { int *bytes; =20 + /* + * Though at the moment MAX_NUMNODES <=3D 1024 in all archs but let's make + * sure it does not exceed S16_MAX otherwise we need to fix node_id type + * in struct obj_stock_pcp. + */ + BUILD_BUG_ON(MAX_NUMNODES >=3D S16_MAX); + if (!stock || READ_ONCE(stock->cached_objcg) !=3D objcg) goto direct; =20 @@ -3166,9 +3174,11 @@ static void __account_obj_stock(struct obj_cgroup *o= bjcg, * Save vmstat data in stock and skip vmstat array update unless * accumulating over a page of vmstat data or when pgdat changes. */ - if (stock->cached_pgdat !=3D pgdat) { + if (stock->node_id =3D=3D NUMA_NO_NODE) { + stock->node_id =3D pgdat->node_id; + } else if (stock->node_id !=3D pgdat->node_id) { /* Flush the existing cached vmstat data */ - struct pglist_data *oldpg =3D stock->cached_pgdat; + struct pglist_data *oldpg =3D NODE_DATA(stock->node_id); =20 if (stock->nr_slab_reclaimable_b) { mod_objcg_mlstate(objcg, oldpg, NR_SLAB_RECLAIMABLE_B, @@ -3180,7 +3190,7 @@ static void __account_obj_stock(struct obj_cgroup *ob= jcg, stock->nr_slab_unreclaimable_b); stock->nr_slab_unreclaimable_b =3D 0; } - stock->cached_pgdat =3D pgdat; + stock->node_id =3D pgdat->node_id; } =20 bytes =3D (idx =3D=3D NR_SLAB_RECLAIMABLE_B) ? &stock->nr_slab_reclaimabl= e_b @@ -3276,19 +3286,21 @@ static void drain_obj_stock(struct obj_stock_pcp *s= tock) * Flush the vmstat data in current stock */ if (stock->nr_slab_reclaimable_b || stock->nr_slab_unreclaimable_b) { + struct pglist_data *oldpg =3D NODE_DATA(stock->node_id); + if (stock->nr_slab_reclaimable_b) { - mod_objcg_mlstate(old, stock->cached_pgdat, + mod_objcg_mlstate(old, oldpg, NR_SLAB_RECLAIMABLE_B, stock->nr_slab_reclaimable_b); stock->nr_slab_reclaimable_b =3D 0; } if (stock->nr_slab_unreclaimable_b) { - mod_objcg_mlstate(old, stock->cached_pgdat, + mod_objcg_mlstate(old, oldpg, NR_SLAB_UNRECLAIMABLE_B, stock->nr_slab_unreclaimable_b); stock->nr_slab_unreclaimable_b =3D 0; } - stock->cached_pgdat =3D NULL; + stock->node_id =3D NUMA_NO_NODE; } =20 WRITE_ONCE(stock->cached_objcg, NULL); --=20 2.53.0-Meta From nobody Sun May 24 20:33:10 2026 Received: from out-176.mta1.migadu.com (out-176.mta1.migadu.com [95.215.58.176]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 5F037226D18 for ; Fri, 22 May 2026 01:19:52 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=95.215.58.176 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1779412793; cv=none; b=l5x+Jomw07BN+6JvblG3AbXp+vIu/rhO8b/62PZ/KjjJqHwAXdjcylRd0VjfeZfLeZ0sD87IQpFkEmt3sKESQKOztdG/Ir62pT2HLraNlXbI0eJxfmUMcgRNWfhchdaF8k2gY73nzJXcv8PQk/l/55XRAonZDJ5Bnag859Wxu1Y= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1779412793; c=relaxed/simple; bh=j3r4Q1ZbbYd2Var5WVJ3HcVYRvGRTTpGMe7MUkK5OR4=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=tjuPTnQ1V4AnUspcfqwgX6k3kXEZAi9dOjAjKujjUTHD45wtvblm5NWEHQzhuPct3sMNf1/6jLr1o4wKbUC6fTfWPw/b/oLCbZh6KN9Mju5ztCPj+AatmfQJ1vxu4qKuoOIooQRv+h5bMTVB5rCjDPPQDBvmFTqyLSJJ+QaEyfk= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=linux.dev; spf=pass smtp.mailfrom=linux.dev; dkim=pass (1024-bit key) header.d=linux.dev header.i=@linux.dev header.b=LBjWDd/H; arc=none smtp.client-ip=95.215.58.176 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=linux.dev Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=linux.dev Authentication-Results: smtp.subspace.kernel.org; dkim=pass (1024-bit key) header.d=linux.dev header.i=@linux.dev header.b="LBjWDd/H" X-Report-Abuse: Please report any abuse attempt to abuse@migadu.com and include these headers. DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=linux.dev; s=key1; t=1779412790; h=from:from:reply-to:subject:subject:date:date:message-id:message-id: to:to:cc:cc:mime-version:mime-version: content-transfer-encoding:content-transfer-encoding: in-reply-to:in-reply-to:references:references; bh=j0DDjB0w4E9tMqU+JEwxbi4iz46ciqDj+g3fDXVBUs8=; b=LBjWDd/HeERmF6Rf8dihT6KKDeCm9NG6KIIBMxZfB69IOMazt2pX6Wn1nfoXxIAKbwbL4z lfv2KcxH0aceH3Bm/26jxoCn9id9qEBK1UyJ2oUDuD3LLhAQx763nh7zneCEZvHkdNufhi qiEAedwMNnd+OQ4XUAgQaNtBDY9ria0= From: Shakeel Butt To: Andrew Morton Cc: Johannes Weiner , Michal Hocko , Roman Gushchin , Muchun Song , Qi Zheng , Alexandre Ghiti , Joshua Hahn , Harry Yoo , Meta kernel team , linux-mm@kvack.org, cgroups@vger.kernel.org, linux-kernel@vger.kernel.org, kernel test robot Subject: [PATCH v2 2/4] memcg: uint16_t for nr_bytes in obj_stock_pcp Date: Thu, 21 May 2026 18:19:06 -0700 Message-ID: <20260522011908.1669332-3-shakeel.butt@linux.dev> In-Reply-To: <20260522011908.1669332-1-shakeel.butt@linux.dev> References: <20260522011908.1669332-1-shakeel.butt@linux.dev> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable X-Migadu-Flow: FLOW_OUT Content-Type: text/plain; charset="utf-8" Currently struct obj_stock_pcp stores nr_bytes in an 'unsigned int' which is 4 bytes on 64-bit machines. Switch the field to uint16_t to shrink the per-CPU cache. The kernel supports PAGE_SIZE_4KB, _8KB, _16KB, _32KB, _64KB and _256KB (see HAVE_PAGE_SIZE_* in arch/Kconfig). After the PAGE_SIZE-aligned flush in __refill_obj_stock(), the sub-page remainder fits in uint16_t up through 64KiB pages where PAGE_SIZE - 1 =3D=3D U16_MAX, but on 256KiB pages PAGE_SIZE - 1 =3D=3D 0x3FFFF exceeds U16_MAX. The accumulator also needs to stay within uint16_t between page-aligned flushes on 64KiB pages where PAGE_SIZE itself is U16_MAX + 1. Accumulate the new total in an 'unsigned int' local, then: 1. Flush whenever the accumulator would hit U16_MAX. Together with the existing allow_uncharge flush at PAGE_SIZE, this keeps the uint16_t safe on PAGE_SIZE <=3D 64KiB. 2. On configs with PAGE_SHIFT > 16 (PAGE_SIZE_256KB on hexagon and powerpc 44x), push any sub-page remainder above U16_MAX into objcg->nr_charged_bytes via atomic_add before storing back, so the store cannot silently truncate. The PAGE_SHIFT > 16 guard folds the branch out at compile time on smaller page sizes. Fixes: 01b9da291c49 ("mm: memcontrol: convert objcg to be per-memcg per-nod= e type") Tested-by: kernel test robot Signed-off-by: Shakeel Butt Reviewed-by: Harry Yoo (Oracle) Acked-by: Muchun Song Acked-by: Qi Zheng Reported-by: kernel test robot --- Changes since v1: - Collected tags - Rearrange fields of obj_stock_pcp (David Laight) - Fix comparison operator (Harry) mm/memcontrol.c | 33 +++++++++++++++++++++++++++------ 1 file changed, 27 insertions(+), 6 deletions(-) diff --git a/mm/memcontrol.c b/mm/memcontrol.c index d7c162946719..e4f00a8159d5 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -2019,8 +2019,8 @@ static DEFINE_PER_CPU_ALIGNED(struct memcg_stock_pcp,= memcg_stock) =3D { =20 struct obj_stock_pcp { local_trylock_t lock; - unsigned int nr_bytes; struct obj_cgroup *cached_objcg; + uint16_t nr_bytes; int16_t node_id; int nr_slab_reclaimable_b; int nr_slab_unreclaimable_b; @@ -3331,6 +3331,7 @@ static void __refill_obj_stock(struct obj_cgroup *obj= cg, bool allow_uncharge) { unsigned int nr_pages =3D 0; + unsigned int stock_nr_bytes; =20 if (!stock) { nr_pages =3D nr_bytes >> PAGE_SHIFT; @@ -3339,21 +3340,41 @@ static void __refill_obj_stock(struct obj_cgroup *o= bjcg, goto out; } =20 + stock_nr_bytes =3D stock->nr_bytes; if (READ_ONCE(stock->cached_objcg) !=3D objcg) { /* reset if necessary */ drain_obj_stock(stock); obj_cgroup_get(objcg); - stock->nr_bytes =3D atomic_read(&objcg->nr_charged_bytes) + stock_nr_bytes =3D atomic_read(&objcg->nr_charged_bytes) ? atomic_xchg(&objcg->nr_charged_bytes, 0) : 0; WRITE_ONCE(stock->cached_objcg, objcg); =20 allow_uncharge =3D true; /* Allow uncharge when objcg changes */ } - stock->nr_bytes +=3D nr_bytes; + stock_nr_bytes +=3D nr_bytes; + + /* Since stock->nr_bytes is uint16_t, don't refill >=3D U16_MAX */ + if ((allow_uncharge && (stock_nr_bytes > PAGE_SIZE)) || + stock_nr_bytes > U16_MAX) { + nr_pages =3D stock_nr_bytes >> PAGE_SHIFT; + stock_nr_bytes &=3D (PAGE_SIZE - 1); + + /* + * On configs with PAGE_SHIFT > 16 (PAGE_SIZE_256KB on + * hexagon and powerpc 44x), the sub-page remainder can + * still exceed U16_MAX. Push the excess back to + * objcg->nr_charged_bytes so the store into uint16_t + * cannot silently truncate; folded out at compile time + * on smaller page sizes. + */ + if (PAGE_SHIFT > 16 && stock_nr_bytes > U16_MAX) { + unsigned int kept =3D stock_nr_bytes & U16_MAX; =20 - if (allow_uncharge && (stock->nr_bytes > PAGE_SIZE)) { - nr_pages =3D stock->nr_bytes >> PAGE_SHIFT; - stock->nr_bytes &=3D (PAGE_SIZE - 1); + atomic_add(stock_nr_bytes - kept, + &objcg->nr_charged_bytes); + stock_nr_bytes =3D kept; + } } + stock->nr_bytes =3D stock_nr_bytes; =20 out: if (nr_pages) --=20 2.53.0-Meta From nobody Sun May 24 20:33:10 2026 Received: from out-189.mta0.migadu.com (out-189.mta0.migadu.com [91.218.175.189]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id EC1652367CF for ; Fri, 22 May 2026 01:20:14 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=91.218.175.189 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1779412816; cv=none; b=JkV066qZpSpitrJzTZhXnF0dRG/PspUoJNuWbftfLKUkqZKM0U3MkU95gkRPCT0Gig1f6ZXCXBV+7DYFwKn3D/N+A5gQrmw96djB53l+Nx1e1iHYlIzLUTTfizWJbwIPYU17dT1XcLhcArbXzGtAl6vJAxp1y6m1ammrz/gvL+8= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1779412816; c=relaxed/simple; bh=vJDjCztTbemGieM7E9AKDabvaixnTahDNI3PCkVOjzI=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=M5j3IgRNjqjyDuCrGo2SLsHPSEkLGl+4JdhqZvxvjXsiqrYGSTsj3G4nFUt1zxkJQQT86Uhwgv3C+EDB8bPilhDcpshMTM/trM5ncT72497P269VYfKAQvjiHVIMgp2UrxBvU7eIJSwSr/Eyl7KKabRNBxD9RbM+d01oE23kN2g= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=linux.dev; spf=pass smtp.mailfrom=linux.dev; dkim=pass (1024-bit key) header.d=linux.dev header.i=@linux.dev header.b=jy/r4gRI; arc=none smtp.client-ip=91.218.175.189 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=linux.dev Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=linux.dev Authentication-Results: smtp.subspace.kernel.org; dkim=pass (1024-bit key) header.d=linux.dev header.i=@linux.dev header.b="jy/r4gRI" X-Report-Abuse: Please report any abuse attempt to abuse@migadu.com and include these headers. DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=linux.dev; s=key1; t=1779412813; h=from:from:reply-to:subject:subject:date:date:message-id:message-id: to:to:cc:cc:mime-version:mime-version: content-transfer-encoding:content-transfer-encoding: in-reply-to:in-reply-to:references:references; bh=vHNkZOw2UW3I07VSJDEzsPCBLxzvWAd9K3Bgudx7KQM=; b=jy/r4gRIe+INC0fMlfJlaVAV5x+Nb+kKW1ZjbpReCKKEkZqCimx5YRDWT3Q3+hCVkcV9UX 2YgRNKrwM6EBzzzk6gUti1L+SEw9QMo2ynp9MTl9SL4EgNljClhgVgTeceWaHS2AmnOZWe xndOWTYnRO2WY7oCNwoD4m2/fxcPkao= From: Shakeel Butt To: Andrew Morton Cc: Johannes Weiner , Michal Hocko , Roman Gushchin , Muchun Song , Qi Zheng , Alexandre Ghiti , Joshua Hahn , Harry Yoo , Meta kernel team , linux-mm@kvack.org, cgroups@vger.kernel.org, linux-kernel@vger.kernel.org, kernel test robot Subject: [PATCH v2 3/4] memcg: int16_t for cached slab stats Date: Thu, 21 May 2026 18:19:07 -0700 Message-ID: <20260522011908.1669332-4-shakeel.butt@linux.dev> In-Reply-To: <20260522011908.1669332-1-shakeel.butt@linux.dev> References: <20260522011908.1669332-1-shakeel.butt@linux.dev> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable X-Migadu-Flow: FLOW_OUT Content-Type: text/plain; charset="utf-8" Currently struct obj_stock_pcp stores cached slab stats in 'int' which is 4 bytes per counter on 64-bit machines. Switch them to int16_t to shrink the cached metadata. The existing PAGE_SIZE flush in __account_obj_stock() bounds *bytes at PAGE_SIZE on 4KiB and 16KiB page archs, well within int16_t. On 64KiB pages PAGE_SIZE is well above S16_MAX so that flush never fires, and a sufficiently long run of accumulations would overflow the cache. Add an explicit S16_MAX guard before each add: when the next add would push abs(*bytes) past S16_MAX, fold the cached value into @nr and flush directly via mod_objcg_mlstate() before the accumulation. Fixes: 01b9da291c49 ("mm: memcontrol: convert objcg to be per-memcg per-nod= e type") Tested-by: kernel test robot Signed-off-by: Shakeel Butt Reviewed-by: Harry Yoo (Oracle) Acked-by: Muchun Song Acked-by: Qi Zheng Reported-by: kernel test robot --- Changes since v2: - Collected tags mm/memcontrol.c | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/mm/memcontrol.c b/mm/memcontrol.c index e4f00a8159d5..78c02451312b 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -2022,8 +2022,8 @@ struct obj_stock_pcp { struct obj_cgroup *cached_objcg; uint16_t nr_bytes; int16_t node_id; - int nr_slab_reclaimable_b; - int nr_slab_unreclaimable_b; + int16_t nr_slab_reclaimable_b; + int16_t nr_slab_unreclaimable_b; =20 struct work_struct work; unsigned long flags; @@ -3158,7 +3158,7 @@ static void __account_obj_stock(struct obj_cgroup *ob= jcg, struct obj_stock_pcp *stock, int nr, struct pglist_data *pgdat, enum node_stat_item idx) { - int *bytes; + int16_t *bytes; =20 /* * Though at the moment MAX_NUMNODES <=3D 1024 in all archs but let's make @@ -3195,6 +3195,16 @@ static void __account_obj_stock(struct obj_cgroup *o= bjcg, =20 bytes =3D (idx =3D=3D NR_SLAB_RECLAIMABLE_B) ? &stock->nr_slab_reclaimabl= e_b : &stock->nr_slab_unreclaimable_b; + /* + * To avoid overflow or underflow, flush directly if accumulating @nr + * would push the cached value past S16_MAX. + */ + if (abs(nr + *bytes) > S16_MAX) { + nr +=3D *bytes; + *bytes =3D 0; + goto direct; + } + /* * Even for large object >=3D PAGE_SIZE, the vmstat data will still be * cached locally at least once before pushing it out. --=20 2.53.0-Meta From nobody Sun May 24 20:33:10 2026 Received: from out-180.mta0.migadu.com (out-180.mta0.migadu.com [91.218.175.180]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id EAF542367CF for ; Fri, 22 May 2026 01:20:23 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=91.218.175.180 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1779412825; cv=none; b=Qvuy6D8IXBvfzabR2sZAfRT2yxErePXANuDDNyu4+WDrmEMzznz1WJN4O5dXEwOXidf2q78IOXgqkH2nDNm909rmWUSipzPEoeDd14l2lHLQfTVPuVfF/6OLUMG4O9gRS7DADBoL2wLuSZUfNWhjlm/qyYNi3Mf59/krFJ9m4gU= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1779412825; c=relaxed/simple; bh=lHTtEkVV9dleQGADmhZiZqxF/hqpeo/T+1k4eAAGw5o=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=TZOaT7rpaOJ5X+JDiUO3lInRvim/qZxbvltDo12aeVW+ADS54mQ4lEVugwcRVfjhny3q4Y573adejWwiBYWapvyV15/aXxi5Fbxp6equ47M2NHziErcwCNqu6E+r/tj8g8xEqXeOAePjQIsU8u+FzDnEUrPiHuHwZ5D36Jfg3TM= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=linux.dev; spf=pass smtp.mailfrom=linux.dev; dkim=pass (1024-bit key) header.d=linux.dev header.i=@linux.dev header.b=aeFFxBqi; arc=none smtp.client-ip=91.218.175.180 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=linux.dev Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=linux.dev Authentication-Results: smtp.subspace.kernel.org; dkim=pass (1024-bit key) header.d=linux.dev header.i=@linux.dev header.b="aeFFxBqi" X-Report-Abuse: Please report any abuse attempt to abuse@migadu.com and include these headers. DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=linux.dev; s=key1; t=1779412822; h=from:from:reply-to:subject:subject:date:date:message-id:message-id: to:to:cc:cc:mime-version:mime-version: content-transfer-encoding:content-transfer-encoding: in-reply-to:in-reply-to:references:references; bh=mlNjNHDfX0di3XL59bBl+Q1N8mPFOAdebVVOjRv4HBc=; b=aeFFxBqiMwvPBNGunCTlbkXP6dV8fKhIvqc4g3guRjXOlvjY+erJhhFwA/mOrFRKh8UBTI CD7UV30JQZpdOnxg0Fn1nQNm1lcCY10X8x/iqAAaB5Cvh+HO5gWsUbmCfL0HUfTqYpYrRJ iEfJmO8xZUFjqZjBf8MXOFNUYu/VNVk= From: Shakeel Butt To: Andrew Morton Cc: Johannes Weiner , Michal Hocko , Roman Gushchin , Muchun Song , Qi Zheng , Alexandre Ghiti , Joshua Hahn , Harry Yoo , Meta kernel team , linux-mm@kvack.org, cgroups@vger.kernel.org, linux-kernel@vger.kernel.org, kernel test robot Subject: [PATCH v2 4/4] memcg: multi objcg charge support Date: Thu, 21 May 2026 18:19:08 -0700 Message-ID: <20260522011908.1669332-5-shakeel.butt@linux.dev> In-Reply-To: <20260522011908.1669332-1-shakeel.butt@linux.dev> References: <20260522011908.1669332-1-shakeel.butt@linux.dev> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable X-Migadu-Flow: FLOW_OUT Content-Type: text/plain; charset="utf-8" Commit 01b9da291c49 ("mm: memcontrol: convert objcg to be per-memcg per-node type") split a memcg's single obj_cgroup into one per NUMA node so that reparenting LRU folios can take per-node lru locks. As a side effect, the per-CPU obj_stock_pcp -- which caches exactly one cached_objcg -- thrashes on workloads where threads of the same memcg run on different NUMA nodes. The kernel test robot reported a 67.7% regression on stress-ng.switch.ops_per_sec from this pattern. Mirror the multi-slot pattern already used by memcg_stock_pcp: turn nr_bytes and cached_objcg into NR_OBJ_STOCK-element arrays, scan all slots on consume/refill/account, prefer empty slots when inserting, and evict a random slot only when full. With multiple slots a CPU can hold the per-node objcg variants of one memcg plus a few siblings without ever forcing a drain. A single int8_t index records which slot the cached slab stats belong to; the stats are flushed on slot or pgdat change. With NR_OBJ_STOCK =3D 5 the layout (verified with pahole) is: offset 0 : lock(1) + index(1) + node_id(2) + slab stats(4) =3D 8B offset 8 : nr_bytes[5] =3D 10B offset 18 : padding =3D 6B offset 24 : cached[5] =3D 40B offset 64 : (line 2) work_struct + flags (cold) so consume_obj_stock, refill_obj_stock and the slab account path each touch exactly one 64-byte cache line on non-debug 64-bit builds. Reported-by: kernel test robot Closes: https://lore.kernel.org/oe-lkp/202605121641.b6a60cb0-lkp@intel.com Fixes: 01b9da291c49 ("mm: memcontrol: convert objcg to be per-memcg per-nod= e type") Signed-off-by: Shakeel Butt Tested-by: kernel test robot --- Changes since v1: - Use round robin for drain mm/memcontrol.c | 188 ++++++++++++++++++++++++++++++++++-------------- 1 file changed, 136 insertions(+), 52 deletions(-) diff --git a/mm/memcontrol.c b/mm/memcontrol.c index 78c02451312b..ba17633b0bd0 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -150,14 +150,14 @@ static void obj_cgroup_release(struct percpu_ref *ref) * However, it can be PAGE_SIZE or (x * PAGE_SIZE). * * The following sequence can lead to it: - * 1) CPU0: objcg =3D=3D stock->cached_objcg + * 1) CPU0: objcg cached in one of stock->cached[i] * 2) CPU1: we do a small allocation (e.g. 92 bytes), * PAGE_SIZE bytes are charged * 3) CPU1: a process from another memcg is allocating something, * the stock if flushed, * objcg->nr_charged_bytes =3D PAGE_SIZE - 92 * 5) CPU0: we do release this object, - * 92 bytes are added to stock->nr_bytes + * 92 bytes are added to stock->nr_bytes[i] * 6) CPU0: stock is flushed, * 92 bytes are added to objcg->nr_charged_bytes * @@ -2017,25 +2017,40 @@ static DEFINE_PER_CPU_ALIGNED(struct memcg_stock_pc= p, memcg_stock) =3D { .lock =3D INIT_LOCAL_TRYLOCK(lock), }; =20 +/* + * NR_OBJ_STOCK is sized so the entire hot path of obj_stock_pcp + * (lock, accounting metadata, nr_bytes[] and cached[]) fits within a + * single 64-byte cache line on non-debug 64-bit builds. With 5 slots: + * lock(1) + index(1) + node_id(2) + slab stats(4) + nr_bytes(10) + * + pad(6) + cached(40) =3D=3D 64 bytes. + * A CPU can thus consume/refill/account against five different objcgs + * (typically per-node variants of the same memcg) while incurring at + * most one cache miss on the stock. + */ +#define NR_OBJ_STOCK 5 struct obj_stock_pcp { local_trylock_t lock; - struct obj_cgroup *cached_objcg; - uint16_t nr_bytes; + int8_t index; int16_t node_id; int16_t nr_slab_reclaimable_b; int16_t nr_slab_unreclaimable_b; + uint16_t nr_bytes[NR_OBJ_STOCK]; + struct obj_cgroup *cached[NR_OBJ_STOCK]; =20 struct work_struct work; unsigned long flags; + uint8_t drain_idx; }; =20 static DEFINE_PER_CPU_ALIGNED(struct obj_stock_pcp, obj_stock) =3D { .lock =3D INIT_LOCAL_TRYLOCK(lock), + .index =3D -1, .node_id =3D NUMA_NO_NODE, }; =20 static DEFINE_MUTEX(percpu_charge_mutex); =20 +static void drain_obj_stock_slot(struct obj_stock_pcp *stock, int i); static void drain_obj_stock(struct obj_stock_pcp *stock); static bool obj_stock_flush_required(struct obj_stock_pcp *stock, struct mem_cgroup *root_memcg); @@ -3153,12 +3168,13 @@ static void unlock_stock(struct obj_stock_pcp *stoc= k) local_unlock(&obj_stock.lock); } =20 -/* Call after __refill_obj_stock() to ensure stock->cached_objg =3D=3D obj= cg */ +/* Call after __refill_obj_stock() so a slot for objcg exists in the stock= */ static void __account_obj_stock(struct obj_cgroup *objcg, struct obj_stock_pcp *stock, int nr, struct pglist_data *pgdat, enum node_stat_item idx) { int16_t *bytes; + int i; =20 /* * Though at the moment MAX_NUMNODES <=3D 1024 in all archs but let's make @@ -3167,29 +3183,39 @@ static void __account_obj_stock(struct obj_cgroup *= objcg, */ BUILD_BUG_ON(MAX_NUMNODES >=3D S16_MAX); =20 - if (!stock || READ_ONCE(stock->cached_objcg) !=3D objcg) + if (!stock) + goto direct; + + for (i =3D 0; i < NR_OBJ_STOCK; ++i) { + if (READ_ONCE(stock->cached[i]) =3D=3D objcg) + break; + } + if (i =3D=3D NR_OBJ_STOCK) goto direct; =20 /* * Save vmstat data in stock and skip vmstat array update unless - * accumulating over a page of vmstat data or when pgdat changes. + * accumulating over a page of vmstat data or when the objcg slot or + * pgdat the stats belong to changes. */ - if (stock->node_id =3D=3D NUMA_NO_NODE) { + if (stock->index < 0) { + stock->index =3D i; stock->node_id =3D pgdat->node_id; - } else if (stock->node_id !=3D pgdat->node_id) { - /* Flush the existing cached vmstat data */ + } else if (stock->index !=3D i || stock->node_id !=3D pgdat->node_id) { + struct obj_cgroup *old =3D READ_ONCE(stock->cached[stock->index]); struct pglist_data *oldpg =3D NODE_DATA(stock->node_id); =20 if (stock->nr_slab_reclaimable_b) { - mod_objcg_mlstate(objcg, oldpg, NR_SLAB_RECLAIMABLE_B, + mod_objcg_mlstate(old, oldpg, NR_SLAB_RECLAIMABLE_B, stock->nr_slab_reclaimable_b); stock->nr_slab_reclaimable_b =3D 0; } if (stock->nr_slab_unreclaimable_b) { - mod_objcg_mlstate(objcg, oldpg, NR_SLAB_UNRECLAIMABLE_B, + mod_objcg_mlstate(old, oldpg, NR_SLAB_UNRECLAIMABLE_B, stock->nr_slab_unreclaimable_b); stock->nr_slab_unreclaimable_b =3D 0; } + stock->index =3D i; stock->node_id =3D pgdat->node_id; } =20 @@ -3230,10 +3256,16 @@ static bool __consume_obj_stock(struct obj_cgroup *= objcg, struct obj_stock_pcp *stock, unsigned int nr_bytes) { - if (objcg =3D=3D READ_ONCE(stock->cached_objcg) && - stock->nr_bytes >=3D nr_bytes) { - stock->nr_bytes -=3D nr_bytes; - return true; + int i; + + for (i =3D 0; i < NR_OBJ_STOCK; ++i) { + if (READ_ONCE(stock->cached[i]) !=3D objcg) + continue; + if (stock->nr_bytes[i] >=3D nr_bytes) { + stock->nr_bytes[i] -=3D nr_bytes; + return true; + } + return false; } =20 return false; @@ -3254,16 +3286,42 @@ static bool consume_obj_stock(struct obj_cgroup *ob= jcg, unsigned int nr_bytes) return ret; } =20 -static void drain_obj_stock(struct obj_stock_pcp *stock) +/* Flush the cached slab stats (if any) back to their owning objcg/pgdat. = */ +static void drain_obj_stock_stats(struct obj_stock_pcp *stock) { - struct obj_cgroup *old =3D READ_ONCE(stock->cached_objcg); + struct obj_cgroup *old; + struct pglist_data *oldpg; + + if (stock->index < 0) + return; + + old =3D READ_ONCE(stock->cached[stock->index]); + oldpg =3D NODE_DATA(stock->node_id); + + if (stock->nr_slab_reclaimable_b) { + mod_objcg_mlstate(old, oldpg, NR_SLAB_RECLAIMABLE_B, + stock->nr_slab_reclaimable_b); + stock->nr_slab_reclaimable_b =3D 0; + } + if (stock->nr_slab_unreclaimable_b) { + mod_objcg_mlstate(old, oldpg, NR_SLAB_UNRECLAIMABLE_B, + stock->nr_slab_unreclaimable_b); + stock->nr_slab_unreclaimable_b =3D 0; + } + stock->index =3D -1; + stock->node_id =3D NUMA_NO_NODE; +} + +static void drain_obj_stock_slot(struct obj_stock_pcp *stock, int i) +{ + struct obj_cgroup *old =3D READ_ONCE(stock->cached[i]); =20 if (!old) return; =20 - if (stock->nr_bytes) { - unsigned int nr_pages =3D stock->nr_bytes >> PAGE_SHIFT; - unsigned int nr_bytes =3D stock->nr_bytes & (PAGE_SIZE - 1); + if (stock->nr_bytes[i]) { + unsigned int nr_pages =3D stock->nr_bytes[i] >> PAGE_SHIFT; + unsigned int nr_bytes =3D stock->nr_bytes[i] & (PAGE_SIZE - 1); =20 if (nr_pages) { struct mem_cgroup *memcg; @@ -3289,46 +3347,43 @@ static void drain_obj_stock(struct obj_stock_pcp *s= tock) * so it might be changed in the future. */ atomic_add(nr_bytes, &old->nr_charged_bytes); - stock->nr_bytes =3D 0; + stock->nr_bytes[i] =3D 0; } =20 - /* - * Flush the vmstat data in current stock - */ - if (stock->nr_slab_reclaimable_b || stock->nr_slab_unreclaimable_b) { - struct pglist_data *oldpg =3D NODE_DATA(stock->node_id); - - if (stock->nr_slab_reclaimable_b) { - mod_objcg_mlstate(old, oldpg, - NR_SLAB_RECLAIMABLE_B, - stock->nr_slab_reclaimable_b); - stock->nr_slab_reclaimable_b =3D 0; - } - if (stock->nr_slab_unreclaimable_b) { - mod_objcg_mlstate(old, oldpg, - NR_SLAB_UNRECLAIMABLE_B, - stock->nr_slab_unreclaimable_b); - stock->nr_slab_unreclaimable_b =3D 0; - } - stock->node_id =3D NUMA_NO_NODE; - } + /* Flush vmstat data when its owning slot is being drained. */ + if (stock->index =3D=3D i) + drain_obj_stock_stats(stock); =20 - WRITE_ONCE(stock->cached_objcg, NULL); + WRITE_ONCE(stock->cached[i], NULL); obj_cgroup_put(old); } =20 +static void drain_obj_stock(struct obj_stock_pcp *stock) +{ + int i; + + for (i =3D 0; i < NR_OBJ_STOCK; ++i) + drain_obj_stock_slot(stock, i); +} + static bool obj_stock_flush_required(struct obj_stock_pcp *stock, struct mem_cgroup *root_memcg) { - struct obj_cgroup *objcg =3D READ_ONCE(stock->cached_objcg); + struct obj_cgroup *objcg; struct mem_cgroup *memcg; bool flush =3D false; + int i; =20 rcu_read_lock(); - if (objcg) { + for (i =3D 0; i < NR_OBJ_STOCK; ++i) { + objcg =3D READ_ONCE(stock->cached[i]); + if (!objcg) + continue; memcg =3D obj_cgroup_memcg(objcg); - if (memcg && mem_cgroup_is_descendant(memcg, root_memcg)) + if (memcg && mem_cgroup_is_descendant(memcg, root_memcg)) { flush =3D true; + break; + } } rcu_read_unlock(); =20 @@ -3342,6 +3397,7 @@ static void __refill_obj_stock(struct obj_cgroup *obj= cg, { unsigned int nr_pages =3D 0; unsigned int stock_nr_bytes; + int i, slot =3D -1, empty_slot =3D -1; =20 if (!stock) { nr_pages =3D nr_bytes >> PAGE_SHIFT; @@ -3350,19 +3406,47 @@ static void __refill_obj_stock(struct obj_cgroup *o= bjcg, goto out; } =20 - stock_nr_bytes =3D stock->nr_bytes; - if (READ_ONCE(stock->cached_objcg) !=3D objcg) { /* reset if necessary */ - drain_obj_stock(stock); + for (i =3D 0; i < NR_OBJ_STOCK; ++i) { + struct obj_cgroup *cached =3D READ_ONCE(stock->cached[i]); + + if (!cached) { + if (empty_slot =3D=3D -1) + empty_slot =3D i; + continue; + } + if (cached =3D=3D objcg) { + slot =3D i; + break; + } + } + + if (slot =3D=3D -1) { + slot =3D empty_slot; + if (slot =3D=3D -1) { + slot =3D stock->drain_idx++; + if (stock->drain_idx =3D=3D NR_OBJ_STOCK) + stock->drain_idx =3D 0; + drain_obj_stock_slot(stock, slot); + } obj_cgroup_get(objcg); + /* + * Keep the xchg result in the unsigned int local; storing + * it directly into stock->nr_bytes[slot] (uint16_t) would + * silently truncate values >=3D U16_MAX and bypass the flush + * guard below, leaking page-counter charges. + */ stock_nr_bytes =3D atomic_read(&objcg->nr_charged_bytes) ? atomic_xchg(&objcg->nr_charged_bytes, 0) : 0; - WRITE_ONCE(stock->cached_objcg, objcg); + WRITE_ONCE(stock->cached[slot], objcg); =20 allow_uncharge =3D true; /* Allow uncharge when objcg changes */ + } else { + stock_nr_bytes =3D stock->nr_bytes[slot]; } + stock_nr_bytes +=3D nr_bytes; =20 - /* Since stock->nr_bytes is uint16_t, don't refill >=3D U16_MAX */ + /* nr_bytes[] is uint16_t; flush if we would refill >=3D U16_MAX. */ if ((allow_uncharge && (stock_nr_bytes > PAGE_SIZE)) || stock_nr_bytes > U16_MAX) { nr_pages =3D stock_nr_bytes >> PAGE_SHIFT; @@ -3384,7 +3468,7 @@ static void __refill_obj_stock(struct obj_cgroup *obj= cg, stock_nr_bytes =3D kept; } } - stock->nr_bytes =3D stock_nr_bytes; + stock->nr_bytes[slot] =3D stock_nr_bytes; =20 out: if (nr_pages) --=20 2.53.0-Meta