From nobody Wed Dec 17 22:40:58 2025 Received: from out-181.mta0.migadu.com (out-181.mta0.migadu.com [91.218.175.181]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 20F1B151998 for ; Fri, 14 Mar 2025 06:15:31 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=91.218.175.181 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1741932933; cv=none; b=enJ3vTKaIKA9wsoFy9+b7ybQ1ivrwPVcBOqLTpILq+m4ziJvgAF+ZCnwyZDW7Q7m/wtsqDhzGeLGc4JUiJNqX3MmMuos7PhYC9m03XtEmJDoENsuvZN/KLWEb9jC85F27O7WUOiYNFNyrXWpYMoFhS4QLtqYmmKGk31Tqqy0haA= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1741932933; c=relaxed/simple; bh=rR4VRO/qCn9tibn4IiHZRnKTQxDzvVTiAx2JBG5cZJY=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=l1chhc3hgqpNicCeG6h11v0xZ9ostsUrvRl9NbZeQj+u/71CLDHoRVJ7W0+xh09s97SypezrP4sdlMkPQ71o2DTjd/rlfmloe5TpB7I+UmmyHBkc2nFxkSTbKFez3VCpozsHfm7trM+MIPduSQMI2GNOM9SBvk4EoRQJi+woTFc= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=linux.dev; spf=pass smtp.mailfrom=linux.dev; dkim=pass (1024-bit key) header.d=linux.dev header.i=@linux.dev header.b=IFAk3ny3; arc=none smtp.client-ip=91.218.175.181 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=linux.dev Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=linux.dev Authentication-Results: smtp.subspace.kernel.org; dkim=pass (1024-bit key) header.d=linux.dev header.i=@linux.dev header.b="IFAk3ny3" X-Report-Abuse: Please report any abuse attempt to abuse@migadu.com and include these headers. DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=linux.dev; s=key1; t=1741932928; h=from:from:reply-to:subject:subject:date:date:message-id:message-id: to:to:cc:cc:mime-version:mime-version: content-transfer-encoding:content-transfer-encoding: in-reply-to:in-reply-to:references:references; bh=cmS5hv6xwjGOhhAkz0IaR50zSoZuuoSBRO/S5eqp4mA=; b=IFAk3ny3UqE/iQCJa7lOm0f18gE7d5o7rhlz8Rsi6NYU4Rbjj03rJ/JizQmPAY0O6BVqup yEGTkuAJggm5Z9B13K0mwm3oUigTkR+xQtR8TVgwbpmnqyYUpzzkXSWkBOkqt3KwZEJQDf A6ZNhSvExzpbiw9PsZYYi1jj0cgmXsA= From: Shakeel Butt To: Andrew Morton Cc: Johannes Weiner , Michal Hocko , Roman Gushchin , Muchun Song , Vlastimil Babka , Sebastian Andrzej Siewior , linux-mm@kvack.org, cgroups@vger.kernel.org, linux-kernel@vger.kernel.org, Meta kernel team Subject: [RFC PATCH 01/10] memcg: remove root memcg check from refill_stock Date: Thu, 13 Mar 2025 23:15:02 -0700 Message-ID: <20250314061511.1308152-2-shakeel.butt@linux.dev> In-Reply-To: <20250314061511.1308152-1-shakeel.butt@linux.dev> References: <20250314061511.1308152-1-shakeel.butt@linux.dev> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable X-Migadu-Flow: FLOW_OUT Content-Type: text/plain; charset="utf-8" refill_stock can not be called with root memcg, so there is no need to check it. Signed-off-by: Shakeel Butt --- mm/memcontrol.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/mm/memcontrol.c b/mm/memcontrol.c index b29433eb17fa..c09a32e93d39 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -1883,6 +1883,7 @@ static void __refill_stock(struct mem_cgroup *memcg, = unsigned int nr_pages) drain_stock(stock); } =20 +/* Should never be called with root_mem_cgroup. */ static void refill_stock(struct mem_cgroup *memcg, unsigned int nr_pages) { unsigned long flags; @@ -1892,8 +1893,6 @@ static void refill_stock(struct mem_cgroup *memcg, un= signed int nr_pages) * In case of unlikely failure to lock percpu stock_lock * uncharge memcg directly. */ - if (mem_cgroup_is_root(memcg)) - return; page_counter_uncharge(&memcg->memory, nr_pages); if (do_memsw_account()) page_counter_uncharge(&memcg->memsw, nr_pages); --=20 2.47.1 From nobody Wed Dec 17 22:40:58 2025 Received: from out-182.mta0.migadu.com (out-182.mta0.migadu.com [91.218.175.182]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id CD82B566A for ; Fri, 14 Mar 2025 06:15:33 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=91.218.175.182 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1741932935; cv=none; b=WNuKP3Eca3T/CqS5XbpmFMJlemLujXlNMaYlMGJYbpJvQ7PWS6YByEHLxqVow6RYWUaB7NNU1ifbrHns5VeyGlH04SQEqb9tlZ5/T44XKRFHCb8I9D0Q69Berhkggqil0T8j8ILRPZkZpb3leEmD6gFW2NuBXwbp5voyCZMAZio= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1741932935; c=relaxed/simple; bh=W/+dT0JeMfCNVWYLzWA9JLxOlfjfaJDgqzXPPWZp9qA=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=qc6q4i1XwfBkIjE+RnMwy0uzhw6jy3tyRqUrjLgE7Genp8/p+ZZq3eqWK4xnijrVncErniPvOqR9/cd5UG6GTURaVy23N6hUiMKVk4+ufcnCRDl1k/anBs9Iqt2VP/gA0Ad3vOq0buUtV/XQa8C0pRcS0K6hQD7/FYaDrm29L+U= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=linux.dev; spf=pass smtp.mailfrom=linux.dev; dkim=pass (1024-bit key) header.d=linux.dev header.i=@linux.dev header.b=FeQvgZ0y; arc=none smtp.client-ip=91.218.175.182 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=linux.dev Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=linux.dev Authentication-Results: smtp.subspace.kernel.org; dkim=pass (1024-bit key) header.d=linux.dev header.i=@linux.dev header.b="FeQvgZ0y" X-Report-Abuse: Please report any abuse attempt to abuse@migadu.com and include these headers. DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=linux.dev; s=key1; t=1741932932; h=from:from:reply-to:subject:subject:date:date:message-id:message-id: to:to:cc:cc:mime-version:mime-version: content-transfer-encoding:content-transfer-encoding: in-reply-to:in-reply-to:references:references; bh=4gcQPtwIR/hVm1yRr4cRacE1bGrzLqevTybXQMDqJ14=; b=FeQvgZ0yXEZ1YmVQs936nWoNHKfUiPJcFKFvpqK16/CwDxmnMaVmA7QENZTzMSIQbHs7WV BdIx05IJWLFoRUTmqjG0ITtgCcDG/oQDslGjV7+zAa+Ar1pyCIytzoVik0svc/mmlR+U1G xzLDhIBey5JN/Ok/Tv5oW3YZhsF2aIA= From: Shakeel Butt To: Andrew Morton Cc: Johannes Weiner , Michal Hocko , Roman Gushchin , Muchun Song , Vlastimil Babka , Sebastian Andrzej Siewior , linux-mm@kvack.org, cgroups@vger.kernel.org, linux-kernel@vger.kernel.org, Meta kernel team Subject: [RFC PATCH 02/10] memcg: decouple drain_obj_stock from local stock Date: Thu, 13 Mar 2025 23:15:03 -0700 Message-ID: <20250314061511.1308152-3-shakeel.butt@linux.dev> In-Reply-To: <20250314061511.1308152-1-shakeel.butt@linux.dev> References: <20250314061511.1308152-1-shakeel.butt@linux.dev> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable X-Migadu-Flow: FLOW_OUT Content-Type: text/plain; charset="utf-8" Currently drain_obj_stock() can potentially call __refill_stock which accesses local cpu stock and thus requires memcg stock's local_lock. However if we look at the code paths leading to drain_obj_stock(), there is never a good reason to refill the memcg stock at all from it. At the moment, drain_obj_stock can be called from reclaim, hotplug cpu teardown, mod_objcg_state() and refill_obj_stock(). For reclaim and hotplug there is no need to refill. For the other two paths, most probably the newly switched objcg would be used in near future and thus no need to refill stock with the older objcg. In addition, __refill_stock() from drain_obj_stock() happens on rare cases, so performance is not really an issue. Let's just uncharge directly instead of refill which will also decouple drain_obj_stock from local cpu stock and local_lock requirements. Signed-off-by: Shakeel Butt Acked-by: Vlastimil Babka --- mm/memcontrol.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/mm/memcontrol.c b/mm/memcontrol.c index c09a32e93d39..28cb75b5bc66 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -2855,7 +2855,12 @@ static struct obj_cgroup *drain_obj_stock(struct mem= cg_stock_pcp *stock) =20 mod_memcg_state(memcg, MEMCG_KMEM, -nr_pages); memcg1_account_kmem(memcg, -nr_pages); - __refill_stock(memcg, nr_pages); + if (!mem_cgroup_is_root(memcg)) { + page_counter_uncharge(&memcg->memory, nr_pages); + if (do_memsw_account()) + page_counter_uncharge(&memcg->memsw, + nr_pages); + } =20 css_put(&memcg->css); } --=20 2.47.1 From nobody Wed Dec 17 22:40:58 2025 Received: from out-189.mta1.migadu.com (out-189.mta1.migadu.com [95.215.58.189]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 2FBDA18C039 for ; Fri, 14 Mar 2025 06:15:46 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=95.215.58.189 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1741932949; cv=none; b=qUPQGmTKskSUhIPImbYKFgowIJ4ry1EXYPh4Lqy9UfvNQ/fDDpZvSxSTic93U05GA8Z2wtfNPRnGXvy8jkPPEqRZ9XgXtLyylmxVGm7uuMeNyr9EVlpg5OEJz6eibrwkSrAT+2r5c7Wmoj1SPu8flO5AE0cGq2qxkMVEcQTu7Rw= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1741932949; c=relaxed/simple; bh=zuchniEhDmUBC9X7u8R4xdltILbNO0cwprPM/OBuxLE=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=RanDjRvGw51KYWIoYnyS6eXVEenmBxvIozj8tf054tf1T4E/OT+0kctpuTig/2AnSrkgH/VLZ5H+5aEojBm31DGFXJ6aLyD+cxZHfq+hj7q38bX5Xgiu2Fn1mrAmR9hJdKOREXZHF4cYEODMX48dgbxcoQBNI5pe0I3eaVQf+aA= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=linux.dev; spf=pass smtp.mailfrom=linux.dev; dkim=pass (1024-bit key) header.d=linux.dev header.i=@linux.dev header.b=K7191N8V; arc=none smtp.client-ip=95.215.58.189 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=linux.dev Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=linux.dev Authentication-Results: smtp.subspace.kernel.org; dkim=pass (1024-bit key) header.d=linux.dev header.i=@linux.dev header.b="K7191N8V" X-Report-Abuse: Please report any abuse attempt to abuse@migadu.com and include these headers. DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=linux.dev; s=key1; t=1741932935; h=from:from:reply-to:subject:subject:date:date:message-id:message-id: to:to:cc:cc:mime-version:mime-version: content-transfer-encoding:content-transfer-encoding: in-reply-to:in-reply-to:references:references; bh=Nc+pYdC/xOdelwTJP89jkCPlJBy7KhpTV1fBT5yIKFQ=; b=K7191N8VG4pwJwKAtbd57HzRApbLYVKRwODKb5dG+bBi82YQH9CObHi+r08ica5eqBu7tJ xD77JPOtkH2IbGmEsx+h8XqYr/yBfI8NF7nycM1cLg4A2nsTvABOYTPnxQkKGJENlzd59V yqtmNKe3bpW0fqRXy5bQUx64hkyFLiY= From: Shakeel Butt To: Andrew Morton Cc: Johannes Weiner , Michal Hocko , Roman Gushchin , Muchun Song , Vlastimil Babka , Sebastian Andrzej Siewior , linux-mm@kvack.org, cgroups@vger.kernel.org, linux-kernel@vger.kernel.org, Meta kernel team Subject: [RFC PATCH 03/10] memcg: introduce memcg_uncharge Date: Thu, 13 Mar 2025 23:15:04 -0700 Message-ID: <20250314061511.1308152-4-shakeel.butt@linux.dev> In-Reply-To: <20250314061511.1308152-1-shakeel.butt@linux.dev> References: <20250314061511.1308152-1-shakeel.butt@linux.dev> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable X-Migadu-Flow: FLOW_OUT Content-Type: text/plain; charset="utf-8" At multiple places in memcontrol.c, the memory and memsw page counters are being uncharged. This is error-prone. Let's move the functionality to a newly introduced memcg_uncharge and call it from all those places. Signed-off-by: Shakeel Butt Acked-by: Vlastimil Babka --- mm/memcontrol.c | 28 ++++++++++++---------------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/mm/memcontrol.c b/mm/memcontrol.c index 28cb75b5bc66..b54e3a1d23bd 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -1816,6 +1816,13 @@ static bool consume_stock(struct mem_cgroup *memcg, = unsigned int nr_pages, return ret; } =20 +static void memcg_uncharge(struct mem_cgroup *memcg, unsigned int nr_pages) +{ + page_counter_uncharge(&memcg->memory, nr_pages); + if (do_memsw_account()) + page_counter_uncharge(&memcg->memsw, nr_pages); +} + /* * Returns stocks cached in percpu and reset cached information. */ @@ -1828,10 +1835,7 @@ static void drain_stock(struct memcg_stock_pcp *stoc= k) return; =20 if (stock_pages) { - page_counter_uncharge(&old->memory, stock_pages); - if (do_memsw_account()) - page_counter_uncharge(&old->memsw, stock_pages); - + memcg_uncharge(old, stock_pages); WRITE_ONCE(stock->nr_pages, 0); } =20 @@ -1893,9 +1897,7 @@ static void refill_stock(struct mem_cgroup *memcg, un= signed int nr_pages) * In case of unlikely failure to lock percpu stock_lock * uncharge memcg directly. */ - page_counter_uncharge(&memcg->memory, nr_pages); - if (do_memsw_account()) - page_counter_uncharge(&memcg->memsw, nr_pages); + memcg_uncharge(memcg, nr_pages); return; } __refill_stock(memcg, nr_pages); @@ -2855,12 +2857,8 @@ static struct obj_cgroup *drain_obj_stock(struct mem= cg_stock_pcp *stock) =20 mod_memcg_state(memcg, MEMCG_KMEM, -nr_pages); memcg1_account_kmem(memcg, -nr_pages); - if (!mem_cgroup_is_root(memcg)) { - page_counter_uncharge(&memcg->memory, nr_pages); - if (do_memsw_account()) - page_counter_uncharge(&memcg->memsw, - nr_pages); - } + if (!mem_cgroup_is_root(memcg)) + memcg_uncharge(memcg, nr_pages); =20 css_put(&memcg->css); } @@ -4689,9 +4687,7 @@ static inline void uncharge_gather_clear(struct uncha= rge_gather *ug) static void uncharge_batch(const struct uncharge_gather *ug) { if (ug->nr_memory) { - page_counter_uncharge(&ug->memcg->memory, ug->nr_memory); - if (do_memsw_account()) - page_counter_uncharge(&ug->memcg->memsw, ug->nr_memory); + memcg_uncharge(ug->memcg, ug->nr_memory); if (ug->nr_kmem) { mod_memcg_state(ug->memcg, MEMCG_KMEM, -ug->nr_kmem); memcg1_account_kmem(ug->memcg, -ug->nr_kmem); --=20 2.47.1 From nobody Wed Dec 17 22:40:58 2025 Received: from out-172.mta0.migadu.com (out-172.mta0.migadu.com [91.218.175.172]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 1AA72153808 for ; Fri, 14 Mar 2025 06:15:39 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=91.218.175.172 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1741932941; cv=none; b=ZmFS8ricgKgFYDJyU3PmKZllMlAM2yEpv6ok7nKGWTalSPuaHHMAqQCJHXQgroa80m7ia57OLP/q+WxQZjo/hARQD81+ADhh9O0CM+rGP99QwaaloDNklku0KrdvvdhmbazAyFw3CT3fikSVRZEs9vgX4YBSSgCQS7pb4t4bAi8= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1741932941; c=relaxed/simple; bh=q7a/l+3dxZyatbmA+x05mtthRfHxYV4Q69VBf6eryFQ=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=XDvyygzNjhgEt7aL9WNf+rXc6+Z26AlyVhC8njs95STFLQWFcI7YBiBLTezmS+sTqkikbY+1Dz7O8A7N7BFjOCs1Nxs7ygY//7dBBtanP4/JblpEZUdafWq5EbdrnFadAwiP1TNQ1MHUHQdRmJhOZaeVb1CWjGwq6u3qyJYRcUw= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=linux.dev; spf=pass smtp.mailfrom=linux.dev; dkim=pass (1024-bit key) header.d=linux.dev header.i=@linux.dev header.b=lW53X41+; arc=none smtp.client-ip=91.218.175.172 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=linux.dev Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=linux.dev Authentication-Results: smtp.subspace.kernel.org; dkim=pass (1024-bit key) header.d=linux.dev header.i=@linux.dev header.b="lW53X41+" X-Report-Abuse: Please report any abuse attempt to abuse@migadu.com and include these headers. DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=linux.dev; s=key1; t=1741932938; h=from:from:reply-to:subject:subject:date:date:message-id:message-id: to:to:cc:cc:mime-version:mime-version: content-transfer-encoding:content-transfer-encoding: in-reply-to:in-reply-to:references:references; bh=GHfMqiybfg/wadzE/r4pC5kSUqcreJExX0T8UoeIBCE=; b=lW53X41+bXtE/Pccr68GzJy8M1OGwPSERF3WpfrSeM1+eRyR6ZO5RUE+LGGoKkOl7MQME4 6FhWXE8Ek04ZXXxR8nAGVb7rCa4kXqwDs+rNp9NJ0Yia1I8mVy/nQvi2NZZY0xRfeuZimx iF9NPX2PD5aazVpGSYv6Fk0c5u38YjU= From: Shakeel Butt To: Andrew Morton Cc: Johannes Weiner , Michal Hocko , Roman Gushchin , Muchun Song , Vlastimil Babka , Sebastian Andrzej Siewior , linux-mm@kvack.org, cgroups@vger.kernel.org, linux-kernel@vger.kernel.org, Meta kernel team Subject: [RFC PATCH 04/10] memcg: manually inline __refill_stock Date: Thu, 13 Mar 2025 23:15:05 -0700 Message-ID: <20250314061511.1308152-5-shakeel.butt@linux.dev> In-Reply-To: <20250314061511.1308152-1-shakeel.butt@linux.dev> References: <20250314061511.1308152-1-shakeel.butt@linux.dev> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable X-Migadu-Flow: FLOW_OUT Content-Type: text/plain; charset="utf-8" There are no more multiple callers of __refill_stock(), so simply inline it to refill_stock(). Signed-off-by: Shakeel Butt Acked-by: Vlastimil Babka --- mm/memcontrol.c | 32 ++++++++++++-------------------- 1 file changed, 12 insertions(+), 20 deletions(-) diff --git a/mm/memcontrol.c b/mm/memcontrol.c index b54e3a1d23bd..7054b0ebd207 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -1865,14 +1865,21 @@ static void drain_local_stock(struct work_struct *d= ummy) obj_cgroup_put(old); } =20 -/* - * Cache charges(val) to local per_cpu area. - * This will be consumed by consume_stock() function, later. - */ -static void __refill_stock(struct mem_cgroup *memcg, unsigned int nr_pages) +/* Should never be called with root_mem_cgroup. */ +static void refill_stock(struct mem_cgroup *memcg, unsigned int nr_pages) { struct memcg_stock_pcp *stock; unsigned int stock_pages; + unsigned long flags; + + if (!localtry_trylock_irqsave(&memcg_stock.stock_lock, flags)) { + /* + * In case of unlikely failure to lock percpu stock_lock + * uncharge memcg directly. + */ + memcg_uncharge(memcg, nr_pages); + return; + } =20 stock =3D this_cpu_ptr(&memcg_stock); if (READ_ONCE(stock->cached) !=3D memcg) { /* reset if necessary */ @@ -1885,22 +1892,7 @@ static void __refill_stock(struct mem_cgroup *memcg,= unsigned int nr_pages) =20 if (stock_pages > MEMCG_CHARGE_BATCH) drain_stock(stock); -} =20 -/* Should never be called with root_mem_cgroup. */ -static void refill_stock(struct mem_cgroup *memcg, unsigned int nr_pages) -{ - unsigned long flags; - - if (!localtry_trylock_irqsave(&memcg_stock.stock_lock, flags)) { - /* - * In case of unlikely failure to lock percpu stock_lock - * uncharge memcg directly. - */ - memcg_uncharge(memcg, nr_pages); - return; - } - __refill_stock(memcg, nr_pages); localtry_unlock_irqrestore(&memcg_stock.stock_lock, flags); } =20 --=20 2.47.1 From nobody Wed Dec 17 22:40:58 2025 Received: from out-178.mta1.migadu.com (out-178.mta1.migadu.com [95.215.58.178]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 2E65D186E56 for ; Fri, 14 Mar 2025 06:15:43 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=95.215.58.178 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1741932944; cv=none; b=Ntb8dTZvOvVja6LIdhwSMNBcbkgySAkosgcZAMu2O/sOaSdrOxHuA9cROF2+b+/Nug3ghY6cFRzMgDAKfxDqmRo9mOKowbMUjkJj+BAxjoqqU+Iw2y0eUvmRhRWzwBC4roJay3IOXKusRtcCQO5BSQoL9fQMaCUDeLARSwks/7A= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1741932944; c=relaxed/simple; bh=ivi2Ai5pbO72ygi8JeAdlS/MqXoqGjtJwT11DBtAaM4=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=gL68zFgmxDTUWIAM1q1DVaLpooe3bTFiVFUUkFs3cFPfQsCRIEI9OVaQcLPt+Qp82tNlgt1iXge0mFJHpFBW5XOCW7La50dV2LZIAMsB/q+YSqCXvz4iaWbtH/b4rRSLdxRHaliFWuKHKqELE9TY1qXhq4vPup9AJZBZkHeQFdM= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=linux.dev; spf=pass smtp.mailfrom=linux.dev; dkim=pass (1024-bit key) header.d=linux.dev header.i=@linux.dev header.b=A3ycuLnE; arc=none smtp.client-ip=95.215.58.178 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=linux.dev Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=linux.dev Authentication-Results: smtp.subspace.kernel.org; dkim=pass (1024-bit key) header.d=linux.dev header.i=@linux.dev header.b="A3ycuLnE" X-Report-Abuse: Please report any abuse attempt to abuse@migadu.com and include these headers. DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=linux.dev; s=key1; t=1741932941; h=from:from:reply-to:subject:subject:date:date:message-id:message-id: to:to:cc:cc:mime-version:mime-version: content-transfer-encoding:content-transfer-encoding: in-reply-to:in-reply-to:references:references; bh=lcDoiTd7xEiImi4APdwYNjABkW5nrlOItPgvjNpxcds=; b=A3ycuLnENDh5V07GWrH8Jxn+FR3OaxeVX1PN7A7UGAgmaark0HHfSnFiszqsZ4vp8xDkjs fPjVhxtQ7rRtNBIuopLUKdw0S2PB/6Ooq+Es0ODBtRgCobG9H0wTXDPAQOxpVL/z4Rv7uc 1aWOQV+aKTZuhIGf3oAQCTRqjvv0WUM= From: Shakeel Butt To: Andrew Morton Cc: Johannes Weiner , Michal Hocko , Roman Gushchin , Muchun Song , Vlastimil Babka , Sebastian Andrzej Siewior , linux-mm@kvack.org, cgroups@vger.kernel.org, linux-kernel@vger.kernel.org, Meta kernel team Subject: [RFC PATCH 05/10] memcg: no refilling stock from obj_cgroup_release Date: Thu, 13 Mar 2025 23:15:06 -0700 Message-ID: <20250314061511.1308152-6-shakeel.butt@linux.dev> In-Reply-To: <20250314061511.1308152-1-shakeel.butt@linux.dev> References: <20250314061511.1308152-1-shakeel.butt@linux.dev> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable X-Migadu-Flow: FLOW_OUT Content-Type: text/plain; charset="utf-8" obj_cgroup_release is called when all the references to the objcg has been released i.e. no more memory objects are pointing to it. Most probably objcg->memcg will be pointing to some ancestor memcg and at the moment, in obj_cgroup_release, the kernel call obj_cgroup_uncharge_pages() to uncharge last remaining memory. However obj_cgroup_uncharge_pages() refills the local stock. There is no need to refill the local stock with some ancestor memcg and flush the local stock. In addition this removes the requirement to only call obj_cgroup_put() outside of local_lock. Signed-off-by: Shakeel Butt Acked-by: Vlastimil Babka --- mm/memcontrol.c | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/mm/memcontrol.c b/mm/memcontrol.c index 7054b0ebd207..83db180455a1 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -129,8 +129,7 @@ bool mem_cgroup_kmem_disabled(void) return cgroup_memory_nokmem; } =20 -static void obj_cgroup_uncharge_pages(struct obj_cgroup *objcg, - unsigned int nr_pages); +static void memcg_uncharge(struct mem_cgroup *memcg, unsigned int nr_pages= ); =20 static void obj_cgroup_release(struct percpu_ref *ref) { @@ -163,8 +162,16 @@ static void obj_cgroup_release(struct percpu_ref *ref) WARN_ON_ONCE(nr_bytes & (PAGE_SIZE - 1)); nr_pages =3D nr_bytes >> PAGE_SHIFT; =20 - if (nr_pages) - obj_cgroup_uncharge_pages(objcg, nr_pages); + if (nr_pages) { + struct mem_cgroup *memcg; + + memcg =3D get_mem_cgroup_from_objcg(objcg); + mod_memcg_state(memcg, MEMCG_KMEM, -nr_pages); + memcg1_account_kmem(memcg, -nr_pages); + if (!mem_cgroup_is_root(memcg)) + memcg_uncharge(memcg, nr_pages); + css_put(&memcg->css); + } =20 spin_lock_irqsave(&objcg_lock, flags); list_del(&objcg->list); --=20 2.47.1 From nobody Wed Dec 17 22:40:58 2025 Received: from out-170.mta1.migadu.com (out-170.mta1.migadu.com [95.215.58.170]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 976BD18CBEC for ; Fri, 14 Mar 2025 06:15:46 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=95.215.58.170 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1741932948; cv=none; b=BMmhyrmZyf5NkxikUzU59rXIA6ORBZl1lUzUJ6ftn0USkHA3E1aG5AFCRJt+G6iH3XZqKlHgfI8I/PWOD/thqJeIAz01dxuB+U00KLMqypkuQgbpGmwmujlkMa7xZHRWtDATw3XenONuulKUvvvRQrHDiSsaLEtNDVeeSImp614= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1741932948; c=relaxed/simple; bh=LloJnKkIR/QdFvsz1WTvP10su9GsOgJ3e0RXRBD9G0s=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=tE1LwWZbkfEAqmyJzLo7RdRwrs2yusVGXtRQEiL5PaYXzrk2E2T4oR2ezlOV/QGH1IibTYjVB5Qj36k01M0qMXz1TT78rAjSpf7Nj7PWismyW4/A/cVtiqFHpRlVuYBhaQW6sS5J7Cqr6lkv0Ib0t2ikbpHDH8sGb6vdi4aBDnI= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=linux.dev; spf=pass smtp.mailfrom=linux.dev; dkim=pass (1024-bit key) header.d=linux.dev header.i=@linux.dev header.b=V274faEo; arc=none smtp.client-ip=95.215.58.170 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=linux.dev Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=linux.dev Authentication-Results: smtp.subspace.kernel.org; dkim=pass (1024-bit key) header.d=linux.dev header.i=@linux.dev header.b="V274faEo" X-Report-Abuse: Please report any abuse attempt to abuse@migadu.com and include these headers. DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=linux.dev; s=key1; t=1741932944; h=from:from:reply-to:subject:subject:date:date:message-id:message-id: to:to:cc:cc:mime-version:mime-version: content-transfer-encoding:content-transfer-encoding: in-reply-to:in-reply-to:references:references; bh=tHvct7NUO2otKVtcPkikbwH9IxeqmMsC6tyfV+hMsCI=; b=V274faEovU31wH6hmPF8Pl1ql5SCdkECGFNsM9aIg8fqkshlyZQjJkHlOiibZ5eK8EBIaF mzT2xavCrHmwrG2e71ysA2lRB0K2dCx5x92LDIJjzGPXTyyBhrFDUktB4v80nkwy47GVhp OIu/u1LFsSewxNzC4GuEWlmMp1zITI8= From: Shakeel Butt To: Andrew Morton Cc: Johannes Weiner , Michal Hocko , Roman Gushchin , Muchun Song , Vlastimil Babka , Sebastian Andrzej Siewior , linux-mm@kvack.org, cgroups@vger.kernel.org, linux-kernel@vger.kernel.org, Meta kernel team Subject: [RFC PATCH 06/10] memcg: do obj_cgroup_put inside drain_obj_stock Date: Thu, 13 Mar 2025 23:15:07 -0700 Message-ID: <20250314061511.1308152-7-shakeel.butt@linux.dev> In-Reply-To: <20250314061511.1308152-1-shakeel.butt@linux.dev> References: <20250314061511.1308152-1-shakeel.butt@linux.dev> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable X-Migadu-Flow: FLOW_OUT Content-Type: text/plain; charset="utf-8" Previously we could not call obj_cgroup_put() inside the local lock because on the put on the last reference, the release function obj_cgroup_release() may try to re-acquire the local lock. However that chain has been broken. Now simply do obj_cgroup_put() inside drain_obj_stock() instead of returning the old objcg. Signed-off-by: Shakeel Butt --- mm/memcontrol.c | 37 +++++++++++-------------------------- 1 file changed, 11 insertions(+), 26 deletions(-) diff --git a/mm/memcontrol.c b/mm/memcontrol.c index 83db180455a1..3c4de384b5a0 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -1778,7 +1778,7 @@ static DEFINE_PER_CPU(struct memcg_stock_pcp, memcg_s= tock) =3D { }; static DEFINE_MUTEX(percpu_charge_mutex); =20 -static struct obj_cgroup *drain_obj_stock(struct memcg_stock_pcp *stock); +static void drain_obj_stock(struct memcg_stock_pcp *stock); static bool obj_stock_flush_required(struct memcg_stock_pcp *stock, struct mem_cgroup *root_memcg); =20 @@ -1853,7 +1853,6 @@ static void drain_stock(struct memcg_stock_pcp *stock) static void drain_local_stock(struct work_struct *dummy) { struct memcg_stock_pcp *stock; - struct obj_cgroup *old =3D NULL; unsigned long flags; =20 /* @@ -1864,12 +1863,11 @@ static void drain_local_stock(struct work_struct *d= ummy) localtry_lock_irqsave(&memcg_stock.stock_lock, flags); =20 stock =3D this_cpu_ptr(&memcg_stock); - old =3D drain_obj_stock(stock); + drain_obj_stock(stock); drain_stock(stock); clear_bit(FLUSHING_CACHED_CHARGE, &stock->flags); =20 localtry_unlock_irqrestore(&memcg_stock.stock_lock, flags); - obj_cgroup_put(old); } =20 /* Should never be called with root_mem_cgroup. */ @@ -1951,18 +1949,16 @@ void drain_all_stock(struct mem_cgroup *root_memcg) static int memcg_hotplug_cpu_dead(unsigned int cpu) { struct memcg_stock_pcp *stock; - struct obj_cgroup *old; unsigned long flags; =20 stock =3D &per_cpu(memcg_stock, cpu); =20 /* drain_obj_stock requires stock_lock */ localtry_lock_irqsave(&memcg_stock.stock_lock, flags); - old =3D drain_obj_stock(stock); + drain_obj_stock(stock); localtry_unlock_irqrestore(&memcg_stock.stock_lock, flags); =20 drain_stock(stock); - obj_cgroup_put(old); =20 return 0; } @@ -2745,24 +2741,20 @@ void __memcg_kmem_uncharge_page(struct page *page, = int order) } =20 /* Replace the stock objcg with objcg, return the old objcg */ -static struct obj_cgroup *replace_stock_objcg(struct memcg_stock_pcp *stoc= k, - struct obj_cgroup *objcg) +static void replace_stock_objcg(struct memcg_stock_pcp *stock, + struct obj_cgroup *objcg) { - struct obj_cgroup *old =3D NULL; - - old =3D drain_obj_stock(stock); + drain_obj_stock(stock); obj_cgroup_get(objcg); stock->nr_bytes =3D atomic_read(&objcg->nr_charged_bytes) ? atomic_xchg(&objcg->nr_charged_bytes, 0) : 0; WRITE_ONCE(stock->cached_objcg, objcg); - return old; } =20 static void mod_objcg_state(struct obj_cgroup *objcg, struct pglist_data *= pgdat, enum node_stat_item idx, int nr) { struct memcg_stock_pcp *stock; - struct obj_cgroup *old =3D NULL; unsigned long flags; int *bytes; =20 @@ -2775,7 +2767,7 @@ static void mod_objcg_state(struct obj_cgroup *objcg,= struct pglist_data *pgdat, * changes. */ if (READ_ONCE(stock->cached_objcg) !=3D objcg) { - old =3D replace_stock_objcg(stock, objcg); + replace_stock_objcg(stock, objcg); stock->cached_pgdat =3D pgdat; } else if (stock->cached_pgdat !=3D pgdat) { /* Flush the existing cached vmstat data */ @@ -2816,7 +2808,6 @@ static void mod_objcg_state(struct obj_cgroup *objcg,= struct pglist_data *pgdat, __mod_objcg_mlstate(objcg, pgdat, idx, nr); =20 localtry_unlock_irqrestore(&memcg_stock.stock_lock, flags); - obj_cgroup_put(old); } =20 static bool consume_obj_stock(struct obj_cgroup *objcg, unsigned int nr_by= tes) @@ -2838,12 +2829,12 @@ static bool consume_obj_stock(struct obj_cgroup *ob= jcg, unsigned int nr_bytes) return ret; } =20 -static struct obj_cgroup *drain_obj_stock(struct memcg_stock_pcp *stock) +static void drain_obj_stock(struct memcg_stock_pcp *stock) { struct obj_cgroup *old =3D READ_ONCE(stock->cached_objcg); =20 if (!old) - return NULL; + return; =20 if (stock->nr_bytes) { unsigned int nr_pages =3D stock->nr_bytes >> PAGE_SHIFT; @@ -2896,11 +2887,7 @@ static struct obj_cgroup *drain_obj_stock(struct mem= cg_stock_pcp *stock) } =20 WRITE_ONCE(stock->cached_objcg, NULL); - /* - * The `old' objects needs to be released by the caller via - * obj_cgroup_put() outside of memcg_stock_pcp::stock_lock. - */ - return old; + obj_cgroup_put(old); } =20 static bool obj_stock_flush_required(struct memcg_stock_pcp *stock, @@ -2922,7 +2909,6 @@ static void refill_obj_stock(struct obj_cgroup *objcg= , unsigned int nr_bytes, bool allow_uncharge) { struct memcg_stock_pcp *stock; - struct obj_cgroup *old =3D NULL; unsigned long flags; unsigned int nr_pages =3D 0; =20 @@ -2930,7 +2916,7 @@ static void refill_obj_stock(struct obj_cgroup *objcg= , unsigned int nr_bytes, =20 stock =3D this_cpu_ptr(&memcg_stock); if (READ_ONCE(stock->cached_objcg) !=3D objcg) { /* reset if necessary */ - old =3D replace_stock_objcg(stock, objcg); + replace_stock_objcg(stock, objcg); allow_uncharge =3D true; /* Allow uncharge when objcg changes */ } stock->nr_bytes +=3D nr_bytes; @@ -2941,7 +2927,6 @@ static void refill_obj_stock(struct obj_cgroup *objcg= , unsigned int nr_bytes, } =20 localtry_unlock_irqrestore(&memcg_stock.stock_lock, flags); - obj_cgroup_put(old); =20 if (nr_pages) obj_cgroup_uncharge_pages(objcg, nr_pages); --=20 2.47.1 From nobody Wed Dec 17 22:40:58 2025 Received: from out-188.mta0.migadu.com (out-188.mta0.migadu.com [91.218.175.188]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 0C84B18D621 for ; Fri, 14 Mar 2025 06:15:48 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=91.218.175.188 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1741932950; cv=none; b=UY/Upk3bOp8x0LOKRzn7ez4/rOoOAW1KkLhEfrGQgRRTf4FMpqJXkCRTo7gUfHIIPUTyHKwcP5dDKv3f+mi5ria8tquAJ2T+A7S1hWKnkqx+xmoUJy3hfoO7gq84okBTbB/gchUpJ2IyLcPdPYIVKoIpYEzkeLBKOn34y6kX/bM= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1741932950; c=relaxed/simple; bh=hPS0VKbMCZoTwVyonLwRfLAvT0JQ45gt7M89UuV5fWc=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=FWduI6nF7GQpGtFTY1TASVRNGRwmLPhsAA384TUv7w+rN6rKvcpZshAZb7ntXPDiO6gjBmoNB0pckzD3OsudA9jJSlVFvpGXcGxEK8+AgJ05HiFrpR5vW7ckZ0E6kceZ3Q53ReXfz/dHf88UIyI89Tugbdf58LM0lg2jWrIrp4Y= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=linux.dev; spf=pass smtp.mailfrom=linux.dev; dkim=pass (1024-bit key) header.d=linux.dev header.i=@linux.dev header.b=NzN2MoMP; arc=none smtp.client-ip=91.218.175.188 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=linux.dev Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=linux.dev Authentication-Results: smtp.subspace.kernel.org; dkim=pass (1024-bit key) header.d=linux.dev header.i=@linux.dev header.b="NzN2MoMP" X-Report-Abuse: Please report any abuse attempt to abuse@migadu.com and include these headers. DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=linux.dev; s=key1; t=1741932947; h=from:from:reply-to:subject:subject:date:date:message-id:message-id: to:to:cc:cc:mime-version:mime-version: content-transfer-encoding:content-transfer-encoding: in-reply-to:in-reply-to:references:references; bh=T/vOofDJKYjkd25R4JYNyNwQOb8YH2uyqN/2EfEOuLI=; b=NzN2MoMPHTfdCiaak06HUc74OdZQIvqLAIVFB9SwXO8961DljO4pkXWKDSORvkMJpuOHCT mHFDQrUN3h5Kvgdym+MQodLK/7gwfoGuo9VU27qQeiMZAjNPLBgsbxldVen9H+7J4RVwG7 h535Oyanb8IayGBPoUMgidcRMWV3aoY= From: Shakeel Butt To: Andrew Morton Cc: Johannes Weiner , Michal Hocko , Roman Gushchin , Muchun Song , Vlastimil Babka , Sebastian Andrzej Siewior , linux-mm@kvack.org, cgroups@vger.kernel.org, linux-kernel@vger.kernel.org, Meta kernel team Subject: [RFC PATCH 07/10] memcg: use __mod_memcg_state in drain_obj_stock Date: Thu, 13 Mar 2025 23:15:08 -0700 Message-ID: <20250314061511.1308152-8-shakeel.butt@linux.dev> In-Reply-To: <20250314061511.1308152-1-shakeel.butt@linux.dev> References: <20250314061511.1308152-1-shakeel.butt@linux.dev> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable X-Migadu-Flow: FLOW_OUT Content-Type: text/plain; charset="utf-8" For non-PREEMPT_RT kernels, drain_obj_stock() is always called with irq disabled, so we can use __mod_memcg_state() instead of mod_memcg_state(). For PREEMPT_RT, we need to add memcg_stats_[un]lock in __mod_memcg_state(). Signed-off-by: Shakeel Butt Reviewed-by: Sebastian Andrzej Siewior --- mm/memcontrol.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mm/memcontrol.c b/mm/memcontrol.c index 3c4de384b5a0..dfe9c2eb7816 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -707,10 +707,12 @@ void __mod_memcg_state(struct mem_cgroup *memcg, enum= memcg_stat_item idx, if (WARN_ONCE(BAD_STAT_IDX(i), "%s: missing stat item %d\n", __func__, id= x)) return; =20 + memcg_stats_lock(); __this_cpu_add(memcg->vmstats_percpu->state[i], val); val =3D memcg_state_val_in_pages(idx, val); memcg_rstat_updated(memcg, val); trace_mod_memcg_state(memcg, idx, val); + memcg_stats_unlock(); } =20 #ifdef CONFIG_MEMCG_V1 @@ -2845,7 +2847,7 @@ static void drain_obj_stock(struct memcg_stock_pcp *s= tock) =20 memcg =3D get_mem_cgroup_from_objcg(old); =20 - mod_memcg_state(memcg, MEMCG_KMEM, -nr_pages); + __mod_memcg_state(memcg, MEMCG_KMEM, -nr_pages); memcg1_account_kmem(memcg, -nr_pages); if (!mem_cgroup_is_root(memcg)) memcg_uncharge(memcg, nr_pages); --=20 2.47.1 From nobody Wed Dec 17 22:40:58 2025 Received: from out-182.mta1.migadu.com (out-182.mta1.migadu.com [95.215.58.182]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 2078B190472 for ; Fri, 14 Mar 2025 06:15:51 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=95.215.58.182 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1741932954; cv=none; b=d163mjyBKr0NjtTOeVPtX84E5iL18EJkwikzN7BBWdO1c+LpfADBRJtFBZwS0PVix1tNFakgHwgwzE8gmBqcvb8YS9Z1JRu/t4xRYCd1SqXVsZRFckOIH7TixXAx3T3Me2L34Nq2fUHFl805FZLuJ73q/7UZ3niGf9sqqeIwQ/o= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1741932954; c=relaxed/simple; bh=DOSr7hMbbaWwE7vTOYq2OQeJLGOdT9916XhVh5YD7mI=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=Vqax2Zr23zRUtPqerO5SPxEGp6K8GCTIzR58avsw+86uQwxSifc2J3vjvcsxkRbD642wwXs9d3z4pDA6NzqERwJnTpg3rOIrvAPus6G4wNS8ditHVv/aeZbHHuYfmi4z2rwm+A2ktBYlaCg3iu0dbIXuaTd/99KtLZ+ASiT5xMY= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=linux.dev; spf=pass smtp.mailfrom=linux.dev; dkim=pass (1024-bit key) header.d=linux.dev header.i=@linux.dev header.b=vhAfmg//; arc=none smtp.client-ip=95.215.58.182 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=linux.dev Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=linux.dev Authentication-Results: smtp.subspace.kernel.org; dkim=pass (1024-bit key) header.d=linux.dev header.i=@linux.dev header.b="vhAfmg//" X-Report-Abuse: Please report any abuse attempt to abuse@migadu.com and include these headers. DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=linux.dev; s=key1; t=1741932950; h=from:from:reply-to:subject:subject:date:date:message-id:message-id: to:to:cc:cc:mime-version:mime-version: content-transfer-encoding:content-transfer-encoding: in-reply-to:in-reply-to:references:references; bh=5fE+tfATBs+u3OomjlvcpPVaZLFEEe7to/tRcynNlOg=; b=vhAfmg//mk5ht4EXslf1UpxrBhoU+6iHYa0Fdob5JRoSmvL+p5JjLl+AsS2rCYbTMyvkEM Uk2TH3CtPmlMCFALAhBM+VV80oT4jcCeobDv8g/awWwP8Xvz3ivJmp4jmuhu6Wr1xd0gwF /Ajsj8Phh2rDKYPW9ZJL0S9rCQRj/i0= From: Shakeel Butt To: Andrew Morton Cc: Johannes Weiner , Michal Hocko , Roman Gushchin , Muchun Song , Vlastimil Babka , Sebastian Andrzej Siewior , linux-mm@kvack.org, cgroups@vger.kernel.org, linux-kernel@vger.kernel.org, Meta kernel team Subject: [RFC PATCH 08/10] memcg: assert in_task for couple of local_lock holders Date: Thu, 13 Mar 2025 23:15:09 -0700 Message-ID: <20250314061511.1308152-9-shakeel.butt@linux.dev> In-Reply-To: <20250314061511.1308152-1-shakeel.butt@linux.dev> References: <20250314061511.1308152-1-shakeel.butt@linux.dev> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable X-Migadu-Flow: FLOW_OUT Content-Type: text/plain; charset="utf-8" The drain_local_stock() and memcg_hotplug_cpu_dead() only run in task context, so there is no need to localtry_trylock_irqsave() the local stock_lock in those functions. The plan is to convert all stock_lock users which can be called in multiple context to use localtry_trylock_irqsave() and subsequently switch to non-irq disabling interface. So, for functions which are never called in non-task context, this patch adds the asserts. Signed-off-by: Shakeel Butt --- mm/memcontrol.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/mm/memcontrol.c b/mm/memcontrol.c index dfe9c2eb7816..c803d2f5e322 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -1857,6 +1857,8 @@ static void drain_local_stock(struct work_struct *dum= my) struct memcg_stock_pcp *stock; unsigned long flags; =20 + lockdep_assert_once(in_task()); + /* * The only protection from cpu hotplug (memcg_hotplug_cpu_dead) vs. * drain_stock races is that we always operate on local CPU stock @@ -1953,6 +1955,8 @@ static int memcg_hotplug_cpu_dead(unsigned int cpu) struct memcg_stock_pcp *stock; unsigned long flags; =20 + lockdep_assert_once(in_task()); + stock =3D &per_cpu(memcg_stock, cpu); =20 /* drain_obj_stock requires stock_lock */ --=20 2.47.1 From nobody Wed Dec 17 22:40:58 2025 Received: from out-177.mta0.migadu.com (out-177.mta0.migadu.com [91.218.175.177]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id E75D319B586 for ; Fri, 14 Mar 2025 06:15:54 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=91.218.175.177 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1741932957; cv=none; b=WudkiI/q+nfwdOE3xVLA71VT94nHbrW9oqqNiPio4NRZLi+ZLFqm2wTbOezxQ0llhPAFpaQQOXmL6gCVUh67Y6BZBSY3u5j3zu+I6pAOBzk3Xbdd6VZc0dZcVls4BgM0t/FrKDNyV+S4cQ5C25cNFcw3JqI53vRJtLxODMGZBl4= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1741932957; c=relaxed/simple; bh=0zD7s3jtU8T0QnDwNFPVi23rov59ZSfE8r2niGUwTpE=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=F4d8nJq6Sjr2fmGJuPeG6IxvlfQaLgYzE+TnvjOlkyr7P+9VlsOoZjEqYUJRu/KymwQ13mS0ytGC09Or6jglhWivPjmBcAi3aCIpdkvvM2KqDJ2mdFuv3hARzKggVWktSiYKKP9pQmXs7xI7KKLgwhOuHI8h5lCVKRg5+x7po5c= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=linux.dev; spf=pass smtp.mailfrom=linux.dev; dkim=pass (1024-bit key) header.d=linux.dev header.i=@linux.dev header.b=kEdIKGlD; arc=none smtp.client-ip=91.218.175.177 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=linux.dev Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=linux.dev Authentication-Results: smtp.subspace.kernel.org; dkim=pass (1024-bit key) header.d=linux.dev header.i=@linux.dev header.b="kEdIKGlD" X-Report-Abuse: Please report any abuse attempt to abuse@migadu.com and include these headers. DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=linux.dev; s=key1; t=1741932952; h=from:from:reply-to:subject:subject:date:date:message-id:message-id: to:to:cc:cc:mime-version:mime-version: content-transfer-encoding:content-transfer-encoding: in-reply-to:in-reply-to:references:references; bh=MpyYup93rtdpPqOazjOWpUMNns5c+BBQgx0og/O/C6c=; b=kEdIKGlDaWDMSsPANjgdCoWUUq9Aqr4mi5OYIz7rsALJ4sjMBJg3BQ1uxepzQ2QiLXHTeS rgI7A3Z1vT15uePUZX+pSwvZjlpzip9/HvgWyN0cmiA5TbQ+zAeduhWHSB0dERoJH14Q2S Fot0pjHwPM3UuiKee4avcxnTc57xAL0= From: Shakeel Butt To: Andrew Morton Cc: Johannes Weiner , Michal Hocko , Roman Gushchin , Muchun Song , Vlastimil Babka , Sebastian Andrzej Siewior , linux-mm@kvack.org, cgroups@vger.kernel.org, linux-kernel@vger.kernel.org, Meta kernel team Subject: [RFC PATCH 09/10] memcg: trylock stock for objcg Date: Thu, 13 Mar 2025 23:15:10 -0700 Message-ID: <20250314061511.1308152-10-shakeel.butt@linux.dev> In-Reply-To: <20250314061511.1308152-1-shakeel.butt@linux.dev> References: <20250314061511.1308152-1-shakeel.butt@linux.dev> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable X-Migadu-Flow: FLOW_OUT Content-Type: text/plain; charset="utf-8" To make objcg stock functions work without disabling irq, we need to convert those function to use localtry_trylock_irqsave() instead of localtry_lock_irqsave(). This patch for now just does the conversion and later patch will eliminate the irq disabling code. Signed-off-by: Shakeel Butt --- mm/memcontrol.c | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/mm/memcontrol.c b/mm/memcontrol.c index c803d2f5e322..ba5d004049d3 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -2764,7 +2764,11 @@ static void mod_objcg_state(struct obj_cgroup *objcg= , struct pglist_data *pgdat, unsigned long flags; int *bytes; =20 - localtry_lock_irqsave(&memcg_stock.stock_lock, flags); + if (!localtry_trylock_irqsave(&memcg_stock.stock_lock, flags)) { + __mod_objcg_mlstate(objcg, pgdat, idx, nr); + return; + } + stock =3D this_cpu_ptr(&memcg_stock); =20 /* @@ -2822,7 +2826,8 @@ static bool consume_obj_stock(struct obj_cgroup *objc= g, unsigned int nr_bytes) unsigned long flags; bool ret =3D false; =20 - localtry_lock_irqsave(&memcg_stock.stock_lock, flags); + if (!localtry_trylock_irqsave(&memcg_stock.stock_lock, flags)) + return ret; =20 stock =3D this_cpu_ptr(&memcg_stock); if (objcg =3D=3D READ_ONCE(stock->cached_objcg) && stock->nr_bytes >=3D n= r_bytes) { @@ -2918,7 +2923,10 @@ static void refill_obj_stock(struct obj_cgroup *objc= g, unsigned int nr_bytes, unsigned long flags; unsigned int nr_pages =3D 0; =20 - localtry_lock_irqsave(&memcg_stock.stock_lock, flags); + if (!localtry_trylock_irqsave(&memcg_stock.stock_lock, flags)) { + atomic_add(nr_bytes, &objcg->nr_charged_bytes); + return; + } =20 stock =3D this_cpu_ptr(&memcg_stock); if (READ_ONCE(stock->cached_objcg) !=3D objcg) { /* reset if necessary */ --=20 2.47.1 From nobody Wed Dec 17 22:40:58 2025 Received: from out-171.mta0.migadu.com (out-171.mta0.migadu.com [91.218.175.171]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 8BA61199249 for ; Fri, 14 Mar 2025 06:15:57 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=91.218.175.171 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1741932959; cv=none; b=XRVoEZITmzosZnZ4y4YS1t4GQ7tEGYALciE9ORm1UvUTrCvmnRb9tfhMtjLimxywMYkb58NOBFbiSMaJCMqkGlftwI9bKmJAF4d7mP8tjc4zNVP9V1OqaLIeOFK2aN7ClT8IZq4/1DqpRubz+Z5oQAOUck/+EQZhdpEo+l3hycc= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1741932959; c=relaxed/simple; bh=4tRYEgx4s2dGQWYal44m1GqRv+xFVFSmf6A26cqiTaY=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=iaSpUymHiYREUAI2RYgsR07Pl1RXrw4/l05DLlNzDXKBlMcE29QkLcmJJXsxxDLTjR4gOXcrDC5LKduWoUZ0A9PcsO+TZITQBZDTejyJy2AjJWqJsusM/EO2vrlSRl3wTAGXVq4qgVQQN4eXsp6/ZSb7zGLv5b9MRSzAeq7KFE4= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=linux.dev; spf=pass smtp.mailfrom=linux.dev; dkim=pass (1024-bit key) header.d=linux.dev header.i=@linux.dev header.b=mYEbdysE; arc=none smtp.client-ip=91.218.175.171 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=linux.dev Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=linux.dev Authentication-Results: smtp.subspace.kernel.org; dkim=pass (1024-bit key) header.d=linux.dev header.i=@linux.dev header.b="mYEbdysE" X-Report-Abuse: Please report any abuse attempt to abuse@migadu.com and include these headers. DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=linux.dev; s=key1; t=1741932955; h=from:from:reply-to:subject:subject:date:date:message-id:message-id: to:to:cc:cc:mime-version:mime-version: content-transfer-encoding:content-transfer-encoding: in-reply-to:in-reply-to:references:references; bh=Xw8AvDJaEiUMuHgyRNtkqcMvSjXYO4544yPP6QKBNZE=; b=mYEbdysEUSC+zM5Wxo6KARPQbRM+HtLiduFhPb20JvLfYT1Htqt3W955s6pM5F7W/XSXLA nSL+qfEuasBoi5U2vDZB22BZQ4/YlY81kfFMHRajmjZlWxx8jAIR113xghAv/rPdhjWTr3 Z8nwH3ZhCOSHPMA4YlqvUW9uRNC4Hjw= From: Shakeel Butt To: Andrew Morton Cc: Johannes Weiner , Michal Hocko , Roman Gushchin , Muchun Song , Vlastimil Babka , Sebastian Andrzej Siewior , linux-mm@kvack.org, cgroups@vger.kernel.org, linux-kernel@vger.kernel.org, Meta kernel team Subject: [RFC PATCH 10/10] memcg: no more irq disabling for stock locks Date: Thu, 13 Mar 2025 23:15:11 -0700 Message-ID: <20250314061511.1308152-11-shakeel.butt@linux.dev> In-Reply-To: <20250314061511.1308152-1-shakeel.butt@linux.dev> References: <20250314061511.1308152-1-shakeel.butt@linux.dev> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable X-Migadu-Flow: FLOW_OUT Content-Type: text/plain; charset="utf-8" Let's switch all memcg_stock locks acquire and release places to not disable and enable irqs. There are two still functions (i.e. mod_objcg_state() and drain_obj_stock) which needs to disable irqs to update the stats on non-RT kernels. For now add a simple wrapper for that functionality. Signed-off-by: Shakeel Butt --- mm/memcontrol.c | 83 ++++++++++++++++++++++++++++++++----------------- 1 file changed, 54 insertions(+), 29 deletions(-) diff --git a/mm/memcontrol.c b/mm/memcontrol.c index ba5d004049d3..fa28efa298f4 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -1796,22 +1796,17 @@ static bool obj_stock_flush_required(struct memcg_s= tock_pcp *stock, * * returns true if successful, false otherwise. */ -static bool consume_stock(struct mem_cgroup *memcg, unsigned int nr_pages, - gfp_t gfp_mask) +static bool consume_stock(struct mem_cgroup *memcg, unsigned int nr_pages) { struct memcg_stock_pcp *stock; unsigned int stock_pages; - unsigned long flags; bool ret =3D false; =20 if (nr_pages > MEMCG_CHARGE_BATCH) return ret; =20 - if (!localtry_trylock_irqsave(&memcg_stock.stock_lock, flags)) { - if (!gfpflags_allow_spinning(gfp_mask)) + if (!localtry_trylock(&memcg_stock.stock_lock)) return ret; - localtry_lock_irqsave(&memcg_stock.stock_lock, flags); - } =20 stock =3D this_cpu_ptr(&memcg_stock); stock_pages =3D READ_ONCE(stock->nr_pages); @@ -1820,7 +1815,7 @@ static bool consume_stock(struct mem_cgroup *memcg, u= nsigned int nr_pages, ret =3D true; } =20 - localtry_unlock_irqrestore(&memcg_stock.stock_lock, flags); + localtry_unlock(&memcg_stock.stock_lock); =20 return ret; } @@ -1855,7 +1850,6 @@ static void drain_stock(struct memcg_stock_pcp *stock) static void drain_local_stock(struct work_struct *dummy) { struct memcg_stock_pcp *stock; - unsigned long flags; =20 lockdep_assert_once(in_task()); =20 @@ -1864,14 +1858,14 @@ static void drain_local_stock(struct work_struct *d= ummy) * drain_stock races is that we always operate on local CPU stock * here with IRQ disabled */ - localtry_lock_irqsave(&memcg_stock.stock_lock, flags); + localtry_lock(&memcg_stock.stock_lock); =20 stock =3D this_cpu_ptr(&memcg_stock); drain_obj_stock(stock); drain_stock(stock); clear_bit(FLUSHING_CACHED_CHARGE, &stock->flags); =20 - localtry_unlock_irqrestore(&memcg_stock.stock_lock, flags); + localtry_unlock(&memcg_stock.stock_lock); } =20 /* Should never be called with root_mem_cgroup. */ @@ -1879,9 +1873,8 @@ static void refill_stock(struct mem_cgroup *memcg, un= signed int nr_pages) { struct memcg_stock_pcp *stock; unsigned int stock_pages; - unsigned long flags; =20 - if (!localtry_trylock_irqsave(&memcg_stock.stock_lock, flags)) { + if (!localtry_trylock(&memcg_stock.stock_lock)) { /* * In case of unlikely failure to lock percpu stock_lock * uncharge memcg directly. @@ -1902,7 +1895,7 @@ static void refill_stock(struct mem_cgroup *memcg, un= signed int nr_pages) if (stock_pages > MEMCG_CHARGE_BATCH) drain_stock(stock); =20 - localtry_unlock_irqrestore(&memcg_stock.stock_lock, flags); + localtry_unlock(&memcg_stock.stock_lock); } =20 /* @@ -1953,17 +1946,12 @@ void drain_all_stock(struct mem_cgroup *root_memcg) static int memcg_hotplug_cpu_dead(unsigned int cpu) { struct memcg_stock_pcp *stock; - unsigned long flags; =20 lockdep_assert_once(in_task()); =20 stock =3D &per_cpu(memcg_stock, cpu); =20 - /* drain_obj_stock requires stock_lock */ - localtry_lock_irqsave(&memcg_stock.stock_lock, flags); drain_obj_stock(stock); - localtry_unlock_irqrestore(&memcg_stock.stock_lock, flags); - drain_stock(stock); =20 return 0; @@ -2254,7 +2242,7 @@ static int try_charge_memcg(struct mem_cgroup *memcg,= gfp_t gfp_mask, unsigned long pflags; =20 retry: - if (consume_stock(memcg, nr_pages, gfp_mask)) + if (consume_stock(memcg, nr_pages)) return 0; =20 if (!gfpflags_allow_spinning(gfp_mask)) @@ -2757,6 +2745,28 @@ static void replace_stock_objcg(struct memcg_stock_p= cp *stock, WRITE_ONCE(stock->cached_objcg, objcg); } =20 +static unsigned long rt_lock(void) +{ +#ifdef CONFIG_PREEMPT_RT + migrate_disable(); + return 0; +#else + unsigned long flags =3D 0; + + local_irq_save(flags); + return flags; +#endif +} + +static void rt_unlock(unsigned long flags) +{ +#ifdef CONFIG_PREEMPT_RT + migrate_enable(); +#else + local_irq_restore(flags); +#endif +} + static void mod_objcg_state(struct obj_cgroup *objcg, struct pglist_data *= pgdat, enum node_stat_item idx, int nr) { @@ -2764,7 +2774,8 @@ static void mod_objcg_state(struct obj_cgroup *objcg,= struct pglist_data *pgdat, unsigned long flags; int *bytes; =20 - if (!localtry_trylock_irqsave(&memcg_stock.stock_lock, flags)) { + if (!localtry_trylock(&memcg_stock.stock_lock)) { + /* Do we need mix_rt_[un]lock here too. */ __mod_objcg_mlstate(objcg, pgdat, idx, nr); return; } @@ -2783,6 +2794,8 @@ static void mod_objcg_state(struct obj_cgroup *objcg,= struct pglist_data *pgdat, /* Flush the existing cached vmstat data */ struct pglist_data *oldpg =3D stock->cached_pgdat; =20 + flags =3D rt_lock(); + if (stock->nr_slab_reclaimable_b) { __mod_objcg_mlstate(objcg, oldpg, NR_SLAB_RECLAIMABLE_B, stock->nr_slab_reclaimable_b); @@ -2793,6 +2806,8 @@ static void mod_objcg_state(struct obj_cgroup *objcg,= struct pglist_data *pgdat, stock->nr_slab_unreclaimable_b); stock->nr_slab_unreclaimable_b =3D 0; } + + rt_unlock(flags); stock->cached_pgdat =3D pgdat; } =20 @@ -2814,19 +2829,21 @@ static void mod_objcg_state(struct obj_cgroup *objc= g, struct pglist_data *pgdat, nr =3D 0; } } - if (nr) + if (nr) { + flags =3D rt_lock(); __mod_objcg_mlstate(objcg, pgdat, idx, nr); + rt_unlock(flags); + } =20 - localtry_unlock_irqrestore(&memcg_stock.stock_lock, flags); + localtry_unlock(&memcg_stock.stock_lock); } =20 static bool consume_obj_stock(struct obj_cgroup *objcg, unsigned int nr_by= tes) { struct memcg_stock_pcp *stock; - unsigned long flags; bool ret =3D false; =20 - if (!localtry_trylock_irqsave(&memcg_stock.stock_lock, flags)) + if (!localtry_trylock(&memcg_stock.stock_lock)) return ret; =20 stock =3D this_cpu_ptr(&memcg_stock); @@ -2835,7 +2852,7 @@ static bool consume_obj_stock(struct obj_cgroup *objc= g, unsigned int nr_bytes) ret =3D true; } =20 - localtry_unlock_irqrestore(&memcg_stock.stock_lock, flags); + localtry_unlock(&memcg_stock.stock_lock); =20 return ret; } @@ -2843,10 +2860,16 @@ static bool consume_obj_stock(struct obj_cgroup *ob= jcg, unsigned int nr_bytes) static void drain_obj_stock(struct memcg_stock_pcp *stock) { struct obj_cgroup *old =3D READ_ONCE(stock->cached_objcg); + unsigned long flags; + bool locked =3D stock->nr_bytes || stock->nr_slab_reclaimable_b || + stock->nr_slab_unreclaimable_b; =20 if (!old) return; =20 + if (locked) + flags =3D rt_lock(); + if (stock->nr_bytes) { unsigned int nr_pages =3D stock->nr_bytes >> PAGE_SHIFT; unsigned int nr_bytes =3D stock->nr_bytes & (PAGE_SIZE - 1); @@ -2897,6 +2920,9 @@ static void drain_obj_stock(struct memcg_stock_pcp *s= tock) stock->cached_pgdat =3D NULL; } =20 + if (locked) + rt_unlock(flags); + WRITE_ONCE(stock->cached_objcg, NULL); obj_cgroup_put(old); } @@ -2920,10 +2946,9 @@ static void refill_obj_stock(struct obj_cgroup *objc= g, unsigned int nr_bytes, bool allow_uncharge) { struct memcg_stock_pcp *stock; - unsigned long flags; unsigned int nr_pages =3D 0; =20 - if (!localtry_trylock_irqsave(&memcg_stock.stock_lock, flags)) { + if (!localtry_trylock(&memcg_stock.stock_lock)) { atomic_add(nr_bytes, &objcg->nr_charged_bytes); return; } @@ -2940,7 +2965,7 @@ static void refill_obj_stock(struct obj_cgroup *objcg= , unsigned int nr_bytes, stock->nr_bytes &=3D (PAGE_SIZE - 1); } =20 - localtry_unlock_irqrestore(&memcg_stock.stock_lock, flags); + localtry_unlock(&memcg_stock.stock_lock); =20 if (nr_pages) obj_cgroup_uncharge_pages(objcg, nr_pages); --=20 2.47.1