From nobody Fri Dec 19 22:01:03 2025 Received: from out-179.mta0.migadu.com (out-179.mta0.migadu.com [91.218.175.179]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id A99962222CB for ; Fri, 16 May 2025 06:49:41 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=91.218.175.179 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1747378183; cv=none; b=eLFM7RHS7WmKiepCRoLXdJ+28IztNyP3sb3iLjTvxlHeEdhsRkSQg+Z3A5BPBeIVJ5XGY5ufrss9JF/dmYulWyBKsygEbiyC9tpgRN4p6RrmG7C1NBkWouIcUthBcBEC4SxEgPSl0S42kdnOeaQeI+nT5QmEPKXGvTjosVtt5hg= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1747378183; c=relaxed/simple; bh=sg+sQl+gHXroJGyjSMpwjQg6chV+GnJCGvWkX8PFwZg=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=TRPpHQ7Z7HPWD6W/M9l7UPZHgiIJwiFR1dw58cU05yTTJW6FxnXNswOrzPTTueKZlnyG5ph1Dx2/L+S0okqeQ5oWECd0WVOVRQiEtBsiQPAzDEuoNX7NrbjvJ6DirLU6psgfQ1e3pCdHBm4DgL/G7SGNE2VIFaa5r28QBK1Jjzg= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=linux.dev; spf=pass smtp.mailfrom=linux.dev; dkim=pass (1024-bit key) header.d=linux.dev header.i=@linux.dev header.b=wABxfM9x; arc=none smtp.client-ip=91.218.175.179 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=linux.dev Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=linux.dev Authentication-Results: smtp.subspace.kernel.org; dkim=pass (1024-bit key) header.d=linux.dev header.i=@linux.dev header.b="wABxfM9x" X-Report-Abuse: Please report any abuse attempt to abuse@migadu.com and include these headers. DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=linux.dev; s=key1; t=1747378169; h=from:from:reply-to:subject:subject:date:date:message-id:message-id: to:to:cc:cc:mime-version:mime-version: content-transfer-encoding:content-transfer-encoding: in-reply-to:in-reply-to:references:references; bh=IzHgteX+DiRHg0P96tNBM3iI8xKb/ZAbEXKlmYyzoHo=; b=wABxfM9xp6FgM9v7ejZlbjDnAnjdGnnbgbCx0VXaexVniB7dq1QHA/LwEkFWZ2av2p6uch Y7ipUI4zAY5p+Gx5upGd0zVpPwPO7iKgevbaZ/Asg1fGjb5XE2IlI2TQDykpYvV6YNXEk/ YbXBrcuLYceHt0ut41nZtsrfUbe3YS8= From: Shakeel Butt To: Andrew Morton Cc: Johannes Weiner , Michal Hocko , Roman Gushchin , Muchun Song , Vlastimil Babka , Alexei Starovoitov , Sebastian Andrzej Siewior , Harry Yoo , Yosry Ahmed , Peter Zijlstra , Mathieu Desnoyers , bpf@vger.kernel.org, linux-mm@kvack.org, cgroups@vger.kernel.org, linux-kernel@vger.kernel.org, Meta kernel team Subject: [PATCH 1/5] memcg: disable kmem charging in nmi for unsupported arch Date: Thu, 15 May 2025 23:49:08 -0700 Message-ID: <20250516064912.1515065-2-shakeel.butt@linux.dev> In-Reply-To: <20250516064912.1515065-1-shakeel.butt@linux.dev> References: <20250516064912.1515065-1-shakeel.butt@linux.dev> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable X-Migadu-Flow: FLOW_OUT Content-Type: text/plain; charset="utf-8" The memcg accounting and stats uses this_cpu* and atomic* ops. There are archs which define CONFIG_HAVE_NMI but does not define CONFIG_ARCH_HAS_NMI_SAFE_THIS_CPU_OPS and ARCH_HAVE_NMI_SAFE_CMPXCHG, so memcg accounting for such archs in nmi context is not possible to support. Let's just disable memcg accounting in nmi context for such archs. Signed-off-by: Shakeel Butt --- include/linux/memcontrol.h | 5 +++++ mm/memcontrol.c | 15 +++++++++++++++ 2 files changed, 20 insertions(+) diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h index f7848f73f41c..53920528821f 100644 --- a/include/linux/memcontrol.h +++ b/include/linux/memcontrol.h @@ -62,6 +62,11 @@ struct mem_cgroup_reclaim_cookie { =20 #ifdef CONFIG_MEMCG =20 +#if defined(CONFIG_ARCH_HAS_NMI_SAFE_THIS_CPU_OPS) || \ + !defined(CONFIG_HAVE_NMI) || defined(ARCH_HAVE_NMI_SAFE_CMPXCHG) +#define MEMCG_SUPPORTS_NMI_CHARGING +#endif + #define MEM_CGROUP_ID_SHIFT 16 =20 struct mem_cgroup_id { diff --git a/mm/memcontrol.c b/mm/memcontrol.c index e17b698f6243..dface07f69bb 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -2647,11 +2647,26 @@ static struct obj_cgroup *current_objcg_update(void) return objcg; } =20 +#ifdef MEMCG_SUPPORTS_NMI_CHARGING +static inline bool nmi_charging_allowed(void) +{ + return true; +} +#else +static inline bool nmi_charging_allowed(void) +{ + return false; +} +#endif + __always_inline struct obj_cgroup *current_obj_cgroup(void) { struct mem_cgroup *memcg; struct obj_cgroup *objcg; =20 + if (in_nmi() && !nmi_charging_allowed()) + return NULL; + if (in_task()) { memcg =3D current->active_memcg; if (unlikely(memcg)) --=20 2.47.1 From nobody Fri Dec 19 22:01:03 2025 Received: from out-174.mta0.migadu.com (out-174.mta0.migadu.com [91.218.175.174]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 06AEA227E83 for ; Fri, 16 May 2025 06:49:44 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=91.218.175.174 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1747378188; cv=none; b=oOAo0xBZhM7QmqQ5Hkeu7lzbrUn9pLY0HgfTwTZ9otlLQ/3vMOEK9G8AUwL9RZcqkXPPUvYXFD7rUrBQhtYoC700p7JP6y6t0kLXAYfmysmwNm66t1yCsVUN3DPWSDDCR4isRDlKE1/rAHKoCRuaIYHZADdAzcIEpMaZy3WVFoQ= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1747378188; c=relaxed/simple; bh=tEf3IX+SKfryTgjjjr+5wg6/0dH9CvOVe0W0Gxmney4=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=oVSRY+mbfZ9yN57dziKU4eHHUOBWkllM1nQBH/8B/L0QTb572wMCsZM+kNBaaBfeYHL9jDqU7Zzg/Mcx4r+bCXn2Dyqxi0wIdugG/koRtcDXIiCsyJAN9Kvp5fpcn1LHslUqTQLQ7j+EVkiJw4jQc78hxRm8pjTisNypiFDYAUI= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=linux.dev; spf=pass smtp.mailfrom=linux.dev; dkim=pass (1024-bit key) header.d=linux.dev header.i=@linux.dev header.b=WSR4/oQs; arc=none smtp.client-ip=91.218.175.174 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=linux.dev Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=linux.dev Authentication-Results: smtp.subspace.kernel.org; dkim=pass (1024-bit key) header.d=linux.dev header.i=@linux.dev header.b="WSR4/oQs" X-Report-Abuse: Please report any abuse attempt to abuse@migadu.com and include these headers. DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=linux.dev; s=key1; t=1747378183; h=from:from:reply-to:subject:subject:date:date:message-id:message-id: to:to:cc:cc:mime-version:mime-version: content-transfer-encoding:content-transfer-encoding: in-reply-to:in-reply-to:references:references; bh=wjgeN2qjXaFT2yhGcnf6ojnEDqDgQ6jiTAUoDbWljT0=; b=WSR4/oQsdzkHgky5kgsfe7Q3covs1cTkeEfb8OaN46IWjcemjgSUOJOSDHT4tf9q8JeusI f3f8hayqZo9C32j3hvK/L+zPzmYInvz8GgshCBLLJP5qKhCIT7a0GiQI/HtJW0fGIIkigw dj+8morZoI+7sBShj5rXkhBOH0jZGtk= From: Shakeel Butt To: Andrew Morton Cc: Johannes Weiner , Michal Hocko , Roman Gushchin , Muchun Song , Vlastimil Babka , Alexei Starovoitov , Sebastian Andrzej Siewior , Harry Yoo , Yosry Ahmed , Peter Zijlstra , Mathieu Desnoyers , bpf@vger.kernel.org, linux-mm@kvack.org, cgroups@vger.kernel.org, linux-kernel@vger.kernel.org, Meta kernel team Subject: [PATCH 2/5] memcg: nmi safe memcg stats for specific archs Date: Thu, 15 May 2025 23:49:09 -0700 Message-ID: <20250516064912.1515065-3-shakeel.butt@linux.dev> In-Reply-To: <20250516064912.1515065-1-shakeel.butt@linux.dev> References: <20250516064912.1515065-1-shakeel.butt@linux.dev> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable X-Migadu-Flow: FLOW_OUT Content-Type: text/plain; charset="utf-8" There are archs which have NMI but does not support this_cpu_* ops safely in the nmi context but they support safe atomic ops in nmi context. For such archs, let's add infra to use atomic ops for the memcg stats which can be updated in nmi. At the moment, the memcg stats which get updated in the objcg charging path are MEMCG_KMEM, NR_SLAB_RECLAIMABLE_B & NR_SLAB_UNRECLAIMABLE_B. Rather than adding support for all memcg stats to be nmi safe, let's just add infra to make these three stats nmi safe which this patch is doing. Signed-off-by: Shakeel Butt Acked-by: Vlastimil Babka --- include/linux/memcontrol.h | 20 ++++++++++++++-- mm/memcontrol.c | 49 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 67 insertions(+), 2 deletions(-) diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h index 53920528821f..b10ae2388c27 100644 --- a/include/linux/memcontrol.h +++ b/include/linux/memcontrol.h @@ -62,9 +62,15 @@ struct mem_cgroup_reclaim_cookie { =20 #ifdef CONFIG_MEMCG =20 -#if defined(CONFIG_ARCH_HAS_NMI_SAFE_THIS_CPU_OPS) || \ - !defined(CONFIG_HAVE_NMI) || defined(ARCH_HAVE_NMI_SAFE_CMPXCHG) +#if defined(CONFIG_ARCH_HAS_NMI_SAFE_THIS_CPU_OPS) || !defined(CONFIG_HAVE= _NMI) + +#define MEMCG_SUPPORTS_NMI_CHARGING + +#elif defined(ARCH_HAVE_NMI_SAFE_CMPXCHG) + #define MEMCG_SUPPORTS_NMI_CHARGING +#define MEMCG_NMI_NEED_ATOMIC + #endif =20 #define MEM_CGROUP_ID_SHIFT 16 @@ -118,6 +124,12 @@ struct mem_cgroup_per_node { CACHELINE_PADDING(_pad2_); unsigned long lru_zone_size[MAX_NR_ZONES][NR_LRU_LISTS]; struct mem_cgroup_reclaim_iter iter; + +#ifdef MEMCG_NMI_NEED_ATOMIC + /* slab stats for nmi context */ + atomic_t slab_reclaimable; + atomic_t slab_unreclaimable; +#endif }; =20 struct mem_cgroup_threshold { @@ -241,6 +253,10 @@ struct mem_cgroup { atomic_long_t memory_events[MEMCG_NR_MEMORY_EVENTS]; atomic_long_t memory_events_local[MEMCG_NR_MEMORY_EVENTS]; =20 +#ifdef MEMCG_NMI_NEED_ATOMIC + /* MEMCG_KMEM for nmi context */ + atomic_t kmem_stat; +#endif /* * Hint of reclaim pressure for socket memroy management. Note * that this indicator should NOT be used in legacy cgroup mode diff --git a/mm/memcontrol.c b/mm/memcontrol.c index dface07f69bb..102fdec3f49e 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -3979,6 +3979,53 @@ static void mem_cgroup_stat_aggregate(struct aggrega= te_control *ac) } } =20 +#ifdef MEMCG_NMI_NEED_ATOMIC +static void flush_nmi_stats(struct mem_cgroup *memcg, struct mem_cgroup *p= arent, + int cpu) +{ + int nid; + + if (atomic_read(&memcg->kmem_stat)) { + int kmem =3D atomic_xchg(&memcg->kmem_stat, 0); + int index =3D memcg_stats_index(MEMCG_KMEM); + + memcg->vmstats->state[index] +=3D kmem; + if (parent) + parent->vmstats->state_pending[index] +=3D kmem; + } + + for_each_node_state(nid, N_MEMORY) { + struct mem_cgroup_per_node *pn =3D memcg->nodeinfo[nid]; + struct lruvec_stats *lstats =3D pn->lruvec_stats; + struct lruvec_stats *plstats =3D NULL; + + if (parent) + plstats =3D parent->nodeinfo[nid]->lruvec_stats; + + if (atomic_read(&pn->slab_reclaimable)) { + int slab =3D atomic_xchg(&pn->slab_reclaimable, 0); + int index =3D memcg_stats_index(NR_SLAB_RECLAIMABLE_B); + + lstats->state[index] +=3D slab; + if (plstats) + plstats->state_pending[index] +=3D slab; + } + if (atomic_read(&pn->slab_unreclaimable)) { + int slab =3D atomic_xchg(&pn->slab_unreclaimable, 0); + int index =3D memcg_stats_index(NR_SLAB_UNRECLAIMABLE_B); + + lstats->state[index] +=3D slab; + if (plstats) + plstats->state_pending[index] +=3D slab; + } + } +} +#else +static void flush_nmi_stats(struct mem_cgroup *memcg, struct mem_cgroup *p= arent, + int cpu) +{} +#endif + static void mem_cgroup_css_rstat_flush(struct cgroup_subsys_state *css, in= t cpu) { struct mem_cgroup *memcg =3D mem_cgroup_from_css(css); @@ -3987,6 +4034,8 @@ static void mem_cgroup_css_rstat_flush(struct cgroup_= subsys_state *css, int cpu) struct aggregate_control ac; int nid; =20 + flush_nmi_stats(memcg, parent, cpu); + statc =3D per_cpu_ptr(memcg->vmstats_percpu, cpu); =20 ac =3D (struct aggregate_control) { --=20 2.47.1 From nobody Fri Dec 19 22:01:03 2025 Received: from out-178.mta1.migadu.com (out-178.mta1.migadu.com [95.215.58.178]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 3ECA6226D04 for ; Fri, 16 May 2025 06:49:58 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=95.215.58.178 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1747378200; cv=none; b=GwGRb8UGPobtWum/KFsw8qwmaV/0wR6H9aeICHD7xeTz7bDFzeEcQfVSc/b7g4D51/pBn4/L7yNT8R19ZD7zcXlnoJIfFkpWJ5+QxRb2NEcPUOjY1T4z3Ndv3cy+5kjllV8Ccg1uruxkAz103tYork6YTk3h4AFhQ96BFuwrxso= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1747378200; c=relaxed/simple; bh=/T3ZsKT8VNYbD3xOoG59SO/PpCy0LoJnigPxVN5Air4=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=gQccbAEItCkd9F6p8S5qUczZDltq0qzedjGV4Ow6CwcYI4p2OPSuFUB5Ryg7HGlkEh/ju+n8ZupZDXmgS+1WxJIz0WdIttH8feClNUZd5gOx2GPgo6b4cOGYiubeva1HujCQYtII8xydpBDz1SWApo3Xsv0yzwP4cTtPcKigyg0= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=linux.dev; spf=pass smtp.mailfrom=linux.dev; dkim=pass (1024-bit key) header.d=linux.dev header.i=@linux.dev header.b=eUrZE+qT; arc=none smtp.client-ip=95.215.58.178 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=linux.dev Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=linux.dev Authentication-Results: smtp.subspace.kernel.org; dkim=pass (1024-bit key) header.d=linux.dev header.i=@linux.dev header.b="eUrZE+qT" X-Report-Abuse: Please report any abuse attempt to abuse@migadu.com and include these headers. DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=linux.dev; s=key1; t=1747378196; h=from:from:reply-to:subject:subject:date:date:message-id:message-id: to:to:cc:cc:mime-version:mime-version: content-transfer-encoding:content-transfer-encoding: in-reply-to:in-reply-to:references:references; bh=CI0Bp2IqmZG3mdccHhETvzHB03Kjr+cVmvCiU8mxhk8=; b=eUrZE+qTBmqtnw5qgpMu5Xy5nqqfp4nL5OQwO/cMx1nhkP5cl2G1N0FzLVHsYX+AHOAX0R 6r+IDuqrlnNMWzF5B7mNhK+NW7AjBBMC6xFKS6uO7B3olBoUWEoUYgewvnNkpuTfE3GGub 1Bvoop0NQFZkgT+rvHVR4ZkpVSP33pk= From: Shakeel Butt To: Andrew Morton Cc: Johannes Weiner , Michal Hocko , Roman Gushchin , Muchun Song , Vlastimil Babka , Alexei Starovoitov , Sebastian Andrzej Siewior , Harry Yoo , Yosry Ahmed , Peter Zijlstra , Mathieu Desnoyers , bpf@vger.kernel.org, linux-mm@kvack.org, cgroups@vger.kernel.org, linux-kernel@vger.kernel.org, Meta kernel team Subject: [PATCH 3/5] memcg: add nmi-safe update for MEMCG_KMEM Date: Thu, 15 May 2025 23:49:10 -0700 Message-ID: <20250516064912.1515065-4-shakeel.butt@linux.dev> In-Reply-To: <20250516064912.1515065-1-shakeel.butt@linux.dev> References: <20250516064912.1515065-1-shakeel.butt@linux.dev> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable X-Migadu-Flow: FLOW_OUT Content-Type: text/plain; charset="utf-8" The objcg based kmem charging and uncharging code path needs to update MEMCG_KMEM appropriately. Let's add support to update MEMCG_KMEM in nmi-safe way for those code paths. Signed-off-by: Shakeel Butt Acked-by: Vlastimil Babka --- mm/memcontrol.c | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/mm/memcontrol.c b/mm/memcontrol.c index 102fdec3f49e..899a31e6b087 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -2729,6 +2729,23 @@ struct obj_cgroup *get_obj_cgroup_from_folio(struct = folio *folio) return objcg; } =20 +#ifdef MEMCG_NMI_NEED_ATOMIC +static inline void account_kmem_nmi_safe(struct mem_cgroup *memcg, int val) +{ + if (likely(!in_nmi())) { + mod_memcg_state(memcg, MEMCG_KMEM, val); + } else { + /* TODO: add to cgroup update tree once it is nmi-safe. */ + atomic_add(val, &memcg->kmem_stat); + } +} +#else +static inline void account_kmem_nmi_safe(struct mem_cgroup *memcg, int val) +{ + mod_memcg_state(memcg, MEMCG_KMEM, val); +} +#endif + /* * obj_cgroup_uncharge_pages: uncharge a number of kernel pages from a obj= cg * @objcg: object cgroup to uncharge @@ -2741,7 +2758,7 @@ static void obj_cgroup_uncharge_pages(struct obj_cgro= up *objcg, =20 memcg =3D get_mem_cgroup_from_objcg(objcg); =20 - mod_memcg_state(memcg, MEMCG_KMEM, -nr_pages); + account_kmem_nmi_safe(memcg, -nr_pages); memcg1_account_kmem(memcg, -nr_pages); if (!mem_cgroup_is_root(memcg)) refill_stock(memcg, nr_pages); @@ -2769,7 +2786,7 @@ static int obj_cgroup_charge_pages(struct obj_cgroup = *objcg, gfp_t gfp, if (ret) goto out; =20 - mod_memcg_state(memcg, MEMCG_KMEM, nr_pages); + account_kmem_nmi_safe(memcg, nr_pages); memcg1_account_kmem(memcg, nr_pages); out: css_put(&memcg->css); --=20 2.47.1 From nobody Fri Dec 19 22:01:03 2025 Received: from out-187.mta1.migadu.com (out-187.mta1.migadu.com [95.215.58.187]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id D762E224259 for ; Fri, 16 May 2025 06:50:10 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=95.215.58.187 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1747378212; cv=none; b=keZM7YtVR5nrJlAp0cNaspt4XSGBxRJJgiDj6ra1bSNuZ/WSvugldlWhcT9GCOpSbGYAZuJrwYlUVOLDOO2YXh0kyrY0NvepW+603Kt/GXF9iv+fIYXqbgC1y+RZ08fHcGaz40S5wiOJHohyScScoNhMTUXXSltXPY2iYE8qoUE= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1747378212; c=relaxed/simple; bh=6rsRUJUmm+2rmMLShujDbilkS0Q6IW2UZei1tehxkH8=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=N22y5/Ezpz6NMX7wUbNZtdqk6HC86HJ4ENtJoPxy62FaaaTfJKZOyjup5i/4DdA0VPGXPxweMEQs0nYhKPwHZP7sWIbYjPXq6niDovb+Qd8xfgPS8yXOTSh6rIsAlBceG8DdoPlgM7mEVCmQSb+O8WX3f9gAwys1er5/4C3fLow= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=linux.dev; spf=pass smtp.mailfrom=linux.dev; dkim=pass (1024-bit key) header.d=linux.dev header.i=@linux.dev header.b=YmY1dR1Q; arc=none smtp.client-ip=95.215.58.187 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=linux.dev Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=linux.dev Authentication-Results: smtp.subspace.kernel.org; dkim=pass (1024-bit key) header.d=linux.dev header.i=@linux.dev header.b="YmY1dR1Q" X-Report-Abuse: Please report any abuse attempt to abuse@migadu.com and include these headers. DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=linux.dev; s=key1; t=1747378208; h=from:from:reply-to:subject:subject:date:date:message-id:message-id: to:to:cc:cc:mime-version:mime-version: content-transfer-encoding:content-transfer-encoding: in-reply-to:in-reply-to:references:references; bh=jVWd6gOAyWb026W16IrpTsGOo2lVltd8hq2g+0iLsvs=; b=YmY1dR1QPKkAid5qt778SbUFTBoxhhl1j871LE9ZdYQj2beQaEombFog1irFez8NJxEOC8 hkryeE7Epcg0y8rxeMYkxetCKfXska0Q9ms78R7EgFhGlaXpNtWBnBj9aZs+jNo4KKEBPO cOEtRs6Rqdh/mwvxQxR/Fc16uABoLeo= From: Shakeel Butt To: Andrew Morton Cc: Johannes Weiner , Michal Hocko , Roman Gushchin , Muchun Song , Vlastimil Babka , Alexei Starovoitov , Sebastian Andrzej Siewior , Harry Yoo , Yosry Ahmed , Peter Zijlstra , Mathieu Desnoyers , bpf@vger.kernel.org, linux-mm@kvack.org, cgroups@vger.kernel.org, linux-kernel@vger.kernel.org, Meta kernel team Subject: [PATCH 4/5] memcg: nmi-safe slab stats updates Date: Thu, 15 May 2025 23:49:11 -0700 Message-ID: <20250516064912.1515065-5-shakeel.butt@linux.dev> In-Reply-To: <20250516064912.1515065-1-shakeel.butt@linux.dev> References: <20250516064912.1515065-1-shakeel.butt@linux.dev> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable X-Migadu-Flow: FLOW_OUT Content-Type: text/plain; charset="utf-8" The objcg based kmem [un]charging can be called in nmi context and it may need to update NR_SLAB_[UN]RECLAIMABLE_B stats. So, let's correctly handle the updates of these stats in the nmi context. Signed-off-by: Shakeel Butt Acked-by: Vlastimil Babka --- mm/memcontrol.c | 36 +++++++++++++++++++++++++++++++++--- 1 file changed, 33 insertions(+), 3 deletions(-) diff --git a/mm/memcontrol.c b/mm/memcontrol.c index 899a31e6b087..85519ce37f18 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -2517,17 +2517,47 @@ static void commit_charge(struct folio *folio, stru= ct mem_cgroup *memcg) folio->memcg_data =3D (unsigned long)memcg; } =20 +#ifdef MEMCG_NMI_NEED_ATOMIC +static inline void account_slab_nmi_safe(struct mem_cgroup *memcg, + struct pglist_data *pgdat, + enum node_stat_item idx, int nr) +{ + struct lruvec *lruvec; + + if (likely(!in_nmi())) { + lruvec =3D mem_cgroup_lruvec(memcg, pgdat); + mod_memcg_lruvec_state(lruvec, idx, nr); + } else { + struct mem_cgroup_per_node *pn =3D memcg->nodeinfo[pgdat->node_id]; + + /* TODO: add to cgroup update tree once it is nmi-safe. */ + if (idx =3D=3D NR_SLAB_RECLAIMABLE_B) + atomic_add(nr, &pn->slab_reclaimable); + else + atomic_add(nr, &pn->slab_unreclaimable); + } +} +#else +static inline void account_slab_nmi_safe(struct mem_cgroup *memcg, + struct pglist_data *pgdat, + enum node_stat_item idx, int nr) +{ + struct lruvec *lruvec; + + lruvec =3D mem_cgroup_lruvec(memcg, pgdat); + mod_memcg_lruvec_state(lruvec, idx, nr); +} +#endif + static inline void mod_objcg_mlstate(struct obj_cgroup *objcg, struct pglist_data *pgdat, enum node_stat_item idx, int nr) { struct mem_cgroup *memcg; - struct lruvec *lruvec; =20 rcu_read_lock(); memcg =3D obj_cgroup_memcg(objcg); - lruvec =3D mem_cgroup_lruvec(memcg, pgdat); - mod_memcg_lruvec_state(lruvec, idx, nr); + account_slab_nmi_safe(memcg, pgdat, idx, nr); rcu_read_unlock(); } =20 --=20 2.47.1 From nobody Fri Dec 19 22:01:03 2025 Received: from out-187.mta1.migadu.com (out-187.mta1.migadu.com [95.215.58.187]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id C6269229B07 for ; Fri, 16 May 2025 06:50:15 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=95.215.58.187 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1747378217; cv=none; b=eh6lcSoiOp77qhiUhNpleUyTRC1CR+oL5X4HV+H6rpeQMEJJAjX1vbRJ3ftlDo60uiXKSYgkfxoDJPMpPM8c60ovyIcscs6P46MkRudy9ejsb5NDQMcBUkzM3sH9/LNXEgQdkkmGl9bBUWn9Aa03PWujaZwrjb3qAEczaFtT5es= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1747378217; c=relaxed/simple; bh=TN0aHevHnfhcWVGThCRs5qFeEEsKBRAAG6+XURYRKRA=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=jxyhWq27smQ5fiwBQQTpHg6NA4KkzXESdzq28e5uniCxY9skEj49X/7icQYvUayKuZZQcuLNe+tXSqhiHheUhARzwbFUeGpauFqmn2t4jZtdlsZPWwE2Kzlu8VQ9faK6/aNjXMqqubXejRSyOiKIg5V16PiPrmIrebjlJmcAlb4= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=linux.dev; spf=pass smtp.mailfrom=linux.dev; dkim=pass (1024-bit key) header.d=linux.dev header.i=@linux.dev header.b=siOP1XI+; arc=none smtp.client-ip=95.215.58.187 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=linux.dev Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=linux.dev Authentication-Results: smtp.subspace.kernel.org; dkim=pass (1024-bit key) header.d=linux.dev header.i=@linux.dev header.b="siOP1XI+" X-Report-Abuse: Please report any abuse attempt to abuse@migadu.com and include these headers. DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=linux.dev; s=key1; t=1747378214; h=from:from:reply-to:subject:subject:date:date:message-id:message-id: to:to:cc:cc:mime-version:mime-version: content-transfer-encoding:content-transfer-encoding: in-reply-to:in-reply-to:references:references; bh=rMkhHE3/mY4LwdWFLkKMdTI5kaoAKFQu40eQRc8Gq7c=; b=siOP1XI+4UsJf4QcPdJTLENFgNL3vd8HYedjMVroCwakJlLDn86JD4Qc4+nqwIm8+F87sI kqlqdNKpEZ4nDYd76Re7gsHiCM1qQT38MtpIP7DJ7FJAY/lWAM3GbK7iGbZo2mD3WJdZyJ 4jB0niGQ+ZYUpCNX1Oa1k50J3tj2xwE= From: Shakeel Butt To: Andrew Morton Cc: Johannes Weiner , Michal Hocko , Roman Gushchin , Muchun Song , Vlastimil Babka , Alexei Starovoitov , Sebastian Andrzej Siewior , Harry Yoo , Yosry Ahmed , Peter Zijlstra , Mathieu Desnoyers , bpf@vger.kernel.org, linux-mm@kvack.org, cgroups@vger.kernel.org, linux-kernel@vger.kernel.org, Meta kernel team Subject: [PATCH 5/5] memcg: make memcg_rstat_updated nmi safe Date: Thu, 15 May 2025 23:49:12 -0700 Message-ID: <20250516064912.1515065-6-shakeel.butt@linux.dev> In-Reply-To: <20250516064912.1515065-1-shakeel.butt@linux.dev> References: <20250516064912.1515065-1-shakeel.butt@linux.dev> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable X-Migadu-Flow: FLOW_OUT Content-Type: text/plain; charset="utf-8" memcg: convert stats_updates to atomic_t Currently kernel maintains memory related stats updates per-cgroup to optimize stats flushing. The stats_updates is defined as atomic64_t which is not nmi-safe on some archs. Actually we don't really need 64bit atomic as the max value stats_updates can get should be less than nr_cpus * MEMCG_CHARGE_BATCH. A normal atomic_t should suffice. Also the function cgroup_rstat_updated() is still not nmi-safe but there is parallel effort to make it nmi-safe, so until then let's ignore it in the nmi context. Signed-off-by: Shakeel Butt Acked-by: Vlastimil Babka --- mm/memcontrol.c | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/mm/memcontrol.c b/mm/memcontrol.c index 85519ce37f18..06d5d5407b00 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -533,7 +533,7 @@ struct memcg_vmstats { unsigned long events_pending[NR_MEMCG_EVENTS]; =20 /* Stats updates since the last flush */ - atomic64_t stats_updates; + atomic_t stats_updates; }; =20 /* @@ -559,7 +559,7 @@ static u64 flush_last_time; =20 static bool memcg_vmstats_needs_flush(struct memcg_vmstats *vmstats) { - return atomic64_read(&vmstats->stats_updates) > + return atomic_read(&vmstats->stats_updates) > MEMCG_CHARGE_BATCH * num_online_cpus(); } =20 @@ -573,7 +573,9 @@ static inline void memcg_rstat_updated(struct mem_cgrou= p *memcg, int val, if (!val) return; =20 - cgroup_rstat_updated(memcg->css.cgroup, cpu); + /* TODO: add to cgroup update tree once it is nmi-safe. */ + if (!in_nmi()) + cgroup_rstat_updated(memcg->css.cgroup, cpu); statc_pcpu =3D memcg->vmstats_percpu; for (; statc_pcpu; statc_pcpu =3D statc->parent_pcpu) { statc =3D this_cpu_ptr(statc_pcpu); @@ -591,7 +593,7 @@ static inline void memcg_rstat_updated(struct mem_cgrou= p *memcg, int val, continue; =20 stats_updates =3D this_cpu_xchg(statc_pcpu->stats_updates, 0); - atomic64_add(stats_updates, &statc->vmstats->stats_updates); + atomic_add(stats_updates, &statc->vmstats->stats_updates); } } =20 @@ -599,7 +601,7 @@ static void __mem_cgroup_flush_stats(struct mem_cgroup = *memcg, bool force) { bool needs_flush =3D memcg_vmstats_needs_flush(memcg->vmstats); =20 - trace_memcg_flush_stats(memcg, atomic64_read(&memcg->vmstats->stats_updat= es), + trace_memcg_flush_stats(memcg, atomic_read(&memcg->vmstats->stats_updates= ), force, needs_flush); =20 if (!force && !needs_flush) @@ -4132,8 +4134,8 @@ static void mem_cgroup_css_rstat_flush(struct cgroup_= subsys_state *css, int cpu) } WRITE_ONCE(statc->stats_updates, 0); /* We are in a per-cpu loop here, only do the atomic write once */ - if (atomic64_read(&memcg->vmstats->stats_updates)) - atomic64_set(&memcg->vmstats->stats_updates, 0); + if (atomic_read(&memcg->vmstats->stats_updates)) + atomic_set(&memcg->vmstats->stats_updates, 0); } =20 static void mem_cgroup_fork(struct task_struct *task) --=20 2.47.1