From nobody Thu Dec 18 07:13:13 2025 Received: from out-171.mta0.migadu.com (out-171.mta0.migadu.com [91.218.175.171]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 8BA61199249 for ; Fri, 14 Mar 2025 06:15:57 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=91.218.175.171 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1741932959; cv=none; b=XRVoEZITmzosZnZ4y4YS1t4GQ7tEGYALciE9ORm1UvUTrCvmnRb9tfhMtjLimxywMYkb58NOBFbiSMaJCMqkGlftwI9bKmJAF4d7mP8tjc4zNVP9V1OqaLIeOFK2aN7ClT8IZq4/1DqpRubz+Z5oQAOUck/+EQZhdpEo+l3hycc= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1741932959; c=relaxed/simple; bh=4tRYEgx4s2dGQWYal44m1GqRv+xFVFSmf6A26cqiTaY=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=iaSpUymHiYREUAI2RYgsR07Pl1RXrw4/l05DLlNzDXKBlMcE29QkLcmJJXsxxDLTjR4gOXcrDC5LKduWoUZ0A9PcsO+TZITQBZDTejyJy2AjJWqJsusM/EO2vrlSRl3wTAGXVq4qgVQQN4eXsp6/ZSb7zGLv5b9MRSzAeq7KFE4= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=linux.dev; spf=pass smtp.mailfrom=linux.dev; dkim=pass (1024-bit key) header.d=linux.dev header.i=@linux.dev header.b=mYEbdysE; arc=none smtp.client-ip=91.218.175.171 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=linux.dev Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=linux.dev Authentication-Results: smtp.subspace.kernel.org; dkim=pass (1024-bit key) header.d=linux.dev header.i=@linux.dev header.b="mYEbdysE" X-Report-Abuse: Please report any abuse attempt to abuse@migadu.com and include these headers. DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=linux.dev; s=key1; t=1741932955; h=from:from:reply-to:subject:subject:date:date:message-id:message-id: to:to:cc:cc:mime-version:mime-version: content-transfer-encoding:content-transfer-encoding: in-reply-to:in-reply-to:references:references; bh=Xw8AvDJaEiUMuHgyRNtkqcMvSjXYO4544yPP6QKBNZE=; b=mYEbdysEUSC+zM5Wxo6KARPQbRM+HtLiduFhPb20JvLfYT1Htqt3W955s6pM5F7W/XSXLA nSL+qfEuasBoi5U2vDZB22BZQ4/YlY81kfFMHRajmjZlWxx8jAIR113xghAv/rPdhjWTr3 Z8nwH3ZhCOSHPMA4YlqvUW9uRNC4Hjw= From: Shakeel Butt To: Andrew Morton Cc: Johannes Weiner , Michal Hocko , Roman Gushchin , Muchun Song , Vlastimil Babka , Sebastian Andrzej Siewior , linux-mm@kvack.org, cgroups@vger.kernel.org, linux-kernel@vger.kernel.org, Meta kernel team Subject: [RFC PATCH 10/10] memcg: no more irq disabling for stock locks Date: Thu, 13 Mar 2025 23:15:11 -0700 Message-ID: <20250314061511.1308152-11-shakeel.butt@linux.dev> In-Reply-To: <20250314061511.1308152-1-shakeel.butt@linux.dev> References: <20250314061511.1308152-1-shakeel.butt@linux.dev> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable X-Migadu-Flow: FLOW_OUT Content-Type: text/plain; charset="utf-8" Let's switch all memcg_stock locks acquire and release places to not disable and enable irqs. There are two still functions (i.e. mod_objcg_state() and drain_obj_stock) which needs to disable irqs to update the stats on non-RT kernels. For now add a simple wrapper for that functionality. Signed-off-by: Shakeel Butt --- mm/memcontrol.c | 83 ++++++++++++++++++++++++++++++++----------------- 1 file changed, 54 insertions(+), 29 deletions(-) diff --git a/mm/memcontrol.c b/mm/memcontrol.c index ba5d004049d3..fa28efa298f4 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -1796,22 +1796,17 @@ static bool obj_stock_flush_required(struct memcg_s= tock_pcp *stock, * * returns true if successful, false otherwise. */ -static bool consume_stock(struct mem_cgroup *memcg, unsigned int nr_pages, - gfp_t gfp_mask) +static bool consume_stock(struct mem_cgroup *memcg, unsigned int nr_pages) { struct memcg_stock_pcp *stock; unsigned int stock_pages; - unsigned long flags; bool ret =3D false; =20 if (nr_pages > MEMCG_CHARGE_BATCH) return ret; =20 - if (!localtry_trylock_irqsave(&memcg_stock.stock_lock, flags)) { - if (!gfpflags_allow_spinning(gfp_mask)) + if (!localtry_trylock(&memcg_stock.stock_lock)) return ret; - localtry_lock_irqsave(&memcg_stock.stock_lock, flags); - } =20 stock =3D this_cpu_ptr(&memcg_stock); stock_pages =3D READ_ONCE(stock->nr_pages); @@ -1820,7 +1815,7 @@ static bool consume_stock(struct mem_cgroup *memcg, u= nsigned int nr_pages, ret =3D true; } =20 - localtry_unlock_irqrestore(&memcg_stock.stock_lock, flags); + localtry_unlock(&memcg_stock.stock_lock); =20 return ret; } @@ -1855,7 +1850,6 @@ static void drain_stock(struct memcg_stock_pcp *stock) static void drain_local_stock(struct work_struct *dummy) { struct memcg_stock_pcp *stock; - unsigned long flags; =20 lockdep_assert_once(in_task()); =20 @@ -1864,14 +1858,14 @@ static void drain_local_stock(struct work_struct *d= ummy) * drain_stock races is that we always operate on local CPU stock * here with IRQ disabled */ - localtry_lock_irqsave(&memcg_stock.stock_lock, flags); + localtry_lock(&memcg_stock.stock_lock); =20 stock =3D this_cpu_ptr(&memcg_stock); drain_obj_stock(stock); drain_stock(stock); clear_bit(FLUSHING_CACHED_CHARGE, &stock->flags); =20 - localtry_unlock_irqrestore(&memcg_stock.stock_lock, flags); + localtry_unlock(&memcg_stock.stock_lock); } =20 /* Should never be called with root_mem_cgroup. */ @@ -1879,9 +1873,8 @@ static void refill_stock(struct mem_cgroup *memcg, un= signed int nr_pages) { struct memcg_stock_pcp *stock; unsigned int stock_pages; - unsigned long flags; =20 - if (!localtry_trylock_irqsave(&memcg_stock.stock_lock, flags)) { + if (!localtry_trylock(&memcg_stock.stock_lock)) { /* * In case of unlikely failure to lock percpu stock_lock * uncharge memcg directly. @@ -1902,7 +1895,7 @@ static void refill_stock(struct mem_cgroup *memcg, un= signed int nr_pages) if (stock_pages > MEMCG_CHARGE_BATCH) drain_stock(stock); =20 - localtry_unlock_irqrestore(&memcg_stock.stock_lock, flags); + localtry_unlock(&memcg_stock.stock_lock); } =20 /* @@ -1953,17 +1946,12 @@ void drain_all_stock(struct mem_cgroup *root_memcg) static int memcg_hotplug_cpu_dead(unsigned int cpu) { struct memcg_stock_pcp *stock; - unsigned long flags; =20 lockdep_assert_once(in_task()); =20 stock =3D &per_cpu(memcg_stock, cpu); =20 - /* drain_obj_stock requires stock_lock */ - localtry_lock_irqsave(&memcg_stock.stock_lock, flags); drain_obj_stock(stock); - localtry_unlock_irqrestore(&memcg_stock.stock_lock, flags); - drain_stock(stock); =20 return 0; @@ -2254,7 +2242,7 @@ static int try_charge_memcg(struct mem_cgroup *memcg,= gfp_t gfp_mask, unsigned long pflags; =20 retry: - if (consume_stock(memcg, nr_pages, gfp_mask)) + if (consume_stock(memcg, nr_pages)) return 0; =20 if (!gfpflags_allow_spinning(gfp_mask)) @@ -2757,6 +2745,28 @@ static void replace_stock_objcg(struct memcg_stock_p= cp *stock, WRITE_ONCE(stock->cached_objcg, objcg); } =20 +static unsigned long rt_lock(void) +{ +#ifdef CONFIG_PREEMPT_RT + migrate_disable(); + return 0; +#else + unsigned long flags =3D 0; + + local_irq_save(flags); + return flags; +#endif +} + +static void rt_unlock(unsigned long flags) +{ +#ifdef CONFIG_PREEMPT_RT + migrate_enable(); +#else + local_irq_restore(flags); +#endif +} + static void mod_objcg_state(struct obj_cgroup *objcg, struct pglist_data *= pgdat, enum node_stat_item idx, int nr) { @@ -2764,7 +2774,8 @@ static void mod_objcg_state(struct obj_cgroup *objcg,= struct pglist_data *pgdat, unsigned long flags; int *bytes; =20 - if (!localtry_trylock_irqsave(&memcg_stock.stock_lock, flags)) { + if (!localtry_trylock(&memcg_stock.stock_lock)) { + /* Do we need mix_rt_[un]lock here too. */ __mod_objcg_mlstate(objcg, pgdat, idx, nr); return; } @@ -2783,6 +2794,8 @@ static void mod_objcg_state(struct obj_cgroup *objcg,= struct pglist_data *pgdat, /* Flush the existing cached vmstat data */ struct pglist_data *oldpg =3D stock->cached_pgdat; =20 + flags =3D rt_lock(); + if (stock->nr_slab_reclaimable_b) { __mod_objcg_mlstate(objcg, oldpg, NR_SLAB_RECLAIMABLE_B, stock->nr_slab_reclaimable_b); @@ -2793,6 +2806,8 @@ static void mod_objcg_state(struct obj_cgroup *objcg,= struct pglist_data *pgdat, stock->nr_slab_unreclaimable_b); stock->nr_slab_unreclaimable_b =3D 0; } + + rt_unlock(flags); stock->cached_pgdat =3D pgdat; } =20 @@ -2814,19 +2829,21 @@ static void mod_objcg_state(struct obj_cgroup *objc= g, struct pglist_data *pgdat, nr =3D 0; } } - if (nr) + if (nr) { + flags =3D rt_lock(); __mod_objcg_mlstate(objcg, pgdat, idx, nr); + rt_unlock(flags); + } =20 - localtry_unlock_irqrestore(&memcg_stock.stock_lock, flags); + localtry_unlock(&memcg_stock.stock_lock); } =20 static bool consume_obj_stock(struct obj_cgroup *objcg, unsigned int nr_by= tes) { struct memcg_stock_pcp *stock; - unsigned long flags; bool ret =3D false; =20 - if (!localtry_trylock_irqsave(&memcg_stock.stock_lock, flags)) + if (!localtry_trylock(&memcg_stock.stock_lock)) return ret; =20 stock =3D this_cpu_ptr(&memcg_stock); @@ -2835,7 +2852,7 @@ static bool consume_obj_stock(struct obj_cgroup *objc= g, unsigned int nr_bytes) ret =3D true; } =20 - localtry_unlock_irqrestore(&memcg_stock.stock_lock, flags); + localtry_unlock(&memcg_stock.stock_lock); =20 return ret; } @@ -2843,10 +2860,16 @@ static bool consume_obj_stock(struct obj_cgroup *ob= jcg, unsigned int nr_bytes) static void drain_obj_stock(struct memcg_stock_pcp *stock) { struct obj_cgroup *old =3D READ_ONCE(stock->cached_objcg); + unsigned long flags; + bool locked =3D stock->nr_bytes || stock->nr_slab_reclaimable_b || + stock->nr_slab_unreclaimable_b; =20 if (!old) return; =20 + if (locked) + flags =3D rt_lock(); + if (stock->nr_bytes) { unsigned int nr_pages =3D stock->nr_bytes >> PAGE_SHIFT; unsigned int nr_bytes =3D stock->nr_bytes & (PAGE_SIZE - 1); @@ -2897,6 +2920,9 @@ static void drain_obj_stock(struct memcg_stock_pcp *s= tock) stock->cached_pgdat =3D NULL; } =20 + if (locked) + rt_unlock(flags); + WRITE_ONCE(stock->cached_objcg, NULL); obj_cgroup_put(old); } @@ -2920,10 +2946,9 @@ static void refill_obj_stock(struct obj_cgroup *objc= g, unsigned int nr_bytes, bool allow_uncharge) { struct memcg_stock_pcp *stock; - unsigned long flags; unsigned int nr_pages =3D 0; =20 - if (!localtry_trylock_irqsave(&memcg_stock.stock_lock, flags)) { + if (!localtry_trylock(&memcg_stock.stock_lock)) { atomic_add(nr_bytes, &objcg->nr_charged_bytes); return; } @@ -2940,7 +2965,7 @@ static void refill_obj_stock(struct obj_cgroup *objcg= , unsigned int nr_bytes, stock->nr_bytes &=3D (PAGE_SIZE - 1); } =20 - localtry_unlock_irqrestore(&memcg_stock.stock_lock, flags); + localtry_unlock(&memcg_stock.stock_lock); =20 if (nr_pages) obj_cgroup_uncharge_pages(objcg, nr_pages); --=20 2.47.1