From nobody Wed Nov 27 04:31:30 2024 Received: from mail-pg1-f179.google.com (mail-pg1-f179.google.com [209.85.215.179]) (using TLSv1.2 with cipher ECDHE-RSA-AES128-GCM-SHA256 (128/128 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 7F03C43ACB; Mon, 14 Oct 2024 03:11:18 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=209.85.215.179 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1728875480; cv=none; b=W7hgFuE82ddrEhin1SpVRA5dt0G8vUu8th0Lsfcdr5s0tK1ykG+iCfhUdfvMG2LfzmBbaJNUSsh2qnQI7hiOwqgh/9qO8kpcJXRLzW+/E3yCaIlfhj1svP8OtoSJJbIj3Tiv3tLtKvoTfi/qbieRJBp0/Kr5n9Pqa49d5s7rzEQ= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1728875480; c=relaxed/simple; bh=mD4RJEhAin69c0Coj5xGRt+ncUmUyzzgnKmub6srj84=; h=From:To:Cc:Subject:Date:Message-Id:In-Reply-To:References: MIME-Version; b=LvC0vyfzFYogthiENv+At8oh4f8qA/gEeLkiK0cb4YeFT2V5+yUQ+L+mXCSxoQM41mD7LmvsQyOPtD0V1ueq4mfnJGce88gFXNcgkqN1+qmGPotrBxYW0ZQl1AUe52LsMN61xnqvumVcffL5TK0BsaGxGMSbc+Ik3jBPbBIvkqU= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=gmail.com; spf=pass smtp.mailfrom=gmail.com; dkim=pass (2048-bit key) header.d=gmail.com header.i=@gmail.com header.b=i+FEkloY; arc=none smtp.client-ip=209.85.215.179 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=gmail.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=gmail.com Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=gmail.com header.i=@gmail.com header.b="i+FEkloY" Received: by mail-pg1-f179.google.com with SMTP id 41be03b00d2f7-7ea6a4f287bso881926a12.3; Sun, 13 Oct 2024 20:11:18 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=gmail.com; s=20230601; t=1728875478; x=1729480278; darn=vger.kernel.org; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:from:to:cc:subject:date :message-id:reply-to; bh=ZDYsYYa+owJ/7qI8MxzT4IGktWSYUFxVm242eY9jgz8=; b=i+FEkloYGbtATs4xjoWl+d7DhwA1TGeqk9ONjIT2Opj9mHRu5BFE5v8C+iqLuf/93A xCAqR9Nc93hISfHIx/71sad7V9+f0581d4rC33CRcEu5PYxnhg6Kh43OUgh4GyixI2/n Jc+2XF/B54YsVxIUH09bdxWfAKJcDQExP3PVAizhnkMu1gCfBELl4cnCdYZNtrgWAbZc yvrItqSDzTJzla+Bu21eRKHaOYGuKcUvTac0PAFKw/cq1qAlMY7aPGJZF8pMpWZtYcWQ vGj1CyTiVDVeNx+U8a6CCK0/QASuEngoB/pvNwnrT4HdmyXZJ+VEVV6d7SakFcXlOiTw HJhg== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20230601; t=1728875478; x=1729480278; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:x-gm-message-state:from:to:cc :subject:date:message-id:reply-to; bh=ZDYsYYa+owJ/7qI8MxzT4IGktWSYUFxVm242eY9jgz8=; b=eon9c+U9XFmqckThxYbNnTPEFeEzeK4z1vfE7DbwEUqATn5XIf1j+BRRRGCkVCtO9A 5l+O61cOCxlArck0BwbmmZ/TncfZNixje4++XSE8j9/VeDJrDjardZhiRpcySbOa1Sij ozhsCS2grtaYOzmlWAhtLMRbBmHlYhlXtdPRZstGt5Gw6qt231qfm8d0PB4rTYPAibfi bLLm3E1rrtsufCvMdPsXXFnUa6Xat6d29WIyNE57bKwDsInqrVPIU1RW6L/q+2m3z6wW 5yJ8bkiC7Qz8CYa6FgSoz25ZO8EIAIAxO5/oPLqDpUEK6JPPVG3atBViDy68J3Sf3+JS oPxA== X-Forwarded-Encrypted: i=1; AJvYcCUw909gOT9Tfvwzn0UuGVrYZ7arUMmcOPUcrFK1DUeOJOavZWQL9Meizt/R8yVudnWqq1DkQuyG17MKRRM=@vger.kernel.org X-Gm-Message-State: AOJu0YwEvfAoYPyeu/ojO0iN3ioYxuZ6IzeLHIYKe3zWCfBlWt56UnZQ KAZkbeORE8DMeiz6i3wz+by+zW0DTfG6C6oUW4xKRMH5WZXRzGZi X-Google-Smtp-Source: AGHT+IESsY50CaRkq7+Sv8BAcwt26tbNnmoJIQEwykjCyVcBMvJ9VxsQ+1v5tXM0gUMi/wxZdfW79w== X-Received: by 2002:a05:6a20:438d:b0:1d6:2378:cffd with SMTP id adf61e73a8af0-1d8c93f5dbemr10657209637.0.1728875477718; Sun, 13 Oct 2024 20:11:17 -0700 (PDT) Received: from localhost.localdomain ([39.144.103.177]) by smtp.gmail.com with ESMTPSA id 41be03b00d2f7-7ea6e396a71sm2987514a12.73.2024.10.13.20.11.11 (version=TLS1_2 cipher=ECDHE-ECDSA-AES128-GCM-SHA256 bits=128/128); Sun, 13 Oct 2024 20:11:17 -0700 (PDT) From: Yafang Shao To: mingo@redhat.com, peterz@infradead.org, juri.lelli@redhat.com, vincent.guittot@linaro.org, dietmar.eggemann@arm.com, rostedt@goodmis.org, bsegall@google.com, mgorman@suse.de, vschneid@redhat.com, hannes@cmpxchg.org, surenb@google.com Cc: cgroups@vger.kernel.org, linux-kernel@vger.kernel.org, Yafang Shao Subject: [PATCH v3 bpf-next 1/4] sched: Define sched_clock_irqtime as static key Date: Mon, 14 Oct 2024 11:10:54 +0800 Message-Id: <20241014031057.8199-2-laoar.shao@gmail.com> X-Mailer: git-send-email 2.30.1 (Apple Git-130) In-Reply-To: <20241014031057.8199-1-laoar.shao@gmail.com> References: <20241014031057.8199-1-laoar.shao@gmail.com> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Since CPU time accounting is a performance-critical path, let's define sched_clock_irqtime as a static key to minimize potential overhead. Signed-off-by: Yafang Shao --- kernel/sched/cputime.c | 16 +++++++--------- kernel/sched/sched.h | 1 + 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/kernel/sched/cputime.c b/kernel/sched/cputime.c index 0bed0fa1acd9..d0b6ea737d04 100644 --- a/kernel/sched/cputime.c +++ b/kernel/sched/cputime.c @@ -7,6 +7,8 @@ #include #endif =20 +DEFINE_STATIC_KEY_FALSE(sched_clock_irqtime); + #ifdef CONFIG_IRQ_TIME_ACCOUNTING =20 /* @@ -22,16 +24,14 @@ */ DEFINE_PER_CPU(struct irqtime, cpu_irqtime); =20 -static int sched_clock_irqtime; - void enable_sched_clock_irqtime(void) { - sched_clock_irqtime =3D 1; + static_branch_enable(&sched_clock_irqtime); } =20 void disable_sched_clock_irqtime(void) { - sched_clock_irqtime =3D 0; + static_branch_disable(&sched_clock_irqtime); } =20 static void irqtime_account_delta(struct irqtime *irqtime, u64 delta, @@ -57,7 +57,7 @@ void irqtime_account_irq(struct task_struct *curr, unsign= ed int offset) s64 delta; int cpu; =20 - if (!sched_clock_irqtime) + if (!static_branch_likely(&sched_clock_irqtime)) return; =20 cpu =3D smp_processor_id(); @@ -90,8 +90,6 @@ static u64 irqtime_tick_accounted(u64 maxtime) =20 #else /* CONFIG_IRQ_TIME_ACCOUNTING */ =20 -#define sched_clock_irqtime (0) - static u64 irqtime_tick_accounted(u64 dummy) { return 0; @@ -478,7 +476,7 @@ void account_process_tick(struct task_struct *p, int us= er_tick) if (vtime_accounting_enabled_this_cpu()) return; =20 - if (sched_clock_irqtime) { + if (static_branch_likely(&sched_clock_irqtime)) { irqtime_account_process_tick(p, user_tick, 1); return; } @@ -507,7 +505,7 @@ void account_idle_ticks(unsigned long ticks) { u64 cputime, steal; =20 - if (sched_clock_irqtime) { + if (static_branch_likely(&sched_clock_irqtime)) { irqtime_account_idle_ticks(ticks); return; } diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 8063db62b027..db7d541eebff 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -3174,6 +3174,7 @@ struct irqtime { }; =20 DECLARE_PER_CPU(struct irqtime, cpu_irqtime); +DECLARE_STATIC_KEY_FALSE(sched_clock_irqtime); =20 /* * Returns the irqtime minus the softirq time computed by ksoftirqd. --=20 2.43.5 From nobody Wed Nov 27 04:31:30 2024 Received: from mail-pg1-f172.google.com (mail-pg1-f172.google.com [209.85.215.172]) (using TLSv1.2 with cipher ECDHE-RSA-AES128-GCM-SHA256 (128/128 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 52EEB43ACB; Mon, 14 Oct 2024 03:11:25 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=209.85.215.172 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1728875486; cv=none; b=Tx0HKBgiEDgJvsid3Grojg85U6C5mdn5/6JU0ry63HNy7KhPrhHyawd1Ekny9+XOBnwMOkRZnpY15hF+N6ScIcb6pVzgppGPFxtObx0hVfAY1wOiYPtE6ZwUKAAiOed1iHEYu0jQw1A/GJYX8gytL75kv0NH6s2r7G1BqJncwn8= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1728875486; c=relaxed/simple; bh=Eb8d5lyXFhSllOGcwU1VmnjYWiuaJB9lPUZZG8MTkDE=; h=From:To:Cc:Subject:Date:Message-Id:In-Reply-To:References: MIME-Version; b=ny6iv7oRHXKrTFcwvV58V2t80W4OJHj1XumhHgSo26oEys2aRvmbHN322eCsqCHGh4bn3lchvg964njRPxp5LuWscCjsxpeZ5B7mA51H0Zqevyi62D08T5DimRDhwYVc7nyutXkX3k1BQMUM2UpjHLIgQjk3v32f8IVgNomW/mU= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=gmail.com; spf=pass smtp.mailfrom=gmail.com; dkim=pass (2048-bit key) header.d=gmail.com header.i=@gmail.com header.b=jc0yLBcs; arc=none smtp.client-ip=209.85.215.172 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=gmail.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=gmail.com Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=gmail.com header.i=@gmail.com header.b="jc0yLBcs" Received: by mail-pg1-f172.google.com with SMTP id 41be03b00d2f7-7ea7e2ff5ceso433917a12.2; Sun, 13 Oct 2024 20:11:25 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=gmail.com; s=20230601; t=1728875485; x=1729480285; darn=vger.kernel.org; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:from:to:cc:subject:date :message-id:reply-to; bh=DCA0RTe4B5CkvnInMX18gUuR5vgyuSITkXk+PtSiiZc=; b=jc0yLBcsWqkpRBKhZe0WWV7LfdjszG2PPu1EvPT+qxlzSAYQqD/lBrPUtMv0G0s9aG fsDBZVC87eIzfblSos/kwwdwv7IRmRX273kIasz1a7d+U63/kJjgb3BIOZh/JNdd1Jq6 2eUgo0eLXYXLg9sUp8buFGOWtbG4opgMuddXHowztRXVEUkSdccQBJqeK9nsjGXNt11m FgAa+l61yI8H5CgLjrjC6eWY/3nhb/CLK/ifLXjOlGb+xuaJ0R4AY5X+6XbYWEijX4pU bzEYhCHB2QR0uuIqhddShRb5F85YKZjqLXcNP3dQaomcN9KTIjAjgIEwUZVriwgRJWeS IQbQ== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20230601; t=1728875485; x=1729480285; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:x-gm-message-state:from:to:cc :subject:date:message-id:reply-to; bh=DCA0RTe4B5CkvnInMX18gUuR5vgyuSITkXk+PtSiiZc=; b=kPdQfc73KziKrZg5ZDo7awtijmFfpsGqeTriT3E1+7gIdFKyZfEs3627xyKt9ldiR7 WMVkaMFKezDyvzYFMJ9mp5vWFXwFM5z/ZXokN/G6hSlnOU0S703HYgi4BQb2GZBWIYkk E2wvshpxJoxgX4Fc3VIGpX0EfePCQx6y4nc9H5QESSYke+lRtAozQ49Cti9AZaQL17Rv bihSOjgBhnZAYZcVxk2WEH8Odzy6nkPDNHb1Ne284ZKA5hmzAvlbBGDuzN7Q+arVugNS 4JQf/M/r69DW9m6sBKmol72Pmcdwk+Eu+RWknlvU0dN/ngk/OxIBxEfJe8rW5J865X6L tFzw== X-Forwarded-Encrypted: i=1; AJvYcCVfvBnPk9hWsKKyK9RGPsQORaSqm228KNKmxsVvDxnJkX5Ert/Ha9Qw77AsYNYr42tsTdkLDtrhr7nyiTQ=@vger.kernel.org X-Gm-Message-State: AOJu0YwytB/4gdkWDmDSGVbPHrRfIKvyuqutw9pl3vP0+sXKKv9SaC5j HjQWtLusE/0Z8XUsx+J0YeexOa3JLMlufAcRDJ263ctOR97Nn+Gj X-Google-Smtp-Source: AGHT+IE7En5d/SWK9qXnmm9Y+WwUVVPi4obtfuxQzcdnLZbQ4AUsoBTQRGBT8Dje7/JHIXvcBbgX4w== X-Received: by 2002:a05:6a21:3998:b0:1d7:1277:8d12 with SMTP id adf61e73a8af0-1d8bcf5c061mr14099715637.29.1728875484643; Sun, 13 Oct 2024 20:11:24 -0700 (PDT) Received: from localhost.localdomain ([39.144.103.177]) by smtp.gmail.com with ESMTPSA id 41be03b00d2f7-7ea6e396a71sm2987514a12.73.2024.10.13.20.11.18 (version=TLS1_2 cipher=ECDHE-ECDSA-AES128-GCM-SHA256 bits=128/128); Sun, 13 Oct 2024 20:11:24 -0700 (PDT) From: Yafang Shao To: mingo@redhat.com, peterz@infradead.org, juri.lelli@redhat.com, vincent.guittot@linaro.org, dietmar.eggemann@arm.com, rostedt@goodmis.org, bsegall@google.com, mgorman@suse.de, vschneid@redhat.com, hannes@cmpxchg.org, surenb@google.com Cc: cgroups@vger.kernel.org, linux-kernel@vger.kernel.org, Yafang Shao Subject: [PATCH v3 bpf-next 2/4] sched: Don't account irq time if sched_clock_irqtime is disabled Date: Mon, 14 Oct 2024 11:10:55 +0800 Message-Id: <20241014031057.8199-3-laoar.shao@gmail.com> X-Mailer: git-send-email 2.30.1 (Apple Git-130) In-Reply-To: <20241014031057.8199-1-laoar.shao@gmail.com> References: <20241014031057.8199-1-laoar.shao@gmail.com> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" sched_clock_irqtime may be disabled due to the clock source, in which case IRQ time should not be accounted. Let's add a conditional check to avoid unnecessary logic. Signed-off-by: Yafang Shao --- kernel/sched/core.c | 44 +++++++++++++++++++++++--------------------- 1 file changed, 23 insertions(+), 21 deletions(-) diff --git a/kernel/sched/core.c b/kernel/sched/core.c index b6cc1cf499d6..8b633a14a60f 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -735,29 +735,31 @@ static void update_rq_clock_task(struct rq *rq, s64 d= elta) s64 __maybe_unused steal =3D 0, irq_delta =3D 0; =20 #ifdef CONFIG_IRQ_TIME_ACCOUNTING - irq_delta =3D irq_time_read(cpu_of(rq)) - rq->prev_irq_time; + if (static_branch_likely(&sched_clock_irqtime)) { + irq_delta =3D irq_time_read(cpu_of(rq)) - rq->prev_irq_time; =20 - /* - * Since irq_time is only updated on {soft,}irq_exit, we might run into - * this case when a previous update_rq_clock() happened inside a - * {soft,}IRQ region. - * - * When this happens, we stop ->clock_task and only update the - * prev_irq_time stamp to account for the part that fit, so that a next - * update will consume the rest. This ensures ->clock_task is - * monotonic. - * - * It does however cause some slight miss-attribution of {soft,}IRQ - * time, a more accurate solution would be to update the irq_time using - * the current rq->clock timestamp, except that would require using - * atomic ops. - */ - if (irq_delta > delta) - irq_delta =3D delta; + /* + * Since irq_time is only updated on {soft,}irq_exit, we might run into + * this case when a previous update_rq_clock() happened inside a + * {soft,}IRQ region. + * + * When this happens, we stop ->clock_task and only update the + * prev_irq_time stamp to account for the part that fit, so that a next + * update will consume the rest. This ensures ->clock_task is + * monotonic. + * + * It does however cause some slight miss-attribution of {soft,}IRQ + * time, a more accurate solution would be to update the irq_time using + * the current rq->clock timestamp, except that would require using + * atomic ops. + */ + if (irq_delta > delta) + irq_delta =3D delta; =20 - rq->prev_irq_time +=3D irq_delta; - delta -=3D irq_delta; - delayacct_irq(rq->curr, irq_delta); + rq->prev_irq_time +=3D irq_delta; + delta -=3D irq_delta; + delayacct_irq(rq->curr, irq_delta); + } #endif #ifdef CONFIG_PARAVIRT_TIME_ACCOUNTING if (static_key_false((¶virt_steal_rq_enabled))) { --=20 2.43.5 From nobody Wed Nov 27 04:31:30 2024 Received: from mail-pg1-f171.google.com (mail-pg1-f171.google.com [209.85.215.171]) (using TLSv1.2 with cipher ECDHE-RSA-AES128-GCM-SHA256 (128/128 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 163511BF2B; Mon, 14 Oct 2024 03:11:31 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=209.85.215.171 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1728875493; cv=none; b=n8zyjOIOjmWKOVhPck+VrVs+XvkanXddaLz6E9BGjHrFnrPPd1ZH0AOgYQjdB1OhkynoPo8LxBUDS0WdzESJkJqNoELiRRDxKsl0uHTy9I+tiYeTlCdyFJ2S9Z9MDJubYzf1llKHnRzLz4qQHUnlS16ba27Zrz6uZKaRnCnGgzw= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1728875493; c=relaxed/simple; bh=UATzxBpzvyjT4QxUNGjwaJojge+8LbeUNVFkaxtJjgs=; h=From:To:Cc:Subject:Date:Message-Id:In-Reply-To:References: MIME-Version; b=WDKDVG8z2ULDLKtpnOkMpDTUwe3sPG0gYRI+DoYTY1yxwNlp2RMhlLM/AjYpERsSCSsNZNdqmmV+E8Sh4TvxY1X6UFvCzjth0vPHyoCRb9emIB6JEsUsHaVCdbheDc1sDNkMoR1ua1vfB9z7BNBdKVKd4o5C98Mc36hJB6UDTRM= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=gmail.com; spf=pass smtp.mailfrom=gmail.com; dkim=pass (2048-bit key) header.d=gmail.com header.i=@gmail.com header.b=HdnJnzf7; arc=none smtp.client-ip=209.85.215.171 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=gmail.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=gmail.com Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=gmail.com header.i=@gmail.com header.b="HdnJnzf7" Received: by mail-pg1-f171.google.com with SMTP id 41be03b00d2f7-7ea76a12c32so950772a12.1; Sun, 13 Oct 2024 20:11:31 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=gmail.com; s=20230601; t=1728875491; x=1729480291; darn=vger.kernel.org; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:from:to:cc:subject:date :message-id:reply-to; bh=UTcsvLBt6qYLHkMhyyOrBJnopX+R4q5PsgT0wkU2hcE=; b=HdnJnzf7rRuxKNznejClZgRBhmcYN81NCaI7ti8sZhHy7pgcb/iGlH0ChMK35so5WU cvxzEaL6Tme/8tbVQg5iBbWF6lRMNy6tRFYILqiLnz0TKWROGANZ5MnbQnhmqlSmwg6Y TU3XBpjtWrbthmjiJgPsx9ODSLV+cBS9YF2WcSOErcK5HSk5lgI+L+XaAwUVD/WarySQ 9qwqygrF0jMp2smm4n/CWHlIYouY8jDITtt0vjkOYg4AdYmXUl7LcM0TN3ith2+CUhYn B8Zhm+2PTk9TIc4l+An9ikCIFm3eWNll4z5cxQv9q6U2KYNB5ZlWy2xPScu12y0yBTwv H/Lw== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20230601; t=1728875491; x=1729480291; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:x-gm-message-state:from:to:cc :subject:date:message-id:reply-to; bh=UTcsvLBt6qYLHkMhyyOrBJnopX+R4q5PsgT0wkU2hcE=; b=C54EHo7bEVFrvrVjPni8wj1tMK3iCfoYMGeMuxgik6hOQgO43hfkFflWoAqLsPLok2 OQ4DlIDAn2/gnIkd/RceevW1J6bRr5JwI61SNpfRgueWGsrAEYfqr+8xQRJoysgVmMIE X9v7EgSTN3KVW2P8yHKsk6tzMwPilhaRSzxUezeCJnwfY5rpuugU6sXCazuQH9juIr3u YE0B9ERYJ0WEMDxq1p6tDMo4kHNXEl7w4iAnPKpy90MzoCUiZu4JQuryt+GRLtHvaebW zDtecpK5IfBqwAkmrS2nDEbl8+kr9z9ZUNdBjENY84VpP1HuSG2oZZ4g74BpHWJxJ0oA gQRA== X-Forwarded-Encrypted: i=1; AJvYcCUAAJEGR4HWWJ7NTmUh4BlbAWzMZ620egArzVBbqQxxaMERzB1Rwkfd8edAJeN+5PpW1l9CASRcUuW5Imk=@vger.kernel.org X-Gm-Message-State: AOJu0YyNiUfBRcL5tRTuoXLUmgdzezu3YzXp5Mt0x1p14elPJa95DkDd YuaC2O4GXUGzLUVzLo+D8b5HjLKP5m56VHdhB/QupyPepxw2tLFY X-Google-Smtp-Source: AGHT+IGH33yhUOGKrNZb5vm8e8ExVR/UGC7dmxHYpaQ1ZiquElwKkxXSAqmH9hTHJVtpN/wOcpEImA== X-Received: by 2002:a05:6a20:db0c:b0:1d3:293d:4c5a with SMTP id adf61e73a8af0-1d8bcf3e6e8mr13900268637.22.1728875491345; Sun, 13 Oct 2024 20:11:31 -0700 (PDT) Received: from localhost.localdomain ([39.144.103.177]) by smtp.gmail.com with ESMTPSA id 41be03b00d2f7-7ea6e396a71sm2987514a12.73.2024.10.13.20.11.25 (version=TLS1_2 cipher=ECDHE-ECDSA-AES128-GCM-SHA256 bits=128/128); Sun, 13 Oct 2024 20:11:30 -0700 (PDT) From: Yafang Shao To: mingo@redhat.com, peterz@infradead.org, juri.lelli@redhat.com, vincent.guittot@linaro.org, dietmar.eggemann@arm.com, rostedt@goodmis.org, bsegall@google.com, mgorman@suse.de, vschneid@redhat.com, hannes@cmpxchg.org, surenb@google.com Cc: cgroups@vger.kernel.org, linux-kernel@vger.kernel.org, Yafang Shao Subject: [PATCH v3 bpf-next 3/4] sched, psi: Don't account irq time if sched_clock_irqtime is disabled Date: Mon, 14 Oct 2024 11:10:56 +0800 Message-Id: <20241014031057.8199-4-laoar.shao@gmail.com> X-Mailer: git-send-email 2.30.1 (Apple Git-130) In-Reply-To: <20241014031057.8199-1-laoar.shao@gmail.com> References: <20241014031057.8199-1-laoar.shao@gmail.com> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" sched_clock_irqtime may be disabled due to the clock source. When disabled, irq_time_read() won't change over time, so there is nothing to account. We can save iterating the whole hierarchy on every tick and context switch. Signed-off-by: Yafang Shao Acked-by: Johannes Weiner --- kernel/sched/psi.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/kernel/sched/psi.c b/kernel/sched/psi.c index 020d58967d4e..49d9c75be0c8 100644 --- a/kernel/sched/psi.c +++ b/kernel/sched/psi.c @@ -1000,7 +1000,8 @@ void psi_account_irqtime(struct rq *rq, struct task_s= truct *curr, struct task_st u64 now, irq; s64 delta; =20 - if (static_branch_likely(&psi_disabled)) + if (static_branch_likely(&psi_disabled) || + !static_branch_likely(&sched_clock_irqtime)) return; =20 if (!curr->pid) --=20 2.43.5 From nobody Wed Nov 27 04:31:30 2024 Received: from mail-pf1-f174.google.com (mail-pf1-f174.google.com [209.85.210.174]) (using TLSv1.2 with cipher ECDHE-RSA-AES128-GCM-SHA256 (128/128 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id A5DF97D07D; Mon, 14 Oct 2024 03:11:40 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=209.85.210.174 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1728875502; cv=none; b=Ff0oimoRmxFR0sACGJ5OXvh56z3EqsUfX4kwnS/61ILdlmqwcSCEDt0VQasM44/lzgivZmhNn1ifM029DAqzr+OH4J11Sg8di5CdCa6jupja02C6X6QOp91PmA7IrLlQ3jccjmToXeo0kRQuqRHBrQqI38X1IQ4huQSCmAN9HD4= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1728875502; c=relaxed/simple; bh=HJP6a2h9/x54OljTWBeq6TmIslcq6mJSw5J+xx53u48=; h=From:To:Cc:Subject:Date:Message-Id:In-Reply-To:References: MIME-Version:Content-Type; b=SdUxz+qYjAi2qQcKHgJBxXTWHyl4Op2sjozh13E1u6gamG9f3mJAOU3+hmKOk5wLk6yxkjrfjIzswI+HsU0DRFBj6nbdcjxOMJgshVQCrESSfrkVuyBxJJ67DLVvNuMPobM1gTelWV3tDAkviho3AnA2OJVYGpIBxMLtI8uIdxI= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=gmail.com; spf=pass smtp.mailfrom=gmail.com; dkim=pass (2048-bit key) header.d=gmail.com header.i=@gmail.com header.b=jSsytke/; arc=none smtp.client-ip=209.85.210.174 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=gmail.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=gmail.com Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=gmail.com header.i=@gmail.com header.b="jSsytke/" Received: by mail-pf1-f174.google.com with SMTP id d2e1a72fcca58-71e5b2814f6so744482b3a.2; Sun, 13 Oct 2024 20:11:40 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=gmail.com; s=20230601; t=1728875500; x=1729480300; darn=vger.kernel.org; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:from:to:cc:subject:date :message-id:reply-to; bh=z5golT4Suxm5vj1Uhhhzt+VjmPN8PE6WAYyWdar7nxo=; b=jSsytke/p6Wn0U+zrADoiPZS34cK1e8JxWOvqtQ8mZ0V5dTlvFMj9ZMYgQ75zvclqp hdzosZmXXbnumOp/2NZogCqEx2tHjdaMZ7k5OEzgDJVHY+AdCHQNOExc59PLPfa5rt5g ifMuOdm/XdzVINdMqTKJxbbWkO9l9nIfkNAsEG/9fI7u+dln+Ou19JFrLFvfRJSkc060 gcj5vFuJ+qDbPxoVJgbUu6g91uIDXwzVsdJpYxuIX0Z+gaYNmN8k4yPHRWNKAC8sW1UX T6iErCjo/eV9dd3faqqutkJLOVlZ00l5CYTDKyB5wwJxCcu5nKHa41McQlkYpLYFu3w9 eaxg== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20230601; t=1728875500; x=1729480300; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:x-gm-message-state:from:to:cc :subject:date:message-id:reply-to; bh=z5golT4Suxm5vj1Uhhhzt+VjmPN8PE6WAYyWdar7nxo=; b=uZsRlHY5I/ztp10RYj4KFgcUeCuvLpeZxDPlW0yPRqSk4PU3nCeF9F49yDV9AgnGOL sJSxTTVw9snOT1Q6mdGMQpbQDo3Q/EjbLZXXeBH8DsgO10n2V/Sa214+g8Z7dMgH79h6 MfY198qPdUPNMeuD3x0AhGZQcoR6tZg3/YwZQKnN+ioRclbwCyITXzBFZjAR3WcEk84X r2caPfWo7QiTDPqjgWO/Ggv7qAO5IKqZMT/4/SjoWwZ+uN9XjjDVUVdIbRIvj42iY2LK o0ZJQ4zYNJg6mu2/DVPfxwEsz3UCvNtPQYiRUteumsYQXmbVqsn/ciF6Chb0iI27NqES H65Q== X-Forwarded-Encrypted: i=1; AJvYcCWiLm9+zu4BYrlaTGHuqFEd/gP17w+w4VLgphlBB6MsGj2aMPnxzg4jEYfoX3Dsd2gLnkpADKZ86fO80HU=@vger.kernel.org X-Gm-Message-State: AOJu0YwnBRhA33mH3iNlag/BCJ++t0WlIc9MQ4JvbmfAx8vV2Xxa0Bh7 jfhmoJMLsBIR2Puuvz5THQ/4Mc/eWRfeLsQW1Z8sD9t8rh6tpgS1 X-Google-Smtp-Source: AGHT+IG/gwGJyn/d5hrBvoW9Srcb8iZ8RocW213bzEkYEyqINf9XYM4y39l3wHXlYWsXKOl+p5EwhQ== X-Received: by 2002:a05:6a21:393:b0:1d2:f00e:47bb with SMTP id adf61e73a8af0-1d8bcf3e69fmr14923929637.21.1728875499822; Sun, 13 Oct 2024 20:11:39 -0700 (PDT) Received: from localhost.localdomain ([39.144.103.177]) by smtp.gmail.com with ESMTPSA id 41be03b00d2f7-7ea6e396a71sm2987514a12.73.2024.10.13.20.11.32 (version=TLS1_2 cipher=ECDHE-ECDSA-AES128-GCM-SHA256 bits=128/128); Sun, 13 Oct 2024 20:11:39 -0700 (PDT) From: Yafang Shao To: mingo@redhat.com, peterz@infradead.org, juri.lelli@redhat.com, vincent.guittot@linaro.org, dietmar.eggemann@arm.com, rostedt@goodmis.org, bsegall@google.com, mgorman@suse.de, vschneid@redhat.com, hannes@cmpxchg.org, surenb@google.com Cc: cgroups@vger.kernel.org, linux-kernel@vger.kernel.org, Yafang Shao Subject: [PATCH v3 bpf-next 4/4] sched: Fix cgroup irq accounting for CONFIG_IRQ_TIME_ACCOUNTING Date: Mon, 14 Oct 2024 11:10:57 +0800 Message-Id: <20241014031057.8199-5-laoar.shao@gmail.com> X-Mailer: git-send-email 2.30.1 (Apple Git-130) In-Reply-To: <20241014031057.8199-1-laoar.shao@gmail.com> References: <20241014031057.8199-1-laoar.shao@gmail.com> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Type: text/plain; charset="utf-8" Content-Transfer-Encoding: quoted-printable After enabling CONFIG_IRQ_TIME_ACCOUNTING to monitor IRQ pressure in our container environment, we observed several noticeable behavioral changes. One of our IRQ-heavy services, such as Redis, reported a significant reduction in CPU usage after upgrading to the new kernel with CONFIG_IRQ_TIME_ACCOUNTING enabled. However, despite adding more threads to handle an increased workload, the CPU usage could not be raised. In other words, even though the container=E2=80=99s CPU usage appeared low, it= was unable to process more workloads to utilize additional CPU resources, which caused issues. This behavior can be demonstrated using netperf: function start_server() { for j in `seq 1 3`; do netserver -p $[12345+j] > /dev/null & done } server_ip=3D$1 function start_client() { # That applies to cgroup2 as well. mkdir -p /sys/fs/cgroup/cpuacct/test echo $$ > /sys/fs/cgroup/cpuacct/test/cgroup.procs for j in `seq 1 3`; do port=3D$[12345+j] taskset -c 0 netperf -H ${server_ip} -l ${run_time:-30000} \ -t TCP_STREAM -p $port -- -D -m 1k -M 1K -s 8k -S 8k \ > /dev/null & done } start_server start_client We can verify the CPU usage of the test cgroup using cpuacct.stat. The output shows: system: 53 user: 2 The CPU usage of the cgroup is relatively low at around 55%, but this usage doesn't increase, even with more netperf tasks. The reason is that CPU0 is at 100% utilization, as confirmed by mpstat: 02:56:22 PM CPU %usr %nice %sys %iowait %irq %soft %steal = %guest %gnice %idle 02:56:23 PM 0 0.99 0.00 55.45 0.00 0.99 42.57 0.00 = 0.00 0.00 0.00 02:56:23 PM CPU %usr %nice %sys %iowait %irq %soft %steal = %guest %gnice %idle 02:56:24 PM 0 2.00 0.00 55.00 0.00 0.00 43.00 0.00 = 0.00 0.00 0.00 It is clear that the %soft is not accounted into the cgroup of the interrupted task. This behavior is unexpected. We should account for IRQ time to the cgroup to reflect the pressure the group is under. After a thorough analysis, I discovered that this change in behavior is due to commit 305e6835e055 ("sched: Do not account irq time to current task"), which altered whether IRQ time should be charged to the interrupted task. While I agree that a task should not be penalized by random interrupts, the task itself cannot progress while interrupted. Therefore, the interrupted time should be reported to the user. The system metric in cpuacct.stat is crucial in indicating whether a container is under heavy system pressure, including IRQ/softirq activity. Hence, IRQ/softirq time should be accounted for in the cpuacct system usage, which also applies to cgroup2=E2=80=99s rstat. This patch reintroduces IRQ/softirq accounting to cgroups. Signed-off-by: Yafang Shao Cc: Johannes Weiner --- kernel/sched/core.c | 34 ++++++++++++++++++++++++++++++++-- kernel/sched/psi.c | 15 +++------------ kernel/sched/stats.h | 7 ++++--- 3 files changed, 39 insertions(+), 17 deletions(-) diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 8b633a14a60f..a99461c04cab 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -5566,6 +5566,36 @@ __setup("resched_latency_warn_ms=3D", setup_resched_= latency_warn_ms); static inline u64 cpu_resched_latency(struct rq *rq) { return 0; } #endif /* CONFIG_SCHED_DEBUG */ =20 +#ifdef CONFIG_IRQ_TIME_ACCOUNTING +static void account_irqtime(struct rq *rq, struct task_struct *curr, + struct task_struct *prev) +{ + int cpu =3D smp_processor_id(); + u64 now, irq; + s64 delta; + + if (!static_branch_likely(&sched_clock_irqtime)) + return; + + now =3D cpu_clock(cpu); + irq =3D irq_time_read(cpu); + delta =3D (s64)(irq - rq->psi_irq_time); + if (delta < 0) + return; + + rq->psi_irq_time =3D irq; + psi_account_irqtime(rq, curr, prev, now, delta); + cgroup_account_cputime(curr, delta); + /* We account both softirq and irq into softirq */ + cgroup_account_cputime_field(curr, CPUTIME_SOFTIRQ, delta); +} +#else +static inline void account_irqtime(struct rq *rq, struct task_struct *curr, + struct task_struct *prev) +{ +} +#endif + /* * This function gets called by the timer code, with HZ frequency. * We call it with interrupts disabled. @@ -5587,7 +5617,7 @@ void sched_tick(void) rq_lock(rq, &rf); =20 curr =3D rq->curr; - psi_account_irqtime(rq, curr, NULL); + account_irqtime(rq, curr, NULL); =20 update_rq_clock(rq); hw_pressure =3D arch_scale_hw_pressure(cpu_of(rq)); @@ -6667,7 +6697,7 @@ static void __sched notrace __schedule(int sched_mode) ++*switch_count; =20 migrate_disable_switch(rq, prev); - psi_account_irqtime(rq, prev, next); + account_irqtime(rq, prev, next); psi_sched_switch(prev, next, !task_on_rq_queued(prev)); =20 trace_sched_switch(preempt, prev, next, prev_state); diff --git a/kernel/sched/psi.c b/kernel/sched/psi.c index 49d9c75be0c8..ffa8aa372fbd 100644 --- a/kernel/sched/psi.c +++ b/kernel/sched/psi.c @@ -992,16 +992,14 @@ void psi_task_switch(struct task_struct *prev, struct= task_struct *next, } =20 #ifdef CONFIG_IRQ_TIME_ACCOUNTING -void psi_account_irqtime(struct rq *rq, struct task_struct *curr, struct t= ask_struct *prev) +void psi_account_irqtime(struct rq *rq, struct task_struct *curr, struct t= ask_struct *prev, + u64 now, s64 delta) { int cpu =3D task_cpu(curr); struct psi_group *group; struct psi_group_cpu *groupc; - u64 now, irq; - s64 delta; =20 - if (static_branch_likely(&psi_disabled) || - !static_branch_likely(&sched_clock_irqtime)) + if (static_branch_likely(&psi_disabled)) return; =20 if (!curr->pid) @@ -1012,13 +1010,6 @@ void psi_account_irqtime(struct rq *rq, struct task_= struct *curr, struct task_st if (prev && task_psi_group(prev) =3D=3D group) return; =20 - now =3D cpu_clock(cpu); - irq =3D irq_time_read(cpu); - delta =3D (s64)(irq - rq->psi_irq_time); - if (delta < 0) - return; - rq->psi_irq_time =3D irq; - do { if (!group->enabled) continue; diff --git a/kernel/sched/stats.h b/kernel/sched/stats.h index 237780aa3c53..7c5979761021 100644 --- a/kernel/sched/stats.h +++ b/kernel/sched/stats.h @@ -111,10 +111,11 @@ void psi_task_change(struct task_struct *task, int cl= ear, int set); void psi_task_switch(struct task_struct *prev, struct task_struct *next, bool sleep); #ifdef CONFIG_IRQ_TIME_ACCOUNTING -void psi_account_irqtime(struct rq *rq, struct task_struct *curr, struct t= ask_struct *prev); +void psi_account_irqtime(struct rq *rq, struct task_struct *curr, + struct task_struct *prev, u64 now, s64 delta); #else static inline void psi_account_irqtime(struct rq *rq, struct task_struct *= curr, - struct task_struct *prev) {} + struct task_struct *prev, u64 now, s64 delta) {} #endif /*CONFIG_IRQ_TIME_ACCOUNTING */ /* * PSI tracks state that persists across sleeps, such as iowaits and @@ -197,7 +198,7 @@ static inline void psi_sched_switch(struct task_struct = *prev, struct task_struct *next, bool sleep) {} static inline void psi_account_irqtime(struct rq *rq, struct task_struct *= curr, - struct task_struct *prev) {} + struct task_struct *prev, u64 now, s64 delta) {} #endif /* CONFIG_PSI */ =20 #ifdef CONFIG_SCHED_INFO --=20 2.43.5