From nobody Wed Dec 31 13:04:15 2025 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id A2D46C0018A for ; Thu, 2 Nov 2023 15:33:24 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1376812AbjKBPdZ (ORCPT ); Thu, 2 Nov 2023 11:33:25 -0400 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:53246 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1347613AbjKBPdH (ORCPT ); Thu, 2 Nov 2023 11:33:07 -0400 Received: from casper.infradead.org (casper.infradead.org [IPv6:2001:8b0:10b:1236::1]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id A2D0C137 for ; Thu, 2 Nov 2023 08:32:53 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; d=infradead.org; s=casper.20170209; h=Content-Type:MIME-Version:References: Subject:Cc:To:From:Date:Message-Id:Sender:Reply-To:Content-Transfer-Encoding: Content-ID:Content-Description:In-Reply-To; bh=zYnYxbpqthdvjZXSbX3mfTIRB+RNguvN2goMXYp9Xns=; b=cxJLDQdGkB8ClhyhktXpUFBuAg S0Bzic4TiexMyJAqSvdRRhZIsHSgJkNnIyjJ1NPV3+s8AYBlYUkUwUN52YkM0TG1OsBS/5BE98dLa L5AYvGDBuRSTNMWjHiaMkIjI4D5C2itdmbpTaWCIxiRzdhx9PUdna+HAr5siNLbxVEX2tLpdqZilY /E8U61ux02QKGPF8pwMxWogLE5rG2PDt08aiC5OxKDVCRG4L3M6qTAGJt6/QZhsWc+Enj3LOG/CVZ 0Iz3PJwqOz25KYowq8Fe2BPHQYLIf61IR+RhpClDTdl3feTGbF3vE77hrZJVKuzPTiaeJZrVDQOoj 0U6hjr7g==; Received: from j130084.upc-j.chello.nl ([24.132.130.84] helo=noisy.programming.kicks-ass.net) by casper.infradead.org with esmtpsa (Exim 4.94.2 #2 (Red Hat Linux)) id 1qyZgU-0005PB-TJ; Thu, 02 Nov 2023 15:32:39 +0000 Received: by noisy.programming.kicks-ass.net (Postfix, from userid 0) id 3BC543004AB; Thu, 2 Nov 2023 16:32:39 +0100 (CET) Message-Id: <20231102152017.847792675@infradead.org> User-Agent: quilt/0.65 Date: Thu, 02 Nov 2023 16:09:20 +0100 From: Peter Zijlstra To: mingo@kernel.org Cc: linux-kernel@vger.kernel.org, peterz@infradead.org, acme@kernel.org, mark.rutland@arm.com, alexander.shishkin@linux.intel.com, jolsa@kernel.org, namhyung@kernel.org, irogers@google.com, adrian.hunter@intel.com Subject: [PATCH 01/13] perf: Simplify perf_event_alloc() error path References: <20231102150919.719936610@infradead.org> MIME-Version: 1.0 Precedence: bulk List-ID: X-Mailing-List: linux-kernel@vger.kernel.org Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" The error cleanup sequence in perf_event_alloc() is a subset of the existing _free_event() function (it must of course be). Split this out into __free_event() and simplify the error path. Signed-off-by: Peter Zijlstra (Intel) --- include/linux/perf_event.h | 1=20 kernel/events/core.c | 129 ++++++++++++++++++++++------------------= ----- 2 files changed, 66 insertions(+), 64 deletions(-) --- a/include/linux/perf_event.h +++ b/include/linux/perf_event.h @@ -634,6 +634,7 @@ struct swevent_hlist { #define PERF_ATTACH_ITRACE 0x10 #define PERF_ATTACH_SCHED_CB 0x20 #define PERF_ATTACH_CHILD 0x40 +#define PERF_ATTACH_EXCLUSIVE 0x80 =20 struct bpf_prog; struct perf_cgroup; --- a/kernel/events/core.c +++ b/kernel/events/core.c @@ -5094,6 +5094,8 @@ static int exclusive_event_init(struct p return -EBUSY; } =20 + event->attach_state |=3D PERF_ATTACH_EXCLUSIVE; + return 0; } =20 @@ -5101,14 +5103,13 @@ static void exclusive_event_destroy(stru { struct pmu *pmu =3D event->pmu; =20 - if (!is_exclusive_pmu(pmu)) - return; - /* see comment in exclusive_event_init() */ if (event->attach_state & PERF_ATTACH_TASK) atomic_dec(&pmu->exclusive_cnt); else atomic_inc(&pmu->exclusive_cnt); + + event->attach_state &=3D ~PERF_ATTACH_EXCLUSIVE; } =20 static bool exclusive_event_match(struct perf_event *e1, struct perf_event= *e2) @@ -5143,38 +5144,22 @@ static bool exclusive_event_installable( static void perf_addr_filters_splice(struct perf_event *event, struct list_head *head); =20 -static void _free_event(struct perf_event *event) +/* vs perf_event_alloc() error */ +static void __free_event(struct perf_event *event) { - irq_work_sync(&event->pending_irq); - - unaccount_event(event); - - security_perf_event_free(event); - - if (event->rb) { - /* - * Can happen when we close an event with re-directed output. - * - * Since we have a 0 refcount, perf_mmap_close() will skip - * over us; possibly making our ring_buffer_put() the last. - */ - mutex_lock(&event->mmap_mutex); - ring_buffer_attach(event, NULL); - mutex_unlock(&event->mmap_mutex); - } - - if (is_cgroup_event(event)) - perf_detach_cgroup(event); - if (!event->parent) { if (event->attr.sample_type & PERF_SAMPLE_CALLCHAIN) put_callchain_buffers(); } =20 - perf_event_free_bpf_prog(event); - perf_addr_filters_splice(event, NULL); kfree(event->addr_filter_ranges); =20 + if (event->attach_state & PERF_ATTACH_EXCLUSIVE) + exclusive_event_destroy(event); + + if (is_cgroup_event(event)) + perf_detach_cgroup(event); + if (event->destroy) event->destroy(event); =20 @@ -5185,22 +5170,56 @@ static void _free_event(struct perf_even if (event->hw.target) put_task_struct(event->hw.target); =20 - if (event->pmu_ctx) + if (event->pmu_ctx) { + /* + * put_pmu_ctx() needs an event->ctx reference, because of + * epc->ctx. + */ + WARN_ON_ONCE(!event->ctx); + WARN_ON_ONCE(event->pmu_ctx->ctx !=3D event->ctx); put_pmu_ctx(event->pmu_ctx); + } =20 /* - * perf_event_free_task() relies on put_ctx() being 'last', in particular - * all task references must be cleaned up. + * perf_event_free_task() relies on put_ctx() being 'last', in + * particular all task references must be cleaned up. */ if (event->ctx) put_ctx(event->ctx); =20 - exclusive_event_destroy(event); - module_put(event->pmu->module); + if (event->pmu) + module_put(event->pmu->module); =20 call_rcu(&event->rcu_head, free_event_rcu); } =20 +/* vs perf_event_alloc() success */ +static void _free_event(struct perf_event *event) +{ + irq_work_sync(&event->pending_irq); + + unaccount_event(event); + + security_perf_event_free(event); + + if (event->rb) { + /* + * Can happen when we close an event with re-directed output. + * + * Since we have a 0 refcount, perf_mmap_close() will skip + * over us; possibly making our ring_buffer_put() the last. + */ + mutex_lock(&event->mmap_mutex); + ring_buffer_attach(event, NULL); + mutex_unlock(&event->mmap_mutex); + } + + perf_event_free_bpf_prog(event); + perf_addr_filters_splice(event, NULL); + + __free_event(event); +} + /* * Used to free events which have a known refcount of 1, such as in error = paths * where the event isn't exposed yet and inherited events. @@ -11591,8 +11610,10 @@ static int perf_try_init_event(struct pm event->destroy(event); } =20 - if (ret) + if (ret) { + event->pmu =3D NULL; module_put(pmu->module); + } =20 return ret; } @@ -11918,7 +11939,7 @@ perf_event_alloc(struct perf_event_attr * See perf_output_read(). */ if (attr->inherit && (attr->sample_type & PERF_SAMPLE_READ)) - goto err_ns; + goto err; =20 if (!has_branch_stack(event)) event->attr.branch_sample_type =3D 0; @@ -11926,7 +11947,7 @@ perf_event_alloc(struct perf_event_attr pmu =3D perf_init_event(event); if (IS_ERR(pmu)) { err =3D PTR_ERR(pmu); - goto err_ns; + goto err; } =20 /* @@ -11936,24 +11957,24 @@ perf_event_alloc(struct perf_event_attr */ if (pmu->task_ctx_nr =3D=3D perf_invalid_context && (task || cgroup_fd != =3D -1)) { err =3D -EINVAL; - goto err_pmu; + goto err; } =20 if (event->attr.aux_output && !(pmu->capabilities & PERF_PMU_CAP_AUX_OUTPUT)) { err =3D -EOPNOTSUPP; - goto err_pmu; + goto err; } =20 if (cgroup_fd !=3D -1) { err =3D perf_cgroup_connect(cgroup_fd, event, attr, group_leader); if (err) - goto err_pmu; + goto err; } =20 err =3D exclusive_event_init(event); if (err) - goto err_pmu; + goto err; =20 if (has_addr_filter(event)) { event->addr_filter_ranges =3D kcalloc(pmu->nr_addr_filters, @@ -11961,7 +11982,7 @@ perf_event_alloc(struct perf_event_attr GFP_KERNEL); if (!event->addr_filter_ranges) { err =3D -ENOMEM; - goto err_per_task; + goto err; } =20 /* @@ -11986,41 +12007,21 @@ perf_event_alloc(struct perf_event_attr if (event->attr.sample_type & PERF_SAMPLE_CALLCHAIN) { err =3D get_callchain_buffers(attr->sample_max_stack); if (err) - goto err_addr_filters; + goto err; } } =20 err =3D security_perf_event_alloc(event); if (err) - goto err_callchain_buffer; + goto err; =20 /* symmetric to unaccount_event() in _free_event() */ account_event(event); =20 return event; =20 -err_callchain_buffer: - if (!event->parent) { - if (event->attr.sample_type & PERF_SAMPLE_CALLCHAIN) - put_callchain_buffers(); - } -err_addr_filters: - kfree(event->addr_filter_ranges); - -err_per_task: - exclusive_event_destroy(event); - -err_pmu: - if (is_cgroup_event(event)) - perf_detach_cgroup(event); - if (event->destroy) - event->destroy(event); - module_put(pmu->module); -err_ns: - if (event->hw.target) - put_task_struct(event->hw.target); - call_rcu(&event->rcu_head, free_event_rcu); - +err: + __free_event(event); return ERR_PTR(err); } From nobody Wed Dec 31 13:04:15 2025 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id EAB13C4332F for ; Thu, 2 Nov 2023 15:33:15 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1376598AbjKBPdO (ORCPT ); Thu, 2 Nov 2023 11:33:14 -0400 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:33538 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1347522AbjKBPdD (ORCPT ); Thu, 2 Nov 2023 11:33:03 -0400 Received: from casper.infradead.org (casper.infradead.org [IPv6:2001:8b0:10b:1236::1]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id A3D34186 for ; Thu, 2 Nov 2023 08:32:53 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; d=infradead.org; s=casper.20170209; h=Content-Type:MIME-Version:References: Subject:Cc:To:From:Date:Message-Id:Sender:Reply-To:Content-Transfer-Encoding: Content-ID:Content-Description:In-Reply-To; bh=QznUQ9ZVMRK7rQRlb8+/azdh4eJUviFCrGGLgMbGcIA=; b=H8B1Hayd6YjoE5cZpaxqk3O9Dx V5cwjBsVF24xhVkqtJUzR/sYhFil7mq+Ql9ISiJndEi6dSXOkgCs8B4iyQqy97swCY+zC5mNALUkO kB+YQaW+7ntHlf1zw7HwmSLBncjx9y3a9WvFm6XorUgojk2Hk3yaX2vMsX/VUHoTPOsmO9Q4S/yp2 XSkj3+p/xhADYgUrvDymiZ86u1BQwHAtmzT6Z8z6pSeRAIdf446yxbfgxm6T3BIjDD26tTLpnc+AF rVrFnv6lP0D7jQQONZSDddmurPnGxKIs+goi1F7n+ip8pETUiDyh/XyQHus+ERcBYF/CkBxlGi+5g gXx0DEkA==; Received: from j130084.upc-j.chello.nl ([24.132.130.84] helo=noisy.programming.kicks-ass.net) by casper.infradead.org with esmtpsa (Exim 4.94.2 #2 (Red Hat Linux)) id 1qyZgU-0005PC-TU; Thu, 02 Nov 2023 15:32:39 +0000 Received: by noisy.programming.kicks-ass.net (Postfix, from userid 0) id 3FD81300940; Thu, 2 Nov 2023 16:32:39 +0100 (CET) Message-Id: <20231102152017.957609849@infradead.org> User-Agent: quilt/0.65 Date: Thu, 02 Nov 2023 16:09:21 +0100 From: Peter Zijlstra To: mingo@kernel.org Cc: linux-kernel@vger.kernel.org, peterz@infradead.org, acme@kernel.org, mark.rutland@arm.com, alexander.shishkin@linux.intel.com, jolsa@kernel.org, namhyung@kernel.org, irogers@google.com, adrian.hunter@intel.com Subject: [PATCH 02/13] perf: Simplify perf_pmu_register() error path References: <20231102150919.719936610@infradead.org> MIME-Version: 1.0 Precedence: bulk List-ID: X-Mailing-List: linux-kernel@vger.kernel.org Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" The error path of perf_pmu_register() is of course very similar to a subset of perf_pmu_unregister(). Extract this common part in perf_pmu_free() and simplify things. Signed-off-by: Peter Zijlstra (Intel) --- kernel/events/core.c | 51 ++++++++++++++++++++++++----------------------= ----- 1 file changed, 24 insertions(+), 27 deletions(-) --- a/kernel/events/core.c +++ b/kernel/events/core.c @@ -11426,20 +11426,35 @@ static int pmu_dev_alloc(struct pmu *pmu static struct lock_class_key cpuctx_mutex; static struct lock_class_key cpuctx_lock; =20 +static void perf_pmu_free(struct pmu *pmu) +{ + free_percpu(pmu->pmu_disable_count); + if (pmu->type >=3D 0) + idr_remove(&pmu_idr, pmu->type); + if (pmu_bus_running && pmu->dev && pmu->dev !=3D PMU_NULL_DEV) { + if (pmu->nr_addr_filters) + device_remove_file(pmu->dev, &dev_attr_nr_addr_filters); + device_del(pmu->dev); + put_device(pmu->dev); + } + free_pmu_context(pmu); +} + int perf_pmu_register(struct pmu *pmu, const char *name, int type) { int cpu, ret, max =3D PERF_TYPE_MAX; =20 + pmu->type =3D -1; + mutex_lock(&pmus_lock); ret =3D -ENOMEM; pmu->pmu_disable_count =3D alloc_percpu(int); if (!pmu->pmu_disable_count) goto unlock; =20 - pmu->type =3D -1; if (WARN_ONCE(!name, "Can not register anonymous pmu.\n")) { ret =3D -EINVAL; - goto free_pdc; + goto free; } =20 pmu->name =3D name; @@ -11449,23 +11464,22 @@ int perf_pmu_register(struct pmu *pmu, c =20 ret =3D idr_alloc(&pmu_idr, pmu, max, 0, GFP_KERNEL); if (ret < 0) - goto free_pdc; + goto free; =20 WARN_ON(type >=3D 0 && ret !=3D type); =20 - type =3D ret; - pmu->type =3D type; + pmu->type =3D ret; =20 if (pmu_bus_running && !pmu->dev) { ret =3D pmu_dev_alloc(pmu); if (ret) - goto free_idr; + goto free; } =20 ret =3D -ENOMEM; pmu->cpu_pmu_context =3D alloc_percpu(struct perf_cpu_pmu_context); if (!pmu->cpu_pmu_context) - goto free_dev; + goto free; =20 for_each_possible_cpu(cpu) { struct perf_cpu_pmu_context *cpc; @@ -11511,17 +11525,8 @@ int perf_pmu_register(struct pmu *pmu, c =20 return ret; =20 -free_dev: - if (pmu->dev && pmu->dev !=3D PMU_NULL_DEV) { - device_del(pmu->dev); - put_device(pmu->dev); - } - -free_idr: - idr_remove(&pmu_idr, pmu->type); - -free_pdc: - free_percpu(pmu->pmu_disable_count); +free: + perf_pmu_free(pmu); goto unlock; } EXPORT_SYMBOL_GPL(perf_pmu_register); @@ -11538,15 +11543,7 @@ void perf_pmu_unregister(struct pmu *pmu synchronize_srcu(&pmus_srcu); synchronize_rcu(); =20 - free_percpu(pmu->pmu_disable_count); - idr_remove(&pmu_idr, pmu->type); - if (pmu_bus_running && pmu->dev && pmu->dev !=3D PMU_NULL_DEV) { - if (pmu->nr_addr_filters) - device_remove_file(pmu->dev, &dev_attr_nr_addr_filters); - device_del(pmu->dev); - put_device(pmu->dev); - } - free_pmu_context(pmu); + perf_pmu_free(pmu); mutex_unlock(&pmus_lock); } EXPORT_SYMBOL_GPL(perf_pmu_unregister); From nobody Wed Dec 31 13:04:15 2025 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id E6396C4332F for ; Thu, 2 Nov 2023 15:33:08 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1347632AbjKBPdJ (ORCPT ); Thu, 2 Nov 2023 11:33:09 -0400 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:33490 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1345328AbjKBPdA (ORCPT ); Thu, 2 Nov 2023 11:33:00 -0400 Received: from casper.infradead.org (casper.infradead.org [IPv6:2001:8b0:10b:1236::1]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id A0F8AFB for ; Thu, 2 Nov 2023 08:32:53 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; d=infradead.org; s=casper.20170209; h=Content-Type:MIME-Version:References: Subject:Cc:To:From:Date:Message-Id:Sender:Reply-To:Content-Transfer-Encoding: Content-ID:Content-Description:In-Reply-To; bh=ldnPYKUyHMdijInwVZoC1NuQUIqnzmW6aookZYRZh1U=; b=jahtwHNhX+yr0vKRzO1fhTymzt SLnwIxFZMgFhCVyxgCoHHeHHNal4O09ds8khll2mHqBKCDVh1dg2ozVPmfCLnj1HDCscJXKAOx2eX Ee3F4Kb5E8i3CTw3G4yB+MVmOgYao23eRwYHX4UqNBoo4R/keJ47nkef3A28xvpucksuTa99bXHb3 ZW43WHCx7msUMSWBXjVDb+1D1Hb+R6OMJ9U7Jo1IMe6/becw+Moi5BFHjwsuOCeCEbCAiVuL+MJoK iFR/PUCJJU0V312y8oU8NDuqXEHCEFTrLny4nBVxCbIIoheFbGQeGydTLEw4m7Viz9C3J0D2Xgiab tZ/njmDA==; Received: from j130084.upc-j.chello.nl ([24.132.130.84] helo=noisy.programming.kicks-ass.net) by casper.infradead.org with esmtpsa (Exim 4.94.2 #2 (Red Hat Linux)) id 1qyZgU-0005PD-UQ; Thu, 02 Nov 2023 15:32:39 +0000 Received: by noisy.programming.kicks-ass.net (Postfix, from userid 0) id 4397930098F; Thu, 2 Nov 2023 16:32:39 +0100 (CET) Message-Id: <20231102152018.069585432@infradead.org> User-Agent: quilt/0.65 Date: Thu, 02 Nov 2023 16:09:22 +0100 From: Peter Zijlstra To: mingo@kernel.org Cc: linux-kernel@vger.kernel.org, peterz@infradead.org, acme@kernel.org, mark.rutland@arm.com, alexander.shishkin@linux.intel.com, jolsa@kernel.org, namhyung@kernel.org, irogers@google.com, adrian.hunter@intel.com Subject: [PATCH 03/13] perf: Simplify perf_fget_light() References: <20231102150919.719936610@infradead.org> MIME-Version: 1.0 Precedence: bulk List-ID: X-Mailing-List: linux-kernel@vger.kernel.org Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Introduce fdnull and use it to simplify perf_fget_light() to either return a valid struct fd or not -- much like fdget() itself. Signed-off-by: Peter Zijlstra (Intel) --- include/linux/file.h | 7 ++++++- kernel/events/core.c | 22 +++++++++++----------- 2 files changed, 17 insertions(+), 12 deletions(-) --- a/include/linux/file.h +++ b/include/linux/file.h @@ -59,6 +59,8 @@ static inline struct fd __to_fd(unsigned return (struct fd){(struct file *)(v & ~3),v & 3}; } =20 +#define fdnull __to_fd(0) + static inline struct fd fdget(unsigned int fd) { return __to_fd(__fdget(fd)); --- a/kernel/events/core.c +++ b/kernel/events/core.c @@ -5802,18 +5802,17 @@ EXPORT_SYMBOL_GPL(perf_event_period); =20 static const struct file_operations perf_fops; =20 -static inline int perf_fget_light(int fd, struct fd *p) +static inline struct fd perf_fdget(int fd) { struct fd f =3D fdget(fd); if (!f.file) - return -EBADF; + return fdnull; =20 if (f.file->f_op !=3D &perf_fops) { fdput(f); - return -EBADF; + return fdnull; } - *p =3D f; - return 0; + return f; } =20 static int perf_event_set_output(struct perf_event *event, @@ -5864,10 +5863,9 @@ static long _perf_ioctl(struct perf_even int ret; if (arg !=3D -1) { struct perf_event *output_event; - struct fd output; - ret =3D perf_fget_light(arg, &output); - if (ret) - return ret; + struct fd output =3D perf_fdget(arg); + if (!output.file) + return -EBADF; output_event =3D output.file->private_data; ret =3D perf_event_set_output(event, output_event); fdput(output); @@ -12401,9 +12399,11 @@ SYSCALL_DEFINE5(perf_event_open, return event_fd; =20 if (group_fd !=3D -1) { - err =3D perf_fget_light(group_fd, &group); - if (err) + group =3D perf_fdget(group_fd); + if (!group.file) { + err =3D -EBADF; goto err_fd; + } group_leader =3D group.file->private_data; if (flags & PERF_FLAG_FD_OUTPUT) output_event =3D group_leader; From nobody Wed Dec 31 13:04:15 2025 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id 1C55EC4332F for ; Thu, 2 Nov 2023 15:33:06 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S234875AbjKBPdG (ORCPT ); Thu, 2 Nov 2023 11:33:06 -0400 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:33474 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1343650AbjKBPdA (ORCPT ); Thu, 2 Nov 2023 11:33:00 -0400 Received: from desiato.infradead.org (desiato.infradead.org [IPv6:2001:8b0:10b:1:d65d:64ff:fe57:4e05]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id 71EA2182 for ; Thu, 2 Nov 2023 08:32:53 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; d=infradead.org; s=desiato.20200630; h=Content-Type:MIME-Version:References: Subject:Cc:To:From:Date:Message-Id:Sender:Reply-To:Content-Transfer-Encoding: Content-ID:Content-Description:In-Reply-To; bh=gJ0cDE/AWPvauqfBTyp4RE5Jcb6F1MLuevN6bVvwMR4=; b=E9j9R2VdzYpvPH7rwdpNxxMJZL Mc9Uj6K4UmJMbMKg2oBTNF6i6wR6vGVAtWk2bocVeGTSyak6Q+0BB7ILnNi0m3iO37oLJpqIg4XUt La3ST7Gss5YpASywS+2izd9ykg95moqR9MXiDwf2fm1k/qImKpzMOvG28ngIkGF5qweLTe2SWmhU8 v7vJ96pRbrcuHvG+YWR/0Zo5UnaZbbW9iB/gAk3zta5iAMsK3cEB1CWVf8d0AUpzjZ+0f12XihsUw GqDigxj3W1t5lYUcVEHpHs5MwElyg6fUyJZd9H4Snik/8eRKymF61DRXGMNnaW0HsiEzmpo6a/y2J Dx9Xz99A==; Received: from j130084.upc-j.chello.nl ([24.132.130.84] helo=noisy.programming.kicks-ass.net) by desiato.infradead.org with esmtpsa (Exim 4.96 #2 (Red Hat Linux)) id 1qyZgV-006gQM-26; Thu, 02 Nov 2023 15:32:41 +0000 Received: by noisy.programming.kicks-ass.net (Postfix, from userid 0) id 49CE63015B5; Thu, 2 Nov 2023 16:32:39 +0100 (CET) Message-Id: <20231102152018.177137916@infradead.org> User-Agent: quilt/0.65 Date: Thu, 02 Nov 2023 16:09:23 +0100 From: Peter Zijlstra To: mingo@kernel.org Cc: linux-kernel@vger.kernel.org, peterz@infradead.org, acme@kernel.org, mark.rutland@arm.com, alexander.shishkin@linux.intel.com, jolsa@kernel.org, namhyung@kernel.org, irogers@google.com, adrian.hunter@intel.com Subject: [PATCH 04/13] perf: Simplify event_function*() References: <20231102150919.719936610@infradead.org> MIME-Version: 1.0 Precedence: bulk List-ID: X-Mailing-List: linux-kernel@vger.kernel.org Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Use guards to reduce gotos and simplify control flow. Signed-off-by: Peter Zijlstra (Intel) --- kernel/events/core.c | 36 +++++++++++++++++++++--------------- 1 file changed, 21 insertions(+), 15 deletions(-) --- a/kernel/events/core.c +++ b/kernel/events/core.c @@ -214,6 +214,19 @@ struct event_function_struct { void *data; }; =20 +typedef struct { + struct perf_cpu_context *cpuctx; + struct perf_event_context *ctx; +} class_perf_ctx_lock_t; + +static inline void class_perf_ctx_lock_destructor(class_perf_ctx_lock_t *_= T) +{ perf_ctx_unlock(_T->cpuctx, _T->ctx); } + +static inline class_perf_ctx_lock_t +class_perf_ctx_lock_constructor(struct perf_cpu_context *cpuctx, + struct perf_event_context *ctx) +{ perf_ctx_lock(cpuctx, ctx); return (class_perf_ctx_lock_t){ cpuctx, ctx = }; } + static int event_function(void *info) { struct event_function_struct *efs =3D info; @@ -221,20 +234,17 @@ static int event_function(void *info) struct perf_event_context *ctx =3D event->ctx; struct perf_cpu_context *cpuctx =3D this_cpu_ptr(&perf_cpu_context); struct perf_event_context *task_ctx =3D cpuctx->task_ctx; - int ret =3D 0; =20 lockdep_assert_irqs_disabled(); + guard(perf_ctx_lock)(cpuctx, task_ctx); =20 - perf_ctx_lock(cpuctx, task_ctx); /* * Since we do the IPI call without holding ctx->lock things can have * changed, double check we hit the task we set out to hit. */ if (ctx->task) { - if (ctx->task !=3D current) { - ret =3D -ESRCH; - goto unlock; - } + if (ctx->task !=3D current) + return -ESRCH; =20 /* * We only use event_function_call() on established contexts, @@ -254,10 +264,8 @@ static int event_function(void *info) } =20 efs->func(event, cpuctx, ctx, efs->data); -unlock: - perf_ctx_unlock(cpuctx, task_ctx); =20 - return ret; + return 0; } =20 static void event_function_call(struct perf_event *event, event_f func, vo= id *data) @@ -329,11 +337,11 @@ static void event_function_local(struct task_ctx =3D ctx; } =20 - perf_ctx_lock(cpuctx, task_ctx); + guard(perf_ctx_lock)(cpuctx, task_ctx); =20 task =3D ctx->task; if (task =3D=3D TASK_TOMBSTONE) - goto unlock; + return; =20 if (task) { /* @@ -343,18 +351,16 @@ static void event_function_local(struct */ if (ctx->is_active) { if (WARN_ON_ONCE(task !=3D current)) - goto unlock; + return; =20 if (WARN_ON_ONCE(cpuctx->task_ctx !=3D ctx)) - goto unlock; + return; } } else { WARN_ON_ONCE(&cpuctx->ctx !=3D ctx); } =20 func(event, cpuctx, ctx, data); -unlock: - perf_ctx_unlock(cpuctx, task_ctx); } =20 #define PERF_FLAG_ALL (PERF_FLAG_FD_NO_GROUP |\ From nobody Wed Dec 31 13:04:15 2025 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id 5E26EC4167B for ; Thu, 2 Nov 2023 15:33:19 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1376774AbjKBPdT (ORCPT ); Thu, 2 Nov 2023 11:33:19 -0400 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:33530 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1347525AbjKBPdD (ORCPT ); Thu, 2 Nov 2023 11:33:03 -0400 Received: from casper.infradead.org (casper.infradead.org [IPv6:2001:8b0:10b:1236::1]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id A3820185 for ; Thu, 2 Nov 2023 08:32:53 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; d=infradead.org; s=casper.20170209; h=Content-Type:MIME-Version:References: Subject:Cc:To:From:Date:Message-Id:Sender:Reply-To:Content-Transfer-Encoding: Content-ID:Content-Description:In-Reply-To; bh=nVJ2BrBhs84Vo8op91bCrEqQKyYGP8iSvvBesTbvcL8=; b=rABb7LV3Tw27YsY9IlOdGOlTXx gGljf5bb0ix4DLZ9VPLkCYUfnE881R1/OkrO0I9jJxJPXOQNcozuYIdsSlSdQXgYWL+0BfvUIsr8l OdntMdkVRuBIvA5mrZJl2gmABAGqaz97zI56lJ4uYG/mr4NytS0l2BDyigxSySU++od46JThpDDpk 9TPpW9aTs2ZrB74l6XgpGwzgK1QdlrDR1BVgTjF1IJwQmxR8O/BaWivQnGeUc1DV4Co5qmNokddCm bO4jekbhQyrroQtLUQ05nl5LA2G6EM+6L37WhXvk9peSENHrKkp9/Uq6gdbTEeixpxIaJ7P1S4iEA WDoLNhLQ==; Received: from j130084.upc-j.chello.nl ([24.132.130.84] helo=noisy.programming.kicks-ass.net) by casper.infradead.org with esmtpsa (Exim 4.94.2 #2 (Red Hat Linux)) id 1qyZgV-0005PI-Eg; Thu, 02 Nov 2023 15:32:40 +0000 Received: by noisy.programming.kicks-ass.net (Postfix, from userid 0) id 4C239301C46; Thu, 2 Nov 2023 16:32:39 +0100 (CET) Message-Id: <20231102152018.285699719@infradead.org> User-Agent: quilt/0.65 Date: Thu, 02 Nov 2023 16:09:24 +0100 From: Peter Zijlstra To: mingo@kernel.org Cc: linux-kernel@vger.kernel.org, peterz@infradead.org, acme@kernel.org, mark.rutland@arm.com, alexander.shishkin@linux.intel.com, jolsa@kernel.org, namhyung@kernel.org, irogers@google.com, adrian.hunter@intel.com Subject: [PATCH 05/13] perf: Simplify perf_cgroup_connect() References: <20231102150919.719936610@infradead.org> MIME-Version: 1.0 Precedence: bulk List-ID: X-Mailing-List: linux-kernel@vger.kernel.org Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Use CLASS to reduce gotos and simplify control flow. Signed-off-by: Peter Zijlstra (Intel) --- include/linux/file.h | 2 +- kernel/events/core.c | 19 ++++++++----------- 2 files changed, 9 insertions(+), 12 deletions(-) --- a/kernel/events/core.c +++ b/kernel/events/core.c @@ -936,22 +936,20 @@ static inline int perf_cgroup_connect(in { struct perf_cgroup *cgrp; struct cgroup_subsys_state *css; - struct fd f =3D fdget(fd); - int ret =3D 0; + int ret; =20 + CLASS(fd, f)(fd); if (!f.file) return -EBADF; =20 css =3D css_tryget_online_from_dir(f.file->f_path.dentry, &perf_event_cgrp_subsys); - if (IS_ERR(css)) { - ret =3D PTR_ERR(css); - goto out; - } + if (IS_ERR(css)) + return PTR_ERR(css); =20 ret =3D perf_cgroup_ensure_storage(event, css); if (ret) - goto out; + return ret; =20 cgrp =3D container_of(css, struct perf_cgroup, css); event->cgrp =3D cgrp; @@ -963,11 +961,10 @@ static inline int perf_cgroup_connect(in */ if (group_leader && group_leader->cgrp !=3D cgrp) { perf_detach_cgroup(event); - ret =3D -EINVAL; + return -EINVAL; } -out: - fdput(f); - return ret; + + return 0; } =20 static inline void From nobody Wed Dec 31 13:04:15 2025 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id B29F8C4332F for ; Thu, 2 Nov 2023 15:33:00 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1346501AbjKBPdB (ORCPT ); Thu, 2 Nov 2023 11:33:01 -0400 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:33446 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S229575AbjKBPc7 (ORCPT ); Thu, 2 Nov 2023 11:32:59 -0400 Received: from desiato.infradead.org (desiato.infradead.org [IPv6:2001:8b0:10b:1:d65d:64ff:fe57:4e05]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id 72B88138 for ; Thu, 2 Nov 2023 08:32:51 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; d=infradead.org; s=desiato.20200630; h=Content-Type:MIME-Version:References: Subject:Cc:To:From:Date:Message-Id:Sender:Reply-To:Content-Transfer-Encoding: Content-ID:Content-Description:In-Reply-To; bh=TIflEc6HK/22xB9+LAc8fdYfMJHSxjY/dB735CDFq7w=; b=P6kB5/pEItGyT9udySZwn5g3Qv DokDZMGlAiB4g+5YMtKB0RmZbfY/pSLOWVbj5SPF6cIDKqKZn1inehV8/qd0OzY/wn9JdZcifSQj8 3dH2nA6JS9foEyCWzBfy2FYGKj4LmT0pao4ngY0As0C25SUXZ540eeGac7JIDrYE/ebOo/ehFs1XS uEVmLDDifA+yYjw0mpY8V81H06JBxnpUcCgwR66s2d9LTWqPI5szoghluskYsv0VEb+nsMdEQYP/6 iTI4qNM/z+xa/z7t/+iAAIFUnvAPSqkLVuq9byN3ej3VbidX8FqhNvqpNRFC/A+oIEiju20IssCS+ VKZ9UYPA==; Received: from j130084.upc-j.chello.nl ([24.132.130.84] helo=noisy.programming.kicks-ass.net) by desiato.infradead.org with esmtpsa (Exim 4.96 #2 (Red Hat Linux)) id 1qyZgW-006gQO-0R; Thu, 02 Nov 2023 15:32:41 +0000 Received: by noisy.programming.kicks-ass.net (Postfix, from userid 0) id 505FE302187; Thu, 2 Nov 2023 16:32:39 +0100 (CET) Message-Id: <20231102152018.391356347@infradead.org> User-Agent: quilt/0.65 Date: Thu, 02 Nov 2023 16:09:25 +0100 From: Peter Zijlstra To: mingo@kernel.org Cc: linux-kernel@vger.kernel.org, peterz@infradead.org, acme@kernel.org, mark.rutland@arm.com, alexander.shishkin@linux.intel.com, jolsa@kernel.org, namhyung@kernel.org, irogers@google.com, adrian.hunter@intel.com Subject: [PATCH 06/13] perf; Simplify event_sched_in() References: <20231102150919.719936610@infradead.org> MIME-Version: 1.0 Precedence: bulk List-ID: X-Mailing-List: linux-kernel@vger.kernel.org Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Use guards to reduce gotos and simplify control flow. Signed-off-by: Peter Zijlstra (Intel) --- kernel/events/core.c | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) --- a/kernel/events/core.c +++ b/kernel/events/core.c @@ -1153,6 +1153,8 @@ void perf_pmu_enable(struct pmu *pmu) pmu->pmu_enable(pmu); } =20 +DEFINE_GUARD(perf_pmu_disable, struct pmu *, perf_pmu_disable(_T), perf_pm= u_enable(_T)) + static void perf_assert_pmu_disabled(struct pmu *pmu) { WARN_ON_ONCE(*this_cpu_ptr(pmu->pmu_disable_count) =3D=3D 0); @@ -2489,7 +2491,6 @@ event_sched_in(struct perf_event *event, { struct perf_event_pmu_context *epc =3D event->pmu_ctx; struct perf_cpu_pmu_context *cpc =3D this_cpu_ptr(epc->pmu->cpu_pmu_conte= xt); - int ret =3D 0; =20 WARN_ON_ONCE(event->ctx !=3D ctx); =20 @@ -2517,15 +2518,14 @@ event_sched_in(struct perf_event *event, event->hw.interrupts =3D 0; } =20 - perf_pmu_disable(event->pmu); + guard(perf_pmu_disable)(event->pmu); =20 perf_log_itrace_start(event); =20 if (event->pmu->add(event, PERF_EF_START)) { perf_event_set_state(event, PERF_EVENT_STATE_INACTIVE); event->oncpu =3D -1; - ret =3D -EAGAIN; - goto out; + return -EAGAIN; } =20 if (!is_software_event(event)) @@ -2536,10 +2536,7 @@ event_sched_in(struct perf_event *event, if (event->attr.exclusive) cpc->exclusive =3D 1; =20 -out: - perf_pmu_enable(event->pmu); - - return ret; + return 0; } =20 static int From nobody Wed Dec 31 13:04:15 2025 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id 95F9BC4332F for ; Thu, 2 Nov 2023 15:33:27 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1376820AbjKBPd1 (ORCPT ); Thu, 2 Nov 2023 11:33:27 -0400 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:53234 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1347591AbjKBPdH (ORCPT ); Thu, 2 Nov 2023 11:33:07 -0400 Received: from casper.infradead.org (casper.infradead.org [IPv6:2001:8b0:10b:1236::1]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id A415B187 for ; Thu, 2 Nov 2023 08:32:53 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; d=infradead.org; s=casper.20170209; h=Content-Type:MIME-Version:References: Subject:Cc:To:From:Date:Message-Id:Sender:Reply-To:Content-Transfer-Encoding: Content-ID:Content-Description:In-Reply-To; bh=vBUL+P10jAOvg5j6kSlF2qo+oprvGK1yGYCCYSiAqlM=; b=sZEQnqGXe7KSXx/yVjySpREPrn 0QnaDtfJ4YE6CUOXoCoN8KNoCOu8lcspocLrKrgaugUK4GSm+mvfllmLeN/Sk8ePDXVi14V28PqPu swe3JNG+QFhta7azvPzQDhUsBmYHOYZvWV81qUGSURgqFmzBCl/bbj2Ju21q43AWoKbV0hsNZgkVc 5aJ25OIwHdVcZdMmFDUSqr0/feccxlzH71QSjwPdy8wDQBwh5kMASmhhAmZ6+11tzn9n3oOSVJZrj MaGlVb6zrAeiPe6+ODTLQcrwrSGEUym0qiNqyGHhY8TQBpbYymFq1rjuJAA2OOUkPZIQKxnCAKIJV iRcWyoRA==; Received: from j130084.upc-j.chello.nl ([24.132.130.84] helo=noisy.programming.kicks-ass.net) by casper.infradead.org with esmtpsa (Exim 4.94.2 #2 (Red Hat Linux)) id 1qyZgV-0005PJ-Et; Thu, 02 Nov 2023 15:32:40 +0000 Received: by noisy.programming.kicks-ass.net (Postfix, from userid 0) id 54ACC3021E1; Thu, 2 Nov 2023 16:32:39 +0100 (CET) Message-Id: <20231102152018.499897182@infradead.org> User-Agent: quilt/0.65 Date: Thu, 02 Nov 2023 16:09:26 +0100 From: Peter Zijlstra To: mingo@kernel.org Cc: linux-kernel@vger.kernel.org, peterz@infradead.org, acme@kernel.org, mark.rutland@arm.com, alexander.shishkin@linux.intel.com, jolsa@kernel.org, namhyung@kernel.org, irogers@google.com, adrian.hunter@intel.com Subject: [PATCH 07/13] perf: Simplify: __perf_install_in_context() References: <20231102150919.719936610@infradead.org> MIME-Version: 1.0 Precedence: bulk List-ID: X-Mailing-List: linux-kernel@vger.kernel.org Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Signed-off-by: Peter Zijlstra (Intel) --- kernel/events/core.c | 21 +++++++-------------- 1 file changed, 7 insertions(+), 14 deletions(-) --- a/kernel/events/core.c +++ b/kernel/events/core.c @@ -2732,13 +2732,13 @@ static int __perf_install_in_context(vo struct perf_cpu_context *cpuctx =3D this_cpu_ptr(&perf_cpu_context); struct perf_event_context *task_ctx =3D cpuctx->task_ctx; bool reprogram =3D true; - int ret =3D 0; =20 - raw_spin_lock(&cpuctx->ctx.lock); - if (ctx->task) { - raw_spin_lock(&ctx->lock); + if (ctx->task) task_ctx =3D ctx; =20 + guard(perf_ctx_lock)(cpuctx, task_ctx); + + if (ctx->task) { reprogram =3D (ctx->task =3D=3D current); =20 /* @@ -2748,14 +2748,10 @@ static int __perf_install_in_context(vo * If its not running, we don't care, ctx->lock will * serialize against it becoming runnable. */ - if (task_curr(ctx->task) && !reprogram) { - ret =3D -ESRCH; - goto unlock; - } + if (task_curr(ctx->task) && !reprogram) + return -ESRCH; =20 WARN_ON_ONCE(reprogram && cpuctx->task_ctx && cpuctx->task_ctx !=3D ctx); - } else if (task_ctx) { - raw_spin_lock(&task_ctx->lock); } =20 #ifdef CONFIG_CGROUP_PERF @@ -2778,10 +2774,7 @@ static int __perf_install_in_context(vo add_event_to_ctx(event, ctx); } =20 -unlock: - perf_ctx_unlock(cpuctx, task_ctx); - - return ret; + return 0; } =20 static bool exclusive_event_installable(struct perf_event *event, From nobody Wed Dec 31 13:04:15 2025 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id 91943C4332F for ; Thu, 2 Nov 2023 15:33:30 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1376826AbjKBPda (ORCPT ); Thu, 2 Nov 2023 11:33:30 -0400 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:53264 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1347580AbjKBPdK (ORCPT ); Thu, 2 Nov 2023 11:33:10 -0400 Received: from desiato.infradead.org (desiato.infradead.org [IPv6:2001:8b0:10b:1:d65d:64ff:fe57:4e05]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id 6DC63193 for ; Thu, 2 Nov 2023 08:32:54 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; d=infradead.org; s=desiato.20200630; h=Content-Type:MIME-Version:References: Subject:Cc:To:From:Date:Message-Id:Sender:Reply-To:Content-Transfer-Encoding: Content-ID:Content-Description:In-Reply-To; bh=RF7AlZRmRPruwiE6WSJ8fgsVWKVvpbmYdppCd/KBEkQ=; b=pGACWceG5OkXDjHEQOIpl0EsDB tdjDIEfJtqREzGV8uDOeMJF18ilnbvzS9vXNGXp1RVpudhQJEaA9CWjnf1iANtvN6zRhN7FsNbQuU h92x20a4sQCzDow+aPWeGaEVyzSq2ChWnfXXR/tAAqA51m2sWyUQmkK1rMG+eri8nCFFtZnhNPpTf BHNoS/CNPiZ8w+FwstiR6H7zmovfOW9/xu/JKVgsqLgCup6OepD7DwHa3Kcdm0v2t51QoiKYE9BLa liKTm3md9tx3aojk9Ltl9+tIb+l4CgUo1Uigz+iMozgbC8rmymUEVYAlt7vXn8SVpEt2DL6uStp0n fpKbkelw==; Received: from j130084.upc-j.chello.nl ([24.132.130.84] helo=noisy.programming.kicks-ass.net) by desiato.infradead.org with esmtpsa (Exim 4.96 #2 (Red Hat Linux)) id 1qyZgW-006gQN-0R; Thu, 02 Nov 2023 15:32:41 +0000 Received: by noisy.programming.kicks-ass.net (Postfix, from userid 0) id 58794302202; Thu, 2 Nov 2023 16:32:39 +0100 (CET) Message-Id: <20231102152018.626814126@infradead.org> User-Agent: quilt/0.65 Date: Thu, 02 Nov 2023 16:09:27 +0100 From: Peter Zijlstra To: mingo@kernel.org Cc: linux-kernel@vger.kernel.org, peterz@infradead.org, acme@kernel.org, mark.rutland@arm.com, alexander.shishkin@linux.intel.com, jolsa@kernel.org, namhyung@kernel.org, irogers@google.com, adrian.hunter@intel.com Subject: [PATCH 08/13] perf: Simplify: *perf_event_{dis,en}able*() References: <20231102150919.719936610@infradead.org> MIME-Version: 1.0 Precedence: bulk List-ID: X-Mailing-List: linux-kernel@vger.kernel.org Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Signed-off-by: Peter Zijlstra (Intel) --- kernel/events/core.c | 51 ++++++++++++++++++++++------------------------= ----- 1 file changed, 22 insertions(+), 29 deletions(-) --- a/kernel/events/core.c +++ b/kernel/events/core.c @@ -2415,7 +2415,7 @@ static void __perf_event_disable(struct update_cgrp_time_from_event(event); } =20 - perf_pmu_disable(event->pmu_ctx->pmu); + guard(perf_pmu_disable)(event->pmu_ctx->pmu); =20 if (event =3D=3D event->group_leader) group_sched_out(event, ctx); @@ -2424,8 +2424,6 @@ static void __perf_event_disable(struct =20 perf_event_set_state(event, PERF_EVENT_STATE_OFF); perf_cgroup_event_disable(event, ctx); - - perf_pmu_enable(event->pmu_ctx->pmu); } =20 /* @@ -2446,12 +2444,10 @@ static void _perf_event_disable(struct p { struct perf_event_context *ctx =3D event->ctx; =20 - raw_spin_lock_irq(&ctx->lock); - if (event->state <=3D PERF_EVENT_STATE_OFF) { - raw_spin_unlock_irq(&ctx->lock); - return; + scoped_guard (raw_spinlock_irq, &ctx->lock) { + if (event->state <=3D PERF_EVENT_STATE_OFF) + return; } - raw_spin_unlock_irq(&ctx->lock); =20 event_function_call(event, __perf_event_disable, NULL); } @@ -2955,32 +2951,29 @@ static void _perf_event_enable(struct pe { struct perf_event_context *ctx =3D event->ctx; =20 - raw_spin_lock_irq(&ctx->lock); - if (event->state >=3D PERF_EVENT_STATE_INACTIVE || - event->state < PERF_EVENT_STATE_ERROR) { -out: - raw_spin_unlock_irq(&ctx->lock); - return; - } + scoped_guard (raw_spinlock_irq, &ctx->lock) { + if (event->state >=3D PERF_EVENT_STATE_INACTIVE || + event->state < PERF_EVENT_STATE_ERROR) + return; =20 - /* - * If the event is in error state, clear that first. - * - * That way, if we see the event in error state below, we know that it - * has gone back into error state, as distinct from the task having - * been scheduled away before the cross-call arrived. - */ - if (event->state =3D=3D PERF_EVENT_STATE_ERROR) { /* - * Detached SIBLING events cannot leave ERROR state. + * If the event is in error state, clear that first. + * + * That way, if we see the event in error state below, we know that it + * has gone back into error state, as distinct from the task having + * been scheduled away before the cross-call arrived. */ - if (event->event_caps & PERF_EV_CAP_SIBLING && - event->group_leader =3D=3D event) - goto out; + if (event->state =3D=3D PERF_EVENT_STATE_ERROR) { + /* + * Detached SIBLING events cannot leave ERROR state. + */ + if (event->event_caps & PERF_EV_CAP_SIBLING && + event->group_leader =3D=3D event) + return; =20 - event->state =3D PERF_EVENT_STATE_OFF; + event->state =3D PERF_EVENT_STATE_OFF; + } } - raw_spin_unlock_irq(&ctx->lock); =20 event_function_call(event, __perf_event_enable, NULL); } From nobody Wed Dec 31 13:04:15 2025 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id 180B8C4167B for ; Thu, 2 Nov 2023 15:33:33 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1376833AbjKBPdd (ORCPT ); Thu, 2 Nov 2023 11:33:33 -0400 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:53300 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1347638AbjKBPdK (ORCPT ); Thu, 2 Nov 2023 11:33:10 -0400 Received: from casper.infradead.org (casper.infradead.org [IPv6:2001:8b0:10b:1236::1]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id 1B43418E for ; Thu, 2 Nov 2023 08:32:54 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; d=infradead.org; s=casper.20170209; h=Content-Type:MIME-Version:References: Subject:Cc:To:From:Date:Message-Id:Sender:Reply-To:Content-Transfer-Encoding: Content-ID:Content-Description:In-Reply-To; bh=R8zokz8HpLYR/tS6Df4K9QtMMliKooDPknNNGx1Aq0A=; b=V3eJxy3aHR8sSOdEp2w15Upf3d /3SI+v3WkjGk9NFqxZbFJcgZTvJ79bAaWRNVfLFnbTBafrkhG6t62hkQx0oOa30hg/CHkjhSTKs8T cUWG0PDkodBntmFopGQYrkx+PV1dRnXtnJM9cfwRyafcWNHWZSInWD3PWjf6L3pInbgNwtEodyh/r jvbAavQzJNrczRFhhIF08eaT/eWLXij8kmL47lj4Cop0Kvl1GhC2HqMzAxVQg3h4JAbPTV91m5nFo tt4Qg5y/dxqrNQM8HTUXLIUcM7R/Mv5kfFiB4vG5DsijKr4/Ow0TsTbb7u8CqbMTal5AaY71DpjC5 Coafv0AQ==; Received: from j130084.upc-j.chello.nl ([24.132.130.84] helo=noisy.programming.kicks-ass.net) by casper.infradead.org with esmtpsa (Exim 4.94.2 #2 (Red Hat Linux)) id 1qyZgV-0005PK-Fo; Thu, 02 Nov 2023 15:32:40 +0000 Received: by noisy.programming.kicks-ass.net (Postfix, from userid 0) id 5C2CA302D66; Thu, 2 Nov 2023 16:32:39 +0100 (CET) Message-Id: <20231102152018.768730519@infradead.org> User-Agent: quilt/0.65 Date: Thu, 02 Nov 2023 16:09:28 +0100 From: Peter Zijlstra To: mingo@kernel.org Cc: linux-kernel@vger.kernel.org, peterz@infradead.org, acme@kernel.org, mark.rutland@arm.com, alexander.shishkin@linux.intel.com, jolsa@kernel.org, namhyung@kernel.org, irogers@google.com, adrian.hunter@intel.com Subject: [PATCH 09/13] perf: Simplify perf_event_modify_attr() References: <20231102150919.719936610@infradead.org> MIME-Version: 1.0 Precedence: bulk List-ID: X-Mailing-List: linux-kernel@vger.kernel.org Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Signed-off-by: Peter Zijlstra (Intel) --- kernel/events/core.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) --- a/kernel/events/core.c +++ b/kernel/events/core.c @@ -3172,7 +3172,7 @@ static int perf_event_modify_attr(struct =20 WARN_ON_ONCE(event->ctx->parent_ctx); =20 - mutex_lock(&event->child_mutex); + guard(mutex)(&event->child_mutex); /* * Event-type-independent attributes must be copied before event-type * modification, which will validate that final attributes match the @@ -3181,16 +3181,16 @@ static int perf_event_modify_attr(struct perf_event_modify_copy_attr(&event->attr, attr); err =3D func(event, attr); if (err) - goto out; + return err; + list_for_each_entry(child, &event->child_list, child_list) { perf_event_modify_copy_attr(&child->attr, attr); err =3D func(child, attr); if (err) - goto out; + return err; } -out: - mutex_unlock(&event->child_mutex); - return err; + + return 0; } =20 static void __pmu_ctx_sched_out(struct perf_event_pmu_context *pmu_ctx, From nobody Wed Dec 31 13:04:15 2025 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id 05C56C4167D for ; Thu, 2 Nov 2023 15:33:12 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1376467AbjKBPdM (ORCPT ); Thu, 2 Nov 2023 11:33:12 -0400 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:33514 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1347499AbjKBPdD (ORCPT ); Thu, 2 Nov 2023 11:33:03 -0400 Received: from casper.infradead.org (casper.infradead.org [IPv6:2001:8b0:10b:1236::1]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id A2E44184 for ; Thu, 2 Nov 2023 08:32:53 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; d=infradead.org; s=casper.20170209; h=Content-Type:MIME-Version:References: Subject:Cc:To:From:Date:Message-Id:Sender:Reply-To:Content-Transfer-Encoding: Content-ID:Content-Description:In-Reply-To; bh=xk+4iMONbkXiTk0hS6u3U/Ijn5wg/Ea5KL9bAZzCxZc=; b=i+xBFng42EokmYQs8E3C2aCCNt LSR9otznbyM7qctRZPGk2veM5RMeYAnviUjiBi+cvtz8pi6EKQ1THOGe7hFXgYZcZgcLfrRZrcai9 4qYWZgQm/EKq6HpWHXWiMMuDB7HEXuI8HI/Km9jPQ1LA/Tjwd+rmA8poBJITfQZZdrqurCRe8MCKn DeCjWFal0RmOtaVKlO07WMF32k3YpjokeRGgFGV+ZbtU8PJV1eIxA/WVDzbfoOvusZYKqkCVe+lZw DoPjVDgJ4ssLiV3pCs8XLXRhd8OuC6twA9k2bPVNse7KUCKX91ni2IB9foLt76C56OHuFRDD3GgP8 Ft8+a9MA==; Received: from j130084.upc-j.chello.nl ([24.132.130.84] helo=noisy.programming.kicks-ass.net) by casper.infradead.org with esmtpsa (Exim 4.94.2 #2 (Red Hat Linux)) id 1qyZgV-0005PL-Fp; Thu, 02 Nov 2023 15:32:40 +0000 Received: by noisy.programming.kicks-ass.net (Postfix, from userid 0) id 5FF52302D87; Thu, 2 Nov 2023 16:32:39 +0100 (CET) Message-Id: <20231102152018.878930233@infradead.org> User-Agent: quilt/0.65 Date: Thu, 02 Nov 2023 16:09:29 +0100 From: Peter Zijlstra To: mingo@kernel.org Cc: linux-kernel@vger.kernel.org, peterz@infradead.org, acme@kernel.org, mark.rutland@arm.com, alexander.shishkin@linux.intel.com, jolsa@kernel.org, namhyung@kernel.org, irogers@google.com, adrian.hunter@intel.com Subject: [PATCH 10/13] perf: Simplify perf_event_context_sched_in() References: <20231102150919.719936610@infradead.org> MIME-Version: 1.0 Precedence: bulk List-ID: X-Mailing-List: linux-kernel@vger.kernel.org Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Signed-off-by: Peter Zijlstra (Intel) --- kernel/events/core.c | 38 +++++++++++++++----------------------- 1 file changed, 15 insertions(+), 23 deletions(-) --- a/kernel/events/core.c +++ b/kernel/events/core.c @@ -713,6 +713,9 @@ static void perf_ctx_enable(struct perf_ } } =20 +DEFINE_GUARD(perf_ctx_disable, struct perf_event_context *, + perf_ctx_disable(_T, false), perf_ctx_enable(_T, false)) + static void ctx_sched_out(struct perf_event_context *ctx, enum event_type_= t event_type); static void ctx_sched_in(struct perf_event_context *ctx, enum event_type_t= event_type); =20 @@ -3903,31 +3906,27 @@ static void perf_event_context_sched_in( struct perf_cpu_context *cpuctx =3D this_cpu_ptr(&perf_cpu_context); struct perf_event_context *ctx; =20 - rcu_read_lock(); + guard(rcu)(); + ctx =3D rcu_dereference(task->perf_event_ctxp); if (!ctx) - goto rcu_unlock; - - if (cpuctx->task_ctx =3D=3D ctx) { - perf_ctx_lock(cpuctx, ctx); - perf_ctx_disable(ctx, false); - - perf_ctx_sched_task_cb(ctx, true); - - perf_ctx_enable(ctx, false); - perf_ctx_unlock(cpuctx, ctx); - goto rcu_unlock; - } + return; =20 - perf_ctx_lock(cpuctx, ctx); + guard(perf_ctx_lock)(cpuctx, ctx); /* * We must check ctx->nr_events while holding ctx->lock, such * that we serialize against perf_install_in_context(). */ if (!ctx->nr_events) - goto unlock; + return; + + guard(perf_ctx_disable)(ctx); + + if (cpuctx->task_ctx =3D=3D ctx) { + perf_ctx_sched_task_cb(ctx, true); + return; + } =20 - perf_ctx_disable(ctx, false); /* * We want to keep the following priority order: * cpu pinned (that don't need to move), task pinned, @@ -3947,13 +3946,6 @@ static void perf_event_context_sched_in( =20 if (!RB_EMPTY_ROOT(&ctx->pinned_groups.tree)) perf_ctx_enable(&cpuctx->ctx, false); - - perf_ctx_enable(ctx, false); - -unlock: - perf_ctx_unlock(cpuctx, ctx); -rcu_unlock: - rcu_read_unlock(); } =20 /* From nobody Wed Dec 31 13:04:15 2025 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id B62DCC4167B for ; Thu, 2 Nov 2023 15:33:03 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1347552AbjKBPdE (ORCPT ); Thu, 2 Nov 2023 11:33:04 -0400 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:33464 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S234194AbjKBPc7 (ORCPT ); Thu, 2 Nov 2023 11:32:59 -0400 Received: from desiato.infradead.org (desiato.infradead.org [IPv6:2001:8b0:10b:1:d65d:64ff:fe57:4e05]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id D6E9313E for ; Thu, 2 Nov 2023 08:32:52 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; d=infradead.org; s=desiato.20200630; h=Content-Type:MIME-Version:References: Subject:Cc:To:From:Date:Message-Id:Sender:Reply-To:Content-Transfer-Encoding: Content-ID:Content-Description:In-Reply-To; bh=PDycx4fpiS6PTok+N6ddTvPXW/vNvT8Lmj3Yl8Tx4xE=; b=H7HBpu3sR+BkCk0NREtlw1ja8/ A4hW1cSlDpmlE4lz1CP8KaZ53xnBIgZrCfN9oF42tzkgLbjd3eeU+NbY6VjK3H+4ZqUCvk5dFEkHR env3dsiYcwxY+yYoDW+gCRKe3f3ZstEqUjq0hPAc9lkfdnlprOeE8WvbIICxzAZHIgAFaayYPrsY5 2ARgzP/mQrlDPq3E5k5R3va8nhxjkFLs8j9OIWAnSyULaFxPdEh1yX2M8P0Q0QkBX1yQEbCz6HT3j n2xykC6quW6iGR3LHsDsdri9fYklgyI4yOTrMqnEDgB7UWumS4IL/fiJ+yHC0bSOfW24XHBNamoay s83lQYiA==; Received: from j130084.upc-j.chello.nl ([24.132.130.84] helo=noisy.programming.kicks-ass.net) by desiato.infradead.org with esmtpsa (Exim 4.96 #2 (Red Hat Linux)) id 1qyZgW-006gQP-26; Thu, 02 Nov 2023 15:32:41 +0000 Received: by noisy.programming.kicks-ass.net (Postfix, from userid 0) id 63E9A302EAB; Thu, 2 Nov 2023 16:32:39 +0100 (CET) Message-Id: <20231102152018.986157891@infradead.org> User-Agent: quilt/0.65 Date: Thu, 02 Nov 2023 16:09:30 +0100 From: Peter Zijlstra To: mingo@kernel.org Cc: linux-kernel@vger.kernel.org, peterz@infradead.org, acme@kernel.org, mark.rutland@arm.com, alexander.shishkin@linux.intel.com, jolsa@kernel.org, namhyung@kernel.org, irogers@google.com, adrian.hunter@intel.com Subject: [PATCH 11/13] perf: Simplify perf_adjust_freq_unthr_context() References: <20231102150919.719936610@infradead.org> MIME-Version: 1.0 Precedence: bulk List-ID: X-Mailing-List: linux-kernel@vger.kernel.org Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Signed-off-by: Peter Zijlstra (Intel) --- kernel/events/core.c | 51 +++++++++++++++++++++++-----------------------= ----- 1 file changed, 23 insertions(+), 28 deletions(-) --- a/kernel/events/core.c +++ b/kernel/events/core.c @@ -4090,7 +4090,7 @@ perf_adjust_freq_unthr_context(struct pe if (!(ctx->nr_freq || unthrottle)) return; =20 - raw_spin_lock(&ctx->lock); + guard(raw_spinlock)(&ctx->lock); =20 list_for_each_entry_rcu(event, &ctx->event_list, event_entry) { if (event->state !=3D PERF_EVENT_STATE_ACTIVE) @@ -4100,7 +4100,7 @@ perf_adjust_freq_unthr_context(struct pe if (!event_filter_match(event)) continue; =20 - perf_pmu_disable(event->pmu); + guard(perf_pmu_disable)(event->pmu); =20 hwc =3D &event->hw; =20 @@ -4110,34 +4110,29 @@ perf_adjust_freq_unthr_context(struct pe event->pmu->start(event, 0); } =20 - if (!event->attr.freq || !event->attr.sample_freq) - goto next; + if (event->attr.freq && event->attr.sample_freq) { + /* + * stop the event and update event->count + */ + event->pmu->stop(event, PERF_EF_UPDATE); + + now =3D local64_read(&event->count); + delta =3D now - hwc->freq_count_stamp; + hwc->freq_count_stamp =3D now; + + /* + * restart the event + * reload only if value has changed + * we have stopped the event so tell that + * to perf_adjust_period() to avoid stopping it + * twice. + */ + if (delta > 0) + perf_adjust_period(event, period, delta, false); =20 - /* - * stop the event and update event->count - */ - event->pmu->stop(event, PERF_EF_UPDATE); - - now =3D local64_read(&event->count); - delta =3D now - hwc->freq_count_stamp; - hwc->freq_count_stamp =3D now; - - /* - * restart the event - * reload only if value has changed - * we have stopped the event so tell that - * to perf_adjust_period() to avoid stopping it - * twice. - */ - if (delta > 0) - perf_adjust_period(event, period, delta, false); - - event->pmu->start(event, delta > 0 ? PERF_EF_RELOAD : 0); - next: - perf_pmu_enable(event->pmu); + event->pmu->start(event, delta > 0 ? PERF_EF_RELOAD : 0); + } } - - raw_spin_unlock(&ctx->lock); } =20 /* From nobody Wed Dec 31 13:04:15 2025 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id BDE58C4332F for ; Thu, 2 Nov 2023 15:33:21 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1347652AbjKBPdW (ORCPT ); Thu, 2 Nov 2023 11:33:22 -0400 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:53228 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1347590AbjKBPdH (ORCPT ); Thu, 2 Nov 2023 11:33:07 -0400 Received: from casper.infradead.org (casper.infradead.org [IPv6:2001:8b0:10b:1236::1]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id A431B188 for ; Thu, 2 Nov 2023 08:32:53 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; d=infradead.org; s=casper.20170209; h=Content-Type:MIME-Version:References: Subject:Cc:To:From:Date:Message-Id:Sender:Reply-To:Content-Transfer-Encoding: Content-ID:Content-Description:In-Reply-To; bh=0Nln3H/TQf9zjJAhRuDD5Vdsf6AHs8EETAgdoNVjkyE=; b=nsGBHSM3VplY00jpcPdnfQpcXu 7BEkJV8Ta5CZ9VtP1cTK/WwyhPanO1wXOsnQeSEqzyYpl/4xzzhKBKjJ6qHyR0UJT6L2Ml3+iUqI5 gk0RWwVad3PdTiW/XS9sQRzv67oxgLKdtTahHUAHCr0viDrlFgW/IJj+KsviSOmy6wzKDnEJme1IY ywUiHOVSNCDprk/PRAF4UgDv/3A2KutyC2/irWcU5IIMlZFHShESgUa8woK1WyudM81MsUgLSNa99 jeubDbscY0gEu8eACCiABP7yarKTWRJ80x9M60LW9BIejCjboPF0TMBqMeq3t5Wy4tYMjuELTaXAm RRee22qw==; Received: from j130084.upc-j.chello.nl ([24.132.130.84] helo=noisy.programming.kicks-ass.net) by casper.infradead.org with esmtpsa (Exim 4.94.2 #2 (Red Hat Linux)) id 1qyZgV-0005PT-VU; Thu, 02 Nov 2023 15:32:40 +0000 Received: by noisy.programming.kicks-ass.net (Postfix, from userid 0) id 68CF1302EB9; Thu, 2 Nov 2023 16:32:39 +0100 (CET) Message-Id: <20231102152019.099969167@infradead.org> User-Agent: quilt/0.65 Date: Thu, 02 Nov 2023 16:09:31 +0100 From: Peter Zijlstra To: mingo@kernel.org Cc: linux-kernel@vger.kernel.org, peterz@infradead.org, acme@kernel.org, mark.rutland@arm.com, alexander.shishkin@linux.intel.com, jolsa@kernel.org, namhyung@kernel.org, irogers@google.com, adrian.hunter@intel.com Subject: [PATCH 12/13] perf: Simplify perf_event_*_on_exec() References: <20231102150919.719936610@infradead.org> MIME-Version: 1.0 Precedence: bulk List-ID: X-Mailing-List: linux-kernel@vger.kernel.org Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Signed-off-by: Peter Zijlstra (Intel) --- kernel/events/core.c | 88 +++++++++++++++++++++++-----------------------= ----- 1 file changed, 40 insertions(+), 48 deletions(-) --- a/kernel/events/core.c +++ b/kernel/events/core.c @@ -4318,39 +4318,36 @@ static void perf_event_enable_on_exec(st enum event_type_t event_type =3D 0; struct perf_cpu_context *cpuctx; struct perf_event *event; - unsigned long flags; int enabled =3D 0; =20 - local_irq_save(flags); - if (WARN_ON_ONCE(current->perf_event_ctxp !=3D ctx)) - goto out; - - if (!ctx->nr_events) - goto out; - - cpuctx =3D this_cpu_ptr(&perf_cpu_context); - perf_ctx_lock(cpuctx, ctx); - ctx_sched_out(ctx, EVENT_TIME); - - list_for_each_entry(event, &ctx->event_list, event_entry) { - enabled |=3D event_enable_on_exec(event, ctx); - event_type |=3D get_event_type(event); + scoped_guard (irqsave) { + if (WARN_ON_ONCE(current->perf_event_ctxp !=3D ctx)) + return; + + if (!ctx->nr_events) + return; + + cpuctx =3D this_cpu_ptr(&perf_cpu_context); + guard(perf_ctx_lock)(cpuctx, ctx); + + ctx_sched_out(ctx, EVENT_TIME); + + list_for_each_entry(event, &ctx->event_list, event_entry) { + enabled |=3D event_enable_on_exec(event, ctx); + event_type |=3D get_event_type(event); + } + + /* + * Unclone and reschedule this context if we enabled any event. + */ + if (enabled) { + clone_ctx =3D unclone_ctx(ctx); + ctx_resched(cpuctx, ctx, event_type); + } else { + ctx_sched_in(ctx, EVENT_TIME); + } } =20 - /* - * Unclone and reschedule this context if we enabled any event. - */ - if (enabled) { - clone_ctx =3D unclone_ctx(ctx); - ctx_resched(cpuctx, ctx, event_type); - } else { - ctx_sched_in(ctx, EVENT_TIME); - } - perf_ctx_unlock(cpuctx, ctx); - -out: - local_irq_restore(flags); - if (clone_ctx) put_ctx(clone_ctx); } @@ -4367,34 +4364,29 @@ static void perf_event_remove_on_exec(st { struct perf_event_context *clone_ctx =3D NULL; struct perf_event *event, *next; - unsigned long flags; bool modified =3D false; =20 - mutex_lock(&ctx->mutex); + scoped_guard (mutex, &ctx->mutex) { + if (WARN_ON_ONCE(ctx->task !=3D current)) + return; =20 - if (WARN_ON_ONCE(ctx->task !=3D current)) - goto unlock; + list_for_each_entry_safe(event, next, &ctx->event_list, event_entry) { + if (!event->attr.remove_on_exec) + continue; =20 - list_for_each_entry_safe(event, next, &ctx->event_list, event_entry) { - if (!event->attr.remove_on_exec) - continue; + if (!is_kernel_event(event)) + perf_remove_from_owner(event); =20 - if (!is_kernel_event(event)) - perf_remove_from_owner(event); + modified =3D true; =20 - modified =3D true; + perf_event_exit_event(event, ctx); + } =20 - perf_event_exit_event(event, ctx); + guard(raw_spinlock_irqsave)(&ctx->lock); + if (modified) + clone_ctx =3D unclone_ctx(ctx); } =20 - raw_spin_lock_irqsave(&ctx->lock, flags); - if (modified) - clone_ctx =3D unclone_ctx(ctx); - raw_spin_unlock_irqrestore(&ctx->lock, flags); - -unlock: - mutex_unlock(&ctx->mutex); - if (clone_ctx) put_ctx(clone_ctx); } From nobody Wed Dec 31 13:04:15 2025 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id BDA6DC0018A for ; Thu, 2 Nov 2023 15:33:34 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1376841AbjKBPdf (ORCPT ); Thu, 2 Nov 2023 11:33:35 -0400 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:53228 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1347645AbjKBPdL (ORCPT ); Thu, 2 Nov 2023 11:33:11 -0400 Received: from desiato.infradead.org (desiato.infradead.org [IPv6:2001:8b0:10b:1:d65d:64ff:fe57:4e05]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id D67C118B for ; Thu, 2 Nov 2023 08:32:53 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; d=infradead.org; s=desiato.20200630; h=Content-Type:MIME-Version:References: Subject:Cc:To:From:Date:Message-Id:Sender:Reply-To:Content-Transfer-Encoding: Content-ID:Content-Description:In-Reply-To; bh=Flz3lC+6VWh6RSfoR6MvKdhu9gsbmcKYNkChdKqx6SM=; b=BVE+b0jo+rlvT0zgww+y/0ZSgf uqgnmATxegsfIBHQuwLr6nnQVYZ9MPKO+0t5Bct+irYnS8WXY7qExCLBuabnWgL+LaGCQtDERBvrN zhrdJjf+Ecya4i6fjTQP+6i5mgzgSgAt/y+WgMig5SqYPQlBRnQGCIVJkcO5QrSCP3Vgx2wVMXtdJ +aH8yDUdaHZ8vWtymQgf20RM1TKdrStSjsG68uCZAMgBH/2go6qq8x6YDWwKbGfPMAIMRVbe8KsIJ ayi4+yVviScp8ZzPTkcAqgzhGLZZnlmf/VicigTl2ji5mXJMfIcyWp3MIAvrE1L0ZGttHHs5L6Tcr ZmzBrzdQ==; Received: from j130084.upc-j.chello.nl ([24.132.130.84] helo=noisy.programming.kicks-ass.net) by desiato.infradead.org with esmtpsa (Exim 4.96 #2 (Red Hat Linux)) id 1qyZgW-006gQQ-2D; Thu, 02 Nov 2023 15:32:41 +0000 Received: by noisy.programming.kicks-ass.net (Postfix, from userid 0) id 6D39430311A; Thu, 2 Nov 2023 16:32:39 +0100 (CET) Message-Id: <20231102152019.211374348@infradead.org> User-Agent: quilt/0.65 Date: Thu, 02 Nov 2023 16:09:32 +0100 From: Peter Zijlstra To: mingo@kernel.org Cc: linux-kernel@vger.kernel.org, peterz@infradead.org, acme@kernel.org, mark.rutland@arm.com, alexander.shishkin@linux.intel.com, jolsa@kernel.org, namhyung@kernel.org, irogers@google.com, adrian.hunter@intel.com Subject: [PATCH 13/13] perf: Simplify *perf_event_read*() References: <20231102150919.719936610@infradead.org> MIME-Version: 1.0 Precedence: bulk List-ID: X-Mailing-List: linux-kernel@vger.kernel.org Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Signed-off-by: Peter Zijlstra (Intel) --- kernel/events/core.c | 54 ++++++++++++++++------------------------------= ----- 1 file changed, 17 insertions(+), 37 deletions(-) --- a/kernel/events/core.c +++ b/kernel/events/core.c @@ -4434,7 +4434,8 @@ static void __perf_event_read(void *info if (ctx->task && cpuctx->task_ctx !=3D ctx) return; =20 - raw_spin_lock(&ctx->lock); + guard(raw_spinlock)(&ctx->lock); + if (ctx->is_active & EVENT_TIME) { update_context_time(ctx); update_cgrp_time_from_event(event); @@ -4445,12 +4446,12 @@ static void __perf_event_read(void *info perf_event_update_sibling_time(event); =20 if (event->state !=3D PERF_EVENT_STATE_ACTIVE) - goto unlock; + return; =20 if (!data->group) { pmu->read(event); data->ret =3D 0; - goto unlock; + return; } =20 pmu->start_txn(pmu, PERF_PMU_TXN_READ); @@ -4468,9 +4469,6 @@ static void __perf_event_read(void *info } =20 data->ret =3D pmu->commit_txn(pmu); - -unlock: - raw_spin_unlock(&ctx->lock); } =20 static inline u64 perf_event_count(struct perf_event *event) @@ -4501,32 +4499,25 @@ static void calc_timer_values(struct per int perf_event_read_local(struct perf_event *event, u64 *value, u64 *enabled, u64 *running) { - unsigned long flags; int event_oncpu; int event_cpu; - int ret =3D 0; - /* * Disabling interrupts avoids all counter scheduling (context * switches, timer based rotation and IPIs). */ - local_irq_save(flags); + guard(irqsave)(); =20 /* * It must not be an event with inherit set, we cannot read * all child counters from atomic context. */ - if (event->attr.inherit) { - ret =3D -EOPNOTSUPP; - goto out; - } + if (event->attr.inherit) + return -EOPNOTSUPP; =20 /* If this is a per-task event, it must be for current */ if ((event->attach_state & PERF_ATTACH_TASK) && - event->hw.target !=3D current) { - ret =3D -EINVAL; - goto out; - } + event->hw.target !=3D current) + return -EINVAL; =20 /* * Get the event CPU numbers, and adjust them to local if the event is @@ -4537,16 +4528,12 @@ int perf_event_read_local(struct perf_ev =20 /* If this is a per-CPU event, it must be for this CPU */ if (!(event->attach_state & PERF_ATTACH_TASK) && - event_cpu !=3D smp_processor_id()) { - ret =3D -EINVAL; - goto out; - } + event_cpu !=3D smp_processor_id()) + return -EINVAL; =20 /* If this is a pinned event it must be running on this CPU */ - if (event->attr.pinned && event_oncpu !=3D smp_processor_id()) { - ret =3D -EBUSY; - goto out; - } + if (event->attr.pinned && event_oncpu !=3D smp_processor_id()) + return -EBUSY; =20 /* * If the event is currently on this CPU, its either a per-task event, @@ -4566,10 +4553,8 @@ int perf_event_read_local(struct perf_ev if (running) *running =3D __running; } -out: - local_irq_restore(flags); =20 - return ret; + return 0; } =20 static int perf_event_read(struct perf_event *event, bool group) @@ -4603,7 +4588,7 @@ static int perf_event_read(struct perf_e .ret =3D 0, }; =20 - preempt_disable(); + guard(preempt)(); event_cpu =3D __perf_event_read_cpu(event, event_cpu); =20 /* @@ -4617,19 +4602,15 @@ static int perf_event_read(struct perf_e * after this. */ (void)smp_call_function_single(event_cpu, __perf_event_read, &data, 1); - preempt_enable(); ret =3D data.ret; =20 } else if (state =3D=3D PERF_EVENT_STATE_INACTIVE) { struct perf_event_context *ctx =3D event->ctx; - unsigned long flags; =20 - raw_spin_lock_irqsave(&ctx->lock, flags); + guard(raw_spinlock_irqsave)(&ctx->lock); state =3D event->state; - if (state !=3D PERF_EVENT_STATE_INACTIVE) { - raw_spin_unlock_irqrestore(&ctx->lock, flags); + if (state !=3D PERF_EVENT_STATE_INACTIVE) goto again; - } =20 /* * May read while context is not active (e.g., thread is @@ -4643,7 +4624,6 @@ static int perf_event_read(struct perf_e perf_event_update_time(event); if (group) perf_event_update_sibling_time(event); - raw_spin_unlock_irqrestore(&ctx->lock, flags); } =20 return ret;