From nobody Mon Apr 6 13:28:26 2026 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id EBD16C07E9D for ; Tue, 27 Sep 2022 16:02:05 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S232779AbiI0QCE (ORCPT ); Tue, 27 Sep 2022 12:02:04 -0400 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:41076 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S232221AbiI0QBj (ORCPT ); Tue, 27 Sep 2022 12:01:39 -0400 Received: from dfw.source.kernel.org (dfw.source.kernel.org [139.178.84.217]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id 1C3E4100A8B for ; Tue, 27 Sep 2022 09:01:36 -0700 (PDT) Received: from smtp.kernel.org (relay.kernel.org [52.25.139.140]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by dfw.source.kernel.org (Postfix) with ESMTPS id 9CF7761A55 for ; Tue, 27 Sep 2022 16:01:35 +0000 (UTC) Received: by smtp.kernel.org (Postfix) with ESMTPSA id D7659C433B5; Tue, 27 Sep 2022 16:01:34 +0000 (UTC) Received: from rostedt by gandalf.local.home with local (Exim 4.96) (envelope-from ) id 1odD2j-00G2rl-1N; Tue, 27 Sep 2022 12:02:45 -0400 Message-ID: <20220927160244.996770559@goodmis.org> User-Agent: quilt/0.66 Date: Tue, 27 Sep 2022 12:02:24 -0400 From: Steven Rostedt To: linux-kernel@vger.kernel.org Cc: Daniel Bristot de Oliveira , Ingo Molnar , Andrew Morton , Masami Hiramatsu , Tom Zanussi Subject: [for-next][PATCH 08/20] tracing/filter: Call filter predicate functions directly via a switch statement References: <20220927160216.349640304@goodmis.org> MIME-Version: 1.0 Precedence: bulk List-ID: X-Mailing-List: linux-kernel@vger.kernel.org Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" From: "Steven Rostedt (Google)" Due to retpolines, indirect calls are much more expensive than direct calls. The filters have a select set of functions it uses for the predicates. Instead of using function pointers to call them, create a filter_pred_fn_call() function that uses a switch statement to call the predicate functions directly. This gives almost a 10% speedup to the filter logic. Using the histogram benchmark: Before: # event histogram # # trigger info: hist:keys=3Ddelta:vals=3Dhitcount:sort=3Ddelta:size=3D2048= if delta > 0 [active] # { delta: 113 } hitcount: 272 { delta: 114 } hitcount: 840 { delta: 118 } hitcount: 344 { delta: 119 } hitcount: 25428 { delta: 120 } hitcount: 350590 { delta: 121 } hitcount: 1892484 { delta: 122 } hitcount: 6205004 { delta: 123 } hitcount: 11583521 { delta: 124 } hitcount: 37590979 { delta: 125 } hitcount: 108308504 { delta: 126 } hitcount: 131672461 { delta: 127 } hitcount: 88700598 { delta: 128 } hitcount: 65939870 { delta: 129 } hitcount: 45055004 { delta: 130 } hitcount: 33174464 { delta: 131 } hitcount: 31813493 { delta: 132 } hitcount: 29011676 { delta: 133 } hitcount: 22798782 { delta: 134 } hitcount: 22072486 { delta: 135 } hitcount: 17034113 { delta: 136 } hitcount: 8982490 { delta: 137 } hitcount: 2865908 { delta: 138 } hitcount: 980382 { delta: 139 } hitcount: 1651944 { delta: 140 } hitcount: 4112073 { delta: 141 } hitcount: 3963269 { delta: 142 } hitcount: 1712508 { delta: 143 } hitcount: 575941 After: # event histogram # # trigger info: hist:keys=3Ddelta:vals=3Dhitcount:sort=3Ddelta:size=3D2048= if delta > 0 [active] # { delta: 103 } hitcount: 60 { delta: 104 } hitcount: 16966 { delta: 105 } hitcount: 396625 { delta: 106 } hitcount: 3223400 { delta: 107 } hitcount: 12053754 { delta: 108 } hitcount: 20241711 { delta: 109 } hitcount: 14850200 { delta: 110 } hitcount: 4946599 { delta: 111 } hitcount: 3479315 { delta: 112 } hitcount: 18698299 { delta: 113 } hitcount: 62388733 { delta: 114 } hitcount: 95803834 { delta: 115 } hitcount: 58278130 { delta: 116 } hitcount: 15364800 { delta: 117 } hitcount: 5586866 { delta: 118 } hitcount: 2346880 { delta: 119 } hitcount: 1131091 { delta: 120 } hitcount: 620896 { delta: 121 } hitcount: 236652 { delta: 122 } hitcount: 105957 { delta: 123 } hitcount: 119107 { delta: 124 } hitcount: 54494 { delta: 125 } hitcount: 63856 { delta: 126 } hitcount: 64454 { delta: 127 } hitcount: 34818 { delta: 128 } hitcount: 41446 { delta: 129 } hitcount: 51242 { delta: 130 } hitcount: 28361 { delta: 131 } hitcount: 23926 The peak before was 126ns per event, after the peak is 114ns, and the fastest time went from 113ns to 103ns. Link: https://lkml.kernel.org/r/20220906225529.781407172@goodmis.org Cc: Ingo Molnar Cc: Andrew Morton Cc: Masami Hiramatsu Cc: Tom Zanussi Signed-off-by: Steven Rostedt (Google) --- kernel/trace/trace_events_filter.c | 230 ++++++++++++++++++++--------- 1 file changed, 157 insertions(+), 73 deletions(-) diff --git a/kernel/trace/trace_events_filter.c b/kernel/trace/trace_events= _filter.c index c49c689ce4ad..96acc2b71ac7 100644 --- a/kernel/trace/trace_events_filter.c +++ b/kernel/trace/trace_events_filter.c @@ -43,10 +43,33 @@ enum filter_op_ids { OPS }; =20 static const char * ops[] =3D { OPS }; =20 -typedef int (*filter_pred_fn_t) (struct filter_pred *pred, void *event); +enum filter_pred_fn { + FILTER_PRED_FN_NOP, + FILTER_PRED_FN_64, + FILTER_PRED_FN_S64, + FILTER_PRED_FN_U64, + FILTER_PRED_FN_32, + FILTER_PRED_FN_S32, + FILTER_PRED_FN_U32, + FILTER_PRED_FN_16, + FILTER_PRED_FN_S16, + FILTER_PRED_FN_U16, + FILTER_PRED_FN_8, + FILTER_PRED_FN_S8, + FILTER_PRED_FN_U8, + FILTER_PRED_FN_COMM, + FILTER_PRED_FN_STRING, + FILTER_PRED_FN_STRLOC, + FILTER_PRED_FN_STRRELLOC, + FILTER_PRED_FN_PCHAR_USER, + FILTER_PRED_FN_PCHAR, + FILTER_PRED_FN_CPU, + FILTER_PRED_FN_, + FILTER_PRED_TEST_VISITED, +}; =20 struct filter_pred { - filter_pred_fn_t fn; + enum filter_pred_fn fn_num; u64 val; struct regex regex; unsigned short *ops; @@ -603,44 +626,48 @@ predicate_parse(const char *str, int nr_parens, int n= r_preds, return ERR_PTR(ret); } =20 +enum pred_cmp_types { + PRED_CMP_TYPE_NOP, + PRED_CMP_TYPE_LT, + PRED_CMP_TYPE_LE, + PRED_CMP_TYPE_GT, + PRED_CMP_TYPE_GE, + PRED_CMP_TYPE_BAND, +}; + #define DEFINE_COMPARISON_PRED(type) \ -static int filter_pred_LT_##type(struct filter_pred *pred, void *event) \ -{ \ - type *addr =3D (type *)(event + pred->offset); \ - type val =3D (type)pred->val; \ - return *addr < val; \ -} \ -static int filter_pred_LE_##type(struct filter_pred *pred, void *event) \ +static int filter_pred_##type(struct filter_pred *pred, void *event) \ { \ - type *addr =3D (type *)(event + pred->offset); \ - type val =3D (type)pred->val; \ - return *addr <=3D val; \ -} \ -static int filter_pred_GT_##type(struct filter_pred *pred, void *event) \ -{ \ - type *addr =3D (type *)(event + pred->offset); \ - type val =3D (type)pred->val; \ - return *addr > val; \ -} \ -static int filter_pred_GE_##type(struct filter_pred *pred, void *event) \ -{ \ - type *addr =3D (type *)(event + pred->offset); \ - type val =3D (type)pred->val; \ - return *addr >=3D val; \ -} \ -static int filter_pred_BAND_##type(struct filter_pred *pred, void *event) \ -{ \ - type *addr =3D (type *)(event + pred->offset); \ - type val =3D (type)pred->val; \ - return !!(*addr & val); \ -} \ -static const filter_pred_fn_t pred_funcs_##type[] =3D { \ - filter_pred_LE_##type, \ - filter_pred_LT_##type, \ - filter_pred_GE_##type, \ - filter_pred_GT_##type, \ - filter_pred_BAND_##type, \ -}; + switch (pred->op) { \ + case OP_LT: { \ + type *addr =3D (type *)(event + pred->offset); \ + type val =3D (type)pred->val; \ + return *addr < val; \ + } \ + case OP_LE: { \ + type *addr =3D (type *)(event + pred->offset); \ + type val =3D (type)pred->val; \ + return *addr <=3D val; \ + } \ + case OP_GT: { \ + type *addr =3D (type *)(event + pred->offset); \ + type val =3D (type)pred->val; \ + return *addr > val; \ + } \ + case OP_GE: { \ + type *addr =3D (type *)(event + pred->offset); \ + type val =3D (type)pred->val; \ + return *addr >=3D val; \ + } \ + case OP_BAND: { \ + type *addr =3D (type *)(event + pred->offset); \ + type val =3D (type)pred->val; \ + return !!(*addr & val); \ + } \ + default: \ + return 0; \ + } \ +} =20 #define DEFINE_EQUALITY_PRED(size) \ static int filter_pred_##size(struct filter_pred *pred, void *event) \ @@ -849,11 +876,6 @@ static int filter_pred_comm(struct filter_pred *pred, = void *event) return cmp ^ pred->not; } =20 -static int filter_pred_none(struct filter_pred *pred, void *event) -{ - return 0; -} - /* * regex_match_foo - Basic regex callbacks * @@ -999,6 +1021,19 @@ static void filter_build_regex(struct filter_pred *pr= ed) } } =20 + +#ifdef CONFIG_FTRACE_STARTUP_TEST +static int test_pred_visited_fn(struct filter_pred *pred, void *event); +#else +static int test_pred_visited_fn(struct filter_pred *pred, void *event) +{ + return 0; +} +#endif + + +static int filter_pred_fn_call(struct filter_pred *pred, void *event); + /* return 1 if event matches, 0 otherwise (discard) */ int filter_match_preds(struct event_filter *filter, void *rec) { @@ -1016,7 +1051,7 @@ int filter_match_preds(struct event_filter *filter, v= oid *rec) =20 for (i =3D 0; prog[i].pred; i++) { struct filter_pred *pred =3D prog[i].pred; - int match =3D pred->fn(pred, rec); + int match =3D filter_pred_fn_call(pred, rec); if (match =3D=3D prog[i].when_to_branch) i =3D prog[i].target; } @@ -1202,10 +1237,10 @@ int filter_assign_type(const char *type) return FILTER_OTHER; } =20 -static filter_pred_fn_t select_comparison_fn(enum filter_op_ids op, - int field_size, int field_is_signed) +static enum filter_pred_fn select_comparison_fn(enum filter_op_ids op, + int field_size, int field_is_signed) { - filter_pred_fn_t fn =3D NULL; + enum filter_pred_fn fn =3D FILTER_PRED_FN_NOP; int pred_func_index =3D -1; =20 switch (op) { @@ -1214,50 +1249,99 @@ static filter_pred_fn_t select_comparison_fn(enum f= ilter_op_ids op, break; default: if (WARN_ON_ONCE(op < PRED_FUNC_START)) - return NULL; + return fn; pred_func_index =3D op - PRED_FUNC_START; if (WARN_ON_ONCE(pred_func_index > PRED_FUNC_MAX)) - return NULL; + return fn; } =20 switch (field_size) { case 8: if (pred_func_index < 0) - fn =3D filter_pred_64; + fn =3D FILTER_PRED_FN_64; else if (field_is_signed) - fn =3D pred_funcs_s64[pred_func_index]; + fn =3D FILTER_PRED_FN_S64; else - fn =3D pred_funcs_u64[pred_func_index]; + fn =3D FILTER_PRED_FN_U64; break; case 4: if (pred_func_index < 0) - fn =3D filter_pred_32; + fn =3D FILTER_PRED_FN_32; else if (field_is_signed) - fn =3D pred_funcs_s32[pred_func_index]; + fn =3D FILTER_PRED_FN_S32; else - fn =3D pred_funcs_u32[pred_func_index]; + fn =3D FILTER_PRED_FN_U32; break; case 2: if (pred_func_index < 0) - fn =3D filter_pred_16; + fn =3D FILTER_PRED_FN_16; else if (field_is_signed) - fn =3D pred_funcs_s16[pred_func_index]; + fn =3D FILTER_PRED_FN_S16; else - fn =3D pred_funcs_u16[pred_func_index]; + fn =3D FILTER_PRED_FN_U16; break; case 1: if (pred_func_index < 0) - fn =3D filter_pred_8; + fn =3D FILTER_PRED_FN_8; else if (field_is_signed) - fn =3D pred_funcs_s8[pred_func_index]; + fn =3D FILTER_PRED_FN_S8; else - fn =3D pred_funcs_u8[pred_func_index]; + fn =3D FILTER_PRED_FN_U8; break; } =20 return fn; } =20 + +static int filter_pred_fn_call(struct filter_pred *pred, void *event) +{ + switch (pred->fn_num) { + case FILTER_PRED_FN_64: + return filter_pred_64(pred, event); + case FILTER_PRED_FN_S64: + return filter_pred_s64(pred, event); + case FILTER_PRED_FN_U64: + return filter_pred_u64(pred, event); + case FILTER_PRED_FN_32: + return filter_pred_32(pred, event); + case FILTER_PRED_FN_S32: + return filter_pred_s32(pred, event); + case FILTER_PRED_FN_U32: + return filter_pred_u32(pred, event); + case FILTER_PRED_FN_16: + return filter_pred_16(pred, event); + case FILTER_PRED_FN_S16: + return filter_pred_s16(pred, event); + case FILTER_PRED_FN_U16: + return filter_pred_u16(pred, event); + case FILTER_PRED_FN_8: + return filter_pred_8(pred, event); + case FILTER_PRED_FN_S8: + return filter_pred_s8(pred, event); + case FILTER_PRED_FN_U8: + return filter_pred_u8(pred, event); + case FILTER_PRED_FN_COMM: + return filter_pred_comm(pred, event); + case FILTER_PRED_FN_STRING: + return filter_pred_string(pred, event); + case FILTER_PRED_FN_STRLOC: + return filter_pred_strloc(pred, event); + case FILTER_PRED_FN_STRRELLOC: + return filter_pred_strrelloc(pred, event); + case FILTER_PRED_FN_PCHAR_USER: + return filter_pred_pchar_user(pred, event); + case FILTER_PRED_FN_PCHAR: + return filter_pred_pchar(pred, event); + case FILTER_PRED_FN_CPU: + return filter_pred_cpu(pred, event); + case FILTER_PRED_TEST_VISITED: + return test_pred_visited_fn(pred, event); + default: + return 0; + } +} + /* Called when a predicate is encountered by predicate_parse() */ static int parse_pred(const char *str, void *data, int pos, struct filter_parse_error *pe, @@ -1351,7 +1435,7 @@ static int parse_pred(const char *str, void *data, parse_error(pe, FILT_ERR_IP_FIELD_ONLY, pos + i); goto err_free; } - pred->fn =3D filter_pred_none; + pred->fn_num =3D FILTER_PRED_FN_NOP; =20 /* * Quotes are not required, but if they exist then we need @@ -1429,16 +1513,16 @@ static int parse_pred(const char *str, void *data, filter_build_regex(pred); =20 if (field->filter_type =3D=3D FILTER_COMM) { - pred->fn =3D filter_pred_comm; + pred->fn_num =3D FILTER_PRED_FN_COMM; =20 } else if (field->filter_type =3D=3D FILTER_STATIC_STRING) { - pred->fn =3D filter_pred_string; + pred->fn_num =3D FILTER_PRED_FN_STRING; pred->regex.field_len =3D field->size; =20 } else if (field->filter_type =3D=3D FILTER_DYN_STRING) { - pred->fn =3D filter_pred_strloc; + pred->fn_num =3D FILTER_PRED_FN_STRLOC; } else if (field->filter_type =3D=3D FILTER_RDYN_STRING) - pred->fn =3D filter_pred_strrelloc; + pred->fn_num =3D FILTER_PRED_FN_STRRELLOC; else { =20 if (!ustring_per_cpu) { @@ -1449,9 +1533,9 @@ static int parse_pred(const char *str, void *data, } =20 if (ustring) - pred->fn =3D filter_pred_pchar_user; + pred->fn_num =3D FILTER_PRED_FN_PCHAR_USER; else - pred->fn =3D filter_pred_pchar; + pred->fn_num =3D FILTER_PRED_FN_PCHAR; } /* go past the last quote */ i++; @@ -1499,10 +1583,10 @@ static int parse_pred(const char *str, void *data, pred->val =3D val; =20 if (field->filter_type =3D=3D FILTER_CPU) - pred->fn =3D filter_pred_cpu; + pred->fn_num =3D FILTER_PRED_FN_CPU; else { - pred->fn =3D select_comparison_fn(pred->op, field->size, - field->is_signed); + pred->fn_num =3D select_comparison_fn(pred->op, field->size, + field->is_signed); if (pred->op =3D=3D OP_NE) pred->not =3D 1; } @@ -2309,7 +2393,7 @@ static void update_pred_fn(struct event_filter *filte= r, char *fields) struct filter_pred *pred =3D prog[i].pred; struct ftrace_event_field *field =3D pred->field; =20 - WARN_ON_ONCE(!pred->fn); + WARN_ON_ONCE(pred->fn_num =3D=3D FILTER_PRED_FN_NOP); =20 if (!field) { WARN_ONCE(1, "all leafs should have field defined %d", i); @@ -2319,7 +2403,7 @@ static void update_pred_fn(struct event_filter *filte= r, char *fields) if (!strchr(fields, *field->name)) continue; =20 - pred->fn =3D test_pred_visited_fn; + pred->fn_num =3D FILTER_PRED_TEST_VISITED; } } =20 --=20 2.35.1