1. Patchew
  2. linux
  3. View series

[RFC PATCH v1] perf test: Fix perf stat --bpf-counters on hybrid machines

Ian Rogers posted 1 patch 1 week, 5 days ago
There is a newer version of this series
tools/perf/tests/shell/stat_bpf_counters.sh | 35 +++++++++++----------
1 file changed, 18 insertions(+), 17 deletions(-)
[RFC PATCH v1] perf test: Fix perf stat --bpf-counters on hybrid machines
Posted by Ian Rogers 1 week, 5 days ago 
Events on hardware PMUs may open on >1 hardware PMU on hybrid
machines. Unfortunately multiplexing introduces a scaling issue as
reported in:
https://lore.kernel.org/lkml/20250716223924.825772-1-irogers@google.com/

Switch to using task-clock which as a software event runs on all CPUs
and has a single PMU.

Signed-off-by: Ian Rogers <irogers@google.com>
---
 tools/perf/tests/shell/stat_bpf_counters.sh | 35 +++++++++++----------
 1 file changed, 18 insertions(+), 17 deletions(-)

diff --git a/tools/perf/tests/shell/stat_bpf_counters.sh b/tools/perf/tests/shell/stat_bpf_counters.sh
index f43e28a136d3..58f7dc9692ac 100755
--- a/tools/perf/tests/shell/stat_bpf_counters.sh
+++ b/tools/perf/tests/shell/stat_bpf_counters.sh
@@ -5,6 +5,7 @@
 set -e
 
 workload="perf test -w sqrtloop"
+event=task-clock
 
 # check whether $2 is within +/- 20% of $1
 compare_number()
@@ -25,15 +26,15 @@ compare_number()
 
 check_counts()
 {
-	base_instructions=$1
-	bpf_instructions=$2
+	base_count=$(echo "$1"|sed -e 's/[^0-9]*\([0-9]*\).*/\1/')
+	bpf_count=$(echo "$2"|sed -e 's/[^0-9]*\([0-9]*\).*/\1/')
 
-	if [ "$base_instructions" = "<not" ]; then
-		echo "Skipping: instructions event not counted"
+	if [ "$base_count" = "<not" ]; then
+		echo "Skipping: $event event not counted"
 		exit 2
 	fi
-	if [ "$bpf_instructions" = "<not" ]; then
-		echo "Failed: instructions not counted with --bpf-counters"
+	if [ "$bpf_count" = "<not" ]; then
+		echo "Failed: $event not counted with --bpf-counters"
 		exit 1
 	fi
 }
@@ -41,29 +42,29 @@ check_counts()
 test_bpf_counters()
 {
 	printf "Testing --bpf-counters "
-	base_instructions=$(perf stat --no-big-num -e instructions -- $workload 2>&1 | awk '/instructions/ {print $1}')
-	bpf_instructions=$(perf stat --no-big-num --bpf-counters -e instructions -- $workload  2>&1 | awk '/instructions/ {print $1}')
-	check_counts $base_instructions $bpf_instructions
-	compare_number $base_instructions $bpf_instructions
+	base_count=$(perf stat --no-big-num -e "$event" -- $workload 2>&1 | awk "/$event/ {print \$1}")
+	bpf_count=$(perf stat --no-big-num --bpf-counters -e "$event" -- $workload  2>&1 | awk "/$event/ {print \$1}")
+	check_counts "$base_count" "$bpf_count"
+	compare_number "$base_count" "$bpf_count"
 	echo "[Success]"
 }
 
 test_bpf_modifier()
 {
 	printf "Testing bpf event modifier "
-	stat_output=$(perf stat --no-big-num -e instructions/name=base_instructions/,instructions/name=bpf_instructions/b -- $workload 2>&1)
-	base_instructions=$(echo "$stat_output"| awk '/base_instructions/ {print $1}')
-	bpf_instructions=$(echo "$stat_output"| awk '/bpf_instructions/ {print $1}')
-	check_counts $base_instructions $bpf_instructions
-	compare_number $base_instructions $bpf_instructions
+	stat_output=$(perf stat --no-big-num -e "$event/name=base_$event/,$event/name=bpf_$event/b" -- $workload 2>&1)
+	base_count=$(echo "$stat_output"| awk "/base_$event/ {print \$1}")
+	bpf_count=$(echo "$stat_output"| awk "/bpf_$event/ {print \$1}")
+	check_counts "$base_count" "$bpf_count"
+	compare_number "$base_count" "$bpf_count"
 	echo "[Success]"
 }
 
 # skip if --bpf-counters is not supported
-if ! perf stat -e instructions --bpf-counters true > /dev/null 2>&1; then
+if ! perf stat -e $event --bpf-counters true > /dev/null 2>&1; then
 	if [ "$1" = "-v" ]; then
 		echo "Skipping: --bpf-counters not supported"
-		perf --no-pager stat -e instructions --bpf-counters true || true
+		perf --no-pager stat -e $event --bpf-counters true || true
 	fi
 	exit 2
 fi
-- 
2.53.0.1018.g2bb0e51243-goog
Re: [RFC PATCH v1] perf test: Fix perf stat --bpf-counters on hybrid machines
Posted by Ian Rogers 1 week, 5 days ago 
On Mon, Mar 23, 2026 at 1:24 PM Ian Rogers <irogers@google.com> wrote:
>
> Events on hardware PMUs may open on >1 hardware PMU on hybrid
> machines. Unfortunately multiplexing introduces a scaling issue as
> reported in:
> https://lore.kernel.org/lkml/20250716223924.825772-1-irogers@google.com/
>
> Switch to using task-clock which as a software event runs on all CPUs
> and has a single PMU.
>
> Signed-off-by: Ian Rogers <irogers@google.com>

With hindsight, maybe the better thing to do is Namhyung's patch plus
the "--no-scale" option to `perf stat` to sidestep the scaling issue.
It'd be great to make this test not "(exclusive)", it shouldn't be
necessary. You can test for flakiness with the '-r' option to `perf
test` which takes a repeat count.

Thanks,
Ian

> ---
>  tools/perf/tests/shell/stat_bpf_counters.sh | 35 +++++++++++----------
>  1 file changed, 18 insertions(+), 17 deletions(-)
>
> diff --git a/tools/perf/tests/shell/stat_bpf_counters.sh b/tools/perf/tests/shell/stat_bpf_counters.sh
> index f43e28a136d3..58f7dc9692ac 100755
> --- a/tools/perf/tests/shell/stat_bpf_counters.sh
> +++ b/tools/perf/tests/shell/stat_bpf_counters.sh
> @@ -5,6 +5,7 @@
>  set -e
>
>  workload="perf test -w sqrtloop"
> +event=task-clock
>
>  # check whether $2 is within +/- 20% of $1
>  compare_number()
> @@ -25,15 +26,15 @@ compare_number()
>
>  check_counts()
>  {
> -       base_instructions=$1
> -       bpf_instructions=$2
> +       base_count=$(echo "$1"|sed -e 's/[^0-9]*\([0-9]*\).*/\1/')
> +       bpf_count=$(echo "$2"|sed -e 's/[^0-9]*\([0-9]*\).*/\1/')
>
> -       if [ "$base_instructions" = "<not" ]; then
> -               echo "Skipping: instructions event not counted"
> +       if [ "$base_count" = "<not" ]; then
> +               echo "Skipping: $event event not counted"
>                 exit 2
>         fi
> -       if [ "$bpf_instructions" = "<not" ]; then
> -               echo "Failed: instructions not counted with --bpf-counters"
> +       if [ "$bpf_count" = "<not" ]; then
> +               echo "Failed: $event not counted with --bpf-counters"
>                 exit 1
>         fi
>  }
> @@ -41,29 +42,29 @@ check_counts()
>  test_bpf_counters()
>  {
>         printf "Testing --bpf-counters "
> -       base_instructions=$(perf stat --no-big-num -e instructions -- $workload 2>&1 | awk '/instructions/ {print $1}')
> -       bpf_instructions=$(perf stat --no-big-num --bpf-counters -e instructions -- $workload  2>&1 | awk '/instructions/ {print $1}')
> -       check_counts $base_instructions $bpf_instructions
> -       compare_number $base_instructions $bpf_instructions
> +       base_count=$(perf stat --no-big-num -e "$event" -- $workload 2>&1 | awk "/$event/ {print \$1}")
> +       bpf_count=$(perf stat --no-big-num --bpf-counters -e "$event" -- $workload  2>&1 | awk "/$event/ {print \$1}")
> +       check_counts "$base_count" "$bpf_count"
> +       compare_number "$base_count" "$bpf_count"
>         echo "[Success]"
>  }
>
>  test_bpf_modifier()
>  {
>         printf "Testing bpf event modifier "
> -       stat_output=$(perf stat --no-big-num -e instructions/name=base_instructions/,instructions/name=bpf_instructions/b -- $workload 2>&1)
> -       base_instructions=$(echo "$stat_output"| awk '/base_instructions/ {print $1}')
> -       bpf_instructions=$(echo "$stat_output"| awk '/bpf_instructions/ {print $1}')
> -       check_counts $base_instructions $bpf_instructions
> -       compare_number $base_instructions $bpf_instructions
> +       stat_output=$(perf stat --no-big-num -e "$event/name=base_$event/,$event/name=bpf_$event/b" -- $workload 2>&1)
> +       base_count=$(echo "$stat_output"| awk "/base_$event/ {print \$1}")
> +       bpf_count=$(echo "$stat_output"| awk "/bpf_$event/ {print \$1}")
> +       check_counts "$base_count" "$bpf_count"
> +       compare_number "$base_count" "$bpf_count"
>         echo "[Success]"
>  }
>
>  # skip if --bpf-counters is not supported
> -if ! perf stat -e instructions --bpf-counters true > /dev/null 2>&1; then
> +if ! perf stat -e $event --bpf-counters true > /dev/null 2>&1; then
>         if [ "$1" = "-v" ]; then
>                 echo "Skipping: --bpf-counters not supported"
> -               perf --no-pager stat -e instructions --bpf-counters true || true
> +               perf --no-pager stat -e $event --bpf-counters true || true
>         fi
>         exit 2
>  fi
> --
> 2.53.0.1018.g2bb0e51243-goog
>
Re: [RFC PATCH v1] perf test: Fix perf stat --bpf-counters on hybrid machines
Posted by Namhyung Kim 1 week, 4 days ago 
On Mon, Mar 23, 2026 at 01:26:11PM -0700, Ian Rogers wrote:
> On Mon, Mar 23, 2026 at 1:24 PM Ian Rogers <irogers@google.com> wrote:
> >
> > Events on hardware PMUs may open on >1 hardware PMU on hybrid
> > machines. Unfortunately multiplexing introduces a scaling issue as
> > reported in:
> > https://lore.kernel.org/lkml/20250716223924.825772-1-irogers@google.com/
> >
> > Switch to using task-clock which as a software event runs on all CPUs
> > and has a single PMU.
> >
> > Signed-off-by: Ian Rogers <irogers@google.com>
> 
> With hindsight, maybe the better thing to do is Namhyung's patch plus
> the "--no-scale" option to `perf stat` to sidestep the scaling issue.
> It'd be great to make this test not "(exclusive)", it shouldn't be
> necessary. You can test for flakiness with the '-r' option to `perf
> test` which takes a repeat count.

I think your fix is simpler and better. :)

Acked-by: Namhyung Kim <namhyung@kernel.org>

Thanks,
Namhyung
[PATCH v2] perf test: Fix perf stat --bpf-counters on hybrid machines
Posted by Ian Rogers 1 week, 3 days ago 
Events on hardware PMUs may open on >1 hardware PMU on hybrid
machines. Unfortunately multiplexing introduces a scaling issue as
reported in:
https://lore.kernel.org/lkml/20250716223924.825772-1-irogers@google.com/

Switch to using task-clock which as a software event runs on all CPUs
and has a single PMU.

For when hybrid event scaling is fixed, incorporate support for
combining hybrid counts as posted by Namhyung Kim in:
https://lore.kernel.org/lkml/20260319064513.9269-1-namhyung@kernel.org/

Signed-off-by: Ian Rogers <irogers@google.com>
Acked-by: Namhyung Kim <namhyung@kernel.org>
---
v1: https://lore.kernel.org/lkml/20260323202409.194123-1-irogers@google.com/
---
 tools/perf/tests/shell/stat_bpf_counters.sh | 55 ++++++++++++++-------
 1 file changed, 38 insertions(+), 17 deletions(-)

diff --git a/tools/perf/tests/shell/stat_bpf_counters.sh b/tools/perf/tests/shell/stat_bpf_counters.sh
index f43e28a136d3..f8c24ccc7830 100755
--- a/tools/perf/tests/shell/stat_bpf_counters.sh
+++ b/tools/perf/tests/shell/stat_bpf_counters.sh
@@ -5,6 +5,24 @@
 set -e
 
 workload="perf test -w sqrtloop"
+event=task-clock
+
+extract_count()
+{
+	local target_event="$1"
+
+	awk -v target="$target_event" -v i=0 -v c=0 '
+	$0 ~ target {
+		i++;
+		c += $1
+	}
+	END {
+		if (i > 0)
+			printf "%.0f\n", c;
+		else
+			print "<not counted>"
+	}'
+}
 
 # check whether $2 is within +/- 20% of $1
 compare_number()
@@ -25,15 +43,15 @@ compare_number()
 
 check_counts()
 {
-	base_instructions=$1
-	bpf_instructions=$2
+	base_count=$1
+	bpf_count=$2
 
-	if [ "$base_instructions" = "<not" ]; then
-		echo "Skipping: instructions event not counted"
+	if [ "$base_count" = "<not counted>" ]; then
+		echo "Skipping: $event event not counted"
 		exit 2
 	fi
-	if [ "$bpf_instructions" = "<not" ]; then
-		echo "Failed: instructions not counted with --bpf-counters"
+	if [ "$bpf_count" = "<not counted>" ]; then
+		echo "Failed: $event not counted with --bpf-counters"
 		exit 1
 	fi
 }
@@ -41,29 +59,32 @@ check_counts()
 test_bpf_counters()
 {
 	printf "Testing --bpf-counters "
-	base_instructions=$(perf stat --no-big-num -e instructions -- $workload 2>&1 | awk '/instructions/ {print $1}')
-	bpf_instructions=$(perf stat --no-big-num --bpf-counters -e instructions -- $workload  2>&1 | awk '/instructions/ {print $1}')
-	check_counts $base_instructions $bpf_instructions
-	compare_number $base_instructions $bpf_instructions
+	base_count=$(perf stat --no-big-num -e "$event" -- $workload 2>&1 \
+          | extract_count "$event")
+	bpf_count=$(perf stat --no-big-num --bpf-counters -e "$event" -- $workload  2>&1 \
+          | extract_count "$event")
+	check_counts "$base_count" "$bpf_count"
+	compare_number "$base_count" "$bpf_count"
 	echo "[Success]"
 }
 
 test_bpf_modifier()
 {
 	printf "Testing bpf event modifier "
-	stat_output=$(perf stat --no-big-num -e instructions/name=base_instructions/,instructions/name=bpf_instructions/b -- $workload 2>&1)
-	base_instructions=$(echo "$stat_output"| awk '/base_instructions/ {print $1}')
-	bpf_instructions=$(echo "$stat_output"| awk '/bpf_instructions/ {print $1}')
-	check_counts $base_instructions $bpf_instructions
-	compare_number $base_instructions $bpf_instructions
+	stat_output=$(perf stat --no-big-num \
+          -e "$event/name=base_$event/,$event/name=bpf_$event/b" -- $workload 2>&1)
+	base_count=$(echo "$stat_output" | extract_count "base_$event")
+	bpf_count=$(echo "$stat_output"|  extract_count "bpf_$event")
+	check_counts "$base_count" "$bpf_count"
+	compare_number "$base_count" "$bpf_count"
 	echo "[Success]"
 }
 
 # skip if --bpf-counters is not supported
-if ! perf stat -e instructions --bpf-counters true > /dev/null 2>&1; then
+if ! perf stat -e "$event" --bpf-counters true > /dev/null 2>&1; then
 	if [ "$1" = "-v" ]; then
 		echo "Skipping: --bpf-counters not supported"
-		perf --no-pager stat -e instructions --bpf-counters true || true
+		perf --no-pager stat -e "$event" --bpf-counters true || true
 	fi
 	exit 2
 fi
-- 
2.53.0.1018.g2bb0e51243-goog
Re: [PATCH v2] perf test: Fix perf stat --bpf-counters on hybrid machines
Posted by Falcon, Thomas 1 day, 17 hours ago 
On Wed, 2026-03-25 at 10:16 -0700, Ian Rogers wrote:
> Events on hardware PMUs may open on >1 hardware PMU on hybrid
> machines. Unfortunately multiplexing introduces a scaling issue as
> reported in:
> https://lore.kernel.org/lkml/20250716223924.825772-1-irogers@google.com/
> 
> Switch to using task-clock which as a software event runs on all CPUs
> and has a single PMU.
> 
> For when hybrid event scaling is fixed, incorporate support for
> combining hybrid counts as posted by Namhyung Kim in:
> https://lore.kernel.org/lkml/20260319064513.9269-1-namhyung@kernel.org/
> 
> Signed-off-by: Ian Rogers <irogers@google.com>
> Acked-by: Namhyung Kim <namhyung@kernel.org>

$ sudo ./perf test "perf stat --bpf-counters test" -vvv

Before:

152: perf stat --bpf-counters test:
--- start ---
test child forked, pid 444406
Testing --bpf-counters The difference between 400418163 and 11886805947
are greater than 20%.
---- end(-1) ----
152: perf stat --bpf-counters test                                   :
FAILED!

After:

152: perf stat --bpf-counters test:
--- start ---
test child forked, pid 466896
Testing --bpf-counters [Success]
Testing bpf event modifier [Success]
---- end(0) ----
152: perf stat --bpf-counters test                                   :
Ok


Tested-by: Thomas Falcon <thomas.falcon@intel.com>

Thanks,
Tom

> ---
> v1:
> https://lore.kernel.org/lkml/20260323202409.194123-1-irogers@google.com/
> ---
>  tools/perf/tests/shell/stat_bpf_counters.sh | 55 ++++++++++++++-----
> --
>  1 file changed, 38 insertions(+), 17 deletions(-)
> 
> diff --git a/tools/perf/tests/shell/stat_bpf_counters.sh
> b/tools/perf/tests/shell/stat_bpf_counters.sh
> index f43e28a136d3..f8c24ccc7830 100755
> --- a/tools/perf/tests/shell/stat_bpf_counters.sh
> +++ b/tools/perf/tests/shell/stat_bpf_counters.sh
> @@ -5,6 +5,24 @@
>  set -e
>  
>  workload="perf test -w sqrtloop"
> +event=task-clock
> +
> +extract_count()
> +{
> +	local target_event="$1"
> +
> +	awk -v target="$target_event" -v i=0 -v c=0 '
> +	$0 ~ target {
> +		i++;
> +		c += $1
> +	}
> +	END {
> +		if (i > 0)
> +			printf "%.0f\n", c;
> +		else
> +			print "<not counted>"
> +	}'
> +}
>  
>  # check whether $2 is within +/- 20% of $1
>  compare_number()
> @@ -25,15 +43,15 @@ compare_number()
>  
>  check_counts()
>  {
> -	base_instructions=$1
> -	bpf_instructions=$2
> +	base_count=$1
> +	bpf_count=$2
>  
> -	if [ "$base_instructions" = "<not" ]; then
> -		echo "Skipping: instructions event not counted"
> +	if [ "$base_count" = "<not counted>" ]; then
> +		echo "Skipping: $event event not counted"
>  		exit 2
>  	fi
> -	if [ "$bpf_instructions" = "<not" ]; then
> -		echo "Failed: instructions not counted with --bpf-
> counters"
> +	if [ "$bpf_count" = "<not counted>" ]; then
> +		echo "Failed: $event not counted with --bpf-
> counters"
>  		exit 1
>  	fi
>  }
> @@ -41,29 +59,32 @@ check_counts()
>  test_bpf_counters()
>  {
>  	printf "Testing --bpf-counters "
> -	base_instructions=$(perf stat --no-big-num -e instructions -
> - $workload 2>&1 | awk '/instructions/ {print $1}')
> -	bpf_instructions=$(perf stat --no-big-num --bpf-counters -e
> instructions -- $workload  2>&1 | awk '/instructions/ {print $1}')
> -	check_counts $base_instructions $bpf_instructions
> -	compare_number $base_instructions $bpf_instructions
> +	base_count=$(perf stat --no-big-num -e "$event" -- $workload
> 2>&1 \
> +          | extract_count "$event")
> +	bpf_count=$(perf stat --no-big-num --bpf-counters -e
> "$event" -- $workload  2>&1 \
> +          | extract_count "$event")
> +	check_counts "$base_count" "$bpf_count"
> +	compare_number "$base_count" "$bpf_count"
>  	echo "[Success]"
>  }
>  
>  test_bpf_modifier()
>  {
>  	printf "Testing bpf event modifier "
> -	stat_output=$(perf stat --no-big-num -e
> instructions/name=base_instructions/,instructions/name=bpf_instructio
> ns/b -- $workload 2>&1)
> -	base_instructions=$(echo "$stat_output"| awk
> '/base_instructions/ {print $1}')
> -	bpf_instructions=$(echo "$stat_output"| awk
> '/bpf_instructions/ {print $1}')
> -	check_counts $base_instructions $bpf_instructions
> -	compare_number $base_instructions $bpf_instructions
> +	stat_output=$(perf stat --no-big-num \
> +          -e "$event/name=base_$event/,$event/name=bpf_$event/b" --
> $workload 2>&1)
> +	base_count=$(echo "$stat_output" | extract_count
> "base_$event")
> +	bpf_count=$(echo "$stat_output"|  extract_count
> "bpf_$event")
> +	check_counts "$base_count" "$bpf_count"
> +	compare_number "$base_count" "$bpf_count"
>  	echo "[Success]"
>  }
>  
>  # skip if --bpf-counters is not supported
> -if ! perf stat -e instructions --bpf-counters true > /dev/null 2>&1;
> then
> +if ! perf stat -e "$event" --bpf-counters true > /dev/null 2>&1;
> then
>  	if [ "$1" = "-v" ]; then
>  		echo "Skipping: --bpf-counters not supported"
> -		perf --no-pager stat -e instructions --bpf-counters
> true || true
> +		perf --no-pager stat -e "$event" --bpf-counters true
> || true
>  	fi
>  	exit 2
>  fi

Patchew 2.1-devel-ea86937

© 2016 - 2026 Red Hat, Inc.