[v2] perf test record+probe_libc_inet_pton: Make test resilient

[PATCH v2] perf test record+probe_libc_inet_pton: Make test resilient

Posted by Leo Yan 1 year, 2 months ago

The test failed back and forth due to the call chain being heavily
impacted by the libc, which varies across different architectures and
distros.

The libc contains the symbols for "gaih_inet" and "getaddrinfo" in some
cases, but not always.  Moreover, these symbols can be either normal
symbols or dynamic symbols, making it difficult to decide the call chain
entries due to the symbols are inconsistent.

To fix the issue, this commit identifies three call chain entries are
always present.  These entries are matched by iterating through the
lines in the "perf script" result.  The recording attribute max-stack is
set to 4 for the possible maximum call chain depth.

After:

  # perf test -vF pton
  --- start ---
  Pattern: ping[][0-9 \.:]+probe_libc:inet_pton: \([[:xdigit:]]+\)
    Matching: ping  285058 [025] 1219802.466939: probe_libc:inet_pton: (ffffa14b7cf0)
  Pattern: .*inet_pton\+0x[[:xdigit:]]+[[:space:]]\(/usr/lib/aarch64-linux-gnu/libc-2.31.so|inlined\)$
    Matching: ping  285058 [025] 1219802.466939: probe_libc:inet_pton: (ffffa14b7cf0)
    Matching: ffffa14b7cf0 __GI___inet_pton+0x0 (/usr/lib/aarch64-linux-gnu/libc-2.31.so)
  Pattern: .*(\+0x[[:xdigit:]]+|\[unknown\])[[:space:]]\(.*/bin/ping.*\)$
    Matching: ping  285058 [025] 1219802.466939: probe_libc:inet_pton: (ffffa14b7cf0)
    Matching: ffffa14b7cf0 __GI___inet_pton+0x0 (/usr/lib/aarch64-linux-gnu/libc-2.31.so)
    Matching: ffffa1488040 getaddrinfo+0xe8 (/usr/lib/aarch64-linux-gnu/libc-2.31.so)
    Matching: aaaab8672da4 [unknown] (/usr/bin/ping)
  ---- end ----
   82: probe libc's inet_pton & backtrace it with ping                 : Ok

Reported-by: Jing Zhang <renyu.zj@linux.alibaba.com>
Closes: https://lore.kernel.org/linux-perf-users/1728978807-81116-1-git-send-email-renyu.zj@linux.alibaba.com/
Reported-by: Guilherme Amadio <amadio@gentoo.org>
Closes: https://lore.kernel.org/linux-perf-users/Z0X3AYUWkAgfPpWj@x1/T/#m57327e135b156047e37d214a0d453af6ae1e02be
Signed-off-by: Leo Yan <leo.yan@arm.com>
---
Changes from v1: Fixed a typo s/Seatch/Search.

 .../shell/record+probe_libc_inet_pton.sh      | 34 ++++++++++---------
 1 file changed, 18 insertions(+), 16 deletions(-)

diff --git a/tools/perf/tests/shell/record+probe_libc_inet_pton.sh b/tools/perf/tests/shell/record+probe_libc_inet_pton.sh
index 47a26f25db9f..3fbc7a7409af 100755
--- a/tools/perf/tests/shell/record+probe_libc_inet_pton.sh
+++ b/tools/perf/tests/shell/record+probe_libc_inet_pton.sh
@@ -43,17 +43,8 @@ trace_libc_inet_pton_backtrace() {
 		echo "((__GI_)?getaddrinfo|text_to_binary_address)\+0x[[:xdigit:]]+[[:space:]]\($libc|inlined\)$" >> $expected
 		echo "(gaih_inet|main)\+0x[[:xdigit:]]+[[:space:]]\(inlined|.*/bin/ping.*\)$" >> $expected
 		;;
-	ppc64|ppc64le)
-		eventattr='max-stack=4'
-		# Add gaih_inet to expected backtrace only if it is part of libc.
-		if nm $libc | grep -F -q gaih_inet.; then
-			echo "gaih_inet.*\+0x[[:xdigit:]]+[[:space:]]\($libc\)$" >> $expected
-		fi
-		echo "getaddrinfo\+0x[[:xdigit:]]+[[:space:]]\($libc\)$" >> $expected
-		echo ".*(\+0x[[:xdigit:]]+|\[unknown\])[[:space:]]\(.*/bin/ping.*\)$" >> $expected
-		;;
 	*)
-		eventattr='max-stack=3'
+		eventattr='max-stack=4'
 		echo ".*(\+0x[[:xdigit:]]+|\[unknown\])[[:space:]]\(.*/bin/ping.*\)$" >> $expected
 		;;
 	esac
@@ -76,14 +67,25 @@ trace_libc_inet_pton_backtrace() {
 	fi
 	perf script -i $perf_data | tac | grep -m1 ^ping -B9 | tac > $perf_script
 
-	exec 3<$perf_script
 	exec 4<$expected
-	while read line <&3 && read -r pattern <&4; do
+	while read -r pattern <&4; do
+		echo "Pattern: $pattern"
 		[ -z "$pattern" ] && break
-		echo $line
-		echo "$line" | grep -E -q "$pattern"
-		if [ $? -ne 0 ] ; then
-			printf "FAIL: expected backtrace entry \"%s\" got \"%s\"\n" "$pattern" "$line"
+
+		found=0
+
+		# Search lines in the perf script result
+		exec 3<$perf_script
+		while read line <&3; do
+			[ -z "$line" ] && break
+			echo "  Matching: $line"
+			! echo "$line" | grep -E -q "$pattern"
+			found=$?
+			[ $found -eq 1 ] && break
+		done
+
+		if [ $found -ne 1 ] ; then
+			printf "FAIL: Didn't find the expected backtrace entry \"%s\"\n" "$pattern"
 			return 1
 		fi
 	done
-- 
2.34.1

Re: [PATCH v2] perf test record+probe_libc_inet_pton: Make test resilient

Posted by James Clark 1 year, 1 month ago


On 02/12/2024 11:19 am, Leo Yan wrote:
> The test failed back and forth due to the call chain being heavily
> impacted by the libc, which varies across different architectures and
> distros.
> 
> The libc contains the symbols for "gaih_inet" and "getaddrinfo" in some
> cases, but not always.  Moreover, these symbols can be either normal
> symbols or dynamic symbols, making it difficult to decide the call chain
> entries due to the symbols are inconsistent.
> 
> To fix the issue, this commit identifies three call chain entries are
> always present.  These entries are matched by iterating through the
> lines in the "perf script" result.  The recording attribute max-stack is
> set to 4 for the possible maximum call chain depth.
> 
> After:
> 
>    # perf test -vF pton
>    --- start ---
>    Pattern: ping[][0-9 \.:]+probe_libc:inet_pton: \([[:xdigit:]]+\)
>      Matching: ping  285058 [025] 1219802.466939: probe_libc:inet_pton: (ffffa14b7cf0)
>    Pattern: .*inet_pton\+0x[[:xdigit:]]+[[:space:]]\(/usr/lib/aarch64-linux-gnu/libc-2.31.so|inlined\)$
>      Matching: ping  285058 [025] 1219802.466939: probe_libc:inet_pton: (ffffa14b7cf0)
>      Matching: ffffa14b7cf0 __GI___inet_pton+0x0 (/usr/lib/aarch64-linux-gnu/libc-2.31.so)
>    Pattern: .*(\+0x[[:xdigit:]]+|\[unknown\])[[:space:]]\(.*/bin/ping.*\)$
>      Matching: ping  285058 [025] 1219802.466939: probe_libc:inet_pton: (ffffa14b7cf0)
>      Matching: ffffa14b7cf0 __GI___inet_pton+0x0 (/usr/lib/aarch64-linux-gnu/libc-2.31.so)
>      Matching: ffffa1488040 getaddrinfo+0xe8 (/usr/lib/aarch64-linux-gnu/libc-2.31.so)
>      Matching: aaaab8672da4 [unknown] (/usr/bin/ping)
>    ---- end ----
>     82: probe libc's inet_pton & backtrace it with ping                 : Ok
> 
> Reported-by: Jing Zhang <renyu.zj@linux.alibaba.com>
> Closes: https://lore.kernel.org/linux-perf-users/1728978807-81116-1-git-send-email-renyu.zj@linux.alibaba.com/
> Reported-by: Guilherme Amadio <amadio@gentoo.org>
> Closes: https://lore.kernel.org/linux-perf-users/Z0X3AYUWkAgfPpWj@x1/T/#m57327e135b156047e37d214a0d453af6ae1e02be
> Signed-off-by: Leo Yan <leo.yan@arm.com>
> ---
> Changes from v1: Fixed a typo s/Seatch/Search.
> 
>   .../shell/record+probe_libc_inet_pton.sh      | 34 ++++++++++---------
>   1 file changed, 18 insertions(+), 16 deletions(-)
> 

Reviewed-by: James Clark <james.clark@linaro.org>

> diff --git a/tools/perf/tests/shell/record+probe_libc_inet_pton.sh b/tools/perf/tests/shell/record+probe_libc_inet_pton.sh
> index 47a26f25db9f..3fbc7a7409af 100755
> --- a/tools/perf/tests/shell/record+probe_libc_inet_pton.sh
> +++ b/tools/perf/tests/shell/record+probe_libc_inet_pton.sh
> @@ -43,17 +43,8 @@ trace_libc_inet_pton_backtrace() {
>   		echo "((__GI_)?getaddrinfo|text_to_binary_address)\+0x[[:xdigit:]]+[[:space:]]\($libc|inlined\)$" >> $expected
>   		echo "(gaih_inet|main)\+0x[[:xdigit:]]+[[:space:]]\(inlined|.*/bin/ping.*\)$" >> $expected
>   		;;
> -	ppc64|ppc64le)
> -		eventattr='max-stack=4'
> -		# Add gaih_inet to expected backtrace only if it is part of libc.
> -		if nm $libc | grep -F -q gaih_inet.; then
> -			echo "gaih_inet.*\+0x[[:xdigit:]]+[[:space:]]\($libc\)$" >> $expected
> -		fi
> -		echo "getaddrinfo\+0x[[:xdigit:]]+[[:space:]]\($libc\)$" >> $expected
> -		echo ".*(\+0x[[:xdigit:]]+|\[unknown\])[[:space:]]\(.*/bin/ping.*\)$" >> $expected
> -		;;
>   	*)
> -		eventattr='max-stack=3'
> +		eventattr='max-stack=4'
>   		echo ".*(\+0x[[:xdigit:]]+|\[unknown\])[[:space:]]\(.*/bin/ping.*\)$" >> $expected
>   		;;
>   	esac
> @@ -76,14 +67,25 @@ trace_libc_inet_pton_backtrace() {
>   	fi
>   	perf script -i $perf_data | tac | grep -m1 ^ping -B9 | tac > $perf_script
>   
> -	exec 3<$perf_script
>   	exec 4<$expected
> -	while read line <&3 && read -r pattern <&4; do
> +	while read -r pattern <&4; do
> +		echo "Pattern: $pattern"
>   		[ -z "$pattern" ] && break
> -		echo $line
> -		echo "$line" | grep -E -q "$pattern"
> -		if [ $? -ne 0 ] ; then
> -			printf "FAIL: expected backtrace entry \"%s\" got \"%s\"\n" "$pattern" "$line"
> +
> +		found=0
> +
> +		# Search lines in the perf script result
> +		exec 3<$perf_script
> +		while read line <&3; do
> +			[ -z "$line" ] && break
> +			echo "  Matching: $line"
> +			! echo "$line" | grep -E -q "$pattern"
> +			found=$?
> +			[ $found -eq 1 ] && break
> +		done
> +
> +		if [ $found -ne 1 ] ; then
> +			printf "FAIL: Didn't find the expected backtrace entry \"%s\"\n" "$pattern"
>   			return 1
>   		fi
>   	done

Re: [PATCH v2] perf test record+probe_libc_inet_pton: Make test resilient

Posted by Arnaldo Carvalho de Melo 1 year ago

On Tue, Jan 07, 2025 at 04:51:31PM +0000, James Clark wrote:
> 
> 
> On 02/12/2024 11:19 am, Leo Yan wrote:
> > The test failed back and forth due to the call chain being heavily
> > impacted by the libc, which varies across different architectures and
> > distros.
> > 
> > The libc contains the symbols for "gaih_inet" and "getaddrinfo" in some
> > cases, but not always.  Moreover, these symbols can be either normal
> > symbols or dynamic symbols, making it difficult to decide the call chain
> > entries due to the symbols are inconsistent.
> > 
> > To fix the issue, this commit identifies three call chain entries are
> > always present.  These entries are matched by iterating through the
> > lines in the "perf script" result.  The recording attribute max-stack is
> > set to 4 for the possible maximum call chain depth.
> > 
> > After:
> > 
> >    # perf test -vF pton
> >    --- start ---
> >    Pattern: ping[][0-9 \.:]+probe_libc:inet_pton: \([[:xdigit:]]+\)
> >      Matching: ping  285058 [025] 1219802.466939: probe_libc:inet_pton: (ffffa14b7cf0)
> >    Pattern: .*inet_pton\+0x[[:xdigit:]]+[[:space:]]\(/usr/lib/aarch64-linux-gnu/libc-2.31.so|inlined\)$
> >      Matching: ping  285058 [025] 1219802.466939: probe_libc:inet_pton: (ffffa14b7cf0)
> >      Matching: ffffa14b7cf0 __GI___inet_pton+0x0 (/usr/lib/aarch64-linux-gnu/libc-2.31.so)
> >    Pattern: .*(\+0x[[:xdigit:]]+|\[unknown\])[[:space:]]\(.*/bin/ping.*\)$
> >      Matching: ping  285058 [025] 1219802.466939: probe_libc:inet_pton: (ffffa14b7cf0)
> >      Matching: ffffa14b7cf0 __GI___inet_pton+0x0 (/usr/lib/aarch64-linux-gnu/libc-2.31.so)
> >      Matching: ffffa1488040 getaddrinfo+0xe8 (/usr/lib/aarch64-linux-gnu/libc-2.31.so)
> >      Matching: aaaab8672da4 [unknown] (/usr/bin/ping)
> >    ---- end ----
> >     82: probe libc's inet_pton & backtrace it with ping                 : Ok
> > 
> > Reported-by: Jing Zhang <renyu.zj@linux.alibaba.com>
> > Closes: https://lore.kernel.org/linux-perf-users/1728978807-81116-1-git-send-email-renyu.zj@linux.alibaba.com/
> > Reported-by: Guilherme Amadio <amadio@gentoo.org>
> > Closes: https://lore.kernel.org/linux-perf-users/Z0X3AYUWkAgfPpWj@x1/T/#m57327e135b156047e37d214a0d453af6ae1e02be
> > Signed-off-by: Leo Yan <leo.yan@arm.com>
> > ---
> > Changes from v1: Fixed a typo s/Seatch/Search.
> > 
> >   .../shell/record+probe_libc_inet_pton.sh      | 34 ++++++++++---------
> >   1 file changed, 18 insertions(+), 16 deletions(-)
> > 
> 
> Reviewed-by: James Clark <james.clark@linaro.org>

Thanks, applied to perf-tools-next,

- Arnaldo

Re: [PATCH v2] perf test record+probe_libc_inet_pton: Make test resilient

Posted by Thomas Richter 1 year, 2 months ago

On 12/2/24 12:19, Leo Yan wrote:
> The test failed back and forth due to the call chain being heavily
> impacted by the libc, which varies across different architectures and
> distros.
> 

For s390 using 6.13.0.rc1

Tested-by: Thomas Richter <tmricht@linux.ibm.com>

-- 
Thomas Richter, Dept 3303, IBM s390 Linux Development, Boeblingen, Germany
--
IBM Deutschland Research & Development GmbH

Vorsitzender des Aufsichtsrats: Wolfgang Wendt

Geschäftsführung: David Faller

Sitz der Gesellschaft: Böblingen / Registergericht: Amtsgericht Stuttgart, HRB 243294

Re: [PATCH v2] perf test record+probe_libc_inet_pton: Make test resilient

Posted by Namhyung Kim 1 year, 2 months ago

Hello,

On Mon, Dec 02, 2024 at 02:48:01PM +0100, Thomas Richter wrote:
> On 12/2/24 12:19, Leo Yan wrote:
> > The test failed back and forth due to the call chain being heavily
> > impacted by the libc, which varies across different architectures and
> > distros.
> > 
> 
> For s390 using 6.13.0.rc1
> 
> Tested-by: Thomas Richter <tmricht@linux.ibm.com>

Thanks for the test, but I think the patch excludes s390 to be safe.
And I think we can test s390 in the same way.  If you're willing to do
some more tests, Leo can update the patch to include s390 as well.

Thanks,
Namhyung

> 
> -- 
> Thomas Richter, Dept 3303, IBM s390 Linux Development, Boeblingen, Germany
> --
> IBM Deutschland Research & Development GmbH
> 
> Vorsitzender des Aufsichtsrats: Wolfgang Wendt
> 
> Geschäftsführung: David Faller
> 
> Sitz der Gesellschaft: Böblingen / Registergericht: Amtsgericht Stuttgart, HRB 243294

Re: [PATCH v2] perf test record+probe_libc_inet_pton: Make test resilient

Posted by Leo Yan 1 year, 1 month ago

Hi all,

Happy new year!

On Mon, Dec 02, 2024 at 12:28:03PM -0800, Namhyung Kim wrote:
> 
> Hello,
> 
> On Mon, Dec 02, 2024 at 02:48:01PM +0100, Thomas Richter wrote:
> > On 12/2/24 12:19, Leo Yan wrote:
> > > The test failed back and forth due to the call chain being heavily
> > > impacted by the libc, which varies across different architectures and
> > > distros.
> > >
> >
> > For s390 using 6.13.0.rc1
> >
> > Tested-by: Thomas Richter <tmricht@linux.ibm.com>
> 
> Thanks for the test, but I think the patch excludes s390 to be safe.
> And I think we can test s390 in the same way.  If you're willing to do
> some more tests, Leo can update the patch to include s390 as well.

This case continuously fails on Arm64, so let's not forget to land the
fix.

I agreed with Namhyung for consolidation the test as possible.

@Thomas, could you test on s390 for below change?  And if you could
confirm the 'fp' mode for callgraph is supported on s390, then we even
can unify the event attribute:

  eventattr='max-stack=4'

Thanks,
Leo


---8<---

From 71bb457c7b5370693081856e7804ebe62c214301 Mon Sep 17 00:00:00 2001
From: Leo Yan <leo.yan@arm.com>
Date: Mon, 2 Dec 2024 11:19:58 +0000
Subject: [PATCH] perf test record+probe_libc_inet_pton: Make test resilient

The test failed back and forth due to the call chain being heavily
impacted by the libc, which varies across different architectures and
distros.

The libc contains the symbols for "gaih_inet" and "getaddrinfo" in some
cases, but not always.  Moreover, these symbols can be either normal
symbols or dynamic symbols, making it difficult to decide the call chain
entries due to the symbols are inconsistent.

To fix the issue, this commit identifies three call chain entries are
always present.  These entries are matched by iterating through the
lines in the "perf script" result.  The recording attribute max-stack is
set to 4 for the possible maximum call chain depth.

After:

  # perf test -vF pton
  --- start ---
  Pattern: ping[][0-9 \.:]+probe_libc:inet_pton: \([[:xdigit:]]+\)
    Matching: ping  285058 [025] 1219802.466939: probe_libc:inet_pton: (ffffa14b7cf0)
  Pattern: .*inet_pton\+0x[[:xdigit:]]+[[:space:]]\(/usr/lib/aarch64-linux-gnu/libc-2.31.so|inlined\)$
    Matching: ping  285058 [025] 1219802.466939: probe_libc:inet_pton: (ffffa14b7cf0)
    Matching: ffffa14b7cf0 __GI___inet_pton+0x0 (/usr/lib/aarch64-linux-gnu/libc-2.31.so)
  Pattern: .*(\+0x[[:xdigit:]]+|\[unknown\])[[:space:]]\(.*/bin/ping.*\)$
    Matching: ping  285058 [025] 1219802.466939: probe_libc:inet_pton: (ffffa14b7cf0)
    Matching: ffffa14b7cf0 __GI___inet_pton+0x0 (/usr/lib/aarch64-linux-gnu/libc-2.31.so)
    Matching: ffffa1488040 getaddrinfo+0xe8 (/usr/lib/aarch64-linux-gnu/libc-2.31.so)
    Matching: aaaab8672da4 [unknown] (/usr/bin/ping)
  ---- end ----
   82: probe libc's inet_pton & backtrace it with ping                 : Ok

Reported-by: Jing Zhang <renyu.zj@linux.alibaba.com>
Closes: https://lore.kernel.org/linux-perf-users/1728978807-81116-1-git-send-email-renyu.zj@linux.alibaba.com/
Reported-by: Guilherme Amadio <amadio@gentoo.org>
Closes: https://lore.kernel.org/linux-perf-users/Z0X3AYUWkAgfPpWj@x1/T/#m57327e135b156047e37d214a0d453af6ae1e02be
Signed-off-by: Leo Yan <leo.yan@arm.com>
---
 .../shell/record+probe_libc_inet_pton.sh      | 39 ++++++++++---------
 1 file changed, 20 insertions(+), 19 deletions(-)

diff --git a/tools/perf/tests/shell/record+probe_libc_inet_pton.sh b/tools/perf/tests/shell/record+probe_libc_inet_pton.sh
index 47a26f25db9f..326d924641e0 100755
--- a/tools/perf/tests/shell/record+probe_libc_inet_pton.sh
+++ b/tools/perf/tests/shell/record+probe_libc_inet_pton.sh
@@ -37,24 +37,14 @@ trace_libc_inet_pton_backtrace() {
 
 	echo "ping[][0-9 \.:]+$event_name: \([[:xdigit:]]+\)" > $expected
 	echo ".*inet_pton\+0x[[:xdigit:]]+[[:space:]]\($libc|inlined\)$" >> $expected
+	echo ".*(\+0x[[:xdigit:]]+|\[unknown\])[[:space:]]\(.*/bin/ping.*\)$" >> $expected
+
 	case "$(uname -m)" in
 	s390x)
 		eventattr='call-graph=dwarf,max-stack=4'
-		echo "((__GI_)?getaddrinfo|text_to_binary_address)\+0x[[:xdigit:]]+[[:space:]]\($libc|inlined\)$" >> $expected
-		echo "(gaih_inet|main)\+0x[[:xdigit:]]+[[:space:]]\(inlined|.*/bin/ping.*\)$" >> $expected
-		;;
-	ppc64|ppc64le)
-		eventattr='max-stack=4'
-		# Add gaih_inet to expected backtrace only if it is part of libc.
-		if nm $libc | grep -F -q gaih_inet.; then
-			echo "gaih_inet.*\+0x[[:xdigit:]]+[[:space:]]\($libc\)$" >> $expected
-		fi
-		echo "getaddrinfo\+0x[[:xdigit:]]+[[:space:]]\($libc\)$" >> $expected
-		echo ".*(\+0x[[:xdigit:]]+|\[unknown\])[[:space:]]\(.*/bin/ping.*\)$" >> $expected
 		;;
 	*)
-		eventattr='max-stack=3'
-		echo ".*(\+0x[[:xdigit:]]+|\[unknown\])[[:space:]]\(.*/bin/ping.*\)$" >> $expected
+		eventattr='max-stack=4'
 		;;
 	esac
 
@@ -76,14 +66,25 @@ trace_libc_inet_pton_backtrace() {
 	fi
 	perf script -i $perf_data | tac | grep -m1 ^ping -B9 | tac > $perf_script
 
-	exec 3<$perf_script
 	exec 4<$expected
-	while read line <&3 && read -r pattern <&4; do
+	while read -r pattern <&4; do
+		echo "Pattern: $pattern"
 		[ -z "$pattern" ] && break
-		echo $line
-		echo "$line" | grep -E -q "$pattern"
-		if [ $? -ne 0 ] ; then
-			printf "FAIL: expected backtrace entry \"%s\" got \"%s\"\n" "$pattern" "$line"
+
+		found=0
+
+		# Search lines in the perf script result
+		exec 3<$perf_script
+		while read line <&3; do
+			[ -z "$line" ] && break
+			echo "  Matching: $line"
+			! echo "$line" | grep -E -q "$pattern"
+			found=$?
+			[ $found -eq 1 ] && break
+		done
+
+		if [ $found -ne 1 ] ; then
+			printf "FAIL: Didn't find the expected backtrace entry \"%s\"\n" "$pattern"
 			return 1
 		fi
 	done
-- 
2.25.1