Add test to validate end-to-end start/stop sequence for
each remoteproc instances available on target.

Add first test sequence to validated each instance sequencially
to identify any issue while booting each instance.
Add second test sequence to validate all instances concurrently
to identify any race scenario within instances doing bootup.

Additional user argument (--seqdelay) is available to add
delay is seconds, between start/stop sequence. This is added
as different target might have different threshold to start
any instance (default is 5 secs).

Running tests:
	./remoteproc_test.sh --seqdelay 10

Signed-off-by: Wasim Nazir <quic_wasimn@quicinc.com>
---
 MAINTAINERS                                   |   1 +
 tools/testing/selftests/Makefile              |   1 +
 tools/testing/selftests/remoteproc/Makefile   |   4 +
 tools/testing/selftests/remoteproc/config     |   1 +
 .../selftests/remoteproc/remoteproc_test.sh   | 157 ++++++++++++++++++
 5 files changed, 164 insertions(+)
 create mode 100644 tools/testing/selftests/remoteproc/Makefile
 create mode 100644 tools/testing/selftests/remoteproc/config
 create mode 100644 tools/testing/selftests/remoteproc/remoteproc_test.sh


Test output with 4 remoteproc instances:

TAP version 13
1..5
# Testing rproc start/stop sequence for each instance sequencially
# Testing rproc sequence for 4080000.remoteproc
ok 1 4080000.remoteproc
# Testing rproc sequence for 3700000.remoteproc
ok 2 3700000.remoteproc
# Testing rproc sequence for 8a00000.remoteproc
ok 3 8a00000.remoteproc
# Testing rproc sequence for a300000.remoteproc
ok 4 a300000.remoteproc
# Testing rproc start/stop sequence for all instances concurrently
ok 5 for all remoteproc0 remoteproc1 remoteproc2 remoteproc3
# Totals: pass:5 fail:0 xfail:0 xpass:0 skip:0 error:0

Changes in v3:
	- Add user argument for sequence delay (--sedelay).
	- Update commit & add comments.
	- v2: https://lore.kernel.org/all/20240927112132.3927298-1-quic_wasimn@quicinc.com/

diff --git a/MAINTAINERS b/MAINTAINERS
index e9659a5a7fb3..1f8182473be1 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -19481,6 +19481,7 @@ F:	Documentation/staging/remoteproc.rst
 F:	drivers/remoteproc/
 F:	include/linux/remoteproc.h
 F:	include/linux/remoteproc/
+F:	tools/testing/selftests/remoteproc/

 REMOTE PROCESSOR MESSAGING (RPMSG) SUBSYSTEM
 M:	Bjorn Andersson <andersson@kernel.org>
diff --git a/tools/testing/selftests/Makefile b/tools/testing/selftests/Makefile
index 363d031a16f7..78669153be90 100644
--- a/tools/testing/selftests/Makefile
+++ b/tools/testing/selftests/Makefile
@@ -82,6 +82,7 @@ TARGETS += proc
 TARGETS += pstore
 TARGETS += ptrace
 TARGETS += openat2
+TARGETS += remoteproc
 TARGETS += resctrl
 TARGETS += riscv
 TARGETS += rlimits
diff --git a/tools/testing/selftests/remoteproc/Makefile b/tools/testing/selftests/remoteproc/Makefile
new file mode 100644
index 000000000000..a84b3934fd36
--- /dev/null
+++ b/tools/testing/selftests/remoteproc/Makefile
@@ -0,0 +1,4 @@
+# SPDX-License-Identifier: GPL-2.0
+TEST_PROGS := remoteproc_test.sh
+
+include ../lib.mk
diff --git a/tools/testing/selftests/remoteproc/config b/tools/testing/selftests/remoteproc/config
new file mode 100644
index 000000000000..a5c237d2f3b4
--- /dev/null
+++ b/tools/testing/selftests/remoteproc/config
@@ -0,0 +1 @@
+CONFIG_REMOTEPROC=y
diff --git a/tools/testing/selftests/remoteproc/remoteproc_test.sh b/tools/testing/selftests/remoteproc/remoteproc_test.sh
new file mode 100644
index 000000000000..d58c1e10005c
--- /dev/null
+++ b/tools/testing/selftests/remoteproc/remoteproc_test.sh
@@ -0,0 +1,157 @@
+#!/bin/sh
+# SPDX-License-Identifier: GPL-2.0
+#
+# Copyright (c) 2024 Qualcomm Innovation Center, Inc. All rights reserved.
+#
+
+DIR="$(dirname $(readlink -f "$0"))"
+
+KTAP_HELPERS="${DIR}/../kselftest/ktap_helpers.sh"
+if [ -e "$KTAP_HELPERS" ]; then
+    . "$KTAP_HELPERS"
+else
+    echo -n "1..0 # SKIP $KTAP_HELPERS file not found"
+    exit 4
+fi
+
+RPROC_SYS=/sys/class/remoteproc
+RPROC_SEQ_SLEEP=5
+
+rproc_instances=
+num_tests=0
+test_err=0
+
+check_error() {
+	if [ $? -ne 0 ]; then
+		test_err=$((test_err+1))
+		ktap_print_msg "$@"
+	fi
+}
+
+parse_args() {
+	script=${0##*/}
+
+	if [ $# -eq 2 ] && [ "$1" = "--seqdelay" ]; then
+		shift || true
+		RPROC_SEQ_SLEEP=$1
+	else
+		ktap_print_msg "Usage: ${script} --seqdelay <time in secs>"
+		ktap_print_msg "Proceed with default sequence delay = $RPROC_SEQ_SLEEP"
+	fi
+}
+
+rproc_stop_instances() {
+	for instance in ${rproc_instances}; do
+		rproc=${RPROC_SYS}/$instance
+		rproc_name=$(cat $rproc/name)
+		rproc_state=$(cat $rproc/state)
+
+		echo stop > "$rproc/state"
+		check_error "$rproc_name state-stop failed at state $rproc_state"
+	done
+	sleep ${RPROC_SEQ_SLEEP}
+}
+
+rproc_start_instances() {
+	for instance in ${rproc_instances}; do
+		rproc=${RPROC_SYS}/$instance
+		rproc_name=$(cat $rproc/name)
+		rproc_state=$(cat $rproc/state)
+
+		echo start > "$rproc/state"
+		check_error "$rproc_name state-start failed at state $rproc_state"
+	done
+	sleep ${RPROC_SEQ_SLEEP}
+}
+
+rproc_seq_test_instance_one() {
+	instance=$1
+	rproc=${RPROC_SYS}/$instance
+	rproc_name=$(cat $rproc/name)
+	rproc_state=$(cat $rproc/state)
+	ktap_print_msg "Testing rproc sequence for $rproc_name"
+
+	# Reset test_err value
+	test_err=0
+
+	# Begin start/stop sequence
+	echo start > "$rproc/state"
+	check_error "$rproc_name state-start failed at state $rproc_state"
+
+	sleep ${RPROC_SEQ_SLEEP}
+
+	echo stop > "$rproc/state"
+	check_error "$rproc_name state-stop failed at state $rproc_state"
+
+	if [ $test_err -ne 0 ]; then
+		ktap_test_fail "$rproc_name"
+	else
+		ktap_test_pass "$rproc_name"
+	fi
+}
+
+rproc_seq_test_instances_concurrently() {
+	# Reset test_err value
+	test_err=0
+
+	rproc_start_instances
+
+	rproc_stop_instances
+
+	if [ $test_err -ne 0 ]; then
+		ktap_test_fail "for any of $rproc_instances"
+	else
+		ktap_test_pass "for all $rproc_instances"
+	fi
+}
+
+#################################
+### Test starts here
+#################################
+
+ktap_print_header
+
+# Parse user arguments
+parse_args $@
+
+# Check for required sysfs entries
+if [ ! -d "${RPROC_SYS}" ]; then
+	ktap_skip_all "${RPROC_SYS} doesn't exist."
+	exit "${KSFT_SKIP}"
+fi
+
+rproc_instances=$(find ${RPROC_SYS}/remoteproc* -maxdepth 1 -exec basename {} \;)
+num_tests=$(echo ${rproc_instances} | wc -w)
+if [ "${num_tests}" -eq 0 ]; then
+	ktap_skip_all "${RPROC_SYS}/remoteproc* doesn't exist."
+	exit "${KSFT_SKIP}"
+fi
+
+# Total tests will be:
+# 1) Seq tests for each instance sequencially
+# 2) Seq tests for all instances concurrently
+num_tests=$((num_tests+1))
+
+ktap_set_plan "${num_tests}"
+
+### Stop all instances
+#
+# Intention is to stop all running instances. If any instances are not yet
+# started it will be don't care case as test_err is not checked.
+# NOTE: Assuming no instances are in crashed state
+rproc_stop_instances
+
+### Test 1
+ktap_print_msg "Testing rproc start/stop sequence for each instance sequencially"
+for instance in ${rproc_instances}; do
+	rproc_seq_test_instance_one $instance
+done
+
+### Test 2
+ktap_print_msg "Testing rproc start/stop sequence for all instances concurrently"
+rproc_seq_test_instances_concurrently
+
+### Restore all instances
+rproc_start_instances
+
+ktap_finished
--
2.46.1