Test to check if setting PROC_CN_MCAST_NOTIFY in proc connector API, allows
a thread's non-zero exit status to be returned to proc_filter.

The threads.c program creates 2 child threads. 1st thread handles signal
SIGSEGV, and 2nd thread needs to indicate some error condition (value 1)
to the kernel, instead of using pthread_exit() with 1.

In both cases, child sends notify_netlink_thread_exit(exit_code) to kernel,
to let kernel know it has exited abnormally with exit_code.

Compile:
    make thread
    make proc_filter
Run:
    ./threads

Signed-off-by: Anjali Kulkarni <anjali.k.kulkarni@oracle.com>
---
 tools/testing/selftests/connector/Makefile    |  23 +-
 .../testing/selftests/connector/proc_filter.c |  34 ++-
 tools/testing/selftests/connector/thread.c    | 232 ++++++++++++++++++
 .../selftests/connector/thread_filter.c       |  96 ++++++++
 4 files changed, 378 insertions(+), 7 deletions(-)
 create mode 100644 tools/testing/selftests/connector/thread.c
 create mode 100644 tools/testing/selftests/connector/thread_filter.c

diff --git a/tools/testing/selftests/connector/Makefile b/tools/testing/selftests/connector/Makefile
index 92188b9bac5c..bf335826bc3b 100644
--- a/tools/testing/selftests/connector/Makefile
+++ b/tools/testing/selftests/connector/Makefile
@@ -1,5 +1,26 @@
 # SPDX-License-Identifier: GPL-2.0
-CFLAGS += -Wall $(KHDR_INCLUDES)
+KERNEL="../../../.."
+
+CFLAGS += -Wall $(KHDR_INCLUDES) -I $(KERNEL)/include/uapi -I $(KERNEL)/include
+
+proc_filter: proc_filter.o
+	cc proc_filter.o -o proc_filter
+
+proc_filter.o: proc_filter.c
+	cc -c proc_filter.c -o proc_filter.o $(CFLAGS)
+
+thread: thread.o thread_filter.o
+	cc thread.o thread_filter.o -o thread
+
+thread.o: thread.c $(DEPS)
+		cc -c thread.c -o thread.o $(CFLAGS)
+
+thread_filter.o: thread_filter.c
+		cc -c thread_filter.c -o thread_filter.o $(CFLAGS)
+
+define EXTRA_CLEAN
+	rm *.o thread
+endef
 
 TEST_GEN_PROGS = proc_filter
 
diff --git a/tools/testing/selftests/connector/proc_filter.c b/tools/testing/selftests/connector/proc_filter.c
index 4a825b997666..5bf992deb792 100644
--- a/tools/testing/selftests/connector/proc_filter.c
+++ b/tools/testing/selftests/connector/proc_filter.c
@@ -1,4 +1,9 @@
 // SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Author: Anjali Kulkarni <anjali.k.kulkarni@oracle.com>
+ *
+ * Copyright (c) 2024 Oracle and/or its affiliates.
+ */
 
 #include <sys/types.h>
 #include <sys/epoll.h>
@@ -28,6 +33,7 @@
 volatile static int interrupted;
 static int nl_sock, ret_errno, tcount;
 static struct epoll_event evn;
+FILE *file;
 
 static int filter;
 
@@ -37,6 +43,8 @@ static int filter;
 #define Printf ksft_print_msg
 #endif
 
+#define EXIT_LOG
+
 int send_message(void *pinp)
 {
 	char buff[NL_MESSAGE_SIZE];
@@ -146,6 +154,12 @@ int handle_packet(char *buff, int fd, struct proc_event *event)
 		tcount++;
 		switch (event->what) {
 		case PROC_EVENT_EXIT:
+#ifdef EXIT_LOG
+			fprintf(file, "pid %d tgid %d code %d\n",
+				event->event_data.exit.process_pid,
+				event->event_data.exit.process_tgid,
+				event->event_data.exit.exit_code);
+#endif
 			Printf("Exit process %d (tgid %d) with code %d, signal %d\n",
 			       event->event_data.exit.process_pid,
 			       event->event_data.exit.process_tgid,
@@ -279,17 +293,24 @@ int main(int argc, char *argv[])
 		exit(1);
 	}
 
+#ifdef EXIT_LOG
+	file = fopen("exit.log", "w");
+	if (file == NULL) {
+		perror("Error opening file exit.log");
+		close(nl_sock);
+		close(epoll_fd);
+		exit(1);
+	}
+#endif
+
 	while (!interrupted) {
 		err = handle_events(epoll_fd, &proc_ev);
 		if (err < 0) {
 			if (ret_errno == EINTR)
 				continue;
-			if (err == -2)
-				close(nl_sock);
-			if (err == -3) {
-				close(nl_sock);
-				close(epoll_fd);
-			}
+			close(nl_sock);
+			close(epoll_fd);
+			fclose(file);
 			exit(1);
 		}
 	}
@@ -304,6 +325,7 @@ int main(int argc, char *argv[])
 
 	close(epoll_fd);
 	close(nl_sock);
+	fclose(file);
 
 	printf("Done total count: %d\n", tcount);
 	exit(0);
diff --git a/tools/testing/selftests/connector/thread.c b/tools/testing/selftests/connector/thread.c
new file mode 100644
index 000000000000..70c475ce3176
--- /dev/null
+++ b/tools/testing/selftests/connector/thread.c
@@ -0,0 +1,232 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Author: Anjali Kulkarni <anjali.k.kulkarni@oracle.com>
+ *
+ * Copyright (c) 2024 Oracle and/or its affiliates.
+ */
+
+#include <pthread.h>
+#include <stdio.h>
+#include <unistd.h>
+#include <stdlib.h>
+#include <signal.h>
+
+/*
+ * This code tests a thread exit notification when thread exits abnormally.
+ * Normally, when a thread exits abnormally, the kernel is not aware of the
+ * exit code. This is usually only conveyed from child to parent via the
+ * pthread_exit() and pthread_join() calls. Sometimes, however, a parent
+ * process cannot monitor all child processes via pthread_join(), particularly
+ * when there is a huge amount of child processes. In this case, the parent
+ * has created the child with PTHREAD_CREATE_DETACHED attribute.
+ * To fix this problem, either when child wants to convey non-zero exit via
+ * pthread_exit() or in a signal handler, the child can notify the kernel's
+ * connector module it's exit status via a netlink call with new type
+ * PROC_CN_MCAST_NOTIFY. (Implemented in the thread_filter.c file).
+ * This will send the exit code from the child to the kernel, which the kernel
+ * can later return to proc_filter program when the child actually exits.
+ * Compile:
+ *	make thread
+ *	make proc_filter
+ * Run:
+ *	./threads
+ */
+
+extern int notify_netlink_thread_exit(unsigned int exit_code);
+
+static void sigsegvh(int sig)
+{
+	unsigned int exit_code = (unsigned int) sig;
+	/*
+	 * Send any non-zero value to get a notification. Here we are
+	 * sending the signal number for SIGSEGV which is 11
+	 */
+	notify_netlink_thread_exit(exit_code);
+}
+
+void *threadc1(void *ptr)
+{
+	signal(SIGSEGV, sigsegvh);
+
+	*(int *)ptr = gettid();
+
+	printf("Child 1 thread id %d, handling SIGSEGV\n", gettid());
+	sleep(10);
+	pthread_exit(NULL);
+}
+
+void *threadc2(void *ptr)
+{
+	int exit_val = 1;
+
+	*(int *)ptr = gettid();
+
+	printf("Child 2 thread id %d, wants to exit with value %d\n",
+			gettid(), exit_val);
+	sleep(2);
+	notify_netlink_thread_exit(exit_val);
+	pthread_exit(NULL);
+}
+
+static void verify_exit_status(int tid1, int tid2)
+{
+	int found1 = 0, found2 = 0;
+	int pid, tgid, exit_code;
+	size_t size = 1024;
+	FILE *file;
+	char *data;
+	int ret;
+
+	data = malloc(size * sizeof(char));
+	if (data == NULL) {
+		perror("malloc for data failed");
+		exit(1);
+	}
+
+	file = fopen("exit.log", "r");
+	if (file == NULL) {
+		perror("fopen of exit.log failed");
+		free(data);
+		exit(1);
+	}
+
+	while (getline(&data, &size, file) != -1) {
+		ret = sscanf(data, "pid %d tgid %d code %d",
+				&pid, &tgid, &exit_code);
+		if (ret != 3) {
+			perror("sscanf error");
+			free(data);
+			fclose(file);
+			exit(1);
+		}
+
+		if (tgid != getpid())
+			continue;
+
+		if (pid == tid1) {
+			if (exit_code == 11) {
+				printf("Successful notification of SIGSEGV, tid %d\n",
+						pid);
+			} else {
+				printf("Failure SIGSEGV tid %d, exit code %d\n",
+					       pid, exit_code);
+			}
+			found1 = 1;
+		} else if (pid == tid2) {
+			if (exit_code == 1) {
+				printf("Successful notification of thread exit tid %d\n",
+						pid);
+			} else {
+				printf("Failure thread exit tid %d, exit code %d\n",
+					       pid, exit_code);
+			}
+			found2 = 1;
+		}
+	}
+
+	if (!found1)
+		printf("tid %d not present in exit.log file\n", tid1);
+
+	if (!found2)
+		printf("tid %d not present in exit.log file\n", tid2);
+
+	fclose(file);
+	free(data);
+}
+
+static inline void init_threads(pthread_attr_t *attr)
+{
+	int ret;
+
+	ret = pthread_attr_init(attr);
+	if (ret != 0) {
+		perror("pthread_attr_init failed");
+		exit(ret);
+	}
+
+	ret = pthread_attr_setdetachstate(attr, PTHREAD_CREATE_DETACHED);
+	if (ret != 0) {
+		perror("pthread_attr_setdetachstate failed");
+		exit(ret);
+	}
+}
+
+static inline void destroy_thread_attr(pthread_attr_t *attr)
+{
+	int ret;
+
+	ret = pthread_attr_destroy(attr);
+	if (ret != 0) {
+		perror("pthread_attr_destroy failed");
+		exit(ret);
+	}
+}
+
+
+static inline pid_t start_proc_filter(void)
+{
+	pid_t proc_filter = 0;
+
+	proc_filter = fork();
+	if (proc_filter == -1) {
+		perror("fork()");
+		exit(1);
+	}
+
+	if (proc_filter == 0) {
+		char* arr[] = {"proc_filter", "-f", NULL};
+		execv("./proc_filter", arr);
+	}
+	sleep(1);
+	return proc_filter;
+}
+
+int main(int argc, char **argv)
+{
+	pthread_t thread1, thread2;
+	pthread_attr_t attr1, attr2;
+	int tid1, tid2, ret;
+	pid_t proc_filter_pid;
+
+	proc_filter_pid = start_proc_filter();
+
+	init_threads(&attr1);
+	ret = pthread_create(&thread1, &attr1, *threadc1, &tid1);
+	if (ret != 0) {
+		perror("pthread_create failed");
+		exit(ret);
+	}
+
+	init_threads(&attr2);
+	ret = pthread_create(&thread2, &attr2, *threadc2, &tid2);
+	if (ret != 0) {
+		perror("pthread_create failed");
+		exit(ret);
+	}
+
+	sleep(1);
+
+	/* Send SIGSEGV to tid1 */
+	kill(tid1, SIGSEGV);
+
+	/*
+	 * Wait for children to exit or be killed and for exit.log to
+	 * be generated by ./proc_filter
+	 */
+	sleep(2);
+
+	/*
+	 * Kill proc_filter to get exit.log
+	 */
+	kill(proc_filter_pid, SIGINT);
+
+	/* Required to allow kill to be processed */
+	sleep(1);
+
+	verify_exit_status(tid1, tid2);
+
+	destroy_thread_attr(&attr1);
+	destroy_thread_attr(&attr2);
+
+	exit(0);
+}
diff --git a/tools/testing/selftests/connector/thread_filter.c b/tools/testing/selftests/connector/thread_filter.c
new file mode 100644
index 000000000000..3da740aa7537
--- /dev/null
+++ b/tools/testing/selftests/connector/thread_filter.c
@@ -0,0 +1,96 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Author: Anjali Kulkarni <anjali.k.kulkarni@oracle.com>
+ *
+ * Copyright (c) 2024 Oracle and/or its affiliates.
+ */
+
+#include <sys/types.h>
+#include <sys/epoll.h>
+#include <sys/socket.h>
+#include <linux/netlink.h>
+#include <linux/connector.h>
+#include <linux/cn_proc.h>
+
+#include <stddef.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <strings.h>
+#include <errno.h>
+#include <signal.h>
+#include <string.h>
+
+#define NL_MESSAGE_SIZE (sizeof(struct nlmsghdr) + sizeof(struct cn_msg) + \
+			sizeof(struct proc_input))
+
+/*
+ * Send PROC_CN_MCAST_NOTIFY type notification to the connector code in kernel.
+ * This will send the exit_code specified by user to the connector layer, so
+ * it can send a notification for that event to any listening process
+ */
+int send_message(int nl_sock, unsigned int exit_code)
+{
+	char buff[NL_MESSAGE_SIZE];
+	struct nlmsghdr *hdr;
+	struct cn_msg *msg;
+
+	hdr = (struct nlmsghdr *)buff;
+	hdr->nlmsg_len = NL_MESSAGE_SIZE;
+	hdr->nlmsg_type = NLMSG_DONE;
+	hdr->nlmsg_flags = 0;
+	hdr->nlmsg_seq = 0;
+	hdr->nlmsg_pid = getpid();
+
+	msg = (struct cn_msg *)NLMSG_DATA(hdr);
+	msg->id.idx = CN_IDX_PROC;
+	msg->id.val = CN_VAL_PROC;
+	msg->seq = 0;
+	msg->ack = 0;
+	msg->flags = 0;
+
+	msg->len = sizeof(struct proc_input);
+	((struct proc_input *)msg->data)->mcast_op =
+		PROC_CN_MCAST_NOTIFY;
+	((struct proc_input *)msg->data)->uexit_code = exit_code;
+
+	if (send(nl_sock, hdr, hdr->nlmsg_len, 0) == -1) {
+		perror("send failed");
+		return -errno;
+	}
+	return 0;
+}
+
+int notify_netlink_thread_exit(unsigned int exit_code)
+{
+	struct sockaddr_nl sa_nl;
+	int err = 0;
+	int nl_sock;
+
+	nl_sock = socket(PF_NETLINK, SOCK_DGRAM, NETLINK_CONNECTOR);
+
+	if (nl_sock == -1) {
+		perror("socket failed");
+		return -errno;
+	}
+
+	bzero(&sa_nl, sizeof(sa_nl));
+	sa_nl.nl_family = AF_NETLINK;
+	sa_nl.nl_groups = CN_IDX_PROC;
+	sa_nl.nl_pid    = gettid();
+
+	if (bind(nl_sock, (struct sockaddr *)&sa_nl, sizeof(sa_nl)) == -1) {
+		perror("bind failed");
+		close(nl_sock);
+		return -errno;
+	}
+
+	err = send_message(nl_sock, exit_code);
+
+	close(nl_sock);
+
+	if (err < 0)
+		return err;
+
+	return 0;
+}
-- 
2.46.0

On Thu, Oct 17, 2024 at 11:14:36AM -0700, Anjali Kulkarni wrote:
> Test to check if setting PROC_CN_MCAST_NOTIFY in proc connector API, allows
> a thread's non-zero exit status to be returned to proc_filter.
> 
> The threads.c program creates 2 child threads. 1st thread handles signal
> SIGSEGV, and 2nd thread needs to indicate some error condition (value 1)
> to the kernel, instead of using pthread_exit() with 1.
> 
> In both cases, child sends notify_netlink_thread_exit(exit_code) to kernel,
> to let kernel know it has exited abnormally with exit_code.
> 
> Compile:
>     make thread
>     make proc_filter
> Run:
>     ./threads
> 
> Signed-off-by: Anjali Kulkarni <anjali.k.kulkarni@oracle.com>
> ---
>  tools/testing/selftests/connector/Makefile    |  23 +-
>  .../testing/selftests/connector/proc_filter.c |  34 ++-
>  tools/testing/selftests/connector/thread.c    | 232 ++++++++++++++++++
>  .../selftests/connector/thread_filter.c       |  96 ++++++++
>  4 files changed, 378 insertions(+), 7 deletions(-)
>  create mode 100644 tools/testing/selftests/connector/thread.c
>  create mode 100644 tools/testing/selftests/connector/thread_filter.c
> 
> diff --git a/tools/testing/selftests/connector/Makefile b/tools/testing/selftests/connector/Makefile
> index 92188b9bac5c..bf335826bc3b 100644
> --- a/tools/testing/selftests/connector/Makefile
> +++ b/tools/testing/selftests/connector/Makefile
> @@ -1,5 +1,26 @@
>  # SPDX-License-Identifier: GPL-2.0
> -CFLAGS += -Wall $(KHDR_INCLUDES)
> +KERNEL="../../../.."
> +
> +CFLAGS += -Wall $(KHDR_INCLUDES) -I $(KERNEL)/include/uapi -I $(KERNEL)/include
> +
> +proc_filter: proc_filter.o
> +	cc proc_filter.o -o proc_filter
> +
> +proc_filter.o: proc_filter.c
> +	cc -c proc_filter.c -o proc_filter.o $(CFLAGS)
> +
> +thread: thread.o thread_filter.o
> +	cc thread.o thread_filter.o -o thread
> +
> +thread.o: thread.c $(DEPS)
> +		cc -c thread.c -o thread.o $(CFLAGS)
> +
> +thread_filter.o: thread_filter.c
> +		cc -c thread_filter.c -o thread_filter.o $(CFLAGS)
> +
> +define EXTRA_CLEAN
> +	rm *.o thread
> +endef
>  
>  TEST_GEN_PROGS = proc_filter
>  

I am a little confused by this, as it seems to result in user-space
code using kernel headers. Is that expected?

$ make -C tools/testing/selftests/connector
...
cc -c proc_filter.c -o proc_filter.o -Wall -isystem /home/horms/projects/linux/linux/tools/testing/selftests/../../../usr/include -I "../../../.."/include/uapi -I "../../../.."/include -D_GNU_SOURCE=
In file included from ../../../../include/uapi/linux/netlink.h:7,
                 from proc_filter.c:11:
../../../../include/uapi/linux/types.h:10:2: warning: #warning "Attempt to use kernel headers from user space, see https://kernelnewbies.org/KernelHeaders" [-Wcpp]
   10 | #warning "Attempt to use kernel headers from user space, see https://kernelnewbies.org/KernelHeaders"
      |  ^~~~~~~
...

> diff --git a/tools/testing/selftests/connector/thread.c b/tools/testing/selftests/connector/thread.c

...

> +static inline void init_threads(pthread_attr_t *attr)

Please don't use inline in .c files unless there is a demonstrable,
usually performance, reason to do so.

Likewise twice more in this patch and once in patch 1/3.

> +{
> +	int ret;
> +
> +	ret = pthread_attr_init(attr);
> +	if (ret != 0) {
> +		perror("pthread_attr_init failed");
> +		exit(ret);
> +	}
> +
> +	ret = pthread_attr_setdetachstate(attr, PTHREAD_CREATE_DETACHED);
> +	if (ret != 0) {
> +		perror("pthread_attr_setdetachstate failed");
> +		exit(ret);
> +	}
> +}

...

> On Oct 18, 2024, at 3:04 AM, Simon Horman <horms@kernel.org> wrote:
> 
> On Thu, Oct 17, 2024 at 11:14:36AM -0700, Anjali Kulkarni wrote:
>> Test to check if setting PROC_CN_MCAST_NOTIFY in proc connector API, allows
>> a thread's non-zero exit status to be returned to proc_filter.
>> 
>> The threads.c program creates 2 child threads. 1st thread handles signal
>> SIGSEGV, and 2nd thread needs to indicate some error condition (value 1)
>> to the kernel, instead of using pthread_exit() with 1.
>> 
>> In both cases, child sends notify_netlink_thread_exit(exit_code) to kernel,
>> to let kernel know it has exited abnormally with exit_code.
>> 
>> Compile:
>>    make thread
>>    make proc_filter
>> Run:
>>    ./threads
>> 
>> Signed-off-by: Anjali Kulkarni <anjali.k.kulkarni@oracle.com>
>> ---
>> tools/testing/selftests/connector/Makefile    |  23 +-
>> .../testing/selftests/connector/proc_filter.c |  34 ++-
>> tools/testing/selftests/connector/thread.c    | 232 ++++++++++++++++++
>> .../selftests/connector/thread_filter.c       |  96 ++++++++
>> 4 files changed, 378 insertions(+), 7 deletions(-)
>> create mode 100644 tools/testing/selftests/connector/thread.c
>> create mode 100644 tools/testing/selftests/connector/thread_filter.c
>> 
>> diff --git a/tools/testing/selftests/connector/Makefile b/tools/testing/selftests/connector/Makefile
>> index 92188b9bac5c..bf335826bc3b 100644
>> --- a/tools/testing/selftests/connector/Makefile
>> +++ b/tools/testing/selftests/connector/Makefile
>> @@ -1,5 +1,26 @@
>> # SPDX-License-Identifier: GPL-2.0
>> -CFLAGS += -Wall $(KHDR_INCLUDES)
>> +KERNEL="../../../.."
>> +
>> +CFLAGS += -Wall $(KHDR_INCLUDES) -I $(KERNEL)/include/uapi -I $(KERNEL)/include
>> +
>> +proc_filter: proc_filter.o
>> + cc proc_filter.o -o proc_filter
>> +
>> +proc_filter.o: proc_filter.c
>> + cc -c proc_filter.c -o proc_filter.o $(CFLAGS)
>> +
>> +thread: thread.o thread_filter.o
>> + cc thread.o thread_filter.o -o thread
>> +
>> +thread.o: thread.c $(DEPS)
>> + cc -c thread.c -o thread.o $(CFLAGS)
>> +
>> +thread_filter.o: thread_filter.c
>> + cc -c thread_filter.c -o thread_filter.o $(CFLAGS)
>> +
>> +define EXTRA_CLEAN
>> + rm *.o thread
>> +endef
>> 
>> TEST_GEN_PROGS = proc_filter
>> 
> 
> I am a little confused by this, as it seems to result in user-space
> code using kernel headers. Is that expected?

If I do not do this, then it’s not possible to run the selftests on a
system which does not have the built image installed. This allows me to
to test the selftest code in the build tree without having to install the build
kernel on the system, as it uses the header files in the build itself instead
of the ones installed on the system.
I can remove it if required.

Anjali

> 
> $ make -C tools/testing/selftests/connector
> ...
> cc -c proc_filter.c -o proc_filter.o -Wall -isystem /home/horms/projects/linux/linux/tools/testing/selftests/../../../usr/include -I "../../../.."/include/uapi -I "../../../.."/include -D_GNU_SOURCE=
> In file included from ../../../../include/uapi/linux/netlink.h:7,
>                 from proc_filter.c:11:
> ../../../../include/uapi/linux/types.h:10:2: warning: #warning "Attempt to use kernel headers from user space, see https://urldefense.com/v3/__https://kernelnewbies.org/KernelHeaders__;!!ACWV5N9M2RV99hQ!P3LdH9x7VmRu9salLGieUHe27YSyiYXyUoUdHV9Narf8UAdmXLVPyHFi41zJVKFAKVgo9eMevokFdRLwTH8p$ " [-Wcpp]
>   10 | #warning "Attempt to use kernel headers from user space, see https://urldefense.com/v3/__https://kernelnewbies.org/KernelHeaders__;!!ACWV5N9M2RV99hQ!P3LdH9x7VmRu9salLGieUHe27YSyiYXyUoUdHV9Narf8UAdmXLVPyHFi41zJVKFAKVgo9eMevokFdRLwTH8p$ "
>      |  ^~~~~~~
> ...
> 
>> diff --git a/tools/testing/selftests/connector/thread.c b/tools/testing/selftests/connector/thread.c
> 
> ...
> 
>> +static inline void init_threads(pthread_attr_t *attr)
> 
> Please don't use inline in .c files unless there is a demonstrable,
> usually performance, reason to do so.
> 
> Likewise twice more in this patch and once in patch 1/3.
> 
>> +{
>> + int ret;
>> +
>> + ret = pthread_attr_init(attr);
>> + if (ret != 0) {
>> + perror("pthread_attr_init failed");
>> + exit(ret);
>> + }
>> +
>> + ret = pthread_attr_setdetachstate(attr, PTHREAD_CREATE_DETACHED);
>> + if (ret != 0) {
>> + perror("pthread_attr_setdetachstate failed");
>> + exit(ret);
>> + }
>> +}
> 
> ...