From: Menglong Dong <imagedong@tencent.com> In the 1th patch, we do some code cleanup with replease 'sock->sk' with 'sk'. In the 2th patch, we add statistics for mptcp socket in use. And in the 3th patch, we add the testing for this commit. Changes since v4: - rebase to solve merge conflict Changes since v3: - rename MPTCP_DESTROIED to MPTCP_DESTROYED in the 2th patch Changes since v2: - add testing Changes since v1: - split the code cleanup into the 1th patch. - decrease the statistics for listening mptcp socket inuse with mptcp_listen_inuse_dec() - add MPTCP_DESTROIED flags to store if mptcp_destroy_common() was called on the msk. For fallback case, we need to decrease the statistics only once, and mptcp_destroy_common() can be called more than once. Menglong Dong (3): mptcp: introduce 'sk' to replace 'sock->sk' in mptcp_listen() mptcp: add statistics for mptcp socket in use selftest: mptcp: add test for mptcp socket in use net/mptcp/protocol.c | 35 ++++++++++++++++++----- net/mptcp/protocol.h | 1 + net/mptcp/subflow.c | 3 ++ tools/testing/selftests/net/mptcp/diag.sh | 28 ++++++++++++++++++ 4 files changed, 60 insertions(+), 7 deletions(-) -- 2.37.2 From: Menglong Dong <imagedong@tencent.com> 'sock->sk' is used frequently in mptcp_listen(). Therefore, we can introduce the 'sk' and replace 'sock->sk' with it. Signed-off-by: Menglong Dong <imagedong@tencent.com> --- net/mptcp/protocol.c | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/net/mptcp/protocol.c b/net/mptcp/protocol.c index XXXXXXX..XXXXXXX 100644 --- a/net/mptcp/protocol.c +++ b/net/mptcp/protocol.c @@ -XXX,XX +XXX,XX @@ static int mptcp_stream_connect(struct socket *sock, struct sockaddr *uaddr, static int mptcp_listen(struct socket *sock, int backlog) { struct mptcp_sock *msk = mptcp_sk(sock->sk); + struct sock *sk = sock->sk; struct socket *ssock; int err; pr_debug("msk=%p", msk); - lock_sock(sock->sk); + lock_sock(sk); ssock = __mptcp_nmpc_socket(msk); if (!ssock) { err = -EINVAL; @@ -XXX,XX +XXX,XX @@ static int mptcp_listen(struct socket *sock, int backlog) } mptcp_token_destroy(msk); - inet_sk_state_store(sock->sk, TCP_LISTEN); - sock_set_flag(sock->sk, SOCK_RCU_FREE); + inet_sk_state_store(sk, TCP_LISTEN); + sock_set_flag(sk, SOCK_RCU_FREE); err = ssock->ops->listen(ssock, backlog); - inet_sk_state_store(sock->sk, inet_sk_state_load(ssock->sk)); + inet_sk_state_store(sk, inet_sk_state_load(ssock->sk)); if (!err) - mptcp_copy_inaddrs(sock->sk, ssock->sk); + mptcp_copy_inaddrs(sk, ssock->sk); unlock: - release_sock(sock->sk); + release_sock(sk); return err; } -- 2.37.2 From: Menglong Dong <imagedong@tencent.com> Do the statistics of mptcp socket in use with sock_prot_inuse_add(). Therefore, we can get the count of used mptcp socket from /proc/net/protocols: & cat /proc/net/protocols protocol size sockets memory press maxhdr slab module cl co di ac io in de sh ss gs se re sp bi br ha uh gp em MPTCPv6 2048 0 0 no 0 yes kernel y n y y y y y y y y y y n n n y y y n MPTCP 1896 1 0 no 0 yes kernel y n y y y y y y y y y y n n n y y y n Signed-off-by: Menglong Dong <imagedong@tencent.com> --- v5: - rebase to solve merge conflict v4: - rename MPTCP_DESTROIED to MPTCP_DESTROYED v2: - decrease the statistics for listening mptcp socket inuse with mptcp_listen_inuse_dec() - add MPTCP_DESTROIED flags to store if mptcp_destroy_common() was called on the msk. For fallback case, we need to decrease the statistics only once, and mptcp_destroy_common() can be called more than once. --- net/mptcp/protocol.c | 22 +++++++++++++++++++++- net/mptcp/protocol.h | 1 + net/mptcp/subflow.c | 3 +++ 3 files changed, 25 insertions(+), 1 deletion(-) diff --git a/net/mptcp/protocol.c b/net/mptcp/protocol.c index XXXXXXX..XXXXXXX 100644 --- a/net/mptcp/protocol.c +++ b/net/mptcp/protocol.c @@ -XXX,XX +XXX,XX @@ static void mptcp_copy_inaddrs(struct sock *msk, const struct sock *ssk) inet_sk(msk)->inet_rcv_saddr = inet_sk(ssk)->inet_rcv_saddr; } +static void mptcp_listen_inuse_dec(struct sock *sk) +{ + struct mptcp_sock *msk = mptcp_sk(sk); + struct socket *ssock; + + ssock = __mptcp_nmpc_socket(msk); + if (ssock && inet_sk_state_load(ssock->sk) == TCP_LISTEN) + sock_prot_inuse_add(sock_net(sk), sk->sk_prot, -1); +} + static int mptcp_disconnect(struct sock *sk, int flags) { struct mptcp_sock *msk = mptcp_sk(sk); @@ -XXX,XX +XXX,XX @@ static int mptcp_disconnect(struct sock *sk, int flags) if (mptcp_sk(sk)->token) mptcp_event(MPTCP_EVENT_CLOSED, mptcp_sk(sk), NULL, GFP_KERNEL); + mptcp_listen_inuse_dec(sk); /* msk->subflow is still intact, the following will not free the first * subflow */ @@ -XXX,XX +XXX,XX @@ void mptcp_destroy_common(struct mptcp_sock *msk, unsigned int flags) skb_rbtree_purge(&msk->out_of_order_queue); mptcp_data_unlock(sk); + if ((__mptcp_check_fallback(msk) && + !test_and_set_bit(MPTCP_DESTROYED, &msk->flags)) || + !sk_unhashed(sk)) + sock_prot_inuse_add(sock_net(sk), sk->sk_prot, -1); + /* move all the rx fwd alloc into the sk_mem_reclaim_final in * inet_sock_destruct() will dispose it */ @@ -XXX,XX +XXX,XX @@ static void mptcp_destroy(struct sock *sk) { struct mptcp_sock *msk = mptcp_sk(sk); + mptcp_listen_inuse_dec(sk); /* clears msk->subflow, allowing the following to close * even the initial subflow */ @@ -XXX,XX +XXX,XX @@ static int mptcp_connect(struct sock *sk, struct sockaddr *uaddr, int addr_len) mptcp_token_destroy(msk); inet_sk_state_store(sk, TCP_SYN_SENT); + sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1); subflow = mptcp_subflow_ctx(ssock->sk); #ifdef CONFIG_TCP_MD5SIG /* no MPTCP if MD5SIG is enabled on this socket or we may run out of @@ -XXX,XX +XXX,XX @@ static int mptcp_listen(struct socket *sock, int backlog) err = ssock->ops->listen(ssock, backlog); inet_sk_state_store(sk, inet_sk_state_load(ssock->sk)); - if (!err) + if (!err) { + sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1); mptcp_copy_inaddrs(sk, ssock->sk); + } unlock: release_sock(sk); diff --git a/net/mptcp/protocol.h b/net/mptcp/protocol.h index XXXXXXX..XXXXXXX 100644 --- a/net/mptcp/protocol.h +++ b/net/mptcp/protocol.h @@ -XXX,XX +XXX,XX @@ #define MPTCP_WORK_EOF 3 #define MPTCP_FALLBACK_DONE 4 #define MPTCP_WORK_CLOSE_SUBFLOW 5 +#define MPTCP_DESTROYED 6 /* MPTCP socket release cb flags */ #define MPTCP_PUSH_PENDING 1 diff --git a/net/mptcp/subflow.c b/net/mptcp/subflow.c index XXXXXXX..XXXXXXX 100644 --- a/net/mptcp/subflow.c +++ b/net/mptcp/subflow.c @@ -XXX,XX +XXX,XX @@ static struct sock *subflow_syn_recv_sock(const struct sock *sk, mptcp_sk(new_msk)->setsockopt_seq = ctx->setsockopt_seq; mptcp_pm_new_connection(mptcp_sk(new_msk), child, 1); mptcp_token_accept(subflow_req, mptcp_sk(new_msk)); + sock_prot_inuse_add(sock_net(new_msk), + new_msk->sk_prot, + 1); ctx->conn = new_msk; new_msk = NULL; -- 2.37.2 From: Menglong Dong <imagedong@tencent.com> Add the function chk_msk_inuse() to diag.sh, which is used to check the statistics of mptcp socket in use. As mptcp socket in listen state will be closed randomly after 'accept', we need to get the count of listening mptcp socket through 'ss' command. mptcp_connect command with '-r' flag seems won't exit after flush_pids, therefore we need consider this additional statistics. But after the second flush_pids, it will exit. All tests pass. Signed-off-by: Menglong Dong <imagedong@tencent.com> --- tools/testing/selftests/net/mptcp/diag.sh | 28 +++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/tools/testing/selftests/net/mptcp/diag.sh b/tools/testing/selftests/net/mptcp/diag.sh index XXXXXXX..XXXXXXX 100755 --- a/tools/testing/selftests/net/mptcp/diag.sh +++ b/tools/testing/selftests/net/mptcp/diag.sh @@ -XXX,XX +XXX,XX @@ chk_msk_listen() nr=$(ss -Ml $filter | wc -l) } +chk_msk_inuse() +{ + local nr listen_nr + local expected=$1 + + shift 1 + msg=$* + + nr=$(ip netns exec $ns awk '$1~/^MPTCP$/{print $3}' /proc/net/protocols) + listen_nr=$(ss -N $ns -Ml | grep -c LISTEN) + expected=$(($expected+$listen_nr)) + + printf "%-50s" "$msg" + + if [ $nr != $expected ]; then + echo "[ fail ] expected $expected found $nr" + ret=$test_cnt + else + echo "[ ok ]" + fi +} + # $1: ns, $2: port wait_local_port_listen() { @@ -XXX,XX +XXX,XX @@ wait_connected $ns 10000 chk_msk_nr 2 "after MPC handshake " chk_msk_remote_key_nr 2 "....chk remote_key" chk_msk_fallback_nr 0 "....chk no fallback" +chk_msk_inuse 2 "msk in use statistics" flush_pids +# with '-r' flag, client won't exit after flush_pids +chk_msk_inuse 1 "msk in use statistics" echo "a" | \ timeout ${timeout_test} \ @@ -XXX,XX +XXX,XX @@ for I in `seq 1 $NR_CLIENTS`; do done wait_msk_nr $((NR_CLIENTS*2)) "many msk socket present" +chk_msk_inuse $((NR_CLIENTS*2+1)) "msk in use statistics" flush_pids +chk_msk_inuse 0 "msk in use statistics" + exit $ret -- 2.37.2 From: Menglong Dong <imagedong@tencent.com> In the 1th patch, we do some code cleanup with replease 'sock->sk' with 'sk'. In the 2th patch, we add statistics for mptcp socket in use. In the 3th patch, we make mptcp_connect can exit when receive 'SIGUSR1' with '-r' flag. And in the 4th patch, we add the testing for this commit. Changes since v5: - introduce MPTCP_INUSE flag to store if msk is in use, as I find that it's not correct to check is a msk is in use by !sk_unhashed(sk) in mptcp_destroy_common(), because the token can be release in mptcp_check_fastclose() - add the 3th patch - reuse __chk_nr in 4th patch Changes since v4: - rebase to solve merge conflict Changes since v3: - rename MPTCP_DESTROIED to MPTCP_DESTROYED in the 2th patch Changes since v2: - add testing Changes since v1: - split the code cleanup into the 1th patch. - decrease the statistics for listening mptcp socket inuse with mptcp_listen_inuse_dec() - add MPTCP_DESTROIED flags to store if mptcp_destroy_common() was called on the msk. For fallback case, we need to decrease the statistics only once, and mptcp_destroy_common() can be called more than once. Menglong Dong (4): mptcp: introduce 'sk' to replace 'sock->sk' in mptcp_listen() mptcp: add statistics for mptcp socket in use selftest: mptcp: exit from copyfd_io_poll() when receive SIGUSR1 selftest: mptcp: add test for mptcp socket in use net/mptcp/protocol.c | 21 ++++++--- net/mptcp/protocol.h | 13 +++++ net/mptcp/subflow.c | 1 + tools/testing/selftests/net/mptcp/diag.sh | 47 +++++++++++++++++-- .../selftests/net/mptcp/mptcp_connect.c | 4 +- 5 files changed, 72 insertions(+), 14 deletions(-) -- 2.37.2 From: Menglong Dong <imagedong@tencent.com> 'sock->sk' is used frequently in mptcp_listen(). Therefore, we can introduce the 'sk' and replace 'sock->sk' with it. Signed-off-by: Menglong Dong <imagedong@tencent.com> --- net/mptcp/protocol.c | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/net/mptcp/protocol.c b/net/mptcp/protocol.c index XXXXXXX..XXXXXXX 100644 --- a/net/mptcp/protocol.c +++ b/net/mptcp/protocol.c @@ -XXX,XX +XXX,XX @@ static int mptcp_stream_connect(struct socket *sock, struct sockaddr *uaddr, static int mptcp_listen(struct socket *sock, int backlog) { struct mptcp_sock *msk = mptcp_sk(sock->sk); + struct sock *sk = sock->sk; struct socket *ssock; int err; pr_debug("msk=%p", msk); - lock_sock(sock->sk); + lock_sock(sk); ssock = __mptcp_nmpc_socket(msk); if (!ssock) { err = -EINVAL; @@ -XXX,XX +XXX,XX @@ static int mptcp_listen(struct socket *sock, int backlog) } mptcp_token_destroy(msk); - inet_sk_state_store(sock->sk, TCP_LISTEN); - sock_set_flag(sock->sk, SOCK_RCU_FREE); + inet_sk_state_store(sk, TCP_LISTEN); + sock_set_flag(sk, SOCK_RCU_FREE); err = ssock->ops->listen(ssock, backlog); - inet_sk_state_store(sock->sk, inet_sk_state_load(ssock->sk)); + inet_sk_state_store(sk, inet_sk_state_load(ssock->sk)); if (!err) - mptcp_copy_inaddrs(sock->sk, ssock->sk); + mptcp_copy_inaddrs(sk, ssock->sk); unlock: - release_sock(sock->sk); + release_sock(sk); return err; } -- 2.37.2 From: Menglong Dong <imagedong@tencent.com> Do the statistics of mptcp socket in use with sock_prot_inuse_add(). Therefore, we can get the count of used mptcp socket from /proc/net/protocols: & cat /proc/net/protocols protocol size sockets memory press maxhdr slab module cl co di ac io in de sh ss gs se re sp bi br ha uh gp em MPTCPv6 2048 0 0 no 0 yes kernel y n y y y y y y y y y y n n n y y y n MPTCP 1896 1 0 no 0 yes kernel y n y y y y y y y y y y n n n y y y n Signed-off-by: Menglong Dong <imagedong@tencent.com> --- v6: - introduce the 'MPTCP_INUSE' flag and check if msk is in use by it v5: - rebase to solve merge conflict v4: - rename MPTCP_DESTROIED to MPTCP_DESTROYED v2: - decrease the statistics for listening mptcp socket inuse with mptcp_listen_inuse_dec() - add MPTCP_DESTROIED flags to store if mptcp_destroy_common() was called on the msk. For fallback case, we need to decrease the statistics only once, and mptcp_destroy_common() can be called more than once. --- net/mptcp/protocol.c | 8 +++++++- net/mptcp/protocol.h | 13 +++++++++++++ net/mptcp/subflow.c | 1 + 3 files changed, 21 insertions(+), 1 deletion(-) diff --git a/net/mptcp/protocol.c b/net/mptcp/protocol.c index XXXXXXX..XXXXXXX 100644 --- a/net/mptcp/protocol.c +++ b/net/mptcp/protocol.c @@ -XXX,XX +XXX,XX @@ struct sock *mptcp_sk_clone(const struct sock *sk, msk->snd_una = msk->write_seq; msk->wnd_end = msk->snd_nxt + req->rsk_rcv_wnd; msk->setsockopt_seq = mptcp_sk(sk)->setsockopt_seq; + clear_bit(MPTCP_INUSE, &msk->flags); mptcp_init_sched(msk, mptcp_sk(sk)->sched); if (mp_opt->suboptions & OPTIONS_MPTCP_MPC) { @@ -XXX,XX +XXX,XX @@ void mptcp_destroy_common(struct mptcp_sock *msk, unsigned int flags) skb_rbtree_purge(&msk->out_of_order_queue); mptcp_data_unlock(sk); + mptcp_inuse_dec(sk); + /* move all the rx fwd alloc into the sk_mem_reclaim_final in * inet_sock_destruct() will dispose it */ @@ -XXX,XX +XXX,XX @@ static int mptcp_connect(struct sock *sk, struct sockaddr *uaddr, int addr_len) mptcp_token_destroy(msk); inet_sk_state_store(sk, TCP_SYN_SENT); + mptcp_inuse_inc(sk); subflow = mptcp_subflow_ctx(ssock->sk); #ifdef CONFIG_TCP_MD5SIG /* no MPTCP if MD5SIG is enabled on this socket or we may run out of @@ -XXX,XX +XXX,XX @@ static int mptcp_listen(struct socket *sock, int backlog) err = ssock->ops->listen(ssock, backlog); inet_sk_state_store(sk, inet_sk_state_load(ssock->sk)); - if (!err) + if (!err) { + mptcp_inuse_inc(sk); mptcp_copy_inaddrs(sk, ssock->sk); + } unlock: release_sock(sk); diff --git a/net/mptcp/protocol.h b/net/mptcp/protocol.h index XXXXXXX..XXXXXXX 100644 --- a/net/mptcp/protocol.h +++ b/net/mptcp/protocol.h @@ -XXX,XX +XXX,XX @@ #define MPTCP_WORK_EOF 3 #define MPTCP_FALLBACK_DONE 4 #define MPTCP_WORK_CLOSE_SUBFLOW 5 +#define MPTCP_INUSE 6 /* MPTCP socket release cb flags */ #define MPTCP_PUSH_PENDING 1 @@ -XXX,XX +XXX,XX @@ static inline struct mptcp_data_frag *mptcp_rtx_head(const struct sock *sk) return list_first_entry_or_null(&msk->rtx_queue, struct mptcp_data_frag, list); } +static inline void mptcp_inuse_inc(const struct sock *sk) +{ + if (!test_and_set_bit(MPTCP_INUSE, &mptcp_sk(sk)->flags)) + sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1); +} + +static inline void mptcp_inuse_dec(const struct sock *sk) +{ + if (test_and_clear_bit(MPTCP_INUSE, &mptcp_sk(sk)->flags)) + sock_prot_inuse_add(sock_net(sk), sk->sk_prot, -1); +} + struct csum_pseudo_header { __be64 data_seq; __be32 subflow_seq; diff --git a/net/mptcp/subflow.c b/net/mptcp/subflow.c index XXXXXXX..XXXXXXX 100644 --- a/net/mptcp/subflow.c +++ b/net/mptcp/subflow.c @@ -XXX,XX +XXX,XX @@ static struct sock *subflow_syn_recv_sock(const struct sock *sk, mptcp_sk(new_msk)->setsockopt_seq = ctx->setsockopt_seq; mptcp_pm_new_connection(mptcp_sk(new_msk), child, 1); mptcp_token_accept(subflow_req, mptcp_sk(new_msk)); + mptcp_inuse_inc(new_msk); ctx->conn = new_msk; new_msk = NULL; -- 2.37.2 From: Menglong Dong <imagedong@tencent.com> For now, mptcp_connect won't exit after receiving the 'SIGUSR1' signal if '-r' is set. Fix this by skipping poll and sleep in copyfd_io_poll() if 'quit' is set. Signed-off-by: Menglong Dong <imagedong@tencent.com> --- tools/testing/selftests/net/mptcp/mptcp_connect.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/testing/selftests/net/mptcp/mptcp_connect.c b/tools/testing/selftests/net/mptcp/mptcp_connect.c index XXXXXXX..XXXXXXX 100644 --- a/tools/testing/selftests/net/mptcp/mptcp_connect.c +++ b/tools/testing/selftests/net/mptcp/mptcp_connect.c @@ -XXX,XX +XXX,XX @@ static int copyfd_io_poll(int infd, int peerfd, int outfd, bool *in_closed_after char rbuf[8192]; ssize_t len; - if (fds.events == 0) + if (fds.events == 0 || quit) break; switch (poll(&fds, 1, poll_timeout)) { @@ -XXX,XX +XXX,XX @@ static int copyfd_io_poll(int infd, int peerfd, int outfd, bool *in_closed_after } /* leave some time for late join/announce */ - if (cfg_remove) + if (cfg_remove && !quit) usleep(cfg_wait); return 0; -- 2.37.2 From: Menglong Dong <imagedong@tencent.com> Add the function chk_msk_inuse() to diag.sh, which is used to check the statistics of mptcp socket in use. As mptcp socket in listen state will be closed randomly after 'accept', we need to get the count of listening mptcp socket through 'ss' command. All tests pass. Signed-off-by: Menglong Dong <imagedong@tencent.com> --- v6: - reuse __chk_nr to check the number of msk in use --- tools/testing/selftests/net/mptcp/diag.sh | 47 ++++++++++++++++++++--- 1 file changed, 42 insertions(+), 5 deletions(-) diff --git a/tools/testing/selftests/net/mptcp/diag.sh b/tools/testing/selftests/net/mptcp/diag.sh index XXXXXXX..XXXXXXX 100755 --- a/tools/testing/selftests/net/mptcp/diag.sh +++ b/tools/testing/selftests/net/mptcp/diag.sh @@ -XXX,XX +XXX,XX @@ if [ $? -ne 0 ];then exit $ksft_skip fi +get_msk_inuse() +{ + ip netns exec $ns cat /proc/net/protocols | awk '$1~/^MPTCP$/{print $3}' +} + __chk_nr() { - local condition="$1" + local command="$1" local expected=$2 local msg nr shift 2 msg=$* - nr=$(ss -inmHMN $ns | $condition) + nr=$(eval $command) printf "%-50s" "$msg" if [ $nr != $expected ]; then @@ -XXX,XX +XXX,XX @@ __chk_nr() test_cnt=$((test_cnt+1)) } +__chk_msk_nr() +{ + local condition=$1 + shift 1 + + __chk_nr "ss -inmHMN $ns | $condition" $* +} + chk_msk_nr() { - __chk_nr "grep -c token:" $* + __chk_msk_nr "grep -c token:" $* } wait_msk_nr() @@ -XXX,XX +XXX,XX @@ wait_msk_nr() chk_msk_fallback_nr() { - __chk_nr "grep -c fallback" $* + __chk_msk_nr "grep -c fallback" $* } chk_msk_remote_key_nr() { - __chk_nr "grep -c remote_key" $* + __chk_msk_nr "grep -c remote_key" $* } __chk_listen() @@ -XXX,XX +XXX,XX @@ chk_msk_listen() nr=$(ss -Ml $filter | wc -l) } +chk_msk_inuse() +{ + local expected=$1 + local listen_nr + + listen_nr=$(ss -N $ns -Ml | grep -c LISTEN) + expected=$(($expected+$listen_nr)) + shift 1 + + for i in $(seq 10); do + if [ $(get_msk_inuse) -eq $expected ];then + break + fi + sleep 0.1 + done + + __chk_nr get_msk_inuse $expected $* +} + # $1: ns, $2: port wait_local_port_listen() { @@ -XXX,XX +XXX,XX @@ wait_connected $ns 10000 chk_msk_nr 2 "after MPC handshake " chk_msk_remote_key_nr 2 "....chk remote_key" chk_msk_fallback_nr 0 "....chk no fallback" +chk_msk_inuse 2 "chk 2 msk in use" flush_pids +chk_msk_inuse 0 "chk 0 msk in use after flush" echo "a" | \ timeout ${timeout_test} \ @@ -XXX,XX +XXX,XX @@ for I in `seq 1 $NR_CLIENTS`; do done wait_msk_nr $((NR_CLIENTS*2)) "many msk socket present" +chk_msk_inuse $((NR_CLIENTS*2)) "chk many msk in use" flush_pids +chk_msk_inuse 0 "chk 0 msk in use after flush" + exit $ret -- 2.37.2