SMC-R tests the viability of link by sending out TEST_LINK LLC
messages over RoCE fabric when connections on link have been
idle for a time longer than keepalive interval (testlink time).
But using tcp_keepalive_time as testlink time maybe not quite
suitable because it is default no less than two hours[1], which
is too long for single link to find peer dead. The active host
will still use peer-dead link (QP) sending messages, and can't
find out until get IB_WC_RETRY_EXC_ERR error CQEs, which takes
more time than TEST_LINK timeout (SMC_LLC_WAIT_TIME) normally.
So this patch introduces a independent sysctl for SMC-R to set
link keepalive time, in order to detect link down in time. The
default value is 30 seconds.
[1] https://www.rfc-editor.org/rfc/rfc1122#page-101
Signed-off-by: Wen Gu <guwen@linux.alibaba.com>
---
Documentation/networking/smc-sysctl.rst | 7 +++++++
include/net/netns/smc.h | 1 +
net/smc/smc_llc.c | 2 +-
net/smc/smc_llc.h | 1 +
net/smc/smc_sysctl.c | 14 ++++++++++++++
5 files changed, 24 insertions(+), 1 deletion(-)
diff --git a/Documentation/networking/smc-sysctl.rst b/Documentation/networking/smc-sysctl.rst
index 742e90e..f8c3d59 100644
--- a/Documentation/networking/smc-sysctl.rst
+++ b/Documentation/networking/smc-sysctl.rst
@@ -34,3 +34,10 @@ smcr_buf_type - INTEGER
- 1 - Use virtually contiguous buffers
- 2 - Mixed use of the two types. Try physically contiguous buffers first.
If not available, use virtually contiguous buffers then.
+
+smcr_testlink_time - INTEGER
+ How frequently SMC-R link sends out TEST_LINK LLC messages to confirm
+ viability, after the last activity of connections on it. The maximum
+ value is (INT_MAX / HZ) seconds, the minimum value is 1 second.
+
+ Default: 30 seconds.
diff --git a/include/net/netns/smc.h b/include/net/netns/smc.h
index 2adbe2b..d295e2c 100644
--- a/include/net/netns/smc.h
+++ b/include/net/netns/smc.h
@@ -19,5 +19,6 @@ struct netns_smc {
#endif
unsigned int sysctl_autocorking_size;
unsigned int sysctl_smcr_buf_type;
+ int sysctl_smcr_testlink_time;
};
#endif
diff --git a/net/smc/smc_llc.c b/net/smc/smc_llc.c
index 175026a..388bd2e 100644
--- a/net/smc/smc_llc.c
+++ b/net/smc/smc_llc.c
@@ -2127,7 +2127,7 @@ void smc_llc_lgr_init(struct smc_link_group *lgr, struct smc_sock *smc)
init_waitqueue_head(&lgr->llc_flow_waiter);
init_waitqueue_head(&lgr->llc_msg_waiter);
mutex_init(&lgr->llc_conf_mutex);
- lgr->llc_testlink_time = READ_ONCE(net->ipv4.sysctl_tcp_keepalive_time);
+ lgr->llc_testlink_time = READ_ONCE(net->smc.sysctl_smcr_testlink_time) * HZ;
}
/* called after lgr was removed from lgr_list */
diff --git a/net/smc/smc_llc.h b/net/smc/smc_llc.h
index 4404e52..1de9a29 100644
--- a/net/smc/smc_llc.h
+++ b/net/smc/smc_llc.h
@@ -19,6 +19,7 @@
#define SMC_LLC_WAIT_FIRST_TIME (5 * HZ)
#define SMC_LLC_WAIT_TIME (2 * HZ)
+#define SMC_LLC_TESTLINK_DEFAULT_TIME 30
enum smc_llc_reqresp {
SMC_LLC_REQ,
diff --git a/net/smc/smc_sysctl.c b/net/smc/smc_sysctl.c
index 0613868..7f68520 100644
--- a/net/smc/smc_sysctl.c
+++ b/net/smc/smc_sysctl.c
@@ -16,8 +16,12 @@
#include "smc.h"
#include "smc_core.h"
+#include "smc_llc.h"
#include "smc_sysctl.h"
+static int smcr_testlink_time_min = 1;
+static int smcr_testlink_time_max = (INT_MAX / HZ);
+
static struct ctl_table smc_table[] = {
{
.procname = "autocorking_size",
@@ -35,6 +39,15 @@
.extra1 = SYSCTL_ZERO,
.extra2 = SYSCTL_TWO,
},
+ {
+ .procname = "smcr_testlink_time",
+ .data = &init_net.smc.sysctl_smcr_testlink_time,
+ .maxlen = sizeof(int),
+ .mode = 0644,
+ .proc_handler = proc_dointvec_minmax,
+ .extra1 = &smcr_testlink_time_min,
+ .extra2 = &smcr_testlink_time_max,
+ },
{ }
};
@@ -60,6 +73,7 @@ int __net_init smc_sysctl_net_init(struct net *net)
net->smc.sysctl_autocorking_size = SMC_AUTOCORKING_DEFAULT_SIZE;
net->smc.sysctl_smcr_buf_type = SMCR_PHYS_CONT_BUFS;
+ net->smc.sysctl_smcr_testlink_time = SMC_LLC_TESTLINK_DEFAULT_TIME;
return 0;
--
1.8.3.1On Tue, Sep 20, 2022 at 10:53:54AM +0800, Wen Gu wrote:
>SMC-R tests the viability of link by sending out TEST_LINK LLC
>messages over RoCE fabric when connections on link have been
>idle for a time longer than keepalive interval (testlink time).
>
>But using tcp_keepalive_time as testlink time maybe not quite
>suitable because it is default no less than two hours[1], which
>is too long for single link to find peer dead. The active host
>will still use peer-dead link (QP) sending messages, and can't
>find out until get IB_WC_RETRY_EXC_ERR error CQEs, which takes
>more time than TEST_LINK timeout (SMC_LLC_WAIT_TIME) normally.
>
>So this patch introduces a independent sysctl for SMC-R to set
>link keepalive time, in order to detect link down in time. The
>default value is 30 seconds.
>
>[1] https://www.rfc-editor.org/rfc/rfc1122#page-101
>
>Signed-off-by: Wen Gu <guwen@linux.alibaba.com>
>---
> Documentation/networking/smc-sysctl.rst | 7 +++++++
> include/net/netns/smc.h | 1 +
> net/smc/smc_llc.c | 2 +-
> net/smc/smc_llc.h | 1 +
> net/smc/smc_sysctl.c | 14 ++++++++++++++
> 5 files changed, 24 insertions(+), 1 deletion(-)
>
>diff --git a/Documentation/networking/smc-sysctl.rst b/Documentation/networking/smc-sysctl.rst
>index 742e90e..f8c3d59 100644
>--- a/Documentation/networking/smc-sysctl.rst
>+++ b/Documentation/networking/smc-sysctl.rst
>@@ -34,3 +34,10 @@ smcr_buf_type - INTEGER
> - 1 - Use virtually contiguous buffers
> - 2 - Mixed use of the two types. Try physically contiguous buffers first.
> If not available, use virtually contiguous buffers then.
>+
>+smcr_testlink_time - INTEGER
>+ How frequently SMC-R link sends out TEST_LINK LLC messages to confirm
>+ viability, after the last activity of connections on it. The maximum
>+ value is (INT_MAX / HZ) seconds, the minimum value is 1 second.
>+
>+ Default: 30 seconds.
>diff --git a/include/net/netns/smc.h b/include/net/netns/smc.h
>index 2adbe2b..d295e2c 100644
>--- a/include/net/netns/smc.h
>+++ b/include/net/netns/smc.h
>@@ -19,5 +19,6 @@ struct netns_smc {
> #endif
> unsigned int sysctl_autocorking_size;
> unsigned int sysctl_smcr_buf_type;
>+ int sysctl_smcr_testlink_time;
> };
> #endif
>diff --git a/net/smc/smc_llc.c b/net/smc/smc_llc.c
>index 175026a..388bd2e 100644
>--- a/net/smc/smc_llc.c
>+++ b/net/smc/smc_llc.c
>@@ -2127,7 +2127,7 @@ void smc_llc_lgr_init(struct smc_link_group *lgr, struct smc_sock *smc)
> init_waitqueue_head(&lgr->llc_flow_waiter);
> init_waitqueue_head(&lgr->llc_msg_waiter);
> mutex_init(&lgr->llc_conf_mutex);
>- lgr->llc_testlink_time = READ_ONCE(net->ipv4.sysctl_tcp_keepalive_time);
>+ lgr->llc_testlink_time = READ_ONCE(net->smc.sysctl_smcr_testlink_time) * HZ;
> }
>
> /* called after lgr was removed from lgr_list */
>diff --git a/net/smc/smc_llc.h b/net/smc/smc_llc.h
>index 4404e52..1de9a29 100644
>--- a/net/smc/smc_llc.h
>+++ b/net/smc/smc_llc.h
>@@ -19,6 +19,7 @@
>
> #define SMC_LLC_WAIT_FIRST_TIME (5 * HZ)
> #define SMC_LLC_WAIT_TIME (2 * HZ)
>+#define SMC_LLC_TESTLINK_DEFAULT_TIME 30
I'm wondering why we don't follow the upper to macros using (30 * HZ) ?
>
> enum smc_llc_reqresp {
> SMC_LLC_REQ,
>diff --git a/net/smc/smc_sysctl.c b/net/smc/smc_sysctl.c
>index 0613868..7f68520 100644
>--- a/net/smc/smc_sysctl.c
>+++ b/net/smc/smc_sysctl.c
>@@ -16,8 +16,12 @@
>
> #include "smc.h"
> #include "smc_core.h"
>+#include "smc_llc.h"
> #include "smc_sysctl.h"
>
>+static int smcr_testlink_time_min = 1;
>+static int smcr_testlink_time_max = (INT_MAX / HZ);
>+
> static struct ctl_table smc_table[] = {
> {
> .procname = "autocorking_size",
>@@ -35,6 +39,15 @@
> .extra1 = SYSCTL_ZERO,
> .extra2 = SYSCTL_TWO,
> },
>+ {
>+ .procname = "smcr_testlink_time",
>+ .data = &init_net.smc.sysctl_smcr_testlink_time,
>+ .maxlen = sizeof(int),
>+ .mode = 0644,
>+ .proc_handler = proc_dointvec_minmax,
>+ .extra1 = &smcr_testlink_time_min,
>+ .extra2 = &smcr_testlink_time_max,
>+ },
> { }
> };
>
>@@ -60,6 +73,7 @@ int __net_init smc_sysctl_net_init(struct net *net)
>
> net->smc.sysctl_autocorking_size = SMC_AUTOCORKING_DEFAULT_SIZE;
> net->smc.sysctl_smcr_buf_type = SMCR_PHYS_CONT_BUFS;
>+ net->smc.sysctl_smcr_testlink_time = SMC_LLC_TESTLINK_DEFAULT_TIME;
>
> return 0;
>
>--
>1.8.3.1
On 2022/9/20 12:55, dust.li wrote:
> On Tue, Sep 20, 2022 at 10:53:54AM +0800, Wen Gu wrote:
>> SMC-R tests the viability of link by sending out TEST_LINK LLC
>> messages over RoCE fabric when connections on link have been
>> idle for a time longer than keepalive interval (testlink time).
>>
>> But using tcp_keepalive_time as testlink time maybe not quite
>> suitable because it is default no less than two hours[1], which
>> is too long for single link to find peer dead. The active host
>> will still use peer-dead link (QP) sending messages, and can't
>> find out until get IB_WC_RETRY_EXC_ERR error CQEs, which takes
>> more time than TEST_LINK timeout (SMC_LLC_WAIT_TIME) normally.
>>
>> So this patch introduces a independent sysctl for SMC-R to set
>> link keepalive time, in order to detect link down in time. The
>> default value is 30 seconds.
>>
>> [1] https://www.rfc-editor.org/rfc/rfc1122#page-101
>>
>> Signed-off-by: Wen Gu <guwen@linux.alibaba.com>
>> ---
>> /* called after lgr was removed from lgr_list */
>> diff --git a/net/smc/smc_llc.h b/net/smc/smc_llc.h
>> index 4404e52..1de9a29 100644
>> --- a/net/smc/smc_llc.h
>> +++ b/net/smc/smc_llc.h
>> @@ -19,6 +19,7 @@
>>
>> #define SMC_LLC_WAIT_FIRST_TIME (5 * HZ)
>> #define SMC_LLC_WAIT_TIME (2 * HZ)
>> +#define SMC_LLC_TESTLINK_DEFAULT_TIME 30
>
> I'm wondering why we don't follow the upper to macros using (30 * HZ) ?
>
Thanks for the reivew.
Because the value of sysctl_smcr_testlink_time is in seconds, and the value
of llc_testlink_time is jiffies.
I have thought about
1) using proc_dointvec_jiffies as sysctl's proc_handler just like TCP does.
But proc_dointvec_jiffies has no minimum limit, value 0 makes no sense for SMC testlink.
2) using proc_dointvec_ms_jiffies_minmax as proc_handler. But millisecond interval
seems expensive for SMC test link.
So, I choose to use proc_dointvec_minmax, make sysctl_smcr_testlink_time in
seconds, and convert to jiffies when assigning to llc_testlink_time.
Thanks,
Wen Gu.
On Tue, Sep 20, 2022 at 02:23:09PM +0800, Wen Gu wrote: > > >On 2022/9/20 12:55, dust.li wrote: > >> On Tue, Sep 20, 2022 at 10:53:54AM +0800, Wen Gu wrote: >> > SMC-R tests the viability of link by sending out TEST_LINK LLC >> > messages over RoCE fabric when connections on link have been >> > idle for a time longer than keepalive interval (testlink time). >> > >> > But using tcp_keepalive_time as testlink time maybe not quite >> > suitable because it is default no less than two hours[1], which >> > is too long for single link to find peer dead. The active host >> > will still use peer-dead link (QP) sending messages, and can't >> > find out until get IB_WC_RETRY_EXC_ERR error CQEs, which takes >> > more time than TEST_LINK timeout (SMC_LLC_WAIT_TIME) normally. >> > >> > So this patch introduces a independent sysctl for SMC-R to set >> > link keepalive time, in order to detect link down in time. The >> > default value is 30 seconds. >> > >> > [1] https://www.rfc-editor.org/rfc/rfc1122#page-101 >> > >> > Signed-off-by: Wen Gu <guwen@linux.alibaba.com> >> > --- > >> > /* called after lgr was removed from lgr_list */ >> > diff --git a/net/smc/smc_llc.h b/net/smc/smc_llc.h >> > index 4404e52..1de9a29 100644 >> > --- a/net/smc/smc_llc.h >> > +++ b/net/smc/smc_llc.h >> > @@ -19,6 +19,7 @@ >> > >> > #define SMC_LLC_WAIT_FIRST_TIME (5 * HZ) >> > #define SMC_LLC_WAIT_TIME (2 * HZ) >> > +#define SMC_LLC_TESTLINK_DEFAULT_TIME 30 >> >> I'm wondering why we don't follow the upper to macros using (30 * HZ) ? >> >Thanks for the reivew. > >Because the value of sysctl_smcr_testlink_time is in seconds, and the value >of llc_testlink_time is jiffies. > >I have thought about >1) using proc_dointvec_jiffies as sysctl's proc_handler just like TCP does. > But proc_dointvec_jiffies has no minimum limit, value 0 makes no sense for SMC testlink. Maybe 0 means disable the LLC TEST LINK ? >2) using proc_dointvec_ms_jiffies_minmax as proc_handler. But millisecond interval > seems expensive for SMC test link. > >So, I choose to use proc_dointvec_minmax, make sysctl_smcr_testlink_time in >seconds, and convert to jiffies when assigning to llc_testlink_time. If proc_dointvec_jiffies_minmax is really the problem, maybe you can write your own proc handler. Thanks
On 2022/9/20 16:21, dust.li wrote: > On Tue, Sep 20, 2022 at 02:23:09PM +0800, Wen Gu wrote: >> >> >> On 2022/9/20 12:55, dust.li wrote: >> >>> On Tue, Sep 20, 2022 at 10:53:54AM +0800, Wen Gu wrote: >>>> +++ b/net/smc/smc_llc.h >>>> @@ -19,6 +19,7 @@ >>>> >>>> #define SMC_LLC_WAIT_FIRST_TIME (5 * HZ) >>>> #define SMC_LLC_WAIT_TIME (2 * HZ) >>>> +#define SMC_LLC_TESTLINK_DEFAULT_TIME 30 >>> >>> I'm wondering why we don't follow the upper to macros using (30 * HZ) ? >>> >> Thanks for the reivew. >> >> Because the value of sysctl_smcr_testlink_time is in seconds, and the value >> of llc_testlink_time is jiffies. >> >> I have thought about >> 1) using proc_dointvec_jiffies as sysctl's proc_handler just like TCP does. >> But proc_dointvec_jiffies has no minimum limit, value 0 makes no sense for SMC testlink. > > Maybe 0 means disable the LLC TEST LINK ? > > >> 2) using proc_dointvec_ms_jiffies_minmax as proc_handler. But millisecond interval >> seems expensive for SMC test link. >> >> So, I choose to use proc_dointvec_minmax, make sysctl_smcr_testlink_time in >> seconds, and convert to jiffies when assigning to llc_testlink_time. > > If proc_dointvec_jiffies_minmax is really the problem, maybe you can > write your own proc handler. > Oops, I didn't noticed that value 0 means disabling LLC testlink in smc_llc_link_active(). So no need to set the minimum limit and proc_dointvec_jiffies will be fine. I will send a v2 to improve it. Thanks, Wen Gu
On Tue, Sep 20, 2022 at 10:53:54AM +0800, Wen Gu wrote:
> SMC-R tests the viability of link by sending out TEST_LINK LLC
> messages over RoCE fabric when connections on link have been
> idle for a time longer than keepalive interval (testlink time).
>
> But using tcp_keepalive_time as testlink time maybe not quite
> suitable because it is default no less than two hours[1], which
> is too long for single link to find peer dead. The active host
> will still use peer-dead link (QP) sending messages, and can't
> find out until get IB_WC_RETRY_EXC_ERR error CQEs, which takes
> more time than TEST_LINK timeout (SMC_LLC_WAIT_TIME) normally.
>
> So this patch introduces a independent sysctl for SMC-R to set
> link keepalive time, in order to detect link down in time. The
> default value is 30 seconds.
>
> [1] https://www.rfc-editor.org/rfc/rfc1122#page-101
>
> Signed-off-by: Wen Gu <guwen@linux.alibaba.com>
LGTM.
Reviewed-by: Tony Lu <tonylu@linux.alibaba.com>
> ---
> Documentation/networking/smc-sysctl.rst | 7 +++++++
> include/net/netns/smc.h | 1 +
> net/smc/smc_llc.c | 2 +-
> net/smc/smc_llc.h | 1 +
> net/smc/smc_sysctl.c | 14 ++++++++++++++
> 5 files changed, 24 insertions(+), 1 deletion(-)
>
> diff --git a/Documentation/networking/smc-sysctl.rst b/Documentation/networking/smc-sysctl.rst
> index 742e90e..f8c3d59 100644
> --- a/Documentation/networking/smc-sysctl.rst
> +++ b/Documentation/networking/smc-sysctl.rst
> @@ -34,3 +34,10 @@ smcr_buf_type - INTEGER
> - 1 - Use virtually contiguous buffers
> - 2 - Mixed use of the two types. Try physically contiguous buffers first.
> If not available, use virtually contiguous buffers then.
> +
> +smcr_testlink_time - INTEGER
> + How frequently SMC-R link sends out TEST_LINK LLC messages to confirm
> + viability, after the last activity of connections on it. The maximum
> + value is (INT_MAX / HZ) seconds, the minimum value is 1 second.
> +
> + Default: 30 seconds.
> diff --git a/include/net/netns/smc.h b/include/net/netns/smc.h
> index 2adbe2b..d295e2c 100644
> --- a/include/net/netns/smc.h
> +++ b/include/net/netns/smc.h
> @@ -19,5 +19,6 @@ struct netns_smc {
> #endif
> unsigned int sysctl_autocorking_size;
> unsigned int sysctl_smcr_buf_type;
> + int sysctl_smcr_testlink_time;
> };
> #endif
> diff --git a/net/smc/smc_llc.c b/net/smc/smc_llc.c
> index 175026a..388bd2e 100644
> --- a/net/smc/smc_llc.c
> +++ b/net/smc/smc_llc.c
> @@ -2127,7 +2127,7 @@ void smc_llc_lgr_init(struct smc_link_group *lgr, struct smc_sock *smc)
> init_waitqueue_head(&lgr->llc_flow_waiter);
> init_waitqueue_head(&lgr->llc_msg_waiter);
> mutex_init(&lgr->llc_conf_mutex);
> - lgr->llc_testlink_time = READ_ONCE(net->ipv4.sysctl_tcp_keepalive_time);
> + lgr->llc_testlink_time = READ_ONCE(net->smc.sysctl_smcr_testlink_time) * HZ;
> }
>
> /* called after lgr was removed from lgr_list */
> diff --git a/net/smc/smc_llc.h b/net/smc/smc_llc.h
> index 4404e52..1de9a29 100644
> --- a/net/smc/smc_llc.h
> +++ b/net/smc/smc_llc.h
> @@ -19,6 +19,7 @@
>
> #define SMC_LLC_WAIT_FIRST_TIME (5 * HZ)
> #define SMC_LLC_WAIT_TIME (2 * HZ)
> +#define SMC_LLC_TESTLINK_DEFAULT_TIME 30
>
> enum smc_llc_reqresp {
> SMC_LLC_REQ,
> diff --git a/net/smc/smc_sysctl.c b/net/smc/smc_sysctl.c
> index 0613868..7f68520 100644
> --- a/net/smc/smc_sysctl.c
> +++ b/net/smc/smc_sysctl.c
> @@ -16,8 +16,12 @@
>
> #include "smc.h"
> #include "smc_core.h"
> +#include "smc_llc.h"
> #include "smc_sysctl.h"
>
> +static int smcr_testlink_time_min = 1;
> +static int smcr_testlink_time_max = (INT_MAX / HZ);
> +
> static struct ctl_table smc_table[] = {
> {
> .procname = "autocorking_size",
> @@ -35,6 +39,15 @@
> .extra1 = SYSCTL_ZERO,
> .extra2 = SYSCTL_TWO,
> },
> + {
> + .procname = "smcr_testlink_time",
> + .data = &init_net.smc.sysctl_smcr_testlink_time,
> + .maxlen = sizeof(int),
> + .mode = 0644,
> + .proc_handler = proc_dointvec_minmax,
> + .extra1 = &smcr_testlink_time_min,
> + .extra2 = &smcr_testlink_time_max,
> + },
> { }
> };
>
> @@ -60,6 +73,7 @@ int __net_init smc_sysctl_net_init(struct net *net)
>
> net->smc.sysctl_autocorking_size = SMC_AUTOCORKING_DEFAULT_SIZE;
> net->smc.sysctl_smcr_buf_type = SMCR_PHYS_CONT_BUFS;
> + net->smc.sysctl_smcr_testlink_time = SMC_LLC_TESTLINK_DEFAULT_TIME;
>
> return 0;
>
> --
> 1.8.3.1
© 2016 - 2026 Red Hat, Inc.