From: Alistair Francis <alistair.francis@wdc.com>
If the nvme_tcp_try_send() or nvme_tcp_try_recv() functions return
EKEYEXPIRED then the underlying TLS keys need to be updated. This occurs
on an KeyUpdate event.
If the NVMe Target (TLS server) initiates a KeyUpdate this patch will
allow the NVMe layer to process the KeyUpdate request and forward the
request to userspace. Userspace must then update the key to keep the
connection alive.
This patch allows us to handle the NVMe target sending a KeyUpdate
request without aborting the connection. At this time we don't support
initiating a KeyUpdate.
Link: https://datatracker.ietf.org/doc/html/rfc8446#section-4.6.3
Signed-off-by: Alistair Francis <alistair.francis@wdc.com>
---
v4:
- Remove all support for initiating KeyUpdate
- Don't call cancel_work() when updating keys
v3:
- Don't cancel existing handshake requests
v2:
- Don't change the state
- Use a helper function for KeyUpdates
- Continue sending in nvme_tcp_send_all() after a KeyUpdate
- Remove command message using recvmsg
drivers/nvme/host/tcp.c | 60 ++++++++++++++++++++++++++++++++++-------
1 file changed, 51 insertions(+), 9 deletions(-)
diff --git a/drivers/nvme/host/tcp.c b/drivers/nvme/host/tcp.c
index 2696bf97dfac..791e0cc91ad8 100644
--- a/drivers/nvme/host/tcp.c
+++ b/drivers/nvme/host/tcp.c
@@ -172,6 +172,7 @@ struct nvme_tcp_queue {
bool tls_enabled;
u32 rcv_crc;
u32 snd_crc;
+ key_serial_t user_session_id;
__le32 exp_ddgst;
__le32 recv_ddgst;
struct completion tls_complete;
@@ -858,7 +859,10 @@ static void nvme_tcp_handle_c2h_term(struct nvme_tcp_queue *queue,
static int nvme_tcp_recvmsg_pdu(struct nvme_tcp_queue *queue)
{
char *pdu = queue->pdu;
+ char cbuf[CMSG_LEN(sizeof(char))] = {};
struct msghdr msg = {
+ .msg_control = cbuf,
+ .msg_controllen = sizeof(cbuf),
.msg_flags = MSG_DONTWAIT,
};
struct kvec iov = {
@@ -873,12 +877,17 @@ static int nvme_tcp_recvmsg_pdu(struct nvme_tcp_queue *queue)
if (ret <= 0)
return ret;
+ hdr = queue->pdu;
+ if (hdr->type == TLS_HANDSHAKE_KEYUPDATE) {
+ dev_err(queue->ctrl->ctrl.device, "KeyUpdate message\n");
+ return 1;
+ }
+
queue->pdu_remaining -= ret;
queue->pdu_offset += ret;
if (queue->pdu_remaining)
return 0;
- hdr = queue->pdu;
if (unlikely(hdr->hlen != sizeof(struct nvme_tcp_rsp_pdu))) {
if (!nvme_tcp_recv_pdu_supported(hdr->type))
goto unsupported_pdu;
@@ -944,6 +953,7 @@ static int nvme_tcp_recvmsg_data(struct nvme_tcp_queue *queue)
struct request *rq =
nvme_cid_to_rq(nvme_tcp_tagset(queue), pdu->command_id);
struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq);
+ char cbuf[CMSG_LEN(sizeof(char))] = {};
if (nvme_tcp_recv_state(queue) != NVME_TCP_RECV_DATA)
return 0;
@@ -973,12 +983,14 @@ static int nvme_tcp_recvmsg_data(struct nvme_tcp_queue *queue)
memset(&msg, 0, sizeof(msg));
msg.msg_iter = req->iter;
msg.msg_flags = MSG_DONTWAIT;
+ msg.msg_control = cbuf,
+ msg.msg_controllen = sizeof(cbuf),
ret = sock_recvmsg(queue->sock, &msg, msg.msg_flags);
if (ret < 0) {
- dev_err(queue->ctrl->ctrl.device,
- "queue %d failed to receive request %#x data",
- nvme_tcp_queue_id(queue), rq->tag);
+ dev_dbg(queue->ctrl->ctrl.device,
+ "queue %d failed to receive request %#x data, %d",
+ nvme_tcp_queue_id(queue), rq->tag, ret);
return ret;
}
if (queue->data_digest)
@@ -1381,17 +1393,42 @@ static int nvme_tcp_try_recvmsg(struct nvme_tcp_queue *queue)
}
} while (result >= 0);
- if (result < 0 && result != -EAGAIN) {
+ if (result == -EKEYEXPIRED) {
+ return -EKEYEXPIRED;
+ } else if (result == -EAGAIN) {
+ return -EAGAIN;
+ } else if (result < 0) {
dev_err(queue->ctrl->ctrl.device,
"receive failed: %d\n", result);
queue->rd_enabled = false;
nvme_tcp_error_recovery(&queue->ctrl->ctrl);
- } else if (result == -EAGAIN)
- result = 0;
+ }
return result < 0 ? result : (queue->nr_cqe = nr_cqe);
}
+static void update_tls_keys(struct nvme_tcp_queue *queue)
+{
+ int qid = nvme_tcp_queue_id(queue);
+ int ret;
+
+ dev_dbg(queue->ctrl->ctrl.device,
+ "updating key for queue %d\n", qid);
+
+ flush_work(&(queue->ctrl->ctrl).async_event_work);
+
+ ret = nvme_tcp_start_tls(&(queue->ctrl->ctrl),
+ queue, queue->ctrl->ctrl.tls_pskid,
+ HANDSHAKE_KEY_UPDATE_TYPE_RECEIVED);
+
+ if (ret < 0) {
+ dev_err(queue->ctrl->ctrl.device,
+ "failed to update the keys %d\n", ret);
+ nvme_tcp_fail_request(queue->request);
+ nvme_tcp_done_send_req(queue);
+ }
+}
+
static void nvme_tcp_io_work(struct work_struct *w)
{
struct nvme_tcp_queue *queue =
@@ -1414,8 +1451,11 @@ static void nvme_tcp_io_work(struct work_struct *w)
result = nvme_tcp_try_recvmsg(queue);
if (result > 0)
pending = true;
- else if (unlikely(result < 0))
- return;
+ else if (unlikely(result < 0)) {
+ if (result == -EKEYEXPIRED)
+ update_tls_keys(queue);
+ break;
+ }
/* did we get some space after spending time in recv? */
if (nvme_tcp_queue_has_pending(queue) &&
@@ -1723,6 +1763,7 @@ static void nvme_tcp_tls_done(void *data, int status, key_serial_t pskid,
ctrl->ctrl.tls_pskid = key_serial(tls_key);
key_put(tls_key);
queue->tls_err = 0;
+ queue->user_session_id = user_session_id;
}
out_complete:
@@ -1752,6 +1793,7 @@ static int nvme_tcp_start_tls(struct nvme_ctrl *nctrl,
keyring = key_serial(nctrl->opts->keyring);
args.ta_keyring = keyring;
args.ta_timeout_ms = tls_handshake_timeout * 1000;
+ args.user_session_id = queue->user_session_id;
queue->tls_err = -EOPNOTSUPP;
init_completion(&queue->tls_complete);
if (keyupdate == HANDSHAKE_KEY_UPDATE_TYPE_UNSPEC)
--
2.51.0On 10/17/25 06:23, alistair23@gmail.com wrote:
> From: Alistair Francis <alistair.francis@wdc.com>
>
> If the nvme_tcp_try_send() or nvme_tcp_try_recv() functions return
> EKEYEXPIRED then the underlying TLS keys need to be updated. This occurs
> on an KeyUpdate event.
>
> If the NVMe Target (TLS server) initiates a KeyUpdate this patch will
> allow the NVMe layer to process the KeyUpdate request and forward the
> request to userspace. Userspace must then update the key to keep the
> connection alive.
>
> This patch allows us to handle the NVMe target sending a KeyUpdate
> request without aborting the connection. At this time we don't support
> initiating a KeyUpdate.
>
> Link: https://datatracker.ietf.org/doc/html/rfc8446#section-4.6.3
> Signed-off-by: Alistair Francis <alistair.francis@wdc.com>
> ---
> v4:
> - Remove all support for initiating KeyUpdate
> - Don't call cancel_work() when updating keys
> v3:
> - Don't cancel existing handshake requests
> v2:
> - Don't change the state
> - Use a helper function for KeyUpdates
> - Continue sending in nvme_tcp_send_all() after a KeyUpdate
> - Remove command message using recvmsg
>
> drivers/nvme/host/tcp.c | 60 ++++++++++++++++++++++++++++++++++-------
> 1 file changed, 51 insertions(+), 9 deletions(-)
>
> diff --git a/drivers/nvme/host/tcp.c b/drivers/nvme/host/tcp.c
> index 2696bf97dfac..791e0cc91ad8 100644
> --- a/drivers/nvme/host/tcp.c
> +++ b/drivers/nvme/host/tcp.c
> @@ -172,6 +172,7 @@ struct nvme_tcp_queue {
> bool tls_enabled;
> u32 rcv_crc;
> u32 snd_crc;
> + key_serial_t user_session_id;
> __le32 exp_ddgst;
> __le32 recv_ddgst;
> struct completion tls_complete;
> @@ -858,7 +859,10 @@ static void nvme_tcp_handle_c2h_term(struct nvme_tcp_queue *queue,
> static int nvme_tcp_recvmsg_pdu(struct nvme_tcp_queue *queue)
> {
> char *pdu = queue->pdu;
> + char cbuf[CMSG_LEN(sizeof(char))] = {};
> struct msghdr msg = {
> + .msg_control = cbuf,
> + .msg_controllen = sizeof(cbuf),
> .msg_flags = MSG_DONTWAIT,
> };
> struct kvec iov = {
> @@ -873,12 +877,17 @@ static int nvme_tcp_recvmsg_pdu(struct nvme_tcp_queue *queue)
> if (ret <= 0)
> return ret;
>
> + hdr = queue->pdu;
> + if (hdr->type == TLS_HANDSHAKE_KEYUPDATE) {
> + dev_err(queue->ctrl->ctrl.device, "KeyUpdate message\n");
> + return 1;
> + }
> +
> queue->pdu_remaining -= ret;
> queue->pdu_offset += ret;
> if (queue->pdu_remaining)
> return 0;
>
> - hdr = queue->pdu;
> if (unlikely(hdr->hlen != sizeof(struct nvme_tcp_rsp_pdu))) {
> if (!nvme_tcp_recv_pdu_supported(hdr->type))
> goto unsupported_pdu;
> @@ -944,6 +953,7 @@ static int nvme_tcp_recvmsg_data(struct nvme_tcp_queue *queue)
> struct request *rq =
> nvme_cid_to_rq(nvme_tcp_tagset(queue), pdu->command_id);
> struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq);
> + char cbuf[CMSG_LEN(sizeof(char))] = {};
>
> if (nvme_tcp_recv_state(queue) != NVME_TCP_RECV_DATA)
> return 0;
> @@ -973,12 +983,14 @@ static int nvme_tcp_recvmsg_data(struct nvme_tcp_queue *queue)
> memset(&msg, 0, sizeof(msg));
> msg.msg_iter = req->iter;
> msg.msg_flags = MSG_DONTWAIT;
> + msg.msg_control = cbuf,
> + msg.msg_controllen = sizeof(cbuf),
>
Watch out. This is the recvmsg bug Olga had been posting patches for.
Thing is, if there is a control message the networking code will place
the control message payload into the message buffer. But in doing so
it expects the message buffer to be an iovec, not a bio vec.
To handle this properly you'd need to _not_ set the control buffer,
but rather check for 'MSG_CTRUNC' in msg_flags upon return.
Then you have to setup a new message with msg_control set and
a suitable msg_len (5 bytes, wasn't it?) and re-issue recvmsg
with that message.
And keep fingers crossed that you don't get MSG_CTRUNC on every
call to recvmsg() ...
> ret = sock_recvmsg(queue->sock, &msg, msg.msg_flags);
> if (ret < 0) {
> - dev_err(queue->ctrl->ctrl.device,
> - "queue %d failed to receive request %#x data",
> - nvme_tcp_queue_id(queue), rq->tag);
> + dev_dbg(queue->ctrl->ctrl.device,
> + "queue %d failed to receive request %#x data, %d",
> + nvme_tcp_queue_id(queue), rq->tag, ret);
> return ret;
> }
> if (queue->data_digest)
> @@ -1381,17 +1393,42 @@ static int nvme_tcp_try_recvmsg(struct nvme_tcp_queue *queue)
> }
> } while (result >= 0);
>
> - if (result < 0 && result != -EAGAIN) {
> + if (result == -EKEYEXPIRED) {
> + return -EKEYEXPIRED;
> + } else if (result == -EAGAIN) {
> + return -EAGAIN;
> + } else if (result < 0) {
> dev_err(queue->ctrl->ctrl.device,
> "receive failed: %d\n", result);
> queue->rd_enabled = false;
> nvme_tcp_error_recovery(&queue->ctrl->ctrl);
> - } else if (result == -EAGAIN)
> - result = 0;
> + }
>
> return result < 0 ? result : (queue->nr_cqe = nr_cqe);
> }
>
> +static void update_tls_keys(struct nvme_tcp_queue *queue)
> +{
> + int qid = nvme_tcp_queue_id(queue);
> + int ret;
> +
> + dev_dbg(queue->ctrl->ctrl.device,
> + "updating key for queue %d\n", qid);
> +
> + flush_work(&(queue->ctrl->ctrl).async_event_work);
> +
> + ret = nvme_tcp_start_tls(&(queue->ctrl->ctrl),
> + queue, queue->ctrl->ctrl.tls_pskid,
> + HANDSHAKE_KEY_UPDATE_TYPE_RECEIVED);
> +
> + if (ret < 0) {
> + dev_err(queue->ctrl->ctrl.device,
> + "failed to update the keys %d\n", ret);
> + nvme_tcp_fail_request(queue->request);
> + nvme_tcp_done_send_req(queue);
> + }
> +}
> +
> static void nvme_tcp_io_work(struct work_struct *w)
> {
> struct nvme_tcp_queue *queue =
> @@ -1414,8 +1451,11 @@ static void nvme_tcp_io_work(struct work_struct *w)
> result = nvme_tcp_try_recvmsg(queue);
> if (result > 0)
> pending = true;
> - else if (unlikely(result < 0))
> - return;
> + else if (unlikely(result < 0)) {
> + if (result == -EKEYEXPIRED)
> + update_tls_keys(queue);
> + break;
> + }
>
> /* did we get some space after spending time in recv? */
> if (nvme_tcp_queue_has_pending(queue) &&
> @@ -1723,6 +1763,7 @@ static void nvme_tcp_tls_done(void *data, int status, key_serial_t pskid,
> ctrl->ctrl.tls_pskid = key_serial(tls_key);
> key_put(tls_key);
> queue->tls_err = 0;
> + queue->user_session_id = user_session_id;
Hmm. I wonder, do we need to store the generation number somewhere?
Currently the sysfs interface is completely oblivious that a key update
has happened. I really would like to have _some_ indicator there telling
us that a key update had happened, and the generation number would be
ideal here.
> }
>
> out_complete:
> @@ -1752,6 +1793,7 @@ static int nvme_tcp_start_tls(struct nvme_ctrl *nctrl,
> keyring = key_serial(nctrl->opts->keyring);
> args.ta_keyring = keyring;
> args.ta_timeout_ms = tls_handshake_timeout * 1000;
> + args.user_session_id = queue->user_session_id;
> queue->tls_err = -EOPNOTSUPP;
> init_completion(&queue->tls_complete);
> if (keyupdate == HANDSHAKE_KEY_UPDATE_TYPE_UNSPEC)
Chers,
Hannes
--
Dr. Hannes Reinecke Kernel Storage Architect
hare@suse.de +49 911 74053 688
SUSE Software Solutions GmbH, Frankenstr. 146, 90461 Nürnberg
HRB 36809 (AG Nürnberg), GF: I. Totev, A. McDonald, W. Knoblich
On Mon, Oct 20, 2025 at 4:22 PM Hannes Reinecke <hare@suse.de> wrote:
>
> On 10/17/25 06:23, alistair23@gmail.com wrote:
> > From: Alistair Francis <alistair.francis@wdc.com>
> >
> > If the nvme_tcp_try_send() or nvme_tcp_try_recv() functions return
> > EKEYEXPIRED then the underlying TLS keys need to be updated. This occurs
> > on an KeyUpdate event.
> >
> > If the NVMe Target (TLS server) initiates a KeyUpdate this patch will
> > allow the NVMe layer to process the KeyUpdate request and forward the
> > request to userspace. Userspace must then update the key to keep the
> > connection alive.
> >
> > This patch allows us to handle the NVMe target sending a KeyUpdate
> > request without aborting the connection. At this time we don't support
> > initiating a KeyUpdate.
> >
> > Link: https://datatracker.ietf.org/doc/html/rfc8446#section-4.6.3
> > Signed-off-by: Alistair Francis <alistair.francis@wdc.com>
> > ---
> > v4:
> > - Remove all support for initiating KeyUpdate
> > - Don't call cancel_work() when updating keys
> > v3:
> > - Don't cancel existing handshake requests
> > v2:
> > - Don't change the state
> > - Use a helper function for KeyUpdates
> > - Continue sending in nvme_tcp_send_all() after a KeyUpdate
> > - Remove command message using recvmsg
> >
> > drivers/nvme/host/tcp.c | 60 ++++++++++++++++++++++++++++++++++-------
> > 1 file changed, 51 insertions(+), 9 deletions(-)
> >
> > diff --git a/drivers/nvme/host/tcp.c b/drivers/nvme/host/tcp.c
> > index 2696bf97dfac..791e0cc91ad8 100644
> > --- a/drivers/nvme/host/tcp.c
> > +++ b/drivers/nvme/host/tcp.c
> > @@ -172,6 +172,7 @@ struct nvme_tcp_queue {
> > bool tls_enabled;
> > u32 rcv_crc;
> > u32 snd_crc;
> > + key_serial_t user_session_id;
> > __le32 exp_ddgst;
> > __le32 recv_ddgst;
> > struct completion tls_complete;
> > @@ -858,7 +859,10 @@ static void nvme_tcp_handle_c2h_term(struct nvme_tcp_queue *queue,
> > static int nvme_tcp_recvmsg_pdu(struct nvme_tcp_queue *queue)
> > {
> > char *pdu = queue->pdu;
> > + char cbuf[CMSG_LEN(sizeof(char))] = {};
> > struct msghdr msg = {
> > + .msg_control = cbuf,
> > + .msg_controllen = sizeof(cbuf),
> > .msg_flags = MSG_DONTWAIT,
> > };
> > struct kvec iov = {
> > @@ -873,12 +877,17 @@ static int nvme_tcp_recvmsg_pdu(struct nvme_tcp_queue *queue)
> > if (ret <= 0)
> > return ret;
> >
> > + hdr = queue->pdu;
> > + if (hdr->type == TLS_HANDSHAKE_KEYUPDATE) {
> > + dev_err(queue->ctrl->ctrl.device, "KeyUpdate message\n");
> > + return 1;
> > + }
> > +
> > queue->pdu_remaining -= ret;
> > queue->pdu_offset += ret;
> > if (queue->pdu_remaining)
> > return 0;
> >
> > - hdr = queue->pdu;
> > if (unlikely(hdr->hlen != sizeof(struct nvme_tcp_rsp_pdu))) {
> > if (!nvme_tcp_recv_pdu_supported(hdr->type))
> > goto unsupported_pdu;
> > @@ -944,6 +953,7 @@ static int nvme_tcp_recvmsg_data(struct nvme_tcp_queue *queue)
> > struct request *rq =
> > nvme_cid_to_rq(nvme_tcp_tagset(queue), pdu->command_id);
> > struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq);
> > + char cbuf[CMSG_LEN(sizeof(char))] = {};
> >
> > if (nvme_tcp_recv_state(queue) != NVME_TCP_RECV_DATA)
> > return 0;
> > @@ -973,12 +983,14 @@ static int nvme_tcp_recvmsg_data(struct nvme_tcp_queue *queue)
> > memset(&msg, 0, sizeof(msg));
> > msg.msg_iter = req->iter;
> > msg.msg_flags = MSG_DONTWAIT;
> > + msg.msg_control = cbuf,
> > + msg.msg_controllen = sizeof(cbuf),
> >
> Watch out. This is the recvmsg bug Olga had been posting patches for.
> Thing is, if there is a control message the networking code will place
> the control message payload into the message buffer. But in doing so
> it expects the message buffer to be an iovec, not a bio vec.
>
> To handle this properly you'd need to _not_ set the control buffer,
> but rather check for 'MSG_CTRUNC' in msg_flags upon return.
> Then you have to setup a new message with msg_control set and
> a suitable msg_len (5 bytes, wasn't it?) and re-issue recvmsg
> with that message.
Fixed
>
> And keep fingers crossed that you don't get MSG_CTRUNC on every
> call to recvmsg() ...
>
> > ret = sock_recvmsg(queue->sock, &msg, msg.msg_flags);
> > if (ret < 0) {
> > - dev_err(queue->ctrl->ctrl.device,
> > - "queue %d failed to receive request %#x data",
> > - nvme_tcp_queue_id(queue), rq->tag);
> > + dev_dbg(queue->ctrl->ctrl.device,
> > + "queue %d failed to receive request %#x data, %d",
> > + nvme_tcp_queue_id(queue), rq->tag, ret);
> > return ret;
> > }
> > if (queue->data_digest)
> > @@ -1381,17 +1393,42 @@ static int nvme_tcp_try_recvmsg(struct nvme_tcp_queue *queue)
> > }
> > } while (result >= 0);
> >
> > - if (result < 0 && result != -EAGAIN) {
> > + if (result == -EKEYEXPIRED) {
> > + return -EKEYEXPIRED;
> > + } else if (result == -EAGAIN) {
> > + return -EAGAIN;
> > + } else if (result < 0) {
> > dev_err(queue->ctrl->ctrl.device,
> > "receive failed: %d\n", result);
> > queue->rd_enabled = false;
> > nvme_tcp_error_recovery(&queue->ctrl->ctrl);
> > - } else if (result == -EAGAIN)
> > - result = 0;
> > + }
> >
> > return result < 0 ? result : (queue->nr_cqe = nr_cqe);
> > }
> >
> > +static void update_tls_keys(struct nvme_tcp_queue *queue)
> > +{
> > + int qid = nvme_tcp_queue_id(queue);
> > + int ret;
> > +
> > + dev_dbg(queue->ctrl->ctrl.device,
> > + "updating key for queue %d\n", qid);
> > +
> > + flush_work(&(queue->ctrl->ctrl).async_event_work);
> > +
> > + ret = nvme_tcp_start_tls(&(queue->ctrl->ctrl),
> > + queue, queue->ctrl->ctrl.tls_pskid,
> > + HANDSHAKE_KEY_UPDATE_TYPE_RECEIVED);
> > +
> > + if (ret < 0) {
> > + dev_err(queue->ctrl->ctrl.device,
> > + "failed to update the keys %d\n", ret);
> > + nvme_tcp_fail_request(queue->request);
> > + nvme_tcp_done_send_req(queue);
> > + }
> > +}
> > +
> > static void nvme_tcp_io_work(struct work_struct *w)
> > {
> > struct nvme_tcp_queue *queue =
> > @@ -1414,8 +1451,11 @@ static void nvme_tcp_io_work(struct work_struct *w)
> > result = nvme_tcp_try_recvmsg(queue);
> > if (result > 0)
> > pending = true;
> > - else if (unlikely(result < 0))
> > - return;
> > + else if (unlikely(result < 0)) {
> > + if (result == -EKEYEXPIRED)
> > + update_tls_keys(queue);
> > + break;
> > + }
> >
> > /* did we get some space after spending time in recv? */
> > if (nvme_tcp_queue_has_pending(queue) &&
> > @@ -1723,6 +1763,7 @@ static void nvme_tcp_tls_done(void *data, int status, key_serial_t pskid,
> > ctrl->ctrl.tls_pskid = key_serial(tls_key);
> > key_put(tls_key);
> > queue->tls_err = 0;
> > + queue->user_session_id = user_session_id;
>
> Hmm. I wonder, do we need to store the generation number somewhere?
> Currently the sysfs interface is completely oblivious that a key update
> has happened. I really would like to have _some_ indicator there telling
> us that a key update had happened, and the generation number would be
> ideal here.
I don't follow.
The TLS layer will report the number of KeyUpdates that have been
received. Userspace also knows that a KeyUpdate happened as we call to
userspace to handle updating the keys.
Alistair
>
> > }
> >
> > out_complete:
> > @@ -1752,6 +1793,7 @@ static int nvme_tcp_start_tls(struct nvme_ctrl *nctrl,
> > keyring = key_serial(nctrl->opts->keyring);
> > args.ta_keyring = keyring;
> > args.ta_timeout_ms = tls_handshake_timeout * 1000;
> > + args.user_session_id = queue->user_session_id;
> > queue->tls_err = -EOPNOTSUPP;
> > init_completion(&queue->tls_complete);
> > if (keyupdate == HANDSHAKE_KEY_UPDATE_TYPE_UNSPEC)
>
> Chers,
> Hannes
> --
> Dr. Hannes Reinecke Kernel Storage Architect
> hare@suse.de +49 911 74053 688
> SUSE Software Solutions GmbH, Frankenstr. 146, 90461 Nürnberg
> HRB 36809 (AG Nürnberg), GF: I. Totev, A. McDonald, W. Knoblich
On 10/22/25 06:35, Alistair Francis wrote: > On Mon, Oct 20, 2025 at 4:22 PM Hannes Reinecke <hare@suse.de> wrote: >> >> On 10/17/25 06:23, alistair23@gmail.com wrote: >>> From: Alistair Francis <alistair.francis@wdc.com> >>> [ .. ]>>> @@ -1723,6 +1763,7 @@ static void nvme_tcp_tls_done(void *data, int status, key_serial_t pskid, >>> ctrl->ctrl.tls_pskid = key_serial(tls_key); >>> key_put(tls_key); >>> queue->tls_err = 0; >>> + queue->user_session_id = user_session_id; >> >> Hmm. I wonder, do we need to store the generation number somewhere? >> Currently the sysfs interface is completely oblivious that a key update >> has happened. I really would like to have _some_ indicator there telling >> us that a key update had happened, and the generation number would be >> ideal here. > > I don't follow. > > The TLS layer will report the number of KeyUpdates that have been > received. Userspace also knows that a KeyUpdate happened as we call to > userspace to handle updating the keys. > Oh, the tlshd will know that (somehow). But everyone else will not; the 'tls_pskid' contents will stay the the same. Can we have a sysfs attribute reporting the sequence number of the most recent KeyUpdate? Cheers, Hannes -- Dr. Hannes Reinecke Kernel Storage Architect hare@suse.de +49 911 74053 688 SUSE Software Solutions GmbH, Frankenstr. 146, 90461 Nürnberg HRB 36809 (AG Nürnberg), GF: I. Totev, A. McDonald, W. Knoblich
On Wed, Oct 22, 2025 at 4:56 PM Hannes Reinecke <hare@suse.de> wrote: > > On 10/22/25 06:35, Alistair Francis wrote: > > On Mon, Oct 20, 2025 at 4:22 PM Hannes Reinecke <hare@suse.de> wrote: > >> > >> On 10/17/25 06:23, alistair23@gmail.com wrote: > >>> From: Alistair Francis <alistair.francis@wdc.com> > >>> > [ .. ]>>> @@ -1723,6 +1763,7 @@ static void nvme_tcp_tls_done(void > *data, int status, key_serial_t pskid, > >>> ctrl->ctrl.tls_pskid = key_serial(tls_key); > >>> key_put(tls_key); > >>> queue->tls_err = 0; > >>> + queue->user_session_id = user_session_id; > >> > >> Hmm. I wonder, do we need to store the generation number somewhere? > >> Currently the sysfs interface is completely oblivious that a key update > >> has happened. I really would like to have _some_ indicator there telling > >> us that a key update had happened, and the generation number would be > >> ideal here. > > > > I don't follow. > > > > The TLS layer will report the number of KeyUpdates that have been > > received. Userspace also knows that a KeyUpdate happened as we call to > > userspace to handle updating the keys. > > > Oh, the tlshd will know that (somehow). But everyone else will not; the > 'tls_pskid' contents will stay the the same. > Can we have a sysfs attribute reporting the sequence number of the most > recent KeyUpdate? Why do we want to reveal that to userspace though? Realistically it should just be ~2^64 and it'll should remain the same number, even after multiple updates Alistair > Cheers, > Hannes > -- > Dr. Hannes Reinecke Kernel Storage Architect > hare@suse.de +49 911 74053 688 > SUSE Software Solutions GmbH, Frankenstr. 146, 90461 Nürnberg > HRB 36809 (AG Nürnberg), GF: I. Totev, A. McDonald, W. Knoblich
On Fri, Oct 17, 2025 at 02:23:10PM +1000, alistair23@gmail.com wrote:
> From: Alistair Francis <alistair.francis@wdc.com>
>
> If the nvme_tcp_try_send() or nvme_tcp_try_recv() functions return
> EKEYEXPIRED then the underlying TLS keys need to be updated. This occurs
> on an KeyUpdate event.
>
> If the NVMe Target (TLS server) initiates a KeyUpdate this patch will
> allow the NVMe layer to process the KeyUpdate request and forward the
> request to userspace. Userspace must then update the key to keep the
> connection alive.
>
> This patch allows us to handle the NVMe target sending a KeyUpdate
> request without aborting the connection. At this time we don't support
> initiating a KeyUpdate.
>
> Link: https://datatracker.ietf.org/doc/html/rfc8446#section-4.6.3
Totally independent of the current Link-tag flamewar, spec reference
should be in the free flowing commit text.
> + if (result == -EKEYEXPIRED) {
> + return -EKEYEXPIRED;
> + } else if (result == -EAGAIN) {
> + return -EAGAIN;
> + } else if (result < 0) {
returns do not need and should not be followed by else statements.
> dev_err(queue->ctrl->ctrl.device,
> "receive failed: %d\n", result);
> queue->rd_enabled = false;
> nvme_tcp_error_recovery(&queue->ctrl->ctrl);
> + }
>
> return result < 0 ? result : (queue->nr_cqe = nr_cqe);
Also the overall flow here, but old and newly added feels really odd,
up to the point of intentional obfuscation in the last return line.
I'd expect this to be more something like:
if (result < 0) {
if (result != -EKEYEXPIRED && result != -EAGAIN) {
dev_err(queue->ctrl->ctrl.device,
"receive failed: %d\n", result);
queue->rd_enabled = false;
nvme_tcp_error_recovery(&queue->ctrl->ctrl);
}
return result;
}
queue->nr_cqe = nr_cqe;
return nr_cqe;
}
© 2016 - 2026 Red Hat, Inc.