From nobody Wed Nov 5 04:38:10 2025 Delivered-To: importer@patchew.org Received-SPF: pass (zoho.com: domain of gnu.org designates 208.118.235.17 as permitted sender) client-ip=208.118.235.17; envelope-from=qemu-devel-bounces+importer=patchew.org@nongnu.org; helo=lists.gnu.org; Authentication-Results: mx.zohomail.com; spf=pass (zoho.com: domain of gnu.org designates 208.118.235.17 as permitted sender) smtp.mailfrom=qemu-devel-bounces+importer=patchew.org@nongnu.org; dmarc=fail(p=none dis=none) header.from=virtuozzo.com Return-Path: Received: from lists.gnu.org (lists.gnu.org [208.118.235.17]) by mx.zohomail.com with SMTPS id 1533058581034157.74268180221827; Tue, 31 Jul 2018 10:36:21 -0700 (PDT) Received: from localhost ([::1]:60028 helo=lists.gnu.org) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1fkYZH-0005sw-Ln for importer@patchew.org; Tue, 31 Jul 2018 13:36:19 -0400 Received: from eggs.gnu.org ([2001:4830:134:3::10]:36936) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1fkYTs-0001Ui-B2 for qemu-devel@nongnu.org; Tue, 31 Jul 2018 13:30:46 -0400 Received: from Debian-exim by eggs.gnu.org with spam-scanned (Exim 4.71) (envelope-from ) id 1fkYTo-0004FP-8Z for qemu-devel@nongnu.org; Tue, 31 Jul 2018 13:30:44 -0400 Received: from relay.sw.ru ([185.231.240.75]:60558) by eggs.gnu.org with esmtps (TLS1.0:DHE_RSA_AES_256_CBC_SHA1:32) (Exim 4.71) (envelope-from ) id 1fkYTn-0004Du-SP; Tue, 31 Jul 2018 13:30:40 -0400 Received: from vz-out.virtuozzo.com ([185.231.240.5] helo=kvm.sw.ru) by relay.sw.ru with esmtp (Exim 4.90_1) (envelope-from ) id 1fkYTi-00080k-5M; Tue, 31 Jul 2018 20:30:34 +0300 From: Vladimir Sementsov-Ogievskiy To: qemu-devel@nongnu.org, qemu-block@nongnu.org Date: Tue, 31 Jul 2018 20:30:24 +0300 Message-Id: <20180731173033.75467-2-vsementsov@virtuozzo.com> X-Mailer: git-send-email 2.11.1 In-Reply-To: <20180731173033.75467-1-vsementsov@virtuozzo.com> References: <20180731173033.75467-1-vsementsov@virtuozzo.com> X-detected-operating-system: by eggs.gnu.org: GNU/Linux 3.x [fuzzy] X-Received-From: 185.231.240.75 Subject: [Qemu-devel] [PATCH v4 01/10] block/nbd-client: split channel errors from export errors X-BeenThere: qemu-devel@nongnu.org X-Mailman-Version: 2.1.21 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Cc: kwolf@redhat.com, vsementsov@virtuozzo.com, armbru@redhat.com, mreitz@redhat.com, den@openvz.org, pbonzini@redhat.com Errors-To: qemu-devel-bounces+importer=patchew.org@nongnu.org Sender: "Qemu-devel" X-ZohoMail: RSF_0 Z_629925259 SPT_0 Content-Transfer-Encoding: quoted-printable MIME-Version: 1.0 Content-Type: text/plain; charset="utf-8" To implement nbd reconnect in further patches, we need to distinguish error codes, returned by nbd server, from channel errors, to reconnect only in the latter case. Signed-off-by: Vladimir Sementsov-Ogievskiy Reviewed-by: Eric Blake --- block/nbd-client.c | 83 +++++++++++++++++++++++++++++++-------------------= ---- 1 file changed, 47 insertions(+), 36 deletions(-) diff --git a/block/nbd-client.c b/block/nbd-client.c index 9686ecbd5e..1efc3d36cc 100644 --- a/block/nbd-client.c +++ b/block/nbd-client.c @@ -501,11 +501,11 @@ static coroutine_fn int nbd_co_do_receive_one_chunk( */ static coroutine_fn int nbd_co_receive_one_chunk( NBDClientSession *s, uint64_t handle, bool only_structured, - QEMUIOVector *qiov, NBDReply *reply, void **payload, Error **errp) + int *request_ret, QEMUIOVector *qiov, NBDReply *reply, void **payl= oad, + Error **errp) { - int request_ret; int ret =3D nbd_co_do_receive_one_chunk(s, handle, only_structured, - &request_ret, qiov, payload, err= p); + request_ret, qiov, payload, errp= ); =20 if (ret < 0) { s->quit =3D true; @@ -515,7 +515,6 @@ static coroutine_fn int nbd_co_receive_one_chunk( *reply =3D s->reply; } s->reply.handle =3D 0; - ret =3D request_ret; } =20 if (s->read_reply_co) { @@ -527,22 +526,17 @@ static coroutine_fn int nbd_co_receive_one_chunk( =20 typedef struct NBDReplyChunkIter { int ret; - bool fatal; + int request_ret; Error *err; bool done, only_structured; } NBDReplyChunkIter; =20 -static void nbd_iter_error(NBDReplyChunkIter *iter, bool fatal, - int ret, Error **local_err) +static void nbd_iter_channel_error(NBDReplyChunkIter *iter, + int ret, Error **local_err) { assert(ret < 0); =20 - if ((fatal && !iter->fatal) || iter->ret =3D=3D 0) { - if (iter->ret !=3D 0) { - error_free(iter->err); - iter->err =3D NULL; - } - iter->fatal =3D fatal; + if (!iter->ret) { iter->ret =3D ret; error_propagate(&iter->err, *local_err); } else { @@ -552,6 +546,15 @@ static void nbd_iter_error(NBDReplyChunkIter *iter, bo= ol fatal, *local_err =3D NULL; } =20 +static void nbd_iter_request_error(NBDReplyChunkIter *iter, int ret) +{ + assert(ret < 0); + + if (!iter->request_ret) { + iter->request_ret =3D ret; + } +} + /* NBD_FOREACH_REPLY_CHUNK */ #define NBD_FOREACH_REPLY_CHUNK(s, iter, handle, structured, \ @@ -567,13 +570,13 @@ static bool nbd_reply_chunk_iter_receive(NBDClientSes= sion *s, QEMUIOVector *qiov, NBDReply *rep= ly, void **payload) { - int ret; + int ret, request_ret; NBDReply local_reply; NBDStructuredReplyChunk *chunk; Error *local_err =3D NULL; if (s->quit) { error_setg(&local_err, "Connection closed"); - nbd_iter_error(iter, true, -EIO, &local_err); + nbd_iter_channel_error(iter, -EIO, &local_err); goto break_loop; } =20 @@ -587,10 +590,12 @@ static bool nbd_reply_chunk_iter_receive(NBDClientSes= sion *s, } =20 ret =3D nbd_co_receive_one_chunk(s, handle, iter->only_structured, - qiov, reply, payload, &local_err); + &request_ret, qiov, reply, payload, + &local_err); if (ret < 0) { - /* If it is a fatal error s->quit is set by nbd_co_receive_one_chu= nk */ - nbd_iter_error(iter, s->quit, ret, &local_err); + nbd_iter_channel_error(iter, ret, &local_err); + } else if (request_ret < 0) { + nbd_iter_request_error(iter, request_ret); } =20 /* Do not execute the body of NBD_FOREACH_REPLY_CHUNK for simple reply= . */ @@ -627,7 +632,7 @@ break_loop: } =20 static int nbd_co_receive_return_code(NBDClientSession *s, uint64_t handle, - Error **errp) + int *request_ret, Error **errp) { NBDReplyChunkIter iter; =20 @@ -636,12 +641,13 @@ static int nbd_co_receive_return_code(NBDClientSessio= n *s, uint64_t handle, } =20 error_propagate(errp, iter.err); + *request_ret =3D iter.request_ret; return iter.ret; } =20 static int nbd_co_receive_cmdread_reply(NBDClientSession *s, uint64_t hand= le, uint64_t offset, QEMUIOVector *qio= v, - Error **errp) + int *request_ret, Error **errp) { NBDReplyChunkIter iter; NBDReply reply; @@ -666,7 +672,7 @@ static int nbd_co_receive_cmdread_reply(NBDClientSessio= n *s, uint64_t handle, offset, qiov, &local_err); if (ret < 0) { s->quit =3D true; - nbd_iter_error(&iter, true, ret, &local_err); + nbd_iter_channel_error(&iter, ret, &local_err); } break; default: @@ -676,7 +682,7 @@ static int nbd_co_receive_cmdread_reply(NBDClientSessio= n *s, uint64_t handle, error_setg(&local_err, "Unexpected reply type: %d (%s) for CMD_READ", chunk->type, nbd_reply_type_lookup(chunk->type)= ); - nbd_iter_error(&iter, true, -EINVAL, &local_err); + nbd_iter_channel_error(&iter, -EINVAL, &local_err); } } =20 @@ -685,12 +691,14 @@ static int nbd_co_receive_cmdread_reply(NBDClientSess= ion *s, uint64_t handle, } =20 error_propagate(errp, iter.err); + *request_ret =3D iter.request_ret; return iter.ret; } =20 static int nbd_co_receive_blockstatus_reply(NBDClientSession *s, uint64_t handle, uint64_t leng= th, - NBDExtent *extent, Error **err= p) + NBDExtent *extent, + int *request_ret, Error **errp) { NBDReplyChunkIter iter; NBDReply reply; @@ -712,7 +720,7 @@ static int nbd_co_receive_blockstatus_reply(NBDClientSe= ssion *s, if (received) { s->quit =3D true; error_setg(&local_err, "Several BLOCK_STATUS chunks in rep= ly"); - nbd_iter_error(&iter, true, -EINVAL, &local_err); + nbd_iter_channel_error(&iter, -EINVAL, &local_err); } received =3D true; =20 @@ -721,7 +729,7 @@ static int nbd_co_receive_blockstatus_reply(NBDClientSe= ssion *s, &local_err); if (ret < 0) { s->quit =3D true; - nbd_iter_error(&iter, true, ret, &local_err); + nbd_iter_channel_error(&iter, ret, &local_err); } break; default: @@ -731,7 +739,7 @@ static int nbd_co_receive_blockstatus_reply(NBDClientSe= ssion *s, "Unexpected reply type: %d (%s) " "for CMD_BLOCK_STATUS", chunk->type, nbd_reply_type_lookup(chunk->type)= ); - nbd_iter_error(&iter, true, -EINVAL, &local_err); + nbd_iter_channel_error(&iter, -EINVAL, &local_err); } } =20 @@ -746,14 +754,16 @@ static int nbd_co_receive_blockstatus_reply(NBDClient= Session *s, iter.ret =3D -EIO; } } + error_propagate(errp, iter.err); + *request_ret =3D iter.request_ret; return iter.ret; } =20 static int nbd_co_request(BlockDriverState *bs, NBDRequest *request, QEMUIOVector *write_qiov) { - int ret; + int ret, request_ret; Error *local_err =3D NULL; NBDClientSession *client =3D nbd_get_client_session(bs); =20 @@ -769,17 +779,18 @@ static int nbd_co_request(BlockDriverState *bs, NBDRe= quest *request, return ret; } =20 - ret =3D nbd_co_receive_return_code(client, request->handle, &local_err= ); + ret =3D nbd_co_receive_return_code(client, request->handle, + &request_ret, &local_err); if (local_err) { error_report_err(local_err); } - return ret; + return ret ? ret : request_ret; } =20 int nbd_client_co_preadv(BlockDriverState *bs, uint64_t offset, uint64_t bytes, QEMUIOVector *qiov, int flags) { - int ret; + int ret, request_ret; Error *local_err =3D NULL; NBDClientSession *client =3D nbd_get_client_session(bs); NBDRequest request =3D { @@ -800,11 +811,11 @@ int nbd_client_co_preadv(BlockDriverState *bs, uint64= _t offset, } =20 ret =3D nbd_co_receive_cmdread_reply(client, request.handle, offset, q= iov, - &local_err); + &request_ret, &local_err); if (local_err) { error_report_err(local_err); } - return ret; + return ret ? ret : request_ret; } =20 int nbd_client_co_pwritev(BlockDriverState *bs, uint64_t offset, @@ -898,7 +909,7 @@ int coroutine_fn nbd_client_co_block_status(BlockDriver= State *bs, int64_t *pnum, int64_t *map, BlockDriverState **file) { - int64_t ret; + int ret, request_ret; NBDExtent extent =3D { 0 }; NBDClientSession *client =3D nbd_get_client_session(bs); Error *local_err =3D NULL; @@ -923,12 +934,12 @@ int coroutine_fn nbd_client_co_block_status(BlockDriv= erState *bs, } =20 ret =3D nbd_co_receive_blockstatus_reply(client, request.handle, bytes, - &extent, &local_err); + &extent, &request_ret, &local_e= rr); if (local_err) { error_report_err(local_err); } - if (ret < 0) { - return ret; + if (ret < 0 || request_ret < 0) { + return ret ? ret : request_ret; } =20 assert(extent.length); --=20 2.11.1 From nobody Wed Nov 5 04:38:10 2025 Delivered-To: importer@patchew.org Received-SPF: pass (zoho.com: domain of gnu.org designates 208.118.235.17 as permitted sender) client-ip=208.118.235.17; envelope-from=qemu-devel-bounces+importer=patchew.org@nongnu.org; helo=lists.gnu.org; Authentication-Results: mx.zohomail.com; spf=pass (zoho.com: domain of gnu.org designates 208.118.235.17 as permitted sender) smtp.mailfrom=qemu-devel-bounces+importer=patchew.org@nongnu.org; dmarc=fail(p=none dis=none) header.from=virtuozzo.com Return-Path: Received: from lists.gnu.org (208.118.235.17 [208.118.235.17]) by mx.zohomail.com with SMTPS id 1533058735617516.3535969397863; Tue, 31 Jul 2018 10:38:55 -0700 (PDT) Received: from localhost ([::1]:60050 helo=lists.gnu.org) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1fkYbh-0007ss-AW for importer@patchew.org; Tue, 31 Jul 2018 13:38:49 -0400 Received: from eggs.gnu.org ([2001:4830:134:3::10]:36935) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1fkYTs-0001Uh-Ah for qemu-devel@nongnu.org; Tue, 31 Jul 2018 13:30:45 -0400 Received: from Debian-exim by eggs.gnu.org with spam-scanned (Exim 4.71) (envelope-from ) id 1fkYTo-0004F3-5c for qemu-devel@nongnu.org; Tue, 31 Jul 2018 13:30:44 -0400 Received: from relay.sw.ru ([185.231.240.75]:60546) by eggs.gnu.org with esmtps (TLS1.0:DHE_RSA_AES_256_CBC_SHA1:32) (Exim 4.71) (envelope-from ) id 1fkYTn-0004Dr-TY; Tue, 31 Jul 2018 13:30:40 -0400 Received: from vz-out.virtuozzo.com ([185.231.240.5] helo=kvm.sw.ru) by relay.sw.ru with esmtp (Exim 4.90_1) (envelope-from ) id 1fkYTi-00080k-Ho; Tue, 31 Jul 2018 20:30:34 +0300 From: Vladimir Sementsov-Ogievskiy To: qemu-devel@nongnu.org, qemu-block@nongnu.org Date: Tue, 31 Jul 2018 20:30:25 +0300 Message-Id: <20180731173033.75467-3-vsementsov@virtuozzo.com> X-Mailer: git-send-email 2.11.1 In-Reply-To: <20180731173033.75467-1-vsementsov@virtuozzo.com> References: <20180731173033.75467-1-vsementsov@virtuozzo.com> X-detected-operating-system: by eggs.gnu.org: GNU/Linux 3.x [fuzzy] X-Received-From: 185.231.240.75 Subject: [Qemu-devel] [PATCH v4 02/10] block/nbd: move connection code from block/nbd to block/nbd-client X-BeenThere: qemu-devel@nongnu.org X-Mailman-Version: 2.1.21 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Cc: kwolf@redhat.com, vsementsov@virtuozzo.com, armbru@redhat.com, mreitz@redhat.com, den@openvz.org, pbonzini@redhat.com Errors-To: qemu-devel-bounces+importer=patchew.org@nongnu.org Sender: "Qemu-devel" X-ZohoMail: RSF_0 Z_629925259 SPT_0 Content-Transfer-Encoding: quoted-printable MIME-Version: 1.0 Content-Type: text/plain; charset="utf-8" Keep all connection code in one file, to be able to implement reconnect in further patches. Signed-off-by: Vladimir Sementsov-Ogievskiy Reviewed-by: Eric Blake --- block/nbd-client.h | 2 +- block/nbd-client.c | 37 +++++++++++++++++++++++++++++++++++-- block/nbd.c | 40 ++-------------------------------------- 3 files changed, 38 insertions(+), 41 deletions(-) diff --git a/block/nbd-client.h b/block/nbd-client.h index cfc90550b9..2f047ba614 100644 --- a/block/nbd-client.h +++ b/block/nbd-client.h @@ -41,7 +41,7 @@ typedef struct NBDClientSession { NBDClientSession *nbd_get_client_session(BlockDriverState *bs); =20 int nbd_client_init(BlockDriverState *bs, - QIOChannelSocket *sock, + SocketAddress *saddr, const char *export_name, QCryptoTLSCreds *tlscreds, const char *hostname, diff --git a/block/nbd-client.c b/block/nbd-client.c index 1efc3d36cc..a0d8f2523e 100644 --- a/block/nbd-client.c +++ b/block/nbd-client.c @@ -976,8 +976,31 @@ void nbd_client_close(BlockDriverState *bs) nbd_teardown_connection(bs); } =20 +static QIOChannelSocket *nbd_establish_connection(SocketAddress *saddr, + Error **errp) +{ + QIOChannelSocket *sioc; + Error *local_err =3D NULL; + + sioc =3D qio_channel_socket_new(); + qio_channel_set_name(QIO_CHANNEL(sioc), "nbd-client"); + + qio_channel_socket_connect_sync(sioc, + saddr, + &local_err); + if (local_err) { + object_unref(OBJECT(sioc)); + error_propagate(errp, local_err); + return NULL; + } + + qio_channel_set_delay(QIO_CHANNEL(sioc), false); + + return sioc; +} + int nbd_client_init(BlockDriverState *bs, - QIOChannelSocket *sioc, + SocketAddress *saddr, const char *export, QCryptoTLSCreds *tlscreds, const char *hostname, @@ -987,6 +1010,15 @@ int nbd_client_init(BlockDriverState *bs, NBDClientSession *client =3D nbd_get_client_session(bs); int ret; =20 + /* establish TCP connection, return error if it fails + * TODO: Configurable retry-until-timeout behaviour. + */ + QIOChannelSocket *sioc =3D nbd_establish_connection(saddr, errp); + + if (!sioc) { + return -ECONNREFUSED; + } + /* NBD handshake */ logout("session init %s\n", export); qio_channel_set_blocking(QIO_CHANNEL(sioc), true, NULL); @@ -1001,12 +1033,14 @@ int nbd_client_init(BlockDriverState *bs, g_free(client->info.x_dirty_bitmap); if (ret < 0) { logout("Failed to negotiate with the NBD server\n"); + object_unref(OBJECT(sioc)); return ret; } if (client->info.flags & NBD_FLAG_READ_ONLY && !bdrv_is_read_only(bs)) { error_setg(errp, "request for write access conflicts with read-only expo= rt"); + object_unref(OBJECT(sioc)); return -EACCES; } if (client->info.flags & NBD_FLAG_SEND_FUA) { @@ -1020,7 +1054,6 @@ int nbd_client_init(BlockDriverState *bs, qemu_co_mutex_init(&client->send_mutex); qemu_co_queue_init(&client->free_sema); client->sioc =3D sioc; - object_ref(OBJECT(client->sioc)); =20 if (!client->ioc) { client->ioc =3D QIO_CHANNEL(sioc); diff --git a/block/nbd.c b/block/nbd.c index e87699fb73..9db5eded89 100644 --- a/block/nbd.c +++ b/block/nbd.c @@ -295,30 +295,6 @@ NBDClientSession *nbd_get_client_session(BlockDriverSt= ate *bs) return &s->client; } =20 -static QIOChannelSocket *nbd_establish_connection(SocketAddress *saddr, - Error **errp) -{ - QIOChannelSocket *sioc; - Error *local_err =3D NULL; - - sioc =3D qio_channel_socket_new(); - qio_channel_set_name(QIO_CHANNEL(sioc), "nbd-client"); - - qio_channel_socket_connect_sync(sioc, - saddr, - &local_err); - if (local_err) { - object_unref(OBJECT(sioc)); - error_propagate(errp, local_err); - return NULL; - } - - qio_channel_set_delay(QIO_CHANNEL(sioc), false); - - return sioc; -} - - static QCryptoTLSCreds *nbd_get_tls_creds(const char *id, Error **errp) { Object *obj; @@ -394,7 +370,6 @@ static int nbd_open(BlockDriverState *bs, QDict *option= s, int flags, BDRVNBDState *s =3D bs->opaque; QemuOpts *opts =3D NULL; Error *local_err =3D NULL; - QIOChannelSocket *sioc =3D NULL; QCryptoTLSCreds *tlscreds =3D NULL; const char *hostname =3D NULL; int ret =3D -EINVAL; @@ -434,22 +409,11 @@ static int nbd_open(BlockDriverState *bs, QDict *opti= ons, int flags, hostname =3D s->saddr->u.inet.host; } =20 - /* establish TCP connection, return error if it fails - * TODO: Configurable retry-until-timeout behaviour. - */ - sioc =3D nbd_establish_connection(s->saddr, errp); - if (!sioc) { - ret =3D -ECONNREFUSED; - goto error; - } - /* NBD handshake */ - ret =3D nbd_client_init(bs, sioc, s->export, tlscreds, hostname, + ret =3D nbd_client_init(bs, s->saddr, s->export, tlscreds, hostname, qemu_opt_get(opts, "x-dirty-bitmap"), errp); + error: - if (sioc) { - object_unref(OBJECT(sioc)); - } if (tlscreds) { object_unref(OBJECT(tlscreds)); } --=20 2.11.1 From nobody Wed Nov 5 04:38:10 2025 Delivered-To: importer@patchew.org Received-SPF: pass (zoho.com: domain of gnu.org designates 208.118.235.17 as permitted sender) client-ip=208.118.235.17; envelope-from=qemu-devel-bounces+importer=patchew.org@nongnu.org; helo=lists.gnu.org; Authentication-Results: mx.zohomail.com; spf=pass (zoho.com: domain of gnu.org designates 208.118.235.17 as permitted sender) smtp.mailfrom=qemu-devel-bounces+importer=patchew.org@nongnu.org; dmarc=fail(p=none dis=none) header.from=virtuozzo.com Return-Path: Received: from lists.gnu.org (lists.gnu.org [208.118.235.17]) by mx.zohomail.com with SMTPS id 1533058581762232.28886347729974; Tue, 31 Jul 2018 10:36:21 -0700 (PDT) Received: from localhost ([::1]:60027 helo=lists.gnu.org) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1fkYZF-0005rv-1n for importer@patchew.org; Tue, 31 Jul 2018 13:36:17 -0400 Received: from eggs.gnu.org ([2001:4830:134:3::10]:36930) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1fkYTs-0001Ue-9P for qemu-devel@nongnu.org; Tue, 31 Jul 2018 13:30:45 -0400 Received: from Debian-exim by eggs.gnu.org with spam-scanned (Exim 4.71) (envelope-from ) id 1fkYTo-0004Et-48 for qemu-devel@nongnu.org; Tue, 31 Jul 2018 13:30:44 -0400 Received: from relay.sw.ru ([185.231.240.75]:60548) by eggs.gnu.org with esmtps (TLS1.0:DHE_RSA_AES_256_CBC_SHA1:32) (Exim 4.71) (envelope-from ) id 1fkYTn-0004Dt-SR; Tue, 31 Jul 2018 13:30:40 -0400 Received: from vz-out.virtuozzo.com ([185.231.240.5] helo=kvm.sw.ru) by relay.sw.ru with esmtp (Exim 4.90_1) (envelope-from ) id 1fkYTj-00080k-6D; Tue, 31 Jul 2018 20:30:35 +0300 From: Vladimir Sementsov-Ogievskiy To: qemu-devel@nongnu.org, qemu-block@nongnu.org Date: Tue, 31 Jul 2018 20:30:26 +0300 Message-Id: <20180731173033.75467-4-vsementsov@virtuozzo.com> X-Mailer: git-send-email 2.11.1 In-Reply-To: <20180731173033.75467-1-vsementsov@virtuozzo.com> References: <20180731173033.75467-1-vsementsov@virtuozzo.com> X-detected-operating-system: by eggs.gnu.org: GNU/Linux 3.x [fuzzy] X-Received-From: 185.231.240.75 Subject: [Qemu-devel] [PATCH v4 03/10] block/nbd-client: split connection from initialization X-BeenThere: qemu-devel@nongnu.org X-Mailman-Version: 2.1.21 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Cc: kwolf@redhat.com, vsementsov@virtuozzo.com, armbru@redhat.com, mreitz@redhat.com, den@openvz.org, pbonzini@redhat.com Errors-To: qemu-devel-bounces+importer=patchew.org@nongnu.org Sender: "Qemu-devel" X-ZohoMail: RSF_0 Z_629925259 SPT_0 Content-Transfer-Encoding: quoted-printable MIME-Version: 1.0 Content-Type: text/plain; charset="utf-8" Split connection code to reuse it for reconnect. Signed-off-by: Vladimir Sementsov-Ogievskiy Reviewed-by: Eric Blake --- block/nbd-client.c | 33 ++++++++++++++++++++++++--------- 1 file changed, 24 insertions(+), 9 deletions(-) diff --git a/block/nbd-client.c b/block/nbd-client.c index a0d8f2523e..a1813cbfe1 100644 --- a/block/nbd-client.c +++ b/block/nbd-client.c @@ -999,13 +999,13 @@ static QIOChannelSocket *nbd_establish_connection(Soc= ketAddress *saddr, return sioc; } =20 -int nbd_client_init(BlockDriverState *bs, - SocketAddress *saddr, - const char *export, - QCryptoTLSCreds *tlscreds, - const char *hostname, - const char *x_dirty_bitmap, - Error **errp) +static int nbd_client_connect(BlockDriverState *bs, + SocketAddress *saddr, + const char *export, + QCryptoTLSCreds *tlscreds, + const char *hostname, + const char *x_dirty_bitmap, + Error **errp) { NBDClientSession *client =3D nbd_get_client_session(bs); int ret; @@ -1051,8 +1051,6 @@ int nbd_client_init(BlockDriverState *bs, bs->supported_zero_flags |=3D BDRV_REQ_MAY_UNMAP; } =20 - qemu_co_mutex_init(&client->send_mutex); - qemu_co_queue_init(&client->free_sema); client->sioc =3D sioc; =20 if (!client->ioc) { @@ -1069,3 +1067,20 @@ int nbd_client_init(BlockDriverState *bs, logout("Established connection with NBD server\n"); return 0; } + +int nbd_client_init(BlockDriverState *bs, + SocketAddress *saddr, + const char *export, + QCryptoTLSCreds *tlscreds, + const char *hostname, + const char *x_dirty_bitmap, + Error **errp) +{ + NBDClientSession *client =3D nbd_get_client_session(bs); + + qemu_co_mutex_init(&client->send_mutex); + qemu_co_queue_init(&client->free_sema); + + return nbd_client_connect(bs, saddr, export, tlscreds, hostname, + x_dirty_bitmap, errp); +} --=20 2.11.1 From nobody Wed Nov 5 04:38:10 2025 Delivered-To: importer@patchew.org Received-SPF: pass (zoho.com: domain of gnu.org designates 208.118.235.17 as permitted sender) client-ip=208.118.235.17; envelope-from=qemu-devel-bounces+importer=patchew.org@nongnu.org; helo=lists.gnu.org; Authentication-Results: mx.zohomail.com; spf=pass (zoho.com: domain of gnu.org designates 208.118.235.17 as permitted sender) smtp.mailfrom=qemu-devel-bounces+importer=patchew.org@nongnu.org; dmarc=fail(p=none dis=none) header.from=virtuozzo.com Return-Path: Received: from lists.gnu.org (lists.gnu.org [208.118.235.17]) by mx.zohomail.com with SMTPS id 1533058581757479.95465911020256; Tue, 31 Jul 2018 10:36:21 -0700 (PDT) Received: from localhost ([::1]:60026 helo=lists.gnu.org) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1fkYZF-0005rF-Lq for importer@patchew.org; Tue, 31 Jul 2018 13:36:17 -0400 Received: from eggs.gnu.org ([2001:4830:134:3::10]:36941) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1fkYTs-0001Ul-Be for qemu-devel@nongnu.org; Tue, 31 Jul 2018 13:30:45 -0400 Received: from Debian-exim by eggs.gnu.org with spam-scanned (Exim 4.71) (envelope-from ) id 1fkYTo-0004F2-5I for qemu-devel@nongnu.org; Tue, 31 Jul 2018 13:30:44 -0400 Received: from relay.sw.ru ([185.231.240.75]:60560) by eggs.gnu.org with esmtps (TLS1.0:DHE_RSA_AES_256_CBC_SHA1:32) (Exim 4.71) (envelope-from ) id 1fkYTn-0004Dq-T5; Tue, 31 Jul 2018 13:30:40 -0400 Received: from vz-out.virtuozzo.com ([185.231.240.5] helo=kvm.sw.ru) by relay.sw.ru with esmtp (Exim 4.90_1) (envelope-from ) id 1fkYTj-00080k-OL; Tue, 31 Jul 2018 20:30:35 +0300 From: Vladimir Sementsov-Ogievskiy To: qemu-devel@nongnu.org, qemu-block@nongnu.org Date: Tue, 31 Jul 2018 20:30:27 +0300 Message-Id: <20180731173033.75467-5-vsementsov@virtuozzo.com> X-Mailer: git-send-email 2.11.1 In-Reply-To: <20180731173033.75467-1-vsementsov@virtuozzo.com> References: <20180731173033.75467-1-vsementsov@virtuozzo.com> X-detected-operating-system: by eggs.gnu.org: GNU/Linux 3.x [fuzzy] X-Received-From: 185.231.240.75 Subject: [Qemu-devel] [PATCH v4 04/10] block/nbd-client: fix nbd_reply_chunk_iter_receive X-BeenThere: qemu-devel@nongnu.org X-Mailman-Version: 2.1.21 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Cc: kwolf@redhat.com, vsementsov@virtuozzo.com, armbru@redhat.com, mreitz@redhat.com, den@openvz.org, pbonzini@redhat.com Errors-To: qemu-devel-bounces+importer=patchew.org@nongnu.org Sender: "Qemu-devel" X-ZohoMail: RSF_0 Z_629925259 SPT_0 Content-Transfer-Encoding: quoted-printable MIME-Version: 1.0 Content-Type: text/plain; charset="utf-8" Use exported report, not the variable to be reused (should not really matter). Signed-off-by: Vladimir Sementsov-Ogievskiy Reviewed-by: Eric Blake --- block/nbd-client.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/block/nbd-client.c b/block/nbd-client.c index a1813cbfe1..263d1721f9 100644 --- a/block/nbd-client.c +++ b/block/nbd-client.c @@ -599,7 +599,7 @@ static bool nbd_reply_chunk_iter_receive(NBDClientSessi= on *s, } =20 /* Do not execute the body of NBD_FOREACH_REPLY_CHUNK for simple reply= . */ - if (nbd_reply_is_simple(&s->reply) || s->quit) { + if (nbd_reply_is_simple(reply) || s->quit) { goto break_loop; } =20 --=20 2.11.1 From nobody Wed Nov 5 04:38:10 2025 Delivered-To: importer@patchew.org Received-SPF: pass (zoho.com: domain of gnu.org designates 208.118.235.17 as permitted sender) client-ip=208.118.235.17; envelope-from=qemu-devel-bounces+importer=patchew.org@nongnu.org; helo=lists.gnu.org; Authentication-Results: mx.zohomail.com; spf=pass (zoho.com: domain of gnu.org designates 208.118.235.17 as permitted sender) smtp.mailfrom=qemu-devel-bounces+importer=patchew.org@nongnu.org; dmarc=fail(p=none dis=none) header.from=virtuozzo.com Return-Path: Received: from lists.gnu.org (lists.gnu.org [208.118.235.17]) by mx.zohomail.com with SMTPS id 1533058396977906.3133632559843; Tue, 31 Jul 2018 10:33:16 -0700 (PDT) Received: from localhost ([::1]:60008 helo=lists.gnu.org) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1fkYWI-0002vK-04 for importer@patchew.org; Tue, 31 Jul 2018 13:33:14 -0400 Received: from eggs.gnu.org ([2001:4830:134:3::10]:36937) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1fkYTs-0001Uj-BJ for qemu-devel@nongnu.org; Tue, 31 Jul 2018 13:30:45 -0400 Received: from Debian-exim by eggs.gnu.org with spam-scanned (Exim 4.71) (envelope-from ) id 1fkYTo-0004FB-5P for qemu-devel@nongnu.org; Tue, 31 Jul 2018 13:30:44 -0400 Received: from relay.sw.ru ([185.231.240.75]:60564) by eggs.gnu.org with esmtps (TLS1.0:DHE_RSA_AES_256_CBC_SHA1:32) (Exim 4.71) (envelope-from ) id 1fkYTn-0004EG-Ui; Tue, 31 Jul 2018 13:30:40 -0400 Received: from vz-out.virtuozzo.com ([185.231.240.5] helo=kvm.sw.ru) by relay.sw.ru with esmtp (Exim 4.90_1) (envelope-from ) id 1fkYTk-00080k-3A; Tue, 31 Jul 2018 20:30:36 +0300 From: Vladimir Sementsov-Ogievskiy To: qemu-devel@nongnu.org, qemu-block@nongnu.org Date: Tue, 31 Jul 2018 20:30:28 +0300 Message-Id: <20180731173033.75467-6-vsementsov@virtuozzo.com> X-Mailer: git-send-email 2.11.1 In-Reply-To: <20180731173033.75467-1-vsementsov@virtuozzo.com> References: <20180731173033.75467-1-vsementsov@virtuozzo.com> X-detected-operating-system: by eggs.gnu.org: GNU/Linux 3.x [fuzzy] X-Received-From: 185.231.240.75 Subject: [Qemu-devel] [PATCH v4 05/10] block/nbd-client: don't check ioc X-BeenThere: qemu-devel@nongnu.org X-Mailman-Version: 2.1.21 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Cc: kwolf@redhat.com, vsementsov@virtuozzo.com, armbru@redhat.com, mreitz@redhat.com, den@openvz.org, pbonzini@redhat.com Errors-To: qemu-devel-bounces+importer=patchew.org@nongnu.org Sender: "Qemu-devel" X-ZohoMail: RSF_0 Z_629925259 SPT_0 Content-Transfer-Encoding: quoted-printable MIME-Version: 1.0 Content-Type: text/plain; charset="utf-8" We have several paranoiac checks for ioc !=3D NULL. But ioc may become NULL only on close, which should not happen during requests handling. Also, we check ioc only sometimes, not after each yield, which is inconsistent. Let's drop these checks. However, for safety, lets leave asserts instead. Signed-off-by: Vladimir Sementsov-Ogievskiy Reviewed-by: Eric Blake --- block/nbd-client.c | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/block/nbd-client.c b/block/nbd-client.c index 263d1721f9..7eaf0149f0 100644 --- a/block/nbd-client.c +++ b/block/nbd-client.c @@ -51,9 +51,7 @@ static void nbd_teardown_connection(BlockDriverState *bs) { NBDClientSession *client =3D nbd_get_client_session(bs); =20 - if (!client->ioc) { /* Already closed */ - return; - } + assert(client->ioc); =20 /* finish any pending coroutines */ qio_channel_shutdown(client->ioc, @@ -150,10 +148,7 @@ static int nbd_co_send_request(BlockDriverState *bs, rc =3D -EIO; goto err; } - if (!s->ioc) { - rc =3D -EPIPE; - goto err; - } + assert(s->ioc); =20 if (qiov) { qio_channel_set_cork(s->ioc, true); @@ -426,10 +421,11 @@ static coroutine_fn int nbd_co_do_receive_one_chunk( s->requests[i].receiving =3D true; qemu_coroutine_yield(); s->requests[i].receiving =3D false; - if (!s->ioc || s->quit) { + if (s->quit) { error_setg(errp, "Connection closed"); return -EIO; } + assert(s->ioc); =20 assert(s->reply.handle =3D=3D handle); =20 @@ -967,9 +963,7 @@ void nbd_client_close(BlockDriverState *bs) NBDClientSession *client =3D nbd_get_client_session(bs); NBDRequest request =3D { .type =3D NBD_CMD_DISC }; =20 - if (client->ioc =3D=3D NULL) { - return; - } + assert(client->ioc); =20 nbd_send_request(client->ioc, &request); =20 --=20 2.11.1 From nobody Wed Nov 5 04:38:10 2025 Delivered-To: importer@patchew.org Received-SPF: pass (zoho.com: domain of gnu.org designates 208.118.235.17 as permitted sender) client-ip=208.118.235.17; envelope-from=qemu-devel-bounces+importer=patchew.org@nongnu.org; helo=lists.gnu.org; Authentication-Results: mx.zohomail.com; spf=pass (zoho.com: domain of gnu.org designates 208.118.235.17 as permitted sender) smtp.mailfrom=qemu-devel-bounces+importer=patchew.org@nongnu.org; dmarc=fail(p=none dis=none) header.from=virtuozzo.com Return-Path: Received: from lists.gnu.org (208.118.235.17 [208.118.235.17]) by mx.zohomail.com with SMTPS id 1533058736233284.6921281272756; Tue, 31 Jul 2018 10:38:56 -0700 (PDT) Received: from localhost ([::1]:60049 helo=lists.gnu.org) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1fkYbi-0007sh-1J for importer@patchew.org; Tue, 31 Jul 2018 13:38:50 -0400 Received: from eggs.gnu.org ([2001:4830:134:3::10]:36939) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1fkYTs-0001Uk-BL for qemu-devel@nongnu.org; Tue, 31 Jul 2018 13:30:46 -0400 Received: from Debian-exim by eggs.gnu.org with spam-scanned (Exim 4.71) (envelope-from ) id 1fkYTo-0004Fh-OO for qemu-devel@nongnu.org; Tue, 31 Jul 2018 13:30:44 -0400 Received: from relay.sw.ru ([185.231.240.75]:60568) by eggs.gnu.org with esmtps (TLS1.0:DHE_RSA_AES_256_CBC_SHA1:32) (Exim 4.71) (envelope-from ) id 1fkYTo-0004EN-C5; Tue, 31 Jul 2018 13:30:40 -0400 Received: from vz-out.virtuozzo.com ([185.231.240.5] helo=kvm.sw.ru) by relay.sw.ru with esmtp (Exim 4.90_1) (envelope-from ) id 1fkYTl-00080k-4E; Tue, 31 Jul 2018 20:30:37 +0300 From: Vladimir Sementsov-Ogievskiy To: qemu-devel@nongnu.org, qemu-block@nongnu.org Date: Tue, 31 Jul 2018 20:30:29 +0300 Message-Id: <20180731173033.75467-7-vsementsov@virtuozzo.com> X-Mailer: git-send-email 2.11.1 In-Reply-To: <20180731173033.75467-1-vsementsov@virtuozzo.com> References: <20180731173033.75467-1-vsementsov@virtuozzo.com> X-detected-operating-system: by eggs.gnu.org: GNU/Linux 3.x [fuzzy] X-Received-From: 185.231.240.75 Subject: [Qemu-devel] [PATCH v4 06/10] block/nbd-client: move from quit to state X-BeenThere: qemu-devel@nongnu.org X-Mailman-Version: 2.1.21 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Cc: kwolf@redhat.com, vsementsov@virtuozzo.com, armbru@redhat.com, mreitz@redhat.com, den@openvz.org, pbonzini@redhat.com Errors-To: qemu-devel-bounces+importer=patchew.org@nongnu.org Sender: "Qemu-devel" X-ZohoMail: RSF_0 Z_629925259 SPT_0 Content-Transfer-Encoding: quoted-printable MIME-Version: 1.0 Content-Type: text/plain; charset="utf-8" To implement reconnect we need several states for the client: CONNECTED, QUIT and two CONNECTING states. CONNECTING states will be realized in the following patches. This patch implements CONNECTED and QUIT. QUIT means, that we should close the connection and fail all current and further requests (like old quit =3D true). CONNECTED means that connection is ok, we can send requests (like old quit =3D false). For receiving loop we use a comparison of the current state with QUIT, because reconnect will be in the same loop, so it should be looping until the end. Opposite, for requests we use a comparison of the current state with CONNECTED, as we don't want to send requests in CONNECTING states ( which are unreachable now, but will be reachable after the following commits) Signed-off-by: Vladimir Sementsov-Ogievskiy --- block/nbd-client.h | 9 ++++++++- block/nbd-client.c | 55 ++++++++++++++++++++++++++++++++------------------= ---- 2 files changed, 41 insertions(+), 23 deletions(-) diff --git a/block/nbd-client.h b/block/nbd-client.h index 2f047ba614..5367425774 100644 --- a/block/nbd-client.h +++ b/block/nbd-client.h @@ -23,6 +23,13 @@ typedef struct { bool receiving; /* waiting for read_reply_co? */ } NBDClientRequest; =20 +typedef enum NBDClientState { + NBD_CLIENT_CONNECTING_WAIT, + NBD_CLIENT_CONNECTING_NOWAIT, + NBD_CLIENT_CONNECTED, + NBD_CLIENT_QUIT +} NBDClientState; + typedef struct NBDClientSession { QIOChannelSocket *sioc; /* The master data channel */ QIOChannel *ioc; /* The current I/O channel which may differ (eg TLS) = */ @@ -32,10 +39,10 @@ typedef struct NBDClientSession { CoQueue free_sema; Coroutine *read_reply_co; int in_flight; + NBDClientState state; =20 NBDClientRequest requests[MAX_NBD_REQUESTS]; NBDReply reply; - bool quit; } NBDClientSession; =20 NBDClientSession *nbd_get_client_session(BlockDriverState *bs); diff --git a/block/nbd-client.c b/block/nbd-client.c index 7eaf0149f0..a91fd3ea3e 100644 --- a/block/nbd-client.c +++ b/block/nbd-client.c @@ -34,6 +34,12 @@ #define HANDLE_TO_INDEX(bs, handle) ((handle) ^ (uint64_t)(intptr_t)(bs)) #define INDEX_TO_HANDLE(bs, index) ((index) ^ (uint64_t)(intptr_t)(bs)) =20 +/* @ret would be used for reconnect in future */ +static void nbd_channel_error(NBDClientSession *s, int ret) +{ + s->state =3D NBD_CLIENT_QUIT; +} + static void nbd_recv_coroutines_wake_all(NBDClientSession *s) { int i; @@ -73,14 +79,15 @@ static coroutine_fn void nbd_read_reply_entry(void *opa= que) int ret =3D 0; Error *local_err =3D NULL; =20 - while (!s->quit) { + while (s->state !=3D NBD_CLIENT_QUIT) { assert(s->reply.handle =3D=3D 0); ret =3D nbd_receive_reply(s->ioc, &s->reply, &local_err); if (local_err) { error_report_err(local_err); } if (ret <=3D 0) { - break; + nbd_channel_error(s, ret ? ret : -EIO); + continue; } =20 /* There's no need for a mutex on the receive side, because the @@ -93,7 +100,8 @@ static coroutine_fn void nbd_read_reply_entry(void *opaq= ue) !s->requests[i].receiving || (nbd_reply_is_structured(&s->reply) && !s->info.structured_rep= ly)) { - break; + nbd_channel_error(s, -EINVAL); + continue; } =20 /* We're woken up again by the request itself. Note that there @@ -111,7 +119,6 @@ static coroutine_fn void nbd_read_reply_entry(void *opa= que) qemu_coroutine_yield(); } =20 - s->quit =3D true; nbd_recv_coroutines_wake_all(s); s->read_reply_co =3D NULL; } @@ -121,12 +128,18 @@ static int nbd_co_send_request(BlockDriverState *bs, QEMUIOVector *qiov) { NBDClientSession *s =3D nbd_get_client_session(bs); - int rc, i; + int rc, i =3D -1; =20 qemu_co_mutex_lock(&s->send_mutex); while (s->in_flight =3D=3D MAX_NBD_REQUESTS) { qemu_co_queue_wait(&s->free_sema, &s->send_mutex); } + + if (s->state !=3D NBD_CLIENT_CONNECTED) { + rc =3D -EIO; + goto err; + } + s->in_flight++; =20 for (i =3D 0; i < MAX_NBD_REQUESTS; i++) { @@ -144,16 +157,12 @@ static int nbd_co_send_request(BlockDriverState *bs, =20 request->handle =3D INDEX_TO_HANDLE(s, i); =20 - if (s->quit) { - rc =3D -EIO; - goto err; - } assert(s->ioc); =20 if (qiov) { qio_channel_set_cork(s->ioc, true); rc =3D nbd_send_request(s->ioc, request); - if (rc >=3D 0 && !s->quit) { + if (rc >=3D 0 && s->state =3D=3D NBD_CLIENT_CONNECTED) { if (qio_channel_writev_all(s->ioc, qiov->iov, qiov->niov, NULL) < 0) { rc =3D -EIO; @@ -168,9 +177,11 @@ static int nbd_co_send_request(BlockDriverState *bs, =20 err: if (rc < 0) { - s->quit =3D true; - s->requests[i].coroutine =3D NULL; - s->in_flight--; + nbd_channel_error(s, rc); + if (i !=3D -1) { + s->requests[i].coroutine =3D NULL; + s->in_flight--; + } qemu_co_queue_next(&s->free_sema); } qemu_co_mutex_unlock(&s->send_mutex); @@ -421,7 +432,7 @@ static coroutine_fn int nbd_co_do_receive_one_chunk( s->requests[i].receiving =3D true; qemu_coroutine_yield(); s->requests[i].receiving =3D false; - if (s->quit) { + if (s->state !=3D NBD_CLIENT_CONNECTED) { error_setg(errp, "Connection closed"); return -EIO; } @@ -504,7 +515,7 @@ static coroutine_fn int nbd_co_receive_one_chunk( request_ret, qiov, payload, errp= ); =20 if (ret < 0) { - s->quit =3D true; + nbd_channel_error(s, ret); } else { /* For assert at loop start in nbd_read_reply_entry */ if (reply) { @@ -570,7 +581,7 @@ static bool nbd_reply_chunk_iter_receive(NBDClientSessi= on *s, NBDReply local_reply; NBDStructuredReplyChunk *chunk; Error *local_err =3D NULL; - if (s->quit) { + if (s->state !=3D NBD_CLIENT_CONNECTED) { error_setg(&local_err, "Connection closed"); nbd_iter_channel_error(iter, -EIO, &local_err); goto break_loop; @@ -595,7 +606,7 @@ static bool nbd_reply_chunk_iter_receive(NBDClientSessi= on *s, } =20 /* Do not execute the body of NBD_FOREACH_REPLY_CHUNK for simple reply= . */ - if (nbd_reply_is_simple(reply) || s->quit) { + if (nbd_reply_is_simple(reply) || s->state !=3D NBD_CLIENT_CONNECTED) { goto break_loop; } =20 @@ -667,14 +678,14 @@ static int nbd_co_receive_cmdread_reply(NBDClientSess= ion *s, uint64_t handle, ret =3D nbd_parse_offset_hole_payload(&reply.structured, paylo= ad, offset, qiov, &local_err); if (ret < 0) { - s->quit =3D true; + nbd_channel_error(s, ret); nbd_iter_channel_error(&iter, ret, &local_err); } break; default: if (!nbd_reply_type_is_error(chunk->type)) { /* not allowed reply type */ - s->quit =3D true; + nbd_channel_error(s, -EINVAL); error_setg(&local_err, "Unexpected reply type: %d (%s) for CMD_READ", chunk->type, nbd_reply_type_lookup(chunk->type)= ); @@ -714,7 +725,7 @@ static int nbd_co_receive_blockstatus_reply(NBDClientSe= ssion *s, switch (chunk->type) { case NBD_REPLY_TYPE_BLOCK_STATUS: if (received) { - s->quit =3D true; + nbd_channel_error(s, -EINVAL); error_setg(&local_err, "Several BLOCK_STATUS chunks in rep= ly"); nbd_iter_channel_error(&iter, -EINVAL, &local_err); } @@ -724,13 +735,13 @@ static int nbd_co_receive_blockstatus_reply(NBDClient= Session *s, payload, length, extent, &local_err); if (ret < 0) { - s->quit =3D true; + nbd_channel_error(s, ret); nbd_iter_channel_error(&iter, ret, &local_err); } break; default: if (!nbd_reply_type_is_error(chunk->type)) { - s->quit =3D true; + nbd_channel_error(s, -EINVAL); error_setg(&local_err, "Unexpected reply type: %d (%s) " "for CMD_BLOCK_STATUS", --=20 2.11.1 From nobody Wed Nov 5 04:38:10 2025 Delivered-To: importer@patchew.org Received-SPF: pass (zoho.com: domain of gnu.org designates 208.118.235.17 as permitted sender) client-ip=208.118.235.17; envelope-from=qemu-devel-bounces+importer=patchew.org@nongnu.org; helo=lists.gnu.org; Authentication-Results: mx.zohomail.com; spf=pass (zoho.com: domain of gnu.org designates 208.118.235.17 as permitted sender) smtp.mailfrom=qemu-devel-bounces+importer=patchew.org@nongnu.org; dmarc=fail(p=none dis=none) header.from=virtuozzo.com Return-Path: Received: from lists.gnu.org (lists.gnu.org [208.118.235.17]) by mx.zohomail.com with SMTPS id 1533058584045993.5480011940971; Tue, 31 Jul 2018 10:36:24 -0700 (PDT) Received: from localhost ([::1]:60029 helo=lists.gnu.org) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1fkYZK-0005wp-SG for importer@patchew.org; Tue, 31 Jul 2018 13:36:22 -0400 Received: from eggs.gnu.org ([2001:4830:134:3::10]:36934) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1fkYTs-0001Ug-AN for qemu-devel@nongnu.org; Tue, 31 Jul 2018 13:30:46 -0400 Received: from Debian-exim by eggs.gnu.org with spam-scanned (Exim 4.71) (envelope-from ) id 1fkYTp-0004Fv-34 for qemu-devel@nongnu.org; Tue, 31 Jul 2018 13:30:44 -0400 Received: from relay.sw.ru ([185.231.240.75]:60574) by eggs.gnu.org with esmtps (TLS1.0:DHE_RSA_AES_256_CBC_SHA1:32) (Exim 4.71) (envelope-from ) id 1fkYTo-0004EY-Q9; Tue, 31 Jul 2018 13:30:40 -0400 Received: from vz-out.virtuozzo.com ([185.231.240.5] helo=kvm.sw.ru) by relay.sw.ru with esmtp (Exim 4.90_1) (envelope-from ) id 1fkYTl-00080k-My; Tue, 31 Jul 2018 20:30:37 +0300 From: Vladimir Sementsov-Ogievskiy To: qemu-devel@nongnu.org, qemu-block@nongnu.org Date: Tue, 31 Jul 2018 20:30:30 +0300 Message-Id: <20180731173033.75467-8-vsementsov@virtuozzo.com> X-Mailer: git-send-email 2.11.1 In-Reply-To: <20180731173033.75467-1-vsementsov@virtuozzo.com> References: <20180731173033.75467-1-vsementsov@virtuozzo.com> X-detected-operating-system: by eggs.gnu.org: GNU/Linux 3.x [fuzzy] X-Received-From: 185.231.240.75 Subject: [Qemu-devel] [PATCH v4 07/10] block/nbd-client: rename read_reply_co to connection_co X-BeenThere: qemu-devel@nongnu.org X-Mailman-Version: 2.1.21 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Cc: kwolf@redhat.com, vsementsov@virtuozzo.com, armbru@redhat.com, mreitz@redhat.com, den@openvz.org, pbonzini@redhat.com Errors-To: qemu-devel-bounces+importer=patchew.org@nongnu.org Sender: "Qemu-devel" X-ZohoMail: RSF_0 Z_629925259 SPT_0 Content-Transfer-Encoding: quoted-printable MIME-Version: 1.0 Content-Type: text/plain; charset="utf-8" This coroutine will serve nbd reconnects, so, rename it to be something more generic. Signed-off-by: Vladimir Sementsov-Ogievskiy Reviewed-by: Eric Blake --- block/nbd-client.h | 4 ++-- block/nbd-client.c | 24 ++++++++++++------------ 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/block/nbd-client.h b/block/nbd-client.h index 5367425774..e7bda4d3c4 100644 --- a/block/nbd-client.h +++ b/block/nbd-client.h @@ -20,7 +20,7 @@ typedef struct { Coroutine *coroutine; uint64_t offset; /* original offset of the request */ - bool receiving; /* waiting for read_reply_co? */ + bool receiving; /* waiting for connection_co? */ } NBDClientRequest; =20 typedef enum NBDClientState { @@ -37,7 +37,7 @@ typedef struct NBDClientSession { =20 CoMutex send_mutex; CoQueue free_sema; - Coroutine *read_reply_co; + Coroutine *connection_co; int in_flight; NBDClientState state; =20 diff --git a/block/nbd-client.c b/block/nbd-client.c index a91fd3ea3e..c184a9f0e9 100644 --- a/block/nbd-client.c +++ b/block/nbd-client.c @@ -63,7 +63,7 @@ static void nbd_teardown_connection(BlockDriverState *bs) qio_channel_shutdown(client->ioc, QIO_CHANNEL_SHUTDOWN_BOTH, NULL); - BDRV_POLL_WHILE(bs, client->read_reply_co); + BDRV_POLL_WHILE(bs, client->connection_co); =20 nbd_client_detach_aio_context(bs); object_unref(OBJECT(client->sioc)); @@ -72,7 +72,7 @@ static void nbd_teardown_connection(BlockDriverState *bs) client->ioc =3D NULL; } =20 -static coroutine_fn void nbd_read_reply_entry(void *opaque) +static coroutine_fn void nbd_connection_entry(void *opaque) { NBDClientSession *s =3D opaque; uint64_t i; @@ -105,14 +105,14 @@ static coroutine_fn void nbd_read_reply_entry(void *o= paque) } =20 /* We're woken up again by the request itself. Note that there - * is no race between yielding and reentering read_reply_co. This + * is no race between yielding and reentering connection_co. This * is because: * * - if the request runs on the same AioContext, it is only * entered after we yield * * - if the request runs on a different AioContext, reentering - * read_reply_co happens through a bottom half, which can only + * connection_co happens through a bottom half, which can only * run after we yield. */ aio_co_wake(s->requests[i].coroutine); @@ -120,7 +120,7 @@ static coroutine_fn void nbd_read_reply_entry(void *opa= que) } =20 nbd_recv_coroutines_wake_all(s); - s->read_reply_co =3D NULL; + s->connection_co =3D NULL; } =20 static int nbd_co_send_request(BlockDriverState *bs, @@ -428,7 +428,7 @@ static coroutine_fn int nbd_co_do_receive_one_chunk( } *request_ret =3D 0; =20 - /* Wait until we're woken up by nbd_read_reply_entry. */ + /* Wait until we're woken up by nbd_connection_entry. */ s->requests[i].receiving =3D true; qemu_coroutine_yield(); s->requests[i].receiving =3D false; @@ -503,7 +503,7 @@ static coroutine_fn int nbd_co_do_receive_one_chunk( } =20 /* nbd_co_receive_one_chunk - * Read reply, wake up read_reply_co and set s->quit if needed. + * Read reply, wake up connection_co and set s->quit if needed. * Return value is a fatal error code or normal nbd reply error code */ static coroutine_fn int nbd_co_receive_one_chunk( @@ -517,15 +517,15 @@ static coroutine_fn int nbd_co_receive_one_chunk( if (ret < 0) { nbd_channel_error(s, ret); } else { - /* For assert at loop start in nbd_read_reply_entry */ + /* For assert at loop start in nbd_connection_entry */ if (reply) { *reply =3D s->reply; } s->reply.handle =3D 0; } =20 - if (s->read_reply_co) { - aio_co_wake(s->read_reply_co); + if (s->connection_co) { + aio_co_wake(s->connection_co); } =20 return ret; @@ -966,7 +966,7 @@ void nbd_client_attach_aio_context(BlockDriverState *bs, { NBDClientSession *client =3D nbd_get_client_session(bs); qio_channel_attach_aio_context(QIO_CHANNEL(client->ioc), new_context); - aio_co_schedule(new_context, client->read_reply_co); + aio_co_schedule(new_context, client->connection_co); } =20 void nbd_client_close(BlockDriverState *bs) @@ -1066,7 +1066,7 @@ static int nbd_client_connect(BlockDriverState *bs, /* Now that we're connected, set the socket to be non-blocking and * kick the reply mechanism. */ qio_channel_set_blocking(QIO_CHANNEL(sioc), false, NULL); - client->read_reply_co =3D qemu_coroutine_create(nbd_read_reply_entry, = client); + client->connection_co =3D qemu_coroutine_create(nbd_connection_entry, = client); nbd_client_attach_aio_context(bs, bdrv_get_aio_context(bs)); =20 logout("Established connection with NBD server\n"); --=20 2.11.1 From nobody Wed Nov 5 04:38:10 2025 Delivered-To: importer@patchew.org Received-SPF: pass (zoho.com: domain of gnu.org designates 208.118.235.17 as permitted sender) client-ip=208.118.235.17; envelope-from=qemu-devel-bounces+importer=patchew.org@nongnu.org; helo=lists.gnu.org; Authentication-Results: mx.zohomail.com; spf=pass (zoho.com: domain of gnu.org designates 208.118.235.17 as permitted sender) smtp.mailfrom=qemu-devel-bounces+importer=patchew.org@nongnu.org; dmarc=fail(p=none dis=none) header.from=virtuozzo.com Return-Path: Received: from lists.gnu.org (lists.gnu.org [208.118.235.17]) by mx.zohomail.com with SMTPS id 1533058396828199.45510371093997; Tue, 31 Jul 2018 10:33:16 -0700 (PDT) Received: from localhost ([::1]:60006 helo=lists.gnu.org) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1fkYWF-0002u1-Dp for importer@patchew.org; Tue, 31 Jul 2018 13:33:11 -0400 Received: from eggs.gnu.org ([2001:4830:134:3::10]:36929) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1fkYTs-0001Ud-8k for qemu-devel@nongnu.org; Tue, 31 Jul 2018 13:30:45 -0400 Received: from Debian-exim by eggs.gnu.org with spam-scanned (Exim 4.71) (envelope-from ) id 1fkYTo-0004FZ-EH for qemu-devel@nongnu.org; Tue, 31 Jul 2018 13:30:44 -0400 Received: from relay.sw.ru ([185.231.240.75]:60576) by eggs.gnu.org with esmtps (TLS1.0:DHE_RSA_AES_256_CBC_SHA1:32) (Exim 4.71) (envelope-from ) id 1fkYTo-0004Ef-7A; Tue, 31 Jul 2018 13:30:40 -0400 Received: from vz-out.virtuozzo.com ([185.231.240.5] helo=kvm.sw.ru) by relay.sw.ru with esmtp (Exim 4.90_1) (envelope-from ) id 1fkYTm-00080k-8l; Tue, 31 Jul 2018 20:30:38 +0300 From: Vladimir Sementsov-Ogievskiy To: qemu-devel@nongnu.org, qemu-block@nongnu.org Date: Tue, 31 Jul 2018 20:30:31 +0300 Message-Id: <20180731173033.75467-9-vsementsov@virtuozzo.com> X-Mailer: git-send-email 2.11.1 In-Reply-To: <20180731173033.75467-1-vsementsov@virtuozzo.com> References: <20180731173033.75467-1-vsementsov@virtuozzo.com> X-detected-operating-system: by eggs.gnu.org: GNU/Linux 3.x [fuzzy] X-Received-From: 185.231.240.75 Subject: [Qemu-devel] [PATCH v4 08/10] block/nbd: add cmdline and qapi parameter reconnect-delay X-BeenThere: qemu-devel@nongnu.org X-Mailman-Version: 2.1.21 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Cc: kwolf@redhat.com, vsementsov@virtuozzo.com, armbru@redhat.com, mreitz@redhat.com, den@openvz.org, pbonzini@redhat.com Errors-To: qemu-devel-bounces+importer=patchew.org@nongnu.org Sender: "Qemu-devel" X-ZohoMail: RSF_0 Z_629925259 SPT_0 Content-Transfer-Encoding: quoted-printable MIME-Version: 1.0 Content-Type: text/plain; charset="utf-8" Reconnect will be implemented in the following commit, so for now, in semantics below, disconnect itself is a "serious error". Signed-off-by: Vladimir Sementsov-Ogievskiy --- qapi/block-core.json | 12 +++++++++++- block/nbd-client.h | 1 + block/nbd-client.c | 1 + block/nbd.c | 16 +++++++++++++++- 4 files changed, 28 insertions(+), 2 deletions(-) diff --git a/qapi/block-core.json b/qapi/block-core.json index 5b9084a394..cf03402ec5 100644 --- a/qapi/block-core.json +++ b/qapi/block-core.json @@ -3511,13 +3511,23 @@ # traditional "base:allocation" block status (see # NBD_OPT_LIST_META_CONTEXT in the NBD protocol) (since 3= .0) # +# @reconnect-delay: Reconnect delay. On disconnect, nbd client tries to co= nnect +# again, until success or serious error. During first +# @reconnect-delay seconds of reconnecting loop all requ= ests +# are paused and have a chance to rerun, if successful +# connect occures during this time. After @reconnect-del= ay +# seconds all delayed requests are failed and all follow= ing +# requests will be failed to (until successfull reconnec= t). +# Default 300 seconds (Since 3.1) +# # Since: 2.9 ## { 'struct': 'BlockdevOptionsNbd', 'data': { 'server': 'SocketAddress', '*export': 'str', '*tls-creds': 'str', - '*x-dirty-bitmap': 'str' } } + '*x-dirty-bitmap': 'str', + '*reconnect-delay': 'uint32' } } =20 ## # @BlockdevOptionsRaw: diff --git a/block/nbd-client.h b/block/nbd-client.h index e7bda4d3c4..ef8a6a9239 100644 --- a/block/nbd-client.h +++ b/block/nbd-client.h @@ -53,6 +53,7 @@ int nbd_client_init(BlockDriverState *bs, QCryptoTLSCreds *tlscreds, const char *hostname, const char *x_dirty_bitmap, + uint32_t reconnect_delay, Error **errp); void nbd_client_close(BlockDriverState *bs); =20 diff --git a/block/nbd-client.c b/block/nbd-client.c index c184a9f0e9..41e6e6e702 100644 --- a/block/nbd-client.c +++ b/block/nbd-client.c @@ -1079,6 +1079,7 @@ int nbd_client_init(BlockDriverState *bs, QCryptoTLSCreds *tlscreds, const char *hostname, const char *x_dirty_bitmap, + uint32_t reconnect_delay, Error **errp) { NBDClientSession *client =3D nbd_get_client_session(bs); diff --git a/block/nbd.c b/block/nbd.c index 9db5eded89..b40fb5a977 100644 --- a/block/nbd.c +++ b/block/nbd.c @@ -360,6 +360,18 @@ static QemuOptsList nbd_runtime_opts =3D { .help =3D "experimental: expose named dirty bitmap in place of= " "block status", }, + { + .name =3D "reconnect-delay", + .type =3D QEMU_OPT_NUMBER, + .help =3D "Reconnect delay. On disconnect, nbd client tries to" + "connect again, until success or serious error. During" + "first @reconnect-delay seconds of reconnecting loop a= ll" + "requests are paused and have a chance to rerun, if" + "successful connect occures during this time. After" + "@reconnect-delay seconds all delayed requests are fai= led" + "and all following requests will be failed to (until" + "successfull reconnect). Default 300 seconds", + }, { /* end of list */ } }, }; @@ -411,7 +423,9 @@ static int nbd_open(BlockDriverState *bs, QDict *option= s, int flags, =20 /* NBD handshake */ ret =3D nbd_client_init(bs, s->saddr, s->export, tlscreds, hostname, - qemu_opt_get(opts, "x-dirty-bitmap"), errp); + qemu_opt_get(opts, "x-dirty-bitmap"), + qemu_opt_get_number(opts, "reconnect-delay", 300= ), + errp); =20 error: if (tlscreds) { --=20 2.11.1 From nobody Wed Nov 5 04:38:10 2025 Delivered-To: importer@patchew.org Received-SPF: pass (zoho.com: domain of gnu.org designates 208.118.235.17 as permitted sender) client-ip=208.118.235.17; envelope-from=qemu-devel-bounces+importer=patchew.org@nongnu.org; helo=lists.gnu.org; Authentication-Results: mx.zohomail.com; spf=pass (zoho.com: domain of gnu.org designates 208.118.235.17 as permitted sender) smtp.mailfrom=qemu-devel-bounces+importer=patchew.org@nongnu.org; dmarc=fail(p=none dis=none) header.from=virtuozzo.com Return-Path: Received: from lists.gnu.org (208.118.235.17 [208.118.235.17]) by mx.zohomail.com with SMTPS id 1533058398559976.7754990028883; Tue, 31 Jul 2018 10:33:18 -0700 (PDT) Received: from localhost ([::1]:60009 helo=lists.gnu.org) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1fkYWK-0002zq-Sc for importer@patchew.org; Tue, 31 Jul 2018 13:33:16 -0400 Received: from eggs.gnu.org ([2001:4830:134:3::10]:36933) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1fkYTs-0001Uf-AE for qemu-devel@nongnu.org; Tue, 31 Jul 2018 13:30:46 -0400 Received: from Debian-exim by eggs.gnu.org with spam-scanned (Exim 4.71) (envelope-from ) id 1fkYTp-0004Fr-2m for qemu-devel@nongnu.org; Tue, 31 Jul 2018 13:30:44 -0400 Received: from relay.sw.ru ([185.231.240.75]:60582) by eggs.gnu.org with esmtps (TLS1.0:DHE_RSA_AES_256_CBC_SHA1:32) (Exim 4.71) (envelope-from ) id 1fkYTo-0004Em-Jz; Tue, 31 Jul 2018 13:30:41 -0400 Received: from vz-out.virtuozzo.com ([185.231.240.5] helo=kvm.sw.ru) by relay.sw.ru with esmtp (Exim 4.90_1) (envelope-from ) id 1fkYTm-00080k-Vg; Tue, 31 Jul 2018 20:30:39 +0300 From: Vladimir Sementsov-Ogievskiy To: qemu-devel@nongnu.org, qemu-block@nongnu.org Date: Tue, 31 Jul 2018 20:30:32 +0300 Message-Id: <20180731173033.75467-10-vsementsov@virtuozzo.com> X-Mailer: git-send-email 2.11.1 In-Reply-To: <20180731173033.75467-1-vsementsov@virtuozzo.com> References: <20180731173033.75467-1-vsementsov@virtuozzo.com> X-detected-operating-system: by eggs.gnu.org: GNU/Linux 3.x [fuzzy] X-Received-From: 185.231.240.75 Subject: [Qemu-devel] [PATCH v4 09/10] block/nbd-client: nbd reconnect X-BeenThere: qemu-devel@nongnu.org X-Mailman-Version: 2.1.21 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Cc: kwolf@redhat.com, vsementsov@virtuozzo.com, armbru@redhat.com, mreitz@redhat.com, den@openvz.org, pbonzini@redhat.com Errors-To: qemu-devel-bounces+importer=patchew.org@nongnu.org Sender: "Qemu-devel" X-ZohoMail: RSF_0 Z_629925259 SPT_0 Content-Transfer-Encoding: quoted-printable MIME-Version: 1.0 Content-Type: text/plain; charset="utf-8" Implement reconnect. To achieve this: 1. add new modes: connecting-wait: means, that reconnecting is in progress, and there were small number of reconnect attempts, so all requests are waiting for the connection. connecting-nowait: reconnecting is in progress, there were a lot of attempts of reconnect, all requests will return errors. two old modes are used too: connected: normal state quit: exiting after fatal error or on close Possible transitions are: * -> quit connecting-* -> connected connecting-wait -> connecting-nowait (transition is done after reconnect-delay seconds in connecting-wait mode) connected -> connecting-wait 2. Implement reconnect in connection_co. So, in connecting-* mode, connection_co, tries to reconnect unlimited times. 3. Retry nbd queries on channel error, if we are in connecting-wait state. Signed-off-by: Vladimir Sementsov-Ogievskiy --- block/nbd-client.h | 4 + block/nbd-client.c | 304 +++++++++++++++++++++++++++++++++++++++++++------= ---- 2 files changed, 255 insertions(+), 53 deletions(-) diff --git a/block/nbd-client.h b/block/nbd-client.h index ef8a6a9239..52e4ec66be 100644 --- a/block/nbd-client.h +++ b/block/nbd-client.h @@ -40,6 +40,10 @@ typedef struct NBDClientSession { Coroutine *connection_co; int in_flight; NBDClientState state; + bool receiving; + int connect_status; + Error *connect_err; + bool wait_in_flight; =20 NBDClientRequest requests[MAX_NBD_REQUESTS]; NBDReply reply; diff --git a/block/nbd-client.c b/block/nbd-client.c index 41e6e6e702..b09907096d 100644 --- a/block/nbd-client.c +++ b/block/nbd-client.c @@ -34,10 +34,26 @@ #define HANDLE_TO_INDEX(bs, handle) ((handle) ^ (uint64_t)(intptr_t)(bs)) #define INDEX_TO_HANDLE(bs, index) ((index) ^ (uint64_t)(intptr_t)(bs)) =20 -/* @ret would be used for reconnect in future */ +static int nbd_client_connect(BlockDriverState *bs, + SocketAddress *saddr, + const char *export, + QCryptoTLSCreds *tlscreds, + const char *hostname, + const char *x_dirty_bitmap, + Error **errp); + static void nbd_channel_error(NBDClientSession *s, int ret) { - s->state =3D NBD_CLIENT_QUIT; + if (ret =3D=3D -EIO) { + if (s->state =3D=3D NBD_CLIENT_CONNECTED) { + s->state =3D NBD_CLIENT_CONNECTING_WAIT; + } + } else { + if (s->state =3D=3D NBD_CLIENT_CONNECTED) { + qio_channel_shutdown(s->ioc, QIO_CHANNEL_SHUTDOWN_BOTH, NULL); + } + s->state =3D NBD_CLIENT_QUIT; + } } =20 static void nbd_recv_coroutines_wake_all(NBDClientSession *s) @@ -57,33 +73,151 @@ static void nbd_teardown_connection(BlockDriverState *= bs) { NBDClientSession *client =3D nbd_get_client_session(bs); =20 - assert(client->ioc); - - /* finish any pending coroutines */ - qio_channel_shutdown(client->ioc, - QIO_CHANNEL_SHUTDOWN_BOTH, - NULL); + if (client->state =3D=3D NBD_CLIENT_CONNECTED) { + /* finish any pending coroutines */ + assert(client->ioc); + qio_channel_shutdown(client->ioc, QIO_CHANNEL_SHUTDOWN_BOTH, NULL); + } + client->state =3D NBD_CLIENT_QUIT; BDRV_POLL_WHILE(bs, client->connection_co); +} + +typedef struct NBDConnection { + BlockDriverState *bs; + SocketAddress *saddr; + const char *export; + QCryptoTLSCreds *tlscreds; + const char *hostname; + const char *x_dirty_bitmap; + uint32_t reconnect_delay; +} NBDConnection; + +static bool nbd_client_connecting(NBDClientSession *client) +{ + return client->state =3D=3D NBD_CLIENT_CONNECTING_WAIT || + client->state =3D=3D NBD_CLIENT_CONNECTING_NOWAIT; +} + +static bool nbd_client_connecting_wait(NBDClientSession *client) +{ + return client->state =3D=3D NBD_CLIENT_CONNECTING_WAIT; +} + +static coroutine_fn void nbd_reconnect_attempt(NBDConnection *con) +{ + NBDClientSession *s =3D nbd_get_client_session(con->bs); + Error *local_err =3D NULL; + + assert(nbd_client_connecting(s)); + + /* Wait completion of all in-flight requests */ + + qemu_co_mutex_lock(&s->send_mutex); + + while (s->in_flight > 0) { + qemu_co_mutex_unlock(&s->send_mutex); + nbd_recv_coroutines_wake_all(s); + s->wait_in_flight =3D true; + qemu_coroutine_yield(); + s->wait_in_flight =3D false; + qemu_co_mutex_lock(&s->send_mutex); + } + + qemu_co_mutex_unlock(&s->send_mutex); + + /* Now we are sure, that nobody accessing the channel now and nobody + * will try to access the channel, until we set state to CONNECTED + */ + + /* Finalize previous connection if any */ + if (s->ioc) { + nbd_client_detach_aio_context(con->bs); + object_unref(OBJECT(s->sioc)); + s->sioc =3D NULL; + object_unref(OBJECT(s->ioc)); + s->ioc =3D NULL; + } + + s->connect_status =3D nbd_client_connect(con->bs, con->saddr, + con->export, con->tlscreds, + con->hostname, con->x_dirty_bit= map, + &local_err); + error_free(s->connect_err); + s->connect_err =3D NULL; + error_propagate(&s->connect_err, local_err); + local_err =3D NULL; =20 - nbd_client_detach_aio_context(bs); - object_unref(OBJECT(client->sioc)); - client->sioc =3D NULL; - object_unref(OBJECT(client->ioc)); - client->ioc =3D NULL; + if (s->connect_status =3D=3D -EINVAL) { + /* Protocol error or something like this, go to NBD_CLIENT_QUIT */ + nbd_channel_error(s, s->connect_status); + return; + } + + if (s->connect_status < 0) { + /* failed attempt */ + return; + } + + /* successfully connected */ + s->state =3D NBD_CLIENT_CONNECTED; + qemu_co_queue_restart_all(&s->free_sema); +} + +static coroutine_fn void nbd_reconnect_loop(NBDConnection *con) +{ + NBDClientSession *s =3D nbd_get_client_session(con->bs); + uint64_t start_time_ns =3D qemu_clock_get_ns(QEMU_CLOCK_REALTIME); + uint64_t delay_ns =3D con->reconnect_delay * 1000000000UL; + uint64_t timeout =3D 1000000000UL; /* 1 sec */ + uint64_t max_timeout =3D 16000000000UL; /* 16 sec */ + + if (!nbd_client_connecting(s)) { + return; + } + + nbd_reconnect_attempt(con); + + while (nbd_client_connecting(s)) { + if (s->state =3D=3D NBD_CLIENT_CONNECTING_WAIT && + qemu_clock_get_ns(QEMU_CLOCK_REALTIME) - start_time_ns > delay= _ns) + { + s->state =3D NBD_CLIENT_CONNECTING_NOWAIT; + qemu_co_queue_restart_all(&s->free_sema); + } + + qemu_co_sleep_ns(QEMU_CLOCK_REALTIME, timeout); + if (timeout < max_timeout) { + timeout *=3D 2; + } + + nbd_reconnect_attempt(con); + } } =20 static coroutine_fn void nbd_connection_entry(void *opaque) { - NBDClientSession *s =3D opaque; + NBDConnection *con =3D opaque; + NBDClientSession *s =3D nbd_get_client_session(con->bs); uint64_t i; int ret =3D 0; Error *local_err =3D NULL; =20 while (s->state !=3D NBD_CLIENT_QUIT) { + if (nbd_client_connecting(s)) { + nbd_reconnect_loop(con); + } + + if (s->state !=3D NBD_CLIENT_CONNECTED) { + continue; + } + + s->receiving =3D true; assert(s->reply.handle =3D=3D 0); ret =3D nbd_receive_reply(s->ioc, &s->reply, &local_err); + s->receiving =3D false; if (local_err) { error_report_err(local_err); + local_err =3D NULL; } if (ret <=3D 0) { nbd_channel_error(s, ret ? ret : -EIO); @@ -119,8 +253,18 @@ static coroutine_fn void nbd_connection_entry(void *op= aque) qemu_coroutine_yield(); } =20 + qemu_co_queue_restart_all(&s->free_sema); nbd_recv_coroutines_wake_all(s); s->connection_co =3D NULL; + if (s->ioc) { + nbd_client_detach_aio_context(con->bs); + object_unref(OBJECT(s->sioc)); + s->sioc =3D NULL; + object_unref(OBJECT(s->ioc)); + s->ioc =3D NULL; + } + + g_free(con); } =20 static int nbd_co_send_request(BlockDriverState *bs, @@ -131,7 +275,7 @@ static int nbd_co_send_request(BlockDriverState *bs, int rc, i =3D -1; =20 qemu_co_mutex_lock(&s->send_mutex); - while (s->in_flight =3D=3D MAX_NBD_REQUESTS) { + while (s->in_flight =3D=3D MAX_NBD_REQUESTS || nbd_client_connecting_w= ait(s)) { qemu_co_queue_wait(&s->free_sema, &s->send_mutex); } =20 @@ -182,7 +326,11 @@ err: s->requests[i].coroutine =3D NULL; s->in_flight--; } - qemu_co_queue_next(&s->free_sema); + if (s->in_flight =3D=3D 0 && s->wait_in_flight) { + aio_co_wake(s->connection_co); + } else { + qemu_co_queue_next(&s->free_sema); + } } qemu_co_mutex_unlock(&s->send_mutex); return rc; @@ -521,10 +669,13 @@ static coroutine_fn int nbd_co_receive_one_chunk( if (reply) { *reply =3D s->reply; } - s->reply.handle =3D 0; } + s->reply.handle =3D 0; =20 - if (s->connection_co) { + if (s->connection_co && !s->wait_in_flight) { + /* We must check s->wait_in_flight, because we may entered by + * nbd_recv_coroutines_wake_all(), in this case we should not + * wake connection_co here, it will woken by last request. */ aio_co_wake(s->connection_co); } =20 @@ -632,7 +783,11 @@ break_loop: =20 qemu_co_mutex_lock(&s->send_mutex); s->in_flight--; - qemu_co_queue_next(&s->free_sema); + if (s->in_flight =3D=3D 0 && s->wait_in_flight) { + aio_co_wake(s->connection_co); + } else { + qemu_co_queue_next(&s->free_sema); + } qemu_co_mutex_unlock(&s->send_mutex); =20 return false; @@ -781,16 +936,21 @@ static int nbd_co_request(BlockDriverState *bs, NBDRe= quest *request, } else { assert(request->type !=3D NBD_CMD_WRITE); } - ret =3D nbd_co_send_request(bs, request, write_qiov); - if (ret < 0) { - return ret; - } =20 - ret =3D nbd_co_receive_return_code(client, request->handle, - &request_ret, &local_err); - if (local_err) { - error_report_err(local_err); - } + do { + ret =3D nbd_co_send_request(bs, request, write_qiov); + if (ret < 0) { + continue; + } + + ret =3D nbd_co_receive_return_code(client, request->handle, + &request_ret, &local_err); + if (local_err) { + error_report_err(local_err); + local_err =3D NULL; + } + } while (ret < 0 && nbd_client_connecting_wait(client)); + return ret ? ret : request_ret; } =20 @@ -812,16 +972,21 @@ int nbd_client_co_preadv(BlockDriverState *bs, uint64= _t offset, if (!bytes) { return 0; } - ret =3D nbd_co_send_request(bs, &request, NULL); - if (ret < 0) { - return ret; - } =20 - ret =3D nbd_co_receive_cmdread_reply(client, request.handle, offset, q= iov, - &request_ret, &local_err); - if (local_err) { - error_report_err(local_err); - } + do { + ret =3D nbd_co_send_request(bs, &request, NULL); + if (ret < 0) { + continue; + } + + ret =3D nbd_co_receive_cmdread_reply(client, request.handle, offse= t, qiov, + &request_ret, &local_err); + if (local_err) { + error_report_err(local_err); + local_err =3D NULL; + } + } while (ret < 0 && nbd_client_connecting_wait(client)); + return ret ? ret : request_ret; } =20 @@ -935,16 +1100,21 @@ int coroutine_fn nbd_client_co_block_status(BlockDri= verState *bs, return BDRV_BLOCK_DATA; } =20 - ret =3D nbd_co_send_request(bs, &request, NULL); - if (ret < 0) { - return ret; - } + do { + ret =3D nbd_co_send_request(bs, &request, NULL); + if (ret < 0) { + continue; + } + + ret =3D nbd_co_receive_blockstatus_reply(client, request.handle, b= ytes, + &extent, &request_ret, + &local_err); + if (local_err) { + error_report_err(local_err); + local_err =3D NULL; + } + } while (ret < 0 && nbd_client_connecting_wait(client)); =20 - ret =3D nbd_co_receive_blockstatus_reply(client, request.handle, bytes, - &extent, &request_ret, &local_e= rr); - if (local_err) { - error_report_err(local_err); - } if (ret < 0 || request_ret < 0) { return ret ? ret : request_ret; } @@ -966,7 +1136,15 @@ void nbd_client_attach_aio_context(BlockDriverState *= bs, { NBDClientSession *client =3D nbd_get_client_session(bs); qio_channel_attach_aio_context(QIO_CHANNEL(client->ioc), new_context); - aio_co_schedule(new_context, client->connection_co); + if (client->receiving) { + /* We schedule connection_co only if it is waiting for read from t= he + * channel. We should not schedule it if it is some other yield, a= nd if + * it is currently executing (we call nbd_client_attach_aio_contex= t from + * connection code). + */ + + aio_co_schedule(new_context, client->connection_co); + } } =20 void nbd_client_close(BlockDriverState *bs) @@ -974,9 +1152,9 @@ void nbd_client_close(BlockDriverState *bs) NBDClientSession *client =3D nbd_get_client_session(bs); NBDRequest request =3D { .type =3D NBD_CMD_DISC }; =20 - assert(client->ioc); - - nbd_send_request(client->ioc, &request); + if (client->ioc) { + nbd_send_request(client->ioc, &request); + } =20 nbd_teardown_connection(bs); } @@ -1066,7 +1244,6 @@ static int nbd_client_connect(BlockDriverState *bs, /* Now that we're connected, set the socket to be non-blocking and * kick the reply mechanism. */ qio_channel_set_blocking(QIO_CHANNEL(sioc), false, NULL); - client->connection_co =3D qemu_coroutine_create(nbd_connection_entry, = client); nbd_client_attach_aio_context(bs, bdrv_get_aio_context(bs)); =20 logout("Established connection with NBD server\n"); @@ -1082,11 +1259,32 @@ int nbd_client_init(BlockDriverState *bs, uint32_t reconnect_delay, Error **errp) { + int ret; NBDClientSession *client =3D nbd_get_client_session(bs); + NBDConnection *con; =20 qemu_co_mutex_init(&client->send_mutex); qemu_co_queue_init(&client->free_sema); =20 - return nbd_client_connect(bs, saddr, export, tlscreds, hostname, - x_dirty_bitmap, errp); + ret =3D nbd_client_connect(bs, saddr, export, tlscreds, hostname, + x_dirty_bitmap, errp); + if (ret < 0) { + return ret; + } + /* successfully connected */ + client->state =3D NBD_CLIENT_CONNECTED; + + con =3D g_new(NBDConnection, 1); + con->bs =3D bs; + con->saddr =3D saddr; + con->export =3D export; + con->tlscreds =3D tlscreds; + con->hostname =3D hostname; + con->x_dirty_bitmap =3D x_dirty_bitmap; + con->reconnect_delay =3D reconnect_delay; + + client->connection_co =3D qemu_coroutine_create(nbd_connection_entry, = con); + aio_co_schedule(bdrv_get_aio_context(bs), client->connection_co); + + return 0; } --=20 2.11.1 From nobody Wed Nov 5 04:38:10 2025 Delivered-To: importer@patchew.org Received-SPF: pass (zoho.com: domain of gnu.org designates 208.118.235.17 as permitted sender) client-ip=208.118.235.17; envelope-from=qemu-devel-bounces+importer=patchew.org@nongnu.org; helo=lists.gnu.org; Authentication-Results: mx.zohomail.com; spf=pass (zoho.com: domain of gnu.org designates 208.118.235.17 as permitted sender) smtp.mailfrom=qemu-devel-bounces+importer=patchew.org@nongnu.org; dmarc=fail(p=none dis=none) header.from=virtuozzo.com Return-Path: Received: from lists.gnu.org (lists.gnu.org [208.118.235.17]) by mx.zohomail.com with SMTPS id 1533058841380149.2339617288029; Tue, 31 Jul 2018 10:40:41 -0700 (PDT) Received: from localhost ([::1]:60064 helo=lists.gnu.org) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1fkYdU-0000ns-9H for importer@patchew.org; Tue, 31 Jul 2018 13:40:40 -0400 Received: from eggs.gnu.org ([2001:4830:134:3::10]:37003) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1fkYTu-0001Vz-2H for qemu-devel@nongnu.org; Tue, 31 Jul 2018 13:30:47 -0400 Received: from Debian-exim by eggs.gnu.org with spam-scanned (Exim 4.71) (envelope-from ) id 1fkYTp-0004G3-8X for qemu-devel@nongnu.org; Tue, 31 Jul 2018 13:30:46 -0400 Received: from relay.sw.ru ([185.231.240.75]:60584) by eggs.gnu.org with esmtps (TLS1.0:DHE_RSA_AES_256_CBC_SHA1:32) (Exim 4.71) (envelope-from ) id 1fkYTo-0004Ep-Vv; Tue, 31 Jul 2018 13:30:41 -0400 Received: from vz-out.virtuozzo.com ([185.231.240.5] helo=kvm.sw.ru) by relay.sw.ru with esmtp (Exim 4.90_1) (envelope-from ) id 1fkYTn-00080k-AB; Tue, 31 Jul 2018 20:30:39 +0300 From: Vladimir Sementsov-Ogievskiy To: qemu-devel@nongnu.org, qemu-block@nongnu.org Date: Tue, 31 Jul 2018 20:30:33 +0300 Message-Id: <20180731173033.75467-11-vsementsov@virtuozzo.com> X-Mailer: git-send-email 2.11.1 In-Reply-To: <20180731173033.75467-1-vsementsov@virtuozzo.com> References: <20180731173033.75467-1-vsementsov@virtuozzo.com> X-detected-operating-system: by eggs.gnu.org: GNU/Linux 3.x [fuzzy] X-Received-From: 185.231.240.75 Subject: [Qemu-devel] [PATCH v4 10/10] iotests: test nbd reconnect X-BeenThere: qemu-devel@nongnu.org X-Mailman-Version: 2.1.21 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Cc: kwolf@redhat.com, vsementsov@virtuozzo.com, armbru@redhat.com, mreitz@redhat.com, den@openvz.org, pbonzini@redhat.com Errors-To: qemu-devel-bounces+importer=patchew.org@nongnu.org Sender: "Qemu-devel" X-ZohoMail: RSF_0 Z_629925259 SPT_0 Content-Transfer-Encoding: quoted-printable MIME-Version: 1.0 Content-Type: text/plain; charset="utf-8" Add test, which starts backup to nbd target and restarts nbd server during backup. Signed-off-by: Vladimir Sementsov-Ogievskiy --- tests/qemu-iotests/220 | 67 +++++++++++++++++++++++++++++++++++++++= ++++ tests/qemu-iotests/220.out | 7 +++++ tests/qemu-iotests/group | 1 + tests/qemu-iotests/iotests.py | 4 +++ 4 files changed, 79 insertions(+) create mode 100755 tests/qemu-iotests/220 create mode 100644 tests/qemu-iotests/220.out diff --git a/tests/qemu-iotests/220 b/tests/qemu-iotests/220 new file mode 100755 index 0000000000..c9702a7dad --- /dev/null +++ b/tests/qemu-iotests/220 @@ -0,0 +1,67 @@ +#!/usr/bin/env python +# +# Test nbd reconnect +# +# Copyright (c) 2018 Virtuozzo International GmbH +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + +import time + +import iotests +from iotests import qemu_img_create, file_path, qemu_nbd_popen + +disk_a, disk_b, nbd_sock =3D file_path('disk_a', 'disk_b', 'nbd-sock') +nbd_uri =3D 'nbd+unix:///exp?socket=3D' + nbd_sock + +qemu_img_create('-f', iotests.imgfmt, disk_a, '5M') +qemu_img_create('-f', iotests.imgfmt, disk_b, '5M') +srv =3D qemu_nbd_popen('-k', nbd_sock, '-x', 'exp', '-f', iotests.imgfmt, = disk_b) +time.sleep(1) + +vm =3D iotests.VM().add_drive(disk_a) +vm.launch() +vm.hmp_qemu_io('drive0', 'write 0 5M') + +print 'blockdev-add:', vm.qmp('blockdev-add', node_name=3D'backup0', drive= r=3D'raw', + file=3D{'driver':'nbd', + 'export': 'exp', + 'server': {'type': 'unix', + 'path': nbd_sock}}) +print 'blockdev-backup:', vm.qmp('blockdev-backup', device=3D'drive0', + sync=3D'full', target=3D'backup0') + +time.sleep(1) +print 'Kill NBD server' +srv.kill() + +jobs =3D vm.qmp('query-block-jobs')['return'] +if jobs and jobs[0]['offset'] < jobs[0]['len']: + print 'Backup job is still in progress' + +time.sleep(1) + +print 'Start NBD server' +srv =3D qemu_nbd_popen('-k', nbd_sock, '-x', 'exp', '-f', iotests.imgfmt, = disk_b) + +try: + e =3D vm.event_wait('BLOCK_JOB_COMPLETED') + print e['event'], ':', e['data'] +except: + pass + +print 'blockdev-del:', vm.qmp('blockdev-del', node_name=3D'backup0') +srv.kill() +vm.shutdown() diff --git a/tests/qemu-iotests/220.out b/tests/qemu-iotests/220.out new file mode 100644 index 0000000000..dae1a49d9f --- /dev/null +++ b/tests/qemu-iotests/220.out @@ -0,0 +1,7 @@ +blockdev-add: {u'return': {}} +blockdev-backup: {u'return': {}} +Kill NBD server +Backup job is still in progress +Start NBD server +BLOCK_JOB_COMPLETED : {u'device': u'drive0', u'type': u'backup', u'speed':= 0, u'len': 5242880, u'offset': 5242880} +blockdev-del: {u'return': {}} diff --git a/tests/qemu-iotests/group b/tests/qemu-iotests/group index b973dc842d..ee2473c6a3 100644 --- a/tests/qemu-iotests/group +++ b/tests/qemu-iotests/group @@ -219,6 +219,7 @@ 217 rw auto quick 218 rw auto quick 219 rw auto +220 rw auto quick 221 rw auto quick 222 rw auto quick 223 rw auto quick diff --git a/tests/qemu-iotests/iotests.py b/tests/qemu-iotests/iotests.py index 4e67fbbe96..17bc8c8e32 100644 --- a/tests/qemu-iotests/iotests.py +++ b/tests/qemu-iotests/iotests.py @@ -185,6 +185,10 @@ def qemu_nbd(*args): '''Run qemu-nbd in daemon mode and return the parent's exit code''' return subprocess.call(qemu_nbd_args + ['--fork'] + list(args)) =20 +def qemu_nbd_popen(*args): + '''Run qemu-nbd in daemon mode and return the parent's exit code''' + return subprocess.Popen(qemu_nbd_args + ['--persistent'] + list(args)) + def compare_images(img1, img2, fmt1=3Dimgfmt, fmt2=3Dimgfmt): '''Return True if two image files are identical''' return qemu_img('compare', '-f', fmt1, --=20 2.11.1