[PATCH v3 2/2] nbd: Clean up clients more efficiently

Eric Blake posted 2 patches 3 months, 2 weeks ago
[PATCH v3 2/2] nbd: Clean up clients more efficiently
Posted by Eric Blake 3 months, 2 weeks ago
Since an NBD server may be long-living, serving clients that
repeatedly connect and disconnect, it can be more efficient to clean
up after each client disconnects, rather than storing a list of
resources to clean up when the server exits.  Rewrite the list of
known clients to be double-linked so that we can get O(1) deletion to
keep the list pruned to size as clients exit.  This in turn requires
each client to track an opaque pointer of owner information (although
qemu-nbd doesn't need to refer to it).

Signed-off-by: Eric Blake <eblake@redhat.com>
---
 include/block/nbd.h |  4 +++-
 blockdev-nbd.c      | 27 ++++++++++++++++-----------
 nbd/server.c        | 15 ++++++++++++---
 qemu-nbd.c          |  2 +-
 4 files changed, 32 insertions(+), 16 deletions(-)

diff --git a/include/block/nbd.h b/include/block/nbd.h
index 4e7bd6342f9..7dce9b9c35b 100644
--- a/include/block/nbd.h
+++ b/include/block/nbd.h
@@ -405,7 +405,9 @@ NBDExport *nbd_export_find(const char *name);
 void nbd_client_new(QIOChannelSocket *sioc,
                     QCryptoTLSCreds *tlscreds,
                     const char *tlsauthz,
-                    void (*close_fn)(NBDClient *, bool));
+                    void (*close_fn)(NBDClient *, bool),
+                    void *owner);
+void *nbd_client_owner(NBDClient *client);
 void nbd_client_get(NBDClient *client);
 void nbd_client_put(NBDClient *client);

diff --git a/blockdev-nbd.c b/blockdev-nbd.c
index b8f00f402c6..660f89d881e 100644
--- a/blockdev-nbd.c
+++ b/blockdev-nbd.c
@@ -23,7 +23,7 @@

 typedef struct NBDConn {
     QIOChannelSocket *cioc;
-    QSLIST_ENTRY(NBDConn) next;
+    QLIST_ENTRY(NBDConn) next;
 } NBDConn;

 typedef struct NBDServerData {
@@ -32,10 +32,11 @@ typedef struct NBDServerData {
     char *tlsauthz;
     uint32_t max_connections;
     uint32_t connections;
-    QSLIST_HEAD(, NBDConn) conns;
+    QLIST_HEAD(, NBDConn) conns;
 } NBDServerData;

 static NBDServerData *nbd_server;
+static uint32_t nbd_cookie; /* Generation count of nbd_server */
 static int qemu_nbd_connections = -1; /* Non-negative if this is qemu-nbd */

 static void nbd_update_server_watch(NBDServerData *s);
@@ -57,10 +58,16 @@ int nbd_server_max_connections(void)

 static void nbd_blockdev_client_closed(NBDClient *client, bool ignored)
 {
+    NBDConn *conn = nbd_client_owner(client);
+
     assert(qemu_in_main_thread() && nbd_server);

+    object_unref(OBJECT(conn->cioc));
+    QLIST_REMOVE(conn, next);
+    g_free(conn);
+
     nbd_client_put(client);
-    assert(nbd_server->connections > 0);
+    assert(nbd_server && nbd_server->connections > 0);
     nbd_server->connections--;
     nbd_update_server_watch(nbd_server);
 }
@@ -74,12 +81,12 @@ static void nbd_accept(QIONetListener *listener, QIOChannelSocket *cioc,
     nbd_server->connections++;
     object_ref(OBJECT(cioc));
     conn->cioc = cioc;
-    QSLIST_INSERT_HEAD(&nbd_server->conns, conn, next);
+    QLIST_INSERT_HEAD(&nbd_server->conns, conn, next);
     nbd_update_server_watch(nbd_server);

     qio_channel_set_name(QIO_CHANNEL(cioc), "nbd-server");
     nbd_client_new(cioc, nbd_server->tlscreds, nbd_server->tlsauthz,
-                   nbd_blockdev_client_closed);
+                   nbd_blockdev_client_closed, conn);
 }

 static void nbd_update_server_watch(NBDServerData *s)
@@ -93,6 +100,8 @@ static void nbd_update_server_watch(NBDServerData *s)

 static void nbd_server_free(NBDServerData *server)
 {
+    NBDConn *conn, *tmp;
+
     if (!server) {
         return;
     }
@@ -103,14 +112,9 @@ static void nbd_server_free(NBDServerData *server)
      */
     qio_net_listener_disconnect(server->listener);
     object_unref(OBJECT(server->listener));
-    while (!QSLIST_EMPTY(&server->conns)) {
-        NBDConn *conn = QSLIST_FIRST(&server->conns);
-
+    QLIST_FOREACH_SAFE(conn, &server->conns, next, tmp) {
         qio_channel_shutdown(QIO_CHANNEL(conn->cioc), QIO_CHANNEL_SHUTDOWN_BOTH,
                              NULL);
-        object_unref(OBJECT(conn->cioc));
-        QSLIST_REMOVE_HEAD(&server->conns, next);
-        g_free(conn);
     }

     AIO_WAIT_WHILE_UNLOCKED(NULL, server->connections > 0);
@@ -119,6 +123,7 @@ static void nbd_server_free(NBDServerData *server)
         object_unref(OBJECT(server->tlscreds));
     }
     g_free(server->tlsauthz);
+    nbd_cookie++;

     g_free(server);
 }
diff --git a/nbd/server.c b/nbd/server.c
index 892797bb111..90f48b42a47 100644
--- a/nbd/server.c
+++ b/nbd/server.c
@@ -124,6 +124,7 @@ struct NBDMetaContexts {
 struct NBDClient {
     int refcount; /* atomic */
     void (*close_fn)(NBDClient *client, bool negotiated);
+    void *owner;

     QemuMutex lock;

@@ -3205,14 +3206,15 @@ static coroutine_fn void nbd_co_client_start(void *opaque)
 }

 /*
- * Create a new client listener using the given channel @sioc.
+ * Create a new client listener using the given channel @sioc and @owner.
  * Begin servicing it in a coroutine.  When the connection closes, call
- * @close_fn with an indication of whether the client completed negotiation.
+ * @close_fn and an indication of whether the client completed negotiation.
  */
 void nbd_client_new(QIOChannelSocket *sioc,
                     QCryptoTLSCreds *tlscreds,
                     const char *tlsauthz,
-                    void (*close_fn)(NBDClient *, bool))
+                    void (*close_fn)(NBDClient *, bool),
+                    void *owner)
 {
     NBDClient *client;
     Coroutine *co;
@@ -3231,7 +3233,14 @@ void nbd_client_new(QIOChannelSocket *sioc,
     client->ioc = QIO_CHANNEL(sioc);
     object_ref(OBJECT(client->ioc));
     client->close_fn = close_fn;
+    client->owner = owner;

     co = qemu_coroutine_create(nbd_co_client_start, client);
     qemu_coroutine_enter(co);
 }
+
+void *
+nbd_client_owner(NBDClient *client)
+{
+    return client->owner;
+}
diff --git a/qemu-nbd.c b/qemu-nbd.c
index d7b3ccab21c..da6e36a2a34 100644
--- a/qemu-nbd.c
+++ b/qemu-nbd.c
@@ -390,7 +390,7 @@ static void nbd_accept(QIONetListener *listener, QIOChannelSocket *cioc,

     nb_fds++;
     nbd_update_server_watch();
-    nbd_client_new(cioc, tlscreds, tlsauthz, nbd_client_closed);
+    nbd_client_new(cioc, tlscreds, tlsauthz, nbd_client_closed, NULL);
 }

 static void nbd_update_server_watch(void)
-- 
2.45.2
Re: [PATCH v3 2/2] nbd: Clean up clients more efficiently
Posted by Daniel P. Berrangé 3 months, 2 weeks ago
On Mon, Aug 05, 2024 at 09:21:36PM -0500, Eric Blake wrote:
> Since an NBD server may be long-living, serving clients that
> repeatedly connect and disconnect, it can be more efficient to clean
> up after each client disconnects, rather than storing a list of
> resources to clean up when the server exits.  Rewrite the list of
> known clients to be double-linked so that we can get O(1) deletion to
> keep the list pruned to size as clients exit.  This in turn requires
> each client to track an opaque pointer of owner information (although
> qemu-nbd doesn't need to refer to it).

I tend to feel that this needs to be squashed into the previous
patch.  The previous patch effectively creates unbounded memory
usage in the NBD server. ie consider a client that connects and
immediately disconnects, not sending any data, in a tight loop.
It will "leak" NBDConn & QIOChanelSocket pointers for each
iteration of the loop, only to be cleaned up when the NBD Server
is shutdown.

Especially given that we tagged the previous commit with a CVE
and not this commit,  people could be misled into backporting
only the former commit leaving them open to the leak.

> 
> Signed-off-by: Eric Blake <eblake@redhat.com>
> ---
>  include/block/nbd.h |  4 +++-
>  blockdev-nbd.c      | 27 ++++++++++++++++-----------
>  nbd/server.c        | 15 ++++++++++++---
>  qemu-nbd.c          |  2 +-
>  4 files changed, 32 insertions(+), 16 deletions(-)
> 
> diff --git a/include/block/nbd.h b/include/block/nbd.h
> index 4e7bd6342f9..7dce9b9c35b 100644
> --- a/include/block/nbd.h
> +++ b/include/block/nbd.h
> @@ -405,7 +405,9 @@ NBDExport *nbd_export_find(const char *name);
>  void nbd_client_new(QIOChannelSocket *sioc,
>                      QCryptoTLSCreds *tlscreds,
>                      const char *tlsauthz,
> -                    void (*close_fn)(NBDClient *, bool));
> +                    void (*close_fn)(NBDClient *, bool),
> +                    void *owner);
> +void *nbd_client_owner(NBDClient *client);
>  void nbd_client_get(NBDClient *client);
>  void nbd_client_put(NBDClient *client);
> 
> diff --git a/blockdev-nbd.c b/blockdev-nbd.c
> index b8f00f402c6..660f89d881e 100644
> --- a/blockdev-nbd.c
> +++ b/blockdev-nbd.c
> @@ -23,7 +23,7 @@
> 
>  typedef struct NBDConn {
>      QIOChannelSocket *cioc;
> -    QSLIST_ENTRY(NBDConn) next;
> +    QLIST_ENTRY(NBDConn) next;
>  } NBDConn;
> 
>  typedef struct NBDServerData {
> @@ -32,10 +32,11 @@ typedef struct NBDServerData {
>      char *tlsauthz;
>      uint32_t max_connections;
>      uint32_t connections;
> -    QSLIST_HEAD(, NBDConn) conns;
> +    QLIST_HEAD(, NBDConn) conns;
>  } NBDServerData;
> 
>  static NBDServerData *nbd_server;
> +static uint32_t nbd_cookie; /* Generation count of nbd_server */
>  static int qemu_nbd_connections = -1; /* Non-negative if this is qemu-nbd */
> 
>  static void nbd_update_server_watch(NBDServerData *s);
> @@ -57,10 +58,16 @@ int nbd_server_max_connections(void)
> 
>  static void nbd_blockdev_client_closed(NBDClient *client, bool ignored)
>  {
> +    NBDConn *conn = nbd_client_owner(client);
> +
>      assert(qemu_in_main_thread() && nbd_server);
> 
> +    object_unref(OBJECT(conn->cioc));
> +    QLIST_REMOVE(conn, next);
> +    g_free(conn);
> +
>      nbd_client_put(client);
> -    assert(nbd_server->connections > 0);
> +    assert(nbd_server && nbd_server->connections > 0);
>      nbd_server->connections--;
>      nbd_update_server_watch(nbd_server);
>  }
> @@ -74,12 +81,12 @@ static void nbd_accept(QIONetListener *listener, QIOChannelSocket *cioc,
>      nbd_server->connections++;
>      object_ref(OBJECT(cioc));
>      conn->cioc = cioc;
> -    QSLIST_INSERT_HEAD(&nbd_server->conns, conn, next);
> +    QLIST_INSERT_HEAD(&nbd_server->conns, conn, next);
>      nbd_update_server_watch(nbd_server);
> 
>      qio_channel_set_name(QIO_CHANNEL(cioc), "nbd-server");
>      nbd_client_new(cioc, nbd_server->tlscreds, nbd_server->tlsauthz,
> -                   nbd_blockdev_client_closed);
> +                   nbd_blockdev_client_closed, conn);
>  }
> 
>  static void nbd_update_server_watch(NBDServerData *s)
> @@ -93,6 +100,8 @@ static void nbd_update_server_watch(NBDServerData *s)
> 
>  static void nbd_server_free(NBDServerData *server)
>  {
> +    NBDConn *conn, *tmp;
> +
>      if (!server) {
>          return;
>      }
> @@ -103,14 +112,9 @@ static void nbd_server_free(NBDServerData *server)
>       */
>      qio_net_listener_disconnect(server->listener);
>      object_unref(OBJECT(server->listener));
> -    while (!QSLIST_EMPTY(&server->conns)) {
> -        NBDConn *conn = QSLIST_FIRST(&server->conns);
> -
> +    QLIST_FOREACH_SAFE(conn, &server->conns, next, tmp) {
>          qio_channel_shutdown(QIO_CHANNEL(conn->cioc), QIO_CHANNEL_SHUTDOWN_BOTH,
>                               NULL);
> -        object_unref(OBJECT(conn->cioc));
> -        QSLIST_REMOVE_HEAD(&server->conns, next);
> -        g_free(conn);
>      }
> 
>      AIO_WAIT_WHILE_UNLOCKED(NULL, server->connections > 0);
> @@ -119,6 +123,7 @@ static void nbd_server_free(NBDServerData *server)
>          object_unref(OBJECT(server->tlscreds));
>      }
>      g_free(server->tlsauthz);
> +    nbd_cookie++;
> 
>      g_free(server);
>  }
> diff --git a/nbd/server.c b/nbd/server.c
> index 892797bb111..90f48b42a47 100644
> --- a/nbd/server.c
> +++ b/nbd/server.c
> @@ -124,6 +124,7 @@ struct NBDMetaContexts {
>  struct NBDClient {
>      int refcount; /* atomic */
>      void (*close_fn)(NBDClient *client, bool negotiated);
> +    void *owner;
> 
>      QemuMutex lock;
> 
> @@ -3205,14 +3206,15 @@ static coroutine_fn void nbd_co_client_start(void *opaque)
>  }
> 
>  /*
> - * Create a new client listener using the given channel @sioc.
> + * Create a new client listener using the given channel @sioc and @owner.
>   * Begin servicing it in a coroutine.  When the connection closes, call
> - * @close_fn with an indication of whether the client completed negotiation.
> + * @close_fn and an indication of whether the client completed negotiation.
>   */
>  void nbd_client_new(QIOChannelSocket *sioc,
>                      QCryptoTLSCreds *tlscreds,
>                      const char *tlsauthz,
> -                    void (*close_fn)(NBDClient *, bool))
> +                    void (*close_fn)(NBDClient *, bool),
> +                    void *owner)
>  {
>      NBDClient *client;
>      Coroutine *co;
> @@ -3231,7 +3233,14 @@ void nbd_client_new(QIOChannelSocket *sioc,
>      client->ioc = QIO_CHANNEL(sioc);
>      object_ref(OBJECT(client->ioc));
>      client->close_fn = close_fn;
> +    client->owner = owner;
> 
>      co = qemu_coroutine_create(nbd_co_client_start, client);
>      qemu_coroutine_enter(co);
>  }
> +
> +void *
> +nbd_client_owner(NBDClient *client)
> +{
> +    return client->owner;
> +}
> diff --git a/qemu-nbd.c b/qemu-nbd.c
> index d7b3ccab21c..da6e36a2a34 100644
> --- a/qemu-nbd.c
> +++ b/qemu-nbd.c
> @@ -390,7 +390,7 @@ static void nbd_accept(QIONetListener *listener, QIOChannelSocket *cioc,
> 
>      nb_fds++;
>      nbd_update_server_watch();
> -    nbd_client_new(cioc, tlscreds, tlsauthz, nbd_client_closed);
> +    nbd_client_new(cioc, tlscreds, tlsauthz, nbd_client_closed, NULL);
>  }
> 
>  static void nbd_update_server_watch(void)
> -- 
> 2.45.2
> 

With regards,
Daniel
-- 
|: https://berrange.com      -o-    https://www.flickr.com/photos/dberrange :|
|: https://libvirt.org         -o-            https://fstop138.berrange.com :|
|: https://entangle-photo.org    -o-    https://www.instagram.com/dberrange :|
Re: [PATCH v3 2/2] nbd: Clean up clients more efficiently
Posted by Eric Blake 3 months, 2 weeks ago
On Tue, Aug 06, 2024 at 10:32:54AM GMT, Daniel P. Berrangé wrote:
> On Mon, Aug 05, 2024 at 09:21:36PM -0500, Eric Blake wrote:
> > Since an NBD server may be long-living, serving clients that
> > repeatedly connect and disconnect, it can be more efficient to clean
> > up after each client disconnects, rather than storing a list of
> > resources to clean up when the server exits.  Rewrite the list of
> > known clients to be double-linked so that we can get O(1) deletion to
> > keep the list pruned to size as clients exit.  This in turn requires
> > each client to track an opaque pointer of owner information (although
> > qemu-nbd doesn't need to refer to it).
> 
> I tend to feel that this needs to be squashed into the previous
> patch.  The previous patch effectively creates unbounded memory
> usage in the NBD server. ie consider a client that connects and
> immediately disconnects, not sending any data, in a tight loop.
> It will "leak" NBDConn & QIOChanelSocket pointers for each
> iteration of the loop, only to be cleaned up when the NBD Server
> is shutdown.

Hmm. Offline, we observed that qemu's default of unlimited clients may
not be the smartest thing - if max-connections is not set, we may want
to default it to something like 100 (even when MULTI_CONN is
exploited, most clients that take advantage of it will probably only
use 8 or 16 parallel sockets; more than that tends to run into other
limits rather than providing continual improvements).  I can add such
a change in default in v4.  In contrast, qemu-nbd defaults to
max-connections 1 for historical reasons (among others, if you have a
non-persistent server, it is really nice that qemu-nbd automatically
exits after the first client disconnects, rather than needing to clean
up the server yourself), but that's too small for MULTI_CONN to be of
any use.

-- 
Eric Blake, Principal Software Engineer
Red Hat, Inc.
Virtualization:  qemu.org | libguestfs.org


Re: [PATCH v3 2/2] nbd: Clean up clients more efficiently
Posted by Eric Blake 3 months, 2 weeks ago
On Tue, Aug 06, 2024 at 10:32:54AM GMT, Daniel P. Berrangé wrote:
> On Mon, Aug 05, 2024 at 09:21:36PM -0500, Eric Blake wrote:
> > Since an NBD server may be long-living, serving clients that
> > repeatedly connect and disconnect, it can be more efficient to clean
> > up after each client disconnects, rather than storing a list of
> > resources to clean up when the server exits.  Rewrite the list of
> > known clients to be double-linked so that we can get O(1) deletion to
> > keep the list pruned to size as clients exit.  This in turn requires
> > each client to track an opaque pointer of owner information (although
> > qemu-nbd doesn't need to refer to it).
> 
> I tend to feel that this needs to be squashed into the previous
> patch.  The previous patch effectively creates unbounded memory
> usage in the NBD server. ie consider a client that connects and
> immediately disconnects, not sending any data, in a tight loop.
> It will "leak" NBDConn & QIOChanelSocket pointers for each
> iteration of the loop, only to be cleaned up when the NBD Server
> is shutdown.
> 
> Especially given that we tagged the previous commit with a CVE
> and not this commit,  people could be misled into backporting
> only the former commit leaving them open to the leak.

Makes sense.  Will respin v4 along those lines; although I plan to
refactor slightly: have patch 1 pick up the part of this patch that
allows server.c to track a client's owner, then patch 2 be the CVE fix
creating the doubly-linked list of QIOSocketChannel wrappers as
owners.  Also, I realized that nbd_server->connections == 0 is now
effectively redundant with QLIST_EMPTY(&nbd_server->conns), so that's
another cleanup to squash in.

> > @@ -103,14 +112,9 @@ static void nbd_server_free(NBDServerData *server)
> >       */
> >      qio_net_listener_disconnect(server->listener);
> >      object_unref(OBJECT(server->listener));
> > -    while (!QSLIST_EMPTY(&server->conns)) {
> > -        NBDConn *conn = QSLIST_FIRST(&server->conns);
> > -
> > +    QLIST_FOREACH_SAFE(conn, &server->conns, next, tmp) {
> >          qio_channel_shutdown(QIO_CHANNEL(conn->cioc), QIO_CHANNEL_SHUTDOWN_BOTH,
> >                               NULL);
> > -        object_unref(OBJECT(conn->cioc));
> > -        QSLIST_REMOVE_HEAD(&server->conns, next);
> > -        g_free(conn);
> >      }
> > 
> >      AIO_WAIT_WHILE_UNLOCKED(NULL, server->connections > 0);

-- 
Eric Blake, Principal Software Engineer
Red Hat, Inc.
Virtualization:  qemu.org | libguestfs.org


Re: [PATCH v3 2/2] nbd: Clean up clients more efficiently
Posted by Eric Blake 3 months, 2 weeks ago
On Mon, Aug 05, 2024 at 09:21:36PM GMT, Eric Blake wrote:
> Since an NBD server may be long-living, serving clients that
> repeatedly connect and disconnect, it can be more efficient to clean
> up after each client disconnects, rather than storing a list of
> resources to clean up when the server exits.  Rewrite the list of
> known clients to be double-linked so that we can get O(1) deletion to
> keep the list pruned to size as clients exit.  This in turn requires
> each client to track an opaque pointer of owner information (although
> qemu-nbd doesn't need to refer to it).
> 
> Signed-off-by: Eric Blake <eblake@redhat.com>
> ---
>  include/block/nbd.h |  4 +++-
>  blockdev-nbd.c      | 27 ++++++++++++++++-----------
>  nbd/server.c        | 15 ++++++++++++---
>  qemu-nbd.c          |  2 +-
>  4 files changed, 32 insertions(+), 16 deletions(-)
> 
> diff --git a/include/block/nbd.h b/include/block/nbd.h
> index 4e7bd6342f9..7dce9b9c35b 100644
> --- a/include/block/nbd.h
> +++ b/include/block/nbd.h
> @@ -405,7 +405,9 @@ NBDExport *nbd_export_find(const char *name);
>  void nbd_client_new(QIOChannelSocket *sioc,
>                      QCryptoTLSCreds *tlscreds,
>                      const char *tlsauthz,
> -                    void (*close_fn)(NBDClient *, bool));
> +                    void (*close_fn)(NBDClient *, bool),
> +                    void *owner);
> +void *nbd_client_owner(NBDClient *client);
>  void nbd_client_get(NBDClient *client);
>  void nbd_client_put(NBDClient *client);
> 
> diff --git a/blockdev-nbd.c b/blockdev-nbd.c
> index b8f00f402c6..660f89d881e 100644
> --- a/blockdev-nbd.c
> +++ b/blockdev-nbd.c
> @@ -23,7 +23,7 @@
> 
>  typedef struct NBDConn {
>      QIOChannelSocket *cioc;
> -    QSLIST_ENTRY(NBDConn) next;
> +    QLIST_ENTRY(NBDConn) next;
>  } NBDConn;
> 
>  typedef struct NBDServerData {
> @@ -32,10 +32,11 @@ typedef struct NBDServerData {
>      char *tlsauthz;
>      uint32_t max_connections;
>      uint32_t connections;
> -    QSLIST_HEAD(, NBDConn) conns;
> +    QLIST_HEAD(, NBDConn) conns;
>  } NBDServerData;
> 
>  static NBDServerData *nbd_server;
> +static uint32_t nbd_cookie; /* Generation count of nbd_server */

Dead line leftover from v2; gone in my tree now.

>  static int qemu_nbd_connections = -1; /* Non-negative if this is qemu-nbd */
> 
>  static void nbd_update_server_watch(NBDServerData *s);
> @@ -57,10 +58,16 @@ int nbd_server_max_connections(void)
> 
>  static void nbd_blockdev_client_closed(NBDClient *client, bool ignored)
>  {
> +    NBDConn *conn = nbd_client_owner(client);
> +
>      assert(qemu_in_main_thread() && nbd_server);
> 
> +    object_unref(OBJECT(conn->cioc));
> +    QLIST_REMOVE(conn, next);
> +    g_free(conn);
> +
>      nbd_client_put(client);
> -    assert(nbd_server->connections > 0);
> +    assert(nbd_server && nbd_server->connections > 0);

The 'nbd_server && ' in this hunk is redundant with a couple lines above.

>      nbd_server->connections--;
>      nbd_update_server_watch(nbd_server);
>  }
> @@ -74,12 +81,12 @@ static void nbd_accept(QIONetListener *listener, QIOChannelSocket *cioc,
>      nbd_server->connections++;
>      object_ref(OBJECT(cioc));
>      conn->cioc = cioc;
> -    QSLIST_INSERT_HEAD(&nbd_server->conns, conn, next);
> +    QLIST_INSERT_HEAD(&nbd_server->conns, conn, next);
>      nbd_update_server_watch(nbd_server);
> 
>      qio_channel_set_name(QIO_CHANNEL(cioc), "nbd-server");
>      nbd_client_new(cioc, nbd_server->tlscreds, nbd_server->tlsauthz,
> -                   nbd_blockdev_client_closed);
> +                   nbd_blockdev_client_closed, conn);
>  }
> 
>  static void nbd_update_server_watch(NBDServerData *s)
> @@ -93,6 +100,8 @@ static void nbd_update_server_watch(NBDServerData *s)
> 
>  static void nbd_server_free(NBDServerData *server)
>  {
> +    NBDConn *conn, *tmp;
> +
>      if (!server) {
>          return;
>      }
> @@ -103,14 +112,9 @@ static void nbd_server_free(NBDServerData *server)
>       */
>      qio_net_listener_disconnect(server->listener);
>      object_unref(OBJECT(server->listener));
> -    while (!QSLIST_EMPTY(&server->conns)) {
> -        NBDConn *conn = QSLIST_FIRST(&server->conns);
> -
> +    QLIST_FOREACH_SAFE(conn, &server->conns, next, tmp) {
>          qio_channel_shutdown(QIO_CHANNEL(conn->cioc), QIO_CHANNEL_SHUTDOWN_BOTH,
>                               NULL);
> -        object_unref(OBJECT(conn->cioc));
> -        QSLIST_REMOVE_HEAD(&server->conns, next);
> -        g_free(conn);
>      }
> 
>      AIO_WAIT_WHILE_UNLOCKED(NULL, server->connections > 0);
> @@ -119,6 +123,7 @@ static void nbd_server_free(NBDServerData *server)
>          object_unref(OBJECT(server->tlscreds));
>      }
>      g_free(server->tlsauthz);
> +    nbd_cookie++;

One more dead line.

> 
>      g_free(server);
>  }
> diff --git a/nbd/server.c b/nbd/server.c
> index 892797bb111..90f48b42a47 100644
> --- a/nbd/server.c
> +++ b/nbd/server.c
> @@ -124,6 +124,7 @@ struct NBDMetaContexts {
>  struct NBDClient {
>      int refcount; /* atomic */
>      void (*close_fn)(NBDClient *client, bool negotiated);
> +    void *owner;
> 
>      QemuMutex lock;
> 
> @@ -3205,14 +3206,15 @@ static coroutine_fn void nbd_co_client_start(void *opaque)
>  }
> 
>  /*
> - * Create a new client listener using the given channel @sioc.
> + * Create a new client listener using the given channel @sioc and @owner.
>   * Begin servicing it in a coroutine.  When the connection closes, call
> - * @close_fn with an indication of whether the client completed negotiation.
> + * @close_fn and an indication of whether the client completed negotiation.
>   */
>  void nbd_client_new(QIOChannelSocket *sioc,
>                      QCryptoTLSCreds *tlscreds,
>                      const char *tlsauthz,
> -                    void (*close_fn)(NBDClient *, bool))
> +                    void (*close_fn)(NBDClient *, bool),
> +                    void *owner)
>  {
>      NBDClient *client;
>      Coroutine *co;
> @@ -3231,7 +3233,14 @@ void nbd_client_new(QIOChannelSocket *sioc,
>      client->ioc = QIO_CHANNEL(sioc);
>      object_ref(OBJECT(client->ioc));
>      client->close_fn = close_fn;
> +    client->owner = owner;
> 
>      co = qemu_coroutine_create(nbd_co_client_start, client);
>      qemu_coroutine_enter(co);
>  }
> +
> +void *
> +nbd_client_owner(NBDClient *client)
> +{
> +    return client->owner;
> +}
> diff --git a/qemu-nbd.c b/qemu-nbd.c
> index d7b3ccab21c..da6e36a2a34 100644
> --- a/qemu-nbd.c
> +++ b/qemu-nbd.c
> @@ -390,7 +390,7 @@ static void nbd_accept(QIONetListener *listener, QIOChannelSocket *cioc,
> 
>      nb_fds++;
>      nbd_update_server_watch();
> -    nbd_client_new(cioc, tlscreds, tlsauthz, nbd_client_closed);
> +    nbd_client_new(cioc, tlscreds, tlsauthz, nbd_client_closed, NULL);
>  }
> 
>  static void nbd_update_server_watch(void)
> -- 
> 2.45.2
> 
> 

-- 
Eric Blake, Principal Software Engineer
Red Hat, Inc.
Virtualization:  qemu.org | libguestfs.org