[Qemu-devel] [PATCH] vhost-user: delay vhost_user_stop

Marc-André Lureau posted 1 patch 7 years, 1 month ago
Patches applied successfully (tree, apply log)
git fetch https://github.com/patchew-project/qemu tags/patchew/20170224202345.13311-1-marcandre.lureau@redhat.com
Test checkpatch passed
Test docker passed
Test s390x passed
There is a newer version of this series
net/vhost-user.c | 49 +++++++++++++++++++++++++++++++++++++++++++------
1 file changed, 43 insertions(+), 6 deletions(-)
[Qemu-devel] [PATCH] vhost-user: delay vhost_user_stop
Posted by Marc-André Lureau 7 years, 1 month ago
Since commit b0a335e351103bf92f3f9d0bd5759311be8156ac, a socket write
may trigger a disconnect events, calling vhost_user_stop() and clearing
all the vhost_dev strutures holding data that vhost.c functions expect
to remain valid. Delay the cleanup to keep the vhost_dev structure
valid during the vhost.c functions.

Signed-off-by: Marc-André Lureau <marcandre.lureau@redhat.com>
---
 net/vhost-user.c | 49 +++++++++++++++++++++++++++++++++++++++++++------
 1 file changed, 43 insertions(+), 6 deletions(-)

diff --git a/net/vhost-user.c b/net/vhost-user.c
index 77b8110f8c..00573c8ac8 100644
--- a/net/vhost-user.c
+++ b/net/vhost-user.c
@@ -25,6 +25,7 @@ typedef struct VhostUserState {
     guint watch;
     uint64_t acked_features;
     bool started;
+    QEMUBH *chr_closed_bh;
 } VhostUserState;
 
 VHostNetState *vhost_user_get_vhost_net(NetClientState *nc)
@@ -190,9 +191,40 @@ static gboolean net_vhost_user_watch(GIOChannel *chan, GIOCondition cond,
 
     qemu_chr_fe_disconnect(&s->chr);
 
+    s->watch = 0;
     return FALSE;
 }
 
+static void chr_closed_bh(void *opaque)
+{
+    const char *name = opaque;
+    NetClientState *ncs[MAX_QUEUE_NUM];
+    VhostUserState *s;
+    Error *err = NULL;
+    int queues;
+
+    queues = qemu_find_net_clients_except(name, ncs,
+                                          NET_CLIENT_DRIVER_NIC,
+                                          MAX_QUEUE_NUM);
+    assert(queues < MAX_QUEUE_NUM);
+
+    s = DO_UPCAST(VhostUserState, nc, ncs[0]);
+
+    qmp_set_link(name, false, &err);
+    vhost_user_stop(queues, ncs);
+    if (s->watch) {
+        g_source_remove(s->watch);
+    }
+    s->watch = 0;
+
+    qemu_bh_delete(s->chr_closed_bh);
+    s->chr_closed_bh = NULL;
+
+    if (err) {
+        error_report_err(err);
+    }
+}
+
 static void net_vhost_user_event(void *opaque, int event)
 {
     const char *name = opaque;
@@ -212,20 +244,25 @@ static void net_vhost_user_event(void *opaque, int event)
     trace_vhost_user_event(chr->label, event);
     switch (event) {
     case CHR_EVENT_OPENED:
-        s->watch = qemu_chr_fe_add_watch(&s->chr, G_IO_HUP,
-                                         net_vhost_user_watch, s);
         if (vhost_user_start(queues, ncs, &s->chr) < 0) {
             qemu_chr_fe_disconnect(&s->chr);
             return;
         }
+        s->watch = qemu_chr_fe_add_watch(&s->chr, G_IO_HUP,
+                                         net_vhost_user_watch, s);
         qmp_set_link(name, true, &err);
+        s->chr_closed_bh = qemu_bh_new(chr_closed_bh, opaque);
         s->started = true;
         break;
     case CHR_EVENT_CLOSED:
-        qmp_set_link(name, false, &err);
-        vhost_user_stop(queues, ncs);
-        g_source_remove(s->watch);
-        s->watch = 0;
+        /* a close event may happen during a read/write, but vhost
+         * code assumes the vhost_dev remains setup, so delay the
+         * stop & clear to idle.
+         * FIXME: better handle failure in vhost code, remove bh
+         */
+        if (s->chr_closed_bh) {
+            qemu_bh_schedule(s->chr_closed_bh);
+        }
         break;
     }
 
-- 
2.12.0.rc2.3.gc93709801


Re: [Qemu-devel] [PATCH] vhost-user: delay vhost_user_stop
Posted by Marc-André Lureau 7 years, 1 month ago
Hi

On Sat, Feb 25, 2017 at 12:25 AM Marc-André Lureau <
marcandre.lureau@redhat.com> wrote:

> Since commit b0a335e351103bf92f3f9d0bd5759311be8156ac, a socket write
> may trigger a disconnect events, calling vhost_user_stop() and clearing
> all the vhost_dev strutures holding data that vhost.c functions expect
> to remain valid. Delay the cleanup to keep the vhost_dev structure
> valid during the vhost.c functions.
>
> Signed-off-by: Marc-André Lureau <marcandre.lureau@redhat.com>
> ---
>  net/vhost-user.c | 49 +++++++++++++++++++++++++++++++++++++++++++------
>  1 file changed, 43 insertions(+), 6 deletions(-)
>
> diff --git a/net/vhost-user.c b/net/vhost-user.c
> index 77b8110f8c..00573c8ac8 100644
> --- a/net/vhost-user.c
> +++ b/net/vhost-user.c
> @@ -25,6 +25,7 @@ typedef struct VhostUserState {
>      guint watch;
>      uint64_t acked_features;
>      bool started;
> +    QEMUBH *chr_closed_bh;
>  } VhostUserState;
>
>  VHostNetState *vhost_user_get_vhost_net(NetClientState *nc)
> @@ -190,9 +191,40 @@ static gboolean net_vhost_user_watch(GIOChannel
> *chan, GIOCondition cond,
>
>      qemu_chr_fe_disconnect(&s->chr);
>
> +    s->watch = 0;
>      return FALSE;
>  }
>
> +static void chr_closed_bh(void *opaque)
> +{
> +    const char *name = opaque;
> +    NetClientState *ncs[MAX_QUEUE_NUM];
> +    VhostUserState *s;
> +    Error *err = NULL;
> +    int queues;
> +
> +    queues = qemu_find_net_clients_except(name, ncs,
> +                                          NET_CLIENT_DRIVER_NIC,
> +                                          MAX_QUEUE_NUM);
> +    assert(queues < MAX_QUEUE_NUM);
> +
> +    s = DO_UPCAST(VhostUserState, nc, ncs[0]);
> +
> +    qmp_set_link(name, false, &err);
> +    vhost_user_stop(queues, ncs);
> +    if (s->watch) {
> +        g_source_remove(s->watch);
> +    }
> +    s->watch = 0;
> +
> +    qemu_bh_delete(s->chr_closed_bh);
> +    s->chr_closed_bh = NULL;
> +
> +    if (err) {
> +        error_report_err(err);
> +    }
> +}
> +
>  static void net_vhost_user_event(void *opaque, int event)
>  {
>      const char *name = opaque;
> @@ -212,20 +244,25 @@ static void net_vhost_user_event(void *opaque, int
> event)
>      trace_vhost_user_event(chr->label, event);
>      switch (event) {
>      case CHR_EVENT_OPENED:
> -        s->watch = qemu_chr_fe_add_watch(&s->chr, G_IO_HUP,
> -                                         net_vhost_user_watch, s);
>          if (vhost_user_start(queues, ncs, &s->chr) < 0) {
>              qemu_chr_fe_disconnect(&s->chr);
>              return;
>          }
> +        s->watch = qemu_chr_fe_add_watch(&s->chr, G_IO_HUP,
> +                                         net_vhost_user_watch, s);
>          qmp_set_link(name, true, &err);
> +        s->chr_closed_bh = qemu_bh_new(chr_closed_bh, opaque);
>          s->started = true;
>          break;
>      case CHR_EVENT_CLOSED:
> -        qmp_set_link(name, false, &err);
> -        vhost_user_stop(queues, ncs);
> -        g_source_remove(s->watch);
> -        s->watch = 0;
> +        /* a close event may happen during a read/write, but vhost
> +         * code assumes the vhost_dev remains setup, so delay the
> +         * stop & clear to idle.
> +         * FIXME: better handle failure in vhost code, remove bh
> +         */
> +        if (s->chr_closed_bh) {
> +            qemu_bh_schedule(s->chr_closed_bh);
>

And this is also racy if it gets a CHR_EVENT_OPENED before the bh is run.
/me not sure we want to go there.


> +        }
>          break;
>      }
>
> --
> 2.12.0.rc2.3.gc93709801
>
>
> --
Marc-André Lureau
Re: [Qemu-devel] [PATCH] vhost-user: delay vhost_user_stop
Posted by Dr. David Alan Gilbert 7 years, 1 month ago
* Marc-André Lureau (marcandre.lureau@redhat.com) wrote:
> Since commit b0a335e351103bf92f3f9d0bd5759311be8156ac, a socket write
> may trigger a disconnect events, calling vhost_user_stop() and clearing
> all the vhost_dev strutures holding data that vhost.c functions expect
> to remain valid. Delay the cleanup to keep the vhost_dev structure
> valid during the vhost.c functions.
> 
> Signed-off-by: Marc-André Lureau <marcandre.lureau@redhat.com>

This does get me through a 'make check' succesfully.

Dave

> ---
>  net/vhost-user.c | 49 +++++++++++++++++++++++++++++++++++++++++++------
>  1 file changed, 43 insertions(+), 6 deletions(-)
> 
> diff --git a/net/vhost-user.c b/net/vhost-user.c
> index 77b8110f8c..00573c8ac8 100644
> --- a/net/vhost-user.c
> +++ b/net/vhost-user.c
> @@ -25,6 +25,7 @@ typedef struct VhostUserState {
>      guint watch;
>      uint64_t acked_features;
>      bool started;
> +    QEMUBH *chr_closed_bh;
>  } VhostUserState;
>  
>  VHostNetState *vhost_user_get_vhost_net(NetClientState *nc)
> @@ -190,9 +191,40 @@ static gboolean net_vhost_user_watch(GIOChannel *chan, GIOCondition cond,
>  
>      qemu_chr_fe_disconnect(&s->chr);
>  
> +    s->watch = 0;
>      return FALSE;
>  }
>  
> +static void chr_closed_bh(void *opaque)
> +{
> +    const char *name = opaque;
> +    NetClientState *ncs[MAX_QUEUE_NUM];
> +    VhostUserState *s;
> +    Error *err = NULL;
> +    int queues;
> +
> +    queues = qemu_find_net_clients_except(name, ncs,
> +                                          NET_CLIENT_DRIVER_NIC,
> +                                          MAX_QUEUE_NUM);
> +    assert(queues < MAX_QUEUE_NUM);
> +
> +    s = DO_UPCAST(VhostUserState, nc, ncs[0]);
> +
> +    qmp_set_link(name, false, &err);
> +    vhost_user_stop(queues, ncs);
> +    if (s->watch) {
> +        g_source_remove(s->watch);
> +    }
> +    s->watch = 0;
> +
> +    qemu_bh_delete(s->chr_closed_bh);
> +    s->chr_closed_bh = NULL;
> +
> +    if (err) {
> +        error_report_err(err);
> +    }
> +}
> +
>  static void net_vhost_user_event(void *opaque, int event)
>  {
>      const char *name = opaque;
> @@ -212,20 +244,25 @@ static void net_vhost_user_event(void *opaque, int event)
>      trace_vhost_user_event(chr->label, event);
>      switch (event) {
>      case CHR_EVENT_OPENED:
> -        s->watch = qemu_chr_fe_add_watch(&s->chr, G_IO_HUP,
> -                                         net_vhost_user_watch, s);
>          if (vhost_user_start(queues, ncs, &s->chr) < 0) {
>              qemu_chr_fe_disconnect(&s->chr);
>              return;
>          }
> +        s->watch = qemu_chr_fe_add_watch(&s->chr, G_IO_HUP,
> +                                         net_vhost_user_watch, s);
>          qmp_set_link(name, true, &err);
> +        s->chr_closed_bh = qemu_bh_new(chr_closed_bh, opaque);
>          s->started = true;
>          break;
>      case CHR_EVENT_CLOSED:
> -        qmp_set_link(name, false, &err);
> -        vhost_user_stop(queues, ncs);
> -        g_source_remove(s->watch);
> -        s->watch = 0;
> +        /* a close event may happen during a read/write, but vhost
> +         * code assumes the vhost_dev remains setup, so delay the
> +         * stop & clear to idle.
> +         * FIXME: better handle failure in vhost code, remove bh
> +         */
> +        if (s->chr_closed_bh) {
> +            qemu_bh_schedule(s->chr_closed_bh);
> +        }
>          break;
>      }
>  
> -- 
> 2.12.0.rc2.3.gc93709801
> 
--
Dr. David Alan Gilbert / dgilbert@redhat.com / Manchester, UK

Re: [Qemu-devel] [PATCH] vhost-user: delay vhost_user_stop
Posted by Marc-André Lureau 7 years, 1 month ago
Hi

----- Original Message -----
> * Marc-André Lureau (marcandre.lureau@redhat.com) wrote:
> > Since commit b0a335e351103bf92f3f9d0bd5759311be8156ac, a socket write
> > may trigger a disconnect events, calling vhost_user_stop() and clearing
> > all the vhost_dev strutures holding data that vhost.c functions expect
> > to remain valid. Delay the cleanup to keep the vhost_dev structure
> > valid during the vhost.c functions.
> > 
> > Signed-off-by: Marc-André Lureau <marcandre.lureau@redhat.com>
> 
> This does get me through a 'make check' succesfully.
> 

Yes, it's not optimal, but there is so much cleanup to deal with otherwise, than I am inclined to go with this approach for now, and keep the FIXME for 2.10

I'll update the patch to avoid the race on reconnect.

> Dave
> 
> > ---
> >  net/vhost-user.c | 49 +++++++++++++++++++++++++++++++++++++++++++------
> >  1 file changed, 43 insertions(+), 6 deletions(-)
> > 
> > diff --git a/net/vhost-user.c b/net/vhost-user.c
> > index 77b8110f8c..00573c8ac8 100644
> > --- a/net/vhost-user.c
> > +++ b/net/vhost-user.c
> > @@ -25,6 +25,7 @@ typedef struct VhostUserState {
> >      guint watch;
> >      uint64_t acked_features;
> >      bool started;
> > +    QEMUBH *chr_closed_bh;
> >  } VhostUserState;
> >  
> >  VHostNetState *vhost_user_get_vhost_net(NetClientState *nc)
> > @@ -190,9 +191,40 @@ static gboolean net_vhost_user_watch(GIOChannel *chan,
> > GIOCondition cond,
> >  
> >      qemu_chr_fe_disconnect(&s->chr);
> >  
> > +    s->watch = 0;
> >      return FALSE;
> >  }
> >  
> > +static void chr_closed_bh(void *opaque)
> > +{
> > +    const char *name = opaque;
> > +    NetClientState *ncs[MAX_QUEUE_NUM];
> > +    VhostUserState *s;
> > +    Error *err = NULL;
> > +    int queues;
> > +
> > +    queues = qemu_find_net_clients_except(name, ncs,
> > +                                          NET_CLIENT_DRIVER_NIC,
> > +                                          MAX_QUEUE_NUM);
> > +    assert(queues < MAX_QUEUE_NUM);
> > +
> > +    s = DO_UPCAST(VhostUserState, nc, ncs[0]);
> > +
> > +    qmp_set_link(name, false, &err);
> > +    vhost_user_stop(queues, ncs);
> > +    if (s->watch) {
> > +        g_source_remove(s->watch);
> > +    }
> > +    s->watch = 0;
> > +
> > +    qemu_bh_delete(s->chr_closed_bh);
> > +    s->chr_closed_bh = NULL;
> > +
> > +    if (err) {
> > +        error_report_err(err);
> > +    }
> > +}
> > +
> >  static void net_vhost_user_event(void *opaque, int event)
> >  {
> >      const char *name = opaque;
> > @@ -212,20 +244,25 @@ static void net_vhost_user_event(void *opaque, int
> > event)
> >      trace_vhost_user_event(chr->label, event);
> >      switch (event) {
> >      case CHR_EVENT_OPENED:
> > -        s->watch = qemu_chr_fe_add_watch(&s->chr, G_IO_HUP,
> > -                                         net_vhost_user_watch, s);
> >          if (vhost_user_start(queues, ncs, &s->chr) < 0) {
> >              qemu_chr_fe_disconnect(&s->chr);
> >              return;
> >          }
> > +        s->watch = qemu_chr_fe_add_watch(&s->chr, G_IO_HUP,
> > +                                         net_vhost_user_watch, s);
> >          qmp_set_link(name, true, &err);
> > +        s->chr_closed_bh = qemu_bh_new(chr_closed_bh, opaque);
> >          s->started = true;
> >          break;
> >      case CHR_EVENT_CLOSED:
> > -        qmp_set_link(name, false, &err);
> > -        vhost_user_stop(queues, ncs);
> > -        g_source_remove(s->watch);
> > -        s->watch = 0;
> > +        /* a close event may happen during a read/write, but vhost
> > +         * code assumes the vhost_dev remains setup, so delay the
> > +         * stop & clear to idle.
> > +         * FIXME: better handle failure in vhost code, remove bh
> > +         */
> > +        if (s->chr_closed_bh) {
> > +            qemu_bh_schedule(s->chr_closed_bh);
> > +        }
> >          break;
> >      }
> >  
> > --
> > 2.12.0.rc2.3.gc93709801
> > 
> --
> Dr. David Alan Gilbert / dgilbert@redhat.com / Manchester, UK
>