Qemu core dump when stop guest with virtio-blk(remote storage) and iothread

苏思婷 posted 1 patch 3 years, 8 months ago
Failed in applying to current master (apply log)
Qemu core dump when stop guest with virtio-blk(remote storage) and iothread
Posted by 苏思婷 3 years, 8 months ago
Description of problem:
	Qemu core dump when stop guest with virtio-blk(remote storage) and iothread

Version-Release number pf selected component (if applicable):
	kernel version:4.19.36.bsk.9-amd64
	qemu-kvm version:QEMU emulator version 2.12.1

How reproducible:
	100%

Steps to Reproduce:
1. Start guest, one virtio-blk(remote storage) and iothread parameter
	/data00/qemu/x86_64-softmmu/qemu-system-x86_64
	-name guest=instance-03,debug-threads=on
	-kvm /dev/kvm 
	-S
	-object secret,id=masterKey0,format=raw,file=/var/lib/libvirt/qemu/domain-21-instance-03/master-key.aes 
	-machine pc-i440fx-2.12,accel=kvm,usb=off,dump-guest-core=off 
	-cpu qemu64,+kvm_pv_eoi 
	-m 7630 
	-mem-prealloc -mem-path /dev/hugepages/libvirt/qemu/21-instance-03 -smp 4,sockets=4,cores=1,threads=1 
	-object iothread,id=iothread1 -object iothread,id=iothread2 -object iothread,id=iothread3 -object iothread,id=iothread4
	-no-user-config 
	-nodefaults 
	-chardev socket,id=charmonitor,fd=21,server,nowait 
	-mon chardev=charmonitor,id=monitor,mode=control 
	-rtc base=utc,driftfix=slew 
	-global kvm-pit.lost_tick_policy=delay -no-hpet -no-shutdown 
	-boot strict=on -device piix3-usb-uhci,id=usb,bus=pci.0,addr=0x1.0x2 
	-device virtio-serial-pci,id=virtio-serial0,bus=pci.0,addr=0x7 
	-drive file=remote drive path,format=raw,if=none,id=drive-virtio-disk0,cache=none,aio=native,throttling.bps-read=30000000,throttling.bps-write=80000000,throttling.iops-read=800,throttling.iops-write=400 
	-device virtio-blk-pci,iothread=iothread1,scsi=off,bus=pci.0,addr=0x3,drive=drive-virtio-disk0,id=virtio-disk0,bootindex=1,write-cache=on
	-chardev pty,id=charserial0 -device isa-serial,chardev=charserial0,id=serial0 
	-device usb-tablet,id=input0,bus=usb.0,port=1 -vnc 0.0.0.0:0 -k en-us -device cirrus-vga,id=video0,bus=pci.0,addr=0x2 
	-device virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x8 -sandbox on,obsolete=deny,elevateprivileges=deny,spawn=deny,resourcecontrol=deny 
	-device pvpanic,ioport=1285 -msg timestamp=on -d int,unimp,guest_errors
2.stop guest and start guest repeatedly 
	virsh qemu-monitor-command --domain instance-03 '{"execute":"stop", "arguments":{}}'
	virsh qemu-monitor-command --domain instance-03 '{"execute":"cont", "arguments":{}}'

3.Actual results:
   Qemu core dump with error msg:
   	(qemu) qemu: qemu_mutex_unlock_impl: Operation not permitted
   Expected results:
   Guest can stop and start successfully

Additional info:
[Current thread is 1 (Thread 0x7ff188abb700 (LWP 2229116))]
(gdb) bt
#0  __GI_raise (sig=sig@entry=6) at ../sysdeps/unix/sysv/linux/raise.c:51
#1  0x00007ff19d73542a in __GI_abort () at abort.c:89
#2  0x0000560d55a3ca65 in error_exit (err=err@entry=1, msg=msg@entry=0x560d566e9650 <__func__.18241> "qemu_mutex_unlock_impl") at util/qemu-thread-posix.c:37
#3  0x0000560d55e76dc8 in qemu_mutex_unlock_impl (mutex=mutex@entry=0x560d58456e30, file=file@entry=0x560d566e87df "util/async.c", line=line@entry=509) at util/qemu-thread-posix.c:99
#4  0x0000560d55e71580 in aio_context_release (ctx=0x560d58456dd0) at util/async.c:509
#5  0x0000560d55acf344 in virtio_blk_rw_complete (opaque=<optimized out>, ret=0) at /data00/susieqemu/hw/block/virtio-blk.c:126
#6  0x0000560d55dcabc9 in blk_aio_complete (acb=0x7ff17c001bf0) at block/block-backend.c:1345
#7  0x0000560d55e8a25b in coroutine_trampoline (i0=<optimized out>, i1=<optimized out>) at util/coroutine-ucontext.c:116
#8  0x00007ff19d745000 in ?? () from /lib/x86_64-linux-gnu/libc.so.6
#9  0x00007ffd3691e190 in ?? ()
#10 0x0000000000000000 in ?? ()
Backtrace stopped: Cannot access memory at address 0x7fef29cdd000

Thread 3 (Thread 0x7ff1a38f23c0 (LWP 2229113)):
#0  memory_region_ioeventfd_before (a=..., b=...) at /data00/susieqemu/memory.c:185
#1  address_space_add_del_ioeventfds (fds_old_nb=65, fds_old=0x560d59733960, fds_new_nb=64, fds_new=0x560d59ba6d10, as=0x560d598728b0) at /data00/susieqemu/memory.c:794
#2  address_space_update_ioeventfds (as=as@entry=0x560d598728b0) at /data00/susieqemu/memory.c:877
#3  0x0000560d55aabd98 in memory_region_transaction_commit () at /data00/susieqemu/memory.c:1080
#4  0x0000560d55aae260 in memory_region_del_eventfd (mr=mr@entry=0x560d599a2b40, addr=<optimized out>, size=size@entry=0, match_data=<optimized out>, data=<optimized out>, e=<optimized out>) at /data00/susieqemu/memory.c:2274
#5  0x0000560d55d113da in virtio_pci_ioeventfd_assign (d=0x560d599a1e70, notifier=0x560d599db8d8, n=0, assign=<optimized out>) at hw/virtio/virtio-pci.c:268
#6  0x0000560d55d14fef in virtio_bus_set_host_notifier (bus=0x560d599a9f68, n=n@entry=0, assign=assign@entry=false) at hw/virtio/virtio-bus.c:289
#7  0x0000560d55ad2886 in virtio_blk_data_plane_stop (vdev=<optimized out>) at /data00/susieqemu/hw/block/dataplane/virtio-blk.c:295
#8  0x0000560d55d1459e in virtio_bus_stop_ioeventfd (bus=0x560d599a9f68) at hw/virtio/virtio-bus.c:246
#9  0x0000560d55b02c4f in virtio_vmstate_change (opaque=0x560d599a9fe0, running=0, state=RUN_STATE_PAUSED) at /data00/susieqemu/hw/virtio/virtio.c:2236
#10 0x0000560d55ba2517 in vm_state_notify (running=running@entry=0, state=state@entry=RUN_STATE_PAUSED) at vl.c:1649
#11 0x0000560d55a95c3a in do_vm_stop (state=state@entry=RUN_STATE_PAUSED, send_stop=send_stop@entry=true) at /data00/susieqemu/cpus.c:1013
#12 0x0000560d55a96132 in vm_stop (state=state@entry=RUN_STATE_PAUSED) at /data00/susieqemu/cpus.c:2036
#13 0x0000560d55bb652b in qmp_stop (errp=errp@entry=0x7ffd3691e9b0) at qmp.c:106
#14 0x0000560d55bae9aa in qmp_marshal_stop (args=<optimized out>, ret=<optimized out>, errp=0x7ffd3691e9f8) at qapi/qapi-commands-misc.c:784
#15 0x0000560d55e65f89 in do_qmp_dispatch (errp=0x7ffd3691e9f0, request=0x7ffd3691e9f0, cmds=<optimized out>) at qapi/qmp-dispatch.c:111
#16 qmp_dispatch (cmds=<optimized out>, request=request@entry=0x560d58467e60) at qapi/qmp-dispatch.c:164
#17 0x0000560d55a9e95d in monitor_qmp_dispatch_one (req_obj=req_obj@entry=0x560d587b5a20) at /data00/susieqemu/monitor.c:4084
#18 0x0000560d55a9ec99 in monitor_qmp_bh_dispatcher (data=<optimized out>) at /data00/susieqemu/monitor.c:4142
#19 0x0000560d55e70e2e in aio_bh_call (bh=0x560d58472a20) at util/async.c:91
#20 aio_bh_poll (ctx=ctx@entry=0x560d584569e0) at util/async.c:119
#21 0x0000560d55e7470f in aio_dispatch (ctx=0x560d584569e0) at util/aio-posix.c:456
#22 0x0000560d55e70ced in aio_ctx_dispatch (source=<optimized out>, callback=<optimized out>, user_data=<optimized out>) at util/async.c:262
#23 0x00007ff1a0d087f7 in g_main_context_dispatch () from /lib/x86_64-linux-gnu/libglib-2.0.so.0
#24 0x0000560d55e737d8 in glib_pollfds_poll () at util/main-loop.c:217
#25 os_host_main_loop_wait (timeout=<optimized out>) at util/main-loop.c:265
#26 main_loop_wait (nonblocking=<optimized out>) at util/main-loop.c:524
#27 0x0000560d55a41ff7 in main_loop () at vl.c:1980
#28 main (argc=<optimized out>, argv=<optimized out>, envp=<optimized out>) at vl.c:4845

This is because in the virtio_blk_dataplane_stop() function, the context of the iothread will be set to the maincontext in blk_set_aio_context(). In the middle of this context change, the iothread will acquire the original iothread context which is held by the main thread. After the mainthread change the context of the iothread and then release the original iothread context, the iothread will get the old iothread context which is not supposed to happen because the context has been changed to the main thread context. So when the iothread finish virtio_blk_rw_complete() and try to release the main thread context, qemu will crash.

This patch can solve this problem for me, but I am not sure whether this is right, please give me a hand.

diff --git a/block/.block-backend.c.swp b/block/.block-backend.c.swp
deleted file mode 100644
index 0bf897dd..00000000
Binary files a/block/.block-backend.c.swp and /dev/null differ
diff --git a/hw/block/virtio-blk.c b/hw/block/virtio-blk.c
index b9c1b3ba..d8ac782d 100644
--- a/hw/block/virtio-blk.c
+++ b/hw/block/virtio-blk.c
@@ -91,7 +91,12 @@ static void virtio_blk_rw_complete(void *opaque, int ret)
     VirtIOBlock *s = next->dev;
     VirtIODevice *vdev = VIRTIO_DEVICE(s);
     QEMU_LOG(LOG_INFO, "virtio_blk_rw_complete before context_acquire: %p\n", (blk_get_aio_context(s->conf.conf.blk)));
-    aio_context_acquire(blk_get_aio_context(s->conf.conf.blk));
+    static struct timespec ts_sleep;
+    ts_sleep.tv_sec = 0;
+    ts_sleep.tv_nsec = 50 * 1000 * 1000ull;
+    while (aio_context_tryacquire(blk_get_aio_context(s->conf.conf.blk))) {
+        nanosleep(&ts_sleep, NULL);
+    }
     while (next) {
         VirtIOBlockReq *req = next;
         next = req->mr_next;
diff --git a/include/block/aio.h b/include/block/aio.h
index a1d6b9e2..59be498f 100644
--- a/include/block/aio.h
+++ b/include/block/aio.h
@@ -174,6 +174,8 @@ void aio_context_ref(AioContext *ctx);
  */
 void aio_context_unref(AioContext *ctx);
 
+int aio_context_tryacquire(AioContext *ctx);
+
 /* Take ownership of the AioContext.  If the AioContext will be shared between
  * threads, and a thread does not want to be interrupted, it will have to
  * take ownership around calls to aio_poll().  Otherwise, aio_poll()
diff --git a/util/async.c b/util/async.c
index 27fb4901..009f6c5f 100644
--- a/util/async.c
+++ b/util/async.c
@@ -498,6 +498,10 @@ void aio_context_unref(AioContext *ctx)
     g_source_unref(&ctx->source);
 }
 
+int aio_context_acquire(AioContext *ctx) {
+    return qemu_rec_mutex_trylock(&ctx->lock);
+}
+
 void aio_context_acquire(AioContext *ctx)
 {
     qemu_rec_mutex_lock(&ctx->lock);


Re: Qemu core dump when stop guest with virtio-blk(remote storage) and iothread
Posted by Kevin Wolf 3 years, 8 months ago
Am 08.07.2020 um 06:07 hat 苏思婷 geschrieben:
> Description of problem:
> 	Qemu core dump when stop guest with virtio-blk(remote storage) and iothread
> 
> Version-Release number pf selected component (if applicable):
> 	kernel version:4.19.36.bsk.9-amd64
> 	qemu-kvm version:QEMU emulator version 2.12.1

Version 2.12.1 is rather old. Can you retest with a current version? We
fixed a few iothread locking bugs since then.

Kevin