1 | The following changes since commit 67c1115edd98f388ca89dd38322ea3fadf034523: | 1 | The following changes since commit a2376507f615495b1d16685449ce0ea78c2caf9d: |
---|---|---|---|
2 | 2 | ||
3 | Merge remote-tracking branch 'remotes/kraxel/tags/ui-20210323-pull-request' into staging (2021-03-23 23:47:30 +0000) | 3 | Merge remote-tracking branch 'remotes/bonzini-gitlab/tags/for-upstream' into staging (2021-07-24 11:04:57 +0100) |
4 | 4 | ||
5 | are available in the Git repository at: | 5 | are available in the Git repository at: |
6 | 6 | ||
7 | https://gitlab.com/stefanha/qemu.git tags/block-pull-request | 7 | https://gitlab.com/stefanha/qemu.git tags/block-pull-request |
8 | 8 | ||
9 | for you to fetch changes up to 3460fd7f3959d1fa7bcc255796844aa261c805a4: | 9 | for you to fetch changes up to 15a730e7a3aaac180df72cd5730e0617bcf44a5a: |
10 | 10 | ||
11 | migrate-bitmaps-postcopy-test: check that we can't remove in-flight bitmaps (2021-03-24 13:41:19 +0000) | 11 | block/nvme: Fix VFIO_MAP_DMA failed: No space left on device (2021-07-26 09:38:12 +0100) |
12 | 12 | ||
13 | ---------------------------------------------------------------- | 13 | ---------------------------------------------------------------- |
14 | Pull request | 14 | Pull request |
15 | 15 | ||
16 | This dirty bitmap fix solves a crash that can be triggered in the destination | 16 | Phil's block/nvme.c ENOSPC fix for newer Linux kernels that return this errno. |
17 | QEMU process during live migration. | ||
18 | 17 | ||
19 | ---------------------------------------------------------------- | 18 | ---------------------------------------------------------------- |
20 | 19 | ||
21 | Vladimir Sementsov-Ogievskiy (2): | 20 | Philippe Mathieu-Daudé (1): |
22 | migration/block-dirty-bitmap: make incoming disabled bitmaps busy | 21 | block/nvme: Fix VFIO_MAP_DMA failed: No space left on device |
23 | migrate-bitmaps-postcopy-test: check that we can't remove in-flight | ||
24 | bitmaps | ||
25 | 22 | ||
26 | migration/block-dirty-bitmap.c | 6 ++++++ | 23 | block/nvme.c | 22 ++++++++++++++++++++++ |
27 | tests/qemu-iotests/tests/migrate-bitmaps-postcopy-test | 10 ++++++++++ | 24 | 1 file changed, 22 insertions(+) |
28 | 2 files changed, 16 insertions(+) | ||
29 | 25 | ||
30 | -- | 26 | -- |
31 | 2.30.2 | 27 | 2.31.1 |
32 | 28 | diff view generated by jsdifflib |
Deleted patch | |||
---|---|---|---|
1 | From: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com> | ||
2 | 1 | ||
3 | Incoming enabled bitmaps are busy, because we do | ||
4 | bdrv_dirty_bitmap_create_successor() for them. But disabled bitmaps | ||
5 | being migrated are not marked busy, and user can remove them during the | ||
6 | incoming migration. Then we may crash in cancel_incoming_locked() when | ||
7 | try to remove the bitmap that was already removed by user, like this: | ||
8 | |||
9 | #0 qemu_mutex_lock_impl (mutex=0x5593d88c50d1, file=0x559680554b20 | ||
10 | "../block/dirty-bitmap.c", line=64) at ../util/qemu-thread-posix.c:77 | ||
11 | #1 bdrv_dirty_bitmaps_lock (bs=0x5593d88c0ee9) | ||
12 | at ../block/dirty-bitmap.c:64 | ||
13 | #2 bdrv_release_dirty_bitmap (bitmap=0x5596810e9570) | ||
14 | at ../block/dirty-bitmap.c:362 | ||
15 | #3 cancel_incoming_locked (s=0x559680be8208 <dbm_state+40>) | ||
16 | at ../migration/block-dirty-bitmap.c:918 | ||
17 | #4 dirty_bitmap_load (f=0x559681d02b10, opaque=0x559680be81e0 | ||
18 | <dbm_state>, version_id=1) at ../migration/block-dirty-bitmap.c:1194 | ||
19 | #5 vmstate_load (f=0x559681d02b10, se=0x559680fb5810) | ||
20 | at ../migration/savevm.c:908 | ||
21 | #6 qemu_loadvm_section_part_end (f=0x559681d02b10, | ||
22 | mis=0x559680fb4a30) at ../migration/savevm.c:2473 | ||
23 | #7 qemu_loadvm_state_main (f=0x559681d02b10, mis=0x559680fb4a30) | ||
24 | at ../migration/savevm.c:2626 | ||
25 | #8 postcopy_ram_listen_thread (opaque=0x0) | ||
26 | at ../migration/savevm.c:1871 | ||
27 | #9 qemu_thread_start (args=0x5596817ccd10) | ||
28 | at ../util/qemu-thread-posix.c:521 | ||
29 | #10 start_thread () at /lib64/libpthread.so.0 | ||
30 | #11 clone () at /lib64/libc.so.6 | ||
31 | |||
32 | Note bs pointer taken from bitmap: it's definitely bad aligned. That's | ||
33 | because we are in use after free, bitmap is already freed. | ||
34 | |||
35 | So, let's make disabled bitmaps (being migrated) busy during incoming | ||
36 | migration. | ||
37 | |||
38 | Signed-off-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com> | ||
39 | Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com> | ||
40 | Message-Id: <20210322094906.5079-2-vsementsov@virtuozzo.com> | ||
41 | --- | ||
42 | migration/block-dirty-bitmap.c | 6 ++++++ | ||
43 | 1 file changed, 6 insertions(+) | ||
44 | |||
45 | diff --git a/migration/block-dirty-bitmap.c b/migration/block-dirty-bitmap.c | ||
46 | index XXXXXXX..XXXXXXX 100644 | ||
47 | --- a/migration/block-dirty-bitmap.c | ||
48 | +++ b/migration/block-dirty-bitmap.c | ||
49 | @@ -XXX,XX +XXX,XX @@ static int dirty_bitmap_load_start(QEMUFile *f, DBMLoadState *s) | ||
50 | error_report_err(local_err); | ||
51 | return -EINVAL; | ||
52 | } | ||
53 | + } else { | ||
54 | + bdrv_dirty_bitmap_set_busy(s->bitmap, true); | ||
55 | } | ||
56 | |||
57 | b = g_new(LoadBitmapState, 1); | ||
58 | @@ -XXX,XX +XXX,XX @@ static void cancel_incoming_locked(DBMLoadState *s) | ||
59 | assert(!s->before_vm_start_handled || !b->migrated); | ||
60 | if (bdrv_dirty_bitmap_has_successor(b->bitmap)) { | ||
61 | bdrv_reclaim_dirty_bitmap(b->bitmap, &error_abort); | ||
62 | + } else { | ||
63 | + bdrv_dirty_bitmap_set_busy(b->bitmap, false); | ||
64 | } | ||
65 | bdrv_release_dirty_bitmap(b->bitmap); | ||
66 | } | ||
67 | @@ -XXX,XX +XXX,XX @@ static void dirty_bitmap_load_complete(QEMUFile *f, DBMLoadState *s) | ||
68 | |||
69 | if (bdrv_dirty_bitmap_has_successor(s->bitmap)) { | ||
70 | bdrv_reclaim_dirty_bitmap(s->bitmap, &error_abort); | ||
71 | + } else { | ||
72 | + bdrv_dirty_bitmap_set_busy(s->bitmap, false); | ||
73 | } | ||
74 | |||
75 | for (item = s->bitmaps; item; item = g_slist_next(item)) { | ||
76 | -- | ||
77 | 2.30.2 | ||
78 | diff view generated by jsdifflib |
1 | From: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com> | 1 | From: Philippe Mathieu-Daudé <philmd@redhat.com> |
---|---|---|---|
2 | 2 | ||
3 | Check that we can't remove bitmaps being migrated on destination vm. | 3 | When the NVMe block driver was introduced (see commit bdd6a90a9e5, |
4 | The new check proves that previous commit helps. | 4 | January 2018), Linux VFIO_IOMMU_MAP_DMA ioctl was only returning |
5 | -ENOMEM in case of error. The driver was correctly handling the | ||
6 | error path to recycle its volatile IOVA mappings. | ||
5 | 7 | ||
6 | Signed-off-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com> | 8 | To fix CVE-2019-3882, Linux commit 492855939bdb ("vfio/type1: Limit |
9 | DMA mappings per container", April 2019) added the -ENOSPC error to | ||
10 | signal the user exhausted the DMA mappings available for a container. | ||
11 | |||
12 | The block driver started to mis-behave: | ||
13 | |||
14 | qemu-system-x86_64: VFIO_MAP_DMA failed: No space left on device | ||
15 | (qemu) | ||
16 | (qemu) info status | ||
17 | VM status: paused (io-error) | ||
18 | (qemu) c | ||
19 | VFIO_MAP_DMA failed: No space left on device | ||
20 | (qemu) c | ||
21 | VFIO_MAP_DMA failed: No space left on device | ||
22 | |||
23 | (The VM is not resumable from here, hence stuck.) | ||
24 | |||
25 | Fix by handling the new -ENOSPC error (when DMA mappings are | ||
26 | exhausted) without any distinction to the current -ENOMEM error, | ||
27 | so we don't change the behavior on old kernels where the CVE-2019-3882 | ||
28 | fix is not present. | ||
29 | |||
30 | An easy way to reproduce this bug is to restrict the DMA mapping | ||
31 | limit (65535 by default) when loading the VFIO IOMMU module: | ||
32 | |||
33 | # modprobe vfio_iommu_type1 dma_entry_limit=666 | ||
34 | |||
35 | Cc: qemu-stable@nongnu.org | ||
36 | Cc: Fam Zheng <fam@euphon.net> | ||
37 | Cc: Maxim Levitsky <mlevitsk@redhat.com> | ||
38 | Cc: Alex Williamson <alex.williamson@redhat.com> | ||
39 | Reported-by: Michal Prívozník <mprivozn@redhat.com> | ||
40 | Signed-off-by: Philippe Mathieu-Daudé <philmd@redhat.com> | ||
41 | Message-id: 20210723195843.1032825-1-philmd@redhat.com | ||
42 | Fixes: bdd6a90a9e5 ("block: Add VFIO based NVMe driver") | ||
43 | Buglink: https://bugs.launchpad.net/qemu/+bug/1863333 | ||
44 | Resolves: https://gitlab.com/qemu-project/qemu/-/issues/65 | ||
45 | Signed-off-by: Philippe Mathieu-Daudé <philmd@redhat.com> | ||
7 | Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com> | 46 | Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com> |
8 | Message-Id: <20210322094906.5079-3-vsementsov@virtuozzo.com> | ||
9 | --- | 47 | --- |
10 | tests/qemu-iotests/tests/migrate-bitmaps-postcopy-test | 10 ++++++++++ | 48 | block/nvme.c | 22 ++++++++++++++++++++++ |
11 | 1 file changed, 10 insertions(+) | 49 | 1 file changed, 22 insertions(+) |
12 | 50 | ||
13 | diff --git a/tests/qemu-iotests/tests/migrate-bitmaps-postcopy-test b/tests/qemu-iotests/tests/migrate-bitmaps-postcopy-test | 51 | diff --git a/block/nvme.c b/block/nvme.c |
14 | index XXXXXXX..XXXXXXX 100755 | 52 | index XXXXXXX..XXXXXXX 100644 |
15 | --- a/tests/qemu-iotests/tests/migrate-bitmaps-postcopy-test | 53 | --- a/block/nvme.c |
16 | +++ b/tests/qemu-iotests/tests/migrate-bitmaps-postcopy-test | 54 | +++ b/block/nvme.c |
17 | @@ -XXX,XX +XXX,XX @@ class TestDirtyBitmapPostcopyMigration(iotests.QMPTestCase): | 55 | @@ -XXX,XX +XXX,XX @@ try_map: |
18 | self.start_postcopy() | 56 | r = qemu_vfio_dma_map(s->vfio, |
19 | 57 | qiov->iov[i].iov_base, | |
20 | self.vm_b_events += self.vm_b.get_qmp_events() | 58 | len, true, &iova); |
21 | + | 59 | + if (r == -ENOSPC) { |
22 | + # While being here, let's check that we can't remove in-flight bitmaps. | 60 | + /* |
23 | + for vm in (self.vm_a, self.vm_b): | 61 | + * In addition to the -ENOMEM error, the VFIO_IOMMU_MAP_DMA |
24 | + for i in range(0, nb_bitmaps): | 62 | + * ioctl returns -ENOSPC to signal the user exhausted the DMA |
25 | + result = vm.qmp('block-dirty-bitmap-remove', node='drive0', | 63 | + * mappings available for a container since Linux kernel commit |
26 | + name=f'bitmap{i}') | 64 | + * 492855939bdb ("vfio/type1: Limit DMA mappings per container", |
27 | + self.assert_qmp(result, 'error/desc', | 65 | + * April 2019, see CVE-2019-3882). |
28 | + f"Bitmap 'bitmap{i}' is currently in use by " | 66 | + * |
29 | + "another operation and cannot be used") | 67 | + * This block driver already handles this error path by checking |
30 | + | 68 | + * for the -ENOMEM error, so we directly replace -ENOSPC by |
31 | self.vm_b.shutdown() | 69 | + * -ENOMEM. Beside, -ENOSPC has a specific meaning for blockdev |
32 | # recreate vm_b, so there is no incoming option, which prevents | 70 | + * coroutines: it triggers BLOCKDEV_ON_ERROR_ENOSPC and |
33 | # loading bitmaps from disk | 71 | + * BLOCK_ERROR_ACTION_STOP which stops the VM, asking the operator |
72 | + * to add more storage to the blockdev. Not something we can do | ||
73 | + * easily with an IOMMU :) | ||
74 | + */ | ||
75 | + r = -ENOMEM; | ||
76 | + } | ||
77 | if (r == -ENOMEM && retry) { | ||
78 | + /* | ||
79 | + * We exhausted the DMA mappings available for our container: | ||
80 | + * recycle the volatile IOVA mappings. | ||
81 | + */ | ||
82 | retry = false; | ||
83 | trace_nvme_dma_flush_queue_wait(s); | ||
84 | if (s->dma_map_count) { | ||
34 | -- | 85 | -- |
35 | 2.30.2 | 86 | 2.31.1 |
36 | 87 | diff view generated by jsdifflib |