1 | The following changes since commit 19b599f7664b2ebfd0f405fb79c14dd241557452: | 1 | The following changes since commit 474f3938d79ab36b9231c9ad3b5a9314c2aeacde: |
---|---|---|---|
2 | 2 | ||
3 | Merge remote-tracking branch 'remotes/armbru/tags/pull-error-2018-08-27-v2' into staging (2018-08-27 16:44:20 +0100) | 3 | Merge remote-tracking branch 'remotes/amarkovic/tags/mips-queue-jun-21-2019' into staging (2019-06-21 15:40:50 +0100) |
4 | 4 | ||
5 | are available in the Git repository at: | 5 | are available in the Git repository at: |
6 | 6 | ||
7 | https://git.xanclic.moe/XanClic/qemu.git tags/pull-block-2018-08-31-v2 | 7 | https://github.com/XanClic/qemu.git tags/pull-block-2019-06-24 |
8 | 8 | ||
9 | for you to fetch changes up to e21a1c9831fc80ae3f3c1affdfa43350035d8588: | 9 | for you to fetch changes up to ab5d4a30f7f3803ca5106b370969c1b7b54136f8: |
10 | 10 | ||
11 | jobs: remove job_defer_to_main_loop (2018-08-31 16:28:33 +0200) | 11 | iotests: Fix 205 for concurrent runs (2019-06-24 16:01:40 +0200) |
12 | 12 | ||
13 | ---------------------------------------------------------------- | 13 | ---------------------------------------------------------------- |
14 | Block patches: | 14 | Block patches: |
15 | - (Block) job exit refactoring, part 1 | 15 | - The SSH block driver now uses libssh instead of libssh2 |
16 | (removing job_defer_to_main_loop()) | 16 | - The VMDK block driver gets read-only support for the seSparse |
17 | - test-bdrv-drain leak fix | 17 | subformat |
18 | - Various fixes | ||
19 | |||
20 | --- | ||
21 | |||
22 | v2: | ||
23 | - Squashed Pino's fix for pre-0.8 libssh into the libssh patch | ||
18 | 24 | ||
19 | ---------------------------------------------------------------- | 25 | ---------------------------------------------------------------- |
20 | John Snow (9): | 26 | Anton Nefedov (1): |
21 | jobs: change start callback to run callback | 27 | iotest 134: test cluster-misaligned encrypted write |
22 | jobs: canonize Error object | ||
23 | jobs: add exit shim | ||
24 | block/commit: utilize job_exit shim | ||
25 | block/mirror: utilize job_exit shim | ||
26 | jobs: utilize job_exit shim | ||
27 | block/backup: make function variables consistently named | ||
28 | jobs: remove ret argument to job_completed; privatize it | ||
29 | jobs: remove job_defer_to_main_loop | ||
30 | 28 | ||
31 | Marc-André Lureau (1): | 29 | Klaus Birkelund Jensen (1): |
32 | tests: fix bdrv-drain leak | 30 | nvme: do not advertise support for unsupported arbitration mechanism |
33 | 31 | ||
34 | include/qemu/job.h | 70 ++++++++++++++++----------------- | 32 | Max Reitz (1): |
35 | block/backup.c | 81 ++++++++++++++++----------------------- | 33 | iotests: Fix 205 for concurrent runs |
36 | block/commit.c | 29 +++++--------- | 34 | |
37 | block/create.c | 19 +++------ | 35 | Pino Toscano (1): |
38 | block/mirror.c | 39 ++++++++----------- | 36 | ssh: switch from libssh2 to libssh |
39 | block/stream.c | 29 ++++++-------- | 37 | |
40 | job-qmp.c | 5 ++- | 38 | Sam Eiderman (3): |
41 | job.c | 73 ++++++++++++----------------------- | 39 | vmdk: Fix comment regarding max l1_size coverage |
42 | tests/test-bdrv-drain.c | 14 +++---- | 40 | vmdk: Reduce the max bound for L1 table size |
43 | tests/test-blockjob-txn.c | 25 +++++------- | 41 | vmdk: Add read-only support for seSparse snapshots |
44 | tests/test-blockjob.c | 17 ++++---- | 42 | |
45 | trace-events | 2 +- | 43 | Vladimir Sementsov-Ogievskiy (1): |
46 | 12 files changed, 161 insertions(+), 242 deletions(-) | 44 | blockdev: enable non-root nodes for transaction drive-backup source |
45 | |||
46 | configure | 65 +- | ||
47 | block/Makefile.objs | 6 +- | ||
48 | block/ssh.c | 652 ++++++++++-------- | ||
49 | block/vmdk.c | 372 +++++++++- | ||
50 | blockdev.c | 2 +- | ||
51 | hw/block/nvme.c | 1 - | ||
52 | .travis.yml | 4 +- | ||
53 | block/trace-events | 14 +- | ||
54 | docs/qemu-block-drivers.texi | 2 +- | ||
55 | .../dockerfiles/debian-win32-cross.docker | 1 - | ||
56 | .../dockerfiles/debian-win64-cross.docker | 1 - | ||
57 | tests/docker/dockerfiles/fedora.docker | 4 +- | ||
58 | tests/docker/dockerfiles/ubuntu.docker | 2 +- | ||
59 | tests/docker/dockerfiles/ubuntu1804.docker | 2 +- | ||
60 | tests/qemu-iotests/059.out | 2 +- | ||
61 | tests/qemu-iotests/134 | 9 + | ||
62 | tests/qemu-iotests/134.out | 10 + | ||
63 | tests/qemu-iotests/205 | 2 +- | ||
64 | tests/qemu-iotests/207 | 54 +- | ||
65 | tests/qemu-iotests/207.out | 2 +- | ||
66 | 20 files changed, 823 insertions(+), 384 deletions(-) | ||
47 | 67 | ||
48 | -- | 68 | -- |
49 | 2.17.1 | 69 | 2.21.0 |
50 | 70 | ||
51 | 71 | diff view generated by jsdifflib |
Deleted patch | |||
---|---|---|---|
1 | From: Marc-André Lureau <marcandre.lureau@redhat.com> | ||
2 | 1 | ||
3 | Spotted by ASAN: | ||
4 | |||
5 | ================================================================= | ||
6 | ==5378==ERROR: LeakSanitizer: detected memory leaks | ||
7 | |||
8 | Direct leak of 65536 byte(s) in 1 object(s) allocated from: | ||
9 | #0 0x7f788f83bc48 in malloc (/lib64/libasan.so.5+0xeec48) | ||
10 | #1 0x7f788c9923c5 in g_malloc (/lib64/libglib-2.0.so.0+0x523c5) | ||
11 | #2 0x5622a1fe37bc in coroutine_trampoline /home/elmarco/src/qq/util/coroutine-ucontext.c:116 | ||
12 | #3 0x7f788a15d75f in __correctly_grouped_prefixwc (/lib64/libc.so.6+0x4c75f) | ||
13 | |||
14 | (Broken in commit 4c8158e359d.) | ||
15 | |||
16 | Signed-off-by: Marc-André Lureau <marcandre.lureau@redhat.com> | ||
17 | Message-id: 20180809114417.28718-3-marcandre.lureau@redhat.com | ||
18 | Signed-off-by: Max Reitz <mreitz@redhat.com> | ||
19 | --- | ||
20 | tests/test-bdrv-drain.c | 1 + | ||
21 | 1 file changed, 1 insertion(+) | ||
22 | |||
23 | diff --git a/tests/test-bdrv-drain.c b/tests/test-bdrv-drain.c | ||
24 | index XXXXXXX..XXXXXXX 100644 | ||
25 | --- a/tests/test-bdrv-drain.c | ||
26 | +++ b/tests/test-bdrv-drain.c | ||
27 | @@ -XXX,XX +XXX,XX @@ static void coroutine_fn test_co_delete_by_drain(void *opaque) | ||
28 | } | ||
29 | |||
30 | dbdd->done = true; | ||
31 | + g_free(buffer); | ||
32 | } | ||
33 | |||
34 | /** | ||
35 | -- | ||
36 | 2.17.1 | ||
37 | |||
38 | diff view generated by jsdifflib |
1 | From: John Snow <jsnow@redhat.com> | 1 | From: Klaus Birkelund Jensen <klaus@birkelund.eu> |
---|---|---|---|
2 | 2 | ||
3 | Now that the job infrastructure is handling the job_completed call for | 3 | The device mistakenly reports that the Weighted Round Robin with Urgent |
4 | all implemented jobs, we can remove the interface that allowed jobs to | 4 | Priority Class arbitration mechanism is supported. |
5 | schedule their own completion. | ||
6 | 5 | ||
7 | Signed-off-by: John Snow <jsnow@redhat.com> | 6 | It is not. |
8 | Reviewed-by: Max Reitz <mreitz@redhat.com> | 7 | |
9 | Message-id: 20180830015734.19765-10-jsnow@redhat.com | 8 | Signed-off-by: Klaus Birkelund Jensen <klaus.jensen@cnexlabs.com> |
9 | Message-id: 20190606092530.14206-1-klaus@birkelund.eu | ||
10 | Acked-by: Maxim Levitsky <mlevitsk@redhat.com> | ||
10 | Signed-off-by: Max Reitz <mreitz@redhat.com> | 11 | Signed-off-by: Max Reitz <mreitz@redhat.com> |
11 | --- | 12 | --- |
12 | include/qemu/job.h | 17 ----------------- | 13 | hw/block/nvme.c | 1 - |
13 | job.c | 40 ++-------------------------------------- | 14 | 1 file changed, 1 deletion(-) |
14 | 2 files changed, 2 insertions(+), 55 deletions(-) | ||
15 | 15 | ||
16 | diff --git a/include/qemu/job.h b/include/qemu/job.h | 16 | diff --git a/hw/block/nvme.c b/hw/block/nvme.c |
17 | index XXXXXXX..XXXXXXX 100644 | 17 | index XXXXXXX..XXXXXXX 100644 |
18 | --- a/include/qemu/job.h | 18 | --- a/hw/block/nvme.c |
19 | +++ b/include/qemu/job.h | 19 | +++ b/hw/block/nvme.c |
20 | @@ -XXX,XX +XXX,XX @@ void job_finalize(Job *job, Error **errp); | 20 | @@ -XXX,XX +XXX,XX @@ static void nvme_realize(PCIDevice *pci_dev, Error **errp) |
21 | */ | 21 | n->bar.cap = 0; |
22 | void job_dismiss(Job **job, Error **errp); | 22 | NVME_CAP_SET_MQES(n->bar.cap, 0x7ff); |
23 | 23 | NVME_CAP_SET_CQR(n->bar.cap, 1); | |
24 | -typedef void JobDeferToMainLoopFn(Job *job, void *opaque); | 24 | - NVME_CAP_SET_AMS(n->bar.cap, 1); |
25 | - | 25 | NVME_CAP_SET_TO(n->bar.cap, 0xf); |
26 | -/** | 26 | NVME_CAP_SET_CSS(n->bar.cap, 1); |
27 | - * @job: The job | 27 | NVME_CAP_SET_MPSMAX(n->bar.cap, 4); |
28 | - * @fn: The function to run in the main loop | ||
29 | - * @opaque: The opaque value that is passed to @fn | ||
30 | - * | ||
31 | - * This function must be called by the main job coroutine just before it | ||
32 | - * returns. @fn is executed in the main loop with the job AioContext acquired. | ||
33 | - * | ||
34 | - * Block jobs must call bdrv_unref(), bdrv_close(), and anything that uses | ||
35 | - * bdrv_drain_all() in the main loop. | ||
36 | - * | ||
37 | - * The @job AioContext is held while @fn executes. | ||
38 | - */ | ||
39 | -void job_defer_to_main_loop(Job *job, JobDeferToMainLoopFn *fn, void *opaque); | ||
40 | - | ||
41 | /** | ||
42 | * Synchronously finishes the given @job. If @finish is given, it is called to | ||
43 | * trigger completion or cancellation of the job. | ||
44 | diff --git a/job.c b/job.c | ||
45 | index XXXXXXX..XXXXXXX 100644 | ||
46 | --- a/job.c | ||
47 | +++ b/job.c | ||
48 | @@ -XXX,XX +XXX,XX @@ static void coroutine_fn job_co_entry(void *opaque) | ||
49 | assert(job && job->driver && job->driver->run); | ||
50 | job_pause_point(job); | ||
51 | job->ret = job->driver->run(job, &job->err); | ||
52 | - if (!job->deferred_to_main_loop) { | ||
53 | - job->deferred_to_main_loop = true; | ||
54 | - aio_bh_schedule_oneshot(qemu_get_aio_context(), | ||
55 | - job_exit, | ||
56 | - job); | ||
57 | - } | ||
58 | + job->deferred_to_main_loop = true; | ||
59 | + aio_bh_schedule_oneshot(qemu_get_aio_context(), job_exit, job); | ||
60 | } | ||
61 | |||
62 | |||
63 | @@ -XXX,XX +XXX,XX @@ void job_complete(Job *job, Error **errp) | ||
64 | job->driver->complete(job, errp); | ||
65 | } | ||
66 | |||
67 | - | ||
68 | -typedef struct { | ||
69 | - Job *job; | ||
70 | - JobDeferToMainLoopFn *fn; | ||
71 | - void *opaque; | ||
72 | -} JobDeferToMainLoopData; | ||
73 | - | ||
74 | -static void job_defer_to_main_loop_bh(void *opaque) | ||
75 | -{ | ||
76 | - JobDeferToMainLoopData *data = opaque; | ||
77 | - Job *job = data->job; | ||
78 | - AioContext *aio_context = job->aio_context; | ||
79 | - | ||
80 | - aio_context_acquire(aio_context); | ||
81 | - data->fn(data->job, data->opaque); | ||
82 | - aio_context_release(aio_context); | ||
83 | - | ||
84 | - g_free(data); | ||
85 | -} | ||
86 | - | ||
87 | -void job_defer_to_main_loop(Job *job, JobDeferToMainLoopFn *fn, void *opaque) | ||
88 | -{ | ||
89 | - JobDeferToMainLoopData *data = g_malloc(sizeof(*data)); | ||
90 | - data->job = job; | ||
91 | - data->fn = fn; | ||
92 | - data->opaque = opaque; | ||
93 | - job->deferred_to_main_loop = true; | ||
94 | - | ||
95 | - aio_bh_schedule_oneshot(qemu_get_aio_context(), | ||
96 | - job_defer_to_main_loop_bh, data); | ||
97 | -} | ||
98 | - | ||
99 | int job_finish_sync(Job *job, void (*finish)(Job *, Error **errp), Error **errp) | ||
100 | { | ||
101 | Error *local_err = NULL; | ||
102 | -- | 28 | -- |
103 | 2.17.1 | 29 | 2.21.0 |
104 | 30 | ||
105 | 31 | diff view generated by jsdifflib |
1 | From: John Snow <jsnow@redhat.com> | 1 | From: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com> |
---|---|---|---|
2 | 2 | ||
3 | Rename opaque_job to job to be consistent with other job implementations. | 3 | We forget to enable it for transaction .prepare, while it is already |
4 | Rename 'job', the BackupBlockJob object, to 's' to also be consistent. | 4 | enabled in do_drive_backup since commit a2d665c1bc362 |
5 | "blockdev: loosen restrictions on drive-backup source node" | ||
5 | 6 | ||
6 | Suggested-by: Eric Blake <eblake@redhat.com> | 7 | Signed-off-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com> |
7 | Signed-off-by: John Snow <jsnow@redhat.com> | 8 | Message-id: 20190618140804.59214-1-vsementsov@virtuozzo.com |
8 | Reviewed-by: Max Reitz <mreitz@redhat.com> | 9 | Reviewed-by: John Snow <jsnow@redhat.com> |
9 | Message-id: 20180830015734.19765-8-jsnow@redhat.com | ||
10 | Signed-off-by: Max Reitz <mreitz@redhat.com> | 10 | Signed-off-by: Max Reitz <mreitz@redhat.com> |
11 | --- | 11 | --- |
12 | block/backup.c | 62 +++++++++++++++++++++++++------------------------- | 12 | blockdev.c | 2 +- |
13 | 1 file changed, 31 insertions(+), 31 deletions(-) | 13 | 1 file changed, 1 insertion(+), 1 deletion(-) |
14 | 14 | ||
15 | diff --git a/block/backup.c b/block/backup.c | 15 | diff --git a/blockdev.c b/blockdev.c |
16 | index XXXXXXX..XXXXXXX 100644 | 16 | index XXXXXXX..XXXXXXX 100644 |
17 | --- a/block/backup.c | 17 | --- a/blockdev.c |
18 | +++ b/block/backup.c | 18 | +++ b/blockdev.c |
19 | @@ -XXX,XX +XXX,XX @@ static void backup_incremental_init_copy_bitmap(BackupBlockJob *job) | 19 | @@ -XXX,XX +XXX,XX @@ static void drive_backup_prepare(BlkActionState *common, Error **errp) |
20 | bdrv_dirty_iter_free(dbi); | 20 | assert(common->action->type == TRANSACTION_ACTION_KIND_DRIVE_BACKUP); |
21 | } | 21 | backup = common->action->u.drive_backup.data; |
22 | 22 | ||
23 | -static int coroutine_fn backup_run(Job *opaque_job, Error **errp) | 23 | - bs = qmp_get_root_bs(backup->device, errp); |
24 | +static int coroutine_fn backup_run(Job *job, Error **errp) | 24 | + bs = bdrv_lookup_bs(backup->device, backup->device, errp); |
25 | { | 25 | if (!bs) { |
26 | - BackupBlockJob *job = container_of(opaque_job, BackupBlockJob, common.job); | 26 | return; |
27 | - BlockDriverState *bs = blk_bs(job->common.blk); | ||
28 | + BackupBlockJob *s = container_of(job, BackupBlockJob, common.job); | ||
29 | + BlockDriverState *bs = blk_bs(s->common.blk); | ||
30 | int64_t offset, nb_clusters; | ||
31 | int ret = 0; | ||
32 | |||
33 | - QLIST_INIT(&job->inflight_reqs); | ||
34 | - qemu_co_rwlock_init(&job->flush_rwlock); | ||
35 | + QLIST_INIT(&s->inflight_reqs); | ||
36 | + qemu_co_rwlock_init(&s->flush_rwlock); | ||
37 | |||
38 | - nb_clusters = DIV_ROUND_UP(job->len, job->cluster_size); | ||
39 | - job_progress_set_remaining(&job->common.job, job->len); | ||
40 | + nb_clusters = DIV_ROUND_UP(s->len, s->cluster_size); | ||
41 | + job_progress_set_remaining(job, s->len); | ||
42 | |||
43 | - job->copy_bitmap = hbitmap_alloc(nb_clusters, 0); | ||
44 | - if (job->sync_mode == MIRROR_SYNC_MODE_INCREMENTAL) { | ||
45 | - backup_incremental_init_copy_bitmap(job); | ||
46 | + s->copy_bitmap = hbitmap_alloc(nb_clusters, 0); | ||
47 | + if (s->sync_mode == MIRROR_SYNC_MODE_INCREMENTAL) { | ||
48 | + backup_incremental_init_copy_bitmap(s); | ||
49 | } else { | ||
50 | - hbitmap_set(job->copy_bitmap, 0, nb_clusters); | ||
51 | + hbitmap_set(s->copy_bitmap, 0, nb_clusters); | ||
52 | } | 27 | } |
53 | |||
54 | |||
55 | - job->before_write.notify = backup_before_write_notify; | ||
56 | - bdrv_add_before_write_notifier(bs, &job->before_write); | ||
57 | + s->before_write.notify = backup_before_write_notify; | ||
58 | + bdrv_add_before_write_notifier(bs, &s->before_write); | ||
59 | |||
60 | - if (job->sync_mode == MIRROR_SYNC_MODE_NONE) { | ||
61 | + if (s->sync_mode == MIRROR_SYNC_MODE_NONE) { | ||
62 | /* All bits are set in copy_bitmap to allow any cluster to be copied. | ||
63 | * This does not actually require them to be copied. */ | ||
64 | - while (!job_is_cancelled(&job->common.job)) { | ||
65 | + while (!job_is_cancelled(job)) { | ||
66 | /* Yield until the job is cancelled. We just let our before_write | ||
67 | * notify callback service CoW requests. */ | ||
68 | - job_yield(&job->common.job); | ||
69 | + job_yield(job); | ||
70 | } | ||
71 | - } else if (job->sync_mode == MIRROR_SYNC_MODE_INCREMENTAL) { | ||
72 | - ret = backup_run_incremental(job); | ||
73 | + } else if (s->sync_mode == MIRROR_SYNC_MODE_INCREMENTAL) { | ||
74 | + ret = backup_run_incremental(s); | ||
75 | } else { | ||
76 | /* Both FULL and TOP SYNC_MODE's require copying.. */ | ||
77 | - for (offset = 0; offset < job->len; | ||
78 | - offset += job->cluster_size) { | ||
79 | + for (offset = 0; offset < s->len; | ||
80 | + offset += s->cluster_size) { | ||
81 | bool error_is_read; | ||
82 | int alloced = 0; | ||
83 | |||
84 | - if (yield_and_check(job)) { | ||
85 | + if (yield_and_check(s)) { | ||
86 | break; | ||
87 | } | ||
88 | |||
89 | - if (job->sync_mode == MIRROR_SYNC_MODE_TOP) { | ||
90 | + if (s->sync_mode == MIRROR_SYNC_MODE_TOP) { | ||
91 | int i; | ||
92 | int64_t n; | ||
93 | |||
94 | /* Check to see if these blocks are already in the | ||
95 | * backing file. */ | ||
96 | |||
97 | - for (i = 0; i < job->cluster_size;) { | ||
98 | + for (i = 0; i < s->cluster_size;) { | ||
99 | /* bdrv_is_allocated() only returns true/false based | ||
100 | * on the first set of sectors it comes across that | ||
101 | * are are all in the same state. | ||
102 | @@ -XXX,XX +XXX,XX @@ static int coroutine_fn backup_run(Job *opaque_job, Error **errp) | ||
103 | * needed but at some point that is always the case. */ | ||
104 | alloced = | ||
105 | bdrv_is_allocated(bs, offset + i, | ||
106 | - job->cluster_size - i, &n); | ||
107 | + s->cluster_size - i, &n); | ||
108 | i += n; | ||
109 | |||
110 | if (alloced || n == 0) { | ||
111 | @@ -XXX,XX +XXX,XX @@ static int coroutine_fn backup_run(Job *opaque_job, Error **errp) | ||
112 | if (alloced < 0) { | ||
113 | ret = alloced; | ||
114 | } else { | ||
115 | - ret = backup_do_cow(job, offset, job->cluster_size, | ||
116 | + ret = backup_do_cow(s, offset, s->cluster_size, | ||
117 | &error_is_read, false); | ||
118 | } | ||
119 | if (ret < 0) { | ||
120 | /* Depending on error action, fail now or retry cluster */ | ||
121 | BlockErrorAction action = | ||
122 | - backup_error_action(job, error_is_read, -ret); | ||
123 | + backup_error_action(s, error_is_read, -ret); | ||
124 | if (action == BLOCK_ERROR_ACTION_REPORT) { | ||
125 | break; | ||
126 | } else { | ||
127 | - offset -= job->cluster_size; | ||
128 | + offset -= s->cluster_size; | ||
129 | continue; | ||
130 | } | ||
131 | } | ||
132 | } | ||
133 | } | ||
134 | |||
135 | - notifier_with_return_remove(&job->before_write); | ||
136 | + notifier_with_return_remove(&s->before_write); | ||
137 | |||
138 | /* wait until pending backup_do_cow() calls have completed */ | ||
139 | - qemu_co_rwlock_wrlock(&job->flush_rwlock); | ||
140 | - qemu_co_rwlock_unlock(&job->flush_rwlock); | ||
141 | - hbitmap_free(job->copy_bitmap); | ||
142 | + qemu_co_rwlock_wrlock(&s->flush_rwlock); | ||
143 | + qemu_co_rwlock_unlock(&s->flush_rwlock); | ||
144 | + hbitmap_free(s->copy_bitmap); | ||
145 | |||
146 | return ret; | ||
147 | } | ||
148 | -- | 28 | -- |
149 | 2.17.1 | 29 | 2.21.0 |
150 | 30 | ||
151 | 31 | diff view generated by jsdifflib |
1 | From: John Snow <jsnow@redhat.com> | 1 | From: Anton Nefedov <anton.nefedov@virtuozzo.com> |
---|---|---|---|
2 | 2 | ||
3 | Change the manual deferment to mirror_exit into the implicit | 3 | COW (even empty/zero) areas require encryption too |
4 | callback to job_exit and the mirror_exit callback. | ||
5 | 4 | ||
6 | This does change the order of some bdrv_unref calls and job_completed, | 5 | Signed-off-by: Anton Nefedov <anton.nefedov@virtuozzo.com> |
7 | but thanks to the new context in which we call .exit, this is safe to | 6 | Reviewed-by: Eric Blake <eblake@redhat.com> |
8 | defer the possible flushing of any nodes to the job_finalize_single | ||
9 | cleanup stage. | ||
10 | |||
11 | Signed-off-by: John Snow <jsnow@redhat.com> | ||
12 | Message-id: 20180830015734.19765-6-jsnow@redhat.com | ||
13 | Reviewed-by: Max Reitz <mreitz@redhat.com> | 7 | Reviewed-by: Max Reitz <mreitz@redhat.com> |
14 | Reviewed-by: Jeff Cody <jcody@redhat.com> | 8 | Reviewed-by: Alberto Garcia <berto@igalia.com> |
9 | Message-id: 20190516143028.81155-1-anton.nefedov@virtuozzo.com | ||
15 | Signed-off-by: Max Reitz <mreitz@redhat.com> | 10 | Signed-off-by: Max Reitz <mreitz@redhat.com> |
16 | --- | 11 | --- |
17 | block/mirror.c | 29 +++++++++++------------------ | 12 | tests/qemu-iotests/134 | 9 +++++++++ |
18 | 1 file changed, 11 insertions(+), 18 deletions(-) | 13 | tests/qemu-iotests/134.out | 10 ++++++++++ |
14 | 2 files changed, 19 insertions(+) | ||
19 | 15 | ||
20 | diff --git a/block/mirror.c b/block/mirror.c | 16 | diff --git a/tests/qemu-iotests/134 b/tests/qemu-iotests/134 |
17 | index XXXXXXX..XXXXXXX 100755 | ||
18 | --- a/tests/qemu-iotests/134 | ||
19 | +++ b/tests/qemu-iotests/134 | ||
20 | @@ -XXX,XX +XXX,XX @@ echo | ||
21 | echo "== reading whole image ==" | ||
22 | $QEMU_IO --object $SECRET -c "read 0 $size" --image-opts $IMGSPEC | _filter_qemu_io | _filter_testdir | ||
23 | |||
24 | +echo | ||
25 | +echo "== rewriting cluster part ==" | ||
26 | +$QEMU_IO --object $SECRET -c "write -P 0xb 512 512" --image-opts $IMGSPEC | _filter_qemu_io | _filter_testdir | ||
27 | + | ||
28 | +echo | ||
29 | +echo "== verify pattern ==" | ||
30 | +$QEMU_IO --object $SECRET -c "read -P 0 0 512" --image-opts $IMGSPEC | _filter_qemu_io | _filter_testdir | ||
31 | +$QEMU_IO --object $SECRET -c "read -P 0xb 512 512" --image-opts $IMGSPEC | _filter_qemu_io | _filter_testdir | ||
32 | + | ||
33 | echo | ||
34 | echo "== rewriting whole image ==" | ||
35 | $QEMU_IO --object $SECRET -c "write -P 0xa 0 $size" --image-opts $IMGSPEC | _filter_qemu_io | _filter_testdir | ||
36 | diff --git a/tests/qemu-iotests/134.out b/tests/qemu-iotests/134.out | ||
21 | index XXXXXXX..XXXXXXX 100644 | 37 | index XXXXXXX..XXXXXXX 100644 |
22 | --- a/block/mirror.c | 38 | --- a/tests/qemu-iotests/134.out |
23 | +++ b/block/mirror.c | 39 | +++ b/tests/qemu-iotests/134.out |
24 | @@ -XXX,XX +XXX,XX @@ static void mirror_wait_for_all_io(MirrorBlockJob *s) | 40 | @@ -XXX,XX +XXX,XX @@ Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 encryption=on encrypt. |
25 | } | 41 | read 134217728/134217728 bytes at offset 0 |
26 | } | 42 | 128 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) |
27 | 43 | ||
28 | -typedef struct { | 44 | +== rewriting cluster part == |
29 | - int ret; | 45 | +wrote 512/512 bytes at offset 512 |
30 | -} MirrorExitData; | 46 | +512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) |
31 | - | ||
32 | -static void mirror_exit(Job *job, void *opaque) | ||
33 | +static void mirror_exit(Job *job) | ||
34 | { | ||
35 | MirrorBlockJob *s = container_of(job, MirrorBlockJob, common.job); | ||
36 | BlockJob *bjob = &s->common; | ||
37 | - MirrorExitData *data = opaque; | ||
38 | MirrorBDSOpaque *bs_opaque = s->mirror_top_bs->opaque; | ||
39 | AioContext *replace_aio_context = NULL; | ||
40 | BlockDriverState *src = s->mirror_top_bs->backing->bs; | ||
41 | BlockDriverState *target_bs = blk_bs(s->target); | ||
42 | BlockDriverState *mirror_top_bs = s->mirror_top_bs; | ||
43 | Error *local_err = NULL; | ||
44 | + int ret = job->ret; | ||
45 | |||
46 | bdrv_release_dirty_bitmap(src, s->dirty_bitmap); | ||
47 | |||
48 | - /* Make sure that the source BDS doesn't go away before we called | ||
49 | - * job_completed(). */ | ||
50 | + /* Make sure that the source BDS doesn't go away during bdrv_replace_node, | ||
51 | + * before we can call bdrv_drained_end */ | ||
52 | bdrv_ref(src); | ||
53 | bdrv_ref(mirror_top_bs); | ||
54 | bdrv_ref(target_bs); | ||
55 | @@ -XXX,XX +XXX,XX @@ static void mirror_exit(Job *job, void *opaque) | ||
56 | bdrv_set_backing_hd(target_bs, backing, &local_err); | ||
57 | if (local_err) { | ||
58 | error_report_err(local_err); | ||
59 | - data->ret = -EPERM; | ||
60 | + ret = -EPERM; | ||
61 | } | ||
62 | } | ||
63 | } | ||
64 | @@ -XXX,XX +XXX,XX @@ static void mirror_exit(Job *job, void *opaque) | ||
65 | aio_context_acquire(replace_aio_context); | ||
66 | } | ||
67 | |||
68 | - if (s->should_complete && data->ret == 0) { | ||
69 | + if (s->should_complete && ret == 0) { | ||
70 | BlockDriverState *to_replace = src; | ||
71 | if (s->to_replace) { | ||
72 | to_replace = s->to_replace; | ||
73 | @@ -XXX,XX +XXX,XX @@ static void mirror_exit(Job *job, void *opaque) | ||
74 | bdrv_drained_end(target_bs); | ||
75 | if (local_err) { | ||
76 | error_report_err(local_err); | ||
77 | - data->ret = -EPERM; | ||
78 | + ret = -EPERM; | ||
79 | } | ||
80 | } | ||
81 | if (s->to_replace) { | ||
82 | @@ -XXX,XX +XXX,XX @@ static void mirror_exit(Job *job, void *opaque) | ||
83 | blk_insert_bs(bjob->blk, mirror_top_bs, &error_abort); | ||
84 | |||
85 | bs_opaque->job = NULL; | ||
86 | - job_completed(job, data->ret); | ||
87 | |||
88 | - g_free(data); | ||
89 | bdrv_drained_end(src); | ||
90 | bdrv_unref(mirror_top_bs); | ||
91 | bdrv_unref(src); | ||
92 | + | 47 | + |
93 | + job->ret = ret; | 48 | +== verify pattern == |
94 | } | 49 | +read 512/512 bytes at offset 0 |
95 | 50 | +512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) | |
96 | static void mirror_throttle(MirrorBlockJob *s) | 51 | +read 512/512 bytes at offset 512 |
97 | @@ -XXX,XX +XXX,XX @@ static int mirror_flush(MirrorBlockJob *s) | 52 | +512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) |
98 | static int coroutine_fn mirror_run(Job *job, Error **errp) | 53 | + |
99 | { | 54 | == rewriting whole image == |
100 | MirrorBlockJob *s = container_of(job, MirrorBlockJob, common.job); | 55 | wrote 134217728/134217728 bytes at offset 0 |
101 | - MirrorExitData *data; | 56 | 128 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) |
102 | BlockDriverState *bs = s->mirror_top_bs->backing->bs; | ||
103 | BlockDriverState *target_bs = blk_bs(s->target); | ||
104 | bool need_drain = true; | ||
105 | @@ -XXX,XX +XXX,XX @@ immediate_exit: | ||
106 | g_free(s->in_flight_bitmap); | ||
107 | bdrv_dirty_iter_free(s->dbi); | ||
108 | |||
109 | - data = g_malloc(sizeof(*data)); | ||
110 | - data->ret = ret; | ||
111 | - | ||
112 | if (need_drain) { | ||
113 | bdrv_drained_begin(bs); | ||
114 | } | ||
115 | |||
116 | - job_defer_to_main_loop(&s->common.job, mirror_exit, data); | ||
117 | return ret; | ||
118 | } | ||
119 | |||
120 | @@ -XXX,XX +XXX,XX @@ static const BlockJobDriver mirror_job_driver = { | ||
121 | .user_resume = block_job_user_resume, | ||
122 | .drain = block_job_drain, | ||
123 | .run = mirror_run, | ||
124 | + .exit = mirror_exit, | ||
125 | .pause = mirror_pause, | ||
126 | .complete = mirror_complete, | ||
127 | }, | ||
128 | @@ -XXX,XX +XXX,XX @@ static const BlockJobDriver commit_active_job_driver = { | ||
129 | .user_resume = block_job_user_resume, | ||
130 | .drain = block_job_drain, | ||
131 | .run = mirror_run, | ||
132 | + .exit = mirror_exit, | ||
133 | .pause = mirror_pause, | ||
134 | .complete = mirror_complete, | ||
135 | }, | ||
136 | -- | 57 | -- |
137 | 2.17.1 | 58 | 2.21.0 |
138 | 59 | ||
139 | 60 | diff view generated by jsdifflib |
1 | From: John Snow <jsnow@redhat.com> | 1 | From: Sam Eiderman <shmuel.eiderman@oracle.com> |
---|---|---|---|
2 | 2 | ||
3 | Change the manual deferment to commit_complete into the implicit | 3 | Commit b0651b8c246d ("vmdk: Move l1_size check into vmdk_add_extent") |
4 | callback to job_exit, renaming commit_complete to commit_exit. | 4 | extended the l1_size check from VMDK4 to VMDK3 but did not update the |
5 | default coverage in the moved comment. | ||
5 | 6 | ||
6 | This conversion does change the timing of when job_completed is | 7 | The previous vmdk4 calculation: |
7 | called to after the bdrv_replace_node and bdrv_unref calls, which | ||
8 | could have implications for bjob->blk which will now be put down | ||
9 | after this cleanup. | ||
10 | 8 | ||
11 | Kevin highlights that we did not take any permissions for that backend | 9 | (512 * 1024 * 1024) * 512(l2 entries) * 65536(grain) = 16PB |
12 | at job creation time, so it is safe to reorder these operations. | ||
13 | 10 | ||
14 | Signed-off-by: John Snow <jsnow@redhat.com> | 11 | The added vmdk3 calculation: |
12 | |||
13 | (512 * 1024 * 1024) * 4096(l2 entries) * 512(grain) = 1PB | ||
14 | |||
15 | Adding the calculation of vmdk3 to the comment. | ||
16 | |||
17 | In any case, VMware does not offer virtual disks more than 2TB for | ||
18 | vmdk4/vmdk3 or 64TB for the new undocumented seSparse format which is | ||
19 | not implemented yet in qemu. | ||
20 | |||
21 | Reviewed-by: Karl Heubaum <karl.heubaum@oracle.com> | ||
22 | Reviewed-by: Eyal Moscovici <eyal.moscovici@oracle.com> | ||
23 | Reviewed-by: Liran Alon <liran.alon@oracle.com> | ||
24 | Reviewed-by: Arbel Moshe <arbel.moshe@oracle.com> | ||
25 | Signed-off-by: Sam Eiderman <shmuel.eiderman@oracle.com> | ||
26 | Message-id: 20190620091057.47441-2-shmuel.eiderman@oracle.com | ||
27 | Reviewed-by: yuchenlin <yuchenlin@synology.com> | ||
15 | Reviewed-by: Max Reitz <mreitz@redhat.com> | 28 | Reviewed-by: Max Reitz <mreitz@redhat.com> |
16 | Message-id: 20180830015734.19765-5-jsnow@redhat.com | ||
17 | Reviewed-by: Jeff Cody <jcody@redhat.com> | ||
18 | Signed-off-by: Max Reitz <mreitz@redhat.com> | 29 | Signed-off-by: Max Reitz <mreitz@redhat.com> |
19 | --- | 30 | --- |
20 | block/commit.c | 22 +++++----------------- | 31 | block/vmdk.c | 11 ++++++++--- |
21 | 1 file changed, 5 insertions(+), 17 deletions(-) | 32 | 1 file changed, 8 insertions(+), 3 deletions(-) |
22 | 33 | ||
23 | diff --git a/block/commit.c b/block/commit.c | 34 | diff --git a/block/vmdk.c b/block/vmdk.c |
24 | index XXXXXXX..XXXXXXX 100644 | 35 | index XXXXXXX..XXXXXXX 100644 |
25 | --- a/block/commit.c | 36 | --- a/block/vmdk.c |
26 | +++ b/block/commit.c | 37 | +++ b/block/vmdk.c |
27 | @@ -XXX,XX +XXX,XX @@ static int coroutine_fn commit_populate(BlockBackend *bs, BlockBackend *base, | 38 | @@ -XXX,XX +XXX,XX @@ static int vmdk_add_extent(BlockDriverState *bs, |
28 | return 0; | 39 | return -EFBIG; |
29 | } | 40 | } |
30 | 41 | if (l1_size > 512 * 1024 * 1024) { | |
31 | -typedef struct { | 42 | - /* Although with big capacity and small l1_entry_sectors, we can get a |
32 | - int ret; | 43 | + /* |
33 | -} CommitCompleteData; | 44 | + * Although with big capacity and small l1_entry_sectors, we can get a |
34 | - | 45 | * big l1_size, we don't want unbounded value to allocate the table. |
35 | -static void commit_complete(Job *job, void *opaque) | 46 | - * Limit it to 512M, which is 16PB for default cluster and L2 table |
36 | +static void commit_exit(Job *job) | 47 | - * size */ |
37 | { | 48 | + * Limit it to 512M, which is: |
38 | CommitBlockJob *s = container_of(job, CommitBlockJob, common.job); | 49 | + * 16PB - for default "Hosted Sparse Extent" (VMDK4) |
39 | BlockJob *bjob = &s->common; | 50 | + * cluster size: 64KB, L2 table size: 512 entries |
40 | - CommitCompleteData *data = opaque; | 51 | + * 1PB - for default "ESXi Host Sparse Extent" (VMDK3/vmfsSparse) |
41 | BlockDriverState *top = blk_bs(s->top); | 52 | + * cluster size: 512B, L2 table size: 4096 entries |
42 | BlockDriverState *base = blk_bs(s->base); | 53 | + */ |
43 | BlockDriverState *commit_top_bs = s->commit_top_bs; | 54 | error_setg(errp, "L1 size too big"); |
44 | - int ret = data->ret; | 55 | return -EFBIG; |
45 | bool remove_commit_top_bs = false; | 56 | } |
46 | |||
47 | /* Make sure commit_top_bs and top stay around until bdrv_replace_node() */ | ||
48 | @@ -XXX,XX +XXX,XX @@ static void commit_complete(Job *job, void *opaque) | ||
49 | * the normal backing chain can be restored. */ | ||
50 | blk_unref(s->base); | ||
51 | |||
52 | - if (!job_is_cancelled(job) && ret == 0) { | ||
53 | + if (!job_is_cancelled(job) && job->ret == 0) { | ||
54 | /* success */ | ||
55 | - ret = bdrv_drop_intermediate(s->commit_top_bs, base, | ||
56 | - s->backing_file_str); | ||
57 | + job->ret = bdrv_drop_intermediate(s->commit_top_bs, base, | ||
58 | + s->backing_file_str); | ||
59 | } else { | ||
60 | /* XXX Can (or should) we somehow keep 'consistent read' blocked even | ||
61 | * after the failed/cancelled commit job is gone? If we already wrote | ||
62 | @@ -XXX,XX +XXX,XX @@ static void commit_complete(Job *job, void *opaque) | ||
63 | * bdrv_set_backing_hd() to fail. */ | ||
64 | block_job_remove_all_bdrv(bjob); | ||
65 | |||
66 | - job_completed(job, ret); | ||
67 | - g_free(data); | ||
68 | - | ||
69 | /* If bdrv_drop_intermediate() didn't already do that, remove the commit | ||
70 | * filter driver from the backing chain. Do this as the final step so that | ||
71 | * the 'consistent read' permission can be granted. */ | ||
72 | @@ -XXX,XX +XXX,XX @@ static void commit_complete(Job *job, void *opaque) | ||
73 | static int coroutine_fn commit_run(Job *job, Error **errp) | ||
74 | { | ||
75 | CommitBlockJob *s = container_of(job, CommitBlockJob, common.job); | ||
76 | - CommitCompleteData *data; | ||
77 | int64_t offset; | ||
78 | uint64_t delay_ns = 0; | ||
79 | int ret = 0; | ||
80 | @@ -XXX,XX +XXX,XX @@ static int coroutine_fn commit_run(Job *job, Error **errp) | ||
81 | out: | ||
82 | qemu_vfree(buf); | ||
83 | |||
84 | - data = g_malloc(sizeof(*data)); | ||
85 | - data->ret = ret; | ||
86 | - job_defer_to_main_loop(&s->common.job, commit_complete, data); | ||
87 | return ret; | ||
88 | } | ||
89 | |||
90 | @@ -XXX,XX +XXX,XX @@ static const BlockJobDriver commit_job_driver = { | ||
91 | .user_resume = block_job_user_resume, | ||
92 | .drain = block_job_drain, | ||
93 | .run = commit_run, | ||
94 | + .exit = commit_exit, | ||
95 | }, | ||
96 | }; | ||
97 | |||
98 | -- | 57 | -- |
99 | 2.17.1 | 58 | 2.21.0 |
100 | 59 | ||
101 | 60 | diff view generated by jsdifflib |
1 | From: John Snow <jsnow@redhat.com> | 1 | From: Sam Eiderman <shmuel.eiderman@oracle.com> |
---|---|---|---|
2 | 2 | ||
3 | Jobs are now expected to return their retcode on the stack, from the | 3 | 512M of L1 entries is a very loose bound, only 32M are required to store |
4 | .run callback, so we can remove that argument. | 4 | the maximal supported VMDK file size of 2TB. |
5 | 5 | ||
6 | job_cancel does not need to set -ECANCELED because job_completed will | 6 | Fixed qemu-iotest 59# - now failure occures before on impossible L1 |
7 | update the return code itself if the job was canceled. | 7 | table size. |
8 | 8 | ||
9 | While we're here, make job_completed static to job.c and remove it from | 9 | Reviewed-by: Karl Heubaum <karl.heubaum@oracle.com> |
10 | job.h; move the documentation of return code to the .run() callback and | 10 | Reviewed-by: Eyal Moscovici <eyal.moscovici@oracle.com> |
11 | to the job->ret property, accordingly. | 11 | Reviewed-by: Liran Alon <liran.alon@oracle.com> |
12 | 12 | Reviewed-by: Arbel Moshe <arbel.moshe@oracle.com> | |
13 | Signed-off-by: John Snow <jsnow@redhat.com> | 13 | Signed-off-by: Sam Eiderman <shmuel.eiderman@oracle.com> |
14 | Message-id: 20180830015734.19765-9-jsnow@redhat.com | 14 | Message-id: 20190620091057.47441-3-shmuel.eiderman@oracle.com |
15 | Reviewed-by: Max Reitz <mreitz@redhat.com> | 15 | Reviewed-by: Max Reitz <mreitz@redhat.com> |
16 | Signed-off-by: Max Reitz <mreitz@redhat.com> | 16 | Signed-off-by: Max Reitz <mreitz@redhat.com> |
17 | --- | 17 | --- |
18 | include/qemu/job.h | 28 +++++++++++++++------------- | 18 | block/vmdk.c | 13 +++++++------ |
19 | job.c | 11 ++++++----- | 19 | tests/qemu-iotests/059.out | 2 +- |
20 | trace-events | 2 +- | 20 | 2 files changed, 8 insertions(+), 7 deletions(-) |
21 | 3 files changed, 22 insertions(+), 19 deletions(-) | ||
22 | 21 | ||
23 | diff --git a/include/qemu/job.h b/include/qemu/job.h | 22 | diff --git a/block/vmdk.c b/block/vmdk.c |
24 | index XXXXXXX..XXXXXXX 100644 | 23 | index XXXXXXX..XXXXXXX 100644 |
25 | --- a/include/qemu/job.h | 24 | --- a/block/vmdk.c |
26 | +++ b/include/qemu/job.h | 25 | +++ b/block/vmdk.c |
27 | @@ -XXX,XX +XXX,XX @@ typedef struct Job { | 26 | @@ -XXX,XX +XXX,XX @@ static int vmdk_add_extent(BlockDriverState *bs, |
28 | /** Estimated progress_current value at the completion of the job */ | 27 | error_setg(errp, "Invalid granularity, image may be corrupt"); |
29 | int64_t progress_total; | 28 | return -EFBIG; |
30 | 29 | } | |
31 | - /** ret code passed to job_completed. */ | 30 | - if (l1_size > 512 * 1024 * 1024) { |
32 | + /** | 31 | + if (l1_size > 32 * 1024 * 1024) { |
33 | + * Return code from @run and/or @prepare callback(s). | 32 | /* |
34 | + * Not final until the job has reached the CONCLUDED status. | 33 | * Although with big capacity and small l1_entry_sectors, we can get a |
35 | + * 0 on success, -errno on failure. | 34 | * big l1_size, we don't want unbounded value to allocate the table. |
36 | + */ | 35 | - * Limit it to 512M, which is: |
37 | int ret; | 36 | - * 16PB - for default "Hosted Sparse Extent" (VMDK4) |
38 | 37 | - * cluster size: 64KB, L2 table size: 512 entries | |
39 | /** | 38 | - * 1PB - for default "ESXi Host Sparse Extent" (VMDK3/vmfsSparse) |
40 | @@ -XXX,XX +XXX,XX @@ struct JobDriver { | 39 | - * cluster size: 512B, L2 table size: 4096 entries |
41 | /** Enum describing the operation */ | 40 | + * Limit it to 32M, which is enough to store: |
42 | JobType job_type; | 41 | + * 8TB - for both VMDK3 & VMDK4 with |
43 | 42 | + * minimal cluster size: 512B | |
44 | - /** Mandatory: Entrypoint for the Coroutine. */ | 43 | + * minimal L2 table size: 512 entries |
45 | + /** | 44 | + * 8 TB is still more than the maximal value supported for |
46 | + * Mandatory: Entrypoint for the Coroutine. | 45 | + * VMDK3 & VMDK4 which is 2TB. |
47 | + * | 46 | */ |
48 | + * This callback will be invoked when moving from CREATED to RUNNING. | 47 | error_setg(errp, "L1 size too big"); |
49 | + * | 48 | return -EFBIG; |
50 | + * If this callback returns nonzero, the job transaction it is part of is | 49 | diff --git a/tests/qemu-iotests/059.out b/tests/qemu-iotests/059.out |
51 | + * aborted. If it returns zero, the job moves into the WAITING state. If it | ||
52 | + * is the last job to complete in its transaction, all jobs in the | ||
53 | + * transaction move from WAITING to PENDING. | ||
54 | + */ | ||
55 | int coroutine_fn (*run)(Job *job, Error **errp); | ||
56 | |||
57 | /** | ||
58 | @@ -XXX,XX +XXX,XX @@ void job_early_fail(Job *job); | ||
59 | /** Moves the @job from RUNNING to READY */ | ||
60 | void job_transition_to_ready(Job *job); | ||
61 | |||
62 | -/** | ||
63 | - * @job: The job being completed. | ||
64 | - * @ret: The status code. | ||
65 | - * | ||
66 | - * Marks @job as completed. If @ret is non-zero, the job transaction it is part | ||
67 | - * of is aborted. If @ret is zero, the job moves into the WAITING state. If it | ||
68 | - * is the last job to complete in its transaction, all jobs in the transaction | ||
69 | - * move from WAITING to PENDING. | ||
70 | - */ | ||
71 | -void job_completed(Job *job, int ret); | ||
72 | - | ||
73 | /** Asynchronously complete the specified @job. */ | ||
74 | void job_complete(Job *job, Error **errp); | ||
75 | |||
76 | diff --git a/job.c b/job.c | ||
77 | index XXXXXXX..XXXXXXX 100644 | 50 | index XXXXXXX..XXXXXXX 100644 |
78 | --- a/job.c | 51 | --- a/tests/qemu-iotests/059.out |
79 | +++ b/job.c | 52 | +++ b/tests/qemu-iotests/059.out |
80 | @@ -XXX,XX +XXX,XX @@ void job_drain(Job *job) | 53 | @@ -XXX,XX +XXX,XX @@ Offset Length Mapped to File |
81 | } | 54 | 0x140000000 0x10000 0x50000 TEST_DIR/t-s003.vmdk |
82 | } | 55 | |
83 | 56 | === Testing afl image with a very large capacity === | |
84 | +static void job_completed(Job *job); | 57 | -qemu-img: Can't get image size 'TEST_DIR/afl9.IMGFMT': File too large |
85 | + | 58 | +qemu-img: Could not open 'TEST_DIR/afl9.IMGFMT': L1 size too big |
86 | static void job_exit(void *opaque) | 59 | *** done |
87 | { | ||
88 | Job *job = (Job *)opaque; | ||
89 | @@ -XXX,XX +XXX,XX @@ static void job_exit(void *opaque) | ||
90 | job->driver->exit(job); | ||
91 | aio_context_release(aio_context); | ||
92 | } | ||
93 | - job_completed(job, job->ret); | ||
94 | + job_completed(job); | ||
95 | } | ||
96 | |||
97 | /** | ||
98 | @@ -XXX,XX +XXX,XX @@ static void job_completed_txn_success(Job *job) | ||
99 | } | ||
100 | } | ||
101 | |||
102 | -void job_completed(Job *job, int ret) | ||
103 | +static void job_completed(Job *job) | ||
104 | { | ||
105 | assert(job && job->txn && !job_is_completed(job)); | ||
106 | |||
107 | - job->ret = ret; | ||
108 | job_update_rc(job); | ||
109 | - trace_job_completed(job, ret, job->ret); | ||
110 | + trace_job_completed(job, job->ret); | ||
111 | if (job->ret) { | ||
112 | job_completed_txn_abort(job); | ||
113 | } else { | ||
114 | @@ -XXX,XX +XXX,XX @@ void job_cancel(Job *job, bool force) | ||
115 | } | ||
116 | job_cancel_async(job, force); | ||
117 | if (!job_started(job)) { | ||
118 | - job_completed(job, -ECANCELED); | ||
119 | + job_completed(job); | ||
120 | } else if (job->deferred_to_main_loop) { | ||
121 | job_completed_txn_abort(job); | ||
122 | } else { | ||
123 | diff --git a/trace-events b/trace-events | ||
124 | index XXXXXXX..XXXXXXX 100644 | ||
125 | --- a/trace-events | ||
126 | +++ b/trace-events | ||
127 | @@ -XXX,XX +XXX,XX @@ gdbstub_err_checksum_incorrect(uint8_t expected, uint8_t got) "got command packe | ||
128 | # job.c | ||
129 | job_state_transition(void *job, int ret, const char *legal, const char *s0, const char *s1) "job %p (ret: %d) attempting %s transition (%s-->%s)" | ||
130 | job_apply_verb(void *job, const char *state, const char *verb, const char *legal) "job %p in state %s; applying verb %s (%s)" | ||
131 | -job_completed(void *job, int ret, int jret) "job %p ret %d corrected ret %d" | ||
132 | +job_completed(void *job, int ret) "job %p ret %d" | ||
133 | |||
134 | # job-qmp.c | ||
135 | qmp_job_cancel(void *job) "job %p" | ||
136 | -- | 60 | -- |
137 | 2.17.1 | 61 | 2.21.0 |
138 | 62 | ||
139 | 63 | diff view generated by jsdifflib |
1 | From: John Snow <jsnow@redhat.com> | 1 | From: Sam Eiderman <shmuel.eiderman@oracle.com> |
---|---|---|---|
2 | 2 | ||
3 | Presently we codify the entry point for a job as the "start" callback, | 3 | Until ESXi 6.5 VMware used the vmfsSparse format for snapshots (VMDK3 in |
4 | but a more apt name would be "run" to clarify the idea that when this | 4 | QEMU). |
5 | function returns we consider the job to have "finished," except for | 5 | |
6 | any cleanup which occurs in separate callbacks later. | 6 | This format was lacking in the following: |
7 | 7 | ||
8 | As part of this clarification, change the signature to include an error | 8 | * Grain directory (L1) and grain table (L2) entries were 32-bit, |
9 | object and a return code. The error ptr is not yet used, and the return | 9 | allowing access to only 2TB (slightly less) of data. |
10 | code while captured, will be overwritten by actions in the job_completed | 10 | * The grain size (default) was 512 bytes - leading to data |
11 | function. | 11 | fragmentation and many grain tables. |
12 | 12 | * For space reclamation purposes, it was necessary to find all the | |
13 | Signed-off-by: John Snow <jsnow@redhat.com> | 13 | grains which are not pointed to by any grain table - so a reverse |
14 | Reviewed-by: Max Reitz <mreitz@redhat.com> | 14 | mapping of "offset of grain in vmdk" to "grain table" must be |
15 | Message-id: 20180830015734.19765-2-jsnow@redhat.com | 15 | constructed - which takes large amounts of CPU/RAM. |
16 | Reviewed-by: Jeff Cody <jcody@redhat.com> | 16 | |
17 | The format specification can be found in VMware's documentation: | ||
18 | https://www.vmware.com/support/developer/vddk/vmdk_50_technote.pdf | ||
19 | |||
20 | In ESXi 6.5, to support snapshot files larger than 2TB, a new format was | ||
21 | introduced: SESparse (Space Efficient). | ||
22 | |||
23 | This format fixes the above issues: | ||
24 | |||
25 | * All entries are now 64-bit. | ||
26 | * The grain size (default) is 4KB. | ||
27 | * Grain directory and grain tables are now located at the beginning | ||
28 | of the file. | ||
29 | + seSparse format reserves space for all grain tables. | ||
30 | + Grain tables can be addressed using an index. | ||
31 | + Grains are located in the end of the file and can also be | ||
32 | addressed with an index. | ||
33 | - seSparse vmdks of large disks (64TB) have huge preallocated | ||
34 | headers - mainly due to L2 tables, even for empty snapshots. | ||
35 | * The header contains a reverse mapping ("backmap") of "offset of | ||
36 | grain in vmdk" to "grain table" and a bitmap ("free bitmap") which | ||
37 | specifies for each grain - whether it is allocated or not. | ||
38 | Using these data structures we can implement space reclamation | ||
39 | efficiently. | ||
40 | * Due to the fact that the header now maintains two mappings: | ||
41 | * The regular one (grain directory & grain tables) | ||
42 | * A reverse one (backmap and free bitmap) | ||
43 | These data structures can lose consistency upon crash and result | ||
44 | in a corrupted VMDK. | ||
45 | Therefore, a journal is also added to the VMDK and is replayed | ||
46 | when the VMware reopens the file after a crash. | ||
47 | |||
48 | Since ESXi 6.7 - SESparse is the only snapshot format available. | ||
49 | |||
50 | Unfortunately, VMware does not provide documentation regarding the new | ||
51 | seSparse format. | ||
52 | |||
53 | This commit is based on black-box research of the seSparse format. | ||
54 | Various in-guest block operations and their effect on the snapshot file | ||
55 | were tested. | ||
56 | |||
57 | The only VMware provided source of information (regarding the underlying | ||
58 | implementation) was a log file on the ESXi: | ||
59 | |||
60 | /var/log/hostd.log | ||
61 | |||
62 | Whenever an seSparse snapshot is created - the log is being populated | ||
63 | with seSparse records. | ||
64 | |||
65 | Relevant log records are of the form: | ||
66 | |||
67 | [...] Const Header: | ||
68 | [...] constMagic = 0xcafebabe | ||
69 | [...] version = 2.1 | ||
70 | [...] capacity = 204800 | ||
71 | [...] grainSize = 8 | ||
72 | [...] grainTableSize = 64 | ||
73 | [...] flags = 0 | ||
74 | [...] Extents: | ||
75 | [...] Header : <1 : 1> | ||
76 | [...] JournalHdr : <2 : 2> | ||
77 | [...] Journal : <2048 : 2048> | ||
78 | [...] GrainDirectory : <4096 : 2048> | ||
79 | [...] GrainTables : <6144 : 2048> | ||
80 | [...] FreeBitmap : <8192 : 2048> | ||
81 | [...] BackMap : <10240 : 2048> | ||
82 | [...] Grain : <12288 : 204800> | ||
83 | [...] Volatile Header: | ||
84 | [...] volatileMagic = 0xcafecafe | ||
85 | [...] FreeGTNumber = 0 | ||
86 | [...] nextTxnSeqNumber = 0 | ||
87 | [...] replayJournal = 0 | ||
88 | |||
89 | The sizes that are seen in the log file are in sectors. | ||
90 | Extents are of the following format: <offset : size> | ||
91 | |||
92 | This commit is a strict implementation which enforces: | ||
93 | * magics | ||
94 | * version number 2.1 | ||
95 | * grain size of 8 sectors (4KB) | ||
96 | * grain table size of 64 sectors | ||
97 | * zero flags | ||
98 | * extent locations | ||
99 | |||
100 | Additionally, this commit proivdes only a subset of the functionality | ||
101 | offered by seSparse's format: | ||
102 | * Read-only | ||
103 | * No journal replay | ||
104 | * No space reclamation | ||
105 | * No unmap support | ||
106 | |||
107 | Hence, journal header, journal, free bitmap and backmap extents are | ||
108 | unused, only the "classic" (L1 -> L2 -> data) grain access is | ||
109 | implemented. | ||
110 | |||
111 | However there are several differences in the grain access itself. | ||
112 | Grain directory (L1): | ||
113 | * Grain directory entries are indexes (not offsets) to grain | ||
114 | tables. | ||
115 | * Valid grain directory entries have their highest nibble set to | ||
116 | 0x1. | ||
117 | * Since grain tables are always located in the beginning of the | ||
118 | file - the index can fit into 32 bits - so we can use its low | ||
119 | part if it's valid. | ||
120 | Grain table (L2): | ||
121 | * Grain table entries are indexes (not offsets) to grains. | ||
122 | * If the highest nibble of the entry is: | ||
123 | 0x0: | ||
124 | The grain in not allocated. | ||
125 | The rest of the bytes are 0. | ||
126 | 0x1: | ||
127 | The grain is unmapped - guest sees a zero grain. | ||
128 | The rest of the bits point to the previously mapped grain, | ||
129 | see 0x3 case. | ||
130 | 0x2: | ||
131 | The grain is zero. | ||
132 | 0x3: | ||
133 | The grain is allocated - to get the index calculate: | ||
134 | ((entry & 0x0fff000000000000) >> 48) | | ||
135 | ((entry & 0x0000ffffffffffff) << 12) | ||
136 | * The difference between 0x1 and 0x2 is that 0x1 is an unallocated | ||
137 | grain which results from the guest using sg_unmap to unmap the | ||
138 | grain - but the grain itself still exists in the grain extent - a | ||
139 | space reclamation procedure should delete it. | ||
140 | Unmapping a zero grain has no effect (0x2 will not change to 0x1) | ||
141 | but unmapping an unallocated grain will (0x0 to 0x1) - naturally. | ||
142 | |||
143 | In order to implement seSparse some fields had to be changed to support | ||
144 | both 32-bit and 64-bit entry sizes. | ||
145 | |||
146 | Reviewed-by: Karl Heubaum <karl.heubaum@oracle.com> | ||
147 | Reviewed-by: Eyal Moscovici <eyal.moscovici@oracle.com> | ||
148 | Reviewed-by: Arbel Moshe <arbel.moshe@oracle.com> | ||
149 | Signed-off-by: Sam Eiderman <shmuel.eiderman@oracle.com> | ||
150 | Message-id: 20190620091057.47441-4-shmuel.eiderman@oracle.com | ||
17 | Signed-off-by: Max Reitz <mreitz@redhat.com> | 151 | Signed-off-by: Max Reitz <mreitz@redhat.com> |
18 | --- | 152 | --- |
19 | include/qemu/job.h | 2 +- | 153 | block/vmdk.c | 358 ++++++++++++++++++++++++++++++++++++++++++++++++--- |
20 | block/backup.c | 7 ++++--- | 154 | 1 file changed, 342 insertions(+), 16 deletions(-) |
21 | block/commit.c | 7 ++++--- | 155 | |
22 | block/create.c | 8 +++++--- | 156 | diff --git a/block/vmdk.c b/block/vmdk.c |
23 | block/mirror.c | 10 ++++++---- | ||
24 | block/stream.c | 7 ++++--- | ||
25 | job.c | 6 +++--- | ||
26 | tests/test-bdrv-drain.c | 7 ++++--- | ||
27 | tests/test-blockjob-txn.c | 16 ++++++++-------- | ||
28 | tests/test-blockjob.c | 7 ++++--- | ||
29 | 10 files changed, 43 insertions(+), 34 deletions(-) | ||
30 | |||
31 | diff --git a/include/qemu/job.h b/include/qemu/job.h | ||
32 | index XXXXXXX..XXXXXXX 100644 | 157 | index XXXXXXX..XXXXXXX 100644 |
33 | --- a/include/qemu/job.h | 158 | --- a/block/vmdk.c |
34 | +++ b/include/qemu/job.h | 159 | +++ b/block/vmdk.c |
35 | @@ -XXX,XX +XXX,XX @@ struct JobDriver { | 160 | @@ -XXX,XX +XXX,XX @@ typedef struct { |
36 | JobType job_type; | 161 | uint16_t compressAlgorithm; |
37 | 162 | } QEMU_PACKED VMDK4Header; | |
38 | /** Mandatory: Entrypoint for the Coroutine. */ | 163 | |
39 | - CoroutineEntry *start; | 164 | +typedef struct VMDKSESparseConstHeader { |
40 | + int coroutine_fn (*run)(Job *job, Error **errp); | 165 | + uint64_t magic; |
41 | 166 | + uint64_t version; | |
42 | /** | 167 | + uint64_t capacity; |
43 | * If the callback is not NULL, it will be invoked when the job transitions | 168 | + uint64_t grain_size; |
44 | diff --git a/block/backup.c b/block/backup.c | 169 | + uint64_t grain_table_size; |
45 | index XXXXXXX..XXXXXXX 100644 | 170 | + uint64_t flags; |
46 | --- a/block/backup.c | 171 | + uint64_t reserved1; |
47 | +++ b/block/backup.c | 172 | + uint64_t reserved2; |
48 | @@ -XXX,XX +XXX,XX @@ static void backup_incremental_init_copy_bitmap(BackupBlockJob *job) | 173 | + uint64_t reserved3; |
49 | bdrv_dirty_iter_free(dbi); | 174 | + uint64_t reserved4; |
50 | } | 175 | + uint64_t volatile_header_offset; |
51 | 176 | + uint64_t volatile_header_size; | |
52 | -static void coroutine_fn backup_run(void *opaque) | 177 | + uint64_t journal_header_offset; |
53 | +static int coroutine_fn backup_run(Job *opaque_job, Error **errp) | 178 | + uint64_t journal_header_size; |
54 | { | 179 | + uint64_t journal_offset; |
55 | - BackupBlockJob *job = opaque; | 180 | + uint64_t journal_size; |
56 | + BackupBlockJob *job = container_of(opaque_job, BackupBlockJob, common.job); | 181 | + uint64_t grain_dir_offset; |
57 | BackupCompleteData *data; | 182 | + uint64_t grain_dir_size; |
58 | BlockDriverState *bs = blk_bs(job->common.blk); | 183 | + uint64_t grain_tables_offset; |
59 | int64_t offset, nb_clusters; | 184 | + uint64_t grain_tables_size; |
60 | @@ -XXX,XX +XXX,XX @@ static void coroutine_fn backup_run(void *opaque) | 185 | + uint64_t free_bitmap_offset; |
61 | data = g_malloc(sizeof(*data)); | 186 | + uint64_t free_bitmap_size; |
62 | data->ret = ret; | 187 | + uint64_t backmap_offset; |
63 | job_defer_to_main_loop(&job->common.job, backup_complete, data); | 188 | + uint64_t backmap_size; |
64 | + return ret; | 189 | + uint64_t grains_offset; |
65 | } | 190 | + uint64_t grains_size; |
66 | 191 | + uint8_t pad[304]; | |
67 | static const BlockJobDriver backup_job_driver = { | 192 | +} QEMU_PACKED VMDKSESparseConstHeader; |
68 | @@ -XXX,XX +XXX,XX @@ static const BlockJobDriver backup_job_driver = { | 193 | + |
69 | .free = block_job_free, | 194 | +typedef struct VMDKSESparseVolatileHeader { |
70 | .user_resume = block_job_user_resume, | 195 | + uint64_t magic; |
71 | .drain = block_job_drain, | 196 | + uint64_t free_gt_number; |
72 | - .start = backup_run, | 197 | + uint64_t next_txn_seq_number; |
73 | + .run = backup_run, | 198 | + uint64_t replay_journal; |
74 | .commit = backup_commit, | 199 | + uint8_t pad[480]; |
75 | .abort = backup_abort, | 200 | +} QEMU_PACKED VMDKSESparseVolatileHeader; |
76 | .clean = backup_clean, | 201 | + |
77 | diff --git a/block/commit.c b/block/commit.c | 202 | #define L2_CACHE_SIZE 16 |
78 | index XXXXXXX..XXXXXXX 100644 | 203 | |
79 | --- a/block/commit.c | 204 | typedef struct VmdkExtent { |
80 | +++ b/block/commit.c | 205 | @@ -XXX,XX +XXX,XX @@ typedef struct VmdkExtent { |
81 | @@ -XXX,XX +XXX,XX @@ static void commit_complete(Job *job, void *opaque) | 206 | bool compressed; |
82 | bdrv_unref(top); | 207 | bool has_marker; |
83 | } | 208 | bool has_zero_grain; |
84 | 209 | + bool sesparse; | |
85 | -static void coroutine_fn commit_run(void *opaque) | 210 | + uint64_t sesparse_l2_tables_offset; |
86 | +static int coroutine_fn commit_run(Job *job, Error **errp) | 211 | + uint64_t sesparse_clusters_offset; |
87 | { | 212 | + int32_t entry_size; |
88 | - CommitBlockJob *s = opaque; | 213 | int version; |
89 | + CommitBlockJob *s = container_of(job, CommitBlockJob, common.job); | 214 | int64_t sectors; |
90 | CommitCompleteData *data; | 215 | int64_t end_sector; |
91 | int64_t offset; | 216 | int64_t flat_start_offset; |
92 | uint64_t delay_ns = 0; | 217 | int64_t l1_table_offset; |
93 | @@ -XXX,XX +XXX,XX @@ out: | 218 | int64_t l1_backup_table_offset; |
94 | data = g_malloc(sizeof(*data)); | 219 | - uint32_t *l1_table; |
95 | data->ret = ret; | 220 | + void *l1_table; |
96 | job_defer_to_main_loop(&s->common.job, commit_complete, data); | 221 | uint32_t *l1_backup_table; |
97 | + return ret; | 222 | unsigned int l1_size; |
98 | } | 223 | uint32_t l1_entry_sectors; |
99 | 224 | ||
100 | static const BlockJobDriver commit_job_driver = { | 225 | unsigned int l2_size; |
101 | @@ -XXX,XX +XXX,XX @@ static const BlockJobDriver commit_job_driver = { | 226 | - uint32_t *l2_cache; |
102 | .free = block_job_free, | 227 | + void *l2_cache; |
103 | .user_resume = block_job_user_resume, | 228 | uint32_t l2_cache_offsets[L2_CACHE_SIZE]; |
104 | .drain = block_job_drain, | 229 | uint32_t l2_cache_counts[L2_CACHE_SIZE]; |
105 | - .start = commit_run, | 230 | |
106 | + .run = commit_run, | 231 | @@ -XXX,XX +XXX,XX @@ static int vmdk_add_extent(BlockDriverState *bs, |
107 | }, | 232 | * minimal L2 table size: 512 entries |
108 | }; | 233 | * 8 TB is still more than the maximal value supported for |
109 | 234 | * VMDK3 & VMDK4 which is 2TB. | |
110 | diff --git a/block/create.c b/block/create.c | 235 | + * 64TB - for "ESXi seSparse Extent" |
111 | index XXXXXXX..XXXXXXX 100644 | 236 | + * minimal cluster size: 512B (default is 4KB) |
112 | --- a/block/create.c | 237 | + * L2 table size: 4096 entries (const). |
113 | +++ b/block/create.c | 238 | + * 64TB is more than the maximal value supported for |
114 | @@ -XXX,XX +XXX,XX @@ static void blockdev_create_complete(Job *job, void *opaque) | 239 | + * seSparse VMDKs (which is slightly less than 64TB) |
115 | job_completed(job, s->ret, s->err); | 240 | */ |
116 | } | 241 | error_setg(errp, "L1 size too big"); |
117 | 242 | return -EFBIG; | |
118 | -static void coroutine_fn blockdev_create_run(void *opaque) | 243 | @@ -XXX,XX +XXX,XX @@ static int vmdk_add_extent(BlockDriverState *bs, |
119 | +static int coroutine_fn blockdev_create_run(Job *job, Error **errp) | 244 | extent->l2_size = l2_size; |
120 | { | 245 | extent->cluster_sectors = flat ? sectors : cluster_sectors; |
121 | - BlockdevCreateJob *s = opaque; | 246 | extent->next_cluster_sector = ROUND_UP(nb_sectors, cluster_sectors); |
122 | + BlockdevCreateJob *s = container_of(job, BlockdevCreateJob, common); | 247 | + extent->entry_size = sizeof(uint32_t); |
123 | 248 | ||
124 | job_progress_set_remaining(&s->common, 1); | 249 | if (s->num_extents > 1) { |
125 | s->ret = s->drv->bdrv_co_create(s->opts, &s->err); | 250 | extent->end_sector = (*(extent - 1)).end_sector + extent->sectors; |
126 | @@ -XXX,XX +XXX,XX @@ static void coroutine_fn blockdev_create_run(void *opaque) | 251 | @@ -XXX,XX +XXX,XX @@ static int vmdk_init_tables(BlockDriverState *bs, VmdkExtent *extent, |
127 | 252 | int i; | |
128 | qapi_free_BlockdevCreateOptions(s->opts); | 253 | |
129 | job_defer_to_main_loop(&s->common, blockdev_create_complete, NULL); | 254 | /* read the L1 table */ |
130 | + | 255 | - l1_size = extent->l1_size * sizeof(uint32_t); |
131 | + return s->ret; | 256 | + l1_size = extent->l1_size * extent->entry_size; |
132 | } | 257 | extent->l1_table = g_try_malloc(l1_size); |
133 | 258 | if (l1_size && extent->l1_table == NULL) { | |
134 | static const JobDriver blockdev_create_job_driver = { | 259 | return -ENOMEM; |
135 | .instance_size = sizeof(BlockdevCreateJob), | 260 | @@ -XXX,XX +XXX,XX @@ static int vmdk_init_tables(BlockDriverState *bs, VmdkExtent *extent, |
136 | .job_type = JOB_TYPE_CREATE, | 261 | goto fail_l1; |
137 | - .start = blockdev_create_run, | 262 | } |
138 | + .run = blockdev_create_run, | 263 | for (i = 0; i < extent->l1_size; i++) { |
139 | }; | 264 | - le32_to_cpus(&extent->l1_table[i]); |
140 | 265 | + if (extent->entry_size == sizeof(uint64_t)) { | |
141 | void qmp_blockdev_create(const char *job_id, BlockdevCreateOptions *options, | 266 | + le64_to_cpus((uint64_t *)extent->l1_table + i); |
142 | diff --git a/block/mirror.c b/block/mirror.c | 267 | + } else { |
143 | index XXXXXXX..XXXXXXX 100644 | 268 | + assert(extent->entry_size == sizeof(uint32_t)); |
144 | --- a/block/mirror.c | 269 | + le32_to_cpus((uint32_t *)extent->l1_table + i); |
145 | +++ b/block/mirror.c | 270 | + } |
146 | @@ -XXX,XX +XXX,XX @@ static int mirror_flush(MirrorBlockJob *s) | 271 | } |
272 | |||
273 | if (extent->l1_backup_table_offset) { | ||
274 | + assert(!extent->sesparse); | ||
275 | extent->l1_backup_table = g_try_malloc(l1_size); | ||
276 | if (l1_size && extent->l1_backup_table == NULL) { | ||
277 | ret = -ENOMEM; | ||
278 | @@ -XXX,XX +XXX,XX @@ static int vmdk_init_tables(BlockDriverState *bs, VmdkExtent *extent, | ||
279 | } | ||
280 | |||
281 | extent->l2_cache = | ||
282 | - g_new(uint32_t, extent->l2_size * L2_CACHE_SIZE); | ||
283 | + g_malloc(extent->entry_size * extent->l2_size * L2_CACHE_SIZE); | ||
284 | return 0; | ||
285 | fail_l1b: | ||
286 | g_free(extent->l1_backup_table); | ||
287 | @@ -XXX,XX +XXX,XX @@ static int vmdk_open_vmfs_sparse(BlockDriverState *bs, | ||
147 | return ret; | 288 | return ret; |
148 | } | 289 | } |
149 | 290 | ||
150 | -static void coroutine_fn mirror_run(void *opaque) | 291 | +#define SESPARSE_CONST_HEADER_MAGIC UINT64_C(0x00000000cafebabe) |
151 | +static int coroutine_fn mirror_run(Job *job, Error **errp) | 292 | +#define SESPARSE_VOLATILE_HEADER_MAGIC UINT64_C(0x00000000cafecafe) |
293 | + | ||
294 | +/* Strict checks - format not officially documented */ | ||
295 | +static int check_se_sparse_const_header(VMDKSESparseConstHeader *header, | ||
296 | + Error **errp) | ||
297 | +{ | ||
298 | + header->magic = le64_to_cpu(header->magic); | ||
299 | + header->version = le64_to_cpu(header->version); | ||
300 | + header->grain_size = le64_to_cpu(header->grain_size); | ||
301 | + header->grain_table_size = le64_to_cpu(header->grain_table_size); | ||
302 | + header->flags = le64_to_cpu(header->flags); | ||
303 | + header->reserved1 = le64_to_cpu(header->reserved1); | ||
304 | + header->reserved2 = le64_to_cpu(header->reserved2); | ||
305 | + header->reserved3 = le64_to_cpu(header->reserved3); | ||
306 | + header->reserved4 = le64_to_cpu(header->reserved4); | ||
307 | + | ||
308 | + header->volatile_header_offset = | ||
309 | + le64_to_cpu(header->volatile_header_offset); | ||
310 | + header->volatile_header_size = le64_to_cpu(header->volatile_header_size); | ||
311 | + | ||
312 | + header->journal_header_offset = le64_to_cpu(header->journal_header_offset); | ||
313 | + header->journal_header_size = le64_to_cpu(header->journal_header_size); | ||
314 | + | ||
315 | + header->journal_offset = le64_to_cpu(header->journal_offset); | ||
316 | + header->journal_size = le64_to_cpu(header->journal_size); | ||
317 | + | ||
318 | + header->grain_dir_offset = le64_to_cpu(header->grain_dir_offset); | ||
319 | + header->grain_dir_size = le64_to_cpu(header->grain_dir_size); | ||
320 | + | ||
321 | + header->grain_tables_offset = le64_to_cpu(header->grain_tables_offset); | ||
322 | + header->grain_tables_size = le64_to_cpu(header->grain_tables_size); | ||
323 | + | ||
324 | + header->free_bitmap_offset = le64_to_cpu(header->free_bitmap_offset); | ||
325 | + header->free_bitmap_size = le64_to_cpu(header->free_bitmap_size); | ||
326 | + | ||
327 | + header->backmap_offset = le64_to_cpu(header->backmap_offset); | ||
328 | + header->backmap_size = le64_to_cpu(header->backmap_size); | ||
329 | + | ||
330 | + header->grains_offset = le64_to_cpu(header->grains_offset); | ||
331 | + header->grains_size = le64_to_cpu(header->grains_size); | ||
332 | + | ||
333 | + if (header->magic != SESPARSE_CONST_HEADER_MAGIC) { | ||
334 | + error_setg(errp, "Bad const header magic: 0x%016" PRIx64, | ||
335 | + header->magic); | ||
336 | + return -EINVAL; | ||
337 | + } | ||
338 | + | ||
339 | + if (header->version != 0x0000000200000001) { | ||
340 | + error_setg(errp, "Unsupported version: 0x%016" PRIx64, | ||
341 | + header->version); | ||
342 | + return -ENOTSUP; | ||
343 | + } | ||
344 | + | ||
345 | + if (header->grain_size != 8) { | ||
346 | + error_setg(errp, "Unsupported grain size: %" PRIu64, | ||
347 | + header->grain_size); | ||
348 | + return -ENOTSUP; | ||
349 | + } | ||
350 | + | ||
351 | + if (header->grain_table_size != 64) { | ||
352 | + error_setg(errp, "Unsupported grain table size: %" PRIu64, | ||
353 | + header->grain_table_size); | ||
354 | + return -ENOTSUP; | ||
355 | + } | ||
356 | + | ||
357 | + if (header->flags != 0) { | ||
358 | + error_setg(errp, "Unsupported flags: 0x%016" PRIx64, | ||
359 | + header->flags); | ||
360 | + return -ENOTSUP; | ||
361 | + } | ||
362 | + | ||
363 | + if (header->reserved1 != 0 || header->reserved2 != 0 || | ||
364 | + header->reserved3 != 0 || header->reserved4 != 0) { | ||
365 | + error_setg(errp, "Unsupported reserved bits:" | ||
366 | + " 0x%016" PRIx64 " 0x%016" PRIx64 | ||
367 | + " 0x%016" PRIx64 " 0x%016" PRIx64, | ||
368 | + header->reserved1, header->reserved2, | ||
369 | + header->reserved3, header->reserved4); | ||
370 | + return -ENOTSUP; | ||
371 | + } | ||
372 | + | ||
373 | + /* check that padding is 0 */ | ||
374 | + if (!buffer_is_zero(header->pad, sizeof(header->pad))) { | ||
375 | + error_setg(errp, "Unsupported non-zero const header padding"); | ||
376 | + return -ENOTSUP; | ||
377 | + } | ||
378 | + | ||
379 | + return 0; | ||
380 | +} | ||
381 | + | ||
382 | +static int check_se_sparse_volatile_header(VMDKSESparseVolatileHeader *header, | ||
383 | + Error **errp) | ||
384 | +{ | ||
385 | + header->magic = le64_to_cpu(header->magic); | ||
386 | + header->free_gt_number = le64_to_cpu(header->free_gt_number); | ||
387 | + header->next_txn_seq_number = le64_to_cpu(header->next_txn_seq_number); | ||
388 | + header->replay_journal = le64_to_cpu(header->replay_journal); | ||
389 | + | ||
390 | + if (header->magic != SESPARSE_VOLATILE_HEADER_MAGIC) { | ||
391 | + error_setg(errp, "Bad volatile header magic: 0x%016" PRIx64, | ||
392 | + header->magic); | ||
393 | + return -EINVAL; | ||
394 | + } | ||
395 | + | ||
396 | + if (header->replay_journal) { | ||
397 | + error_setg(errp, "Image is dirty, Replaying journal not supported"); | ||
398 | + return -ENOTSUP; | ||
399 | + } | ||
400 | + | ||
401 | + /* check that padding is 0 */ | ||
402 | + if (!buffer_is_zero(header->pad, sizeof(header->pad))) { | ||
403 | + error_setg(errp, "Unsupported non-zero volatile header padding"); | ||
404 | + return -ENOTSUP; | ||
405 | + } | ||
406 | + | ||
407 | + return 0; | ||
408 | +} | ||
409 | + | ||
410 | +static int vmdk_open_se_sparse(BlockDriverState *bs, | ||
411 | + BdrvChild *file, | ||
412 | + int flags, Error **errp) | ||
413 | +{ | ||
414 | + int ret; | ||
415 | + VMDKSESparseConstHeader const_header; | ||
416 | + VMDKSESparseVolatileHeader volatile_header; | ||
417 | + VmdkExtent *extent; | ||
418 | + | ||
419 | + ret = bdrv_apply_auto_read_only(bs, | ||
420 | + "No write support for seSparse images available", errp); | ||
421 | + if (ret < 0) { | ||
422 | + return ret; | ||
423 | + } | ||
424 | + | ||
425 | + assert(sizeof(const_header) == SECTOR_SIZE); | ||
426 | + | ||
427 | + ret = bdrv_pread(file, 0, &const_header, sizeof(const_header)); | ||
428 | + if (ret < 0) { | ||
429 | + bdrv_refresh_filename(file->bs); | ||
430 | + error_setg_errno(errp, -ret, | ||
431 | + "Could not read const header from file '%s'", | ||
432 | + file->bs->filename); | ||
433 | + return ret; | ||
434 | + } | ||
435 | + | ||
436 | + /* check const header */ | ||
437 | + ret = check_se_sparse_const_header(&const_header, errp); | ||
438 | + if (ret < 0) { | ||
439 | + return ret; | ||
440 | + } | ||
441 | + | ||
442 | + assert(sizeof(volatile_header) == SECTOR_SIZE); | ||
443 | + | ||
444 | + ret = bdrv_pread(file, | ||
445 | + const_header.volatile_header_offset * SECTOR_SIZE, | ||
446 | + &volatile_header, sizeof(volatile_header)); | ||
447 | + if (ret < 0) { | ||
448 | + bdrv_refresh_filename(file->bs); | ||
449 | + error_setg_errno(errp, -ret, | ||
450 | + "Could not read volatile header from file '%s'", | ||
451 | + file->bs->filename); | ||
452 | + return ret; | ||
453 | + } | ||
454 | + | ||
455 | + /* check volatile header */ | ||
456 | + ret = check_se_sparse_volatile_header(&volatile_header, errp); | ||
457 | + if (ret < 0) { | ||
458 | + return ret; | ||
459 | + } | ||
460 | + | ||
461 | + ret = vmdk_add_extent(bs, file, false, | ||
462 | + const_header.capacity, | ||
463 | + const_header.grain_dir_offset * SECTOR_SIZE, | ||
464 | + 0, | ||
465 | + const_header.grain_dir_size * | ||
466 | + SECTOR_SIZE / sizeof(uint64_t), | ||
467 | + const_header.grain_table_size * | ||
468 | + SECTOR_SIZE / sizeof(uint64_t), | ||
469 | + const_header.grain_size, | ||
470 | + &extent, | ||
471 | + errp); | ||
472 | + if (ret < 0) { | ||
473 | + return ret; | ||
474 | + } | ||
475 | + | ||
476 | + extent->sesparse = true; | ||
477 | + extent->sesparse_l2_tables_offset = const_header.grain_tables_offset; | ||
478 | + extent->sesparse_clusters_offset = const_header.grains_offset; | ||
479 | + extent->entry_size = sizeof(uint64_t); | ||
480 | + | ||
481 | + ret = vmdk_init_tables(bs, extent, errp); | ||
482 | + if (ret) { | ||
483 | + /* free extent allocated by vmdk_add_extent */ | ||
484 | + vmdk_free_last_extent(bs); | ||
485 | + } | ||
486 | + | ||
487 | + return ret; | ||
488 | +} | ||
489 | + | ||
490 | static int vmdk_open_desc_file(BlockDriverState *bs, int flags, char *buf, | ||
491 | QDict *options, Error **errp); | ||
492 | |||
493 | @@ -XXX,XX +XXX,XX @@ static int vmdk_parse_extents(const char *desc, BlockDriverState *bs, | ||
494 | * RW [size in sectors] SPARSE "file-name.vmdk" | ||
495 | * RW [size in sectors] VMFS "file-name.vmdk" | ||
496 | * RW [size in sectors] VMFSSPARSE "file-name.vmdk" | ||
497 | + * RW [size in sectors] SESPARSE "file-name.vmdk" | ||
498 | */ | ||
499 | flat_offset = -1; | ||
500 | matches = sscanf(p, "%10s %" SCNd64 " %10s \"%511[^\n\r\"]\" %" SCNd64, | ||
501 | @@ -XXX,XX +XXX,XX @@ static int vmdk_parse_extents(const char *desc, BlockDriverState *bs, | ||
502 | |||
503 | if (sectors <= 0 || | ||
504 | (strcmp(type, "FLAT") && strcmp(type, "SPARSE") && | ||
505 | - strcmp(type, "VMFS") && strcmp(type, "VMFSSPARSE")) || | ||
506 | + strcmp(type, "VMFS") && strcmp(type, "VMFSSPARSE") && | ||
507 | + strcmp(type, "SESPARSE")) || | ||
508 | (strcmp(access, "RW"))) { | ||
509 | continue; | ||
510 | } | ||
511 | @@ -XXX,XX +XXX,XX @@ static int vmdk_parse_extents(const char *desc, BlockDriverState *bs, | ||
512 | return ret; | ||
513 | } | ||
514 | extent = &s->extents[s->num_extents - 1]; | ||
515 | + } else if (!strcmp(type, "SESPARSE")) { | ||
516 | + ret = vmdk_open_se_sparse(bs, extent_file, bs->open_flags, errp); | ||
517 | + if (ret) { | ||
518 | + bdrv_unref_child(bs, extent_file); | ||
519 | + return ret; | ||
520 | + } | ||
521 | + extent = &s->extents[s->num_extents - 1]; | ||
522 | } else { | ||
523 | error_setg(errp, "Unsupported extent type '%s'", type); | ||
524 | bdrv_unref_child(bs, extent_file); | ||
525 | @@ -XXX,XX +XXX,XX @@ static int vmdk_open_desc_file(BlockDriverState *bs, int flags, char *buf, | ||
526 | if (strcmp(ct, "monolithicFlat") && | ||
527 | strcmp(ct, "vmfs") && | ||
528 | strcmp(ct, "vmfsSparse") && | ||
529 | + strcmp(ct, "seSparse") && | ||
530 | strcmp(ct, "twoGbMaxExtentSparse") && | ||
531 | strcmp(ct, "twoGbMaxExtentFlat")) { | ||
532 | error_setg(errp, "Unsupported image type '%s'", ct); | ||
533 | @@ -XXX,XX +XXX,XX @@ static int get_cluster_offset(BlockDriverState *bs, | ||
152 | { | 534 | { |
153 | - MirrorBlockJob *s = opaque; | 535 | unsigned int l1_index, l2_offset, l2_index; |
154 | + MirrorBlockJob *s = container_of(job, MirrorBlockJob, common.job); | 536 | int min_index, i, j; |
155 | MirrorExitData *data; | 537 | - uint32_t min_count, *l2_table; |
156 | BlockDriverState *bs = s->mirror_top_bs->backing->bs; | 538 | + uint32_t min_count; |
157 | BlockDriverState *target_bs = blk_bs(s->target); | 539 | + void *l2_table; |
158 | @@ -XXX,XX +XXX,XX @@ immediate_exit: | 540 | bool zeroed = false; |
159 | if (need_drain) { | 541 | int64_t ret; |
160 | bdrv_drained_begin(bs); | 542 | int64_t cluster_sector; |
161 | } | 543 | + unsigned int l2_size_bytes = extent->l2_size * extent->entry_size; |
162 | + | 544 | |
163 | job_defer_to_main_loop(&s->common.job, mirror_exit, data); | 545 | if (m_data) { |
164 | + return ret; | 546 | m_data->valid = 0; |
165 | } | 547 | @@ -XXX,XX +XXX,XX @@ static int get_cluster_offset(BlockDriverState *bs, |
166 | 548 | if (l1_index >= extent->l1_size) { | |
167 | static void mirror_complete(Job *job, Error **errp) | 549 | return VMDK_ERROR; |
168 | @@ -XXX,XX +XXX,XX @@ static const BlockJobDriver mirror_job_driver = { | 550 | } |
169 | .free = block_job_free, | 551 | - l2_offset = extent->l1_table[l1_index]; |
170 | .user_resume = block_job_user_resume, | 552 | + if (extent->sesparse) { |
171 | .drain = block_job_drain, | 553 | + uint64_t l2_offset_u64; |
172 | - .start = mirror_run, | 554 | + |
173 | + .run = mirror_run, | 555 | + assert(extent->entry_size == sizeof(uint64_t)); |
174 | .pause = mirror_pause, | 556 | + |
175 | .complete = mirror_complete, | 557 | + l2_offset_u64 = ((uint64_t *)extent->l1_table)[l1_index]; |
176 | }, | 558 | + if (l2_offset_u64 == 0) { |
177 | @@ -XXX,XX +XXX,XX @@ static const BlockJobDriver commit_active_job_driver = { | 559 | + l2_offset = 0; |
178 | .free = block_job_free, | 560 | + } else if ((l2_offset_u64 & 0xffffffff00000000) != 0x1000000000000000) { |
179 | .user_resume = block_job_user_resume, | 561 | + /* |
180 | .drain = block_job_drain, | 562 | + * Top most nibble is 0x1 if grain table is allocated. |
181 | - .start = mirror_run, | 563 | + * strict check - top most 4 bytes must be 0x10000000 since max |
182 | + .run = mirror_run, | 564 | + * supported size is 64TB for disk - so no more than 64TB / 16MB |
183 | .pause = mirror_pause, | 565 | + * grain directories which is smaller than uint32, |
184 | .complete = mirror_complete, | 566 | + * where 16MB is the only supported default grain table coverage. |
185 | }, | 567 | + */ |
186 | diff --git a/block/stream.c b/block/stream.c | 568 | + return VMDK_ERROR; |
187 | index XXXXXXX..XXXXXXX 100644 | 569 | + } else { |
188 | --- a/block/stream.c | 570 | + l2_offset_u64 = l2_offset_u64 & 0x00000000ffffffff; |
189 | +++ b/block/stream.c | 571 | + l2_offset_u64 = extent->sesparse_l2_tables_offset + |
190 | @@ -XXX,XX +XXX,XX @@ out: | 572 | + l2_offset_u64 * l2_size_bytes / SECTOR_SIZE; |
191 | g_free(data); | 573 | + if (l2_offset_u64 > 0x00000000ffffffff) { |
192 | } | 574 | + return VMDK_ERROR; |
193 | 575 | + } | |
194 | -static void coroutine_fn stream_run(void *opaque) | 576 | + l2_offset = (unsigned int)(l2_offset_u64); |
195 | +static int coroutine_fn stream_run(Job *job, Error **errp) | 577 | + } |
196 | { | 578 | + } else { |
197 | - StreamBlockJob *s = opaque; | 579 | + assert(extent->entry_size == sizeof(uint32_t)); |
198 | + StreamBlockJob *s = container_of(job, StreamBlockJob, common.job); | 580 | + l2_offset = ((uint32_t *)extent->l1_table)[l1_index]; |
199 | StreamCompleteData *data; | 581 | + } |
200 | BlockBackend *blk = s->common.blk; | 582 | if (!l2_offset) { |
201 | BlockDriverState *bs = blk_bs(blk); | 583 | return VMDK_UNALLOC; |
202 | @@ -XXX,XX +XXX,XX @@ out: | 584 | } |
203 | data = g_malloc(sizeof(*data)); | 585 | @@ -XXX,XX +XXX,XX @@ static int get_cluster_offset(BlockDriverState *bs, |
204 | data->ret = ret; | 586 | extent->l2_cache_counts[j] >>= 1; |
205 | job_defer_to_main_loop(&s->common.job, stream_complete, data); | 587 | } |
206 | + return ret; | 588 | } |
207 | } | 589 | - l2_table = extent->l2_cache + (i * extent->l2_size); |
208 | 590 | + l2_table = (char *)extent->l2_cache + (i * l2_size_bytes); | |
209 | static const BlockJobDriver stream_job_driver = { | 591 | goto found; |
210 | @@ -XXX,XX +XXX,XX @@ static const BlockJobDriver stream_job_driver = { | ||
211 | .instance_size = sizeof(StreamBlockJob), | ||
212 | .job_type = JOB_TYPE_STREAM, | ||
213 | .free = block_job_free, | ||
214 | - .start = stream_run, | ||
215 | + .run = stream_run, | ||
216 | .user_resume = block_job_user_resume, | ||
217 | .drain = block_job_drain, | ||
218 | }, | ||
219 | diff --git a/job.c b/job.c | ||
220 | index XXXXXXX..XXXXXXX 100644 | ||
221 | --- a/job.c | ||
222 | +++ b/job.c | ||
223 | @@ -XXX,XX +XXX,XX @@ static void coroutine_fn job_co_entry(void *opaque) | ||
224 | { | ||
225 | Job *job = opaque; | ||
226 | |||
227 | - assert(job && job->driver && job->driver->start); | ||
228 | + assert(job && job->driver && job->driver->run); | ||
229 | job_pause_point(job); | ||
230 | - job->driver->start(job); | ||
231 | + job->ret = job->driver->run(job, NULL); | ||
232 | } | ||
233 | |||
234 | |||
235 | void job_start(Job *job) | ||
236 | { | ||
237 | assert(job && !job_started(job) && job->paused && | ||
238 | - job->driver && job->driver->start); | ||
239 | + job->driver && job->driver->run); | ||
240 | job->co = qemu_coroutine_create(job_co_entry, job); | ||
241 | job->pause_count--; | ||
242 | job->busy = true; | ||
243 | diff --git a/tests/test-bdrv-drain.c b/tests/test-bdrv-drain.c | ||
244 | index XXXXXXX..XXXXXXX 100644 | ||
245 | --- a/tests/test-bdrv-drain.c | ||
246 | +++ b/tests/test-bdrv-drain.c | ||
247 | @@ -XXX,XX +XXX,XX @@ static void test_job_completed(Job *job, void *opaque) | ||
248 | job_completed(job, 0, NULL); | ||
249 | } | ||
250 | |||
251 | -static void coroutine_fn test_job_start(void *opaque) | ||
252 | +static int coroutine_fn test_job_run(Job *job, Error **errp) | ||
253 | { | ||
254 | - TestBlockJob *s = opaque; | ||
255 | + TestBlockJob *s = container_of(job, TestBlockJob, common.job); | ||
256 | |||
257 | job_transition_to_ready(&s->common.job); | ||
258 | while (!s->should_complete) { | ||
259 | @@ -XXX,XX +XXX,XX @@ static void coroutine_fn test_job_start(void *opaque) | ||
260 | } | ||
261 | |||
262 | job_defer_to_main_loop(&s->common.job, test_job_completed, NULL); | ||
263 | + return 0; | ||
264 | } | ||
265 | |||
266 | static void test_job_complete(Job *job, Error **errp) | ||
267 | @@ -XXX,XX +XXX,XX @@ BlockJobDriver test_job_driver = { | ||
268 | .free = block_job_free, | ||
269 | .user_resume = block_job_user_resume, | ||
270 | .drain = block_job_drain, | ||
271 | - .start = test_job_start, | ||
272 | + .run = test_job_run, | ||
273 | .complete = test_job_complete, | ||
274 | }, | ||
275 | }; | ||
276 | diff --git a/tests/test-blockjob-txn.c b/tests/test-blockjob-txn.c | ||
277 | index XXXXXXX..XXXXXXX 100644 | ||
278 | --- a/tests/test-blockjob-txn.c | ||
279 | +++ b/tests/test-blockjob-txn.c | ||
280 | @@ -XXX,XX +XXX,XX @@ static void test_block_job_complete(Job *job, void *opaque) | ||
281 | bdrv_unref(bs); | ||
282 | } | ||
283 | |||
284 | -static void coroutine_fn test_block_job_run(void *opaque) | ||
285 | +static int coroutine_fn test_block_job_run(Job *job, Error **errp) | ||
286 | { | ||
287 | - TestBlockJob *s = opaque; | ||
288 | - BlockJob *job = &s->common; | ||
289 | + TestBlockJob *s = container_of(job, TestBlockJob, common.job); | ||
290 | |||
291 | while (s->iterations--) { | ||
292 | if (s->use_timer) { | ||
293 | - job_sleep_ns(&job->job, 0); | ||
294 | + job_sleep_ns(job, 0); | ||
295 | } else { | ||
296 | - job_yield(&job->job); | ||
297 | + job_yield(job); | ||
298 | } | 592 | } |
299 | 593 | } | |
300 | - if (job_is_cancelled(&job->job)) { | 594 | @@ -XXX,XX +XXX,XX @@ static int get_cluster_offset(BlockDriverState *bs, |
301 | + if (job_is_cancelled(job)) { | 595 | min_index = i; |
302 | break; | ||
303 | } | 596 | } |
304 | } | 597 | } |
305 | 598 | - l2_table = extent->l2_cache + (min_index * extent->l2_size); | |
306 | - job_defer_to_main_loop(&job->job, test_block_job_complete, | 599 | + l2_table = (char *)extent->l2_cache + (min_index * l2_size_bytes); |
307 | + job_defer_to_main_loop(job, test_block_job_complete, | 600 | BLKDBG_EVENT(extent->file, BLKDBG_L2_LOAD); |
308 | (void *)(intptr_t)s->rc); | 601 | if (bdrv_pread(extent->file, |
309 | + return s->rc; | 602 | (int64_t)l2_offset * 512, |
310 | } | 603 | l2_table, |
311 | 604 | - extent->l2_size * sizeof(uint32_t) | |
312 | typedef struct { | 605 | - ) != extent->l2_size * sizeof(uint32_t)) { |
313 | @@ -XXX,XX +XXX,XX @@ static const BlockJobDriver test_block_job_driver = { | 606 | + l2_size_bytes |
314 | .free = block_job_free, | 607 | + ) != l2_size_bytes) { |
315 | .user_resume = block_job_user_resume, | 608 | return VMDK_ERROR; |
316 | .drain = block_job_drain, | 609 | } |
317 | - .start = test_block_job_run, | 610 | |
318 | + .run = test_block_job_run, | 611 | @@ -XXX,XX +XXX,XX @@ static int get_cluster_offset(BlockDriverState *bs, |
319 | }, | 612 | extent->l2_cache_counts[min_index] = 1; |
320 | }; | 613 | found: |
321 | 614 | l2_index = ((offset >> 9) / extent->cluster_sectors) % extent->l2_size; | |
322 | diff --git a/tests/test-blockjob.c b/tests/test-blockjob.c | 615 | - cluster_sector = le32_to_cpu(l2_table[l2_index]); |
323 | index XXXXXXX..XXXXXXX 100644 | 616 | |
324 | --- a/tests/test-blockjob.c | 617 | - if (extent->has_zero_grain && cluster_sector == VMDK_GTE_ZEROED) { |
325 | +++ b/tests/test-blockjob.c | 618 | - zeroed = true; |
326 | @@ -XXX,XX +XXX,XX @@ static void cancel_job_complete(Job *job, Error **errp) | 619 | + if (extent->sesparse) { |
327 | s->should_complete = true; | 620 | + cluster_sector = le64_to_cpu(((uint64_t *)l2_table)[l2_index]); |
328 | } | 621 | + switch (cluster_sector & 0xf000000000000000) { |
329 | 622 | + case 0x0000000000000000: | |
330 | -static void coroutine_fn cancel_job_start(void *opaque) | 623 | + /* unallocated grain */ |
331 | +static int coroutine_fn cancel_job_run(Job *job, Error **errp) | 624 | + if (cluster_sector != 0) { |
332 | { | 625 | + return VMDK_ERROR; |
333 | - CancelJob *s = opaque; | 626 | + } |
334 | + CancelJob *s = container_of(job, CancelJob, common.job); | 627 | + break; |
335 | 628 | + case 0x1000000000000000: | |
336 | while (!s->should_complete) { | 629 | + /* scsi-unmapped grain - fallthrough */ |
337 | if (job_is_cancelled(&s->common.job)) { | 630 | + case 0x2000000000000000: |
338 | @@ -XXX,XX +XXX,XX @@ static void coroutine_fn cancel_job_start(void *opaque) | 631 | + /* zero grain */ |
339 | 632 | + zeroed = true; | |
340 | defer: | 633 | + break; |
341 | job_defer_to_main_loop(&s->common.job, cancel_job_completed, s); | 634 | + case 0x3000000000000000: |
342 | + return 0; | 635 | + /* allocated grain */ |
343 | } | 636 | + cluster_sector = (((cluster_sector & 0x0fff000000000000) >> 48) | |
344 | 637 | + ((cluster_sector & 0x0000ffffffffffff) << 12)); | |
345 | static const BlockJobDriver test_cancel_driver = { | 638 | + cluster_sector = extent->sesparse_clusters_offset + |
346 | @@ -XXX,XX +XXX,XX @@ static const BlockJobDriver test_cancel_driver = { | 639 | + cluster_sector * extent->cluster_sectors; |
347 | .free = block_job_free, | 640 | + break; |
348 | .user_resume = block_job_user_resume, | 641 | + default: |
349 | .drain = block_job_drain, | 642 | + return VMDK_ERROR; |
350 | - .start = cancel_job_start, | 643 | + } |
351 | + .run = cancel_job_run, | 644 | + } else { |
352 | .complete = cancel_job_complete, | 645 | + cluster_sector = le32_to_cpu(((uint32_t *)l2_table)[l2_index]); |
353 | }, | 646 | + |
354 | }; | 647 | + if (extent->has_zero_grain && cluster_sector == VMDK_GTE_ZEROED) { |
648 | + zeroed = true; | ||
649 | + } | ||
650 | } | ||
651 | |||
652 | if (!cluster_sector || zeroed) { | ||
653 | if (!allocate) { | ||
654 | return zeroed ? VMDK_ZEROED : VMDK_UNALLOC; | ||
655 | } | ||
656 | + assert(!extent->sesparse); | ||
657 | |||
658 | if (extent->next_cluster_sector >= VMDK_EXTENT_MAX_SECTORS) { | ||
659 | return VMDK_ERROR; | ||
660 | @@ -XXX,XX +XXX,XX @@ static int get_cluster_offset(BlockDriverState *bs, | ||
661 | m_data->l1_index = l1_index; | ||
662 | m_data->l2_index = l2_index; | ||
663 | m_data->l2_offset = l2_offset; | ||
664 | - m_data->l2_cache_entry = &l2_table[l2_index]; | ||
665 | + m_data->l2_cache_entry = ((uint32_t *)l2_table) + l2_index; | ||
666 | } | ||
667 | } | ||
668 | *cluster_offset = cluster_sector << BDRV_SECTOR_BITS; | ||
669 | @@ -XXX,XX +XXX,XX @@ static int vmdk_pwritev(BlockDriverState *bs, uint64_t offset, | ||
670 | if (!extent) { | ||
671 | return -EIO; | ||
672 | } | ||
673 | + if (extent->sesparse) { | ||
674 | + return -ENOTSUP; | ||
675 | + } | ||
676 | offset_in_cluster = vmdk_find_offset_in_cluster(extent, offset); | ||
677 | n_bytes = MIN(bytes, extent->cluster_sectors * BDRV_SECTOR_SIZE | ||
678 | - offset_in_cluster); | ||
355 | -- | 679 | -- |
356 | 2.17.1 | 680 | 2.21.0 |
357 | 681 | ||
358 | 682 | diff view generated by jsdifflib |
Deleted patch | |||
---|---|---|---|
1 | From: John Snow <jsnow@redhat.com> | ||
2 | 1 | ||
3 | Jobs presently use both an Error object in the case of the create job, | ||
4 | and char strings in the case of generic errors elsewhere. | ||
5 | |||
6 | Unify the two paths as just j->err, and remove the extra argument from | ||
7 | job_completed. The integer error code for job_completed is kept for now, | ||
8 | to be removed shortly in a separate patch. | ||
9 | |||
10 | Signed-off-by: John Snow <jsnow@redhat.com> | ||
11 | Message-id: 20180830015734.19765-3-jsnow@redhat.com | ||
12 | [mreitz: Dropped a superfluous g_strdup()] | ||
13 | Reviewed-by: Eric Blake <eblake@redhat.com> | ||
14 | Signed-off-by: Max Reitz <mreitz@redhat.com> | ||
15 | --- | ||
16 | include/qemu/job.h | 14 ++++++++------ | ||
17 | block/backup.c | 2 +- | ||
18 | block/commit.c | 2 +- | ||
19 | block/create.c | 5 ++--- | ||
20 | block/mirror.c | 2 +- | ||
21 | block/stream.c | 2 +- | ||
22 | job-qmp.c | 5 +++-- | ||
23 | job.c | 18 ++++++------------ | ||
24 | tests/test-bdrv-drain.c | 2 +- | ||
25 | tests/test-blockjob-txn.c | 2 +- | ||
26 | tests/test-blockjob.c | 2 +- | ||
27 | 11 files changed, 26 insertions(+), 30 deletions(-) | ||
28 | |||
29 | diff --git a/include/qemu/job.h b/include/qemu/job.h | ||
30 | index XXXXXXX..XXXXXXX 100644 | ||
31 | --- a/include/qemu/job.h | ||
32 | +++ b/include/qemu/job.h | ||
33 | @@ -XXX,XX +XXX,XX @@ typedef struct Job { | ||
34 | /** Estimated progress_current value at the completion of the job */ | ||
35 | int64_t progress_total; | ||
36 | |||
37 | - /** Error string for a failed job (NULL if, and only if, job->ret == 0) */ | ||
38 | - char *error; | ||
39 | - | ||
40 | /** ret code passed to job_completed. */ | ||
41 | int ret; | ||
42 | |||
43 | + /** | ||
44 | + * Error object for a failed job. | ||
45 | + * If job->ret is nonzero and an error object was not set, it will be set | ||
46 | + * to strerror(-job->ret) during job_completed. | ||
47 | + */ | ||
48 | + Error *err; | ||
49 | + | ||
50 | /** The completion function that will be called when the job completes. */ | ||
51 | BlockCompletionFunc *cb; | ||
52 | |||
53 | @@ -XXX,XX +XXX,XX @@ void job_transition_to_ready(Job *job); | ||
54 | /** | ||
55 | * @job: The job being completed. | ||
56 | * @ret: The status code. | ||
57 | - * @error: The error message for a failing job (only with @ret < 0). If @ret is | ||
58 | - * negative, but NULL is given for @error, strerror() is used. | ||
59 | * | ||
60 | * Marks @job as completed. If @ret is non-zero, the job transaction it is part | ||
61 | * of is aborted. If @ret is zero, the job moves into the WAITING state. If it | ||
62 | * is the last job to complete in its transaction, all jobs in the transaction | ||
63 | * move from WAITING to PENDING. | ||
64 | */ | ||
65 | -void job_completed(Job *job, int ret, Error *error); | ||
66 | +void job_completed(Job *job, int ret); | ||
67 | |||
68 | /** Asynchronously complete the specified @job. */ | ||
69 | void job_complete(Job *job, Error **errp); | ||
70 | diff --git a/block/backup.c b/block/backup.c | ||
71 | index XXXXXXX..XXXXXXX 100644 | ||
72 | --- a/block/backup.c | ||
73 | +++ b/block/backup.c | ||
74 | @@ -XXX,XX +XXX,XX @@ static void backup_complete(Job *job, void *opaque) | ||
75 | { | ||
76 | BackupCompleteData *data = opaque; | ||
77 | |||
78 | - job_completed(job, data->ret, NULL); | ||
79 | + job_completed(job, data->ret); | ||
80 | g_free(data); | ||
81 | } | ||
82 | |||
83 | diff --git a/block/commit.c b/block/commit.c | ||
84 | index XXXXXXX..XXXXXXX 100644 | ||
85 | --- a/block/commit.c | ||
86 | +++ b/block/commit.c | ||
87 | @@ -XXX,XX +XXX,XX @@ static void commit_complete(Job *job, void *opaque) | ||
88 | * bdrv_set_backing_hd() to fail. */ | ||
89 | block_job_remove_all_bdrv(bjob); | ||
90 | |||
91 | - job_completed(job, ret, NULL); | ||
92 | + job_completed(job, ret); | ||
93 | g_free(data); | ||
94 | |||
95 | /* If bdrv_drop_intermediate() didn't already do that, remove the commit | ||
96 | diff --git a/block/create.c b/block/create.c | ||
97 | index XXXXXXX..XXXXXXX 100644 | ||
98 | --- a/block/create.c | ||
99 | +++ b/block/create.c | ||
100 | @@ -XXX,XX +XXX,XX @@ typedef struct BlockdevCreateJob { | ||
101 | BlockDriver *drv; | ||
102 | BlockdevCreateOptions *opts; | ||
103 | int ret; | ||
104 | - Error *err; | ||
105 | } BlockdevCreateJob; | ||
106 | |||
107 | static void blockdev_create_complete(Job *job, void *opaque) | ||
108 | { | ||
109 | BlockdevCreateJob *s = container_of(job, BlockdevCreateJob, common); | ||
110 | |||
111 | - job_completed(job, s->ret, s->err); | ||
112 | + job_completed(job, s->ret); | ||
113 | } | ||
114 | |||
115 | static int coroutine_fn blockdev_create_run(Job *job, Error **errp) | ||
116 | @@ -XXX,XX +XXX,XX @@ static int coroutine_fn blockdev_create_run(Job *job, Error **errp) | ||
117 | BlockdevCreateJob *s = container_of(job, BlockdevCreateJob, common); | ||
118 | |||
119 | job_progress_set_remaining(&s->common, 1); | ||
120 | - s->ret = s->drv->bdrv_co_create(s->opts, &s->err); | ||
121 | + s->ret = s->drv->bdrv_co_create(s->opts, errp); | ||
122 | job_progress_update(&s->common, 1); | ||
123 | |||
124 | qapi_free_BlockdevCreateOptions(s->opts); | ||
125 | diff --git a/block/mirror.c b/block/mirror.c | ||
126 | index XXXXXXX..XXXXXXX 100644 | ||
127 | --- a/block/mirror.c | ||
128 | +++ b/block/mirror.c | ||
129 | @@ -XXX,XX +XXX,XX @@ static void mirror_exit(Job *job, void *opaque) | ||
130 | blk_insert_bs(bjob->blk, mirror_top_bs, &error_abort); | ||
131 | |||
132 | bs_opaque->job = NULL; | ||
133 | - job_completed(job, data->ret, NULL); | ||
134 | + job_completed(job, data->ret); | ||
135 | |||
136 | g_free(data); | ||
137 | bdrv_drained_end(src); | ||
138 | diff --git a/block/stream.c b/block/stream.c | ||
139 | index XXXXXXX..XXXXXXX 100644 | ||
140 | --- a/block/stream.c | ||
141 | +++ b/block/stream.c | ||
142 | @@ -XXX,XX +XXX,XX @@ out: | ||
143 | } | ||
144 | |||
145 | g_free(s->backing_file_str); | ||
146 | - job_completed(job, data->ret, NULL); | ||
147 | + job_completed(job, data->ret); | ||
148 | g_free(data); | ||
149 | } | ||
150 | |||
151 | diff --git a/job-qmp.c b/job-qmp.c | ||
152 | index XXXXXXX..XXXXXXX 100644 | ||
153 | --- a/job-qmp.c | ||
154 | +++ b/job-qmp.c | ||
155 | @@ -XXX,XX +XXX,XX @@ static JobInfo *job_query_single(Job *job, Error **errp) | ||
156 | .status = job->status, | ||
157 | .current_progress = job->progress_current, | ||
158 | .total_progress = job->progress_total, | ||
159 | - .has_error = !!job->error, | ||
160 | - .error = g_strdup(job->error), | ||
161 | + .has_error = !!job->err, | ||
162 | + .error = job->err ? \ | ||
163 | + g_strdup(error_get_pretty(job->err)) : NULL, | ||
164 | }; | ||
165 | |||
166 | return info; | ||
167 | diff --git a/job.c b/job.c | ||
168 | index XXXXXXX..XXXXXXX 100644 | ||
169 | --- a/job.c | ||
170 | +++ b/job.c | ||
171 | @@ -XXX,XX +XXX,XX @@ void job_unref(Job *job) | ||
172 | |||
173 | QLIST_REMOVE(job, job_list); | ||
174 | |||
175 | - g_free(job->error); | ||
176 | + error_free(job->err); | ||
177 | g_free(job->id); | ||
178 | g_free(job); | ||
179 | } | ||
180 | @@ -XXX,XX +XXX,XX @@ static void coroutine_fn job_co_entry(void *opaque) | ||
181 | |||
182 | assert(job && job->driver && job->driver->run); | ||
183 | job_pause_point(job); | ||
184 | - job->ret = job->driver->run(job, NULL); | ||
185 | + job->ret = job->driver->run(job, &job->err); | ||
186 | } | ||
187 | |||
188 | |||
189 | @@ -XXX,XX +XXX,XX @@ static void job_update_rc(Job *job) | ||
190 | job->ret = -ECANCELED; | ||
191 | } | ||
192 | if (job->ret) { | ||
193 | - if (!job->error) { | ||
194 | - job->error = g_strdup(strerror(-job->ret)); | ||
195 | + if (!job->err) { | ||
196 | + error_setg(&job->err, "%s", strerror(-job->ret)); | ||
197 | } | ||
198 | job_state_transition(job, JOB_STATUS_ABORTING); | ||
199 | } | ||
200 | @@ -XXX,XX +XXX,XX @@ static void job_completed_txn_success(Job *job) | ||
201 | } | ||
202 | } | ||
203 | |||
204 | -void job_completed(Job *job, int ret, Error *error) | ||
205 | +void job_completed(Job *job, int ret) | ||
206 | { | ||
207 | assert(job && job->txn && !job_is_completed(job)); | ||
208 | |||
209 | job->ret = ret; | ||
210 | - if (error) { | ||
211 | - assert(job->ret < 0); | ||
212 | - job->error = g_strdup(error_get_pretty(error)); | ||
213 | - error_free(error); | ||
214 | - } | ||
215 | - | ||
216 | job_update_rc(job); | ||
217 | trace_job_completed(job, ret, job->ret); | ||
218 | if (job->ret) { | ||
219 | @@ -XXX,XX +XXX,XX @@ void job_cancel(Job *job, bool force) | ||
220 | } | ||
221 | job_cancel_async(job, force); | ||
222 | if (!job_started(job)) { | ||
223 | - job_completed(job, -ECANCELED, NULL); | ||
224 | + job_completed(job, -ECANCELED); | ||
225 | } else if (job->deferred_to_main_loop) { | ||
226 | job_completed_txn_abort(job); | ||
227 | } else { | ||
228 | diff --git a/tests/test-bdrv-drain.c b/tests/test-bdrv-drain.c | ||
229 | index XXXXXXX..XXXXXXX 100644 | ||
230 | --- a/tests/test-bdrv-drain.c | ||
231 | +++ b/tests/test-bdrv-drain.c | ||
232 | @@ -XXX,XX +XXX,XX @@ typedef struct TestBlockJob { | ||
233 | |||
234 | static void test_job_completed(Job *job, void *opaque) | ||
235 | { | ||
236 | - job_completed(job, 0, NULL); | ||
237 | + job_completed(job, 0); | ||
238 | } | ||
239 | |||
240 | static int coroutine_fn test_job_run(Job *job, Error **errp) | ||
241 | diff --git a/tests/test-blockjob-txn.c b/tests/test-blockjob-txn.c | ||
242 | index XXXXXXX..XXXXXXX 100644 | ||
243 | --- a/tests/test-blockjob-txn.c | ||
244 | +++ b/tests/test-blockjob-txn.c | ||
245 | @@ -XXX,XX +XXX,XX @@ static void test_block_job_complete(Job *job, void *opaque) | ||
246 | rc = -ECANCELED; | ||
247 | } | ||
248 | |||
249 | - job_completed(job, rc, NULL); | ||
250 | + job_completed(job, rc); | ||
251 | bdrv_unref(bs); | ||
252 | } | ||
253 | |||
254 | diff --git a/tests/test-blockjob.c b/tests/test-blockjob.c | ||
255 | index XXXXXXX..XXXXXXX 100644 | ||
256 | --- a/tests/test-blockjob.c | ||
257 | +++ b/tests/test-blockjob.c | ||
258 | @@ -XXX,XX +XXX,XX @@ static void cancel_job_completed(Job *job, void *opaque) | ||
259 | { | ||
260 | CancelJob *s = opaque; | ||
261 | s->completed = true; | ||
262 | - job_completed(job, 0, NULL); | ||
263 | + job_completed(job, 0); | ||
264 | } | ||
265 | |||
266 | static void cancel_job_complete(Job *job, Error **errp) | ||
267 | -- | ||
268 | 2.17.1 | ||
269 | |||
270 | diff view generated by jsdifflib |
1 | From: John Snow <jsnow@redhat.com> | 1 | From: Pino Toscano <ptoscano@redhat.com> |
---|---|---|---|
2 | 2 | ||
3 | Utilize the job_exit shim by not calling job_defer_to_main_loop, and | 3 | Rewrite the implementation of the ssh block driver to use libssh instead |
4 | where applicable, converting the deferred callback into the job_exit | 4 | of libssh2. The libssh library has various advantages over libssh2: |
5 | callback. | 5 | - easier API for authentication (for example for using ssh-agent) |
6 | - easier API for known_hosts handling | ||
7 | - supports newer types of keys in known_hosts | ||
6 | 8 | ||
7 | This converts backup, stream, create, and the unit tests all at once. | 9 | Use APIs/features available in libssh 0.8 conditionally, to support |
8 | Most of these jobs do not see any changes to the order in which they | 10 | older versions (which are not recommended though). |
9 | clean up their resources, except the test-blockjob-txn test, which | ||
10 | now puts down its bs before job_completed is called. | ||
11 | 11 | ||
12 | This is safe for the same reason the reordering in the mirror job is | 12 | Adjust the iotest 207 according to the different error message, and to |
13 | safe, because job_completed no longer runs under two locks, making | 13 | find the default key type for localhost (to properly compare the |
14 | the unref safe even if it causes a flush. | 14 | fingerprint with). |
15 | Contributed-by: Max Reitz <mreitz@redhat.com> | ||
15 | 16 | ||
16 | Signed-off-by: John Snow <jsnow@redhat.com> | 17 | Adjust the various Docker/Travis scripts to use libssh when available |
17 | Reviewed-by: Max Reitz <mreitz@redhat.com> | 18 | instead of libssh2. The mingw/mxe testing is dropped for now, as there |
18 | Message-id: 20180830015734.19765-7-jsnow@redhat.com | 19 | are no packages for it. |
20 | |||
21 | Signed-off-by: Pino Toscano <ptoscano@redhat.com> | ||
22 | Tested-by: Philippe Mathieu-Daudé <philmd@redhat.com> | ||
23 | Acked-by: Alex Bennée <alex.bennee@linaro.org> | ||
24 | Message-id: 20190620200840.17655-1-ptoscano@redhat.com | ||
25 | Reviewed-by: Philippe Mathieu-Daudé <philmd@redhat.com> | ||
26 | Message-id: 5873173.t2JhDm7DL7@lindworm.usersys.redhat.com | ||
19 | Signed-off-by: Max Reitz <mreitz@redhat.com> | 27 | Signed-off-by: Max Reitz <mreitz@redhat.com> |
20 | --- | 28 | --- |
21 | block/backup.c | 16 ---------------- | 29 | configure | 65 +- |
22 | block/create.c | 14 +++----------- | 30 | block/Makefile.objs | 6 +- |
23 | block/stream.c | 22 +++++++--------------- | 31 | block/ssh.c | 652 ++++++++++-------- |
24 | tests/test-bdrv-drain.c | 6 ------ | 32 | .travis.yml | 4 +- |
25 | tests/test-blockjob-txn.c | 11 ++--------- | 33 | block/trace-events | 14 +- |
26 | tests/test-blockjob.c | 10 ++++------ | 34 | docs/qemu-block-drivers.texi | 2 +- |
27 | 6 files changed, 16 insertions(+), 63 deletions(-) | 35 | .../dockerfiles/debian-win32-cross.docker | 1 - |
36 | .../dockerfiles/debian-win64-cross.docker | 1 - | ||
37 | tests/docker/dockerfiles/fedora.docker | 4 +- | ||
38 | tests/docker/dockerfiles/ubuntu.docker | 2 +- | ||
39 | tests/docker/dockerfiles/ubuntu1804.docker | 2 +- | ||
40 | tests/qemu-iotests/207 | 54 +- | ||
41 | tests/qemu-iotests/207.out | 2 +- | ||
42 | 13 files changed, 449 insertions(+), 360 deletions(-) | ||
28 | 43 | ||
29 | diff --git a/block/backup.c b/block/backup.c | 44 | diff --git a/configure b/configure |
45 | index XXXXXXX..XXXXXXX 100755 | ||
46 | --- a/configure | ||
47 | +++ b/configure | ||
48 | @@ -XXX,XX +XXX,XX @@ auth_pam="" | ||
49 | vte="" | ||
50 | virglrenderer="" | ||
51 | tpm="" | ||
52 | -libssh2="" | ||
53 | +libssh="" | ||
54 | live_block_migration="yes" | ||
55 | numa="" | ||
56 | tcmalloc="no" | ||
57 | @@ -XXX,XX +XXX,XX @@ for opt do | ||
58 | ;; | ||
59 | --enable-tpm) tpm="yes" | ||
60 | ;; | ||
61 | - --disable-libssh2) libssh2="no" | ||
62 | + --disable-libssh) libssh="no" | ||
63 | ;; | ||
64 | - --enable-libssh2) libssh2="yes" | ||
65 | + --enable-libssh) libssh="yes" | ||
66 | ;; | ||
67 | --disable-live-block-migration) live_block_migration="no" | ||
68 | ;; | ||
69 | @@ -XXX,XX +XXX,XX @@ disabled with --disable-FEATURE, default is enabled if available: | ||
70 | coroutine-pool coroutine freelist (better performance) | ||
71 | glusterfs GlusterFS backend | ||
72 | tpm TPM support | ||
73 | - libssh2 ssh block device support | ||
74 | + libssh ssh block device support | ||
75 | numa libnuma support | ||
76 | libxml2 for Parallels image format | ||
77 | tcmalloc tcmalloc support | ||
78 | @@ -XXX,XX +XXX,XX @@ EOF | ||
79 | fi | ||
80 | |||
81 | ########################################## | ||
82 | -# libssh2 probe | ||
83 | -min_libssh2_version=1.2.8 | ||
84 | -if test "$libssh2" != "no" ; then | ||
85 | - if $pkg_config --atleast-version=$min_libssh2_version libssh2; then | ||
86 | - libssh2_cflags=$($pkg_config libssh2 --cflags) | ||
87 | - libssh2_libs=$($pkg_config libssh2 --libs) | ||
88 | - libssh2=yes | ||
89 | +# libssh probe | ||
90 | +if test "$libssh" != "no" ; then | ||
91 | + if $pkg_config --exists libssh; then | ||
92 | + libssh_cflags=$($pkg_config libssh --cflags) | ||
93 | + libssh_libs=$($pkg_config libssh --libs) | ||
94 | + libssh=yes | ||
95 | else | ||
96 | - if test "$libssh2" = "yes" ; then | ||
97 | - error_exit "libssh2 >= $min_libssh2_version required for --enable-libssh2" | ||
98 | + if test "$libssh" = "yes" ; then | ||
99 | + error_exit "libssh required for --enable-libssh" | ||
100 | fi | ||
101 | - libssh2=no | ||
102 | + libssh=no | ||
103 | fi | ||
104 | fi | ||
105 | |||
106 | ########################################## | ||
107 | -# libssh2_sftp_fsync probe | ||
108 | +# Check for libssh 0.8 | ||
109 | +# This is done like this instead of using the LIBSSH_VERSION_* and | ||
110 | +# SSH_VERSION_* macros because some distributions in the past shipped | ||
111 | +# snapshots of the future 0.8 from Git, and those snapshots did not | ||
112 | +# have updated version numbers (still referring to 0.7.0). | ||
113 | |||
114 | -if test "$libssh2" = "yes"; then | ||
115 | +if test "$libssh" = "yes"; then | ||
116 | cat > $TMPC <<EOF | ||
117 | -#include <stdio.h> | ||
118 | -#include <libssh2.h> | ||
119 | -#include <libssh2_sftp.h> | ||
120 | -int main(void) { | ||
121 | - LIBSSH2_SESSION *session; | ||
122 | - LIBSSH2_SFTP *sftp; | ||
123 | - LIBSSH2_SFTP_HANDLE *sftp_handle; | ||
124 | - session = libssh2_session_init (); | ||
125 | - sftp = libssh2_sftp_init (session); | ||
126 | - sftp_handle = libssh2_sftp_open (sftp, "/", 0, 0); | ||
127 | - libssh2_sftp_fsync (sftp_handle); | ||
128 | - return 0; | ||
129 | -} | ||
130 | +#include <libssh/libssh.h> | ||
131 | +int main(void) { return ssh_get_server_publickey(NULL, NULL); } | ||
132 | EOF | ||
133 | - # libssh2_cflags/libssh2_libs defined in previous test. | ||
134 | - if compile_prog "$libssh2_cflags" "$libssh2_libs" ; then | ||
135 | - QEMU_CFLAGS="-DHAS_LIBSSH2_SFTP_FSYNC $QEMU_CFLAGS" | ||
136 | + if compile_prog "$libssh_cflags" "$libssh_libs"; then | ||
137 | + libssh_cflags="-DHAVE_LIBSSH_0_8 $libssh_cflags" | ||
138 | fi | ||
139 | fi | ||
140 | |||
141 | @@ -XXX,XX +XXX,XX @@ echo "GlusterFS support $glusterfs" | ||
142 | echo "gcov $gcov_tool" | ||
143 | echo "gcov enabled $gcov" | ||
144 | echo "TPM support $tpm" | ||
145 | -echo "libssh2 support $libssh2" | ||
146 | +echo "libssh support $libssh" | ||
147 | echo "QOM debugging $qom_cast_debug" | ||
148 | echo "Live block migration $live_block_migration" | ||
149 | echo "lzo support $lzo" | ||
150 | @@ -XXX,XX +XXX,XX @@ if test "$glusterfs_iocb_has_stat" = "yes" ; then | ||
151 | echo "CONFIG_GLUSTERFS_IOCB_HAS_STAT=y" >> $config_host_mak | ||
152 | fi | ||
153 | |||
154 | -if test "$libssh2" = "yes" ; then | ||
155 | - echo "CONFIG_LIBSSH2=m" >> $config_host_mak | ||
156 | - echo "LIBSSH2_CFLAGS=$libssh2_cflags" >> $config_host_mak | ||
157 | - echo "LIBSSH2_LIBS=$libssh2_libs" >> $config_host_mak | ||
158 | +if test "$libssh" = "yes" ; then | ||
159 | + echo "CONFIG_LIBSSH=m" >> $config_host_mak | ||
160 | + echo "LIBSSH_CFLAGS=$libssh_cflags" >> $config_host_mak | ||
161 | + echo "LIBSSH_LIBS=$libssh_libs" >> $config_host_mak | ||
162 | fi | ||
163 | |||
164 | if test "$live_block_migration" = "yes" ; then | ||
165 | diff --git a/block/Makefile.objs b/block/Makefile.objs | ||
30 | index XXXXXXX..XXXXXXX 100644 | 166 | index XXXXXXX..XXXXXXX 100644 |
31 | --- a/block/backup.c | 167 | --- a/block/Makefile.objs |
32 | +++ b/block/backup.c | 168 | +++ b/block/Makefile.objs |
33 | @@ -XXX,XX +XXX,XX @@ static BlockErrorAction backup_error_action(BackupBlockJob *job, | 169 | @@ -XXX,XX +XXX,XX @@ block-obj-$(CONFIG_CURL) += curl.o |
34 | } | 170 | block-obj-$(CONFIG_RBD) += rbd.o |
171 | block-obj-$(CONFIG_GLUSTERFS) += gluster.o | ||
172 | block-obj-$(CONFIG_VXHS) += vxhs.o | ||
173 | -block-obj-$(CONFIG_LIBSSH2) += ssh.o | ||
174 | +block-obj-$(CONFIG_LIBSSH) += ssh.o | ||
175 | block-obj-y += accounting.o dirty-bitmap.o | ||
176 | block-obj-y += write-threshold.o | ||
177 | block-obj-y += backup.o | ||
178 | @@ -XXX,XX +XXX,XX @@ rbd.o-libs := $(RBD_LIBS) | ||
179 | gluster.o-cflags := $(GLUSTERFS_CFLAGS) | ||
180 | gluster.o-libs := $(GLUSTERFS_LIBS) | ||
181 | vxhs.o-libs := $(VXHS_LIBS) | ||
182 | -ssh.o-cflags := $(LIBSSH2_CFLAGS) | ||
183 | -ssh.o-libs := $(LIBSSH2_LIBS) | ||
184 | +ssh.o-cflags := $(LIBSSH_CFLAGS) | ||
185 | +ssh.o-libs := $(LIBSSH_LIBS) | ||
186 | block-obj-dmg-bz2-$(CONFIG_BZIP2) += dmg-bz2.o | ||
187 | block-obj-$(if $(CONFIG_DMG),m,n) += $(block-obj-dmg-bz2-y) | ||
188 | dmg-bz2.o-libs := $(BZIP2_LIBS) | ||
189 | diff --git a/block/ssh.c b/block/ssh.c | ||
190 | index XXXXXXX..XXXXXXX 100644 | ||
191 | --- a/block/ssh.c | ||
192 | +++ b/block/ssh.c | ||
193 | @@ -XXX,XX +XXX,XX @@ | ||
194 | |||
195 | #include "qemu/osdep.h" | ||
196 | |||
197 | -#include <libssh2.h> | ||
198 | -#include <libssh2_sftp.h> | ||
199 | +#include <libssh/libssh.h> | ||
200 | +#include <libssh/sftp.h> | ||
201 | |||
202 | #include "block/block_int.h" | ||
203 | #include "block/qdict.h" | ||
204 | @@ -XXX,XX +XXX,XX @@ | ||
205 | #include "trace.h" | ||
206 | |||
207 | /* | ||
208 | - * TRACE_LIBSSH2=<bitmask> enables tracing in libssh2 itself. Note | ||
209 | - * that this requires that libssh2 was specially compiled with the | ||
210 | - * `./configure --enable-debug' option, so most likely you will have | ||
211 | - * to compile it yourself. The meaning of <bitmask> is described | ||
212 | - * here: http://www.libssh2.org/libssh2_trace.html | ||
213 | + * TRACE_LIBSSH=<level> enables tracing in libssh itself. | ||
214 | + * The meaning of <level> is described here: | ||
215 | + * http://api.libssh.org/master/group__libssh__log.html | ||
216 | */ | ||
217 | -#define TRACE_LIBSSH2 0 /* or try: LIBSSH2_TRACE_SFTP */ | ||
218 | +#define TRACE_LIBSSH 0 /* see: SSH_LOG_* */ | ||
219 | |||
220 | typedef struct BDRVSSHState { | ||
221 | /* Coroutine. */ | ||
222 | @@ -XXX,XX +XXX,XX @@ typedef struct BDRVSSHState { | ||
223 | |||
224 | /* SSH connection. */ | ||
225 | int sock; /* socket */ | ||
226 | - LIBSSH2_SESSION *session; /* ssh session */ | ||
227 | - LIBSSH2_SFTP *sftp; /* sftp session */ | ||
228 | - LIBSSH2_SFTP_HANDLE *sftp_handle; /* sftp remote file handle */ | ||
229 | + ssh_session session; /* ssh session */ | ||
230 | + sftp_session sftp; /* sftp session */ | ||
231 | + sftp_file sftp_handle; /* sftp remote file handle */ | ||
232 | |||
233 | - /* See ssh_seek() function below. */ | ||
234 | - int64_t offset; | ||
235 | - bool offset_op_read; | ||
236 | - | ||
237 | - /* File attributes at open. We try to keep the .filesize field | ||
238 | + /* | ||
239 | + * File attributes at open. We try to keep the .size field | ||
240 | * updated if it changes (eg by writing at the end of the file). | ||
241 | */ | ||
242 | - LIBSSH2_SFTP_ATTRIBUTES attrs; | ||
243 | + sftp_attributes attrs; | ||
244 | |||
245 | InetSocketAddress *inet; | ||
246 | |||
247 | @@ -XXX,XX +XXX,XX @@ static void ssh_state_init(BDRVSSHState *s) | ||
248 | { | ||
249 | memset(s, 0, sizeof *s); | ||
250 | s->sock = -1; | ||
251 | - s->offset = -1; | ||
252 | qemu_co_mutex_init(&s->lock); | ||
35 | } | 253 | } |
36 | 254 | ||
37 | -typedef struct { | 255 | @@ -XXX,XX +XXX,XX @@ static void ssh_state_free(BDRVSSHState *s) |
38 | - int ret; | 256 | { |
39 | -} BackupCompleteData; | 257 | g_free(s->user); |
258 | |||
259 | + if (s->attrs) { | ||
260 | + sftp_attributes_free(s->attrs); | ||
261 | + } | ||
262 | if (s->sftp_handle) { | ||
263 | - libssh2_sftp_close(s->sftp_handle); | ||
264 | + sftp_close(s->sftp_handle); | ||
265 | } | ||
266 | if (s->sftp) { | ||
267 | - libssh2_sftp_shutdown(s->sftp); | ||
268 | + sftp_free(s->sftp); | ||
269 | } | ||
270 | if (s->session) { | ||
271 | - libssh2_session_disconnect(s->session, | ||
272 | - "from qemu ssh client: " | ||
273 | - "user closed the connection"); | ||
274 | - libssh2_session_free(s->session); | ||
275 | - } | ||
276 | - if (s->sock >= 0) { | ||
277 | - close(s->sock); | ||
278 | + ssh_disconnect(s->session); | ||
279 | + ssh_free(s->session); /* This frees s->sock */ | ||
280 | } | ||
281 | } | ||
282 | |||
283 | @@ -XXX,XX +XXX,XX @@ session_error_setg(Error **errp, BDRVSSHState *s, const char *fs, ...) | ||
284 | va_end(args); | ||
285 | |||
286 | if (s->session) { | ||
287 | - char *ssh_err; | ||
288 | + const char *ssh_err; | ||
289 | int ssh_err_code; | ||
290 | |||
291 | - /* This is not an errno. See <libssh2.h>. */ | ||
292 | - ssh_err_code = libssh2_session_last_error(s->session, | ||
293 | - &ssh_err, NULL, 0); | ||
294 | - error_setg(errp, "%s: %s (libssh2 error code: %d)", | ||
295 | + /* This is not an errno. See <libssh/libssh.h>. */ | ||
296 | + ssh_err = ssh_get_error(s->session); | ||
297 | + ssh_err_code = ssh_get_error_code(s->session); | ||
298 | + error_setg(errp, "%s: %s (libssh error code: %d)", | ||
299 | msg, ssh_err, ssh_err_code); | ||
300 | } else { | ||
301 | error_setg(errp, "%s", msg); | ||
302 | @@ -XXX,XX +XXX,XX @@ sftp_error_setg(Error **errp, BDRVSSHState *s, const char *fs, ...) | ||
303 | va_end(args); | ||
304 | |||
305 | if (s->sftp) { | ||
306 | - char *ssh_err; | ||
307 | + const char *ssh_err; | ||
308 | int ssh_err_code; | ||
309 | - unsigned long sftp_err_code; | ||
310 | + int sftp_err_code; | ||
311 | |||
312 | - /* This is not an errno. See <libssh2.h>. */ | ||
313 | - ssh_err_code = libssh2_session_last_error(s->session, | ||
314 | - &ssh_err, NULL, 0); | ||
315 | - /* See <libssh2_sftp.h>. */ | ||
316 | - sftp_err_code = libssh2_sftp_last_error((s)->sftp); | ||
317 | + /* This is not an errno. See <libssh/libssh.h>. */ | ||
318 | + ssh_err = ssh_get_error(s->session); | ||
319 | + ssh_err_code = ssh_get_error_code(s->session); | ||
320 | + /* See <libssh/sftp.h>. */ | ||
321 | + sftp_err_code = sftp_get_error(s->sftp); | ||
322 | |||
323 | error_setg(errp, | ||
324 | - "%s: %s (libssh2 error code: %d, sftp error code: %lu)", | ||
325 | + "%s: %s (libssh error code: %d, sftp error code: %d)", | ||
326 | msg, ssh_err, ssh_err_code, sftp_err_code); | ||
327 | } else { | ||
328 | error_setg(errp, "%s", msg); | ||
329 | @@ -XXX,XX +XXX,XX @@ sftp_error_setg(Error **errp, BDRVSSHState *s, const char *fs, ...) | ||
330 | |||
331 | static void sftp_error_trace(BDRVSSHState *s, const char *op) | ||
332 | { | ||
333 | - char *ssh_err; | ||
334 | + const char *ssh_err; | ||
335 | int ssh_err_code; | ||
336 | - unsigned long sftp_err_code; | ||
337 | + int sftp_err_code; | ||
338 | |||
339 | - /* This is not an errno. See <libssh2.h>. */ | ||
340 | - ssh_err_code = libssh2_session_last_error(s->session, | ||
341 | - &ssh_err, NULL, 0); | ||
342 | - /* See <libssh2_sftp.h>. */ | ||
343 | - sftp_err_code = libssh2_sftp_last_error((s)->sftp); | ||
344 | + /* This is not an errno. See <libssh/libssh.h>. */ | ||
345 | + ssh_err = ssh_get_error(s->session); | ||
346 | + ssh_err_code = ssh_get_error_code(s->session); | ||
347 | + /* See <libssh/sftp.h>. */ | ||
348 | + sftp_err_code = sftp_get_error(s->sftp); | ||
349 | |||
350 | trace_sftp_error(op, ssh_err, ssh_err_code, sftp_err_code); | ||
351 | } | ||
352 | @@ -XXX,XX +XXX,XX @@ static void ssh_parse_filename(const char *filename, QDict *options, | ||
353 | parse_uri(filename, options, errp); | ||
354 | } | ||
355 | |||
356 | -static int check_host_key_knownhosts(BDRVSSHState *s, | ||
357 | - const char *host, int port, Error **errp) | ||
358 | +static int check_host_key_knownhosts(BDRVSSHState *s, Error **errp) | ||
359 | { | ||
360 | - const char *home; | ||
361 | - char *knh_file = NULL; | ||
362 | - LIBSSH2_KNOWNHOSTS *knh = NULL; | ||
363 | - struct libssh2_knownhost *found; | ||
364 | - int ret, r; | ||
365 | - const char *hostkey; | ||
366 | - size_t len; | ||
367 | - int type; | ||
40 | - | 368 | - |
41 | -static void backup_complete(Job *job, void *opaque) | 369 | - hostkey = libssh2_session_hostkey(s->session, &len, &type); |
370 | - if (!hostkey) { | ||
371 | + int ret; | ||
372 | +#ifdef HAVE_LIBSSH_0_8 | ||
373 | + enum ssh_known_hosts_e state; | ||
374 | + int r; | ||
375 | + ssh_key pubkey; | ||
376 | + enum ssh_keytypes_e pubkey_type; | ||
377 | + unsigned char *server_hash = NULL; | ||
378 | + size_t server_hash_len; | ||
379 | + char *fingerprint = NULL; | ||
380 | + | ||
381 | + state = ssh_session_is_known_server(s->session); | ||
382 | + trace_ssh_server_status(state); | ||
383 | + | ||
384 | + switch (state) { | ||
385 | + case SSH_KNOWN_HOSTS_OK: | ||
386 | + /* OK */ | ||
387 | + trace_ssh_check_host_key_knownhosts(); | ||
388 | + break; | ||
389 | + case SSH_KNOWN_HOSTS_CHANGED: | ||
390 | ret = -EINVAL; | ||
391 | - session_error_setg(errp, s, "failed to read remote host key"); | ||
392 | + r = ssh_get_server_publickey(s->session, &pubkey); | ||
393 | + if (r == 0) { | ||
394 | + r = ssh_get_publickey_hash(pubkey, SSH_PUBLICKEY_HASH_SHA256, | ||
395 | + &server_hash, &server_hash_len); | ||
396 | + pubkey_type = ssh_key_type(pubkey); | ||
397 | + ssh_key_free(pubkey); | ||
398 | + } | ||
399 | + if (r == 0) { | ||
400 | + fingerprint = ssh_get_fingerprint_hash(SSH_PUBLICKEY_HASH_SHA256, | ||
401 | + server_hash, | ||
402 | + server_hash_len); | ||
403 | + ssh_clean_pubkey_hash(&server_hash); | ||
404 | + } | ||
405 | + if (fingerprint) { | ||
406 | + error_setg(errp, | ||
407 | + "host key (%s key with fingerprint %s) does not match " | ||
408 | + "the one in known_hosts; this may be a possible attack", | ||
409 | + ssh_key_type_to_char(pubkey_type), fingerprint); | ||
410 | + ssh_string_free_char(fingerprint); | ||
411 | + } else { | ||
412 | + error_setg(errp, | ||
413 | + "host key does not match the one in known_hosts; this " | ||
414 | + "may be a possible attack"); | ||
415 | + } | ||
416 | goto out; | ||
417 | - } | ||
418 | - | ||
419 | - knh = libssh2_knownhost_init(s->session); | ||
420 | - if (!knh) { | ||
421 | + case SSH_KNOWN_HOSTS_OTHER: | ||
422 | ret = -EINVAL; | ||
423 | - session_error_setg(errp, s, | ||
424 | - "failed to initialize known hosts support"); | ||
425 | + error_setg(errp, | ||
426 | + "host key for this server not found, another type exists"); | ||
427 | + goto out; | ||
428 | + case SSH_KNOWN_HOSTS_UNKNOWN: | ||
429 | + ret = -EINVAL; | ||
430 | + error_setg(errp, "no host key was found in known_hosts"); | ||
431 | + goto out; | ||
432 | + case SSH_KNOWN_HOSTS_NOT_FOUND: | ||
433 | + ret = -ENOENT; | ||
434 | + error_setg(errp, "known_hosts file not found"); | ||
435 | + goto out; | ||
436 | + case SSH_KNOWN_HOSTS_ERROR: | ||
437 | + ret = -EINVAL; | ||
438 | + error_setg(errp, "error while checking the host"); | ||
439 | + goto out; | ||
440 | + default: | ||
441 | + ret = -EINVAL; | ||
442 | + error_setg(errp, "error while checking for known server (%d)", state); | ||
443 | goto out; | ||
444 | } | ||
445 | +#else /* !HAVE_LIBSSH_0_8 */ | ||
446 | + int state; | ||
447 | |||
448 | - home = getenv("HOME"); | ||
449 | - if (home) { | ||
450 | - knh_file = g_strdup_printf("%s/.ssh/known_hosts", home); | ||
451 | - } else { | ||
452 | - knh_file = g_strdup_printf("/root/.ssh/known_hosts"); | ||
453 | - } | ||
454 | - | ||
455 | - /* Read all known hosts from OpenSSH-style known_hosts file. */ | ||
456 | - libssh2_knownhost_readfile(knh, knh_file, LIBSSH2_KNOWNHOST_FILE_OPENSSH); | ||
457 | + state = ssh_is_server_known(s->session); | ||
458 | + trace_ssh_server_status(state); | ||
459 | |||
460 | - r = libssh2_knownhost_checkp(knh, host, port, hostkey, len, | ||
461 | - LIBSSH2_KNOWNHOST_TYPE_PLAIN| | ||
462 | - LIBSSH2_KNOWNHOST_KEYENC_RAW, | ||
463 | - &found); | ||
464 | - switch (r) { | ||
465 | - case LIBSSH2_KNOWNHOST_CHECK_MATCH: | ||
466 | + switch (state) { | ||
467 | + case SSH_SERVER_KNOWN_OK: | ||
468 | /* OK */ | ||
469 | - trace_ssh_check_host_key_knownhosts(found->key); | ||
470 | + trace_ssh_check_host_key_knownhosts(); | ||
471 | break; | ||
472 | - case LIBSSH2_KNOWNHOST_CHECK_MISMATCH: | ||
473 | + case SSH_SERVER_KNOWN_CHANGED: | ||
474 | ret = -EINVAL; | ||
475 | - session_error_setg(errp, s, | ||
476 | - "host key does not match the one in known_hosts" | ||
477 | - " (found key %s)", found->key); | ||
478 | + error_setg(errp, | ||
479 | + "host key does not match the one in known_hosts; this " | ||
480 | + "may be a possible attack"); | ||
481 | goto out; | ||
482 | - case LIBSSH2_KNOWNHOST_CHECK_NOTFOUND: | ||
483 | + case SSH_SERVER_FOUND_OTHER: | ||
484 | ret = -EINVAL; | ||
485 | - session_error_setg(errp, s, "no host key was found in known_hosts"); | ||
486 | + error_setg(errp, | ||
487 | + "host key for this server not found, another type exists"); | ||
488 | + goto out; | ||
489 | + case SSH_SERVER_FILE_NOT_FOUND: | ||
490 | + ret = -ENOENT; | ||
491 | + error_setg(errp, "known_hosts file not found"); | ||
492 | goto out; | ||
493 | - case LIBSSH2_KNOWNHOST_CHECK_FAILURE: | ||
494 | + case SSH_SERVER_NOT_KNOWN: | ||
495 | ret = -EINVAL; | ||
496 | - session_error_setg(errp, s, | ||
497 | - "failure matching the host key with known_hosts"); | ||
498 | + error_setg(errp, "no host key was found in known_hosts"); | ||
499 | + goto out; | ||
500 | + case SSH_SERVER_ERROR: | ||
501 | + ret = -EINVAL; | ||
502 | + error_setg(errp, "server error"); | ||
503 | goto out; | ||
504 | default: | ||
505 | ret = -EINVAL; | ||
506 | - session_error_setg(errp, s, "unknown error matching the host key" | ||
507 | - " with known_hosts (%d)", r); | ||
508 | + error_setg(errp, "error while checking for known server (%d)", state); | ||
509 | goto out; | ||
510 | } | ||
511 | +#endif /* !HAVE_LIBSSH_0_8 */ | ||
512 | |||
513 | /* known_hosts checking successful. */ | ||
514 | ret = 0; | ||
515 | |||
516 | out: | ||
517 | - if (knh != NULL) { | ||
518 | - libssh2_knownhost_free(knh); | ||
519 | - } | ||
520 | - g_free(knh_file); | ||
521 | return ret; | ||
522 | } | ||
523 | |||
524 | @@ -XXX,XX +XXX,XX @@ static int compare_fingerprint(const unsigned char *fingerprint, size_t len, | ||
525 | |||
526 | static int | ||
527 | check_host_key_hash(BDRVSSHState *s, const char *hash, | ||
528 | - int hash_type, size_t fingerprint_len, Error **errp) | ||
529 | + enum ssh_publickey_hash_type type, Error **errp) | ||
530 | { | ||
531 | - const char *fingerprint; | ||
532 | - | ||
533 | - fingerprint = libssh2_hostkey_hash(s->session, hash_type); | ||
534 | - if (!fingerprint) { | ||
535 | + int r; | ||
536 | + ssh_key pubkey; | ||
537 | + unsigned char *server_hash; | ||
538 | + size_t server_hash_len; | ||
539 | + | ||
540 | +#ifdef HAVE_LIBSSH_0_8 | ||
541 | + r = ssh_get_server_publickey(s->session, &pubkey); | ||
542 | +#else | ||
543 | + r = ssh_get_publickey(s->session, &pubkey); | ||
544 | +#endif | ||
545 | + if (r != SSH_OK) { | ||
546 | session_error_setg(errp, s, "failed to read remote host key"); | ||
547 | return -EINVAL; | ||
548 | } | ||
549 | |||
550 | - if(compare_fingerprint((unsigned char *) fingerprint, fingerprint_len, | ||
551 | - hash) != 0) { | ||
552 | + r = ssh_get_publickey_hash(pubkey, type, &server_hash, &server_hash_len); | ||
553 | + ssh_key_free(pubkey); | ||
554 | + if (r != 0) { | ||
555 | + session_error_setg(errp, s, | ||
556 | + "failed reading the hash of the server SSH key"); | ||
557 | + return -EINVAL; | ||
558 | + } | ||
559 | + | ||
560 | + r = compare_fingerprint(server_hash, server_hash_len, hash); | ||
561 | + ssh_clean_pubkey_hash(&server_hash); | ||
562 | + if (r != 0) { | ||
563 | error_setg(errp, "remote host key does not match host_key_check '%s'", | ||
564 | hash); | ||
565 | return -EPERM; | ||
566 | @@ -XXX,XX +XXX,XX @@ check_host_key_hash(BDRVSSHState *s, const char *hash, | ||
567 | return 0; | ||
568 | } | ||
569 | |||
570 | -static int check_host_key(BDRVSSHState *s, const char *host, int port, | ||
571 | - SshHostKeyCheck *hkc, Error **errp) | ||
572 | +static int check_host_key(BDRVSSHState *s, SshHostKeyCheck *hkc, Error **errp) | ||
573 | { | ||
574 | SshHostKeyCheckMode mode; | ||
575 | |||
576 | @@ -XXX,XX +XXX,XX @@ static int check_host_key(BDRVSSHState *s, const char *host, int port, | ||
577 | case SSH_HOST_KEY_CHECK_MODE_HASH: | ||
578 | if (hkc->u.hash.type == SSH_HOST_KEY_CHECK_HASH_TYPE_MD5) { | ||
579 | return check_host_key_hash(s, hkc->u.hash.hash, | ||
580 | - LIBSSH2_HOSTKEY_HASH_MD5, 16, errp); | ||
581 | + SSH_PUBLICKEY_HASH_MD5, errp); | ||
582 | } else if (hkc->u.hash.type == SSH_HOST_KEY_CHECK_HASH_TYPE_SHA1) { | ||
583 | return check_host_key_hash(s, hkc->u.hash.hash, | ||
584 | - LIBSSH2_HOSTKEY_HASH_SHA1, 20, errp); | ||
585 | + SSH_PUBLICKEY_HASH_SHA1, errp); | ||
586 | } | ||
587 | g_assert_not_reached(); | ||
588 | break; | ||
589 | case SSH_HOST_KEY_CHECK_MODE_KNOWN_HOSTS: | ||
590 | - return check_host_key_knownhosts(s, host, port, errp); | ||
591 | + return check_host_key_knownhosts(s, errp); | ||
592 | default: | ||
593 | g_assert_not_reached(); | ||
594 | } | ||
595 | @@ -XXX,XX +XXX,XX @@ static int check_host_key(BDRVSSHState *s, const char *host, int port, | ||
596 | return -EINVAL; | ||
597 | } | ||
598 | |||
599 | -static int authenticate(BDRVSSHState *s, const char *user, Error **errp) | ||
600 | +static int authenticate(BDRVSSHState *s, Error **errp) | ||
601 | { | ||
602 | int r, ret; | ||
603 | - const char *userauthlist; | ||
604 | - LIBSSH2_AGENT *agent = NULL; | ||
605 | - struct libssh2_agent_publickey *identity; | ||
606 | - struct libssh2_agent_publickey *prev_identity = NULL; | ||
607 | + int method; | ||
608 | |||
609 | - userauthlist = libssh2_userauth_list(s->session, user, strlen(user)); | ||
610 | - if (strstr(userauthlist, "publickey") == NULL) { | ||
611 | + /* Try to authenticate with the "none" method. */ | ||
612 | + r = ssh_userauth_none(s->session, NULL); | ||
613 | + if (r == SSH_AUTH_ERROR) { | ||
614 | ret = -EPERM; | ||
615 | - error_setg(errp, | ||
616 | - "remote server does not support \"publickey\" authentication"); | ||
617 | + session_error_setg(errp, s, "failed to authenticate using none " | ||
618 | + "authentication"); | ||
619 | goto out; | ||
620 | - } | ||
621 | - | ||
622 | - /* Connect to ssh-agent and try each identity in turn. */ | ||
623 | - agent = libssh2_agent_init(s->session); | ||
624 | - if (!agent) { | ||
625 | - ret = -EINVAL; | ||
626 | - session_error_setg(errp, s, "failed to initialize ssh-agent support"); | ||
627 | - goto out; | ||
628 | - } | ||
629 | - if (libssh2_agent_connect(agent)) { | ||
630 | - ret = -ECONNREFUSED; | ||
631 | - session_error_setg(errp, s, "failed to connect to ssh-agent"); | ||
632 | - goto out; | ||
633 | - } | ||
634 | - if (libssh2_agent_list_identities(agent)) { | ||
635 | - ret = -EINVAL; | ||
636 | - session_error_setg(errp, s, | ||
637 | - "failed requesting identities from ssh-agent"); | ||
638 | + } else if (r == SSH_AUTH_SUCCESS) { | ||
639 | + /* Authenticated! */ | ||
640 | + ret = 0; | ||
641 | goto out; | ||
642 | } | ||
643 | |||
644 | - for(;;) { | ||
645 | - r = libssh2_agent_get_identity(agent, &identity, prev_identity); | ||
646 | - if (r == 1) { /* end of list */ | ||
647 | - break; | ||
648 | - } | ||
649 | - if (r < 0) { | ||
650 | + method = ssh_userauth_list(s->session, NULL); | ||
651 | + trace_ssh_auth_methods(method); | ||
652 | + | ||
653 | + /* | ||
654 | + * Try to authenticate with publickey, using the ssh-agent | ||
655 | + * if available. | ||
656 | + */ | ||
657 | + if (method & SSH_AUTH_METHOD_PUBLICKEY) { | ||
658 | + r = ssh_userauth_publickey_auto(s->session, NULL, NULL); | ||
659 | + if (r == SSH_AUTH_ERROR) { | ||
660 | ret = -EINVAL; | ||
661 | - session_error_setg(errp, s, | ||
662 | - "failed to obtain identity from ssh-agent"); | ||
663 | + session_error_setg(errp, s, "failed to authenticate using " | ||
664 | + "publickey authentication"); | ||
665 | goto out; | ||
666 | - } | ||
667 | - r = libssh2_agent_userauth(agent, user, identity); | ||
668 | - if (r == 0) { | ||
669 | + } else if (r == SSH_AUTH_SUCCESS) { | ||
670 | /* Authenticated! */ | ||
671 | ret = 0; | ||
672 | goto out; | ||
673 | } | ||
674 | - /* Failed to authenticate with this identity, try the next one. */ | ||
675 | - prev_identity = identity; | ||
676 | } | ||
677 | |||
678 | ret = -EPERM; | ||
679 | @@ -XXX,XX +XXX,XX @@ static int authenticate(BDRVSSHState *s, const char *user, Error **errp) | ||
680 | "and the identities held by your ssh-agent"); | ||
681 | |||
682 | out: | ||
683 | - if (agent != NULL) { | ||
684 | - /* Note: libssh2 implementation implicitly calls | ||
685 | - * libssh2_agent_disconnect if necessary. | ||
686 | - */ | ||
687 | - libssh2_agent_free(agent); | ||
688 | - } | ||
689 | - | ||
690 | return ret; | ||
691 | } | ||
692 | |||
693 | @@ -XXX,XX +XXX,XX @@ static int connect_to_ssh(BDRVSSHState *s, BlockdevOptionsSsh *opts, | ||
694 | int ssh_flags, int creat_mode, Error **errp) | ||
695 | { | ||
696 | int r, ret; | ||
697 | - long port = 0; | ||
698 | + unsigned int port = 0; | ||
699 | + int new_sock = -1; | ||
700 | |||
701 | if (opts->has_user) { | ||
702 | s->user = g_strdup(opts->user); | ||
703 | @@ -XXX,XX +XXX,XX @@ static int connect_to_ssh(BDRVSSHState *s, BlockdevOptionsSsh *opts, | ||
704 | s->inet = opts->server; | ||
705 | opts->server = NULL; | ||
706 | |||
707 | - if (qemu_strtol(s->inet->port, NULL, 10, &port) < 0) { | ||
708 | + if (qemu_strtoui(s->inet->port, NULL, 10, &port) < 0) { | ||
709 | error_setg(errp, "Use only numeric port value"); | ||
710 | ret = -EINVAL; | ||
711 | goto err; | ||
712 | } | ||
713 | |||
714 | /* Open the socket and connect. */ | ||
715 | - s->sock = inet_connect_saddr(s->inet, errp); | ||
716 | - if (s->sock < 0) { | ||
717 | + new_sock = inet_connect_saddr(s->inet, errp); | ||
718 | + if (new_sock < 0) { | ||
719 | ret = -EIO; | ||
720 | goto err; | ||
721 | } | ||
722 | |||
723 | + /* | ||
724 | + * Try to disable the Nagle algorithm on TCP sockets to reduce latency, | ||
725 | + * but do not fail if it cannot be disabled. | ||
726 | + */ | ||
727 | + r = socket_set_nodelay(new_sock); | ||
728 | + if (r < 0) { | ||
729 | + warn_report("can't set TCP_NODELAY for the ssh server %s: %s", | ||
730 | + s->inet->host, strerror(errno)); | ||
731 | + } | ||
732 | + | ||
733 | /* Create SSH session. */ | ||
734 | - s->session = libssh2_session_init(); | ||
735 | + s->session = ssh_new(); | ||
736 | if (!s->session) { | ||
737 | ret = -EINVAL; | ||
738 | - session_error_setg(errp, s, "failed to initialize libssh2 session"); | ||
739 | + session_error_setg(errp, s, "failed to initialize libssh session"); | ||
740 | goto err; | ||
741 | } | ||
742 | |||
743 | -#if TRACE_LIBSSH2 != 0 | ||
744 | - libssh2_trace(s->session, TRACE_LIBSSH2); | ||
745 | -#endif | ||
746 | + /* | ||
747 | + * Make sure we are in blocking mode during the connection and | ||
748 | + * authentication phases. | ||
749 | + */ | ||
750 | + ssh_set_blocking(s->session, 1); | ||
751 | |||
752 | - r = libssh2_session_handshake(s->session, s->sock); | ||
753 | - if (r != 0) { | ||
754 | + r = ssh_options_set(s->session, SSH_OPTIONS_USER, s->user); | ||
755 | + if (r < 0) { | ||
756 | + ret = -EINVAL; | ||
757 | + session_error_setg(errp, s, | ||
758 | + "failed to set the user in the libssh session"); | ||
759 | + goto err; | ||
760 | + } | ||
761 | + | ||
762 | + r = ssh_options_set(s->session, SSH_OPTIONS_HOST, s->inet->host); | ||
763 | + if (r < 0) { | ||
764 | + ret = -EINVAL; | ||
765 | + session_error_setg(errp, s, | ||
766 | + "failed to set the host in the libssh session"); | ||
767 | + goto err; | ||
768 | + } | ||
769 | + | ||
770 | + if (port > 0) { | ||
771 | + r = ssh_options_set(s->session, SSH_OPTIONS_PORT, &port); | ||
772 | + if (r < 0) { | ||
773 | + ret = -EINVAL; | ||
774 | + session_error_setg(errp, s, | ||
775 | + "failed to set the port in the libssh session"); | ||
776 | + goto err; | ||
777 | + } | ||
778 | + } | ||
779 | + | ||
780 | + r = ssh_options_set(s->session, SSH_OPTIONS_COMPRESSION, "none"); | ||
781 | + if (r < 0) { | ||
782 | + ret = -EINVAL; | ||
783 | + session_error_setg(errp, s, | ||
784 | + "failed to disable the compression in the libssh " | ||
785 | + "session"); | ||
786 | + goto err; | ||
787 | + } | ||
788 | + | ||
789 | + /* Read ~/.ssh/config. */ | ||
790 | + r = ssh_options_parse_config(s->session, NULL); | ||
791 | + if (r < 0) { | ||
792 | + ret = -EINVAL; | ||
793 | + session_error_setg(errp, s, "failed to parse ~/.ssh/config"); | ||
794 | + goto err; | ||
795 | + } | ||
796 | + | ||
797 | + r = ssh_options_set(s->session, SSH_OPTIONS_FD, &new_sock); | ||
798 | + if (r < 0) { | ||
799 | + ret = -EINVAL; | ||
800 | + session_error_setg(errp, s, | ||
801 | + "failed to set the socket in the libssh session"); | ||
802 | + goto err; | ||
803 | + } | ||
804 | + /* libssh took ownership of the socket. */ | ||
805 | + s->sock = new_sock; | ||
806 | + new_sock = -1; | ||
807 | + | ||
808 | + /* Connect. */ | ||
809 | + r = ssh_connect(s->session); | ||
810 | + if (r != SSH_OK) { | ||
811 | ret = -EINVAL; | ||
812 | session_error_setg(errp, s, "failed to establish SSH session"); | ||
813 | goto err; | ||
814 | } | ||
815 | |||
816 | /* Check the remote host's key against known_hosts. */ | ||
817 | - ret = check_host_key(s, s->inet->host, port, opts->host_key_check, errp); | ||
818 | + ret = check_host_key(s, opts->host_key_check, errp); | ||
819 | if (ret < 0) { | ||
820 | goto err; | ||
821 | } | ||
822 | |||
823 | /* Authenticate. */ | ||
824 | - ret = authenticate(s, s->user, errp); | ||
825 | + ret = authenticate(s, errp); | ||
826 | if (ret < 0) { | ||
827 | goto err; | ||
828 | } | ||
829 | |||
830 | /* Start SFTP. */ | ||
831 | - s->sftp = libssh2_sftp_init(s->session); | ||
832 | + s->sftp = sftp_new(s->session); | ||
833 | if (!s->sftp) { | ||
834 | - session_error_setg(errp, s, "failed to initialize sftp handle"); | ||
835 | + session_error_setg(errp, s, "failed to create sftp handle"); | ||
836 | + ret = -EINVAL; | ||
837 | + goto err; | ||
838 | + } | ||
839 | + | ||
840 | + r = sftp_init(s->sftp); | ||
841 | + if (r < 0) { | ||
842 | + sftp_error_setg(errp, s, "failed to initialize sftp handle"); | ||
843 | ret = -EINVAL; | ||
844 | goto err; | ||
845 | } | ||
846 | |||
847 | /* Open the remote file. */ | ||
848 | trace_ssh_connect_to_ssh(opts->path, ssh_flags, creat_mode); | ||
849 | - s->sftp_handle = libssh2_sftp_open(s->sftp, opts->path, ssh_flags, | ||
850 | - creat_mode); | ||
851 | + s->sftp_handle = sftp_open(s->sftp, opts->path, ssh_flags, creat_mode); | ||
852 | if (!s->sftp_handle) { | ||
853 | - session_error_setg(errp, s, "failed to open remote file '%s'", | ||
854 | - opts->path); | ||
855 | + sftp_error_setg(errp, s, "failed to open remote file '%s'", | ||
856 | + opts->path); | ||
857 | ret = -EINVAL; | ||
858 | goto err; | ||
859 | } | ||
860 | |||
861 | - r = libssh2_sftp_fstat(s->sftp_handle, &s->attrs); | ||
862 | - if (r < 0) { | ||
863 | + /* Make sure the SFTP file is handled in blocking mode. */ | ||
864 | + sftp_file_set_blocking(s->sftp_handle); | ||
865 | + | ||
866 | + s->attrs = sftp_fstat(s->sftp_handle); | ||
867 | + if (!s->attrs) { | ||
868 | sftp_error_setg(errp, s, "failed to read file attributes"); | ||
869 | return -EINVAL; | ||
870 | } | ||
871 | @@ -XXX,XX +XXX,XX @@ static int connect_to_ssh(BDRVSSHState *s, BlockdevOptionsSsh *opts, | ||
872 | return 0; | ||
873 | |||
874 | err: | ||
875 | + if (s->attrs) { | ||
876 | + sftp_attributes_free(s->attrs); | ||
877 | + } | ||
878 | + s->attrs = NULL; | ||
879 | if (s->sftp_handle) { | ||
880 | - libssh2_sftp_close(s->sftp_handle); | ||
881 | + sftp_close(s->sftp_handle); | ||
882 | } | ||
883 | s->sftp_handle = NULL; | ||
884 | if (s->sftp) { | ||
885 | - libssh2_sftp_shutdown(s->sftp); | ||
886 | + sftp_free(s->sftp); | ||
887 | } | ||
888 | s->sftp = NULL; | ||
889 | if (s->session) { | ||
890 | - libssh2_session_disconnect(s->session, | ||
891 | - "from qemu ssh client: " | ||
892 | - "error opening connection"); | ||
893 | - libssh2_session_free(s->session); | ||
894 | + ssh_disconnect(s->session); | ||
895 | + ssh_free(s->session); | ||
896 | } | ||
897 | s->session = NULL; | ||
898 | + s->sock = -1; | ||
899 | + if (new_sock >= 0) { | ||
900 | + close(new_sock); | ||
901 | + } | ||
902 | |||
903 | return ret; | ||
904 | } | ||
905 | @@ -XXX,XX +XXX,XX @@ static int ssh_file_open(BlockDriverState *bs, QDict *options, int bdrv_flags, | ||
906 | |||
907 | ssh_state_init(s); | ||
908 | |||
909 | - ssh_flags = LIBSSH2_FXF_READ; | ||
910 | + ssh_flags = 0; | ||
911 | if (bdrv_flags & BDRV_O_RDWR) { | ||
912 | - ssh_flags |= LIBSSH2_FXF_WRITE; | ||
913 | + ssh_flags |= O_RDWR; | ||
914 | + } else { | ||
915 | + ssh_flags |= O_RDONLY; | ||
916 | } | ||
917 | |||
918 | opts = ssh_parse_options(options, errp); | ||
919 | @@ -XXX,XX +XXX,XX @@ static int ssh_file_open(BlockDriverState *bs, QDict *options, int bdrv_flags, | ||
920 | } | ||
921 | |||
922 | /* Go non-blocking. */ | ||
923 | - libssh2_session_set_blocking(s->session, 0); | ||
924 | + ssh_set_blocking(s->session, 0); | ||
925 | |||
926 | qapi_free_BlockdevOptionsSsh(opts); | ||
927 | |||
928 | return 0; | ||
929 | |||
930 | err: | ||
931 | - if (s->sock >= 0) { | ||
932 | - close(s->sock); | ||
933 | - } | ||
934 | - s->sock = -1; | ||
935 | - | ||
936 | qapi_free_BlockdevOptionsSsh(opts); | ||
937 | |||
938 | return ret; | ||
939 | @@ -XXX,XX +XXX,XX @@ static int ssh_grow_file(BDRVSSHState *s, int64_t offset, Error **errp) | ||
940 | { | ||
941 | ssize_t ret; | ||
942 | char c[1] = { '\0' }; | ||
943 | - int was_blocking = libssh2_session_get_blocking(s->session); | ||
944 | + int was_blocking = ssh_is_blocking(s->session); | ||
945 | |||
946 | /* offset must be strictly greater than the current size so we do | ||
947 | * not overwrite anything */ | ||
948 | - assert(offset > 0 && offset > s->attrs.filesize); | ||
949 | + assert(offset > 0 && offset > s->attrs->size); | ||
950 | |||
951 | - libssh2_session_set_blocking(s->session, 1); | ||
952 | + ssh_set_blocking(s->session, 1); | ||
953 | |||
954 | - libssh2_sftp_seek64(s->sftp_handle, offset - 1); | ||
955 | - ret = libssh2_sftp_write(s->sftp_handle, c, 1); | ||
956 | + sftp_seek64(s->sftp_handle, offset - 1); | ||
957 | + ret = sftp_write(s->sftp_handle, c, 1); | ||
958 | |||
959 | - libssh2_session_set_blocking(s->session, was_blocking); | ||
960 | + ssh_set_blocking(s->session, was_blocking); | ||
961 | |||
962 | if (ret < 0) { | ||
963 | sftp_error_setg(errp, s, "Failed to grow file"); | ||
964 | return -EIO; | ||
965 | } | ||
966 | |||
967 | - s->attrs.filesize = offset; | ||
968 | + s->attrs->size = offset; | ||
969 | return 0; | ||
970 | } | ||
971 | |||
972 | @@ -XXX,XX +XXX,XX @@ static int ssh_co_create(BlockdevCreateOptions *options, Error **errp) | ||
973 | ssh_state_init(&s); | ||
974 | |||
975 | ret = connect_to_ssh(&s, opts->location, | ||
976 | - LIBSSH2_FXF_READ|LIBSSH2_FXF_WRITE| | ||
977 | - LIBSSH2_FXF_CREAT|LIBSSH2_FXF_TRUNC, | ||
978 | + O_RDWR | O_CREAT | O_TRUNC, | ||
979 | 0644, errp); | ||
980 | if (ret < 0) { | ||
981 | goto fail; | ||
982 | @@ -XXX,XX +XXX,XX @@ static int ssh_has_zero_init(BlockDriverState *bs) | ||
983 | /* Assume false, unless we can positively prove it's true. */ | ||
984 | int has_zero_init = 0; | ||
985 | |||
986 | - if (s->attrs.flags & LIBSSH2_SFTP_ATTR_PERMISSIONS) { | ||
987 | - if (s->attrs.permissions & LIBSSH2_SFTP_S_IFREG) { | ||
988 | - has_zero_init = 1; | ||
989 | - } | ||
990 | + if (s->attrs->type == SSH_FILEXFER_TYPE_REGULAR) { | ||
991 | + has_zero_init = 1; | ||
992 | } | ||
993 | |||
994 | return has_zero_init; | ||
995 | @@ -XXX,XX +XXX,XX @@ static coroutine_fn void co_yield(BDRVSSHState *s, BlockDriverState *bs) | ||
996 | .co = qemu_coroutine_self() | ||
997 | }; | ||
998 | |||
999 | - r = libssh2_session_block_directions(s->session); | ||
1000 | + r = ssh_get_poll_flags(s->session); | ||
1001 | |||
1002 | - if (r & LIBSSH2_SESSION_BLOCK_INBOUND) { | ||
1003 | + if (r & SSH_READ_PENDING) { | ||
1004 | rd_handler = restart_coroutine; | ||
1005 | } | ||
1006 | - if (r & LIBSSH2_SESSION_BLOCK_OUTBOUND) { | ||
1007 | + if (r & SSH_WRITE_PENDING) { | ||
1008 | wr_handler = restart_coroutine; | ||
1009 | } | ||
1010 | |||
1011 | @@ -XXX,XX +XXX,XX @@ static coroutine_fn void co_yield(BDRVSSHState *s, BlockDriverState *bs) | ||
1012 | trace_ssh_co_yield_back(s->sock); | ||
1013 | } | ||
1014 | |||
1015 | -/* SFTP has a function `libssh2_sftp_seek64' which seeks to a position | ||
1016 | - * in the remote file. Notice that it just updates a field in the | ||
1017 | - * sftp_handle structure, so there is no network traffic and it cannot | ||
1018 | - * fail. | ||
1019 | - * | ||
1020 | - * However, `libssh2_sftp_seek64' does have a catastrophic effect on | ||
1021 | - * performance since it causes the handle to throw away all in-flight | ||
1022 | - * reads and buffered readahead data. Therefore this function tries | ||
1023 | - * to be intelligent about when to call the underlying libssh2 function. | ||
1024 | - */ | ||
1025 | -#define SSH_SEEK_WRITE 0 | ||
1026 | -#define SSH_SEEK_READ 1 | ||
1027 | -#define SSH_SEEK_FORCE 2 | ||
1028 | - | ||
1029 | -static void ssh_seek(BDRVSSHState *s, int64_t offset, int flags) | ||
42 | -{ | 1030 | -{ |
43 | - BackupCompleteData *data = opaque; | 1031 | - bool op_read = (flags & SSH_SEEK_READ) != 0; |
1032 | - bool force = (flags & SSH_SEEK_FORCE) != 0; | ||
44 | - | 1033 | - |
45 | - job_completed(job, data->ret); | 1034 | - if (force || op_read != s->offset_op_read || offset != s->offset) { |
46 | - g_free(data); | 1035 | - trace_ssh_seek(offset); |
1036 | - libssh2_sftp_seek64(s->sftp_handle, offset); | ||
1037 | - s->offset = offset; | ||
1038 | - s->offset_op_read = op_read; | ||
1039 | - } | ||
47 | -} | 1040 | -} |
48 | - | 1041 | - |
49 | static bool coroutine_fn yield_and_check(BackupBlockJob *job) | 1042 | static coroutine_fn int ssh_read(BDRVSSHState *s, BlockDriverState *bs, |
1043 | int64_t offset, size_t size, | ||
1044 | QEMUIOVector *qiov) | ||
1045 | @@ -XXX,XX +XXX,XX @@ static coroutine_fn int ssh_read(BDRVSSHState *s, BlockDriverState *bs, | ||
1046 | |||
1047 | trace_ssh_read(offset, size); | ||
1048 | |||
1049 | - ssh_seek(s, offset, SSH_SEEK_READ); | ||
1050 | + trace_ssh_seek(offset); | ||
1051 | + sftp_seek64(s->sftp_handle, offset); | ||
1052 | |||
1053 | /* This keeps track of the current iovec element ('i'), where we | ||
1054 | * will write to next ('buf'), and the end of the current iovec | ||
1055 | @@ -XXX,XX +XXX,XX @@ static coroutine_fn int ssh_read(BDRVSSHState *s, BlockDriverState *bs, | ||
1056 | buf = i->iov_base; | ||
1057 | end_of_vec = i->iov_base + i->iov_len; | ||
1058 | |||
1059 | - /* libssh2 has a hard-coded limit of 2000 bytes per request, | ||
1060 | - * although it will also do readahead behind our backs. Therefore | ||
1061 | - * we may have to do repeated reads here until we have read 'size' | ||
1062 | - * bytes. | ||
1063 | - */ | ||
1064 | for (got = 0; got < size; ) { | ||
1065 | + size_t request_read_size; | ||
1066 | again: | ||
1067 | - trace_ssh_read_buf(buf, end_of_vec - buf); | ||
1068 | - r = libssh2_sftp_read(s->sftp_handle, buf, end_of_vec - buf); | ||
1069 | - trace_ssh_read_return(r); | ||
1070 | + /* | ||
1071 | + * The size of SFTP packets is limited to 32K bytes, so limit | ||
1072 | + * the amount of data requested to 16K, as libssh currently | ||
1073 | + * does not handle multiple requests on its own. | ||
1074 | + */ | ||
1075 | + request_read_size = MIN(end_of_vec - buf, 16384); | ||
1076 | + trace_ssh_read_buf(buf, end_of_vec - buf, request_read_size); | ||
1077 | + r = sftp_read(s->sftp_handle, buf, request_read_size); | ||
1078 | + trace_ssh_read_return(r, sftp_get_error(s->sftp)); | ||
1079 | |||
1080 | - if (r == LIBSSH2_ERROR_EAGAIN || r == LIBSSH2_ERROR_TIMEOUT) { | ||
1081 | + if (r == SSH_AGAIN) { | ||
1082 | co_yield(s, bs); | ||
1083 | goto again; | ||
1084 | } | ||
1085 | - if (r < 0) { | ||
1086 | - sftp_error_trace(s, "read"); | ||
1087 | - s->offset = -1; | ||
1088 | - return -EIO; | ||
1089 | - } | ||
1090 | - if (r == 0) { | ||
1091 | + if (r == SSH_EOF || (r == 0 && sftp_get_error(s->sftp) == SSH_FX_EOF)) { | ||
1092 | /* EOF: Short read so pad the buffer with zeroes and return it. */ | ||
1093 | qemu_iovec_memset(qiov, got, 0, size - got); | ||
1094 | return 0; | ||
1095 | } | ||
1096 | + if (r <= 0) { | ||
1097 | + sftp_error_trace(s, "read"); | ||
1098 | + return -EIO; | ||
1099 | + } | ||
1100 | |||
1101 | got += r; | ||
1102 | buf += r; | ||
1103 | - s->offset += r; | ||
1104 | if (buf >= end_of_vec && got < size) { | ||
1105 | i++; | ||
1106 | buf = i->iov_base; | ||
1107 | @@ -XXX,XX +XXX,XX @@ static int ssh_write(BDRVSSHState *s, BlockDriverState *bs, | ||
1108 | |||
1109 | trace_ssh_write(offset, size); | ||
1110 | |||
1111 | - ssh_seek(s, offset, SSH_SEEK_WRITE); | ||
1112 | + trace_ssh_seek(offset); | ||
1113 | + sftp_seek64(s->sftp_handle, offset); | ||
1114 | |||
1115 | /* This keeps track of the current iovec element ('i'), where we | ||
1116 | * will read from next ('buf'), and the end of the current iovec | ||
1117 | @@ -XXX,XX +XXX,XX @@ static int ssh_write(BDRVSSHState *s, BlockDriverState *bs, | ||
1118 | end_of_vec = i->iov_base + i->iov_len; | ||
1119 | |||
1120 | for (written = 0; written < size; ) { | ||
1121 | + size_t request_write_size; | ||
1122 | again: | ||
1123 | - trace_ssh_write_buf(buf, end_of_vec - buf); | ||
1124 | - r = libssh2_sftp_write(s->sftp_handle, buf, end_of_vec - buf); | ||
1125 | - trace_ssh_write_return(r); | ||
1126 | + /* | ||
1127 | + * Avoid too large data packets, as libssh currently does not | ||
1128 | + * handle multiple requests on its own. | ||
1129 | + */ | ||
1130 | + request_write_size = MIN(end_of_vec - buf, 131072); | ||
1131 | + trace_ssh_write_buf(buf, end_of_vec - buf, request_write_size); | ||
1132 | + r = sftp_write(s->sftp_handle, buf, request_write_size); | ||
1133 | + trace_ssh_write_return(r, sftp_get_error(s->sftp)); | ||
1134 | |||
1135 | - if (r == LIBSSH2_ERROR_EAGAIN || r == LIBSSH2_ERROR_TIMEOUT) { | ||
1136 | + if (r == SSH_AGAIN) { | ||
1137 | co_yield(s, bs); | ||
1138 | goto again; | ||
1139 | } | ||
1140 | if (r < 0) { | ||
1141 | sftp_error_trace(s, "write"); | ||
1142 | - s->offset = -1; | ||
1143 | return -EIO; | ||
1144 | } | ||
1145 | - /* The libssh2 API is very unclear about this. A comment in | ||
1146 | - * the code says "nothing was acked, and no EAGAIN was | ||
1147 | - * received!" which apparently means that no data got sent | ||
1148 | - * out, and the underlying channel didn't return any EAGAIN | ||
1149 | - * indication. I think this is a bug in either libssh2 or | ||
1150 | - * OpenSSH (server-side). In any case, forcing a seek (to | ||
1151 | - * discard libssh2 internal buffers), and then trying again | ||
1152 | - * works for me. | ||
1153 | - */ | ||
1154 | - if (r == 0) { | ||
1155 | - ssh_seek(s, offset + written, SSH_SEEK_WRITE|SSH_SEEK_FORCE); | ||
1156 | - co_yield(s, bs); | ||
1157 | - goto again; | ||
1158 | - } | ||
1159 | |||
1160 | written += r; | ||
1161 | buf += r; | ||
1162 | - s->offset += r; | ||
1163 | if (buf >= end_of_vec && written < size) { | ||
1164 | i++; | ||
1165 | buf = i->iov_base; | ||
1166 | end_of_vec = i->iov_base + i->iov_len; | ||
1167 | } | ||
1168 | |||
1169 | - if (offset + written > s->attrs.filesize) | ||
1170 | - s->attrs.filesize = offset + written; | ||
1171 | + if (offset + written > s->attrs->size) { | ||
1172 | + s->attrs->size = offset + written; | ||
1173 | + } | ||
1174 | } | ||
1175 | |||
1176 | return 0; | ||
1177 | @@ -XXX,XX +XXX,XX @@ static void unsafe_flush_warning(BDRVSSHState *s, const char *what) | ||
1178 | } | ||
1179 | } | ||
1180 | |||
1181 | -#ifdef HAS_LIBSSH2_SFTP_FSYNC | ||
1182 | +#ifdef HAVE_LIBSSH_0_8 | ||
1183 | |||
1184 | static coroutine_fn int ssh_flush(BDRVSSHState *s, BlockDriverState *bs) | ||
50 | { | 1185 | { |
51 | uint64_t delay_ns; | 1186 | int r; |
52 | @@ -XXX,XX +XXX,XX @@ static void backup_incremental_init_copy_bitmap(BackupBlockJob *job) | 1187 | |
53 | static int coroutine_fn backup_run(Job *opaque_job, Error **errp) | 1188 | trace_ssh_flush(); |
54 | { | 1189 | + |
55 | BackupBlockJob *job = container_of(opaque_job, BackupBlockJob, common.job); | 1190 | + if (!sftp_extension_supported(s->sftp, "fsync@openssh.com", "1")) { |
56 | - BackupCompleteData *data; | 1191 | + unsafe_flush_warning(s, "OpenSSH >= 6.3"); |
57 | BlockDriverState *bs = blk_bs(job->common.blk); | 1192 | + return 0; |
58 | int64_t offset, nb_clusters; | 1193 | + } |
59 | int ret = 0; | 1194 | again: |
60 | @@ -XXX,XX +XXX,XX @@ static int coroutine_fn backup_run(Job *opaque_job, Error **errp) | 1195 | - r = libssh2_sftp_fsync(s->sftp_handle); |
61 | qemu_co_rwlock_unlock(&job->flush_rwlock); | 1196 | - if (r == LIBSSH2_ERROR_EAGAIN || r == LIBSSH2_ERROR_TIMEOUT) { |
62 | hbitmap_free(job->copy_bitmap); | 1197 | + r = sftp_fsync(s->sftp_handle); |
63 | 1198 | + if (r == SSH_AGAIN) { | |
64 | - data = g_malloc(sizeof(*data)); | 1199 | co_yield(s, bs); |
65 | - data->ret = ret; | 1200 | goto again; |
66 | - job_defer_to_main_loop(&job->common.job, backup_complete, data); | 1201 | } |
1202 | - if (r == LIBSSH2_ERROR_SFTP_PROTOCOL && | ||
1203 | - libssh2_sftp_last_error(s->sftp) == LIBSSH2_FX_OP_UNSUPPORTED) { | ||
1204 | - unsafe_flush_warning(s, "OpenSSH >= 6.3"); | ||
1205 | - return 0; | ||
1206 | - } | ||
1207 | if (r < 0) { | ||
1208 | sftp_error_trace(s, "fsync"); | ||
1209 | return -EIO; | ||
1210 | @@ -XXX,XX +XXX,XX @@ static coroutine_fn int ssh_co_flush(BlockDriverState *bs) | ||
67 | return ret; | 1211 | return ret; |
68 | } | 1212 | } |
69 | 1213 | ||
70 | diff --git a/block/create.c b/block/create.c | 1214 | -#else /* !HAS_LIBSSH2_SFTP_FSYNC */ |
71 | index XXXXXXX..XXXXXXX 100644 | 1215 | +#else /* !HAVE_LIBSSH_0_8 */ |
72 | --- a/block/create.c | 1216 | |
73 | +++ b/block/create.c | 1217 | static coroutine_fn int ssh_co_flush(BlockDriverState *bs) |
74 | @@ -XXX,XX +XXX,XX @@ typedef struct BlockdevCreateJob { | ||
75 | Job common; | ||
76 | BlockDriver *drv; | ||
77 | BlockdevCreateOptions *opts; | ||
78 | - int ret; | ||
79 | } BlockdevCreateJob; | ||
80 | |||
81 | -static void blockdev_create_complete(Job *job, void *opaque) | ||
82 | -{ | ||
83 | - BlockdevCreateJob *s = container_of(job, BlockdevCreateJob, common); | ||
84 | - | ||
85 | - job_completed(job, s->ret); | ||
86 | -} | ||
87 | - | ||
88 | static int coroutine_fn blockdev_create_run(Job *job, Error **errp) | ||
89 | { | 1218 | { |
90 | BlockdevCreateJob *s = container_of(job, BlockdevCreateJob, common); | 1219 | BDRVSSHState *s = bs->opaque; |
91 | + int ret; | 1220 | |
92 | 1221 | - unsafe_flush_warning(s, "libssh2 >= 1.4.4"); | |
93 | job_progress_set_remaining(&s->common, 1); | 1222 | + unsafe_flush_warning(s, "libssh >= 0.8.0"); |
94 | - s->ret = s->drv->bdrv_co_create(s->opts, errp); | ||
95 | + ret = s->drv->bdrv_co_create(s->opts, errp); | ||
96 | job_progress_update(&s->common, 1); | ||
97 | |||
98 | qapi_free_BlockdevCreateOptions(s->opts); | ||
99 | - job_defer_to_main_loop(&s->common, blockdev_create_complete, NULL); | ||
100 | |||
101 | - return s->ret; | ||
102 | + return ret; | ||
103 | } | ||
104 | |||
105 | static const JobDriver blockdev_create_job_driver = { | ||
106 | diff --git a/block/stream.c b/block/stream.c | ||
107 | index XXXXXXX..XXXXXXX 100644 | ||
108 | --- a/block/stream.c | ||
109 | +++ b/block/stream.c | ||
110 | @@ -XXX,XX +XXX,XX @@ static int coroutine_fn stream_populate(BlockBackend *blk, | ||
111 | return blk_co_preadv(blk, offset, qiov.size, &qiov, BDRV_REQ_COPY_ON_READ); | ||
112 | } | ||
113 | |||
114 | -typedef struct { | ||
115 | - int ret; | ||
116 | -} StreamCompleteData; | ||
117 | - | ||
118 | -static void stream_complete(Job *job, void *opaque) | ||
119 | +static void stream_exit(Job *job) | ||
120 | { | ||
121 | StreamBlockJob *s = container_of(job, StreamBlockJob, common.job); | ||
122 | BlockJob *bjob = &s->common; | ||
123 | - StreamCompleteData *data = opaque; | ||
124 | BlockDriverState *bs = blk_bs(bjob->blk); | ||
125 | BlockDriverState *base = s->base; | ||
126 | Error *local_err = NULL; | ||
127 | + int ret = job->ret; | ||
128 | |||
129 | - if (!job_is_cancelled(job) && bs->backing && data->ret == 0) { | ||
130 | + if (!job_is_cancelled(job) && bs->backing && ret == 0) { | ||
131 | const char *base_id = NULL, *base_fmt = NULL; | ||
132 | if (base) { | ||
133 | base_id = s->backing_file_str; | ||
134 | @@ -XXX,XX +XXX,XX @@ static void stream_complete(Job *job, void *opaque) | ||
135 | base_fmt = base->drv->format_name; | ||
136 | } | ||
137 | } | ||
138 | - data->ret = bdrv_change_backing_file(bs, base_id, base_fmt); | ||
139 | + ret = bdrv_change_backing_file(bs, base_id, base_fmt); | ||
140 | bdrv_set_backing_hd(bs, base, &local_err); | ||
141 | if (local_err) { | ||
142 | error_report_err(local_err); | ||
143 | - data->ret = -EPERM; | ||
144 | + ret = -EPERM; | ||
145 | goto out; | ||
146 | } | ||
147 | } | ||
148 | @@ -XXX,XX +XXX,XX @@ out: | ||
149 | } | ||
150 | |||
151 | g_free(s->backing_file_str); | ||
152 | - job_completed(job, data->ret); | ||
153 | - g_free(data); | ||
154 | + job->ret = ret; | ||
155 | } | ||
156 | |||
157 | static int coroutine_fn stream_run(Job *job, Error **errp) | ||
158 | { | ||
159 | StreamBlockJob *s = container_of(job, StreamBlockJob, common.job); | ||
160 | - StreamCompleteData *data; | ||
161 | BlockBackend *blk = s->common.blk; | ||
162 | BlockDriverState *bs = blk_bs(blk); | ||
163 | BlockDriverState *base = s->base; | ||
164 | @@ -XXX,XX +XXX,XX @@ static int coroutine_fn stream_run(Job *job, Error **errp) | ||
165 | |||
166 | out: | ||
167 | /* Modify backing chain and close BDSes in main loop */ | ||
168 | - data = g_malloc(sizeof(*data)); | ||
169 | - data->ret = ret; | ||
170 | - job_defer_to_main_loop(&s->common.job, stream_complete, data); | ||
171 | return ret; | ||
172 | } | ||
173 | |||
174 | @@ -XXX,XX +XXX,XX @@ static const BlockJobDriver stream_job_driver = { | ||
175 | .job_type = JOB_TYPE_STREAM, | ||
176 | .free = block_job_free, | ||
177 | .run = stream_run, | ||
178 | + .exit = stream_exit, | ||
179 | .user_resume = block_job_user_resume, | ||
180 | .drain = block_job_drain, | ||
181 | }, | ||
182 | diff --git a/tests/test-bdrv-drain.c b/tests/test-bdrv-drain.c | ||
183 | index XXXXXXX..XXXXXXX 100644 | ||
184 | --- a/tests/test-bdrv-drain.c | ||
185 | +++ b/tests/test-bdrv-drain.c | ||
186 | @@ -XXX,XX +XXX,XX @@ typedef struct TestBlockJob { | ||
187 | bool should_complete; | ||
188 | } TestBlockJob; | ||
189 | |||
190 | -static void test_job_completed(Job *job, void *opaque) | ||
191 | -{ | ||
192 | - job_completed(job, 0); | ||
193 | -} | ||
194 | - | ||
195 | static int coroutine_fn test_job_run(Job *job, Error **errp) | ||
196 | { | ||
197 | TestBlockJob *s = container_of(job, TestBlockJob, common.job); | ||
198 | @@ -XXX,XX +XXX,XX @@ static int coroutine_fn test_job_run(Job *job, Error **errp) | ||
199 | job_pause_point(&s->common.job); | ||
200 | } | ||
201 | |||
202 | - job_defer_to_main_loop(&s->common.job, test_job_completed, NULL); | ||
203 | return 0; | 1223 | return 0; |
204 | } | 1224 | } |
205 | 1225 | ||
206 | diff --git a/tests/test-blockjob-txn.c b/tests/test-blockjob-txn.c | 1226 | -#endif /* !HAS_LIBSSH2_SFTP_FSYNC */ |
1227 | +#endif /* !HAVE_LIBSSH_0_8 */ | ||
1228 | |||
1229 | static int64_t ssh_getlength(BlockDriverState *bs) | ||
1230 | { | ||
1231 | BDRVSSHState *s = bs->opaque; | ||
1232 | int64_t length; | ||
1233 | |||
1234 | - /* Note we cannot make a libssh2 call here. */ | ||
1235 | - length = (int64_t) s->attrs.filesize; | ||
1236 | + /* Note we cannot make a libssh call here. */ | ||
1237 | + length = (int64_t) s->attrs->size; | ||
1238 | trace_ssh_getlength(length); | ||
1239 | |||
1240 | return length; | ||
1241 | @@ -XXX,XX +XXX,XX @@ static int coroutine_fn ssh_co_truncate(BlockDriverState *bs, int64_t offset, | ||
1242 | return -ENOTSUP; | ||
1243 | } | ||
1244 | |||
1245 | - if (offset < s->attrs.filesize) { | ||
1246 | + if (offset < s->attrs->size) { | ||
1247 | error_setg(errp, "ssh driver does not support shrinking files"); | ||
1248 | return -ENOTSUP; | ||
1249 | } | ||
1250 | |||
1251 | - if (offset == s->attrs.filesize) { | ||
1252 | + if (offset == s->attrs->size) { | ||
1253 | return 0; | ||
1254 | } | ||
1255 | |||
1256 | @@ -XXX,XX +XXX,XX @@ static void bdrv_ssh_init(void) | ||
1257 | { | ||
1258 | int r; | ||
1259 | |||
1260 | - r = libssh2_init(0); | ||
1261 | + r = ssh_init(); | ||
1262 | if (r != 0) { | ||
1263 | - fprintf(stderr, "libssh2 initialization failed, %d\n", r); | ||
1264 | + fprintf(stderr, "libssh initialization failed, %d\n", r); | ||
1265 | exit(EXIT_FAILURE); | ||
1266 | } | ||
1267 | |||
1268 | +#if TRACE_LIBSSH != 0 | ||
1269 | + ssh_set_log_level(TRACE_LIBSSH); | ||
1270 | +#endif | ||
1271 | + | ||
1272 | bdrv_register(&bdrv_ssh); | ||
1273 | } | ||
1274 | |||
1275 | diff --git a/.travis.yml b/.travis.yml | ||
207 | index XXXXXXX..XXXXXXX 100644 | 1276 | index XXXXXXX..XXXXXXX 100644 |
208 | --- a/tests/test-blockjob-txn.c | 1277 | --- a/.travis.yml |
209 | +++ b/tests/test-blockjob-txn.c | 1278 | +++ b/.travis.yml |
210 | @@ -XXX,XX +XXX,XX @@ typedef struct { | 1279 | @@ -XXX,XX +XXX,XX @@ addons: |
211 | int *result; | 1280 | - libseccomp-dev |
212 | } TestBlockJob; | 1281 | - libspice-protocol-dev |
213 | 1282 | - libspice-server-dev | |
214 | -static void test_block_job_complete(Job *job, void *opaque) | 1283 | - - libssh2-1-dev |
215 | +static void test_block_job_exit(Job *job) | 1284 | + - libssh-dev |
216 | { | 1285 | - liburcu-dev |
217 | BlockJob *bjob = container_of(job, BlockJob, job); | 1286 | - libusb-1.0-0-dev |
218 | BlockDriverState *bs = blk_bs(bjob->blk); | 1287 | - libvte-2.91-dev |
219 | - int rc = (intptr_t)opaque; | 1288 | @@ -XXX,XX +XXX,XX @@ matrix: |
220 | 1289 | - libseccomp-dev | |
221 | - if (job_is_cancelled(job)) { | 1290 | - libspice-protocol-dev |
222 | - rc = -ECANCELED; | 1291 | - libspice-server-dev |
223 | - } | 1292 | - - libssh2-1-dev |
1293 | + - libssh-dev | ||
1294 | - liburcu-dev | ||
1295 | - libusb-1.0-0-dev | ||
1296 | - libvte-2.91-dev | ||
1297 | diff --git a/block/trace-events b/block/trace-events | ||
1298 | index XXXXXXX..XXXXXXX 100644 | ||
1299 | --- a/block/trace-events | ||
1300 | +++ b/block/trace-events | ||
1301 | @@ -XXX,XX +XXX,XX @@ nbd_client_connect_success(const char *export_name) "export '%s'" | ||
1302 | # ssh.c | ||
1303 | ssh_restart_coroutine(void *co) "co=%p" | ||
1304 | ssh_flush(void) "fsync" | ||
1305 | -ssh_check_host_key_knownhosts(const char *key) "host key OK: %s" | ||
1306 | +ssh_check_host_key_knownhosts(void) "host key OK" | ||
1307 | ssh_connect_to_ssh(char *path, int flags, int mode) "opening file %s flags=0x%x creat_mode=0%o" | ||
1308 | ssh_co_yield(int sock, void *rd_handler, void *wr_handler) "s->sock=%d rd_handler=%p wr_handler=%p" | ||
1309 | ssh_co_yield_back(int sock) "s->sock=%d - back" | ||
1310 | ssh_getlength(int64_t length) "length=%" PRIi64 | ||
1311 | ssh_co_create_opts(uint64_t size) "total_size=%" PRIu64 | ||
1312 | ssh_read(int64_t offset, size_t size) "offset=%" PRIi64 " size=%zu" | ||
1313 | -ssh_read_buf(void *buf, size_t size) "sftp_read buf=%p size=%zu" | ||
1314 | -ssh_read_return(ssize_t ret) "sftp_read returned %zd" | ||
1315 | +ssh_read_buf(void *buf, size_t size, size_t actual_size) "sftp_read buf=%p size=%zu (actual size=%zu)" | ||
1316 | +ssh_read_return(ssize_t ret, int sftp_err) "sftp_read returned %zd (sftp error=%d)" | ||
1317 | ssh_write(int64_t offset, size_t size) "offset=%" PRIi64 " size=%zu" | ||
1318 | -ssh_write_buf(void *buf, size_t size) "sftp_write buf=%p size=%zu" | ||
1319 | -ssh_write_return(ssize_t ret) "sftp_write returned %zd" | ||
1320 | +ssh_write_buf(void *buf, size_t size, size_t actual_size) "sftp_write buf=%p size=%zu (actual size=%zu)" | ||
1321 | +ssh_write_return(ssize_t ret, int sftp_err) "sftp_write returned %zd (sftp error=%d)" | ||
1322 | ssh_seek(int64_t offset) "seeking to offset=%" PRIi64 | ||
1323 | +ssh_auth_methods(int methods) "auth methods=0x%x" | ||
1324 | +ssh_server_status(int status) "server status=%d" | ||
1325 | |||
1326 | # curl.c | ||
1327 | curl_timer_cb(long timeout_ms) "timer callback timeout_ms %ld" | ||
1328 | @@ -XXX,XX +XXX,XX @@ sheepdog_snapshot_create(const char *sn_name, const char *id) "%s %s" | ||
1329 | sheepdog_snapshot_create_inode(const char *name, uint32_t snap, uint32_t vdi) "s->inode: name %s snap_id 0x%" PRIx32 " vdi 0x%" PRIx32 | ||
1330 | |||
1331 | # ssh.c | ||
1332 | -sftp_error(const char *op, const char *ssh_err, int ssh_err_code, unsigned long sftp_err_code) "%s failed: %s (libssh2 error code: %d, sftp error code: %lu)" | ||
1333 | +sftp_error(const char *op, const char *ssh_err, int ssh_err_code, int sftp_err_code) "%s failed: %s (libssh error code: %d, sftp error code: %d)" | ||
1334 | diff --git a/docs/qemu-block-drivers.texi b/docs/qemu-block-drivers.texi | ||
1335 | index XXXXXXX..XXXXXXX 100644 | ||
1336 | --- a/docs/qemu-block-drivers.texi | ||
1337 | +++ b/docs/qemu-block-drivers.texi | ||
1338 | @@ -XXX,XX +XXX,XX @@ print a warning when @code{fsync} is not supported: | ||
1339 | |||
1340 | warning: ssh server @code{ssh.example.com:22} does not support fsync | ||
1341 | |||
1342 | -With sufficiently new versions of libssh2 and OpenSSH, @code{fsync} is | ||
1343 | +With sufficiently new versions of libssh and OpenSSH, @code{fsync} is | ||
1344 | supported. | ||
1345 | |||
1346 | @node disk_images_nvme | ||
1347 | diff --git a/tests/docker/dockerfiles/debian-win32-cross.docker b/tests/docker/dockerfiles/debian-win32-cross.docker | ||
1348 | index XXXXXXX..XXXXXXX 100644 | ||
1349 | --- a/tests/docker/dockerfiles/debian-win32-cross.docker | ||
1350 | +++ b/tests/docker/dockerfiles/debian-win32-cross.docker | ||
1351 | @@ -XXX,XX +XXX,XX @@ RUN DEBIAN_FRONTEND=noninteractive eatmydata \ | ||
1352 | mxe-$TARGET-w64-mingw32.shared-curl \ | ||
1353 | mxe-$TARGET-w64-mingw32.shared-glib \ | ||
1354 | mxe-$TARGET-w64-mingw32.shared-libgcrypt \ | ||
1355 | - mxe-$TARGET-w64-mingw32.shared-libssh2 \ | ||
1356 | mxe-$TARGET-w64-mingw32.shared-libusb1 \ | ||
1357 | mxe-$TARGET-w64-mingw32.shared-lzo \ | ||
1358 | mxe-$TARGET-w64-mingw32.shared-nettle \ | ||
1359 | diff --git a/tests/docker/dockerfiles/debian-win64-cross.docker b/tests/docker/dockerfiles/debian-win64-cross.docker | ||
1360 | index XXXXXXX..XXXXXXX 100644 | ||
1361 | --- a/tests/docker/dockerfiles/debian-win64-cross.docker | ||
1362 | +++ b/tests/docker/dockerfiles/debian-win64-cross.docker | ||
1363 | @@ -XXX,XX +XXX,XX @@ RUN DEBIAN_FRONTEND=noninteractive eatmydata \ | ||
1364 | mxe-$TARGET-w64-mingw32.shared-curl \ | ||
1365 | mxe-$TARGET-w64-mingw32.shared-glib \ | ||
1366 | mxe-$TARGET-w64-mingw32.shared-libgcrypt \ | ||
1367 | - mxe-$TARGET-w64-mingw32.shared-libssh2 \ | ||
1368 | mxe-$TARGET-w64-mingw32.shared-libusb1 \ | ||
1369 | mxe-$TARGET-w64-mingw32.shared-lzo \ | ||
1370 | mxe-$TARGET-w64-mingw32.shared-nettle \ | ||
1371 | diff --git a/tests/docker/dockerfiles/fedora.docker b/tests/docker/dockerfiles/fedora.docker | ||
1372 | index XXXXXXX..XXXXXXX 100644 | ||
1373 | --- a/tests/docker/dockerfiles/fedora.docker | ||
1374 | +++ b/tests/docker/dockerfiles/fedora.docker | ||
1375 | @@ -XXX,XX +XXX,XX @@ ENV PACKAGES \ | ||
1376 | libpng-devel \ | ||
1377 | librbd-devel \ | ||
1378 | libseccomp-devel \ | ||
1379 | - libssh2-devel \ | ||
1380 | + libssh-devel \ | ||
1381 | libubsan \ | ||
1382 | libusbx-devel \ | ||
1383 | libxml2-devel \ | ||
1384 | @@ -XXX,XX +XXX,XX @@ ENV PACKAGES \ | ||
1385 | mingw32-gtk3 \ | ||
1386 | mingw32-libjpeg-turbo \ | ||
1387 | mingw32-libpng \ | ||
1388 | - mingw32-libssh2 \ | ||
1389 | mingw32-libtasn1 \ | ||
1390 | mingw32-nettle \ | ||
1391 | mingw32-pixman \ | ||
1392 | @@ -XXX,XX +XXX,XX @@ ENV PACKAGES \ | ||
1393 | mingw64-gtk3 \ | ||
1394 | mingw64-libjpeg-turbo \ | ||
1395 | mingw64-libpng \ | ||
1396 | - mingw64-libssh2 \ | ||
1397 | mingw64-libtasn1 \ | ||
1398 | mingw64-nettle \ | ||
1399 | mingw64-pixman \ | ||
1400 | diff --git a/tests/docker/dockerfiles/ubuntu.docker b/tests/docker/dockerfiles/ubuntu.docker | ||
1401 | index XXXXXXX..XXXXXXX 100644 | ||
1402 | --- a/tests/docker/dockerfiles/ubuntu.docker | ||
1403 | +++ b/tests/docker/dockerfiles/ubuntu.docker | ||
1404 | @@ -XXX,XX +XXX,XX @@ ENV PACKAGES flex bison \ | ||
1405 | libsnappy-dev \ | ||
1406 | libspice-protocol-dev \ | ||
1407 | libspice-server-dev \ | ||
1408 | - libssh2-1-dev \ | ||
1409 | + libssh-dev \ | ||
1410 | libusb-1.0-0-dev \ | ||
1411 | libusbredirhost-dev \ | ||
1412 | libvdeplug-dev \ | ||
1413 | diff --git a/tests/docker/dockerfiles/ubuntu1804.docker b/tests/docker/dockerfiles/ubuntu1804.docker | ||
1414 | index XXXXXXX..XXXXXXX 100644 | ||
1415 | --- a/tests/docker/dockerfiles/ubuntu1804.docker | ||
1416 | +++ b/tests/docker/dockerfiles/ubuntu1804.docker | ||
1417 | @@ -XXX,XX +XXX,XX @@ ENV PACKAGES flex bison \ | ||
1418 | libsnappy-dev \ | ||
1419 | libspice-protocol-dev \ | ||
1420 | libspice-server-dev \ | ||
1421 | - libssh2-1-dev \ | ||
1422 | + libssh-dev \ | ||
1423 | libusb-1.0-0-dev \ | ||
1424 | libusbredirhost-dev \ | ||
1425 | libvdeplug-dev \ | ||
1426 | diff --git a/tests/qemu-iotests/207 b/tests/qemu-iotests/207 | ||
1427 | index XXXXXXX..XXXXXXX 100755 | ||
1428 | --- a/tests/qemu-iotests/207 | ||
1429 | +++ b/tests/qemu-iotests/207 | ||
1430 | @@ -XXX,XX +XXX,XX @@ with iotests.FilePath('t.img') as disk_path, \ | ||
1431 | |||
1432 | iotests.img_info_log(remote_path) | ||
1433 | |||
1434 | - md5_key = subprocess.check_output( | ||
1435 | - 'ssh-keyscan -t rsa 127.0.0.1 2>/dev/null | grep -v "\\^#" | ' + | ||
1436 | - 'cut -d" " -f3 | base64 -d | md5sum -b | cut -d" " -f1', | ||
1437 | - shell=True).rstrip().decode('ascii') | ||
1438 | + keys = subprocess.check_output( | ||
1439 | + 'ssh-keyscan 127.0.0.1 2>/dev/null | grep -v "\\^#" | ' + | ||
1440 | + 'cut -d" " -f3', | ||
1441 | + shell=True).rstrip().decode('ascii').split('\n') | ||
1442 | + | ||
1443 | + # Mappings of base64 representations to digests | ||
1444 | + md5_keys = {} | ||
1445 | + sha1_keys = {} | ||
1446 | + | ||
1447 | + for key in keys: | ||
1448 | + md5_keys[key] = subprocess.check_output( | ||
1449 | + 'echo %s | base64 -d | md5sum -b | cut -d" " -f1' % key, | ||
1450 | + shell=True).rstrip().decode('ascii') | ||
1451 | + | ||
1452 | + sha1_keys[key] = subprocess.check_output( | ||
1453 | + 'echo %s | base64 -d | sha1sum -b | cut -d" " -f1' % key, | ||
1454 | + shell=True).rstrip().decode('ascii') | ||
1455 | |||
1456 | vm.launch() | ||
1457 | + | ||
1458 | + # Find correct key first | ||
1459 | + matching_key = None | ||
1460 | + for key in keys: | ||
1461 | + result = vm.qmp('blockdev-add', | ||
1462 | + driver='ssh', node_name='node0', path=disk_path, | ||
1463 | + server={ | ||
1464 | + 'host': '127.0.0.1', | ||
1465 | + 'port': '22', | ||
1466 | + }, host_key_check={ | ||
1467 | + 'mode': 'hash', | ||
1468 | + 'type': 'md5', | ||
1469 | + 'hash': md5_keys[key], | ||
1470 | + }) | ||
1471 | + | ||
1472 | + if 'error' not in result: | ||
1473 | + vm.qmp('blockdev-del', node_name='node0') | ||
1474 | + matching_key = key | ||
1475 | + break | ||
1476 | + | ||
1477 | + if matching_key is None: | ||
1478 | + vm.shutdown() | ||
1479 | + iotests.notrun('Did not find a key that fits 127.0.0.1') | ||
1480 | + | ||
1481 | blockdev_create(vm, { 'driver': 'ssh', | ||
1482 | 'location': { | ||
1483 | 'path': disk_path, | ||
1484 | @@ -XXX,XX +XXX,XX @@ with iotests.FilePath('t.img') as disk_path, \ | ||
1485 | 'host-key-check': { | ||
1486 | 'mode': 'hash', | ||
1487 | 'type': 'md5', | ||
1488 | - 'hash': md5_key, | ||
1489 | + 'hash': md5_keys[matching_key], | ||
1490 | } | ||
1491 | }, | ||
1492 | 'size': 8388608 }) | ||
1493 | @@ -XXX,XX +XXX,XX @@ with iotests.FilePath('t.img') as disk_path, \ | ||
1494 | |||
1495 | iotests.img_info_log(remote_path) | ||
1496 | |||
1497 | - sha1_key = subprocess.check_output( | ||
1498 | - 'ssh-keyscan -t rsa 127.0.0.1 2>/dev/null | grep -v "\\^#" | ' + | ||
1499 | - 'cut -d" " -f3 | base64 -d | sha1sum -b | cut -d" " -f1', | ||
1500 | - shell=True).rstrip().decode('ascii') | ||
224 | - | 1501 | - |
225 | - job_completed(job, rc); | 1502 | vm.launch() |
226 | bdrv_unref(bs); | 1503 | blockdev_create(vm, { 'driver': 'ssh', |
227 | } | 1504 | 'location': { |
228 | 1505 | @@ -XXX,XX +XXX,XX @@ with iotests.FilePath('t.img') as disk_path, \ | |
229 | @@ -XXX,XX +XXX,XX @@ static int coroutine_fn test_block_job_run(Job *job, Error **errp) | 1506 | 'host-key-check': { |
230 | } | 1507 | 'mode': 'hash', |
231 | } | 1508 | 'type': 'sha1', |
232 | 1509 | - 'hash': sha1_key, | |
233 | - job_defer_to_main_loop(job, test_block_job_complete, | 1510 | + 'hash': sha1_keys[matching_key], |
234 | - (void *)(intptr_t)s->rc); | 1511 | } |
235 | return s->rc; | 1512 | }, |
236 | } | 1513 | 'size': 4194304 }) |
237 | 1514 | diff --git a/tests/qemu-iotests/207.out b/tests/qemu-iotests/207.out | |
238 | @@ -XXX,XX +XXX,XX @@ static const BlockJobDriver test_block_job_driver = { | ||
239 | .user_resume = block_job_user_resume, | ||
240 | .drain = block_job_drain, | ||
241 | .run = test_block_job_run, | ||
242 | + .exit = test_block_job_exit, | ||
243 | }, | ||
244 | }; | ||
245 | |||
246 | diff --git a/tests/test-blockjob.c b/tests/test-blockjob.c | ||
247 | index XXXXXXX..XXXXXXX 100644 | 1515 | index XXXXXXX..XXXXXXX 100644 |
248 | --- a/tests/test-blockjob.c | 1516 | --- a/tests/qemu-iotests/207.out |
249 | +++ b/tests/test-blockjob.c | 1517 | +++ b/tests/qemu-iotests/207.out |
250 | @@ -XXX,XX +XXX,XX @@ typedef struct CancelJob { | 1518 | @@ -XXX,XX +XXX,XX @@ virtual size: 4 MiB (4194304 bytes) |
251 | bool completed; | 1519 | |
252 | } CancelJob; | 1520 | {"execute": "blockdev-create", "arguments": {"job-id": "job0", "options": {"driver": "ssh", "location": {"host-key-check": {"mode": "none"}, "path": "/this/is/not/an/existing/path", "server": {"host": "127.0.0.1", "port": "22"}}, "size": 4194304}}} |
253 | 1521 | {"return": {}} | |
254 | -static void cancel_job_completed(Job *job, void *opaque) | 1522 | -Job failed: failed to open remote file '/this/is/not/an/existing/path': Failed opening remote file (libssh2 error code: -31) |
255 | +static void cancel_job_exit(Job *job) | 1523 | +Job failed: failed to open remote file '/this/is/not/an/existing/path': SFTP server: No such file (libssh error code: 1, sftp error code: 2) |
256 | { | 1524 | {"execute": "job-dismiss", "arguments": {"id": "job0"}} |
257 | - CancelJob *s = opaque; | 1525 | {"return": {}} |
258 | + CancelJob *s = container_of(job, CancelJob, common.job); | 1526 | |
259 | s->completed = true; | ||
260 | - job_completed(job, 0); | ||
261 | } | ||
262 | |||
263 | static void cancel_job_complete(Job *job, Error **errp) | ||
264 | @@ -XXX,XX +XXX,XX @@ static int coroutine_fn cancel_job_run(Job *job, Error **errp) | ||
265 | |||
266 | while (!s->should_complete) { | ||
267 | if (job_is_cancelled(&s->common.job)) { | ||
268 | - goto defer; | ||
269 | + return 0; | ||
270 | } | ||
271 | |||
272 | if (!job_is_ready(&s->common.job) && s->should_converge) { | ||
273 | @@ -XXX,XX +XXX,XX @@ static int coroutine_fn cancel_job_run(Job *job, Error **errp) | ||
274 | job_sleep_ns(&s->common.job, 100000); | ||
275 | } | ||
276 | |||
277 | - defer: | ||
278 | - job_defer_to_main_loop(&s->common.job, cancel_job_completed, s); | ||
279 | return 0; | ||
280 | } | ||
281 | |||
282 | @@ -XXX,XX +XXX,XX @@ static const BlockJobDriver test_cancel_driver = { | ||
283 | .user_resume = block_job_user_resume, | ||
284 | .drain = block_job_drain, | ||
285 | .run = cancel_job_run, | ||
286 | + .exit = cancel_job_exit, | ||
287 | .complete = cancel_job_complete, | ||
288 | }, | ||
289 | }; | ||
290 | -- | 1527 | -- |
291 | 2.17.1 | 1528 | 2.21.0 |
292 | 1529 | ||
293 | 1530 | diff view generated by jsdifflib |
1 | From: John Snow <jsnow@redhat.com> | 1 | Tests should place their files into the test directory. This includes |
---|---|---|---|
2 | Unix sockets. 205 currently fails to do so, which prevents it from | ||
3 | being run concurrently. | ||
2 | 4 | ||
3 | All jobs do the same thing when they leave their running loop: | 5 | Signed-off-by: Max Reitz <mreitz@redhat.com> |
4 | - Store the return code in a structure | 6 | Message-id: 20190618210238.9524-1-mreitz@redhat.com |
5 | - wait to receive this structure in the main thread | 7 | Reviewed-by: Eric Blake <eblake@redhat.com> |
6 | - signal job completion via job_completed | ||
7 | |||
8 | Few jobs do anything beyond exactly this. Consolidate this exit | ||
9 | logic for a net reduction in SLOC. | ||
10 | |||
11 | More seriously, when we utilize job_defer_to_main_loop_bh to call | ||
12 | a function that calls job_completed, job_finalize_single will run | ||
13 | in a context where it has recursively taken the aio_context lock, | ||
14 | which can cause hangs if it puts down a reference that causes a flush. | ||
15 | |||
16 | You can observe this in practice by looking at mirror_exit's careful | ||
17 | placement of job_completed and bdrv_unref calls. | ||
18 | |||
19 | If we centralize job exiting, we can signal job completion from outside | ||
20 | of the aio_context, which should allow for job cleanup code to run with | ||
21 | only one lock, which makes cleanup callbacks less tricky to write. | ||
22 | |||
23 | Signed-off-by: John Snow <jsnow@redhat.com> | ||
24 | Reviewed-by: Max Reitz <mreitz@redhat.com> | ||
25 | Message-id: 20180830015734.19765-4-jsnow@redhat.com | ||
26 | Reviewed-by: Jeff Cody <jcody@redhat.com> | ||
27 | Signed-off-by: Max Reitz <mreitz@redhat.com> | 8 | Signed-off-by: Max Reitz <mreitz@redhat.com> |
28 | --- | 9 | --- |
29 | include/qemu/job.h | 11 +++++++++++ | 10 | tests/qemu-iotests/205 | 2 +- |
30 | job.c | 18 ++++++++++++++++++ | 11 | 1 file changed, 1 insertion(+), 1 deletion(-) |
31 | 2 files changed, 29 insertions(+) | ||
32 | 12 | ||
33 | diff --git a/include/qemu/job.h b/include/qemu/job.h | 13 | diff --git a/tests/qemu-iotests/205 b/tests/qemu-iotests/205 |
34 | index XXXXXXX..XXXXXXX 100644 | 14 | index XXXXXXX..XXXXXXX 100755 |
35 | --- a/include/qemu/job.h | 15 | --- a/tests/qemu-iotests/205 |
36 | +++ b/include/qemu/job.h | 16 | +++ b/tests/qemu-iotests/205 |
37 | @@ -XXX,XX +XXX,XX @@ struct JobDriver { | 17 | @@ -XXX,XX +XXX,XX @@ import iotests |
38 | */ | 18 | import time |
39 | void (*drain)(Job *job); | 19 | from iotests import qemu_img_create, qemu_io, filter_qemu_io, QemuIoInteractive |
40 | 20 | ||
41 | + /** | 21 | -nbd_sock = 'nbd_sock' |
42 | + * If the callback is not NULL, exit will be invoked from the main thread | 22 | +nbd_sock = os.path.join(iotests.test_dir, 'nbd_sock') |
43 | + * when the job's coroutine has finished, but before transactional | 23 | nbd_uri = 'nbd+unix:///exp?socket=' + nbd_sock |
44 | + * convergence; before @prepare or @abort. | 24 | disk = os.path.join(iotests.test_dir, 'disk') |
45 | + * | ||
46 | + * FIXME TODO: This callback is only temporary to transition remaining jobs | ||
47 | + * to prepare/commit/abort/clean callbacks and will be removed before 3.1. | ||
48 | + * is released. | ||
49 | + */ | ||
50 | + void (*exit)(Job *job); | ||
51 | + | ||
52 | /** | ||
53 | * If the callback is not NULL, prepare will be invoked when all the jobs | ||
54 | * belonging to the same transaction complete; or upon this job's completion | ||
55 | diff --git a/job.c b/job.c | ||
56 | index XXXXXXX..XXXXXXX 100644 | ||
57 | --- a/job.c | ||
58 | +++ b/job.c | ||
59 | @@ -XXX,XX +XXX,XX @@ void job_drain(Job *job) | ||
60 | } | ||
61 | } | ||
62 | |||
63 | +static void job_exit(void *opaque) | ||
64 | +{ | ||
65 | + Job *job = (Job *)opaque; | ||
66 | + AioContext *aio_context = job->aio_context; | ||
67 | + | ||
68 | + if (job->driver->exit) { | ||
69 | + aio_context_acquire(aio_context); | ||
70 | + job->driver->exit(job); | ||
71 | + aio_context_release(aio_context); | ||
72 | + } | ||
73 | + job_completed(job, job->ret); | ||
74 | +} | ||
75 | |||
76 | /** | ||
77 | * All jobs must allow a pause point before entering their job proper. This | ||
78 | @@ -XXX,XX +XXX,XX @@ static void coroutine_fn job_co_entry(void *opaque) | ||
79 | assert(job && job->driver && job->driver->run); | ||
80 | job_pause_point(job); | ||
81 | job->ret = job->driver->run(job, &job->err); | ||
82 | + if (!job->deferred_to_main_loop) { | ||
83 | + job->deferred_to_main_loop = true; | ||
84 | + aio_bh_schedule_oneshot(qemu_get_aio_context(), | ||
85 | + job_exit, | ||
86 | + job); | ||
87 | + } | ||
88 | } | ||
89 | |||
90 | 25 | ||
91 | -- | 26 | -- |
92 | 2.17.1 | 27 | 2.21.0 |
93 | 28 | ||
94 | 29 | diff view generated by jsdifflib |