1 | The following changes since commit 33d609990621dea6c7d056c86f707b8811320ac1: | 1 | The following changes since commit 23919ddfd56135cad3cb468a8f54d5a595f024f4: |
---|---|---|---|
2 | 2 | ||
3 | Merge remote-tracking branch 'remotes/kevin/tags/for-upstream' into staging (2019-06-18 17:00:52 +0100) | 3 | Merge remote-tracking branch 'remotes/aperard/tags/pull-xen-20190827' into staging (2019-08-27 15:52:36 +0100) |
4 | 4 | ||
5 | are available in the Git repository at: | 5 | are available in the Git repository at: |
6 | 6 | ||
7 | https://github.com/XanClic/qemu.git tags/pull-block-2019-06-21 | 7 | https://github.com/XanClic/qemu.git tags/pull-block-2019-08-27 |
8 | 8 | ||
9 | for you to fetch changes up to e2a76186f7948b8b75d1b2b52638de7c2f7f7472: | 9 | for you to fetch changes up to bb043c056cffcc2f3ce88bfdaf2e76e455c09e2c: |
10 | 10 | ||
11 | iotests: Fix 205 for concurrent runs (2019-06-21 14:40:28 +0200) | 11 | iotests: Unify cache mode quoting (2019-08-27 19:48:44 +0200) |
12 | 12 | ||
13 | ---------------------------------------------------------------- | 13 | ---------------------------------------------------------------- |
14 | Block patches: | 14 | Block patches: |
15 | - The SSH block driver now uses libssh instead of libssh2 | 15 | - qemu-io now accepts a file to read a write pattern from |
16 | - The VMDK block driver gets read-only support for the seSparse | 16 | - Ensure that raw files have their first block allocated so we can probe |
17 | subformat | 17 | the O_DIRECT alignment if necessary |
18 | - Various fixes | 18 | - Various fixes |
19 | 19 | ||
20 | ---------------------------------------------------------------- | 20 | ---------------------------------------------------------------- |
21 | Anton Nefedov (1): | 21 | Denis Plotnikov (1): |
22 | iotest 134: test cluster-misaligned encrypted write | 22 | qemu-io: add pattern file for write command |
23 | 23 | ||
24 | Klaus Birkelund Jensen (1): | 24 | Max Reitz (7): |
25 | nvme: do not advertise support for unsupported arbitration mechanism | 25 | iotests: Fix _filter_img_create() |
26 | vmdk: Use bdrv_dirname() for relative extent paths | ||
27 | iotests: Keep testing broken relative extent paths | ||
28 | vmdk: Reject invalid compressed writes | ||
29 | iotests: Disable broken streamOptimized tests | ||
30 | iotests: Disable 110 for vmdk.twoGbMaxExtentSparse | ||
31 | iotests: Disable 126 for flat vmdk subformats | ||
26 | 32 | ||
27 | Max Reitz (1): | 33 | Nir Soffer (3): |
28 | iotests: Fix 205 for concurrent runs | 34 | block: posix: Always allocate the first block |
35 | iotests: Test allocate_first_block() with O_DIRECT | ||
36 | iotests: Unify cache mode quoting | ||
29 | 37 | ||
30 | Pino Toscano (1): | 38 | Stefan Hajnoczi (1): |
31 | ssh: switch from libssh2 to libssh | 39 | file-posix: fix request_alignment typo |
32 | 40 | ||
33 | Sam Eiderman (3): | 41 | Thomas Huth (2): |
34 | vmdk: Fix comment regarding max l1_size coverage | 42 | iotests: Check for enabled drivers before testing them |
35 | vmdk: Reduce the max bound for L1 table size | 43 | tests/check-block: Skip iotests when sanitizers are enabled |
36 | vmdk: Add read-only support for seSparse snapshots | ||
37 | 44 | ||
38 | Vladimir Sementsov-Ogievskiy (1): | 45 | Vladimir Sementsov-Ogievskiy (1): |
39 | blockdev: enable non-root nodes for transaction drive-backup source | 46 | block: fix permission update in bdrv_replace_node |
40 | 47 | ||
41 | configure | 65 +- | 48 | block.c | 5 +- |
42 | block/Makefile.objs | 6 +- | 49 | block/file-posix.c | 53 +++++++++- |
43 | block/ssh.c | 673 ++++++++++-------- | 50 | block/vmdk.c | 64 ++++++++---- |
44 | block/vmdk.c | 372 +++++++++- | 51 | qemu-io-cmds.c | 99 +++++++++++++++++-- |
45 | blockdev.c | 2 +- | 52 | tests/check-block.sh | 5 + |
46 | hw/block/nvme.c | 1 - | 53 | tests/qemu-iotests/002 | 1 + |
47 | .travis.yml | 4 +- | 54 | tests/qemu-iotests/003 | 1 + |
48 | block/trace-events | 14 +- | 55 | tests/qemu-iotests/005 | 3 +- |
49 | docs/qemu-block-drivers.texi | 2 +- | 56 | tests/qemu-iotests/009 | 1 + |
50 | .../dockerfiles/debian-win32-cross.docker | 1 - | 57 | tests/qemu-iotests/010 | 1 + |
51 | .../dockerfiles/debian-win64-cross.docker | 1 - | 58 | tests/qemu-iotests/011 | 1 + |
52 | tests/docker/dockerfiles/fedora.docker | 4 +- | 59 | tests/qemu-iotests/017 | 3 +- |
53 | tests/docker/dockerfiles/ubuntu.docker | 2 +- | 60 | tests/qemu-iotests/018 | 3 +- |
54 | tests/docker/dockerfiles/ubuntu1804.docker | 2 +- | 61 | tests/qemu-iotests/019 | 3 +- |
55 | tests/qemu-iotests/059.out | 2 +- | 62 | tests/qemu-iotests/020 | 3 +- |
56 | tests/qemu-iotests/134 | 9 + | 63 | tests/qemu-iotests/026 | 4 +- |
57 | tests/qemu-iotests/134.out | 10 + | 64 | tests/qemu-iotests/027 | 1 + |
58 | tests/qemu-iotests/205 | 2 +- | 65 | tests/qemu-iotests/032 | 1 + |
59 | tests/qemu-iotests/207 | 54 +- | 66 | tests/qemu-iotests/033 | 1 + |
60 | tests/qemu-iotests/207.out | 2 +- | 67 | tests/qemu-iotests/034 | 3 +- |
61 | 20 files changed, 844 insertions(+), 384 deletions(-) | 68 | tests/qemu-iotests/037 | 3 +- |
69 | tests/qemu-iotests/039 | 4 +- | ||
70 | tests/qemu-iotests/052 | 2 +- | ||
71 | tests/qemu-iotests/059 | 34 ++++++- | ||
72 | tests/qemu-iotests/059.out | 26 +++-- | ||
73 | tests/qemu-iotests/063 | 3 +- | ||
74 | tests/qemu-iotests/071 | 1 + | ||
75 | tests/qemu-iotests/072 | 1 + | ||
76 | tests/qemu-iotests/081 | 4 +- | ||
77 | tests/qemu-iotests/091 | 4 +- | ||
78 | tests/qemu-iotests/099 | 1 + | ||
79 | tests/qemu-iotests/105 | 3 +- | ||
80 | tests/qemu-iotests/110 | 3 +- | ||
81 | tests/qemu-iotests/120 | 1 + | ||
82 | tests/qemu-iotests/126 | 2 + | ||
83 | tests/qemu-iotests/{150.out => 150.out.qcow2} | 0 | ||
84 | tests/qemu-iotests/150.out.raw | 12 +++ | ||
85 | tests/qemu-iotests/162 | 4 +- | ||
86 | tests/qemu-iotests/175 | 47 +++++++-- | ||
87 | tests/qemu-iotests/175.out | 16 ++- | ||
88 | tests/qemu-iotests/178.out.qcow2 | 4 +- | ||
89 | tests/qemu-iotests/184 | 1 + | ||
90 | tests/qemu-iotests/186 | 1 + | ||
91 | tests/qemu-iotests/197 | 1 + | ||
92 | tests/qemu-iotests/215 | 1 + | ||
93 | tests/qemu-iotests/221.out | 12 ++- | ||
94 | tests/qemu-iotests/251 | 1 + | ||
95 | tests/qemu-iotests/253.out | 12 ++- | ||
96 | tests/qemu-iotests/common.filter | 4 +- | ||
97 | tests/qemu-iotests/common.rc | 14 +++ | ||
98 | 50 files changed, 391 insertions(+), 87 deletions(-) | ||
99 | rename tests/qemu-iotests/{150.out => 150.out.qcow2} (100%) | ||
100 | create mode 100644 tests/qemu-iotests/150.out.raw | ||
62 | 101 | ||
63 | -- | 102 | -- |
64 | 2.21.0 | 103 | 2.21.0 |
65 | 104 | ||
66 | 105 | diff view generated by jsdifflib |
1 | From: Pino Toscano <ptoscano@redhat.com> | 1 | From: Denis Plotnikov <dplotnikov@virtuozzo.com> |
---|---|---|---|
2 | 2 | ||
3 | Rewrite the implementation of the ssh block driver to use libssh instead | 3 | The patch allows to provide a pattern file for write |
4 | of libssh2. The libssh library has various advantages over libssh2: | 4 | command. There was no similar ability before. |
5 | - easier API for authentication (for example for using ssh-agent) | ||
6 | - easier API for known_hosts handling | ||
7 | - supports newer types of keys in known_hosts | ||
8 | 5 | ||
9 | Use APIs/features available in libssh 0.8 conditionally, to support | 6 | Signed-off-by: Denis Plotnikov <dplotnikov@virtuozzo.com> |
10 | older versions (which are not recommended though). | 7 | Message-id: 20190820164616.4072-1-dplotnikov@virtuozzo.com |
11 | 8 | Reviewed-by: Eric Blake <eblake@redhat.com> | |
12 | Adjust the iotest 207 according to the different error message, and to | 9 | [mreitz: Keep optstring in alphabetical order] |
13 | find the default key type for localhost (to properly compare the | ||
14 | fingerprint with). | ||
15 | Contributed-by: Max Reitz <mreitz@redhat.com> | ||
16 | |||
17 | Adjust the various Docker/Travis scripts to use libssh when available | ||
18 | instead of libssh2. The mingw/mxe testing is dropped for now, as there | ||
19 | are no packages for it. | ||
20 | |||
21 | Signed-off-by: Pino Toscano <ptoscano@redhat.com> | ||
22 | Tested-by: Philippe Mathieu-Daudé <philmd@redhat.com> | ||
23 | Acked-by: Alex Bennée <alex.bennee@linaro.org> | ||
24 | Message-id: 20190620200840.17655-1-ptoscano@redhat.com | ||
25 | Reviewed-by: Philippe Mathieu-Daudé <philmd@redhat.com> | ||
26 | Signed-off-by: Max Reitz <mreitz@redhat.com> | 10 | Signed-off-by: Max Reitz <mreitz@redhat.com> |
27 | --- | 11 | --- |
28 | configure | 65 +- | 12 | qemu-io-cmds.c | 99 +++++++++++++++++++++++++++++++++++++++++++++++--- |
29 | block/Makefile.objs | 6 +- | 13 | 1 file changed, 93 insertions(+), 6 deletions(-) |
30 | block/ssh.c | 673 ++++++++++-------- | ||
31 | .travis.yml | 4 +- | ||
32 | block/trace-events | 14 +- | ||
33 | docs/qemu-block-drivers.texi | 2 +- | ||
34 | .../dockerfiles/debian-win32-cross.docker | 1 - | ||
35 | .../dockerfiles/debian-win64-cross.docker | 1 - | ||
36 | tests/docker/dockerfiles/fedora.docker | 4 +- | ||
37 | tests/docker/dockerfiles/ubuntu.docker | 2 +- | ||
38 | tests/docker/dockerfiles/ubuntu1804.docker | 2 +- | ||
39 | tests/qemu-iotests/207 | 54 +- | ||
40 | tests/qemu-iotests/207.out | 2 +- | ||
41 | 13 files changed, 470 insertions(+), 360 deletions(-) | ||
42 | 14 | ||
43 | diff --git a/configure b/configure | 15 | diff --git a/qemu-io-cmds.c b/qemu-io-cmds.c |
44 | index XXXXXXX..XXXXXXX 100755 | ||
45 | --- a/configure | ||
46 | +++ b/configure | ||
47 | @@ -XXX,XX +XXX,XX @@ auth_pam="" | ||
48 | vte="" | ||
49 | virglrenderer="" | ||
50 | tpm="" | ||
51 | -libssh2="" | ||
52 | +libssh="" | ||
53 | live_block_migration="yes" | ||
54 | numa="" | ||
55 | tcmalloc="no" | ||
56 | @@ -XXX,XX +XXX,XX @@ for opt do | ||
57 | ;; | ||
58 | --enable-tpm) tpm="yes" | ||
59 | ;; | ||
60 | - --disable-libssh2) libssh2="no" | ||
61 | + --disable-libssh) libssh="no" | ||
62 | ;; | ||
63 | - --enable-libssh2) libssh2="yes" | ||
64 | + --enable-libssh) libssh="yes" | ||
65 | ;; | ||
66 | --disable-live-block-migration) live_block_migration="no" | ||
67 | ;; | ||
68 | @@ -XXX,XX +XXX,XX @@ disabled with --disable-FEATURE, default is enabled if available: | ||
69 | coroutine-pool coroutine freelist (better performance) | ||
70 | glusterfs GlusterFS backend | ||
71 | tpm TPM support | ||
72 | - libssh2 ssh block device support | ||
73 | + libssh ssh block device support | ||
74 | numa libnuma support | ||
75 | libxml2 for Parallels image format | ||
76 | tcmalloc tcmalloc support | ||
77 | @@ -XXX,XX +XXX,XX @@ EOF | ||
78 | fi | ||
79 | |||
80 | ########################################## | ||
81 | -# libssh2 probe | ||
82 | -min_libssh2_version=1.2.8 | ||
83 | -if test "$libssh2" != "no" ; then | ||
84 | - if $pkg_config --atleast-version=$min_libssh2_version libssh2; then | ||
85 | - libssh2_cflags=$($pkg_config libssh2 --cflags) | ||
86 | - libssh2_libs=$($pkg_config libssh2 --libs) | ||
87 | - libssh2=yes | ||
88 | +# libssh probe | ||
89 | +if test "$libssh" != "no" ; then | ||
90 | + if $pkg_config --exists libssh; then | ||
91 | + libssh_cflags=$($pkg_config libssh --cflags) | ||
92 | + libssh_libs=$($pkg_config libssh --libs) | ||
93 | + libssh=yes | ||
94 | else | ||
95 | - if test "$libssh2" = "yes" ; then | ||
96 | - error_exit "libssh2 >= $min_libssh2_version required for --enable-libssh2" | ||
97 | + if test "$libssh" = "yes" ; then | ||
98 | + error_exit "libssh required for --enable-libssh" | ||
99 | fi | ||
100 | - libssh2=no | ||
101 | + libssh=no | ||
102 | fi | ||
103 | fi | ||
104 | |||
105 | ########################################## | ||
106 | -# libssh2_sftp_fsync probe | ||
107 | +# Check for libssh 0.8 | ||
108 | +# This is done like this instead of using the LIBSSH_VERSION_* and | ||
109 | +# SSH_VERSION_* macros because some distributions in the past shipped | ||
110 | +# snapshots of the future 0.8 from Git, and those snapshots did not | ||
111 | +# have updated version numbers (still referring to 0.7.0). | ||
112 | |||
113 | -if test "$libssh2" = "yes"; then | ||
114 | +if test "$libssh" = "yes"; then | ||
115 | cat > $TMPC <<EOF | ||
116 | -#include <stdio.h> | ||
117 | -#include <libssh2.h> | ||
118 | -#include <libssh2_sftp.h> | ||
119 | -int main(void) { | ||
120 | - LIBSSH2_SESSION *session; | ||
121 | - LIBSSH2_SFTP *sftp; | ||
122 | - LIBSSH2_SFTP_HANDLE *sftp_handle; | ||
123 | - session = libssh2_session_init (); | ||
124 | - sftp = libssh2_sftp_init (session); | ||
125 | - sftp_handle = libssh2_sftp_open (sftp, "/", 0, 0); | ||
126 | - libssh2_sftp_fsync (sftp_handle); | ||
127 | - return 0; | ||
128 | -} | ||
129 | +#include <libssh/libssh.h> | ||
130 | +int main(void) { return ssh_get_server_publickey(NULL, NULL); } | ||
131 | EOF | ||
132 | - # libssh2_cflags/libssh2_libs defined in previous test. | ||
133 | - if compile_prog "$libssh2_cflags" "$libssh2_libs" ; then | ||
134 | - QEMU_CFLAGS="-DHAS_LIBSSH2_SFTP_FSYNC $QEMU_CFLAGS" | ||
135 | + if compile_prog "$libssh_cflags" "$libssh_libs"; then | ||
136 | + libssh_cflags="-DHAVE_LIBSSH_0_8 $libssh_cflags" | ||
137 | fi | ||
138 | fi | ||
139 | |||
140 | @@ -XXX,XX +XXX,XX @@ echo "GlusterFS support $glusterfs" | ||
141 | echo "gcov $gcov_tool" | ||
142 | echo "gcov enabled $gcov" | ||
143 | echo "TPM support $tpm" | ||
144 | -echo "libssh2 support $libssh2" | ||
145 | +echo "libssh support $libssh" | ||
146 | echo "QOM debugging $qom_cast_debug" | ||
147 | echo "Live block migration $live_block_migration" | ||
148 | echo "lzo support $lzo" | ||
149 | @@ -XXX,XX +XXX,XX @@ if test "$glusterfs_iocb_has_stat" = "yes" ; then | ||
150 | echo "CONFIG_GLUSTERFS_IOCB_HAS_STAT=y" >> $config_host_mak | ||
151 | fi | ||
152 | |||
153 | -if test "$libssh2" = "yes" ; then | ||
154 | - echo "CONFIG_LIBSSH2=m" >> $config_host_mak | ||
155 | - echo "LIBSSH2_CFLAGS=$libssh2_cflags" >> $config_host_mak | ||
156 | - echo "LIBSSH2_LIBS=$libssh2_libs" >> $config_host_mak | ||
157 | +if test "$libssh" = "yes" ; then | ||
158 | + echo "CONFIG_LIBSSH=m" >> $config_host_mak | ||
159 | + echo "LIBSSH_CFLAGS=$libssh_cflags" >> $config_host_mak | ||
160 | + echo "LIBSSH_LIBS=$libssh_libs" >> $config_host_mak | ||
161 | fi | ||
162 | |||
163 | if test "$live_block_migration" = "yes" ; then | ||
164 | diff --git a/block/Makefile.objs b/block/Makefile.objs | ||
165 | index XXXXXXX..XXXXXXX 100644 | 16 | index XXXXXXX..XXXXXXX 100644 |
166 | --- a/block/Makefile.objs | 17 | --- a/qemu-io-cmds.c |
167 | +++ b/block/Makefile.objs | 18 | +++ b/qemu-io-cmds.c |
168 | @@ -XXX,XX +XXX,XX @@ block-obj-$(CONFIG_CURL) += curl.o | 19 | @@ -XXX,XX +XXX,XX @@ static void qemu_io_free(void *p) |
169 | block-obj-$(CONFIG_RBD) += rbd.o | 20 | qemu_vfree(p); |
170 | block-obj-$(CONFIG_GLUSTERFS) += gluster.o | ||
171 | block-obj-$(CONFIG_VXHS) += vxhs.o | ||
172 | -block-obj-$(CONFIG_LIBSSH2) += ssh.o | ||
173 | +block-obj-$(CONFIG_LIBSSH) += ssh.o | ||
174 | block-obj-y += accounting.o dirty-bitmap.o | ||
175 | block-obj-y += write-threshold.o | ||
176 | block-obj-y += backup.o | ||
177 | @@ -XXX,XX +XXX,XX @@ rbd.o-libs := $(RBD_LIBS) | ||
178 | gluster.o-cflags := $(GLUSTERFS_CFLAGS) | ||
179 | gluster.o-libs := $(GLUSTERFS_LIBS) | ||
180 | vxhs.o-libs := $(VXHS_LIBS) | ||
181 | -ssh.o-cflags := $(LIBSSH2_CFLAGS) | ||
182 | -ssh.o-libs := $(LIBSSH2_LIBS) | ||
183 | +ssh.o-cflags := $(LIBSSH_CFLAGS) | ||
184 | +ssh.o-libs := $(LIBSSH_LIBS) | ||
185 | block-obj-dmg-bz2-$(CONFIG_BZIP2) += dmg-bz2.o | ||
186 | block-obj-$(if $(CONFIG_DMG),m,n) += $(block-obj-dmg-bz2-y) | ||
187 | dmg-bz2.o-libs := $(BZIP2_LIBS) | ||
188 | diff --git a/block/ssh.c b/block/ssh.c | ||
189 | index XXXXXXX..XXXXXXX 100644 | ||
190 | --- a/block/ssh.c | ||
191 | +++ b/block/ssh.c | ||
192 | @@ -XXX,XX +XXX,XX @@ | ||
193 | |||
194 | #include "qemu/osdep.h" | ||
195 | |||
196 | -#include <libssh2.h> | ||
197 | -#include <libssh2_sftp.h> | ||
198 | +#include <libssh/libssh.h> | ||
199 | +#include <libssh/sftp.h> | ||
200 | |||
201 | #include "block/block_int.h" | ||
202 | #include "block/qdict.h" | ||
203 | @@ -XXX,XX +XXX,XX @@ | ||
204 | #include "trace.h" | ||
205 | |||
206 | /* | ||
207 | - * TRACE_LIBSSH2=<bitmask> enables tracing in libssh2 itself. Note | ||
208 | - * that this requires that libssh2 was specially compiled with the | ||
209 | - * `./configure --enable-debug' option, so most likely you will have | ||
210 | - * to compile it yourself. The meaning of <bitmask> is described | ||
211 | - * here: http://www.libssh2.org/libssh2_trace.html | ||
212 | + * TRACE_LIBSSH=<level> enables tracing in libssh itself. | ||
213 | + * The meaning of <level> is described here: | ||
214 | + * http://api.libssh.org/master/group__libssh__log.html | ||
215 | */ | ||
216 | -#define TRACE_LIBSSH2 0 /* or try: LIBSSH2_TRACE_SFTP */ | ||
217 | +#define TRACE_LIBSSH 0 /* see: SSH_LOG_* */ | ||
218 | |||
219 | typedef struct BDRVSSHState { | ||
220 | /* Coroutine. */ | ||
221 | @@ -XXX,XX +XXX,XX @@ typedef struct BDRVSSHState { | ||
222 | |||
223 | /* SSH connection. */ | ||
224 | int sock; /* socket */ | ||
225 | - LIBSSH2_SESSION *session; /* ssh session */ | ||
226 | - LIBSSH2_SFTP *sftp; /* sftp session */ | ||
227 | - LIBSSH2_SFTP_HANDLE *sftp_handle; /* sftp remote file handle */ | ||
228 | + ssh_session session; /* ssh session */ | ||
229 | + sftp_session sftp; /* sftp session */ | ||
230 | + sftp_file sftp_handle; /* sftp remote file handle */ | ||
231 | |||
232 | - /* See ssh_seek() function below. */ | ||
233 | - int64_t offset; | ||
234 | - bool offset_op_read; | ||
235 | - | ||
236 | - /* File attributes at open. We try to keep the .filesize field | ||
237 | + /* | ||
238 | + * File attributes at open. We try to keep the .size field | ||
239 | * updated if it changes (eg by writing at the end of the file). | ||
240 | */ | ||
241 | - LIBSSH2_SFTP_ATTRIBUTES attrs; | ||
242 | + sftp_attributes attrs; | ||
243 | |||
244 | InetSocketAddress *inet; | ||
245 | |||
246 | @@ -XXX,XX +XXX,XX @@ static void ssh_state_init(BDRVSSHState *s) | ||
247 | { | ||
248 | memset(s, 0, sizeof *s); | ||
249 | s->sock = -1; | ||
250 | - s->offset = -1; | ||
251 | qemu_co_mutex_init(&s->lock); | ||
252 | } | 21 | } |
253 | 22 | ||
254 | @@ -XXX,XX +XXX,XX @@ static void ssh_state_free(BDRVSSHState *s) | 23 | +/* |
255 | { | 24 | + * qemu_io_alloc_from_file() |
256 | g_free(s->user); | 25 | + * |
257 | 26 | + * Allocates the buffer and populates it with the content of the given file | |
258 | + if (s->attrs) { | 27 | + * up to @len bytes. If the file length is less than @len, then the buffer |
259 | + sftp_attributes_free(s->attrs); | 28 | + * is populated with the file content cyclically. |
260 | + } | 29 | + * |
261 | if (s->sftp_handle) { | 30 | + * @blk - the block backend where the buffer content is going to be written to |
262 | - libssh2_sftp_close(s->sftp_handle); | 31 | + * @len - the buffer length |
263 | + sftp_close(s->sftp_handle); | 32 | + * @file_name - the file to read the content from |
264 | } | 33 | + * |
265 | if (s->sftp) { | 34 | + * Returns: the buffer pointer on success |
266 | - libssh2_sftp_shutdown(s->sftp); | 35 | + * NULL on error |
267 | + sftp_free(s->sftp); | 36 | + */ |
268 | } | 37 | +static void *qemu_io_alloc_from_file(BlockBackend *blk, size_t len, |
269 | if (s->session) { | 38 | + const char *file_name) |
270 | - libssh2_session_disconnect(s->session, | 39 | +{ |
271 | - "from qemu ssh client: " | 40 | + char *buf, *buf_origin; |
272 | - "user closed the connection"); | 41 | + FILE *f = fopen(file_name, "r"); |
273 | - libssh2_session_free(s->session); | 42 | + int pattern_len; |
274 | - } | ||
275 | - if (s->sock >= 0) { | ||
276 | - close(s->sock); | ||
277 | + ssh_disconnect(s->session); | ||
278 | + ssh_free(s->session); /* This frees s->sock */ | ||
279 | } | ||
280 | } | ||
281 | |||
282 | @@ -XXX,XX +XXX,XX @@ session_error_setg(Error **errp, BDRVSSHState *s, const char *fs, ...) | ||
283 | va_end(args); | ||
284 | |||
285 | if (s->session) { | ||
286 | - char *ssh_err; | ||
287 | + const char *ssh_err; | ||
288 | int ssh_err_code; | ||
289 | |||
290 | - /* This is not an errno. See <libssh2.h>. */ | ||
291 | - ssh_err_code = libssh2_session_last_error(s->session, | ||
292 | - &ssh_err, NULL, 0); | ||
293 | - error_setg(errp, "%s: %s (libssh2 error code: %d)", | ||
294 | + /* This is not an errno. See <libssh/libssh.h>. */ | ||
295 | + ssh_err = ssh_get_error(s->session); | ||
296 | + ssh_err_code = ssh_get_error_code(s->session); | ||
297 | + error_setg(errp, "%s: %s (libssh error code: %d)", | ||
298 | msg, ssh_err, ssh_err_code); | ||
299 | } else { | ||
300 | error_setg(errp, "%s", msg); | ||
301 | @@ -XXX,XX +XXX,XX @@ sftp_error_setg(Error **errp, BDRVSSHState *s, const char *fs, ...) | ||
302 | va_end(args); | ||
303 | |||
304 | if (s->sftp) { | ||
305 | - char *ssh_err; | ||
306 | + const char *ssh_err; | ||
307 | int ssh_err_code; | ||
308 | - unsigned long sftp_err_code; | ||
309 | + int sftp_err_code; | ||
310 | |||
311 | - /* This is not an errno. See <libssh2.h>. */ | ||
312 | - ssh_err_code = libssh2_session_last_error(s->session, | ||
313 | - &ssh_err, NULL, 0); | ||
314 | - /* See <libssh2_sftp.h>. */ | ||
315 | - sftp_err_code = libssh2_sftp_last_error((s)->sftp); | ||
316 | + /* This is not an errno. See <libssh/libssh.h>. */ | ||
317 | + ssh_err = ssh_get_error(s->session); | ||
318 | + ssh_err_code = ssh_get_error_code(s->session); | ||
319 | + /* See <libssh/sftp.h>. */ | ||
320 | + sftp_err_code = sftp_get_error(s->sftp); | ||
321 | |||
322 | error_setg(errp, | ||
323 | - "%s: %s (libssh2 error code: %d, sftp error code: %lu)", | ||
324 | + "%s: %s (libssh error code: %d, sftp error code: %d)", | ||
325 | msg, ssh_err, ssh_err_code, sftp_err_code); | ||
326 | } else { | ||
327 | error_setg(errp, "%s", msg); | ||
328 | @@ -XXX,XX +XXX,XX @@ sftp_error_setg(Error **errp, BDRVSSHState *s, const char *fs, ...) | ||
329 | |||
330 | static void sftp_error_trace(BDRVSSHState *s, const char *op) | ||
331 | { | ||
332 | - char *ssh_err; | ||
333 | + const char *ssh_err; | ||
334 | int ssh_err_code; | ||
335 | - unsigned long sftp_err_code; | ||
336 | + int sftp_err_code; | ||
337 | |||
338 | - /* This is not an errno. See <libssh2.h>. */ | ||
339 | - ssh_err_code = libssh2_session_last_error(s->session, | ||
340 | - &ssh_err, NULL, 0); | ||
341 | - /* See <libssh2_sftp.h>. */ | ||
342 | - sftp_err_code = libssh2_sftp_last_error((s)->sftp); | ||
343 | + /* This is not an errno. See <libssh/libssh.h>. */ | ||
344 | + ssh_err = ssh_get_error(s->session); | ||
345 | + ssh_err_code = ssh_get_error_code(s->session); | ||
346 | + /* See <libssh/sftp.h>. */ | ||
347 | + sftp_err_code = sftp_get_error(s->sftp); | ||
348 | |||
349 | trace_sftp_error(op, ssh_err, ssh_err_code, sftp_err_code); | ||
350 | } | ||
351 | @@ -XXX,XX +XXX,XX @@ static void ssh_parse_filename(const char *filename, QDict *options, | ||
352 | parse_uri(filename, options, errp); | ||
353 | } | ||
354 | |||
355 | -static int check_host_key_knownhosts(BDRVSSHState *s, | ||
356 | - const char *host, int port, Error **errp) | ||
357 | +static int check_host_key_knownhosts(BDRVSSHState *s, Error **errp) | ||
358 | { | ||
359 | - const char *home; | ||
360 | - char *knh_file = NULL; | ||
361 | - LIBSSH2_KNOWNHOSTS *knh = NULL; | ||
362 | - struct libssh2_knownhost *found; | ||
363 | - int ret, r; | ||
364 | - const char *hostkey; | ||
365 | - size_t len; | ||
366 | - int type; | ||
367 | - | ||
368 | - hostkey = libssh2_session_hostkey(s->session, &len, &type); | ||
369 | - if (!hostkey) { | ||
370 | + int ret; | ||
371 | + int r; | ||
372 | + ssh_key pubkey; | ||
373 | + enum ssh_keytypes_e pubkey_type; | ||
374 | + unsigned char *server_hash = NULL; | ||
375 | + size_t server_hash_len; | ||
376 | + char *fingerprint = NULL; | ||
377 | +#ifdef HAVE_LIBSSH_0_8 | ||
378 | + enum ssh_known_hosts_e state; | ||
379 | + | 43 | + |
380 | + state = ssh_session_is_known_server(s->session); | 44 | + if (!f) { |
381 | + trace_ssh_server_status(state); | 45 | + perror(file_name); |
382 | + | 46 | + return NULL; |
383 | + switch (state) { | ||
384 | + case SSH_KNOWN_HOSTS_OK: | ||
385 | + /* OK */ | ||
386 | + trace_ssh_check_host_key_knownhosts(); | ||
387 | + break; | ||
388 | + case SSH_KNOWN_HOSTS_CHANGED: | ||
389 | ret = -EINVAL; | ||
390 | - session_error_setg(errp, s, "failed to read remote host key"); | ||
391 | + r = ssh_get_server_publickey(s->session, &pubkey); | ||
392 | + if (r == 0) { | ||
393 | + r = ssh_get_publickey_hash(pubkey, SSH_PUBLICKEY_HASH_SHA256, | ||
394 | + &server_hash, &server_hash_len); | ||
395 | + pubkey_type = ssh_key_type(pubkey); | ||
396 | + ssh_key_free(pubkey); | ||
397 | + } | ||
398 | + if (r == 0) { | ||
399 | + fingerprint = ssh_get_fingerprint_hash(SSH_PUBLICKEY_HASH_SHA256, | ||
400 | + server_hash, | ||
401 | + server_hash_len); | ||
402 | + ssh_clean_pubkey_hash(&server_hash); | ||
403 | + } | ||
404 | + if (fingerprint) { | ||
405 | + error_setg(errp, | ||
406 | + "host key (%s key with fingerprint %s) does not match " | ||
407 | + "the one in known_hosts; this may be a possible attack", | ||
408 | + ssh_key_type_to_char(pubkey_type), fingerprint); | ||
409 | + ssh_string_free_char(fingerprint); | ||
410 | + } else { | ||
411 | + error_setg(errp, | ||
412 | + "host key does not match the one in known_hosts; this " | ||
413 | + "may be a possible attack"); | ||
414 | + } | ||
415 | goto out; | ||
416 | - } | ||
417 | - | ||
418 | - knh = libssh2_knownhost_init(s->session); | ||
419 | - if (!knh) { | ||
420 | + case SSH_KNOWN_HOSTS_OTHER: | ||
421 | ret = -EINVAL; | ||
422 | - session_error_setg(errp, s, | ||
423 | - "failed to initialize known hosts support"); | ||
424 | + error_setg(errp, | ||
425 | + "host key for this server not found, another type exists"); | ||
426 | + goto out; | ||
427 | + case SSH_KNOWN_HOSTS_UNKNOWN: | ||
428 | + ret = -EINVAL; | ||
429 | + error_setg(errp, "no host key was found in known_hosts"); | ||
430 | + goto out; | ||
431 | + case SSH_KNOWN_HOSTS_NOT_FOUND: | ||
432 | + ret = -ENOENT; | ||
433 | + error_setg(errp, "known_hosts file not found"); | ||
434 | + goto out; | ||
435 | + case SSH_KNOWN_HOSTS_ERROR: | ||
436 | + ret = -EINVAL; | ||
437 | + error_setg(errp, "error while checking the host"); | ||
438 | + goto out; | ||
439 | + default: | ||
440 | + ret = -EINVAL; | ||
441 | + error_setg(errp, "error while checking for known server (%d)", state); | ||
442 | goto out; | ||
443 | } | ||
444 | +#else /* !HAVE_LIBSSH_0_8 */ | ||
445 | + int state; | ||
446 | |||
447 | - home = getenv("HOME"); | ||
448 | - if (home) { | ||
449 | - knh_file = g_strdup_printf("%s/.ssh/known_hosts", home); | ||
450 | - } else { | ||
451 | - knh_file = g_strdup_printf("/root/.ssh/known_hosts"); | ||
452 | - } | ||
453 | - | ||
454 | - /* Read all known hosts from OpenSSH-style known_hosts file. */ | ||
455 | - libssh2_knownhost_readfile(knh, knh_file, LIBSSH2_KNOWNHOST_FILE_OPENSSH); | ||
456 | + state = ssh_is_server_known(s->session); | ||
457 | + trace_ssh_server_status(state); | ||
458 | |||
459 | - r = libssh2_knownhost_checkp(knh, host, port, hostkey, len, | ||
460 | - LIBSSH2_KNOWNHOST_TYPE_PLAIN| | ||
461 | - LIBSSH2_KNOWNHOST_KEYENC_RAW, | ||
462 | - &found); | ||
463 | - switch (r) { | ||
464 | - case LIBSSH2_KNOWNHOST_CHECK_MATCH: | ||
465 | + switch (state) { | ||
466 | + case SSH_SERVER_KNOWN_OK: | ||
467 | /* OK */ | ||
468 | - trace_ssh_check_host_key_knownhosts(found->key); | ||
469 | + trace_ssh_check_host_key_knownhosts(); | ||
470 | break; | ||
471 | - case LIBSSH2_KNOWNHOST_CHECK_MISMATCH: | ||
472 | + case SSH_SERVER_KNOWN_CHANGED: | ||
473 | ret = -EINVAL; | ||
474 | - session_error_setg(errp, s, | ||
475 | - "host key does not match the one in known_hosts" | ||
476 | - " (found key %s)", found->key); | ||
477 | + r = ssh_get_publickey(s->session, &pubkey); | ||
478 | + if (r == 0) { | ||
479 | + r = ssh_get_publickey_hash(pubkey, SSH_PUBLICKEY_HASH_SHA1, | ||
480 | + &server_hash, &server_hash_len); | ||
481 | + pubkey_type = ssh_key_type(pubkey); | ||
482 | + ssh_key_free(pubkey); | ||
483 | + } | ||
484 | + if (r == 0) { | ||
485 | + fingerprint = ssh_get_fingerprint_hash(SSH_PUBLICKEY_HASH_SHA1, | ||
486 | + server_hash, | ||
487 | + server_hash_len); | ||
488 | + ssh_clean_pubkey_hash(&server_hash); | ||
489 | + } | ||
490 | + if (fingerprint) { | ||
491 | + error_setg(errp, | ||
492 | + "host key (%s key with fingerprint %s) does not match " | ||
493 | + "the one in known_hosts; this may be a possible attack", | ||
494 | + ssh_key_type_to_char(pubkey_type), fingerprint); | ||
495 | + ssh_string_free_char(fingerprint); | ||
496 | + } else { | ||
497 | + error_setg(errp, | ||
498 | + "host key does not match the one in known_hosts; this " | ||
499 | + "may be a possible attack"); | ||
500 | + } | ||
501 | + goto out; | ||
502 | + case SSH_SERVER_FOUND_OTHER: | ||
503 | + ret = -EINVAL; | ||
504 | + error_setg(errp, | ||
505 | + "host key for this server not found, another type exists"); | ||
506 | + goto out; | ||
507 | + case SSH_SERVER_FILE_NOT_FOUND: | ||
508 | + ret = -ENOENT; | ||
509 | + error_setg(errp, "known_hosts file not found"); | ||
510 | goto out; | ||
511 | - case LIBSSH2_KNOWNHOST_CHECK_NOTFOUND: | ||
512 | + case SSH_SERVER_NOT_KNOWN: | ||
513 | ret = -EINVAL; | ||
514 | - session_error_setg(errp, s, "no host key was found in known_hosts"); | ||
515 | + error_setg(errp, "no host key was found in known_hosts"); | ||
516 | goto out; | ||
517 | - case LIBSSH2_KNOWNHOST_CHECK_FAILURE: | ||
518 | + case SSH_SERVER_ERROR: | ||
519 | ret = -EINVAL; | ||
520 | - session_error_setg(errp, s, | ||
521 | - "failure matching the host key with known_hosts"); | ||
522 | + error_setg(errp, "server error"); | ||
523 | goto out; | ||
524 | default: | ||
525 | ret = -EINVAL; | ||
526 | - session_error_setg(errp, s, "unknown error matching the host key" | ||
527 | - " with known_hosts (%d)", r); | ||
528 | + error_setg(errp, "error while checking for known server (%d)", state); | ||
529 | goto out; | ||
530 | } | ||
531 | +#endif /* !HAVE_LIBSSH_0_8 */ | ||
532 | |||
533 | /* known_hosts checking successful. */ | ||
534 | ret = 0; | ||
535 | |||
536 | out: | ||
537 | - if (knh != NULL) { | ||
538 | - libssh2_knownhost_free(knh); | ||
539 | - } | ||
540 | - g_free(knh_file); | ||
541 | return ret; | ||
542 | } | ||
543 | |||
544 | @@ -XXX,XX +XXX,XX @@ static int compare_fingerprint(const unsigned char *fingerprint, size_t len, | ||
545 | |||
546 | static int | ||
547 | check_host_key_hash(BDRVSSHState *s, const char *hash, | ||
548 | - int hash_type, size_t fingerprint_len, Error **errp) | ||
549 | + enum ssh_publickey_hash_type type, Error **errp) | ||
550 | { | ||
551 | - const char *fingerprint; | ||
552 | - | ||
553 | - fingerprint = libssh2_hostkey_hash(s->session, hash_type); | ||
554 | - if (!fingerprint) { | ||
555 | + int r; | ||
556 | + ssh_key pubkey; | ||
557 | + unsigned char *server_hash; | ||
558 | + size_t server_hash_len; | ||
559 | + | ||
560 | +#ifdef HAVE_LIBSSH_0_8 | ||
561 | + r = ssh_get_server_publickey(s->session, &pubkey); | ||
562 | +#else | ||
563 | + r = ssh_get_publickey(s->session, &pubkey); | ||
564 | +#endif | ||
565 | + if (r != SSH_OK) { | ||
566 | session_error_setg(errp, s, "failed to read remote host key"); | ||
567 | return -EINVAL; | ||
568 | } | ||
569 | |||
570 | - if(compare_fingerprint((unsigned char *) fingerprint, fingerprint_len, | ||
571 | - hash) != 0) { | ||
572 | + r = ssh_get_publickey_hash(pubkey, type, &server_hash, &server_hash_len); | ||
573 | + ssh_key_free(pubkey); | ||
574 | + if (r != 0) { | ||
575 | + session_error_setg(errp, s, | ||
576 | + "failed reading the hash of the server SSH key"); | ||
577 | + return -EINVAL; | ||
578 | + } | 47 | + } |
579 | + | 48 | + |
580 | + r = compare_fingerprint(server_hash, server_hash_len, hash); | 49 | + if (qemuio_misalign) { |
581 | + ssh_clean_pubkey_hash(&server_hash); | 50 | + len += MISALIGN_OFFSET; |
582 | + if (r != 0) { | ||
583 | error_setg(errp, "remote host key does not match host_key_check '%s'", | ||
584 | hash); | ||
585 | return -EPERM; | ||
586 | @@ -XXX,XX +XXX,XX @@ check_host_key_hash(BDRVSSHState *s, const char *hash, | ||
587 | return 0; | ||
588 | } | ||
589 | |||
590 | -static int check_host_key(BDRVSSHState *s, const char *host, int port, | ||
591 | - SshHostKeyCheck *hkc, Error **errp) | ||
592 | +static int check_host_key(BDRVSSHState *s, SshHostKeyCheck *hkc, Error **errp) | ||
593 | { | ||
594 | SshHostKeyCheckMode mode; | ||
595 | |||
596 | @@ -XXX,XX +XXX,XX @@ static int check_host_key(BDRVSSHState *s, const char *host, int port, | ||
597 | case SSH_HOST_KEY_CHECK_MODE_HASH: | ||
598 | if (hkc->u.hash.type == SSH_HOST_KEY_CHECK_HASH_TYPE_MD5) { | ||
599 | return check_host_key_hash(s, hkc->u.hash.hash, | ||
600 | - LIBSSH2_HOSTKEY_HASH_MD5, 16, errp); | ||
601 | + SSH_PUBLICKEY_HASH_MD5, errp); | ||
602 | } else if (hkc->u.hash.type == SSH_HOST_KEY_CHECK_HASH_TYPE_SHA1) { | ||
603 | return check_host_key_hash(s, hkc->u.hash.hash, | ||
604 | - LIBSSH2_HOSTKEY_HASH_SHA1, 20, errp); | ||
605 | + SSH_PUBLICKEY_HASH_SHA1, errp); | ||
606 | } | ||
607 | g_assert_not_reached(); | ||
608 | break; | ||
609 | case SSH_HOST_KEY_CHECK_MODE_KNOWN_HOSTS: | ||
610 | - return check_host_key_knownhosts(s, host, port, errp); | ||
611 | + return check_host_key_knownhosts(s, errp); | ||
612 | default: | ||
613 | g_assert_not_reached(); | ||
614 | } | ||
615 | @@ -XXX,XX +XXX,XX @@ static int check_host_key(BDRVSSHState *s, const char *host, int port, | ||
616 | return -EINVAL; | ||
617 | } | ||
618 | |||
619 | -static int authenticate(BDRVSSHState *s, const char *user, Error **errp) | ||
620 | +static int authenticate(BDRVSSHState *s, Error **errp) | ||
621 | { | ||
622 | int r, ret; | ||
623 | - const char *userauthlist; | ||
624 | - LIBSSH2_AGENT *agent = NULL; | ||
625 | - struct libssh2_agent_publickey *identity; | ||
626 | - struct libssh2_agent_publickey *prev_identity = NULL; | ||
627 | + int method; | ||
628 | |||
629 | - userauthlist = libssh2_userauth_list(s->session, user, strlen(user)); | ||
630 | - if (strstr(userauthlist, "publickey") == NULL) { | ||
631 | + /* Try to authenticate with the "none" method. */ | ||
632 | + r = ssh_userauth_none(s->session, NULL); | ||
633 | + if (r == SSH_AUTH_ERROR) { | ||
634 | ret = -EPERM; | ||
635 | - error_setg(errp, | ||
636 | - "remote server does not support \"publickey\" authentication"); | ||
637 | + session_error_setg(errp, s, "failed to authenticate using none " | ||
638 | + "authentication"); | ||
639 | goto out; | ||
640 | - } | ||
641 | - | ||
642 | - /* Connect to ssh-agent and try each identity in turn. */ | ||
643 | - agent = libssh2_agent_init(s->session); | ||
644 | - if (!agent) { | ||
645 | - ret = -EINVAL; | ||
646 | - session_error_setg(errp, s, "failed to initialize ssh-agent support"); | ||
647 | - goto out; | ||
648 | - } | ||
649 | - if (libssh2_agent_connect(agent)) { | ||
650 | - ret = -ECONNREFUSED; | ||
651 | - session_error_setg(errp, s, "failed to connect to ssh-agent"); | ||
652 | - goto out; | ||
653 | - } | ||
654 | - if (libssh2_agent_list_identities(agent)) { | ||
655 | - ret = -EINVAL; | ||
656 | - session_error_setg(errp, s, | ||
657 | - "failed requesting identities from ssh-agent"); | ||
658 | + } else if (r == SSH_AUTH_SUCCESS) { | ||
659 | + /* Authenticated! */ | ||
660 | + ret = 0; | ||
661 | goto out; | ||
662 | } | ||
663 | |||
664 | - for(;;) { | ||
665 | - r = libssh2_agent_get_identity(agent, &identity, prev_identity); | ||
666 | - if (r == 1) { /* end of list */ | ||
667 | - break; | ||
668 | - } | ||
669 | - if (r < 0) { | ||
670 | + method = ssh_userauth_list(s->session, NULL); | ||
671 | + trace_ssh_auth_methods(method); | ||
672 | + | ||
673 | + /* | ||
674 | + * Try to authenticate with publickey, using the ssh-agent | ||
675 | + * if available. | ||
676 | + */ | ||
677 | + if (method & SSH_AUTH_METHOD_PUBLICKEY) { | ||
678 | + r = ssh_userauth_publickey_auto(s->session, NULL, NULL); | ||
679 | + if (r == SSH_AUTH_ERROR) { | ||
680 | ret = -EINVAL; | ||
681 | - session_error_setg(errp, s, | ||
682 | - "failed to obtain identity from ssh-agent"); | ||
683 | + session_error_setg(errp, s, "failed to authenticate using " | ||
684 | + "publickey authentication"); | ||
685 | goto out; | ||
686 | - } | ||
687 | - r = libssh2_agent_userauth(agent, user, identity); | ||
688 | - if (r == 0) { | ||
689 | + } else if (r == SSH_AUTH_SUCCESS) { | ||
690 | /* Authenticated! */ | ||
691 | ret = 0; | ||
692 | goto out; | ||
693 | } | ||
694 | - /* Failed to authenticate with this identity, try the next one. */ | ||
695 | - prev_identity = identity; | ||
696 | } | ||
697 | |||
698 | ret = -EPERM; | ||
699 | @@ -XXX,XX +XXX,XX @@ static int authenticate(BDRVSSHState *s, const char *user, Error **errp) | ||
700 | "and the identities held by your ssh-agent"); | ||
701 | |||
702 | out: | ||
703 | - if (agent != NULL) { | ||
704 | - /* Note: libssh2 implementation implicitly calls | ||
705 | - * libssh2_agent_disconnect if necessary. | ||
706 | - */ | ||
707 | - libssh2_agent_free(agent); | ||
708 | - } | ||
709 | - | ||
710 | return ret; | ||
711 | } | ||
712 | |||
713 | @@ -XXX,XX +XXX,XX @@ static int connect_to_ssh(BDRVSSHState *s, BlockdevOptionsSsh *opts, | ||
714 | int ssh_flags, int creat_mode, Error **errp) | ||
715 | { | ||
716 | int r, ret; | ||
717 | - long port = 0; | ||
718 | + unsigned int port = 0; | ||
719 | + int new_sock = -1; | ||
720 | |||
721 | if (opts->has_user) { | ||
722 | s->user = g_strdup(opts->user); | ||
723 | @@ -XXX,XX +XXX,XX @@ static int connect_to_ssh(BDRVSSHState *s, BlockdevOptionsSsh *opts, | ||
724 | s->inet = opts->server; | ||
725 | opts->server = NULL; | ||
726 | |||
727 | - if (qemu_strtol(s->inet->port, NULL, 10, &port) < 0) { | ||
728 | + if (qemu_strtoui(s->inet->port, NULL, 10, &port) < 0) { | ||
729 | error_setg(errp, "Use only numeric port value"); | ||
730 | ret = -EINVAL; | ||
731 | goto err; | ||
732 | } | ||
733 | |||
734 | /* Open the socket and connect. */ | ||
735 | - s->sock = inet_connect_saddr(s->inet, errp); | ||
736 | - if (s->sock < 0) { | ||
737 | + new_sock = inet_connect_saddr(s->inet, errp); | ||
738 | + if (new_sock < 0) { | ||
739 | ret = -EIO; | ||
740 | goto err; | ||
741 | } | ||
742 | |||
743 | + /* | ||
744 | + * Try to disable the Nagle algorithm on TCP sockets to reduce latency, | ||
745 | + * but do not fail if it cannot be disabled. | ||
746 | + */ | ||
747 | + r = socket_set_nodelay(new_sock); | ||
748 | + if (r < 0) { | ||
749 | + warn_report("can't set TCP_NODELAY for the ssh server %s: %s", | ||
750 | + s->inet->host, strerror(errno)); | ||
751 | + } | 51 | + } |
752 | + | 52 | + |
753 | /* Create SSH session. */ | 53 | + buf_origin = buf = blk_blockalign(blk, len); |
754 | - s->session = libssh2_session_init(); | 54 | + |
755 | + s->session = ssh_new(); | 55 | + if (qemuio_misalign) { |
756 | if (!s->session) { | 56 | + buf_origin += MISALIGN_OFFSET; |
757 | ret = -EINVAL; | 57 | + buf += MISALIGN_OFFSET; |
758 | - session_error_setg(errp, s, "failed to initialize libssh2 session"); | 58 | + len -= MISALIGN_OFFSET; |
759 | + session_error_setg(errp, s, "failed to initialize libssh session"); | ||
760 | goto err; | ||
761 | } | ||
762 | |||
763 | -#if TRACE_LIBSSH2 != 0 | ||
764 | - libssh2_trace(s->session, TRACE_LIBSSH2); | ||
765 | -#endif | ||
766 | + /* | ||
767 | + * Make sure we are in blocking mode during the connection and | ||
768 | + * authentication phases. | ||
769 | + */ | ||
770 | + ssh_set_blocking(s->session, 1); | ||
771 | |||
772 | - r = libssh2_session_handshake(s->session, s->sock); | ||
773 | - if (r != 0) { | ||
774 | + r = ssh_options_set(s->session, SSH_OPTIONS_USER, s->user); | ||
775 | + if (r < 0) { | ||
776 | + ret = -EINVAL; | ||
777 | + session_error_setg(errp, s, | ||
778 | + "failed to set the user in the libssh session"); | ||
779 | + goto err; | ||
780 | + } | 59 | + } |
781 | + | 60 | + |
782 | + r = ssh_options_set(s->session, SSH_OPTIONS_HOST, s->inet->host); | 61 | + pattern_len = fread(buf_origin, 1, len, f); |
783 | + if (r < 0) { | 62 | + |
784 | + ret = -EINVAL; | 63 | + if (ferror(f)) { |
785 | + session_error_setg(errp, s, | 64 | + perror(file_name); |
786 | + "failed to set the host in the libssh session"); | 65 | + goto error; |
787 | + goto err; | ||
788 | + } | 66 | + } |
789 | + | 67 | + |
790 | + if (port > 0) { | 68 | + if (pattern_len == 0) { |
791 | + r = ssh_options_set(s->session, SSH_OPTIONS_PORT, &port); | 69 | + fprintf(stderr, "%s: file is empty\n", file_name); |
792 | + if (r < 0) { | 70 | + goto error; |
793 | + ret = -EINVAL; | 71 | + } |
794 | + session_error_setg(errp, s, | 72 | + |
795 | + "failed to set the port in the libssh session"); | 73 | + fclose(f); |
796 | + goto err; | 74 | + |
75 | + if (len > pattern_len) { | ||
76 | + len -= pattern_len; | ||
77 | + buf += pattern_len; | ||
78 | + | ||
79 | + while (len > 0) { | ||
80 | + size_t len_to_copy = MIN(pattern_len, len); | ||
81 | + | ||
82 | + memcpy(buf, buf_origin, len_to_copy); | ||
83 | + | ||
84 | + len -= len_to_copy; | ||
85 | + buf += len_to_copy; | ||
797 | + } | 86 | + } |
798 | + } | 87 | + } |
799 | + | 88 | + |
800 | + r = ssh_options_set(s->session, SSH_OPTIONS_COMPRESSION, "none"); | 89 | + return buf_origin; |
801 | + if (r < 0) { | ||
802 | + ret = -EINVAL; | ||
803 | + session_error_setg(errp, s, | ||
804 | + "failed to disable the compression in the libssh " | ||
805 | + "session"); | ||
806 | + goto err; | ||
807 | + } | ||
808 | + | 90 | + |
809 | + /* Read ~/.ssh/config. */ | 91 | +error: |
810 | + r = ssh_options_parse_config(s->session, NULL); | 92 | + qemu_io_free(buf_origin); |
811 | + if (r < 0) { | 93 | + return NULL; |
812 | + ret = -EINVAL; | 94 | +} |
813 | + session_error_setg(errp, s, "failed to parse ~/.ssh/config"); | ||
814 | + goto err; | ||
815 | + } | ||
816 | + | 95 | + |
817 | + r = ssh_options_set(s->session, SSH_OPTIONS_FD, &new_sock); | 96 | static void dump_buffer(const void *buffer, int64_t offset, int64_t len) |
818 | + if (r < 0) { | 97 | { |
819 | + ret = -EINVAL; | 98 | uint64_t i; |
820 | + session_error_setg(errp, s, | 99 | @@ -XXX,XX +XXX,XX @@ static void write_help(void) |
821 | + "failed to set the socket in the libssh session"); | 100 | " -n, -- with -z, don't allow slow fallback\n" |
822 | + goto err; | 101 | " -p, -- ignored for backwards compatibility\n" |
823 | + } | 102 | " -P, -- use different pattern to fill file\n" |
824 | + /* libssh took ownership of the socket. */ | 103 | +" -s, -- use a pattern file to fill the write buffer\n" |
825 | + s->sock = new_sock; | 104 | " -C, -- report statistics in a machine parsable format\n" |
826 | + new_sock = -1; | 105 | " -q, -- quiet mode, do not show I/O statistics\n" |
827 | + | 106 | " -u, -- with -z, allow unmapping\n" |
828 | + /* Connect. */ | 107 | @@ -XXX,XX +XXX,XX @@ static const cmdinfo_t write_cmd = { |
829 | + r = ssh_connect(s->session); | 108 | .perm = BLK_PERM_WRITE, |
830 | + if (r != SSH_OK) { | 109 | .argmin = 2, |
831 | ret = -EINVAL; | 110 | .argmax = -1, |
832 | session_error_setg(errp, s, "failed to establish SSH session"); | 111 | - .args = "[-bcCfnquz] [-P pattern] off len", |
833 | goto err; | 112 | + .args = "[-bcCfnquz] [-P pattern | -s source_file] off len", |
834 | } | 113 | .oneline = "writes a number of bytes at a specified offset", |
835 | 114 | .help = write_help, | |
836 | /* Check the remote host's key against known_hosts. */ | 115 | }; |
837 | - ret = check_host_key(s, s->inet->host, port, opts->host_key_check, errp); | 116 | @@ -XXX,XX +XXX,XX @@ static int write_f(BlockBackend *blk, int argc, char **argv) |
838 | + ret = check_host_key(s, opts->host_key_check, errp); | 117 | { |
839 | if (ret < 0) { | 118 | struct timespec t1, t2; |
840 | goto err; | 119 | bool Cflag = false, qflag = false, bflag = false; |
841 | } | 120 | - bool Pflag = false, zflag = false, cflag = false; |
842 | 121 | + bool Pflag = false, zflag = false, cflag = false, sflag = false; | |
843 | /* Authenticate. */ | 122 | int flags = 0; |
844 | - ret = authenticate(s, s->user, errp); | 123 | int c, cnt, ret; |
845 | + ret = authenticate(s, errp); | 124 | char *buf = NULL; |
846 | if (ret < 0) { | 125 | @@ -XXX,XX +XXX,XX @@ static int write_f(BlockBackend *blk, int argc, char **argv) |
847 | goto err; | 126 | /* Some compilers get confused and warn if this is not initialized. */ |
848 | } | 127 | int64_t total = 0; |
849 | 128 | int pattern = 0xcd; | |
850 | /* Start SFTP. */ | 129 | + const char *file_name = NULL; |
851 | - s->sftp = libssh2_sftp_init(s->session); | 130 | |
852 | + s->sftp = sftp_new(s->session); | 131 | - while ((c = getopt(argc, argv, "bcCfnpP:quz")) != -1) { |
853 | if (!s->sftp) { | 132 | + while ((c = getopt(argc, argv, "bcCfnpP:qs:uz")) != -1) { |
854 | - session_error_setg(errp, s, "failed to initialize sftp handle"); | 133 | switch (c) { |
855 | + session_error_setg(errp, s, "failed to create sftp handle"); | 134 | case 'b': |
856 | + ret = -EINVAL; | 135 | bflag = true; |
857 | + goto err; | 136 | @@ -XXX,XX +XXX,XX @@ static int write_f(BlockBackend *blk, int argc, char **argv) |
858 | + } | 137 | case 'q': |
859 | + | 138 | qflag = true; |
860 | + r = sftp_init(s->sftp); | 139 | break; |
861 | + if (r < 0) { | 140 | + case 's': |
862 | + sftp_error_setg(errp, s, "failed to initialize sftp handle"); | 141 | + sflag = true; |
863 | ret = -EINVAL; | 142 | + file_name = optarg; |
864 | goto err; | 143 | + break; |
865 | } | 144 | case 'u': |
866 | 145 | flags |= BDRV_REQ_MAY_UNMAP; | |
867 | /* Open the remote file. */ | 146 | break; |
868 | trace_ssh_connect_to_ssh(opts->path, ssh_flags, creat_mode); | 147 | @@ -XXX,XX +XXX,XX @@ static int write_f(BlockBackend *blk, int argc, char **argv) |
869 | - s->sftp_handle = libssh2_sftp_open(s->sftp, opts->path, ssh_flags, | ||
870 | - creat_mode); | ||
871 | + s->sftp_handle = sftp_open(s->sftp, opts->path, ssh_flags, creat_mode); | ||
872 | if (!s->sftp_handle) { | ||
873 | - session_error_setg(errp, s, "failed to open remote file '%s'", | ||
874 | - opts->path); | ||
875 | + sftp_error_setg(errp, s, "failed to open remote file '%s'", | ||
876 | + opts->path); | ||
877 | ret = -EINVAL; | ||
878 | goto err; | ||
879 | } | ||
880 | |||
881 | - r = libssh2_sftp_fstat(s->sftp_handle, &s->attrs); | ||
882 | - if (r < 0) { | ||
883 | + /* Make sure the SFTP file is handled in blocking mode. */ | ||
884 | + sftp_file_set_blocking(s->sftp_handle); | ||
885 | + | ||
886 | + s->attrs = sftp_fstat(s->sftp_handle); | ||
887 | + if (!s->attrs) { | ||
888 | sftp_error_setg(errp, s, "failed to read file attributes"); | ||
889 | return -EINVAL; | 148 | return -EINVAL; |
890 | } | 149 | } |
891 | @@ -XXX,XX +XXX,XX @@ static int connect_to_ssh(BDRVSSHState *s, BlockdevOptionsSsh *opts, | 150 | |
892 | return 0; | 151 | - if (zflag && Pflag) { |
893 | 152 | - printf("-z and -P cannot be specified at the same time\n"); | |
894 | err: | 153 | + if (zflag + Pflag + sflag > 1) { |
895 | + if (s->attrs) { | 154 | + printf("Only one of -z, -P, and -s " |
896 | + sftp_attributes_free(s->attrs); | 155 | + "can be specified at the same time\n"); |
897 | + } | 156 | return -EINVAL; |
898 | + s->attrs = NULL; | ||
899 | if (s->sftp_handle) { | ||
900 | - libssh2_sftp_close(s->sftp_handle); | ||
901 | + sftp_close(s->sftp_handle); | ||
902 | } | 157 | } |
903 | s->sftp_handle = NULL; | 158 | |
904 | if (s->sftp) { | 159 | @@ -XXX,XX +XXX,XX @@ static int write_f(BlockBackend *blk, int argc, char **argv) |
905 | - libssh2_sftp_shutdown(s->sftp); | ||
906 | + sftp_free(s->sftp); | ||
907 | } | 160 | } |
908 | s->sftp = NULL; | 161 | |
909 | if (s->session) { | 162 | if (!zflag) { |
910 | - libssh2_session_disconnect(s->session, | 163 | - buf = qemu_io_alloc(blk, count, pattern); |
911 | - "from qemu ssh client: " | 164 | + if (sflag) { |
912 | - "error opening connection"); | 165 | + buf = qemu_io_alloc_from_file(blk, count, file_name); |
913 | - libssh2_session_free(s->session); | 166 | + if (!buf) { |
914 | + ssh_disconnect(s->session); | 167 | + return -EINVAL; |
915 | + ssh_free(s->session); | 168 | + } |
916 | } | 169 | + } else { |
917 | s->session = NULL; | 170 | + buf = qemu_io_alloc(blk, count, pattern); |
918 | + s->sock = -1; | ||
919 | + if (new_sock >= 0) { | ||
920 | + close(new_sock); | ||
921 | + } | ||
922 | |||
923 | return ret; | ||
924 | } | ||
925 | @@ -XXX,XX +XXX,XX @@ static int ssh_file_open(BlockDriverState *bs, QDict *options, int bdrv_flags, | ||
926 | |||
927 | ssh_state_init(s); | ||
928 | |||
929 | - ssh_flags = LIBSSH2_FXF_READ; | ||
930 | + ssh_flags = 0; | ||
931 | if (bdrv_flags & BDRV_O_RDWR) { | ||
932 | - ssh_flags |= LIBSSH2_FXF_WRITE; | ||
933 | + ssh_flags |= O_RDWR; | ||
934 | + } else { | ||
935 | + ssh_flags |= O_RDONLY; | ||
936 | } | ||
937 | |||
938 | opts = ssh_parse_options(options, errp); | ||
939 | @@ -XXX,XX +XXX,XX @@ static int ssh_file_open(BlockDriverState *bs, QDict *options, int bdrv_flags, | ||
940 | } | ||
941 | |||
942 | /* Go non-blocking. */ | ||
943 | - libssh2_session_set_blocking(s->session, 0); | ||
944 | + ssh_set_blocking(s->session, 0); | ||
945 | |||
946 | qapi_free_BlockdevOptionsSsh(opts); | ||
947 | |||
948 | return 0; | ||
949 | |||
950 | err: | ||
951 | - if (s->sock >= 0) { | ||
952 | - close(s->sock); | ||
953 | - } | ||
954 | - s->sock = -1; | ||
955 | - | ||
956 | qapi_free_BlockdevOptionsSsh(opts); | ||
957 | |||
958 | return ret; | ||
959 | @@ -XXX,XX +XXX,XX @@ static int ssh_grow_file(BDRVSSHState *s, int64_t offset, Error **errp) | ||
960 | { | ||
961 | ssize_t ret; | ||
962 | char c[1] = { '\0' }; | ||
963 | - int was_blocking = libssh2_session_get_blocking(s->session); | ||
964 | + int was_blocking = ssh_is_blocking(s->session); | ||
965 | |||
966 | /* offset must be strictly greater than the current size so we do | ||
967 | * not overwrite anything */ | ||
968 | - assert(offset > 0 && offset > s->attrs.filesize); | ||
969 | + assert(offset > 0 && offset > s->attrs->size); | ||
970 | |||
971 | - libssh2_session_set_blocking(s->session, 1); | ||
972 | + ssh_set_blocking(s->session, 1); | ||
973 | |||
974 | - libssh2_sftp_seek64(s->sftp_handle, offset - 1); | ||
975 | - ret = libssh2_sftp_write(s->sftp_handle, c, 1); | ||
976 | + sftp_seek64(s->sftp_handle, offset - 1); | ||
977 | + ret = sftp_write(s->sftp_handle, c, 1); | ||
978 | |||
979 | - libssh2_session_set_blocking(s->session, was_blocking); | ||
980 | + ssh_set_blocking(s->session, was_blocking); | ||
981 | |||
982 | if (ret < 0) { | ||
983 | sftp_error_setg(errp, s, "Failed to grow file"); | ||
984 | return -EIO; | ||
985 | } | ||
986 | |||
987 | - s->attrs.filesize = offset; | ||
988 | + s->attrs->size = offset; | ||
989 | return 0; | ||
990 | } | ||
991 | |||
992 | @@ -XXX,XX +XXX,XX @@ static int ssh_co_create(BlockdevCreateOptions *options, Error **errp) | ||
993 | ssh_state_init(&s); | ||
994 | |||
995 | ret = connect_to_ssh(&s, opts->location, | ||
996 | - LIBSSH2_FXF_READ|LIBSSH2_FXF_WRITE| | ||
997 | - LIBSSH2_FXF_CREAT|LIBSSH2_FXF_TRUNC, | ||
998 | + O_RDWR | O_CREAT | O_TRUNC, | ||
999 | 0644, errp); | ||
1000 | if (ret < 0) { | ||
1001 | goto fail; | ||
1002 | @@ -XXX,XX +XXX,XX @@ static int ssh_has_zero_init(BlockDriverState *bs) | ||
1003 | /* Assume false, unless we can positively prove it's true. */ | ||
1004 | int has_zero_init = 0; | ||
1005 | |||
1006 | - if (s->attrs.flags & LIBSSH2_SFTP_ATTR_PERMISSIONS) { | ||
1007 | - if (s->attrs.permissions & LIBSSH2_SFTP_S_IFREG) { | ||
1008 | - has_zero_init = 1; | ||
1009 | - } | ||
1010 | + if (s->attrs->type == SSH_FILEXFER_TYPE_REGULAR) { | ||
1011 | + has_zero_init = 1; | ||
1012 | } | ||
1013 | |||
1014 | return has_zero_init; | ||
1015 | @@ -XXX,XX +XXX,XX @@ static coroutine_fn void co_yield(BDRVSSHState *s, BlockDriverState *bs) | ||
1016 | .co = qemu_coroutine_self() | ||
1017 | }; | ||
1018 | |||
1019 | - r = libssh2_session_block_directions(s->session); | ||
1020 | + r = ssh_get_poll_flags(s->session); | ||
1021 | |||
1022 | - if (r & LIBSSH2_SESSION_BLOCK_INBOUND) { | ||
1023 | + if (r & SSH_READ_PENDING) { | ||
1024 | rd_handler = restart_coroutine; | ||
1025 | } | ||
1026 | - if (r & LIBSSH2_SESSION_BLOCK_OUTBOUND) { | ||
1027 | + if (r & SSH_WRITE_PENDING) { | ||
1028 | wr_handler = restart_coroutine; | ||
1029 | } | ||
1030 | |||
1031 | @@ -XXX,XX +XXX,XX @@ static coroutine_fn void co_yield(BDRVSSHState *s, BlockDriverState *bs) | ||
1032 | trace_ssh_co_yield_back(s->sock); | ||
1033 | } | ||
1034 | |||
1035 | -/* SFTP has a function `libssh2_sftp_seek64' which seeks to a position | ||
1036 | - * in the remote file. Notice that it just updates a field in the | ||
1037 | - * sftp_handle structure, so there is no network traffic and it cannot | ||
1038 | - * fail. | ||
1039 | - * | ||
1040 | - * However, `libssh2_sftp_seek64' does have a catastrophic effect on | ||
1041 | - * performance since it causes the handle to throw away all in-flight | ||
1042 | - * reads and buffered readahead data. Therefore this function tries | ||
1043 | - * to be intelligent about when to call the underlying libssh2 function. | ||
1044 | - */ | ||
1045 | -#define SSH_SEEK_WRITE 0 | ||
1046 | -#define SSH_SEEK_READ 1 | ||
1047 | -#define SSH_SEEK_FORCE 2 | ||
1048 | - | ||
1049 | -static void ssh_seek(BDRVSSHState *s, int64_t offset, int flags) | ||
1050 | -{ | ||
1051 | - bool op_read = (flags & SSH_SEEK_READ) != 0; | ||
1052 | - bool force = (flags & SSH_SEEK_FORCE) != 0; | ||
1053 | - | ||
1054 | - if (force || op_read != s->offset_op_read || offset != s->offset) { | ||
1055 | - trace_ssh_seek(offset); | ||
1056 | - libssh2_sftp_seek64(s->sftp_handle, offset); | ||
1057 | - s->offset = offset; | ||
1058 | - s->offset_op_read = op_read; | ||
1059 | - } | ||
1060 | -} | ||
1061 | - | ||
1062 | static coroutine_fn int ssh_read(BDRVSSHState *s, BlockDriverState *bs, | ||
1063 | int64_t offset, size_t size, | ||
1064 | QEMUIOVector *qiov) | ||
1065 | @@ -XXX,XX +XXX,XX @@ static coroutine_fn int ssh_read(BDRVSSHState *s, BlockDriverState *bs, | ||
1066 | |||
1067 | trace_ssh_read(offset, size); | ||
1068 | |||
1069 | - ssh_seek(s, offset, SSH_SEEK_READ); | ||
1070 | + trace_ssh_seek(offset); | ||
1071 | + sftp_seek64(s->sftp_handle, offset); | ||
1072 | |||
1073 | /* This keeps track of the current iovec element ('i'), where we | ||
1074 | * will write to next ('buf'), and the end of the current iovec | ||
1075 | @@ -XXX,XX +XXX,XX @@ static coroutine_fn int ssh_read(BDRVSSHState *s, BlockDriverState *bs, | ||
1076 | buf = i->iov_base; | ||
1077 | end_of_vec = i->iov_base + i->iov_len; | ||
1078 | |||
1079 | - /* libssh2 has a hard-coded limit of 2000 bytes per request, | ||
1080 | - * although it will also do readahead behind our backs. Therefore | ||
1081 | - * we may have to do repeated reads here until we have read 'size' | ||
1082 | - * bytes. | ||
1083 | - */ | ||
1084 | for (got = 0; got < size; ) { | ||
1085 | + size_t request_read_size; | ||
1086 | again: | ||
1087 | - trace_ssh_read_buf(buf, end_of_vec - buf); | ||
1088 | - r = libssh2_sftp_read(s->sftp_handle, buf, end_of_vec - buf); | ||
1089 | - trace_ssh_read_return(r); | ||
1090 | + /* | ||
1091 | + * The size of SFTP packets is limited to 32K bytes, so limit | ||
1092 | + * the amount of data requested to 16K, as libssh currently | ||
1093 | + * does not handle multiple requests on its own. | ||
1094 | + */ | ||
1095 | + request_read_size = MIN(end_of_vec - buf, 16384); | ||
1096 | + trace_ssh_read_buf(buf, end_of_vec - buf, request_read_size); | ||
1097 | + r = sftp_read(s->sftp_handle, buf, request_read_size); | ||
1098 | + trace_ssh_read_return(r, sftp_get_error(s->sftp)); | ||
1099 | |||
1100 | - if (r == LIBSSH2_ERROR_EAGAIN || r == LIBSSH2_ERROR_TIMEOUT) { | ||
1101 | + if (r == SSH_AGAIN) { | ||
1102 | co_yield(s, bs); | ||
1103 | goto again; | ||
1104 | } | ||
1105 | - if (r < 0) { | ||
1106 | - sftp_error_trace(s, "read"); | ||
1107 | - s->offset = -1; | ||
1108 | - return -EIO; | ||
1109 | - } | ||
1110 | - if (r == 0) { | ||
1111 | + if (r == SSH_EOF || (r == 0 && sftp_get_error(s->sftp) == SSH_FX_EOF)) { | ||
1112 | /* EOF: Short read so pad the buffer with zeroes and return it. */ | ||
1113 | qemu_iovec_memset(qiov, got, 0, size - got); | ||
1114 | return 0; | ||
1115 | } | ||
1116 | + if (r <= 0) { | ||
1117 | + sftp_error_trace(s, "read"); | ||
1118 | + return -EIO; | ||
1119 | + } | ||
1120 | |||
1121 | got += r; | ||
1122 | buf += r; | ||
1123 | - s->offset += r; | ||
1124 | if (buf >= end_of_vec && got < size) { | ||
1125 | i++; | ||
1126 | buf = i->iov_base; | ||
1127 | @@ -XXX,XX +XXX,XX @@ static int ssh_write(BDRVSSHState *s, BlockDriverState *bs, | ||
1128 | |||
1129 | trace_ssh_write(offset, size); | ||
1130 | |||
1131 | - ssh_seek(s, offset, SSH_SEEK_WRITE); | ||
1132 | + trace_ssh_seek(offset); | ||
1133 | + sftp_seek64(s->sftp_handle, offset); | ||
1134 | |||
1135 | /* This keeps track of the current iovec element ('i'), where we | ||
1136 | * will read from next ('buf'), and the end of the current iovec | ||
1137 | @@ -XXX,XX +XXX,XX @@ static int ssh_write(BDRVSSHState *s, BlockDriverState *bs, | ||
1138 | end_of_vec = i->iov_base + i->iov_len; | ||
1139 | |||
1140 | for (written = 0; written < size; ) { | ||
1141 | + size_t request_write_size; | ||
1142 | again: | ||
1143 | - trace_ssh_write_buf(buf, end_of_vec - buf); | ||
1144 | - r = libssh2_sftp_write(s->sftp_handle, buf, end_of_vec - buf); | ||
1145 | - trace_ssh_write_return(r); | ||
1146 | + /* | ||
1147 | + * Avoid too large data packets, as libssh currently does not | ||
1148 | + * handle multiple requests on its own. | ||
1149 | + */ | ||
1150 | + request_write_size = MIN(end_of_vec - buf, 131072); | ||
1151 | + trace_ssh_write_buf(buf, end_of_vec - buf, request_write_size); | ||
1152 | + r = sftp_write(s->sftp_handle, buf, request_write_size); | ||
1153 | + trace_ssh_write_return(r, sftp_get_error(s->sftp)); | ||
1154 | |||
1155 | - if (r == LIBSSH2_ERROR_EAGAIN || r == LIBSSH2_ERROR_TIMEOUT) { | ||
1156 | + if (r == SSH_AGAIN) { | ||
1157 | co_yield(s, bs); | ||
1158 | goto again; | ||
1159 | } | ||
1160 | if (r < 0) { | ||
1161 | sftp_error_trace(s, "write"); | ||
1162 | - s->offset = -1; | ||
1163 | return -EIO; | ||
1164 | } | ||
1165 | - /* The libssh2 API is very unclear about this. A comment in | ||
1166 | - * the code says "nothing was acked, and no EAGAIN was | ||
1167 | - * received!" which apparently means that no data got sent | ||
1168 | - * out, and the underlying channel didn't return any EAGAIN | ||
1169 | - * indication. I think this is a bug in either libssh2 or | ||
1170 | - * OpenSSH (server-side). In any case, forcing a seek (to | ||
1171 | - * discard libssh2 internal buffers), and then trying again | ||
1172 | - * works for me. | ||
1173 | - */ | ||
1174 | - if (r == 0) { | ||
1175 | - ssh_seek(s, offset + written, SSH_SEEK_WRITE|SSH_SEEK_FORCE); | ||
1176 | - co_yield(s, bs); | ||
1177 | - goto again; | ||
1178 | - } | ||
1179 | |||
1180 | written += r; | ||
1181 | buf += r; | ||
1182 | - s->offset += r; | ||
1183 | if (buf >= end_of_vec && written < size) { | ||
1184 | i++; | ||
1185 | buf = i->iov_base; | ||
1186 | end_of_vec = i->iov_base + i->iov_len; | ||
1187 | } | ||
1188 | |||
1189 | - if (offset + written > s->attrs.filesize) | ||
1190 | - s->attrs.filesize = offset + written; | ||
1191 | + if (offset + written > s->attrs->size) { | ||
1192 | + s->attrs->size = offset + written; | ||
1193 | + } | 171 | + } |
1194 | } | 172 | } |
1195 | 173 | ||
1196 | return 0; | 174 | clock_gettime(CLOCK_MONOTONIC, &t1); |
1197 | @@ -XXX,XX +XXX,XX @@ static void unsafe_flush_warning(BDRVSSHState *s, const char *what) | ||
1198 | } | ||
1199 | } | ||
1200 | |||
1201 | -#ifdef HAS_LIBSSH2_SFTP_FSYNC | ||
1202 | +#ifdef HAVE_LIBSSH_0_8 | ||
1203 | |||
1204 | static coroutine_fn int ssh_flush(BDRVSSHState *s, BlockDriverState *bs) | ||
1205 | { | ||
1206 | int r; | ||
1207 | |||
1208 | trace_ssh_flush(); | ||
1209 | + | ||
1210 | + if (!sftp_extension_supported(s->sftp, "fsync@openssh.com", "1")) { | ||
1211 | + unsafe_flush_warning(s, "OpenSSH >= 6.3"); | ||
1212 | + return 0; | ||
1213 | + } | ||
1214 | again: | ||
1215 | - r = libssh2_sftp_fsync(s->sftp_handle); | ||
1216 | - if (r == LIBSSH2_ERROR_EAGAIN || r == LIBSSH2_ERROR_TIMEOUT) { | ||
1217 | + r = sftp_fsync(s->sftp_handle); | ||
1218 | + if (r == SSH_AGAIN) { | ||
1219 | co_yield(s, bs); | ||
1220 | goto again; | ||
1221 | } | ||
1222 | - if (r == LIBSSH2_ERROR_SFTP_PROTOCOL && | ||
1223 | - libssh2_sftp_last_error(s->sftp) == LIBSSH2_FX_OP_UNSUPPORTED) { | ||
1224 | - unsafe_flush_warning(s, "OpenSSH >= 6.3"); | ||
1225 | - return 0; | ||
1226 | - } | ||
1227 | if (r < 0) { | ||
1228 | sftp_error_trace(s, "fsync"); | ||
1229 | return -EIO; | ||
1230 | @@ -XXX,XX +XXX,XX @@ static coroutine_fn int ssh_co_flush(BlockDriverState *bs) | ||
1231 | return ret; | ||
1232 | } | ||
1233 | |||
1234 | -#else /* !HAS_LIBSSH2_SFTP_FSYNC */ | ||
1235 | +#else /* !HAVE_LIBSSH_0_8 */ | ||
1236 | |||
1237 | static coroutine_fn int ssh_co_flush(BlockDriverState *bs) | ||
1238 | { | ||
1239 | BDRVSSHState *s = bs->opaque; | ||
1240 | |||
1241 | - unsafe_flush_warning(s, "libssh2 >= 1.4.4"); | ||
1242 | + unsafe_flush_warning(s, "libssh >= 0.8.0"); | ||
1243 | return 0; | ||
1244 | } | ||
1245 | |||
1246 | -#endif /* !HAS_LIBSSH2_SFTP_FSYNC */ | ||
1247 | +#endif /* !HAVE_LIBSSH_0_8 */ | ||
1248 | |||
1249 | static int64_t ssh_getlength(BlockDriverState *bs) | ||
1250 | { | ||
1251 | BDRVSSHState *s = bs->opaque; | ||
1252 | int64_t length; | ||
1253 | |||
1254 | - /* Note we cannot make a libssh2 call here. */ | ||
1255 | - length = (int64_t) s->attrs.filesize; | ||
1256 | + /* Note we cannot make a libssh call here. */ | ||
1257 | + length = (int64_t) s->attrs->size; | ||
1258 | trace_ssh_getlength(length); | ||
1259 | |||
1260 | return length; | ||
1261 | @@ -XXX,XX +XXX,XX @@ static int coroutine_fn ssh_co_truncate(BlockDriverState *bs, int64_t offset, | ||
1262 | return -ENOTSUP; | ||
1263 | } | ||
1264 | |||
1265 | - if (offset < s->attrs.filesize) { | ||
1266 | + if (offset < s->attrs->size) { | ||
1267 | error_setg(errp, "ssh driver does not support shrinking files"); | ||
1268 | return -ENOTSUP; | ||
1269 | } | ||
1270 | |||
1271 | - if (offset == s->attrs.filesize) { | ||
1272 | + if (offset == s->attrs->size) { | ||
1273 | return 0; | ||
1274 | } | ||
1275 | |||
1276 | @@ -XXX,XX +XXX,XX @@ static void bdrv_ssh_init(void) | ||
1277 | { | ||
1278 | int r; | ||
1279 | |||
1280 | - r = libssh2_init(0); | ||
1281 | + r = ssh_init(); | ||
1282 | if (r != 0) { | ||
1283 | - fprintf(stderr, "libssh2 initialization failed, %d\n", r); | ||
1284 | + fprintf(stderr, "libssh initialization failed, %d\n", r); | ||
1285 | exit(EXIT_FAILURE); | ||
1286 | } | ||
1287 | |||
1288 | +#if TRACE_LIBSSH != 0 | ||
1289 | + ssh_set_log_level(TRACE_LIBSSH); | ||
1290 | +#endif | ||
1291 | + | ||
1292 | bdrv_register(&bdrv_ssh); | ||
1293 | } | ||
1294 | |||
1295 | diff --git a/.travis.yml b/.travis.yml | ||
1296 | index XXXXXXX..XXXXXXX 100644 | ||
1297 | --- a/.travis.yml | ||
1298 | +++ b/.travis.yml | ||
1299 | @@ -XXX,XX +XXX,XX @@ addons: | ||
1300 | - libseccomp-dev | ||
1301 | - libspice-protocol-dev | ||
1302 | - libspice-server-dev | ||
1303 | - - libssh2-1-dev | ||
1304 | + - libssh-dev | ||
1305 | - liburcu-dev | ||
1306 | - libusb-1.0-0-dev | ||
1307 | - libvte-2.91-dev | ||
1308 | @@ -XXX,XX +XXX,XX @@ matrix: | ||
1309 | - libseccomp-dev | ||
1310 | - libspice-protocol-dev | ||
1311 | - libspice-server-dev | ||
1312 | - - libssh2-1-dev | ||
1313 | + - libssh-dev | ||
1314 | - liburcu-dev | ||
1315 | - libusb-1.0-0-dev | ||
1316 | - libvte-2.91-dev | ||
1317 | diff --git a/block/trace-events b/block/trace-events | ||
1318 | index XXXXXXX..XXXXXXX 100644 | ||
1319 | --- a/block/trace-events | ||
1320 | +++ b/block/trace-events | ||
1321 | @@ -XXX,XX +XXX,XX @@ nbd_client_connect_success(const char *export_name) "export '%s'" | ||
1322 | # ssh.c | ||
1323 | ssh_restart_coroutine(void *co) "co=%p" | ||
1324 | ssh_flush(void) "fsync" | ||
1325 | -ssh_check_host_key_knownhosts(const char *key) "host key OK: %s" | ||
1326 | +ssh_check_host_key_knownhosts(void) "host key OK" | ||
1327 | ssh_connect_to_ssh(char *path, int flags, int mode) "opening file %s flags=0x%x creat_mode=0%o" | ||
1328 | ssh_co_yield(int sock, void *rd_handler, void *wr_handler) "s->sock=%d rd_handler=%p wr_handler=%p" | ||
1329 | ssh_co_yield_back(int sock) "s->sock=%d - back" | ||
1330 | ssh_getlength(int64_t length) "length=%" PRIi64 | ||
1331 | ssh_co_create_opts(uint64_t size) "total_size=%" PRIu64 | ||
1332 | ssh_read(int64_t offset, size_t size) "offset=%" PRIi64 " size=%zu" | ||
1333 | -ssh_read_buf(void *buf, size_t size) "sftp_read buf=%p size=%zu" | ||
1334 | -ssh_read_return(ssize_t ret) "sftp_read returned %zd" | ||
1335 | +ssh_read_buf(void *buf, size_t size, size_t actual_size) "sftp_read buf=%p size=%zu (actual size=%zu)" | ||
1336 | +ssh_read_return(ssize_t ret, int sftp_err) "sftp_read returned %zd (sftp error=%d)" | ||
1337 | ssh_write(int64_t offset, size_t size) "offset=%" PRIi64 " size=%zu" | ||
1338 | -ssh_write_buf(void *buf, size_t size) "sftp_write buf=%p size=%zu" | ||
1339 | -ssh_write_return(ssize_t ret) "sftp_write returned %zd" | ||
1340 | +ssh_write_buf(void *buf, size_t size, size_t actual_size) "sftp_write buf=%p size=%zu (actual size=%zu)" | ||
1341 | +ssh_write_return(ssize_t ret, int sftp_err) "sftp_write returned %zd (sftp error=%d)" | ||
1342 | ssh_seek(int64_t offset) "seeking to offset=%" PRIi64 | ||
1343 | +ssh_auth_methods(int methods) "auth methods=0x%x" | ||
1344 | +ssh_server_status(int status) "server status=%d" | ||
1345 | |||
1346 | # curl.c | ||
1347 | curl_timer_cb(long timeout_ms) "timer callback timeout_ms %ld" | ||
1348 | @@ -XXX,XX +XXX,XX @@ sheepdog_snapshot_create(const char *sn_name, const char *id) "%s %s" | ||
1349 | sheepdog_snapshot_create_inode(const char *name, uint32_t snap, uint32_t vdi) "s->inode: name %s snap_id 0x%" PRIx32 " vdi 0x%" PRIx32 | ||
1350 | |||
1351 | # ssh.c | ||
1352 | -sftp_error(const char *op, const char *ssh_err, int ssh_err_code, unsigned long sftp_err_code) "%s failed: %s (libssh2 error code: %d, sftp error code: %lu)" | ||
1353 | +sftp_error(const char *op, const char *ssh_err, int ssh_err_code, int sftp_err_code) "%s failed: %s (libssh error code: %d, sftp error code: %d)" | ||
1354 | diff --git a/docs/qemu-block-drivers.texi b/docs/qemu-block-drivers.texi | ||
1355 | index XXXXXXX..XXXXXXX 100644 | ||
1356 | --- a/docs/qemu-block-drivers.texi | ||
1357 | +++ b/docs/qemu-block-drivers.texi | ||
1358 | @@ -XXX,XX +XXX,XX @@ print a warning when @code{fsync} is not supported: | ||
1359 | |||
1360 | warning: ssh server @code{ssh.example.com:22} does not support fsync | ||
1361 | |||
1362 | -With sufficiently new versions of libssh2 and OpenSSH, @code{fsync} is | ||
1363 | +With sufficiently new versions of libssh and OpenSSH, @code{fsync} is | ||
1364 | supported. | ||
1365 | |||
1366 | @node disk_images_nvme | ||
1367 | diff --git a/tests/docker/dockerfiles/debian-win32-cross.docker b/tests/docker/dockerfiles/debian-win32-cross.docker | ||
1368 | index XXXXXXX..XXXXXXX 100644 | ||
1369 | --- a/tests/docker/dockerfiles/debian-win32-cross.docker | ||
1370 | +++ b/tests/docker/dockerfiles/debian-win32-cross.docker | ||
1371 | @@ -XXX,XX +XXX,XX @@ RUN DEBIAN_FRONTEND=noninteractive eatmydata \ | ||
1372 | mxe-$TARGET-w64-mingw32.shared-curl \ | ||
1373 | mxe-$TARGET-w64-mingw32.shared-glib \ | ||
1374 | mxe-$TARGET-w64-mingw32.shared-libgcrypt \ | ||
1375 | - mxe-$TARGET-w64-mingw32.shared-libssh2 \ | ||
1376 | mxe-$TARGET-w64-mingw32.shared-libusb1 \ | ||
1377 | mxe-$TARGET-w64-mingw32.shared-lzo \ | ||
1378 | mxe-$TARGET-w64-mingw32.shared-nettle \ | ||
1379 | diff --git a/tests/docker/dockerfiles/debian-win64-cross.docker b/tests/docker/dockerfiles/debian-win64-cross.docker | ||
1380 | index XXXXXXX..XXXXXXX 100644 | ||
1381 | --- a/tests/docker/dockerfiles/debian-win64-cross.docker | ||
1382 | +++ b/tests/docker/dockerfiles/debian-win64-cross.docker | ||
1383 | @@ -XXX,XX +XXX,XX @@ RUN DEBIAN_FRONTEND=noninteractive eatmydata \ | ||
1384 | mxe-$TARGET-w64-mingw32.shared-curl \ | ||
1385 | mxe-$TARGET-w64-mingw32.shared-glib \ | ||
1386 | mxe-$TARGET-w64-mingw32.shared-libgcrypt \ | ||
1387 | - mxe-$TARGET-w64-mingw32.shared-libssh2 \ | ||
1388 | mxe-$TARGET-w64-mingw32.shared-libusb1 \ | ||
1389 | mxe-$TARGET-w64-mingw32.shared-lzo \ | ||
1390 | mxe-$TARGET-w64-mingw32.shared-nettle \ | ||
1391 | diff --git a/tests/docker/dockerfiles/fedora.docker b/tests/docker/dockerfiles/fedora.docker | ||
1392 | index XXXXXXX..XXXXXXX 100644 | ||
1393 | --- a/tests/docker/dockerfiles/fedora.docker | ||
1394 | +++ b/tests/docker/dockerfiles/fedora.docker | ||
1395 | @@ -XXX,XX +XXX,XX @@ ENV PACKAGES \ | ||
1396 | libpng-devel \ | ||
1397 | librbd-devel \ | ||
1398 | libseccomp-devel \ | ||
1399 | - libssh2-devel \ | ||
1400 | + libssh-devel \ | ||
1401 | libubsan \ | ||
1402 | libusbx-devel \ | ||
1403 | libxml2-devel \ | ||
1404 | @@ -XXX,XX +XXX,XX @@ ENV PACKAGES \ | ||
1405 | mingw32-gtk3 \ | ||
1406 | mingw32-libjpeg-turbo \ | ||
1407 | mingw32-libpng \ | ||
1408 | - mingw32-libssh2 \ | ||
1409 | mingw32-libtasn1 \ | ||
1410 | mingw32-nettle \ | ||
1411 | mingw32-pixman \ | ||
1412 | @@ -XXX,XX +XXX,XX @@ ENV PACKAGES \ | ||
1413 | mingw64-gtk3 \ | ||
1414 | mingw64-libjpeg-turbo \ | ||
1415 | mingw64-libpng \ | ||
1416 | - mingw64-libssh2 \ | ||
1417 | mingw64-libtasn1 \ | ||
1418 | mingw64-nettle \ | ||
1419 | mingw64-pixman \ | ||
1420 | diff --git a/tests/docker/dockerfiles/ubuntu.docker b/tests/docker/dockerfiles/ubuntu.docker | ||
1421 | index XXXXXXX..XXXXXXX 100644 | ||
1422 | --- a/tests/docker/dockerfiles/ubuntu.docker | ||
1423 | +++ b/tests/docker/dockerfiles/ubuntu.docker | ||
1424 | @@ -XXX,XX +XXX,XX @@ ENV PACKAGES flex bison \ | ||
1425 | libsnappy-dev \ | ||
1426 | libspice-protocol-dev \ | ||
1427 | libspice-server-dev \ | ||
1428 | - libssh2-1-dev \ | ||
1429 | + libssh-dev \ | ||
1430 | libusb-1.0-0-dev \ | ||
1431 | libusbredirhost-dev \ | ||
1432 | libvdeplug-dev \ | ||
1433 | diff --git a/tests/docker/dockerfiles/ubuntu1804.docker b/tests/docker/dockerfiles/ubuntu1804.docker | ||
1434 | index XXXXXXX..XXXXXXX 100644 | ||
1435 | --- a/tests/docker/dockerfiles/ubuntu1804.docker | ||
1436 | +++ b/tests/docker/dockerfiles/ubuntu1804.docker | ||
1437 | @@ -XXX,XX +XXX,XX @@ ENV PACKAGES flex bison \ | ||
1438 | libsnappy-dev \ | ||
1439 | libspice-protocol-dev \ | ||
1440 | libspice-server-dev \ | ||
1441 | - libssh2-1-dev \ | ||
1442 | + libssh-dev \ | ||
1443 | libusb-1.0-0-dev \ | ||
1444 | libusbredirhost-dev \ | ||
1445 | libvdeplug-dev \ | ||
1446 | diff --git a/tests/qemu-iotests/207 b/tests/qemu-iotests/207 | ||
1447 | index XXXXXXX..XXXXXXX 100755 | ||
1448 | --- a/tests/qemu-iotests/207 | ||
1449 | +++ b/tests/qemu-iotests/207 | ||
1450 | @@ -XXX,XX +XXX,XX @@ with iotests.FilePath('t.img') as disk_path, \ | ||
1451 | |||
1452 | iotests.img_info_log(remote_path) | ||
1453 | |||
1454 | - md5_key = subprocess.check_output( | ||
1455 | - 'ssh-keyscan -t rsa 127.0.0.1 2>/dev/null | grep -v "\\^#" | ' + | ||
1456 | - 'cut -d" " -f3 | base64 -d | md5sum -b | cut -d" " -f1', | ||
1457 | - shell=True).rstrip().decode('ascii') | ||
1458 | + keys = subprocess.check_output( | ||
1459 | + 'ssh-keyscan 127.0.0.1 2>/dev/null | grep -v "\\^#" | ' + | ||
1460 | + 'cut -d" " -f3', | ||
1461 | + shell=True).rstrip().decode('ascii').split('\n') | ||
1462 | + | ||
1463 | + # Mappings of base64 representations to digests | ||
1464 | + md5_keys = {} | ||
1465 | + sha1_keys = {} | ||
1466 | + | ||
1467 | + for key in keys: | ||
1468 | + md5_keys[key] = subprocess.check_output( | ||
1469 | + 'echo %s | base64 -d | md5sum -b | cut -d" " -f1' % key, | ||
1470 | + shell=True).rstrip().decode('ascii') | ||
1471 | + | ||
1472 | + sha1_keys[key] = subprocess.check_output( | ||
1473 | + 'echo %s | base64 -d | sha1sum -b | cut -d" " -f1' % key, | ||
1474 | + shell=True).rstrip().decode('ascii') | ||
1475 | |||
1476 | vm.launch() | ||
1477 | + | ||
1478 | + # Find correct key first | ||
1479 | + matching_key = None | ||
1480 | + for key in keys: | ||
1481 | + result = vm.qmp('blockdev-add', | ||
1482 | + driver='ssh', node_name='node0', path=disk_path, | ||
1483 | + server={ | ||
1484 | + 'host': '127.0.0.1', | ||
1485 | + 'port': '22', | ||
1486 | + }, host_key_check={ | ||
1487 | + 'mode': 'hash', | ||
1488 | + 'type': 'md5', | ||
1489 | + 'hash': md5_keys[key], | ||
1490 | + }) | ||
1491 | + | ||
1492 | + if 'error' not in result: | ||
1493 | + vm.qmp('blockdev-del', node_name='node0') | ||
1494 | + matching_key = key | ||
1495 | + break | ||
1496 | + | ||
1497 | + if matching_key is None: | ||
1498 | + vm.shutdown() | ||
1499 | + iotests.notrun('Did not find a key that fits 127.0.0.1') | ||
1500 | + | ||
1501 | blockdev_create(vm, { 'driver': 'ssh', | ||
1502 | 'location': { | ||
1503 | 'path': disk_path, | ||
1504 | @@ -XXX,XX +XXX,XX @@ with iotests.FilePath('t.img') as disk_path, \ | ||
1505 | 'host-key-check': { | ||
1506 | 'mode': 'hash', | ||
1507 | 'type': 'md5', | ||
1508 | - 'hash': md5_key, | ||
1509 | + 'hash': md5_keys[matching_key], | ||
1510 | } | ||
1511 | }, | ||
1512 | 'size': 8388608 }) | ||
1513 | @@ -XXX,XX +XXX,XX @@ with iotests.FilePath('t.img') as disk_path, \ | ||
1514 | |||
1515 | iotests.img_info_log(remote_path) | ||
1516 | |||
1517 | - sha1_key = subprocess.check_output( | ||
1518 | - 'ssh-keyscan -t rsa 127.0.0.1 2>/dev/null | grep -v "\\^#" | ' + | ||
1519 | - 'cut -d" " -f3 | base64 -d | sha1sum -b | cut -d" " -f1', | ||
1520 | - shell=True).rstrip().decode('ascii') | ||
1521 | - | ||
1522 | vm.launch() | ||
1523 | blockdev_create(vm, { 'driver': 'ssh', | ||
1524 | 'location': { | ||
1525 | @@ -XXX,XX +XXX,XX @@ with iotests.FilePath('t.img') as disk_path, \ | ||
1526 | 'host-key-check': { | ||
1527 | 'mode': 'hash', | ||
1528 | 'type': 'sha1', | ||
1529 | - 'hash': sha1_key, | ||
1530 | + 'hash': sha1_keys[matching_key], | ||
1531 | } | ||
1532 | }, | ||
1533 | 'size': 4194304 }) | ||
1534 | diff --git a/tests/qemu-iotests/207.out b/tests/qemu-iotests/207.out | ||
1535 | index XXXXXXX..XXXXXXX 100644 | ||
1536 | --- a/tests/qemu-iotests/207.out | ||
1537 | +++ b/tests/qemu-iotests/207.out | ||
1538 | @@ -XXX,XX +XXX,XX @@ virtual size: 4 MiB (4194304 bytes) | ||
1539 | |||
1540 | {"execute": "blockdev-create", "arguments": {"job-id": "job0", "options": {"driver": "ssh", "location": {"host-key-check": {"mode": "none"}, "path": "/this/is/not/an/existing/path", "server": {"host": "127.0.0.1", "port": "22"}}, "size": 4194304}}} | ||
1541 | {"return": {}} | ||
1542 | -Job failed: failed to open remote file '/this/is/not/an/existing/path': Failed opening remote file (libssh2 error code: -31) | ||
1543 | +Job failed: failed to open remote file '/this/is/not/an/existing/path': SFTP server: No such file (libssh error code: 1, sftp error code: 2) | ||
1544 | {"execute": "job-dismiss", "arguments": {"id": "job0"}} | ||
1545 | {"return": {}} | ||
1546 | |||
1547 | -- | 175 | -- |
1548 | 2.21.0 | 176 | 2.21.0 |
1549 | 177 | ||
1550 | 178 | diff view generated by jsdifflib |
1 | From: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com> | 1 | From: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com> |
---|---|---|---|
2 | 2 | ||
3 | We forget to enable it for transaction .prepare, while it is already | 3 | It's wrong to OR shared permissions. It may lead to crash on further |
4 | enabled in do_drive_backup since commit a2d665c1bc362 | 4 | permission updates. |
5 | "blockdev: loosen restrictions on drive-backup source node" | 5 | Also, no needs to consider previously calculated permissions, as at |
6 | this point we already bind all new parents and bdrv_get_cumulative_perm | ||
7 | result is enough. So fix the bug by just set permissions by | ||
8 | bdrv_get_cumulative_perm result. | ||
9 | |||
10 | Bug was introduced in long ago 234ac1a9025, in 2.9. | ||
6 | 11 | ||
7 | Signed-off-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com> | 12 | Signed-off-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com> |
8 | Message-id: 20190618140804.59214-1-vsementsov@virtuozzo.com | 13 | Message-id: 20190824100740.61635-1-vsementsov@virtuozzo.com |
9 | Reviewed-by: John Snow <jsnow@redhat.com> | ||
10 | Signed-off-by: Max Reitz <mreitz@redhat.com> | 14 | Signed-off-by: Max Reitz <mreitz@redhat.com> |
11 | --- | 15 | --- |
12 | blockdev.c | 2 +- | 16 | block.c | 5 ++--- |
13 | 1 file changed, 1 insertion(+), 1 deletion(-) | 17 | 1 file changed, 2 insertions(+), 3 deletions(-) |
14 | 18 | ||
15 | diff --git a/blockdev.c b/blockdev.c | 19 | diff --git a/block.c b/block.c |
16 | index XXXXXXX..XXXXXXX 100644 | 20 | index XXXXXXX..XXXXXXX 100644 |
17 | --- a/blockdev.c | 21 | --- a/block.c |
18 | +++ b/blockdev.c | 22 | +++ b/block.c |
19 | @@ -XXX,XX +XXX,XX @@ static void drive_backup_prepare(BlkActionState *common, Error **errp) | 23 | @@ -XXX,XX +XXX,XX @@ void bdrv_replace_node(BlockDriverState *from, BlockDriverState *to, |
20 | assert(common->action->type == TRANSACTION_ACTION_KIND_DRIVE_BACKUP); | 24 | { |
21 | backup = common->action->u.drive_backup.data; | 25 | BdrvChild *c, *next; |
22 | 26 | GSList *list = NULL, *p; | |
23 | - bs = qmp_get_root_bs(backup->device, errp); | 27 | - uint64_t old_perm, old_shared; |
24 | + bs = bdrv_lookup_bs(backup->device, backup->device, errp); | 28 | uint64_t perm = 0, shared = BLK_PERM_ALL; |
25 | if (!bs) { | 29 | int ret; |
26 | return; | 30 | |
31 | @@ -XXX,XX +XXX,XX @@ void bdrv_replace_node(BlockDriverState *from, BlockDriverState *to, | ||
32 | bdrv_unref(from); | ||
27 | } | 33 | } |
34 | |||
35 | - bdrv_get_cumulative_perm(to, &old_perm, &old_shared); | ||
36 | - bdrv_set_perm(to, old_perm | perm, old_shared | shared); | ||
37 | + bdrv_get_cumulative_perm(to, &perm, &shared); | ||
38 | + bdrv_set_perm(to, perm, shared); | ||
39 | |||
40 | out: | ||
41 | g_slist_free(list); | ||
28 | -- | 42 | -- |
29 | 2.21.0 | 43 | 2.21.0 |
30 | 44 | ||
31 | 45 | diff view generated by jsdifflib |
1 | From: Sam Eiderman <shmuel.eiderman@oracle.com> | 1 | From: Nir Soffer <nirsof@gmail.com> |
---|---|---|---|
2 | 2 | ||
3 | 512M of L1 entries is a very loose bound, only 32M are required to store | 3 | When creating an image with preallocation "off" or "falloc", the first |
4 | the maximal supported VMDK file size of 2TB. | 4 | block of the image is typically not allocated. When using Gluster |
5 | 5 | storage backed by XFS filesystem, reading this block using direct I/O | |
6 | Fixed qemu-iotest 59# - now failure occures before on impossible L1 | 6 | succeeds regardless of request length, fooling alignment detection. |
7 | table size. | 7 | |
8 | 8 | In this case we fallback to a safe value (4096) instead of the optimal | |
9 | Reviewed-by: Karl Heubaum <karl.heubaum@oracle.com> | 9 | value (512), which may lead to unneeded data copying when aligning |
10 | Reviewed-by: Eyal Moscovici <eyal.moscovici@oracle.com> | 10 | requests. Allocating the first block avoids the fallback. |
11 | Reviewed-by: Liran Alon <liran.alon@oracle.com> | 11 | |
12 | Reviewed-by: Arbel Moshe <arbel.moshe@oracle.com> | 12 | Since we allocate the first block even with preallocation=off, we no |
13 | Signed-off-by: Sam Eiderman <shmuel.eiderman@oracle.com> | 13 | longer create images with zero disk size: |
14 | Message-id: 20190620091057.47441-3-shmuel.eiderman@oracle.com | 14 | |
15 | $ ./qemu-img create -f raw test.raw 1g | ||
16 | Formatting 'test.raw', fmt=raw size=1073741824 | ||
17 | |||
18 | $ ls -lhs test.raw | ||
19 | 4.0K -rw-r--r--. 1 nsoffer nsoffer 1.0G Aug 16 23:48 test.raw | ||
20 | |||
21 | And converting the image requires additional cluster: | ||
22 | |||
23 | $ ./qemu-img measure -f raw -O qcow2 test.raw | ||
24 | required size: 458752 | ||
25 | fully allocated size: 1074135040 | ||
26 | |||
27 | When using format like vmdk with multiple files per image, we allocate | ||
28 | one block per file: | ||
29 | |||
30 | $ ./qemu-img create -f vmdk -o subformat=twoGbMaxExtentFlat test.vmdk 4g | ||
31 | Formatting 'test.vmdk', fmt=vmdk size=4294967296 compat6=off hwversion=undefined subformat=twoGbMaxExtentFlat | ||
32 | |||
33 | $ ls -lhs test*.vmdk | ||
34 | 4.0K -rw-r--r--. 1 nsoffer nsoffer 2.0G Aug 27 03:23 test-f001.vmdk | ||
35 | 4.0K -rw-r--r--. 1 nsoffer nsoffer 2.0G Aug 27 03:23 test-f002.vmdk | ||
36 | 4.0K -rw-r--r--. 1 nsoffer nsoffer 353 Aug 27 03:23 test.vmdk | ||
37 | |||
38 | I did quick performance test for copying disks with qemu-img convert to | ||
39 | new raw target image to Gluster storage with sector size of 512 bytes: | ||
40 | |||
41 | for i in $(seq 10); do | ||
42 | rm -f dst.raw | ||
43 | sleep 10 | ||
44 | time ./qemu-img convert -f raw -O raw -t none -T none src.raw dst.raw | ||
45 | done | ||
46 | |||
47 | Here is a table comparing the total time spent: | ||
48 | |||
49 | Type Before(s) After(s) Diff(%) | ||
50 | --------------------------------------- | ||
51 | real 530.028 469.123 -11.4 | ||
52 | user 17.204 10.768 -37.4 | ||
53 | sys 17.881 7.011 -60.7 | ||
54 | |||
55 | We can see very clear improvement in CPU usage. | ||
56 | |||
57 | Signed-off-by: Nir Soffer <nsoffer@redhat.com> | ||
58 | Message-id: 20190827010528.8818-2-nsoffer@redhat.com | ||
15 | Reviewed-by: Max Reitz <mreitz@redhat.com> | 59 | Reviewed-by: Max Reitz <mreitz@redhat.com> |
16 | Signed-off-by: Max Reitz <mreitz@redhat.com> | 60 | Signed-off-by: Max Reitz <mreitz@redhat.com> |
17 | --- | 61 | --- |
18 | block/vmdk.c | 13 +++++++------ | 62 | block/file-posix.c | 51 +++++++++++++++++++ |
19 | tests/qemu-iotests/059.out | 2 +- | 63 | tests/qemu-iotests/059.out | 2 +- |
20 | 2 files changed, 8 insertions(+), 7 deletions(-) | 64 | tests/qemu-iotests/{150.out => 150.out.qcow2} | 0 |
21 | 65 | tests/qemu-iotests/150.out.raw | 12 +++++ | |
22 | diff --git a/block/vmdk.c b/block/vmdk.c | 66 | tests/qemu-iotests/175 | 19 ++++--- |
23 | index XXXXXXX..XXXXXXX 100644 | 67 | tests/qemu-iotests/175.out | 8 +-- |
24 | --- a/block/vmdk.c | 68 | tests/qemu-iotests/178.out.qcow2 | 4 +- |
25 | +++ b/block/vmdk.c | 69 | tests/qemu-iotests/221.out | 12 +++-- |
26 | @@ -XXX,XX +XXX,XX @@ static int vmdk_add_extent(BlockDriverState *bs, | 70 | tests/qemu-iotests/253.out | 12 +++-- |
27 | error_setg(errp, "Invalid granularity, image may be corrupt"); | 71 | 9 files changed, 99 insertions(+), 21 deletions(-) |
28 | return -EFBIG; | 72 | rename tests/qemu-iotests/{150.out => 150.out.qcow2} (100%) |
29 | } | 73 | create mode 100644 tests/qemu-iotests/150.out.raw |
30 | - if (l1_size > 512 * 1024 * 1024) { | 74 | |
31 | + if (l1_size > 32 * 1024 * 1024) { | 75 | diff --git a/block/file-posix.c b/block/file-posix.c |
32 | /* | 76 | index XXXXXXX..XXXXXXX 100644 |
33 | * Although with big capacity and small l1_entry_sectors, we can get a | 77 | --- a/block/file-posix.c |
34 | * big l1_size, we don't want unbounded value to allocate the table. | 78 | +++ b/block/file-posix.c |
35 | - * Limit it to 512M, which is: | 79 | @@ -XXX,XX +XXX,XX @@ static int handle_aiocb_discard(void *opaque) |
36 | - * 16PB - for default "Hosted Sparse Extent" (VMDK4) | 80 | return ret; |
37 | - * cluster size: 64KB, L2 table size: 512 entries | 81 | } |
38 | - * 1PB - for default "ESXi Host Sparse Extent" (VMDK3/vmfsSparse) | 82 | |
39 | - * cluster size: 512B, L2 table size: 4096 entries | 83 | +/* |
40 | + * Limit it to 32M, which is enough to store: | 84 | + * Help alignment probing by allocating the first block. |
41 | + * 8TB - for both VMDK3 & VMDK4 with | 85 | + * |
42 | + * minimal cluster size: 512B | 86 | + * When reading with direct I/O from unallocated area on Gluster backed by XFS, |
43 | + * minimal L2 table size: 512 entries | 87 | + * reading succeeds regardless of request length. In this case we fallback to |
44 | + * 8 TB is still more than the maximal value supported for | 88 | + * safe alignment which is not optimal. Allocating the first block avoids this |
45 | + * VMDK3 & VMDK4 which is 2TB. | 89 | + * fallback. |
46 | */ | 90 | + * |
47 | error_setg(errp, "L1 size too big"); | 91 | + * fd may be opened with O_DIRECT, but we don't know the buffer alignment or |
48 | return -EFBIG; | 92 | + * request alignment, so we use safe values. |
93 | + * | ||
94 | + * Returns: 0 on success, -errno on failure. Since this is an optimization, | ||
95 | + * caller may ignore failures. | ||
96 | + */ | ||
97 | +static int allocate_first_block(int fd, size_t max_size) | ||
98 | +{ | ||
99 | + size_t write_size = (max_size < MAX_BLOCKSIZE) | ||
100 | + ? BDRV_SECTOR_SIZE | ||
101 | + : MAX_BLOCKSIZE; | ||
102 | + size_t max_align = MAX(MAX_BLOCKSIZE, getpagesize()); | ||
103 | + void *buf; | ||
104 | + ssize_t n; | ||
105 | + int ret; | ||
106 | + | ||
107 | + buf = qemu_memalign(max_align, write_size); | ||
108 | + memset(buf, 0, write_size); | ||
109 | + | ||
110 | + do { | ||
111 | + n = pwrite(fd, buf, write_size, 0); | ||
112 | + } while (n == -1 && errno == EINTR); | ||
113 | + | ||
114 | + ret = (n == -1) ? -errno : 0; | ||
115 | + | ||
116 | + qemu_vfree(buf); | ||
117 | + return ret; | ||
118 | +} | ||
119 | + | ||
120 | static int handle_aiocb_truncate(void *opaque) | ||
121 | { | ||
122 | RawPosixAIOData *aiocb = opaque; | ||
123 | @@ -XXX,XX +XXX,XX @@ static int handle_aiocb_truncate(void *opaque) | ||
124 | /* posix_fallocate() doesn't set errno. */ | ||
125 | error_setg_errno(errp, -result, | ||
126 | "Could not preallocate new data"); | ||
127 | + } else if (current_length == 0) { | ||
128 | + /* | ||
129 | + * posix_fallocate() uses fallocate() if the filesystem | ||
130 | + * supports it, or fallback to manually writing zeroes. If | ||
131 | + * fallocate() was used, unaligned reads from the fallocated | ||
132 | + * area in raw_probe_alignment() will succeed, hence we need to | ||
133 | + * allocate the first block. | ||
134 | + * | ||
135 | + * Optimize future alignment probing; ignore failures. | ||
136 | + */ | ||
137 | + allocate_first_block(fd, offset); | ||
138 | } | ||
139 | } else { | ||
140 | result = 0; | ||
141 | @@ -XXX,XX +XXX,XX @@ static int handle_aiocb_truncate(void *opaque) | ||
142 | if (ftruncate(fd, offset) != 0) { | ||
143 | result = -errno; | ||
144 | error_setg_errno(errp, -result, "Could not resize file"); | ||
145 | + } else if (current_length == 0 && offset > current_length) { | ||
146 | + /* Optimize future alignment probing; ignore failures. */ | ||
147 | + allocate_first_block(fd, offset); | ||
148 | } | ||
149 | return result; | ||
150 | default: | ||
49 | diff --git a/tests/qemu-iotests/059.out b/tests/qemu-iotests/059.out | 151 | diff --git a/tests/qemu-iotests/059.out b/tests/qemu-iotests/059.out |
50 | index XXXXXXX..XXXXXXX 100644 | 152 | index XXXXXXX..XXXXXXX 100644 |
51 | --- a/tests/qemu-iotests/059.out | 153 | --- a/tests/qemu-iotests/059.out |
52 | +++ b/tests/qemu-iotests/059.out | 154 | +++ b/tests/qemu-iotests/059.out |
53 | @@ -XXX,XX +XXX,XX @@ Offset Length Mapped to File | 155 | @@ -XXX,XX +XXX,XX @@ Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824000 subformat=twoGbMax |
54 | 0x140000000 0x10000 0x50000 TEST_DIR/t-s003.vmdk | 156 | image: TEST_DIR/t.vmdk |
55 | 157 | file format: vmdk | |
56 | === Testing afl image with a very large capacity === | 158 | virtual size: 0.977 TiB (1073741824000 bytes) |
57 | -qemu-img: Can't get image size 'TEST_DIR/afl9.IMGFMT': File too large | 159 | -disk size: 16 KiB |
58 | +qemu-img: Could not open 'TEST_DIR/afl9.IMGFMT': L1 size too big | 160 | +disk size: 1.97 MiB |
161 | Format specific information: | ||
162 | cid: XXXXXXXX | ||
163 | parent cid: XXXXXXXX | ||
164 | diff --git a/tests/qemu-iotests/150.out b/tests/qemu-iotests/150.out.qcow2 | ||
165 | similarity index 100% | ||
166 | rename from tests/qemu-iotests/150.out | ||
167 | rename to tests/qemu-iotests/150.out.qcow2 | ||
168 | diff --git a/tests/qemu-iotests/150.out.raw b/tests/qemu-iotests/150.out.raw | ||
169 | new file mode 100644 | ||
170 | index XXXXXXX..XXXXXXX | ||
171 | --- /dev/null | ||
172 | +++ b/tests/qemu-iotests/150.out.raw | ||
173 | @@ -XXX,XX +XXX,XX @@ | ||
174 | +QA output created by 150 | ||
175 | + | ||
176 | +=== Mapping sparse conversion === | ||
177 | + | ||
178 | +Offset Length File | ||
179 | +0 0x1000 TEST_DIR/t.IMGFMT | ||
180 | + | ||
181 | +=== Mapping non-sparse conversion === | ||
182 | + | ||
183 | +Offset Length File | ||
184 | +0 0x100000 TEST_DIR/t.IMGFMT | ||
185 | +*** done | ||
186 | diff --git a/tests/qemu-iotests/175 b/tests/qemu-iotests/175 | ||
187 | index XXXXXXX..XXXXXXX 100755 | ||
188 | --- a/tests/qemu-iotests/175 | ||
189 | +++ b/tests/qemu-iotests/175 | ||
190 | @@ -XXX,XX +XXX,XX @@ trap "_cleanup; exit \$status" 0 1 2 3 15 | ||
191 | # the file size. This function hides the resulting difference in the | ||
192 | # stat -c '%b' output. | ||
193 | # Parameter 1: Number of blocks an empty file occupies | ||
194 | -# Parameter 2: Image size in bytes | ||
195 | +# Parameter 2: Minimal number of blocks in an image | ||
196 | +# Parameter 3: Image size in bytes | ||
197 | _filter_blocks() | ||
198 | { | ||
199 | extra_blocks=$1 | ||
200 | - img_size=$2 | ||
201 | + min_blocks=$2 | ||
202 | + img_size=$3 | ||
203 | |||
204 | - sed -e "s/blocks=$extra_blocks\\(\$\\|[^0-9]\\)/nothing allocated/" \ | ||
205 | - -e "s/blocks=$((extra_blocks + img_size / 512))\\(\$\\|[^0-9]\\)/everything allocated/" | ||
206 | + sed -e "s/blocks=$min_blocks\\(\$\\|[^0-9]\\)/min allocation/" \ | ||
207 | + -e "s/blocks=$((extra_blocks + img_size / 512))\\(\$\\|[^0-9]\\)/max allocation/" | ||
208 | } | ||
209 | |||
210 | # get standard environment, filters and checks | ||
211 | @@ -XXX,XX +XXX,XX @@ size=$((1 * 1024 * 1024)) | ||
212 | touch "$TEST_DIR/empty" | ||
213 | extra_blocks=$(stat -c '%b' "$TEST_DIR/empty") | ||
214 | |||
215 | +# We always write the first byte; check how many blocks this filesystem | ||
216 | +# allocates to match empty image alloation. | ||
217 | +printf "\0" > "$TEST_DIR/empty" | ||
218 | +min_blocks=$(stat -c '%b' "$TEST_DIR/empty") | ||
219 | + | ||
220 | echo | ||
221 | echo "== creating image with default preallocation ==" | ||
222 | _make_test_img $size | _filter_imgfmt | ||
223 | -stat -c "size=%s, blocks=%b" $TEST_IMG | _filter_blocks $extra_blocks $size | ||
224 | +stat -c "size=%s, blocks=%b" $TEST_IMG | _filter_blocks $extra_blocks $min_blocks $size | ||
225 | |||
226 | for mode in off full falloc; do | ||
227 | echo | ||
228 | echo "== creating image with preallocation $mode ==" | ||
229 | IMGOPTS=preallocation=$mode _make_test_img $size | _filter_imgfmt | ||
230 | - stat -c "size=%s, blocks=%b" $TEST_IMG | _filter_blocks $extra_blocks $size | ||
231 | + stat -c "size=%s, blocks=%b" $TEST_IMG | _filter_blocks $extra_blocks $min_blocks $size | ||
232 | done | ||
233 | |||
234 | # success, all done | ||
235 | diff --git a/tests/qemu-iotests/175.out b/tests/qemu-iotests/175.out | ||
236 | index XXXXXXX..XXXXXXX 100644 | ||
237 | --- a/tests/qemu-iotests/175.out | ||
238 | +++ b/tests/qemu-iotests/175.out | ||
239 | @@ -XXX,XX +XXX,XX @@ QA output created by 175 | ||
240 | |||
241 | == creating image with default preallocation == | ||
242 | Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1048576 | ||
243 | -size=1048576, nothing allocated | ||
244 | +size=1048576, min allocation | ||
245 | |||
246 | == creating image with preallocation off == | ||
247 | Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1048576 preallocation=off | ||
248 | -size=1048576, nothing allocated | ||
249 | +size=1048576, min allocation | ||
250 | |||
251 | == creating image with preallocation full == | ||
252 | Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1048576 preallocation=full | ||
253 | -size=1048576, everything allocated | ||
254 | +size=1048576, max allocation | ||
255 | |||
256 | == creating image with preallocation falloc == | ||
257 | Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1048576 preallocation=falloc | ||
258 | -size=1048576, everything allocated | ||
259 | +size=1048576, max allocation | ||
260 | *** done | ||
261 | diff --git a/tests/qemu-iotests/178.out.qcow2 b/tests/qemu-iotests/178.out.qcow2 | ||
262 | index XXXXXXX..XXXXXXX 100644 | ||
263 | --- a/tests/qemu-iotests/178.out.qcow2 | ||
264 | +++ b/tests/qemu-iotests/178.out.qcow2 | ||
265 | @@ -XXX,XX +XXX,XX @@ converted image file size in bytes: 196608 | ||
266 | == raw input image with data (human) == | ||
267 | |||
268 | Formatting 'TEST_DIR/t.qcow2', fmt=IMGFMT size=1073741824 | ||
269 | -required size: 393216 | ||
270 | +required size: 458752 | ||
271 | fully allocated size: 1074135040 | ||
272 | wrote 512/512 bytes at offset 512 | ||
273 | 512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) | ||
274 | @@ -XXX,XX +XXX,XX @@ converted image file size in bytes: 196608 | ||
275 | |||
276 | Formatting 'TEST_DIR/t.qcow2', fmt=IMGFMT size=1073741824 | ||
277 | { | ||
278 | - "required": 393216, | ||
279 | + "required": 458752, | ||
280 | "fully-allocated": 1074135040 | ||
281 | } | ||
282 | wrote 512/512 bytes at offset 512 | ||
283 | diff --git a/tests/qemu-iotests/221.out b/tests/qemu-iotests/221.out | ||
284 | index XXXXXXX..XXXXXXX 100644 | ||
285 | --- a/tests/qemu-iotests/221.out | ||
286 | +++ b/tests/qemu-iotests/221.out | ||
287 | @@ -XXX,XX +XXX,XX @@ QA output created by 221 | ||
288 | === Check mapping of unaligned raw image === | ||
289 | |||
290 | Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=65537 | ||
291 | -[{ "start": 0, "length": 66048, "depth": 0, "zero": true, "data": false, "offset": OFFSET}] | ||
292 | -[{ "start": 0, "length": 66048, "depth": 0, "zero": true, "data": false, "offset": OFFSET}] | ||
293 | +[{ "start": 0, "length": 4096, "depth": 0, "zero": false, "data": true, "offset": OFFSET}, | ||
294 | +{ "start": 4096, "length": 61952, "depth": 0, "zero": true, "data": false, "offset": OFFSET}] | ||
295 | +[{ "start": 0, "length": 4096, "depth": 0, "zero": false, "data": true, "offset": OFFSET}, | ||
296 | +{ "start": 4096, "length": 61952, "depth": 0, "zero": true, "data": false, "offset": OFFSET}] | ||
297 | wrote 1/1 bytes at offset 65536 | ||
298 | 1 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) | ||
299 | -[{ "start": 0, "length": 65536, "depth": 0, "zero": true, "data": false, "offset": OFFSET}, | ||
300 | +[{ "start": 0, "length": 4096, "depth": 0, "zero": false, "data": true, "offset": OFFSET}, | ||
301 | +{ "start": 4096, "length": 61440, "depth": 0, "zero": true, "data": false, "offset": OFFSET}, | ||
302 | { "start": 65536, "length": 1, "depth": 0, "zero": false, "data": true, "offset": OFFSET}, | ||
303 | { "start": 65537, "length": 511, "depth": 0, "zero": true, "data": false, "offset": OFFSET}] | ||
304 | -[{ "start": 0, "length": 65536, "depth": 0, "zero": true, "data": false, "offset": OFFSET}, | ||
305 | +[{ "start": 0, "length": 4096, "depth": 0, "zero": false, "data": true, "offset": OFFSET}, | ||
306 | +{ "start": 4096, "length": 61440, "depth": 0, "zero": true, "data": false, "offset": OFFSET}, | ||
307 | { "start": 65536, "length": 1, "depth": 0, "zero": false, "data": true, "offset": OFFSET}, | ||
308 | { "start": 65537, "length": 511, "depth": 0, "zero": true, "data": false, "offset": OFFSET}] | ||
309 | *** done | ||
310 | diff --git a/tests/qemu-iotests/253.out b/tests/qemu-iotests/253.out | ||
311 | index XXXXXXX..XXXXXXX 100644 | ||
312 | --- a/tests/qemu-iotests/253.out | ||
313 | +++ b/tests/qemu-iotests/253.out | ||
314 | @@ -XXX,XX +XXX,XX @@ QA output created by 253 | ||
315 | === Check mapping of unaligned raw image === | ||
316 | |||
317 | Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1048575 | ||
318 | -[{ "start": 0, "length": 1048576, "depth": 0, "zero": true, "data": false, "offset": OFFSET}] | ||
319 | -[{ "start": 0, "length": 1048576, "depth": 0, "zero": true, "data": false, "offset": OFFSET}] | ||
320 | +[{ "start": 0, "length": 4096, "depth": 0, "zero": false, "data": true, "offset": OFFSET}, | ||
321 | +{ "start": 4096, "length": 1044480, "depth": 0, "zero": true, "data": false, "offset": OFFSET}] | ||
322 | +[{ "start": 0, "length": 4096, "depth": 0, "zero": false, "data": true, "offset": OFFSET}, | ||
323 | +{ "start": 4096, "length": 1044480, "depth": 0, "zero": true, "data": false, "offset": OFFSET}] | ||
324 | wrote 65535/65535 bytes at offset 983040 | ||
325 | 63.999 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) | ||
326 | -[{ "start": 0, "length": 983040, "depth": 0, "zero": true, "data": false, "offset": OFFSET}, | ||
327 | +[{ "start": 0, "length": 4096, "depth": 0, "zero": false, "data": true, "offset": OFFSET}, | ||
328 | +{ "start": 4096, "length": 978944, "depth": 0, "zero": true, "data": false, "offset": OFFSET}, | ||
329 | { "start": 983040, "length": 65536, "depth": 0, "zero": false, "data": true, "offset": OFFSET}] | ||
330 | -[{ "start": 0, "length": 983040, "depth": 0, "zero": true, "data": false, "offset": OFFSET}, | ||
331 | +[{ "start": 0, "length": 4096, "depth": 0, "zero": false, "data": true, "offset": OFFSET}, | ||
332 | +{ "start": 4096, "length": 978944, "depth": 0, "zero": true, "data": false, "offset": OFFSET}, | ||
333 | { "start": 983040, "length": 65536, "depth": 0, "zero": false, "data": true, "offset": OFFSET}] | ||
59 | *** done | 334 | *** done |
60 | -- | 335 | -- |
61 | 2.21.0 | 336 | 2.21.0 |
62 | 337 | ||
63 | 338 | diff view generated by jsdifflib |
New patch | |||
---|---|---|---|
1 | From: Nir Soffer <nirsof@gmail.com> | ||
1 | 2 | ||
3 | Using block_resize we can test allocate_first_block() with file | ||
4 | descriptor opened with O_DIRECT, ensuring that it works for any size | ||
5 | larger than 4096 bytes. | ||
6 | |||
7 | Testing smaller sizes is tricky as the result depends on the filesystem | ||
8 | used for testing. For example on NFS any size will work since O_DIRECT | ||
9 | does not require any alignment. | ||
10 | |||
11 | Signed-off-by: Nir Soffer <nsoffer@redhat.com> | ||
12 | Reviewed-by: Max Reitz <mreitz@redhat.com> | ||
13 | Message-id: 20190827010528.8818-3-nsoffer@redhat.com | ||
14 | Signed-off-by: Max Reitz <mreitz@redhat.com> | ||
15 | --- | ||
16 | tests/qemu-iotests/175 | 28 ++++++++++++++++++++++++++++ | ||
17 | tests/qemu-iotests/175.out | 8 ++++++++ | ||
18 | 2 files changed, 36 insertions(+) | ||
19 | |||
20 | diff --git a/tests/qemu-iotests/175 b/tests/qemu-iotests/175 | ||
21 | index XXXXXXX..XXXXXXX 100755 | ||
22 | --- a/tests/qemu-iotests/175 | ||
23 | +++ b/tests/qemu-iotests/175 | ||
24 | @@ -XXX,XX +XXX,XX @@ _filter_blocks() | ||
25 | -e "s/blocks=$((extra_blocks + img_size / 512))\\(\$\\|[^0-9]\\)/max allocation/" | ||
26 | } | ||
27 | |||
28 | +# Resize image using block_resize. | ||
29 | +# Parameter 1: image path | ||
30 | +# Parameter 2: new size | ||
31 | +_block_resize() | ||
32 | +{ | ||
33 | + local path=$1 | ||
34 | + local size=$2 | ||
35 | + | ||
36 | + $QEMU -qmp stdio -nographic -nodefaults \ | ||
37 | + -blockdev file,node-name=file,filename=$path,cache.direct=on \ | ||
38 | + <<EOF | ||
39 | +{'execute': 'qmp_capabilities'} | ||
40 | +{'execute': 'block_resize', 'arguments': {'node-name': 'file', 'size': $size}} | ||
41 | +{'execute': 'quit'} | ||
42 | +EOF | ||
43 | +} | ||
44 | + | ||
45 | # get standard environment, filters and checks | ||
46 | . ./common.rc | ||
47 | . ./common.filter | ||
48 | @@ -XXX,XX +XXX,XX @@ _supported_fmt raw | ||
49 | _supported_proto file | ||
50 | _supported_os Linux | ||
51 | |||
52 | +_default_cache_mode none | ||
53 | +_supported_cache_modes none directsync | ||
54 | + | ||
55 | size=$((1 * 1024 * 1024)) | ||
56 | |||
57 | touch "$TEST_DIR/empty" | ||
58 | @@ -XXX,XX +XXX,XX @@ for mode in off full falloc; do | ||
59 | stat -c "size=%s, blocks=%b" $TEST_IMG | _filter_blocks $extra_blocks $min_blocks $size | ||
60 | done | ||
61 | |||
62 | +for new_size in 4096 1048576; do | ||
63 | + echo | ||
64 | + echo "== resize empty image with block_resize ==" | ||
65 | + _make_test_img 0 | _filter_imgfmt | ||
66 | + _block_resize $TEST_IMG $new_size >/dev/null | ||
67 | + stat -c "size=%s, blocks=%b" $TEST_IMG | _filter_blocks $extra_blocks $min_blocks $new_size | ||
68 | +done | ||
69 | + | ||
70 | # success, all done | ||
71 | echo "*** done" | ||
72 | rm -f $seq.full | ||
73 | diff --git a/tests/qemu-iotests/175.out b/tests/qemu-iotests/175.out | ||
74 | index XXXXXXX..XXXXXXX 100644 | ||
75 | --- a/tests/qemu-iotests/175.out | ||
76 | +++ b/tests/qemu-iotests/175.out | ||
77 | @@ -XXX,XX +XXX,XX @@ size=1048576, max allocation | ||
78 | == creating image with preallocation falloc == | ||
79 | Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1048576 preallocation=falloc | ||
80 | size=1048576, max allocation | ||
81 | + | ||
82 | +== resize empty image with block_resize == | ||
83 | +Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=0 | ||
84 | +size=4096, min allocation | ||
85 | + | ||
86 | +== resize empty image with block_resize == | ||
87 | +Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=0 | ||
88 | +size=1048576, min allocation | ||
89 | *** done | ||
90 | -- | ||
91 | 2.21.0 | ||
92 | |||
93 | diff view generated by jsdifflib |
New patch | |||
---|---|---|---|
1 | fe646693acc changed qemu-img create's output so that it no longer prints | ||
2 | single quotes around parameter values. The subformat and adapter_type | ||
3 | filters in _filter_img_create() have never been adapted to that change. | ||
1 | 4 | ||
5 | Fixes: fe646693acc13ac48b98435d14149ab04dc597bc | ||
6 | Signed-off-by: Max Reitz <mreitz@redhat.com> | ||
7 | Reviewed-by: John Snow <jsnow@redhat.com> | ||
8 | Message-id: 20190815153638.4600-2-mreitz@redhat.com | ||
9 | Reviewed-by: John Snow <jsnow@redhat.com> | ||
10 | Signed-off-by: Max Reitz <mreitz@redhat.com> | ||
11 | --- | ||
12 | tests/qemu-iotests/059.out | 16 ++++++++-------- | ||
13 | tests/qemu-iotests/common.filter | 4 ++-- | ||
14 | 2 files changed, 10 insertions(+), 10 deletions(-) | ||
15 | |||
16 | diff --git a/tests/qemu-iotests/059.out b/tests/qemu-iotests/059.out | ||
17 | index XXXXXXX..XXXXXXX 100644 | ||
18 | --- a/tests/qemu-iotests/059.out | ||
19 | +++ b/tests/qemu-iotests/059.out | ||
20 | @@ -XXX,XX +XXX,XX @@ Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 | ||
21 | qemu-io: can't open device TEST_DIR/t.vmdk: L1 size too big | ||
22 | |||
23 | === Testing monolithicFlat creation and opening === | ||
24 | -Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=2147483648 subformat=monolithicFlat | ||
25 | +Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=2147483648 | ||
26 | image: TEST_DIR/t.IMGFMT | ||
27 | file format: IMGFMT | ||
28 | virtual size: 2 GiB (2147483648 bytes) | ||
29 | |||
30 | === Testing monolithicFlat with zeroed_grain === | ||
31 | qemu-img: TEST_DIR/t.IMGFMT: Flat image can't enable zeroed grain | ||
32 | -Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=2147483648 subformat=monolithicFlat | ||
33 | +Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=2147483648 | ||
34 | |||
35 | === Testing big twoGbMaxExtentFlat === | ||
36 | -Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824000 subformat=twoGbMaxExtentFlat | ||
37 | +Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824000 | ||
38 | image: TEST_DIR/t.vmdk | ||
39 | file format: vmdk | ||
40 | virtual size: 0.977 TiB (1073741824000 bytes) | ||
41 | @@ -XXX,XX +XXX,XX @@ Format specific information: | ||
42 | qemu-img: Could not open 'TEST_DIR/t.IMGFMT': Invalid extent line: RW 12582912 VMFS "dummy.IMGFMT" 1 | ||
43 | |||
44 | === Testing truncated sparse === | ||
45 | -Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=107374182400 subformat=monolithicSparse | ||
46 | +Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=107374182400 | ||
47 | qemu-img: Could not open 'TEST_DIR/t.IMGFMT': File truncated, expecting at least 13172736 bytes | ||
48 | |||
49 | === Converting to streamOptimized from image with small cluster size=== | ||
50 | @@ -XXX,XX +XXX,XX @@ wrote 512/512 bytes at offset 10240 | ||
51 | 512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) | ||
52 | |||
53 | === Testing monolithicFlat with internally generated JSON file name === | ||
54 | -Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 subformat=monolithicFlat | ||
55 | +Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 | ||
56 | qemu-io: can't open: Cannot use relative extent paths with VMDK descriptor file 'json:{"image": {"driver": "file", "filename": "TEST_DIR/t.IMGFMT"}, "driver": "blkdebug", "inject-error.0.event": "read_aio"}' | ||
57 | |||
58 | === Testing version 3 === | ||
59 | @@ -XXX,XX +XXX,XX @@ read 512/512 bytes at offset 64931328 | ||
60 | 512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) | ||
61 | |||
62 | === Testing 4TB monolithicFlat creation and IO === | ||
63 | -Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=4398046511104 subformat=monolithicFlat | ||
64 | +Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=4398046511104 | ||
65 | image: TEST_DIR/t.IMGFMT | ||
66 | file format: IMGFMT | ||
67 | virtual size: 4 TiB (4398046511104 bytes) | ||
68 | @@ -XXX,XX +XXX,XX @@ read 1024/1024 bytes at offset 966367641600 | ||
69 | 1 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) | ||
70 | |||
71 | === Testing qemu-img map on extents === | ||
72 | -Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=33285996544 subformat=monolithicSparse | ||
73 | +Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=33285996544 | ||
74 | wrote 1024/1024 bytes at offset 65024 | ||
75 | 1 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) | ||
76 | wrote 1024/1024 bytes at offset 2147483136 | ||
77 | @@ -XXX,XX +XXX,XX @@ Offset Length Mapped to File | ||
78 | 0 0x20000 0x3f0000 TEST_DIR/t.vmdk | ||
79 | 0x7fff0000 0x20000 0x410000 TEST_DIR/t.vmdk | ||
80 | 0x140000000 0x10000 0x430000 TEST_DIR/t.vmdk | ||
81 | -Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=33285996544 subformat=twoGbMaxExtentSparse | ||
82 | +Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=33285996544 | ||
83 | wrote 1024/1024 bytes at offset 65024 | ||
84 | 1 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) | ||
85 | wrote 1024/1024 bytes at offset 2147483136 | ||
86 | diff --git a/tests/qemu-iotests/common.filter b/tests/qemu-iotests/common.filter | ||
87 | index XXXXXXX..XXXXXXX 100644 | ||
88 | --- a/tests/qemu-iotests/common.filter | ||
89 | +++ b/tests/qemu-iotests/common.filter | ||
90 | @@ -XXX,XX +XXX,XX @@ _filter_img_create() | ||
91 | -e "s# compat6=\\(on\\|off\\)##g" \ | ||
92 | -e "s# static=\\(on\\|off\\)##g" \ | ||
93 | -e "s# zeroed_grain=\\(on\\|off\\)##g" \ | ||
94 | - -e "s# subformat='[^']*'##g" \ | ||
95 | - -e "s# adapter_type='[^']*'##g" \ | ||
96 | + -e "s# subformat=[^ ]*##g" \ | ||
97 | + -e "s# adapter_type=[^ ]*##g" \ | ||
98 | -e "s# hwversion=[^ ]*##g" \ | ||
99 | -e "s# lazy_refcounts=\\(on\\|off\\)##g" \ | ||
100 | -e "s# block_size=[0-9]\\+##g" \ | ||
101 | -- | ||
102 | 2.21.0 | ||
103 | |||
104 | diff view generated by jsdifflib |
1 | From: Sam Eiderman <shmuel.eiderman@oracle.com> | 1 | This makes iotest 033 pass with e.g. subformat=monolithicFlat. It also |
---|---|---|---|
2 | turns a former error in 059 into success. | ||
2 | 3 | ||
3 | Until ESXi 6.5 VMware used the vmfsSparse format for snapshots (VMDK3 in | 4 | Signed-off-by: Max Reitz <mreitz@redhat.com> |
4 | QEMU). | 5 | Message-id: 20190815153638.4600-3-mreitz@redhat.com |
5 | 6 | Reviewed-by: John Snow <jsnow@redhat.com> | |
6 | This format was lacking in the following: | ||
7 | |||
8 | * Grain directory (L1) and grain table (L2) entries were 32-bit, | ||
9 | allowing access to only 2TB (slightly less) of data. | ||
10 | * The grain size (default) was 512 bytes - leading to data | ||
11 | fragmentation and many grain tables. | ||
12 | * For space reclamation purposes, it was necessary to find all the | ||
13 | grains which are not pointed to by any grain table - so a reverse | ||
14 | mapping of "offset of grain in vmdk" to "grain table" must be | ||
15 | constructed - which takes large amounts of CPU/RAM. | ||
16 | |||
17 | The format specification can be found in VMware's documentation: | ||
18 | https://www.vmware.com/support/developer/vddk/vmdk_50_technote.pdf | ||
19 | |||
20 | In ESXi 6.5, to support snapshot files larger than 2TB, a new format was | ||
21 | introduced: SESparse (Space Efficient). | ||
22 | |||
23 | This format fixes the above issues: | ||
24 | |||
25 | * All entries are now 64-bit. | ||
26 | * The grain size (default) is 4KB. | ||
27 | * Grain directory and grain tables are now located at the beginning | ||
28 | of the file. | ||
29 | + seSparse format reserves space for all grain tables. | ||
30 | + Grain tables can be addressed using an index. | ||
31 | + Grains are located in the end of the file and can also be | ||
32 | addressed with an index. | ||
33 | - seSparse vmdks of large disks (64TB) have huge preallocated | ||
34 | headers - mainly due to L2 tables, even for empty snapshots. | ||
35 | * The header contains a reverse mapping ("backmap") of "offset of | ||
36 | grain in vmdk" to "grain table" and a bitmap ("free bitmap") which | ||
37 | specifies for each grain - whether it is allocated or not. | ||
38 | Using these data structures we can implement space reclamation | ||
39 | efficiently. | ||
40 | * Due to the fact that the header now maintains two mappings: | ||
41 | * The regular one (grain directory & grain tables) | ||
42 | * A reverse one (backmap and free bitmap) | ||
43 | These data structures can lose consistency upon crash and result | ||
44 | in a corrupted VMDK. | ||
45 | Therefore, a journal is also added to the VMDK and is replayed | ||
46 | when the VMware reopens the file after a crash. | ||
47 | |||
48 | Since ESXi 6.7 - SESparse is the only snapshot format available. | ||
49 | |||
50 | Unfortunately, VMware does not provide documentation regarding the new | ||
51 | seSparse format. | ||
52 | |||
53 | This commit is based on black-box research of the seSparse format. | ||
54 | Various in-guest block operations and their effect on the snapshot file | ||
55 | were tested. | ||
56 | |||
57 | The only VMware provided source of information (regarding the underlying | ||
58 | implementation) was a log file on the ESXi: | ||
59 | |||
60 | /var/log/hostd.log | ||
61 | |||
62 | Whenever an seSparse snapshot is created - the log is being populated | ||
63 | with seSparse records. | ||
64 | |||
65 | Relevant log records are of the form: | ||
66 | |||
67 | [...] Const Header: | ||
68 | [...] constMagic = 0xcafebabe | ||
69 | [...] version = 2.1 | ||
70 | [...] capacity = 204800 | ||
71 | [...] grainSize = 8 | ||
72 | [...] grainTableSize = 64 | ||
73 | [...] flags = 0 | ||
74 | [...] Extents: | ||
75 | [...] Header : <1 : 1> | ||
76 | [...] JournalHdr : <2 : 2> | ||
77 | [...] Journal : <2048 : 2048> | ||
78 | [...] GrainDirectory : <4096 : 2048> | ||
79 | [...] GrainTables : <6144 : 2048> | ||
80 | [...] FreeBitmap : <8192 : 2048> | ||
81 | [...] BackMap : <10240 : 2048> | ||
82 | [...] Grain : <12288 : 204800> | ||
83 | [...] Volatile Header: | ||
84 | [...] volatileMagic = 0xcafecafe | ||
85 | [...] FreeGTNumber = 0 | ||
86 | [...] nextTxnSeqNumber = 0 | ||
87 | [...] replayJournal = 0 | ||
88 | |||
89 | The sizes that are seen in the log file are in sectors. | ||
90 | Extents are of the following format: <offset : size> | ||
91 | |||
92 | This commit is a strict implementation which enforces: | ||
93 | * magics | ||
94 | * version number 2.1 | ||
95 | * grain size of 8 sectors (4KB) | ||
96 | * grain table size of 64 sectors | ||
97 | * zero flags | ||
98 | * extent locations | ||
99 | |||
100 | Additionally, this commit proivdes only a subset of the functionality | ||
101 | offered by seSparse's format: | ||
102 | * Read-only | ||
103 | * No journal replay | ||
104 | * No space reclamation | ||
105 | * No unmap support | ||
106 | |||
107 | Hence, journal header, journal, free bitmap and backmap extents are | ||
108 | unused, only the "classic" (L1 -> L2 -> data) grain access is | ||
109 | implemented. | ||
110 | |||
111 | However there are several differences in the grain access itself. | ||
112 | Grain directory (L1): | ||
113 | * Grain directory entries are indexes (not offsets) to grain | ||
114 | tables. | ||
115 | * Valid grain directory entries have their highest nibble set to | ||
116 | 0x1. | ||
117 | * Since grain tables are always located in the beginning of the | ||
118 | file - the index can fit into 32 bits - so we can use its low | ||
119 | part if it's valid. | ||
120 | Grain table (L2): | ||
121 | * Grain table entries are indexes (not offsets) to grains. | ||
122 | * If the highest nibble of the entry is: | ||
123 | 0x0: | ||
124 | The grain in not allocated. | ||
125 | The rest of the bytes are 0. | ||
126 | 0x1: | ||
127 | The grain is unmapped - guest sees a zero grain. | ||
128 | The rest of the bits point to the previously mapped grain, | ||
129 | see 0x3 case. | ||
130 | 0x2: | ||
131 | The grain is zero. | ||
132 | 0x3: | ||
133 | The grain is allocated - to get the index calculate: | ||
134 | ((entry & 0x0fff000000000000) >> 48) | | ||
135 | ((entry & 0x0000ffffffffffff) << 12) | ||
136 | * The difference between 0x1 and 0x2 is that 0x1 is an unallocated | ||
137 | grain which results from the guest using sg_unmap to unmap the | ||
138 | grain - but the grain itself still exists in the grain extent - a | ||
139 | space reclamation procedure should delete it. | ||
140 | Unmapping a zero grain has no effect (0x2 will not change to 0x1) | ||
141 | but unmapping an unallocated grain will (0x0 to 0x1) - naturally. | ||
142 | |||
143 | In order to implement seSparse some fields had to be changed to support | ||
144 | both 32-bit and 64-bit entry sizes. | ||
145 | |||
146 | Reviewed-by: Karl Heubaum <karl.heubaum@oracle.com> | ||
147 | Reviewed-by: Eyal Moscovici <eyal.moscovici@oracle.com> | ||
148 | Reviewed-by: Arbel Moshe <arbel.moshe@oracle.com> | ||
149 | Signed-off-by: Sam Eiderman <shmuel.eiderman@oracle.com> | ||
150 | Message-id: 20190620091057.47441-4-shmuel.eiderman@oracle.com | ||
151 | Signed-off-by: Max Reitz <mreitz@redhat.com> | 7 | Signed-off-by: Max Reitz <mreitz@redhat.com> |
152 | --- | 8 | --- |
153 | block/vmdk.c | 358 ++++++++++++++++++++++++++++++++++++++++++++++++--- | 9 | block/vmdk.c | 54 ++++++++++++++++++++++++-------------- |
154 | 1 file changed, 342 insertions(+), 16 deletions(-) | 10 | tests/qemu-iotests/059 | 7 +++-- |
11 | tests/qemu-iotests/059.out | 4 ++- | ||
12 | 3 files changed, 42 insertions(+), 23 deletions(-) | ||
155 | 13 | ||
156 | diff --git a/block/vmdk.c b/block/vmdk.c | 14 | diff --git a/block/vmdk.c b/block/vmdk.c |
157 | index XXXXXXX..XXXXXXX 100644 | 15 | index XXXXXXX..XXXXXXX 100644 |
158 | --- a/block/vmdk.c | 16 | --- a/block/vmdk.c |
159 | +++ b/block/vmdk.c | 17 | +++ b/block/vmdk.c |
160 | @@ -XXX,XX +XXX,XX @@ typedef struct { | 18 | @@ -XXX,XX +XXX,XX @@ static const char *next_line(const char *s) |
161 | uint16_t compressAlgorithm; | ||
162 | } QEMU_PACKED VMDK4Header; | ||
163 | |||
164 | +typedef struct VMDKSESparseConstHeader { | ||
165 | + uint64_t magic; | ||
166 | + uint64_t version; | ||
167 | + uint64_t capacity; | ||
168 | + uint64_t grain_size; | ||
169 | + uint64_t grain_table_size; | ||
170 | + uint64_t flags; | ||
171 | + uint64_t reserved1; | ||
172 | + uint64_t reserved2; | ||
173 | + uint64_t reserved3; | ||
174 | + uint64_t reserved4; | ||
175 | + uint64_t volatile_header_offset; | ||
176 | + uint64_t volatile_header_size; | ||
177 | + uint64_t journal_header_offset; | ||
178 | + uint64_t journal_header_size; | ||
179 | + uint64_t journal_offset; | ||
180 | + uint64_t journal_size; | ||
181 | + uint64_t grain_dir_offset; | ||
182 | + uint64_t grain_dir_size; | ||
183 | + uint64_t grain_tables_offset; | ||
184 | + uint64_t grain_tables_size; | ||
185 | + uint64_t free_bitmap_offset; | ||
186 | + uint64_t free_bitmap_size; | ||
187 | + uint64_t backmap_offset; | ||
188 | + uint64_t backmap_size; | ||
189 | + uint64_t grains_offset; | ||
190 | + uint64_t grains_size; | ||
191 | + uint8_t pad[304]; | ||
192 | +} QEMU_PACKED VMDKSESparseConstHeader; | ||
193 | + | ||
194 | +typedef struct VMDKSESparseVolatileHeader { | ||
195 | + uint64_t magic; | ||
196 | + uint64_t free_gt_number; | ||
197 | + uint64_t next_txn_seq_number; | ||
198 | + uint64_t replay_journal; | ||
199 | + uint8_t pad[480]; | ||
200 | +} QEMU_PACKED VMDKSESparseVolatileHeader; | ||
201 | + | ||
202 | #define L2_CACHE_SIZE 16 | ||
203 | |||
204 | typedef struct VmdkExtent { | ||
205 | @@ -XXX,XX +XXX,XX @@ typedef struct VmdkExtent { | ||
206 | bool compressed; | ||
207 | bool has_marker; | ||
208 | bool has_zero_grain; | ||
209 | + bool sesparse; | ||
210 | + uint64_t sesparse_l2_tables_offset; | ||
211 | + uint64_t sesparse_clusters_offset; | ||
212 | + int32_t entry_size; | ||
213 | int version; | ||
214 | int64_t sectors; | ||
215 | int64_t end_sector; | ||
216 | int64_t flat_start_offset; | ||
217 | int64_t l1_table_offset; | ||
218 | int64_t l1_backup_table_offset; | ||
219 | - uint32_t *l1_table; | ||
220 | + void *l1_table; | ||
221 | uint32_t *l1_backup_table; | ||
222 | unsigned int l1_size; | ||
223 | uint32_t l1_entry_sectors; | ||
224 | |||
225 | unsigned int l2_size; | ||
226 | - uint32_t *l2_cache; | ||
227 | + void *l2_cache; | ||
228 | uint32_t l2_cache_offsets[L2_CACHE_SIZE]; | ||
229 | uint32_t l2_cache_counts[L2_CACHE_SIZE]; | ||
230 | |||
231 | @@ -XXX,XX +XXX,XX @@ static int vmdk_add_extent(BlockDriverState *bs, | ||
232 | * minimal L2 table size: 512 entries | ||
233 | * 8 TB is still more than the maximal value supported for | ||
234 | * VMDK3 & VMDK4 which is 2TB. | ||
235 | + * 64TB - for "ESXi seSparse Extent" | ||
236 | + * minimal cluster size: 512B (default is 4KB) | ||
237 | + * L2 table size: 4096 entries (const). | ||
238 | + * 64TB is more than the maximal value supported for | ||
239 | + * seSparse VMDKs (which is slightly less than 64TB) | ||
240 | */ | ||
241 | error_setg(errp, "L1 size too big"); | ||
242 | return -EFBIG; | ||
243 | @@ -XXX,XX +XXX,XX @@ static int vmdk_add_extent(BlockDriverState *bs, | ||
244 | extent->l2_size = l2_size; | ||
245 | extent->cluster_sectors = flat ? sectors : cluster_sectors; | ||
246 | extent->next_cluster_sector = ROUND_UP(nb_sectors, cluster_sectors); | ||
247 | + extent->entry_size = sizeof(uint32_t); | ||
248 | |||
249 | if (s->num_extents > 1) { | ||
250 | extent->end_sector = (*(extent - 1)).end_sector + extent->sectors; | ||
251 | @@ -XXX,XX +XXX,XX @@ static int vmdk_init_tables(BlockDriverState *bs, VmdkExtent *extent, | ||
252 | int i; | ||
253 | |||
254 | /* read the L1 table */ | ||
255 | - l1_size = extent->l1_size * sizeof(uint32_t); | ||
256 | + l1_size = extent->l1_size * extent->entry_size; | ||
257 | extent->l1_table = g_try_malloc(l1_size); | ||
258 | if (l1_size && extent->l1_table == NULL) { | ||
259 | return -ENOMEM; | ||
260 | @@ -XXX,XX +XXX,XX @@ static int vmdk_init_tables(BlockDriverState *bs, VmdkExtent *extent, | ||
261 | goto fail_l1; | ||
262 | } | ||
263 | for (i = 0; i < extent->l1_size; i++) { | ||
264 | - le32_to_cpus(&extent->l1_table[i]); | ||
265 | + if (extent->entry_size == sizeof(uint64_t)) { | ||
266 | + le64_to_cpus((uint64_t *)extent->l1_table + i); | ||
267 | + } else { | ||
268 | + assert(extent->entry_size == sizeof(uint32_t)); | ||
269 | + le32_to_cpus((uint32_t *)extent->l1_table + i); | ||
270 | + } | ||
271 | } | ||
272 | |||
273 | if (extent->l1_backup_table_offset) { | ||
274 | + assert(!extent->sesparse); | ||
275 | extent->l1_backup_table = g_try_malloc(l1_size); | ||
276 | if (l1_size && extent->l1_backup_table == NULL) { | ||
277 | ret = -ENOMEM; | ||
278 | @@ -XXX,XX +XXX,XX @@ static int vmdk_init_tables(BlockDriverState *bs, VmdkExtent *extent, | ||
279 | } | ||
280 | |||
281 | extent->l2_cache = | ||
282 | - g_new(uint32_t, extent->l2_size * L2_CACHE_SIZE); | ||
283 | + g_malloc(extent->entry_size * extent->l2_size * L2_CACHE_SIZE); | ||
284 | return 0; | ||
285 | fail_l1b: | ||
286 | g_free(extent->l1_backup_table); | ||
287 | @@ -XXX,XX +XXX,XX @@ static int vmdk_open_vmfs_sparse(BlockDriverState *bs, | ||
288 | return ret; | ||
289 | } | 19 | } |
290 | 20 | ||
291 | +#define SESPARSE_CONST_HEADER_MAGIC UINT64_C(0x00000000cafebabe) | 21 | static int vmdk_parse_extents(const char *desc, BlockDriverState *bs, |
292 | +#define SESPARSE_VOLATILE_HEADER_MAGIC UINT64_C(0x00000000cafecafe) | 22 | - const char *desc_file_path, QDict *options, |
293 | + | 23 | - Error **errp) |
294 | +/* Strict checks - format not officially documented */ | 24 | + QDict *options, Error **errp) |
295 | +static int check_se_sparse_const_header(VMDKSESparseConstHeader *header, | 25 | { |
296 | + Error **errp) | 26 | int ret; |
297 | +{ | 27 | int matches; |
298 | + header->magic = le64_to_cpu(header->magic); | ||
299 | + header->version = le64_to_cpu(header->version); | ||
300 | + header->grain_size = le64_to_cpu(header->grain_size); | ||
301 | + header->grain_table_size = le64_to_cpu(header->grain_table_size); | ||
302 | + header->flags = le64_to_cpu(header->flags); | ||
303 | + header->reserved1 = le64_to_cpu(header->reserved1); | ||
304 | + header->reserved2 = le64_to_cpu(header->reserved2); | ||
305 | + header->reserved3 = le64_to_cpu(header->reserved3); | ||
306 | + header->reserved4 = le64_to_cpu(header->reserved4); | ||
307 | + | ||
308 | + header->volatile_header_offset = | ||
309 | + le64_to_cpu(header->volatile_header_offset); | ||
310 | + header->volatile_header_size = le64_to_cpu(header->volatile_header_size); | ||
311 | + | ||
312 | + header->journal_header_offset = le64_to_cpu(header->journal_header_offset); | ||
313 | + header->journal_header_size = le64_to_cpu(header->journal_header_size); | ||
314 | + | ||
315 | + header->journal_offset = le64_to_cpu(header->journal_offset); | ||
316 | + header->journal_size = le64_to_cpu(header->journal_size); | ||
317 | + | ||
318 | + header->grain_dir_offset = le64_to_cpu(header->grain_dir_offset); | ||
319 | + header->grain_dir_size = le64_to_cpu(header->grain_dir_size); | ||
320 | + | ||
321 | + header->grain_tables_offset = le64_to_cpu(header->grain_tables_offset); | ||
322 | + header->grain_tables_size = le64_to_cpu(header->grain_tables_size); | ||
323 | + | ||
324 | + header->free_bitmap_offset = le64_to_cpu(header->free_bitmap_offset); | ||
325 | + header->free_bitmap_size = le64_to_cpu(header->free_bitmap_size); | ||
326 | + | ||
327 | + header->backmap_offset = le64_to_cpu(header->backmap_offset); | ||
328 | + header->backmap_size = le64_to_cpu(header->backmap_size); | ||
329 | + | ||
330 | + header->grains_offset = le64_to_cpu(header->grains_offset); | ||
331 | + header->grains_size = le64_to_cpu(header->grains_size); | ||
332 | + | ||
333 | + if (header->magic != SESPARSE_CONST_HEADER_MAGIC) { | ||
334 | + error_setg(errp, "Bad const header magic: 0x%016" PRIx64, | ||
335 | + header->magic); | ||
336 | + return -EINVAL; | ||
337 | + } | ||
338 | + | ||
339 | + if (header->version != 0x0000000200000001) { | ||
340 | + error_setg(errp, "Unsupported version: 0x%016" PRIx64, | ||
341 | + header->version); | ||
342 | + return -ENOTSUP; | ||
343 | + } | ||
344 | + | ||
345 | + if (header->grain_size != 8) { | ||
346 | + error_setg(errp, "Unsupported grain size: %" PRIu64, | ||
347 | + header->grain_size); | ||
348 | + return -ENOTSUP; | ||
349 | + } | ||
350 | + | ||
351 | + if (header->grain_table_size != 64) { | ||
352 | + error_setg(errp, "Unsupported grain table size: %" PRIu64, | ||
353 | + header->grain_table_size); | ||
354 | + return -ENOTSUP; | ||
355 | + } | ||
356 | + | ||
357 | + if (header->flags != 0) { | ||
358 | + error_setg(errp, "Unsupported flags: 0x%016" PRIx64, | ||
359 | + header->flags); | ||
360 | + return -ENOTSUP; | ||
361 | + } | ||
362 | + | ||
363 | + if (header->reserved1 != 0 || header->reserved2 != 0 || | ||
364 | + header->reserved3 != 0 || header->reserved4 != 0) { | ||
365 | + error_setg(errp, "Unsupported reserved bits:" | ||
366 | + " 0x%016" PRIx64 " 0x%016" PRIx64 | ||
367 | + " 0x%016" PRIx64 " 0x%016" PRIx64, | ||
368 | + header->reserved1, header->reserved2, | ||
369 | + header->reserved3, header->reserved4); | ||
370 | + return -ENOTSUP; | ||
371 | + } | ||
372 | + | ||
373 | + /* check that padding is 0 */ | ||
374 | + if (!buffer_is_zero(header->pad, sizeof(header->pad))) { | ||
375 | + error_setg(errp, "Unsupported non-zero const header padding"); | ||
376 | + return -ENOTSUP; | ||
377 | + } | ||
378 | + | ||
379 | + return 0; | ||
380 | +} | ||
381 | + | ||
382 | +static int check_se_sparse_volatile_header(VMDKSESparseVolatileHeader *header, | ||
383 | + Error **errp) | ||
384 | +{ | ||
385 | + header->magic = le64_to_cpu(header->magic); | ||
386 | + header->free_gt_number = le64_to_cpu(header->free_gt_number); | ||
387 | + header->next_txn_seq_number = le64_to_cpu(header->next_txn_seq_number); | ||
388 | + header->replay_journal = le64_to_cpu(header->replay_journal); | ||
389 | + | ||
390 | + if (header->magic != SESPARSE_VOLATILE_HEADER_MAGIC) { | ||
391 | + error_setg(errp, "Bad volatile header magic: 0x%016" PRIx64, | ||
392 | + header->magic); | ||
393 | + return -EINVAL; | ||
394 | + } | ||
395 | + | ||
396 | + if (header->replay_journal) { | ||
397 | + error_setg(errp, "Image is dirty, Replaying journal not supported"); | ||
398 | + return -ENOTSUP; | ||
399 | + } | ||
400 | + | ||
401 | + /* check that padding is 0 */ | ||
402 | + if (!buffer_is_zero(header->pad, sizeof(header->pad))) { | ||
403 | + error_setg(errp, "Unsupported non-zero volatile header padding"); | ||
404 | + return -ENOTSUP; | ||
405 | + } | ||
406 | + | ||
407 | + return 0; | ||
408 | +} | ||
409 | + | ||
410 | +static int vmdk_open_se_sparse(BlockDriverState *bs, | ||
411 | + BdrvChild *file, | ||
412 | + int flags, Error **errp) | ||
413 | +{ | ||
414 | + int ret; | ||
415 | + VMDKSESparseConstHeader const_header; | ||
416 | + VMDKSESparseVolatileHeader volatile_header; | ||
417 | + VmdkExtent *extent; | ||
418 | + | ||
419 | + ret = bdrv_apply_auto_read_only(bs, | ||
420 | + "No write support for seSparse images available", errp); | ||
421 | + if (ret < 0) { | ||
422 | + return ret; | ||
423 | + } | ||
424 | + | ||
425 | + assert(sizeof(const_header) == SECTOR_SIZE); | ||
426 | + | ||
427 | + ret = bdrv_pread(file, 0, &const_header, sizeof(const_header)); | ||
428 | + if (ret < 0) { | ||
429 | + bdrv_refresh_filename(file->bs); | ||
430 | + error_setg_errno(errp, -ret, | ||
431 | + "Could not read const header from file '%s'", | ||
432 | + file->bs->filename); | ||
433 | + return ret; | ||
434 | + } | ||
435 | + | ||
436 | + /* check const header */ | ||
437 | + ret = check_se_sparse_const_header(&const_header, errp); | ||
438 | + if (ret < 0) { | ||
439 | + return ret; | ||
440 | + } | ||
441 | + | ||
442 | + assert(sizeof(volatile_header) == SECTOR_SIZE); | ||
443 | + | ||
444 | + ret = bdrv_pread(file, | ||
445 | + const_header.volatile_header_offset * SECTOR_SIZE, | ||
446 | + &volatile_header, sizeof(volatile_header)); | ||
447 | + if (ret < 0) { | ||
448 | + bdrv_refresh_filename(file->bs); | ||
449 | + error_setg_errno(errp, -ret, | ||
450 | + "Could not read volatile header from file '%s'", | ||
451 | + file->bs->filename); | ||
452 | + return ret; | ||
453 | + } | ||
454 | + | ||
455 | + /* check volatile header */ | ||
456 | + ret = check_se_sparse_volatile_header(&volatile_header, errp); | ||
457 | + if (ret < 0) { | ||
458 | + return ret; | ||
459 | + } | ||
460 | + | ||
461 | + ret = vmdk_add_extent(bs, file, false, | ||
462 | + const_header.capacity, | ||
463 | + const_header.grain_dir_offset * SECTOR_SIZE, | ||
464 | + 0, | ||
465 | + const_header.grain_dir_size * | ||
466 | + SECTOR_SIZE / sizeof(uint64_t), | ||
467 | + const_header.grain_table_size * | ||
468 | + SECTOR_SIZE / sizeof(uint64_t), | ||
469 | + const_header.grain_size, | ||
470 | + &extent, | ||
471 | + errp); | ||
472 | + if (ret < 0) { | ||
473 | + return ret; | ||
474 | + } | ||
475 | + | ||
476 | + extent->sesparse = true; | ||
477 | + extent->sesparse_l2_tables_offset = const_header.grain_tables_offset; | ||
478 | + extent->sesparse_clusters_offset = const_header.grains_offset; | ||
479 | + extent->entry_size = sizeof(uint64_t); | ||
480 | + | ||
481 | + ret = vmdk_init_tables(bs, extent, errp); | ||
482 | + if (ret) { | ||
483 | + /* free extent allocated by vmdk_add_extent */ | ||
484 | + vmdk_free_last_extent(bs); | ||
485 | + } | ||
486 | + | ||
487 | + return ret; | ||
488 | +} | ||
489 | + | ||
490 | static int vmdk_open_desc_file(BlockDriverState *bs, int flags, char *buf, | ||
491 | QDict *options, Error **errp); | ||
492 | |||
493 | @@ -XXX,XX +XXX,XX @@ static int vmdk_parse_extents(const char *desc, BlockDriverState *bs, | 28 | @@ -XXX,XX +XXX,XX @@ static int vmdk_parse_extents(const char *desc, BlockDriverState *bs, |
494 | * RW [size in sectors] SPARSE "file-name.vmdk" | 29 | const char *p, *np; |
495 | * RW [size in sectors] VMFS "file-name.vmdk" | 30 | int64_t sectors = 0; |
496 | * RW [size in sectors] VMFSSPARSE "file-name.vmdk" | 31 | int64_t flat_offset; |
497 | + * RW [size in sectors] SESPARSE "file-name.vmdk" | 32 | + char *desc_file_dir = NULL; |
498 | */ | 33 | char *extent_path; |
499 | flat_offset = -1; | 34 | BdrvChild *extent_file; |
500 | matches = sscanf(p, "%10s %" SCNd64 " %10s \"%511[^\n\r\"]\" %" SCNd64, | 35 | BDRVVmdkState *s = bs->opaque; |
501 | @@ -XXX,XX +XXX,XX @@ static int vmdk_parse_extents(const char *desc, BlockDriverState *bs, | 36 | @@ -XXX,XX +XXX,XX @@ static int vmdk_parse_extents(const char *desc, BlockDriverState *bs, |
502 | |||
503 | if (sectors <= 0 || | ||
504 | (strcmp(type, "FLAT") && strcmp(type, "SPARSE") && | ||
505 | - strcmp(type, "VMFS") && strcmp(type, "VMFSSPARSE")) || | ||
506 | + strcmp(type, "VMFS") && strcmp(type, "VMFSSPARSE") && | ||
507 | + strcmp(type, "SESPARSE")) || | ||
508 | (strcmp(access, "RW"))) { | ||
509 | continue; | 37 | continue; |
510 | } | 38 | } |
39 | |||
40 | - if (!path_is_absolute(fname) && !path_has_protocol(fname) && | ||
41 | - !desc_file_path[0]) | ||
42 | - { | ||
43 | - bdrv_refresh_filename(bs->file->bs); | ||
44 | - error_setg(errp, "Cannot use relative extent paths with VMDK " | ||
45 | - "descriptor file '%s'", bs->file->bs->filename); | ||
46 | - return -EINVAL; | ||
47 | - } | ||
48 | + if (path_is_absolute(fname)) { | ||
49 | + extent_path = g_strdup(fname); | ||
50 | + } else { | ||
51 | + if (!desc_file_dir) { | ||
52 | + desc_file_dir = bdrv_dirname(bs->file->bs, errp); | ||
53 | + if (!desc_file_dir) { | ||
54 | + bdrv_refresh_filename(bs->file->bs); | ||
55 | + error_prepend(errp, "Cannot use relative paths with VMDK " | ||
56 | + "descriptor file '%s': ", | ||
57 | + bs->file->bs->filename); | ||
58 | + ret = -EINVAL; | ||
59 | + goto out; | ||
60 | + } | ||
61 | + } | ||
62 | |||
63 | - extent_path = path_combine(desc_file_path, fname); | ||
64 | + extent_path = g_strconcat(desc_file_dir, fname, NULL); | ||
65 | + } | ||
66 | |||
67 | ret = snprintf(extent_opt_prefix, 32, "extents.%d", s->num_extents); | ||
68 | assert(ret < 32); | ||
511 | @@ -XXX,XX +XXX,XX @@ static int vmdk_parse_extents(const char *desc, BlockDriverState *bs, | 69 | @@ -XXX,XX +XXX,XX @@ static int vmdk_parse_extents(const char *desc, BlockDriverState *bs, |
512 | return ret; | 70 | g_free(extent_path); |
71 | if (local_err) { | ||
72 | error_propagate(errp, local_err); | ||
73 | - return -EINVAL; | ||
74 | + ret = -EINVAL; | ||
75 | + goto out; | ||
76 | } | ||
77 | |||
78 | /* save to extents array */ | ||
79 | @@ -XXX,XX +XXX,XX @@ static int vmdk_parse_extents(const char *desc, BlockDriverState *bs, | ||
80 | 0, 0, 0, 0, 0, &extent, errp); | ||
81 | if (ret < 0) { | ||
82 | bdrv_unref_child(bs, extent_file); | ||
83 | - return ret; | ||
84 | + goto out; | ||
85 | } | ||
86 | extent->flat_start_offset = flat_offset << 9; | ||
87 | } else if (!strcmp(type, "SPARSE") || !strcmp(type, "VMFSSPARSE")) { | ||
88 | @@ -XXX,XX +XXX,XX @@ static int vmdk_parse_extents(const char *desc, BlockDriverState *bs, | ||
89 | g_free(buf); | ||
90 | if (ret) { | ||
91 | bdrv_unref_child(bs, extent_file); | ||
92 | - return ret; | ||
93 | + goto out; | ||
513 | } | 94 | } |
514 | extent = &s->extents[s->num_extents - 1]; | 95 | extent = &s->extents[s->num_extents - 1]; |
515 | + } else if (!strcmp(type, "SESPARSE")) { | 96 | } else if (!strcmp(type, "SESPARSE")) { |
516 | + ret = vmdk_open_se_sparse(bs, extent_file, bs->open_flags, errp); | 97 | ret = vmdk_open_se_sparse(bs, extent_file, bs->open_flags, errp); |
517 | + if (ret) { | 98 | if (ret) { |
518 | + bdrv_unref_child(bs, extent_file); | 99 | bdrv_unref_child(bs, extent_file); |
519 | + return ret; | 100 | - return ret; |
520 | + } | 101 | + goto out; |
521 | + extent = &s->extents[s->num_extents - 1]; | 102 | } |
103 | extent = &s->extents[s->num_extents - 1]; | ||
522 | } else { | 104 | } else { |
523 | error_setg(errp, "Unsupported extent type '%s'", type); | 105 | error_setg(errp, "Unsupported extent type '%s'", type); |
524 | bdrv_unref_child(bs, extent_file); | 106 | bdrv_unref_child(bs, extent_file); |
107 | - return -ENOTSUP; | ||
108 | + ret = -ENOTSUP; | ||
109 | + goto out; | ||
110 | } | ||
111 | extent->type = g_strdup(type); | ||
112 | } | ||
113 | - return 0; | ||
114 | + | ||
115 | + ret = 0; | ||
116 | + goto out; | ||
117 | |||
118 | invalid: | ||
119 | np = next_line(p); | ||
120 | @@ -XXX,XX +XXX,XX @@ invalid: | ||
121 | np--; | ||
122 | } | ||
123 | error_setg(errp, "Invalid extent line: %.*s", (int)(np - p), p); | ||
124 | - return -EINVAL; | ||
125 | + ret = -EINVAL; | ||
126 | + | ||
127 | +out: | ||
128 | + g_free(desc_file_dir); | ||
129 | + return ret; | ||
130 | } | ||
131 | |||
132 | static int vmdk_open_desc_file(BlockDriverState *bs, int flags, char *buf, | ||
525 | @@ -XXX,XX +XXX,XX @@ static int vmdk_open_desc_file(BlockDriverState *bs, int flags, char *buf, | 133 | @@ -XXX,XX +XXX,XX @@ static int vmdk_open_desc_file(BlockDriverState *bs, int flags, char *buf, |
526 | if (strcmp(ct, "monolithicFlat") && | ||
527 | strcmp(ct, "vmfs") && | ||
528 | strcmp(ct, "vmfsSparse") && | ||
529 | + strcmp(ct, "seSparse") && | ||
530 | strcmp(ct, "twoGbMaxExtentSparse") && | ||
531 | strcmp(ct, "twoGbMaxExtentFlat")) { | ||
532 | error_setg(errp, "Unsupported image type '%s'", ct); | ||
533 | @@ -XXX,XX +XXX,XX @@ static int get_cluster_offset(BlockDriverState *bs, | ||
534 | { | ||
535 | unsigned int l1_index, l2_offset, l2_index; | ||
536 | int min_index, i, j; | ||
537 | - uint32_t min_count, *l2_table; | ||
538 | + uint32_t min_count; | ||
539 | + void *l2_table; | ||
540 | bool zeroed = false; | ||
541 | int64_t ret; | ||
542 | int64_t cluster_sector; | ||
543 | + unsigned int l2_size_bytes = extent->l2_size * extent->entry_size; | ||
544 | |||
545 | if (m_data) { | ||
546 | m_data->valid = 0; | ||
547 | @@ -XXX,XX +XXX,XX @@ static int get_cluster_offset(BlockDriverState *bs, | ||
548 | if (l1_index >= extent->l1_size) { | ||
549 | return VMDK_ERROR; | ||
550 | } | 134 | } |
551 | - l2_offset = extent->l1_table[l1_index]; | 135 | s->create_type = g_strdup(ct); |
552 | + if (extent->sesparse) { | 136 | s->desc_offset = 0; |
553 | + uint64_t l2_offset_u64; | 137 | - ret = vmdk_parse_extents(buf, bs, bs->file->bs->exact_filename, options, |
554 | + | 138 | - errp); |
555 | + assert(extent->entry_size == sizeof(uint64_t)); | 139 | + ret = vmdk_parse_extents(buf, bs, options, errp); |
556 | + | 140 | exit: |
557 | + l2_offset_u64 = ((uint64_t *)extent->l1_table)[l1_index]; | 141 | return ret; |
558 | + if (l2_offset_u64 == 0) { | 142 | } |
559 | + l2_offset = 0; | 143 | diff --git a/tests/qemu-iotests/059 b/tests/qemu-iotests/059 |
560 | + } else if ((l2_offset_u64 & 0xffffffff00000000) != 0x1000000000000000) { | 144 | index XXXXXXX..XXXXXXX 100755 |
561 | + /* | 145 | --- a/tests/qemu-iotests/059 |
562 | + * Top most nibble is 0x1 if grain table is allocated. | 146 | +++ b/tests/qemu-iotests/059 |
563 | + * strict check - top most 4 bytes must be 0x10000000 since max | 147 | @@ -XXX,XX +XXX,XX @@ $QEMU_IMG convert -f qcow2 -O vmdk -o subformat=streamOptimized "$TEST_IMG.qcow2 |
564 | + * supported size is 64TB for disk - so no more than 64TB / 16MB | 148 | |
565 | + * grain directories which is smaller than uint32, | 149 | echo |
566 | + * where 16MB is the only supported default grain table coverage. | 150 | echo "=== Testing monolithicFlat with internally generated JSON file name ===" |
567 | + */ | 151 | +# Should work, because bdrv_dirname() works fine with blkdebug |
568 | + return VMDK_ERROR; | 152 | IMGOPTS="subformat=monolithicFlat" _make_test_img 64M |
569 | + } else { | 153 | -$QEMU_IO -c "open -o driver=$IMGFMT,file.driver=blkdebug,file.image.filename=$TEST_IMG,file.inject-error.0.event=read_aio" 2>&1 \ |
570 | + l2_offset_u64 = l2_offset_u64 & 0x00000000ffffffff; | 154 | - | _filter_testdir | _filter_imgfmt |
571 | + l2_offset_u64 = extent->sesparse_l2_tables_offset + | 155 | +$QEMU_IO -c "open -o driver=$IMGFMT,file.driver=blkdebug,file.image.filename=$TEST_IMG,file.inject-error.0.event=read_aio" \ |
572 | + l2_offset_u64 * l2_size_bytes / SECTOR_SIZE; | 156 | + -c info \ |
573 | + if (l2_offset_u64 > 0x00000000ffffffff) { | 157 | + 2>&1 \ |
574 | + return VMDK_ERROR; | 158 | + | _filter_testdir | _filter_imgfmt | _filter_img_info |
575 | + } | 159 | _cleanup_test_img |
576 | + l2_offset = (unsigned int)(l2_offset_u64); | 160 | |
577 | + } | 161 | echo |
578 | + } else { | 162 | diff --git a/tests/qemu-iotests/059.out b/tests/qemu-iotests/059.out |
579 | + assert(extent->entry_size == sizeof(uint32_t)); | 163 | index XXXXXXX..XXXXXXX 100644 |
580 | + l2_offset = ((uint32_t *)extent->l1_table)[l1_index]; | 164 | --- a/tests/qemu-iotests/059.out |
581 | + } | 165 | +++ b/tests/qemu-iotests/059.out |
582 | if (!l2_offset) { | 166 | @@ -XXX,XX +XXX,XX @@ wrote 512/512 bytes at offset 10240 |
583 | return VMDK_UNALLOC; | 167 | |
584 | } | 168 | === Testing monolithicFlat with internally generated JSON file name === |
585 | @@ -XXX,XX +XXX,XX @@ static int get_cluster_offset(BlockDriverState *bs, | 169 | Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 |
586 | extent->l2_cache_counts[j] >>= 1; | 170 | -qemu-io: can't open: Cannot use relative extent paths with VMDK descriptor file 'json:{"image": {"driver": "file", "filename": "TEST_DIR/t.IMGFMT"}, "driver": "blkdebug", "inject-error.0.event": "read_aio"}' |
587 | } | 171 | +format name: IMGFMT |
588 | } | 172 | +cluster size: 0 bytes |
589 | - l2_table = extent->l2_cache + (i * extent->l2_size); | 173 | +vm state offset: 0 bytes |
590 | + l2_table = (char *)extent->l2_cache + (i * l2_size_bytes); | 174 | |
591 | goto found; | 175 | === Testing version 3 === |
592 | } | 176 | image: TEST_DIR/iotest-version3.IMGFMT |
593 | } | ||
594 | @@ -XXX,XX +XXX,XX @@ static int get_cluster_offset(BlockDriverState *bs, | ||
595 | min_index = i; | ||
596 | } | ||
597 | } | ||
598 | - l2_table = extent->l2_cache + (min_index * extent->l2_size); | ||
599 | + l2_table = (char *)extent->l2_cache + (min_index * l2_size_bytes); | ||
600 | BLKDBG_EVENT(extent->file, BLKDBG_L2_LOAD); | ||
601 | if (bdrv_pread(extent->file, | ||
602 | (int64_t)l2_offset * 512, | ||
603 | l2_table, | ||
604 | - extent->l2_size * sizeof(uint32_t) | ||
605 | - ) != extent->l2_size * sizeof(uint32_t)) { | ||
606 | + l2_size_bytes | ||
607 | + ) != l2_size_bytes) { | ||
608 | return VMDK_ERROR; | ||
609 | } | ||
610 | |||
611 | @@ -XXX,XX +XXX,XX @@ static int get_cluster_offset(BlockDriverState *bs, | ||
612 | extent->l2_cache_counts[min_index] = 1; | ||
613 | found: | ||
614 | l2_index = ((offset >> 9) / extent->cluster_sectors) % extent->l2_size; | ||
615 | - cluster_sector = le32_to_cpu(l2_table[l2_index]); | ||
616 | |||
617 | - if (extent->has_zero_grain && cluster_sector == VMDK_GTE_ZEROED) { | ||
618 | - zeroed = true; | ||
619 | + if (extent->sesparse) { | ||
620 | + cluster_sector = le64_to_cpu(((uint64_t *)l2_table)[l2_index]); | ||
621 | + switch (cluster_sector & 0xf000000000000000) { | ||
622 | + case 0x0000000000000000: | ||
623 | + /* unallocated grain */ | ||
624 | + if (cluster_sector != 0) { | ||
625 | + return VMDK_ERROR; | ||
626 | + } | ||
627 | + break; | ||
628 | + case 0x1000000000000000: | ||
629 | + /* scsi-unmapped grain - fallthrough */ | ||
630 | + case 0x2000000000000000: | ||
631 | + /* zero grain */ | ||
632 | + zeroed = true; | ||
633 | + break; | ||
634 | + case 0x3000000000000000: | ||
635 | + /* allocated grain */ | ||
636 | + cluster_sector = (((cluster_sector & 0x0fff000000000000) >> 48) | | ||
637 | + ((cluster_sector & 0x0000ffffffffffff) << 12)); | ||
638 | + cluster_sector = extent->sesparse_clusters_offset + | ||
639 | + cluster_sector * extent->cluster_sectors; | ||
640 | + break; | ||
641 | + default: | ||
642 | + return VMDK_ERROR; | ||
643 | + } | ||
644 | + } else { | ||
645 | + cluster_sector = le32_to_cpu(((uint32_t *)l2_table)[l2_index]); | ||
646 | + | ||
647 | + if (extent->has_zero_grain && cluster_sector == VMDK_GTE_ZEROED) { | ||
648 | + zeroed = true; | ||
649 | + } | ||
650 | } | ||
651 | |||
652 | if (!cluster_sector || zeroed) { | ||
653 | if (!allocate) { | ||
654 | return zeroed ? VMDK_ZEROED : VMDK_UNALLOC; | ||
655 | } | ||
656 | + assert(!extent->sesparse); | ||
657 | |||
658 | if (extent->next_cluster_sector >= VMDK_EXTENT_MAX_SECTORS) { | ||
659 | return VMDK_ERROR; | ||
660 | @@ -XXX,XX +XXX,XX @@ static int get_cluster_offset(BlockDriverState *bs, | ||
661 | m_data->l1_index = l1_index; | ||
662 | m_data->l2_index = l2_index; | ||
663 | m_data->l2_offset = l2_offset; | ||
664 | - m_data->l2_cache_entry = &l2_table[l2_index]; | ||
665 | + m_data->l2_cache_entry = ((uint32_t *)l2_table) + l2_index; | ||
666 | } | ||
667 | } | ||
668 | *cluster_offset = cluster_sector << BDRV_SECTOR_BITS; | ||
669 | @@ -XXX,XX +XXX,XX @@ static int vmdk_pwritev(BlockDriverState *bs, uint64_t offset, | ||
670 | if (!extent) { | ||
671 | return -EIO; | ||
672 | } | ||
673 | + if (extent->sesparse) { | ||
674 | + return -ENOTSUP; | ||
675 | + } | ||
676 | offset_in_cluster = vmdk_find_offset_in_cluster(extent, offset); | ||
677 | n_bytes = MIN(bytes, extent->cluster_sectors * BDRV_SECTOR_SIZE | ||
678 | - offset_in_cluster); | ||
679 | -- | 177 | -- |
680 | 2.21.0 | 178 | 2.21.0 |
681 | 179 | ||
682 | 180 | diff view generated by jsdifflib |
1 | From: Anton Nefedov <anton.nefedov@virtuozzo.com> | 1 | We had a test for a case where relative extent paths did not work, but |
---|---|---|---|
2 | unfortunately we just fixed the underlying problem, so it works now. | ||
3 | This patch adds a new test case that still fails. | ||
2 | 4 | ||
3 | COW (even empty/zero) areas require encryption too | 5 | Signed-off-by: Max Reitz <mreitz@redhat.com> |
4 | 6 | Reviewed-by: John Snow <jsnow@redhat.com> | |
5 | Signed-off-by: Anton Nefedov <anton.nefedov@virtuozzo.com> | 7 | Message-id: 20190815153638.4600-4-mreitz@redhat.com |
6 | Reviewed-by: Eric Blake <eblake@redhat.com> | 8 | Reviewed-by: John Snow <jsnow@redhat.com> |
7 | Reviewed-by: Max Reitz <mreitz@redhat.com> | ||
8 | Reviewed-by: Alberto Garcia <berto@igalia.com> | ||
9 | Message-id: 20190516143028.81155-1-anton.nefedov@virtuozzo.com | ||
10 | Signed-off-by: Max Reitz <mreitz@redhat.com> | 9 | Signed-off-by: Max Reitz <mreitz@redhat.com> |
11 | --- | 10 | --- |
12 | tests/qemu-iotests/134 | 9 +++++++++ | 11 | tests/qemu-iotests/059 | 27 +++++++++++++++++++++++++++ |
13 | tests/qemu-iotests/134.out | 10 ++++++++++ | 12 | tests/qemu-iotests/059.out | 4 ++++ |
14 | 2 files changed, 19 insertions(+) | 13 | 2 files changed, 31 insertions(+) |
15 | 14 | ||
16 | diff --git a/tests/qemu-iotests/134 b/tests/qemu-iotests/134 | 15 | diff --git a/tests/qemu-iotests/059 b/tests/qemu-iotests/059 |
17 | index XXXXXXX..XXXXXXX 100755 | 16 | index XXXXXXX..XXXXXXX 100755 |
18 | --- a/tests/qemu-iotests/134 | 17 | --- a/tests/qemu-iotests/059 |
19 | +++ b/tests/qemu-iotests/134 | 18 | +++ b/tests/qemu-iotests/059 |
20 | @@ -XXX,XX +XXX,XX @@ echo | 19 | @@ -XXX,XX +XXX,XX @@ $QEMU_IMG convert -f qcow2 -O vmdk -o subformat=streamOptimized "$TEST_IMG.qcow2 |
21 | echo "== reading whole image ==" | 20 | |
22 | $QEMU_IO --object $SECRET -c "read 0 $size" --image-opts $IMGSPEC | _filter_qemu_io | _filter_testdir | 21 | echo |
23 | 22 | echo "=== Testing monolithicFlat with internally generated JSON file name ===" | |
24 | +echo | ||
25 | +echo "== rewriting cluster part ==" | ||
26 | +$QEMU_IO --object $SECRET -c "write -P 0xb 512 512" --image-opts $IMGSPEC | _filter_qemu_io | _filter_testdir | ||
27 | + | 23 | + |
28 | +echo | 24 | +echo '--- blkdebug ---' |
29 | +echo "== verify pattern ==" | 25 | # Should work, because bdrv_dirname() works fine with blkdebug |
30 | +$QEMU_IO --object $SECRET -c "read -P 0 0 512" --image-opts $IMGSPEC | _filter_qemu_io | _filter_testdir | 26 | IMGOPTS="subformat=monolithicFlat" _make_test_img 64M |
31 | +$QEMU_IO --object $SECRET -c "read -P 0xb 512 512" --image-opts $IMGSPEC | _filter_qemu_io | _filter_testdir | 27 | $QEMU_IO -c "open -o driver=$IMGFMT,file.driver=blkdebug,file.image.filename=$TEST_IMG,file.inject-error.0.event=read_aio" \ |
28 | @@ -XXX,XX +XXX,XX @@ $QEMU_IO -c "open -o driver=$IMGFMT,file.driver=blkdebug,file.image.filename=$TE | ||
29 | | _filter_testdir | _filter_imgfmt | _filter_img_info | ||
30 | _cleanup_test_img | ||
31 | |||
32 | +echo '--- quorum ---' | ||
33 | +# Should not work, because bdrv_dirname() does not work with quorum | ||
34 | +IMGOPTS="subformat=monolithicFlat" _make_test_img 64M | ||
35 | +cp "$TEST_IMG" "$TEST_IMG.orig" | ||
36 | + | ||
37 | +filename="json:{ | ||
38 | + \"driver\": \"$IMGFMT\", | ||
39 | + \"file\": { | ||
40 | + \"driver\": \"quorum\", | ||
41 | + \"children\": [ { | ||
42 | + \"driver\": \"file\", | ||
43 | + \"filename\": \"$TEST_IMG\" | ||
44 | + }, { | ||
45 | + \"driver\": \"file\", | ||
46 | + \"filename\": \"$TEST_IMG.orig\" | ||
47 | + } ], | ||
48 | + \"vote-threshold\": 1 | ||
49 | + } }" | ||
50 | + | ||
51 | +filename=$(echo "$filename" | tr '\n' ' ' | sed -e 's/\s\+/ /g') | ||
52 | +$QEMU_IMG info "$filename" 2>&1 \ | ||
53 | + | sed -e "s/'json:[^']*'/\$QUORUM_FILE/g" \ | ||
54 | + | _filter_testdir | _filter_imgfmt | _filter_img_info | ||
55 | + | ||
32 | + | 56 | + |
33 | echo | 57 | echo |
34 | echo "== rewriting whole image ==" | 58 | echo "=== Testing version 3 ===" |
35 | $QEMU_IO --object $SECRET -c "write -P 0xa 0 $size" --image-opts $IMGSPEC | _filter_qemu_io | _filter_testdir | 59 | _use_sample_img iotest-version3.vmdk.bz2 |
36 | diff --git a/tests/qemu-iotests/134.out b/tests/qemu-iotests/134.out | 60 | diff --git a/tests/qemu-iotests/059.out b/tests/qemu-iotests/059.out |
37 | index XXXXXXX..XXXXXXX 100644 | 61 | index XXXXXXX..XXXXXXX 100644 |
38 | --- a/tests/qemu-iotests/134.out | 62 | --- a/tests/qemu-iotests/059.out |
39 | +++ b/tests/qemu-iotests/134.out | 63 | +++ b/tests/qemu-iotests/059.out |
40 | @@ -XXX,XX +XXX,XX @@ Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 encryption=on encrypt. | 64 | @@ -XXX,XX +XXX,XX @@ wrote 512/512 bytes at offset 10240 |
41 | read 134217728/134217728 bytes at offset 0 | 65 | 512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) |
42 | 128 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) | 66 | |
43 | 67 | === Testing monolithicFlat with internally generated JSON file name === | |
44 | +== rewriting cluster part == | 68 | +--- blkdebug --- |
45 | +wrote 512/512 bytes at offset 512 | 69 | Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 |
46 | +512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) | 70 | format name: IMGFMT |
47 | + | 71 | cluster size: 0 bytes |
48 | +== verify pattern == | 72 | vm state offset: 0 bytes |
49 | +read 512/512 bytes at offset 0 | 73 | +--- quorum --- |
50 | +512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) | 74 | +Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 |
51 | +read 512/512 bytes at offset 512 | 75 | +qemu-img: Could not open $QUORUM_FILE: Cannot use relative paths with VMDK descriptor file $QUORUM_FILE: Cannot generate a base directory for quorum nodes |
52 | +512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) | 76 | |
53 | + | 77 | === Testing version 3 === |
54 | == rewriting whole image == | 78 | image: TEST_DIR/iotest-version3.IMGFMT |
55 | wrote 134217728/134217728 bytes at offset 0 | ||
56 | 128 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) | ||
57 | -- | 79 | -- |
58 | 2.21.0 | 80 | 2.21.0 |
59 | 81 | ||
60 | 82 | diff view generated by jsdifflib |
1 | From: Sam Eiderman <shmuel.eiderman@oracle.com> | 1 | Compressed writes generally have to write full clusters, not just in |
---|---|---|---|
2 | theory but also in practice when it comes to vmdk's streamOptimized | ||
3 | subformat. It currently is just silently broken for writes with | ||
4 | non-zero in-cluster offsets: | ||
2 | 5 | ||
3 | Commit b0651b8c246d ("vmdk: Move l1_size check into vmdk_add_extent") | 6 | $ qemu-img create -f vmdk -o subformat=streamOptimized foo.vmdk 1M |
4 | extended the l1_size check from VMDK4 to VMDK3 but did not update the | 7 | $ qemu-io -c 'write 4k 4k' -c 'read 4k 4k' foo.vmdk |
5 | default coverage in the moved comment. | 8 | wrote 4096/4096 bytes at offset 4096 |
9 | 4 KiB, 1 ops; 00.01 sec (443.724 KiB/sec and 110.9309 ops/sec) | ||
10 | read failed: Invalid argument | ||
6 | 11 | ||
7 | The previous vmdk4 calculation: | 12 | (The technical reason is that vmdk_write_extent() just writes the |
13 | incomplete compressed data actually to offset 4k. When reading the | ||
14 | data, vmdk_read_extent() looks at offset 0 and finds the compressed data | ||
15 | size to be 0, because that is what it reads from there. This yields an | ||
16 | error.) | ||
8 | 17 | ||
9 | (512 * 1024 * 1024) * 512(l2 entries) * 65536(grain) = 16PB | 18 | For incomplete writes with zero in-cluster offsets, the error path when |
19 | reading the rest of the cluster is a bit different, but the result is | ||
20 | the same: | ||
10 | 21 | ||
11 | The added vmdk3 calculation: | 22 | $ qemu-img create -f vmdk -o subformat=streamOptimized foo.vmdk 1M |
23 | $ qemu-io -c 'write 0k 4k' -c 'read 4k 4k' foo.vmdk | ||
24 | wrote 4096/4096 bytes at offset 0 | ||
25 | 4 KiB, 1 ops; 00.01 sec (362.641 KiB/sec and 90.6603 ops/sec) | ||
26 | read failed: Invalid argument | ||
12 | 27 | ||
13 | (512 * 1024 * 1024) * 4096(l2 entries) * 512(grain) = 1PB | 28 | (Here, vmdk_read_extent() finds the data and then sees that the |
29 | uncompressed data is short.) | ||
14 | 30 | ||
15 | Adding the calculation of vmdk3 to the comment. | 31 | It is better to reject invalid writes than to make the user believe they |
32 | might have succeeded and then fail when trying to read it back. | ||
16 | 33 | ||
17 | In any case, VMware does not offer virtual disks more than 2TB for | 34 | Signed-off-by: Max Reitz <mreitz@redhat.com> |
18 | vmdk4/vmdk3 or 64TB for the new undocumented seSparse format which is | 35 | Reviewed-by: John Snow <jsnow@redhat.com> |
19 | not implemented yet in qemu. | 36 | Message-id: 20190815153638.4600-5-mreitz@redhat.com |
20 | 37 | Reviewed-by: John Snow <jsnow@redhat.com> | |
21 | Reviewed-by: Karl Heubaum <karl.heubaum@oracle.com> | ||
22 | Reviewed-by: Eyal Moscovici <eyal.moscovici@oracle.com> | ||
23 | Reviewed-by: Liran Alon <liran.alon@oracle.com> | ||
24 | Reviewed-by: Arbel Moshe <arbel.moshe@oracle.com> | ||
25 | Signed-off-by: Sam Eiderman <shmuel.eiderman@oracle.com> | ||
26 | Message-id: 20190620091057.47441-2-shmuel.eiderman@oracle.com | ||
27 | Reviewed-by: yuchenlin <yuchenlin@synology.com> | ||
28 | Reviewed-by: Max Reitz <mreitz@redhat.com> | ||
29 | Signed-off-by: Max Reitz <mreitz@redhat.com> | 38 | Signed-off-by: Max Reitz <mreitz@redhat.com> |
30 | --- | 39 | --- |
31 | block/vmdk.c | 11 ++++++++--- | 40 | block/vmdk.c | 10 ++++++++++ |
32 | 1 file changed, 8 insertions(+), 3 deletions(-) | 41 | 1 file changed, 10 insertions(+) |
33 | 42 | ||
34 | diff --git a/block/vmdk.c b/block/vmdk.c | 43 | diff --git a/block/vmdk.c b/block/vmdk.c |
35 | index XXXXXXX..XXXXXXX 100644 | 44 | index XXXXXXX..XXXXXXX 100644 |
36 | --- a/block/vmdk.c | 45 | --- a/block/vmdk.c |
37 | +++ b/block/vmdk.c | 46 | +++ b/block/vmdk.c |
38 | @@ -XXX,XX +XXX,XX @@ static int vmdk_add_extent(BlockDriverState *bs, | 47 | @@ -XXX,XX +XXX,XX @@ static int vmdk_write_extent(VmdkExtent *extent, int64_t cluster_offset, |
39 | return -EFBIG; | 48 | if (extent->compressed) { |
40 | } | 49 | void *compressed_data; |
41 | if (l1_size > 512 * 1024 * 1024) { | 50 | |
42 | - /* Although with big capacity and small l1_entry_sectors, we can get a | 51 | + /* Only whole clusters */ |
43 | + /* | 52 | + if (offset_in_cluster || |
44 | + * Although with big capacity and small l1_entry_sectors, we can get a | 53 | + n_bytes > (extent->cluster_sectors * SECTOR_SIZE) || |
45 | * big l1_size, we don't want unbounded value to allocate the table. | 54 | + (n_bytes < (extent->cluster_sectors * SECTOR_SIZE) && |
46 | - * Limit it to 512M, which is 16PB for default cluster and L2 table | 55 | + offset + n_bytes != extent->end_sector * SECTOR_SIZE)) |
47 | - * size */ | 56 | + { |
48 | + * Limit it to 512M, which is: | 57 | + ret = -EINVAL; |
49 | + * 16PB - for default "Hosted Sparse Extent" (VMDK4) | 58 | + goto out; |
50 | + * cluster size: 64KB, L2 table size: 512 entries | 59 | + } |
51 | + * 1PB - for default "ESXi Host Sparse Extent" (VMDK3/vmfsSparse) | 60 | + |
52 | + * cluster size: 512B, L2 table size: 4096 entries | 61 | if (!extent->has_marker) { |
53 | + */ | 62 | ret = -EINVAL; |
54 | error_setg(errp, "L1 size too big"); | 63 | goto out; |
55 | return -EFBIG; | ||
56 | } | ||
57 | -- | 64 | -- |
58 | 2.21.0 | 65 | 2.21.0 |
59 | 66 | ||
60 | 67 | diff view generated by jsdifflib |
New patch | |||
---|---|---|---|
1 | streamOptimized does not support writes that do not span exactly one | ||
2 | cluster. Furthermore, it cannot rewrite already allocated clusters. | ||
3 | As such, many iotests do not work with it. Disable them. | ||
1 | 4 | ||
5 | Signed-off-by: Max Reitz <mreitz@redhat.com> | ||
6 | Message-id: 20190815153638.4600-6-mreitz@redhat.com | ||
7 | Reviewed-by: John Snow <jsnow@redhat.com> | ||
8 | Signed-off-by: Max Reitz <mreitz@redhat.com> | ||
9 | --- | ||
10 | tests/qemu-iotests/002 | 1 + | ||
11 | tests/qemu-iotests/003 | 1 + | ||
12 | tests/qemu-iotests/005 | 3 ++- | ||
13 | tests/qemu-iotests/009 | 1 + | ||
14 | tests/qemu-iotests/010 | 1 + | ||
15 | tests/qemu-iotests/011 | 1 + | ||
16 | tests/qemu-iotests/017 | 3 ++- | ||
17 | tests/qemu-iotests/018 | 3 ++- | ||
18 | tests/qemu-iotests/019 | 3 ++- | ||
19 | tests/qemu-iotests/020 | 3 ++- | ||
20 | tests/qemu-iotests/027 | 1 + | ||
21 | tests/qemu-iotests/032 | 1 + | ||
22 | tests/qemu-iotests/033 | 1 + | ||
23 | tests/qemu-iotests/034 | 3 ++- | ||
24 | tests/qemu-iotests/037 | 3 ++- | ||
25 | tests/qemu-iotests/063 | 3 ++- | ||
26 | tests/qemu-iotests/072 | 1 + | ||
27 | tests/qemu-iotests/105 | 3 ++- | ||
28 | tests/qemu-iotests/197 | 1 + | ||
29 | tests/qemu-iotests/215 | 1 + | ||
30 | tests/qemu-iotests/251 | 1 + | ||
31 | 21 files changed, 30 insertions(+), 9 deletions(-) | ||
32 | |||
33 | diff --git a/tests/qemu-iotests/002 b/tests/qemu-iotests/002 | ||
34 | index XXXXXXX..XXXXXXX 100755 | ||
35 | --- a/tests/qemu-iotests/002 | ||
36 | +++ b/tests/qemu-iotests/002 | ||
37 | @@ -XXX,XX +XXX,XX @@ trap "_cleanup; exit \$status" 0 1 2 3 15 | ||
38 | |||
39 | _supported_fmt generic | ||
40 | _supported_proto generic | ||
41 | +_unsupported_imgopts "subformat=streamOptimized" | ||
42 | |||
43 | |||
44 | size=128M | ||
45 | diff --git a/tests/qemu-iotests/003 b/tests/qemu-iotests/003 | ||
46 | index XXXXXXX..XXXXXXX 100755 | ||
47 | --- a/tests/qemu-iotests/003 | ||
48 | +++ b/tests/qemu-iotests/003 | ||
49 | @@ -XXX,XX +XXX,XX @@ trap "_cleanup; exit \$status" 0 1 2 3 15 | ||
50 | |||
51 | _supported_fmt generic | ||
52 | _supported_proto generic | ||
53 | +_unsupported_imgopts "subformat=streamOptimized" | ||
54 | |||
55 | size=128M | ||
56 | offset=67M | ||
57 | diff --git a/tests/qemu-iotests/005 b/tests/qemu-iotests/005 | ||
58 | index XXXXXXX..XXXXXXX 100755 | ||
59 | --- a/tests/qemu-iotests/005 | ||
60 | +++ b/tests/qemu-iotests/005 | ||
61 | @@ -XXX,XX +XXX,XX @@ _supported_fmt generic | ||
62 | _supported_proto generic | ||
63 | _supported_os Linux | ||
64 | _unsupported_imgopts "subformat=twoGbMaxExtentFlat" \ | ||
65 | - "subformat=twoGbMaxExtentSparse" | ||
66 | + "subformat=twoGbMaxExtentSparse" \ | ||
67 | + "subformat=streamOptimized" | ||
68 | |||
69 | # vpc is limited to 127GB, so we can't test it here | ||
70 | if [ "$IMGFMT" = "vpc" ]; then | ||
71 | diff --git a/tests/qemu-iotests/009 b/tests/qemu-iotests/009 | ||
72 | index XXXXXXX..XXXXXXX 100755 | ||
73 | --- a/tests/qemu-iotests/009 | ||
74 | +++ b/tests/qemu-iotests/009 | ||
75 | @@ -XXX,XX +XXX,XX @@ trap "_cleanup; exit \$status" 0 1 2 3 15 | ||
76 | |||
77 | _supported_fmt generic | ||
78 | _supported_proto generic | ||
79 | +_unsupported_imgopts "subformat=streamOptimized" | ||
80 | |||
81 | |||
82 | size=6G | ||
83 | diff --git a/tests/qemu-iotests/010 b/tests/qemu-iotests/010 | ||
84 | index XXXXXXX..XXXXXXX 100755 | ||
85 | --- a/tests/qemu-iotests/010 | ||
86 | +++ b/tests/qemu-iotests/010 | ||
87 | @@ -XXX,XX +XXX,XX @@ trap "_cleanup; exit \$status" 0 1 2 3 15 | ||
88 | |||
89 | _supported_fmt generic | ||
90 | _supported_proto generic | ||
91 | +_unsupported_imgopts "subformat=streamOptimized" | ||
92 | |||
93 | |||
94 | size=6G | ||
95 | diff --git a/tests/qemu-iotests/011 b/tests/qemu-iotests/011 | ||
96 | index XXXXXXX..XXXXXXX 100755 | ||
97 | --- a/tests/qemu-iotests/011 | ||
98 | +++ b/tests/qemu-iotests/011 | ||
99 | @@ -XXX,XX +XXX,XX @@ trap "_cleanup; exit \$status" 0 1 2 3 15 | ||
100 | |||
101 | _supported_fmt generic | ||
102 | _supported_proto generic | ||
103 | +_unsupported_imgopts "subformat=streamOptimized" | ||
104 | |||
105 | |||
106 | size=6G | ||
107 | diff --git a/tests/qemu-iotests/017 b/tests/qemu-iotests/017 | ||
108 | index XXXXXXX..XXXXXXX 100755 | ||
109 | --- a/tests/qemu-iotests/017 | ||
110 | +++ b/tests/qemu-iotests/017 | ||
111 | @@ -XXX,XX +XXX,XX @@ trap "_cleanup; exit \$status" 0 1 2 3 15 | ||
112 | _supported_fmt qcow qcow2 vmdk qed | ||
113 | _supported_proto generic | ||
114 | _unsupported_proto vxhs | ||
115 | -_unsupported_imgopts "subformat=monolithicFlat" "subformat=twoGbMaxExtentFlat" | ||
116 | +_unsupported_imgopts "subformat=monolithicFlat" "subformat=twoGbMaxExtentFlat" \ | ||
117 | + "subformat=streamOptimized" | ||
118 | |||
119 | TEST_OFFSETS="0 4294967296" | ||
120 | |||
121 | diff --git a/tests/qemu-iotests/018 b/tests/qemu-iotests/018 | ||
122 | index XXXXXXX..XXXXXXX 100755 | ||
123 | --- a/tests/qemu-iotests/018 | ||
124 | +++ b/tests/qemu-iotests/018 | ||
125 | @@ -XXX,XX +XXX,XX @@ trap "_cleanup; exit \$status" 0 1 2 3 15 | ||
126 | _supported_fmt qcow qcow2 vmdk qed | ||
127 | _supported_proto file | ||
128 | _supported_os Linux | ||
129 | -_unsupported_imgopts "subformat=monolithicFlat" "subformat=twoGbMaxExtentFlat" | ||
130 | +_unsupported_imgopts "subformat=monolithicFlat" "subformat=twoGbMaxExtentFlat" \ | ||
131 | + "streamOptimized" | ||
132 | |||
133 | TEST_OFFSETS="0 4294967296" | ||
134 | |||
135 | diff --git a/tests/qemu-iotests/019 b/tests/qemu-iotests/019 | ||
136 | index XXXXXXX..XXXXXXX 100755 | ||
137 | --- a/tests/qemu-iotests/019 | ||
138 | +++ b/tests/qemu-iotests/019 | ||
139 | @@ -XXX,XX +XXX,XX @@ _supported_proto file | ||
140 | _supported_os Linux | ||
141 | _unsupported_imgopts "subformat=monolithicFlat" \ | ||
142 | "subformat=twoGbMaxExtentFlat" \ | ||
143 | - "subformat=twoGbMaxExtentSparse" | ||
144 | + "subformat=twoGbMaxExtentSparse" \ | ||
145 | + "subformat=streamOptimized" | ||
146 | |||
147 | TEST_OFFSETS="0 4294967296" | ||
148 | CLUSTER_SIZE=65536 | ||
149 | diff --git a/tests/qemu-iotests/020 b/tests/qemu-iotests/020 | ||
150 | index XXXXXXX..XXXXXXX 100755 | ||
151 | --- a/tests/qemu-iotests/020 | ||
152 | +++ b/tests/qemu-iotests/020 | ||
153 | @@ -XXX,XX +XXX,XX @@ _supported_fmt qcow qcow2 vmdk qed | ||
154 | _supported_proto file | ||
155 | _unsupported_imgopts "subformat=monolithicFlat" \ | ||
156 | "subformat=twoGbMaxExtentFlat" \ | ||
157 | - "subformat=twoGbMaxExtentSparse" | ||
158 | + "subformat=twoGbMaxExtentSparse" \ | ||
159 | + "subformat=streamOptimized" | ||
160 | |||
161 | TEST_OFFSETS="0 4294967296" | ||
162 | |||
163 | diff --git a/tests/qemu-iotests/027 b/tests/qemu-iotests/027 | ||
164 | index XXXXXXX..XXXXXXX 100755 | ||
165 | --- a/tests/qemu-iotests/027 | ||
166 | +++ b/tests/qemu-iotests/027 | ||
167 | @@ -XXX,XX +XXX,XX @@ trap "_cleanup; exit \$status" 0 1 2 3 15 | ||
168 | |||
169 | _supported_fmt vmdk qcow qcow2 qed | ||
170 | _supported_proto generic | ||
171 | +_unsupported_imgopts "subformat=streamOptimized" | ||
172 | |||
173 | |||
174 | size=128M | ||
175 | diff --git a/tests/qemu-iotests/032 b/tests/qemu-iotests/032 | ||
176 | index XXXXXXX..XXXXXXX 100755 | ||
177 | --- a/tests/qemu-iotests/032 | ||
178 | +++ b/tests/qemu-iotests/032 | ||
179 | @@ -XXX,XX +XXX,XX @@ trap "_cleanup; exit \$status" 0 1 2 3 15 | ||
180 | # This works for any image format (though unlikely to segfault for raw) | ||
181 | _supported_fmt generic | ||
182 | _supported_proto generic | ||
183 | +_unsupported_imgopts "subformat=streamOptimized" | ||
184 | |||
185 | echo | ||
186 | echo === Prepare image === | ||
187 | diff --git a/tests/qemu-iotests/033 b/tests/qemu-iotests/033 | ||
188 | index XXXXXXX..XXXXXXX 100755 | ||
189 | --- a/tests/qemu-iotests/033 | ||
190 | +++ b/tests/qemu-iotests/033 | ||
191 | @@ -XXX,XX +XXX,XX @@ trap "_cleanup; exit \$status" 0 1 2 3 15 | ||
192 | |||
193 | _supported_fmt generic | ||
194 | _supported_proto generic | ||
195 | +_unsupported_imgopts "subformat=streamOptimized" | ||
196 | |||
197 | |||
198 | size=128M | ||
199 | diff --git a/tests/qemu-iotests/034 b/tests/qemu-iotests/034 | ||
200 | index XXXXXXX..XXXXXXX 100755 | ||
201 | --- a/tests/qemu-iotests/034 | ||
202 | +++ b/tests/qemu-iotests/034 | ||
203 | @@ -XXX,XX +XXX,XX @@ _supported_proto file | ||
204 | _supported_os Linux | ||
205 | _unsupported_imgopts "subformat=monolithicFlat" \ | ||
206 | "subformat=twoGbMaxExtentFlat" \ | ||
207 | - "subformat=twoGbMaxExtentSparse" | ||
208 | + "subformat=twoGbMaxExtentSparse" \ | ||
209 | + "subformat=streamOptimized" | ||
210 | |||
211 | CLUSTER_SIZE=4k | ||
212 | size=128M | ||
213 | diff --git a/tests/qemu-iotests/037 b/tests/qemu-iotests/037 | ||
214 | index XXXXXXX..XXXXXXX 100755 | ||
215 | --- a/tests/qemu-iotests/037 | ||
216 | +++ b/tests/qemu-iotests/037 | ||
217 | @@ -XXX,XX +XXX,XX @@ _supported_fmt qcow qcow2 vmdk qed | ||
218 | _supported_proto file | ||
219 | _unsupported_imgopts "subformat=monolithicFlat" \ | ||
220 | "subformat=twoGbMaxExtentFlat" \ | ||
221 | - "subformat=twoGbMaxExtentSparse" | ||
222 | + "subformat=twoGbMaxExtentSparse" \ | ||
223 | + "subformat=streamOptimized" | ||
224 | |||
225 | CLUSTER_SIZE=4k | ||
226 | size=128M | ||
227 | diff --git a/tests/qemu-iotests/063 b/tests/qemu-iotests/063 | ||
228 | index XXXXXXX..XXXXXXX 100755 | ||
229 | --- a/tests/qemu-iotests/063 | ||
230 | +++ b/tests/qemu-iotests/063 | ||
231 | @@ -XXX,XX +XXX,XX @@ _supported_fmt qcow qcow2 vmdk qed raw | ||
232 | _supported_proto file | ||
233 | _unsupported_imgopts "subformat=monolithicFlat" \ | ||
234 | "subformat=twoGbMaxExtentFlat" \ | ||
235 | - "subformat=twoGbMaxExtentSparse" | ||
236 | + "subformat=twoGbMaxExtentSparse" \ | ||
237 | + "subformat=streamOptimized" | ||
238 | |||
239 | _make_test_img 4M | ||
240 | |||
241 | diff --git a/tests/qemu-iotests/072 b/tests/qemu-iotests/072 | ||
242 | index XXXXXXX..XXXXXXX 100755 | ||
243 | --- a/tests/qemu-iotests/072 | ||
244 | +++ b/tests/qemu-iotests/072 | ||
245 | @@ -XXX,XX +XXX,XX @@ trap "_cleanup; exit \$status" 0 1 2 3 15 | ||
246 | |||
247 | _supported_fmt vpc vmdk vhdx vdi qed qcow2 qcow | ||
248 | _supported_proto file | ||
249 | +_unsupported_imgopts "subformat=streamOptimized" | ||
250 | |||
251 | IMG_SIZE=64M | ||
252 | |||
253 | diff --git a/tests/qemu-iotests/105 b/tests/qemu-iotests/105 | ||
254 | index XXXXXXX..XXXXXXX 100755 | ||
255 | --- a/tests/qemu-iotests/105 | ||
256 | +++ b/tests/qemu-iotests/105 | ||
257 | @@ -XXX,XX +XXX,XX @@ trap "_cleanup; exit \$status" 0 1 2 3 15 | ||
258 | _supported_fmt qcow2 vmdk vhdx qed | ||
259 | _supported_proto generic | ||
260 | _unsupported_imgopts "subformat=twoGbMaxExtentFlat" \ | ||
261 | - "subformat=twoGbMaxExtentSparse" | ||
262 | + "subformat=twoGbMaxExtentSparse" \ | ||
263 | + "subformat=streamOptimized" | ||
264 | |||
265 | echo | ||
266 | echo "creating large image" | ||
267 | diff --git a/tests/qemu-iotests/197 b/tests/qemu-iotests/197 | ||
268 | index XXXXXXX..XXXXXXX 100755 | ||
269 | --- a/tests/qemu-iotests/197 | ||
270 | +++ b/tests/qemu-iotests/197 | ||
271 | @@ -XXX,XX +XXX,XX @@ _supported_fmt generic | ||
272 | _supported_proto generic | ||
273 | # LUKS support may be possible, but it complicates things. | ||
274 | _unsupported_fmt luks | ||
275 | +_unsupported_imgopts "subformat=streamOptimized" | ||
276 | |||
277 | echo | ||
278 | echo '=== Copy-on-read ===' | ||
279 | diff --git a/tests/qemu-iotests/215 b/tests/qemu-iotests/215 | ||
280 | index XXXXXXX..XXXXXXX 100755 | ||
281 | --- a/tests/qemu-iotests/215 | ||
282 | +++ b/tests/qemu-iotests/215 | ||
283 | @@ -XXX,XX +XXX,XX @@ _supported_fmt generic | ||
284 | _supported_proto generic | ||
285 | # LUKS support may be possible, but it complicates things. | ||
286 | _unsupported_fmt luks | ||
287 | +_unsupported_imgopts "subformat=streamOptimized" | ||
288 | |||
289 | echo | ||
290 | echo '=== Copy-on-read ===' | ||
291 | diff --git a/tests/qemu-iotests/251 b/tests/qemu-iotests/251 | ||
292 | index XXXXXXX..XXXXXXX 100755 | ||
293 | --- a/tests/qemu-iotests/251 | ||
294 | +++ b/tests/qemu-iotests/251 | ||
295 | @@ -XXX,XX +XXX,XX @@ trap "_cleanup; exit \$status" 0 1 2 3 15 | ||
296 | _supported_fmt generic | ||
297 | _supported_proto file | ||
298 | _supported_os Linux | ||
299 | +_unsupported_imgopts "subformat=streamOptimized" | ||
300 | |||
301 | if [ "$IMGOPTSSYNTAX" = "true" ]; then | ||
302 | # We use json:{} filenames here, so we cannot work with additional options. | ||
303 | -- | ||
304 | 2.21.0 | ||
305 | |||
306 | diff view generated by jsdifflib |
1 | Tests should place their files into the test directory. This includes | 1 | The error message for the test case where we have a quorum node for |
---|---|---|---|
2 | Unix sockets. 205 currently fails to do so, which prevents it from | 2 | which no directory name can be generated is different: For |
3 | being run concurrently. | 3 | twoGbMaxExtentSparse, it complains that it cannot open the extent file. |
4 | For other (sub)formats, it just notes that it cannot determine the | ||
5 | backing file path. Both are fine, but just disable twoGbMaxExtentSparse | ||
6 | for simplicity's sake. | ||
4 | 7 | ||
5 | Signed-off-by: Max Reitz <mreitz@redhat.com> | 8 | Signed-off-by: Max Reitz <mreitz@redhat.com> |
6 | Message-id: 20190618210238.9524-1-mreitz@redhat.com | 9 | Reviewed-by: John Snow <jsnow@redhat.com> |
7 | Reviewed-by: Eric Blake <eblake@redhat.com> | 10 | Message-id: 20190815153638.4600-7-mreitz@redhat.com |
11 | Reviewed-by: John Snow <jsnow@redhat.com> | ||
8 | Signed-off-by: Max Reitz <mreitz@redhat.com> | 12 | Signed-off-by: Max Reitz <mreitz@redhat.com> |
9 | --- | 13 | --- |
10 | tests/qemu-iotests/205 | 2 +- | 14 | tests/qemu-iotests/110 | 3 ++- |
11 | 1 file changed, 1 insertion(+), 1 deletion(-) | 15 | 1 file changed, 2 insertions(+), 1 deletion(-) |
12 | 16 | ||
13 | diff --git a/tests/qemu-iotests/205 b/tests/qemu-iotests/205 | 17 | diff --git a/tests/qemu-iotests/110 b/tests/qemu-iotests/110 |
14 | index XXXXXXX..XXXXXXX 100755 | 18 | index XXXXXXX..XXXXXXX 100755 |
15 | --- a/tests/qemu-iotests/205 | 19 | --- a/tests/qemu-iotests/110 |
16 | +++ b/tests/qemu-iotests/205 | 20 | +++ b/tests/qemu-iotests/110 |
17 | @@ -XXX,XX +XXX,XX @@ import iotests | 21 | @@ -XXX,XX +XXX,XX @@ trap "_cleanup; exit \$status" 0 1 2 3 15 |
18 | import time | 22 | # Any format supporting backing files |
19 | from iotests import qemu_img_create, qemu_io, filter_qemu_io, QemuIoInteractive | 23 | _supported_fmt qed qcow qcow2 vmdk |
20 | 24 | _supported_proto file | |
21 | -nbd_sock = 'nbd_sock' | 25 | -_unsupported_imgopts "subformat=monolithicFlat" "subformat=twoGbMaxExtentFlat" |
22 | +nbd_sock = os.path.join(iotests.test_dir, 'nbd_sock') | 26 | +_unsupported_imgopts "subformat=monolithicFlat" "subformat=twoGbMaxExtentFlat" \ |
23 | nbd_uri = 'nbd+unix:///exp?socket=' + nbd_sock | 27 | + "subformat=twoGbMaxExtentSparse" |
24 | disk = os.path.join(iotests.test_dir, 'disk') | 28 | |
29 | TEST_IMG_REL=$(basename "$TEST_IMG") | ||
25 | 30 | ||
26 | -- | 31 | -- |
27 | 2.21.0 | 32 | 2.21.0 |
28 | 33 | ||
29 | 34 | diff view generated by jsdifflib |
New patch | |||
---|---|---|---|
1 | iotest 126 requires backing file support, which flat vmdks cannot offer. | ||
2 | Skip this test for such subformats. | ||
1 | 3 | ||
4 | Signed-off-by: Max Reitz <mreitz@redhat.com> | ||
5 | Message-id: 20190815153638.4600-8-mreitz@redhat.com | ||
6 | Reviewed-by: John Snow <jsnow@redhat.com> | ||
7 | Signed-off-by: Max Reitz <mreitz@redhat.com> | ||
8 | --- | ||
9 | tests/qemu-iotests/126 | 2 ++ | ||
10 | 1 file changed, 2 insertions(+) | ||
11 | |||
12 | diff --git a/tests/qemu-iotests/126 b/tests/qemu-iotests/126 | ||
13 | index XXXXXXX..XXXXXXX 100755 | ||
14 | --- a/tests/qemu-iotests/126 | ||
15 | +++ b/tests/qemu-iotests/126 | ||
16 | @@ -XXX,XX +XXX,XX @@ status=1 # failure is the default! | ||
17 | |||
18 | # Needs backing file support | ||
19 | _supported_fmt qcow qcow2 qed vmdk | ||
20 | +_unsupported_imgopts "subformat=monolithicFlat" \ | ||
21 | + "subformat=twoGbMaxExtentFlat" | ||
22 | # This is the default protocol (and we want to test the difference between | ||
23 | # colons which separate a protocol prefix from the rest and colons which are | ||
24 | # just part of the filename, so we cannot test protocols which require a prefix) | ||
25 | -- | ||
26 | 2.21.0 | ||
27 | |||
28 | diff view generated by jsdifflib |
New patch | |||
---|---|---|---|
1 | From: Stefan Hajnoczi <stefanha@redhat.com> | ||
1 | 2 | ||
3 | Fixes: a6b257a08e3d72219f03e461a52152672fec0612 | ||
4 | ("file-posix: Handle undetectable alignment") | ||
5 | Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com> | ||
6 | Message-id: 20190827101328.4062-1-stefanha@redhat.com | ||
7 | Reviewed-by: Thomas Huth <thuth@redhat.com> | ||
8 | Signed-off-by: Max Reitz <mreitz@redhat.com> | ||
9 | --- | ||
10 | block/file-posix.c | 2 +- | ||
11 | 1 file changed, 1 insertion(+), 1 deletion(-) | ||
12 | |||
13 | diff --git a/block/file-posix.c b/block/file-posix.c | ||
14 | index XXXXXXX..XXXXXXX 100644 | ||
15 | --- a/block/file-posix.c | ||
16 | +++ b/block/file-posix.c | ||
17 | @@ -XXX,XX +XXX,XX @@ static void raw_probe_alignment(BlockDriverState *bs, int fd, Error **errp) | ||
18 | for (i = 0; i < ARRAY_SIZE(alignments); i++) { | ||
19 | align = alignments[i]; | ||
20 | if (raw_is_io_aligned(fd, buf + align, max_align)) { | ||
21 | - /* Fallback to request_aligment. */ | ||
22 | + /* Fallback to request_alignment. */ | ||
23 | s->buf_align = (align != 1) ? align : bs->bl.request_alignment; | ||
24 | break; | ||
25 | } | ||
26 | -- | ||
27 | 2.21.0 | ||
28 | |||
29 | diff view generated by jsdifflib |
New patch | |||
---|---|---|---|
1 | From: Thomas Huth <thuth@redhat.com> | ||
1 | 2 | ||
3 | It is possible to enable only a subset of the block drivers with the | ||
4 | "--block-drv-rw-whitelist" option of the "configure" script. All other | ||
5 | drivers are marked as unusable (or only included as read-only with the | ||
6 | "--block-drv-ro-whitelist" option). If an iotest is now using such a | ||
7 | disabled block driver, it is failing - which is bad, since at least the | ||
8 | tests in the "auto" group should be able to deal with this situation. | ||
9 | Thus let's introduce a "_require_drivers" function that can be used by | ||
10 | the shell tests to check for the availability of certain drivers first, | ||
11 | and marks the test as "not run" if one of the drivers is missing. | ||
12 | |||
13 | This patch mainly targets the test in the "auto" group which should | ||
14 | never fail in such a case, but also improves some of the other tests | ||
15 | along the way. Note that we also assume that the "qcow2" and "file" | ||
16 | drivers are always available - otherwise it does not make sense to | ||
17 | run "make check-block" at all (which only tests with qcow2 by default). | ||
18 | |||
19 | Signed-off-by: Thomas Huth <thuth@redhat.com> | ||
20 | Message-id: 20190823133552.11680-1-thuth@redhat.com | ||
21 | Signed-off-by: Max Reitz <mreitz@redhat.com> | ||
22 | --- | ||
23 | tests/qemu-iotests/071 | 1 + | ||
24 | tests/qemu-iotests/081 | 4 +--- | ||
25 | tests/qemu-iotests/099 | 1 + | ||
26 | tests/qemu-iotests/120 | 1 + | ||
27 | tests/qemu-iotests/162 | 4 +--- | ||
28 | tests/qemu-iotests/184 | 1 + | ||
29 | tests/qemu-iotests/186 | 1 + | ||
30 | tests/qemu-iotests/common.rc | 14 ++++++++++++++ | ||
31 | 8 files changed, 21 insertions(+), 6 deletions(-) | ||
32 | |||
33 | diff --git a/tests/qemu-iotests/071 b/tests/qemu-iotests/071 | ||
34 | index XXXXXXX..XXXXXXX 100755 | ||
35 | --- a/tests/qemu-iotests/071 | ||
36 | +++ b/tests/qemu-iotests/071 | ||
37 | @@ -XXX,XX +XXX,XX @@ trap "_cleanup; exit \$status" 0 1 2 3 15 | ||
38 | |||
39 | _supported_fmt qcow2 | ||
40 | _supported_proto file | ||
41 | +_require_drivers blkdebug blkverify | ||
42 | |||
43 | do_run_qemu() | ||
44 | { | ||
45 | diff --git a/tests/qemu-iotests/081 b/tests/qemu-iotests/081 | ||
46 | index XXXXXXX..XXXXXXX 100755 | ||
47 | --- a/tests/qemu-iotests/081 | ||
48 | +++ b/tests/qemu-iotests/081 | ||
49 | @@ -XXX,XX +XXX,XX @@ trap "_cleanup; exit \$status" 0 1 2 3 15 | ||
50 | _supported_fmt raw | ||
51 | _supported_proto file | ||
52 | _supported_os Linux | ||
53 | +_require_drivers quorum | ||
54 | |||
55 | do_run_qemu() | ||
56 | { | ||
57 | @@ -XXX,XX +XXX,XX @@ run_qemu() | ||
58 | | _filter_qemu_io | _filter_generated_node_ids | ||
59 | } | ||
60 | |||
61 | -test_quorum=$($QEMU_IMG --help|grep quorum) | ||
62 | -[ "$test_quorum" = "" ] && _supported_fmt quorum | ||
63 | - | ||
64 | quorum="driver=raw,file.driver=quorum,file.vote-threshold=2" | ||
65 | quorum="$quorum,file.children.0.file.filename=$TEST_DIR/1.raw" | ||
66 | quorum="$quorum,file.children.1.file.filename=$TEST_DIR/2.raw" | ||
67 | diff --git a/tests/qemu-iotests/099 b/tests/qemu-iotests/099 | ||
68 | index XXXXXXX..XXXXXXX 100755 | ||
69 | --- a/tests/qemu-iotests/099 | ||
70 | +++ b/tests/qemu-iotests/099 | ||
71 | @@ -XXX,XX +XXX,XX @@ trap "_cleanup; exit \$status" 0 1 2 3 15 | ||
72 | _supported_fmt qcow qcow2 qed vdi vhdx vmdk vpc | ||
73 | _supported_proto file | ||
74 | _supported_os Linux | ||
75 | +_require_drivers blkdebug blkverify | ||
76 | _unsupported_imgopts "subformat=monolithicFlat" "subformat=twoGbMaxExtentFlat" \ | ||
77 | "subformat=twoGbMaxExtentSparse" | ||
78 | |||
79 | diff --git a/tests/qemu-iotests/120 b/tests/qemu-iotests/120 | ||
80 | index XXXXXXX..XXXXXXX 100755 | ||
81 | --- a/tests/qemu-iotests/120 | ||
82 | +++ b/tests/qemu-iotests/120 | ||
83 | @@ -XXX,XX +XXX,XX @@ trap "_cleanup; exit \$status" 0 1 2 3 15 | ||
84 | _supported_fmt generic | ||
85 | _supported_proto file | ||
86 | _unsupported_fmt luks | ||
87 | +_require_drivers raw | ||
88 | |||
89 | _make_test_img 64M | ||
90 | |||
91 | diff --git a/tests/qemu-iotests/162 b/tests/qemu-iotests/162 | ||
92 | index XXXXXXX..XXXXXXX 100755 | ||
93 | --- a/tests/qemu-iotests/162 | ||
94 | +++ b/tests/qemu-iotests/162 | ||
95 | @@ -XXX,XX +XXX,XX @@ trap "_cleanup; exit \$status" 0 1 2 3 15 | ||
96 | . ./common.filter | ||
97 | |||
98 | _supported_fmt generic | ||
99 | - | ||
100 | -test_ssh=$($QEMU_IMG --help | grep '^Supported formats:.* ssh\( \|$\)') | ||
101 | -[ "$test_ssh" = "" ] && _notrun "ssh support required" | ||
102 | +_require_drivers ssh | ||
103 | |||
104 | echo | ||
105 | echo '=== NBD ===' | ||
106 | diff --git a/tests/qemu-iotests/184 b/tests/qemu-iotests/184 | ||
107 | index XXXXXXX..XXXXXXX 100755 | ||
108 | --- a/tests/qemu-iotests/184 | ||
109 | +++ b/tests/qemu-iotests/184 | ||
110 | @@ -XXX,XX +XXX,XX @@ trap "exit \$status" 0 1 2 3 15 | ||
111 | . ./common.filter | ||
112 | |||
113 | _supported_os Linux | ||
114 | +_require_drivers throttle | ||
115 | |||
116 | do_run_qemu() | ||
117 | { | ||
118 | diff --git a/tests/qemu-iotests/186 b/tests/qemu-iotests/186 | ||
119 | index XXXXXXX..XXXXXXX 100755 | ||
120 | --- a/tests/qemu-iotests/186 | ||
121 | +++ b/tests/qemu-iotests/186 | ||
122 | @@ -XXX,XX +XXX,XX @@ trap "_cleanup; exit \$status" 0 1 2 3 15 | ||
123 | |||
124 | _supported_fmt qcow2 | ||
125 | _supported_proto file | ||
126 | +_require_drivers null-co | ||
127 | |||
128 | if [ "$QEMU_DEFAULT_MACHINE" != "pc" ]; then | ||
129 | _notrun "Requires a PC machine" | ||
130 | diff --git a/tests/qemu-iotests/common.rc b/tests/qemu-iotests/common.rc | ||
131 | index XXXXXXX..XXXXXXX 100644 | ||
132 | --- a/tests/qemu-iotests/common.rc | ||
133 | +++ b/tests/qemu-iotests/common.rc | ||
134 | @@ -XXX,XX +XXX,XX @@ _require_command() | ||
135 | [ -x "$c" ] || _notrun "$1 utility required, skipped this test" | ||
136 | } | ||
137 | |||
138 | +# Check that a set of drivers has been whitelisted in the QEMU binary | ||
139 | +# | ||
140 | +_require_drivers() | ||
141 | +{ | ||
142 | + available=$($QEMU -drive format=help | \ | ||
143 | + sed -e '/Supported formats:/!d' -e 's/Supported formats://') | ||
144 | + for driver | ||
145 | + do | ||
146 | + if ! echo "$available" | grep -q " $driver\( \|$\)"; then | ||
147 | + _notrun "$driver not available" | ||
148 | + fi | ||
149 | + done | ||
150 | +} | ||
151 | + | ||
152 | # make sure this script returns success | ||
153 | true | ||
154 | -- | ||
155 | 2.21.0 | ||
156 | |||
157 | diff view generated by jsdifflib |
New patch | |||
---|---|---|---|
1 | From: Thomas Huth <thuth@redhat.com> | ||
1 | 2 | ||
3 | The sanitizers (especially the address sanitizer from Clang) are | ||
4 | sometimes printing out warnings or false positives - this spoils | ||
5 | the output of the iotests, causing some of the tests to fail. | ||
6 | Thus let's skip the automatic iotests during "make check" when the | ||
7 | user configured QEMU with --enable-sanitizers. | ||
8 | |||
9 | Signed-off-by: Thomas Huth <thuth@redhat.com> | ||
10 | Message-id: 20190823084203.29734-1-thuth@redhat.com | ||
11 | Signed-off-by: Max Reitz <mreitz@redhat.com> | ||
12 | --- | ||
13 | tests/check-block.sh | 5 +++++ | ||
14 | 1 file changed, 5 insertions(+) | ||
15 | |||
16 | diff --git a/tests/check-block.sh b/tests/check-block.sh | ||
17 | index XXXXXXX..XXXXXXX 100755 | ||
18 | --- a/tests/check-block.sh | ||
19 | +++ b/tests/check-block.sh | ||
20 | @@ -XXX,XX +XXX,XX @@ if grep -q "TARGET_GPROF=y" *-softmmu/config-target.mak 2>/dev/null ; then | ||
21 | exit 0 | ||
22 | fi | ||
23 | |||
24 | +if grep -q "CFLAGS.*-fsanitize" config-host.mak 2>/dev/null ; then | ||
25 | + echo "Sanitizers are enabled ==> Not running the qemu-iotests." | ||
26 | + exit 0 | ||
27 | +fi | ||
28 | + | ||
29 | if [ -z "$(find . -name 'qemu-system-*' -print)" ]; then | ||
30 | echo "No qemu-system binary available ==> Not running the qemu-iotests." | ||
31 | exit 0 | ||
32 | -- | ||
33 | 2.21.0 | ||
34 | |||
35 | diff view generated by jsdifflib |
1 | From: Klaus Birkelund Jensen <klaus@birkelund.eu> | 1 | From: Nir Soffer <nirsof@gmail.com> |
---|---|---|---|
2 | 2 | ||
3 | The device mistakenly reports that the Weighted Round Robin with Urgent | 3 | Quoting cache mode is not needed, and most tests use unquoted values. |
4 | Priority Class arbitration mechanism is supported. | 4 | Unify all test to use the same style. |
5 | 5 | ||
6 | It is not. | 6 | Message-id: 20190827173432.7656-1-nsoffer@redhat.com |
7 | 7 | Signed-off-by: Nir Soffer <nsoffer@redhat.com> | |
8 | Signed-off-by: Klaus Birkelund Jensen <klaus.jensen@cnexlabs.com> | ||
9 | Message-id: 20190606092530.14206-1-klaus@birkelund.eu | ||
10 | Acked-by: Maxim Levitsky <mlevitsk@redhat.com> | ||
11 | Signed-off-by: Max Reitz <mreitz@redhat.com> | 8 | Signed-off-by: Max Reitz <mreitz@redhat.com> |
12 | --- | 9 | --- |
13 | hw/block/nvme.c | 1 - | 10 | tests/qemu-iotests/026 | 4 ++-- |
14 | 1 file changed, 1 deletion(-) | 11 | tests/qemu-iotests/039 | 4 ++-- |
12 | tests/qemu-iotests/052 | 2 +- | ||
13 | tests/qemu-iotests/091 | 4 ++-- | ||
14 | 4 files changed, 7 insertions(+), 7 deletions(-) | ||
15 | 15 | ||
16 | diff --git a/hw/block/nvme.c b/hw/block/nvme.c | 16 | diff --git a/tests/qemu-iotests/026 b/tests/qemu-iotests/026 |
17 | index XXXXXXX..XXXXXXX 100644 | 17 | index XXXXXXX..XXXXXXX 100755 |
18 | --- a/hw/block/nvme.c | 18 | --- a/tests/qemu-iotests/026 |
19 | +++ b/hw/block/nvme.c | 19 | +++ b/tests/qemu-iotests/026 |
20 | @@ -XXX,XX +XXX,XX @@ static void nvme_realize(PCIDevice *pci_dev, Error **errp) | 20 | @@ -XXX,XX +XXX,XX @@ trap "_cleanup; exit \$status" 0 1 2 3 15 |
21 | n->bar.cap = 0; | 21 | # Currently only qcow2 supports rebasing |
22 | NVME_CAP_SET_MQES(n->bar.cap, 0x7ff); | 22 | _supported_fmt qcow2 |
23 | NVME_CAP_SET_CQR(n->bar.cap, 1); | 23 | _supported_proto file |
24 | - NVME_CAP_SET_AMS(n->bar.cap, 1); | 24 | -_default_cache_mode "writethrough" |
25 | NVME_CAP_SET_TO(n->bar.cap, 0xf); | 25 | -_supported_cache_modes "writethrough" "none" |
26 | NVME_CAP_SET_CSS(n->bar.cap, 1); | 26 | +_default_cache_mode writethrough |
27 | NVME_CAP_SET_MPSMAX(n->bar.cap, 4); | 27 | +_supported_cache_modes writethrough none |
28 | # The refcount table tests expect a certain minimum width for refcount entries | ||
29 | # (so that the refcount table actually needs to grow); that minimum is 16 bits, | ||
30 | # being the default refcount entry width. | ||
31 | diff --git a/tests/qemu-iotests/039 b/tests/qemu-iotests/039 | ||
32 | index XXXXXXX..XXXXXXX 100755 | ||
33 | --- a/tests/qemu-iotests/039 | ||
34 | +++ b/tests/qemu-iotests/039 | ||
35 | @@ -XXX,XX +XXX,XX @@ trap "_cleanup; exit \$status" 0 1 2 3 15 | ||
36 | _supported_fmt qcow2 | ||
37 | _supported_proto file | ||
38 | _supported_os Linux | ||
39 | -_default_cache_mode "writethrough" | ||
40 | -_supported_cache_modes "writethrough" | ||
41 | +_default_cache_mode writethrough | ||
42 | +_supported_cache_modes writethrough | ||
43 | |||
44 | size=128M | ||
45 | |||
46 | diff --git a/tests/qemu-iotests/052 b/tests/qemu-iotests/052 | ||
47 | index XXXXXXX..XXXXXXX 100755 | ||
48 | --- a/tests/qemu-iotests/052 | ||
49 | +++ b/tests/qemu-iotests/052 | ||
50 | @@ -XXX,XX +XXX,XX @@ _supported_fmt generic | ||
51 | _supported_proto file | ||
52 | |||
53 | # Don't do O_DIRECT on tmpfs | ||
54 | -_supported_cache_modes "writeback" "writethrough" "unsafe" | ||
55 | +_supported_cache_modes writeback writethrough unsafe | ||
56 | |||
57 | size=128M | ||
58 | _make_test_img $size | ||
59 | diff --git a/tests/qemu-iotests/091 b/tests/qemu-iotests/091 | ||
60 | index XXXXXXX..XXXXXXX 100755 | ||
61 | --- a/tests/qemu-iotests/091 | ||
62 | +++ b/tests/qemu-iotests/091 | ||
63 | @@ -XXX,XX +XXX,XX @@ trap "_cleanup; exit \$status" 0 1 2 3 15 | ||
64 | _supported_fmt qcow2 | ||
65 | _supported_proto file | ||
66 | _supported_os Linux | ||
67 | -_default_cache_mode "none" | ||
68 | -_supported_cache_modes "writethrough" "none" "writeback" | ||
69 | +_default_cache_mode none | ||
70 | +_supported_cache_modes writethrough none writeback | ||
71 | |||
72 | size=1G | ||
73 | |||
28 | -- | 74 | -- |
29 | 2.21.0 | 75 | 2.21.0 |
30 | 76 | ||
31 | 77 | diff view generated by jsdifflib |