The flattened format (currently output by qemu) is used by makedumpfile
only when it is outputting a vmcore to a file which is not seekable. The
flattened format functions essentially as a set of instructions of the
form "seek to the given offset, then write the given bytes out".

The flattened format can be reconstructed using makedumpfile -R, or
makedumpfile-R.pl, but it is a slow process because it requires copying
the entire vmcore. The flattened format can also be directly read by
crash, but still, it requires a lengthy reassembly phase.

To sum up, the flattened format is not an ideal one: it should only be
used on files which are actually not seekable. This is the exact
strategy which makedumpfile uses, as seen in the implementation of
"write_buffer()" in makedumpfile [1]. However, Qemu has always used the
flattened format. For compatibility it is best not to change the default
output format without warning. So, add a flag to DumpState which changes
the output to use the normal (i.e. reassembled) format. This flag will
be added to the QMP commands in the next change.

[1]: https://github.com/makedumpfile/makedumpfile/blob/f23bb943568188a2746dbf9b6692668f5a2ac3b6/makedumpfile.c#L5008-L5040

Signed-off-by: Stephen Brennan <stephen.s.brennan@oracle.com>
---
 dump/dump.c           | 38 +++++++++++++++++++++++++++++++-------
 include/sysemu/dump.h |  1 +
 2 files changed, 32 insertions(+), 7 deletions(-)

diff --git a/dump/dump.c b/dump/dump.c
index 74071a1565..fb9040cfbc 100644
--- a/dump/dump.c
+++ b/dump/dump.c
@@ -814,6 +814,16 @@ static int write_start_flat_header(DumpState *s)
     MakedumpfileHeader *mh;
     int ret = 0;
 
+    /* The user can request we not use the flattened format, but
+     * if the file is not seekable, we need to fall back to flattened. */
+    if (s->kdump_reassembled) {
+        if (lseek(s->fd, 0, SEEK_CUR) == (loff_t) -1) {
+            s->kdump_reassembled = false;
+        } else {
+            return 0;
+        }
+    }
+
     QEMU_BUILD_BUG_ON(sizeof *mh > MAX_SIZE_MDF_HEADER);
     mh = g_malloc0(MAX_SIZE_MDF_HEADER);
 
@@ -837,6 +847,10 @@ static int write_end_flat_header(DumpState *s)
 {
     MakedumpfileDataHeader mdh;
 
+    if (s->kdump_reassembled) {
+        return 0;
+    }
+
     mdh.offset = END_FLAG_FLAT_HEADER;
     mdh.buf_size = END_FLAG_FLAT_HEADER;
 
@@ -853,13 +867,21 @@ static int write_buffer(DumpState *s, off_t offset, const void *buf, size_t size
 {
     size_t written_size;
     MakedumpfileDataHeader mdh;
+    loff_t seek_loc;
 
-    mdh.offset = cpu_to_be64(offset);
-    mdh.buf_size = cpu_to_be64(size);
+    if (s->kdump_reassembled) {
+        seek_loc = lseek(s->fd, offset, SEEK_SET);
+        if (seek_loc == (off_t) -1) {
+            return -1;
+        }
+    } else {
+        mdh.offset = cpu_to_be64(offset);
+        mdh.buf_size = cpu_to_be64(size);
 
-    written_size = qemu_write_full(s->fd, &mdh, sizeof(mdh));
-    if (written_size != sizeof(mdh)) {
-        return -1;
+        written_size = qemu_write_full(s->fd, &mdh, sizeof(mdh));
+        if (written_size != sizeof(mdh)) {
+            return -1;
+        }
     }
 
     written_size = qemu_write_full(s->fd, buf, size);
@@ -1775,7 +1797,8 @@ static void vmcoreinfo_update_phys_base(DumpState *s)
 
 static void dump_init(DumpState *s, int fd, bool has_format,
                       DumpGuestMemoryFormat format, bool paging, bool has_filter,
-                      int64_t begin, int64_t length, Error **errp)
+                      int64_t begin, int64_t length, bool kdump_reassembled,
+                      Error **errp)
 {
     ERRP_GUARD();
     VMCoreInfoState *vmci = vmcoreinfo_find();
@@ -1786,6 +1809,7 @@ static void dump_init(DumpState *s, int fd, bool has_format,
     s->has_format = has_format;
     s->format = format;
     s->written_size = 0;
+    s->kdump_reassembled = kdump_reassembled;
 
     /* kdump-compressed is conflict with paging and filter */
     if (has_format && format != DUMP_GUEST_MEMORY_FORMAT_ELF) {
@@ -2168,7 +2192,7 @@ void qmp_dump_guest_memory(bool paging, const char *file,
     dump_state_prepare(s);
 
     dump_init(s, fd, has_format, format, paging, has_begin,
-              begin, length, errp);
+              begin, length, false, errp);
     if (*errp) {
         qatomic_set(&s->status, DUMP_STATUS_FAILED);
         return;
diff --git a/include/sysemu/dump.h b/include/sysemu/dump.h
index e27af8fb34..71ec492fce 100644
--- a/include/sysemu/dump.h
+++ b/include/sysemu/dump.h
@@ -157,6 +157,7 @@ typedef struct DumpState {
     MemoryMappingList list;
     bool resume;
     bool detached;
+    bool kdump_reassembled;
     hwaddr memory_offset;
     int fd;
 
-- 
2.39.3

Hi

On Thu, Sep 14, 2023 at 5:04 AM Stephen Brennan
<stephen.s.brennan@oracle.com> wrote:
>
> The flattened format (currently output by qemu) is used by makedumpfile
> only when it is outputting a vmcore to a file which is not seekable. The
> flattened format functions essentially as a set of instructions of the
> form "seek to the given offset, then write the given bytes out".
>
> The flattened format can be reconstructed using makedumpfile -R, or
> makedumpfile-R.pl, but it is a slow process because it requires copying
> the entire vmcore. The flattened format can also be directly read by
> crash, but still, it requires a lengthy reassembly phase.
>
> To sum up, the flattened format is not an ideal one: it should only be
> used on files which are actually not seekable. This is the exact
> strategy which makedumpfile uses, as seen in the implementation of
> "write_buffer()" in makedumpfile [1]. However, Qemu has always used the
> flattened format. For compatibility it is best not to change the default
> output format without warning. So, add a flag to DumpState which changes
> the output to use the normal (i.e. reassembled) format. This flag will
> be added to the QMP commands in the next change.
>
> [1]: https://github.com/makedumpfile/makedumpfile/blob/f23bb943568188a2746dbf9b6692668f5a2ac3b6/makedumpfile.c#L5008-L5040
>
> Signed-off-by: Stephen Brennan <stephen.s.brennan@oracle.com>
> ---
>  dump/dump.c           | 38 +++++++++++++++++++++++++++++++-------
>  include/sysemu/dump.h |  1 +
>  2 files changed, 32 insertions(+), 7 deletions(-)
>
> diff --git a/dump/dump.c b/dump/dump.c
> index 74071a1565..fb9040cfbc 100644
> --- a/dump/dump.c
> +++ b/dump/dump.c
> @@ -814,6 +814,16 @@ static int write_start_flat_header(DumpState *s)
>      MakedumpfileHeader *mh;
>      int ret = 0;
>
> +    /* The user can request we not use the flattened format, but
> +     * if the file is not seekable, we need to fall back to flattened. */
> +    if (s->kdump_reassembled) {
> +        if (lseek(s->fd, 0, SEEK_CUR) == (loff_t) -1) {
> +            s->kdump_reassembled = false;

Since it was explicitly asked by the user, we better return an error
instead of falling back silently on flattened.


> +        } else {
> +            return 0;
> +        }
> +    }
> +
>      QEMU_BUILD_BUG_ON(sizeof *mh > MAX_SIZE_MDF_HEADER);
>      mh = g_malloc0(MAX_SIZE_MDF_HEADER);
>
> @@ -837,6 +847,10 @@ static int write_end_flat_header(DumpState *s)
>  {
>      MakedumpfileDataHeader mdh;
>
> +    if (s->kdump_reassembled) {
> +        return 0;
> +    }
> +
>      mdh.offset = END_FLAG_FLAT_HEADER;
>      mdh.buf_size = END_FLAG_FLAT_HEADER;
>
> @@ -853,13 +867,21 @@ static int write_buffer(DumpState *s, off_t offset, const void *buf, size_t size
>  {
>      size_t written_size;
>      MakedumpfileDataHeader mdh;
> +    loff_t seek_loc;
>
> -    mdh.offset = cpu_to_be64(offset);
> -    mdh.buf_size = cpu_to_be64(size);
> +    if (s->kdump_reassembled) {
> +        seek_loc = lseek(s->fd, offset, SEEK_SET);
> +        if (seek_loc == (off_t) -1) {
> +            return -1;
> +        }
> +    } else {
> +        mdh.offset = cpu_to_be64(offset);
> +        mdh.buf_size = cpu_to_be64(size);
>
> -    written_size = qemu_write_full(s->fd, &mdh, sizeof(mdh));
> -    if (written_size != sizeof(mdh)) {
> -        return -1;
> +        written_size = qemu_write_full(s->fd, &mdh, sizeof(mdh));
> +        if (written_size != sizeof(mdh)) {
> +            return -1;
> +        }
>      }
>
>      written_size = qemu_write_full(s->fd, buf, size);
> @@ -1775,7 +1797,8 @@ static void vmcoreinfo_update_phys_base(DumpState *s)
>
>  static void dump_init(DumpState *s, int fd, bool has_format,
>                        DumpGuestMemoryFormat format, bool paging, bool has_filter,
> -                      int64_t begin, int64_t length, Error **errp)
> +                      int64_t begin, int64_t length, bool kdump_reassembled,
> +                      Error **errp)
>  {
>      ERRP_GUARD();
>      VMCoreInfoState *vmci = vmcoreinfo_find();
> @@ -1786,6 +1809,7 @@ static void dump_init(DumpState *s, int fd, bool has_format,
>      s->has_format = has_format;
>      s->format = format;
>      s->written_size = 0;
> +    s->kdump_reassembled = kdump_reassembled;
>
>      /* kdump-compressed is conflict with paging and filter */
>      if (has_format && format != DUMP_GUEST_MEMORY_FORMAT_ELF) {
> @@ -2168,7 +2192,7 @@ void qmp_dump_guest_memory(bool paging, const char *file,
>      dump_state_prepare(s);
>
>      dump_init(s, fd, has_format, format, paging, has_begin,
> -              begin, length, errp);
> +              begin, length, false, errp);
>      if (*errp) {
>          qatomic_set(&s->status, DUMP_STATUS_FAILED);
>          return;
> diff --git a/include/sysemu/dump.h b/include/sysemu/dump.h
> index e27af8fb34..71ec492fce 100644
> --- a/include/sysemu/dump.h
> +++ b/include/sysemu/dump.h
> @@ -157,6 +157,7 @@ typedef struct DumpState {
>      MemoryMappingList list;
>      bool resume;
>      bool detached;
> +    bool kdump_reassembled;
>      hwaddr memory_offset;
>      int fd;
>
> --
> 2.39.3
>


--
Marc-André Lureau