[libvirt PATCH v2 06/13] util: extract storage file probe code into virtstoragefileprobe.c

Pavel Hrdina posted 13 patches 5 years ago
[libvirt PATCH v2 06/13] util: extract storage file probe code into virtstoragefileprobe.c
Posted by Pavel Hrdina 5 years ago
This code is not directly relevant to virStorageSource so move it to
separate file.

Signed-off-by: Pavel Hrdina <phrdina@redhat.com>
---
 po/POTFILES.in                        |   1 +
 src/libvirt_private.syms              |   6 +-
 src/qemu/qemu_driver.c                |   1 +
 src/storage/storage_backend_gluster.c |   1 +
 src/storage/storage_util.c            |   1 +
 src/util/meson.build                  |   1 +
 src/util/virstoragefile.c             | 939 +------------------------
 src/util/virstoragefile.h             |  11 -
 src/util/virstoragefileprobe.c        | 967 ++++++++++++++++++++++++++
 src/util/virstoragefileprobe.h        |  44 ++
 10 files changed, 1026 insertions(+), 946 deletions(-)
 create mode 100644 src/util/virstoragefileprobe.c
 create mode 100644 src/util/virstoragefileprobe.h

diff --git a/po/POTFILES.in b/po/POTFILES.in
index e9fc3991f1..19eb15ada0 100644
--- a/po/POTFILES.in
+++ b/po/POTFILES.in
@@ -306,6 +306,7 @@
 @SRCDIR@src/util/virstorageencryption.c
 @SRCDIR@src/util/virstoragefile.c
 @SRCDIR@src/util/virstoragefilebackend.c
+@SRCDIR@src/util/virstoragefileprobe.c
 @SRCDIR@src/util/virstring.c
 @SRCDIR@src/util/virsysinfo.c
 @SRCDIR@src/util/virsystemd.c
diff --git a/src/libvirt_private.syms b/src/libvirt_private.syms
index 84b650cb86..2dfc7e32d5 100644
--- a/src/libvirt_private.syms
+++ b/src/libvirt_private.syms
@@ -3148,7 +3148,6 @@ virStorageFileInit;
 virStorageFileInitAs;
 virStorageFileParseBackingStoreStr;
 virStorageFileParseChainIndex;
-virStorageFileProbeFormat;
 virStorageFileRead;
 virStorageFileReportBrokenChain;
 virStorageFileStat;
@@ -3212,6 +3211,11 @@ virStorageTypeToString;
 virStorageFileBackendRegister;
 
 
+# util/virstoragefileprobe.h
+virStorageFileProbeFormat;
+virStorageFileProbeGetMetadata;
+
+
 # util/virstring.h
 virSkipSpaces;
 virSkipSpacesAndBackslash;
diff --git a/src/qemu/qemu_driver.c b/src/qemu/qemu_driver.c
index 027617deef..34a8fbe233 100644
--- a/src/qemu/qemu_driver.c
+++ b/src/qemu/qemu_driver.c
@@ -83,6 +83,7 @@
 #include "domain_nwfilter.h"
 #include "virhook.h"
 #include "virstoragefile.h"
+#include "virstoragefileprobe.h"
 #include "virfile.h"
 #include "virfdstream.h"
 #include "configmake.h"
diff --git a/src/storage/storage_backend_gluster.c b/src/storage/storage_backend_gluster.c
index 6c99c270da..205a707a17 100644
--- a/src/storage/storage_backend_gluster.c
+++ b/src/storage/storage_backend_gluster.c
@@ -28,6 +28,7 @@
 #include "viralloc.h"
 #include "virerror.h"
 #include "virlog.h"
+#include "virstoragefileprobe.h"
 #include "virstring.h"
 #include "viruri.h"
 #include "storage_util.h"
diff --git a/src/storage/storage_util.c b/src/storage/storage_util.c
index 2e2c7dc68a..4117127d65 100644
--- a/src/storage/storage_util.c
+++ b/src/storage/storage_util.c
@@ -62,6 +62,7 @@
 #include "vircrypto.h"
 #include "viruuid.h"
 #include "virstoragefile.h"
+#include "virstoragefileprobe.h"
 #include "storage_util.h"
 #include "virlog.h"
 #include "virfile.h"
diff --git a/src/util/meson.build b/src/util/meson.build
index 395e70fd38..9fb270fadd 100644
--- a/src/util/meson.build
+++ b/src/util/meson.build
@@ -91,6 +91,7 @@ util_sources = [
   'virstorageencryption.c',
   'virstoragefile.c',
   'virstoragefilebackend.c',
+  'virstoragefileprobe.c',
   'virstring.c',
   'virsysinfo.c',
   'virsystemd.c',
diff --git a/src/util/virstoragefile.c b/src/util/virstoragefile.c
index 13a86f34e5..98a3222d09 100644
--- a/src/util/virstoragefile.c
+++ b/src/util/virstoragefile.c
@@ -22,8 +22,6 @@
 #include <config.h>
 #include "virstoragefile.h"
 
-#include <unistd.h>
-#include <fcntl.h>
 #include "viralloc.h"
 #include "virxml.h"
 #include "viruuid.h"
@@ -32,13 +30,13 @@
 #include "virfile.h"
 #include "vircommand.h"
 #include "virhash.h"
-#include "virendian.h"
 #include "virstring.h"
 #include "viruri.h"
 #include "virbuffer.h"
 #include "virjson.h"
 #include "virstorageencryption.h"
 #include "virstoragefilebackend.h"
+#include "virstoragefileprobe.h"
 #include "virsecret.h"
 #include "virutil.h"
 
@@ -113,650 +111,6 @@ VIR_ENUM_IMPL(virStorageAuth,
               "none", "chap", "ceph",
 );
 
-enum lv_endian {
-    LV_LITTLE_ENDIAN = 1, /* 1234 */
-    LV_BIG_ENDIAN         /* 4321 */
-};
-
-enum {
-    BACKING_STORE_OK,
-    BACKING_STORE_INVALID,
-    BACKING_STORE_ERROR,
-};
-
-#define FILE_TYPE_VERSIONS_LAST 3
-
-struct FileEncryptionInfo {
-    int format; /* Encryption format to assign */
-
-    int magicOffset; /* Byte offset of the magic */
-    const char *magic; /* Optional string of magic */
-
-    enum lv_endian endian; /* Endianness of file format */
-
-    int versionOffset;    /* Byte offset from start of file
-                           * where we find version number,
-                           * -1 to always fail the version test,
-                           * -2 to always pass the version test */
-    int versionSize;      /* Size in bytes of version data (0, 2, or 4) */
-    int versionNumbers[FILE_TYPE_VERSIONS_LAST];
-                          /* Version numbers to validate. Zeroes are ignored. */
-
-    int modeOffset; /* Byte offset of the format native encryption mode */
-    char modeValue; /* Value expected at offset */
-
-    int payloadOffset; /* start offset of the volume data (in 512 byte sectors) */
-};
-
-struct FileTypeInfo {
-    int magicOffset;    /* Byte offset of the magic */
-    const char *magic;  /* Optional string of file magic
-                         * to check at head of file */
-    enum lv_endian endian; /* Endianness of file format */
-
-    int versionOffset;    /* Byte offset from start of file
-                           * where we find version number,
-                           * -1 to always fail the version test,
-                           * -2 to always pass the version test */
-    int versionSize;      /* Size in bytes of version data (0, 2, or 4) */
-    int versionNumbers[FILE_TYPE_VERSIONS_LAST];
-                          /* Version numbers to validate. Zeroes are ignored. */
-    int sizeOffset;       /* Byte offset from start of file
-                           * where we find capacity info,
-                           * -1 to use st_size as capacity */
-    int sizeBytes;        /* Number of bytes for size field */
-    int sizeMultiplier;   /* A scaling factor if size is not in bytes */
-                          /* Store a COW base image path (possibly relative),
-                           * or NULL if there is no COW base image, to RES;
-                           * return BACKING_STORE_* */
-    const struct FileEncryptionInfo *cryptInfo; /* Encryption info */
-    int (*getBackingStore)(char **res, int *format,
-                           const char *buf, size_t buf_size);
-    int (*getFeatures)(virBitmapPtr *features, int format,
-                       char *buf, ssize_t len);
-};
-
-
-static int cowGetBackingStore(char **, int *,
-                              const char *, size_t);
-static int qcowXGetBackingStore(char **, int *,
-                                const char *, size_t);
-static int qcow2GetFeatures(virBitmapPtr *features, int format,
-                            char *buf, ssize_t len);
-static int vmdk4GetBackingStore(char **, int *,
-                                const char *, size_t);
-static int
-qedGetBackingStore(char **, int *, const char *, size_t);
-
-#define QCOWX_HDR_VERSION (4)
-#define QCOWX_HDR_BACKING_FILE_OFFSET (QCOWX_HDR_VERSION+4)
-#define QCOWX_HDR_BACKING_FILE_SIZE (QCOWX_HDR_BACKING_FILE_OFFSET+8)
-#define QCOWX_HDR_IMAGE_SIZE (QCOWX_HDR_BACKING_FILE_SIZE+4+4)
-
-#define QCOW1_HDR_CRYPT (QCOWX_HDR_IMAGE_SIZE+8+1+1+2)
-#define QCOW2_HDR_CRYPT (QCOWX_HDR_IMAGE_SIZE+8)
-
-#define QCOW1_HDR_TOTAL_SIZE (QCOW1_HDR_CRYPT+4+8)
-#define QCOW2_HDR_TOTAL_SIZE (QCOW2_HDR_CRYPT+4+4+8+8+4+4+8)
-
-#define QCOW2_HDR_EXTENSION_END 0
-#define QCOW2_HDR_EXTENSION_BACKING_FORMAT 0xE2792ACA
-
-#define QCOW2v3_HDR_FEATURES_INCOMPATIBLE (QCOW2_HDR_TOTAL_SIZE)
-#define QCOW2v3_HDR_FEATURES_COMPATIBLE (QCOW2v3_HDR_FEATURES_INCOMPATIBLE+8)
-#define QCOW2v3_HDR_FEATURES_AUTOCLEAR (QCOW2v3_HDR_FEATURES_COMPATIBLE+8)
-
-/* The location of the header size [4 bytes] */
-#define QCOW2v3_HDR_SIZE       (QCOW2_HDR_TOTAL_SIZE+8+8+8+4)
-
-#define QED_HDR_FEATURES_OFFSET (4+4+4+4)
-#define QED_HDR_IMAGE_SIZE (QED_HDR_FEATURES_OFFSET+8+8+8+8)
-#define QED_HDR_BACKING_FILE_OFFSET (QED_HDR_IMAGE_SIZE+8)
-#define QED_HDR_BACKING_FILE_SIZE (QED_HDR_BACKING_FILE_OFFSET+4)
-#define QED_F_BACKING_FILE 0x01
-#define QED_F_BACKING_FORMAT_NO_PROBE 0x04
-
-#define PLOOP_IMAGE_SIZE_OFFSET 36
-#define PLOOP_SIZE_MULTIPLIER 512
-
-#define LUKS_HDR_MAGIC_LEN 6
-#define LUKS_HDR_VERSION_LEN 2
-#define LUKS_HDR_CIPHER_NAME_LEN 32
-#define LUKS_HDR_CIPHER_MODE_LEN 32
-#define LUKS_HDR_HASH_SPEC_LEN 32
-#define LUKS_HDR_PAYLOAD_LEN 4
-
-/* Format described by qemu commit id '3e308f20e' */
-#define LUKS_HDR_VERSION_OFFSET LUKS_HDR_MAGIC_LEN
-#define LUKS_HDR_PAYLOAD_OFFSET (LUKS_HDR_MAGIC_LEN+\
-                                 LUKS_HDR_VERSION_LEN+\
-                                 LUKS_HDR_CIPHER_NAME_LEN+\
-                                 LUKS_HDR_CIPHER_MODE_LEN+\
-                                 LUKS_HDR_HASH_SPEC_LEN)
-
-static struct FileEncryptionInfo const luksEncryptionInfo[] = {
-    {
-        .format = VIR_STORAGE_ENCRYPTION_FORMAT_LUKS,
-
-        /* Magic is 'L','U','K','S', 0xBA, 0xBE */
-        .magicOffset = 0,
-        .magic = "\x4c\x55\x4b\x53\xba\xbe",
-        .endian = LV_BIG_ENDIAN,
-
-        .versionOffset  = LUKS_HDR_VERSION_OFFSET,
-        .versionSize = LUKS_HDR_VERSION_LEN,
-        .versionNumbers = {1},
-
-        .modeOffset = -1,
-        .modeValue = -1,
-
-        .payloadOffset = LUKS_HDR_PAYLOAD_OFFSET,
-    },
-    { 0 }
-};
-
-static struct FileEncryptionInfo const qcow1EncryptionInfo[] = {
-    {
-        .format = VIR_STORAGE_ENCRYPTION_FORMAT_QCOW,
-
-        .magicOffset = 0,
-        .magic = NULL,
-        .endian = LV_BIG_ENDIAN,
-
-        .versionOffset  = -1,
-        .versionSize = 0,
-        .versionNumbers = {},
-
-        .modeOffset = QCOW1_HDR_CRYPT,
-        .modeValue = 1,
-
-        .payloadOffset = -1,
-    },
-    { 0 }
-};
-
-static struct FileEncryptionInfo const qcow2EncryptionInfo[] = {
-    {
-        .format = VIR_STORAGE_ENCRYPTION_FORMAT_QCOW,
-
-        .magicOffset = 0,
-        .magic = NULL,
-        .endian = LV_BIG_ENDIAN,
-
-        .versionOffset  = -1,
-        .versionSize = 0,
-        .versionNumbers = {},
-
-        .modeOffset = QCOW2_HDR_CRYPT,
-        .modeValue = 1,
-
-        .payloadOffset = -1,
-    },
-    {
-        .format = VIR_STORAGE_ENCRYPTION_FORMAT_LUKS,
-
-        .magicOffset = 0,
-        .magic = NULL,
-        .endian = LV_BIG_ENDIAN,
-
-        .versionOffset  = -1,
-        .versionSize = 0,
-        .versionNumbers = {},
-
-        .modeOffset = QCOW2_HDR_CRYPT,
-        .modeValue = 2,
-
-        .payloadOffset = -1,
-    },
-    { 0 }
-};
-
-static struct FileTypeInfo const fileTypeInfo[] = {
-    [VIR_STORAGE_FILE_NONE] = { 0, NULL, LV_LITTLE_ENDIAN,
-                                -1, 0, {0}, 0, 0, 0, NULL, NULL, NULL },
-    [VIR_STORAGE_FILE_RAW] = { 0, NULL, LV_LITTLE_ENDIAN,
-                               -1, 0, {0}, 0, 0, 0,
-                               luksEncryptionInfo,
-                               NULL, NULL },
-    [VIR_STORAGE_FILE_DIR] = { 0, NULL, LV_LITTLE_ENDIAN,
-                               -1, 0, {0}, 0, 0, 0, NULL, NULL, NULL },
-    [VIR_STORAGE_FILE_BOCHS] = {
-        /*"Bochs Virtual HD Image", */ /* Untested */
-        0, NULL,
-        LV_LITTLE_ENDIAN, 64, 4, {0x20000},
-        32+16+16+4+4+4+4+4, 8, 1, NULL, NULL, NULL
-    },
-    [VIR_STORAGE_FILE_CLOOP] = {
-        /* #!/bin/sh
-           #V2.0 Format
-           modprobe cloop file=$0 && mount -r -t iso9660 /dev/cloop $1
-        */ /* Untested */
-        0, NULL,
-        LV_LITTLE_ENDIAN, -1, 0, {0},
-        -1, 0, 0, NULL, NULL, NULL
-    },
-    [VIR_STORAGE_FILE_DMG] = {
-        /* XXX QEMU says there's no magic for dmg,
-         * /usr/share/misc/magic lists double magic (both offsets
-         * would have to match) but then disables that check. */
-        0, NULL,
-        0, -1, 0, {0},
-        -1, 0, 0, NULL, NULL, NULL
-    },
-    [VIR_STORAGE_FILE_ISO] = {
-        32769, "CD001",
-        LV_LITTLE_ENDIAN, -2, 0, {0},
-        -1, 0, 0, NULL, NULL, NULL
-    },
-    [VIR_STORAGE_FILE_VPC] = {
-        0, "conectix",
-        LV_BIG_ENDIAN, 12, 4, {0x10000},
-        8 + 4 + 4 + 8 + 4 + 4 + 2 + 2 + 4, 8, 1, NULL, NULL, NULL
-    },
-    /* TODO: add getBackingStore function */
-    [VIR_STORAGE_FILE_VDI] = {
-        64, "\x7f\x10\xda\xbe",
-        LV_LITTLE_ENDIAN, 68, 4, {0x00010001},
-        64 + 5 * 4 + 256 + 7 * 4, 8, 1, NULL, NULL, NULL},
-
-    /* Not direct file formats, but used for various drivers */
-    [VIR_STORAGE_FILE_FAT] = { 0, NULL, LV_LITTLE_ENDIAN,
-                               -1, 0, {0}, 0, 0, 0, NULL, NULL, NULL },
-    [VIR_STORAGE_FILE_VHD] = { 0, NULL, LV_LITTLE_ENDIAN,
-                               -1, 0, {0}, 0, 0, 0, NULL, NULL, NULL },
-    [VIR_STORAGE_FILE_PLOOP] = { 0, "WithouFreSpacExt", LV_LITTLE_ENDIAN,
-                                 -2, 0, {0}, PLOOP_IMAGE_SIZE_OFFSET, 0,
-                                 PLOOP_SIZE_MULTIPLIER, NULL, NULL, NULL },
-
-    /* All formats with a backing store probe below here */
-    [VIR_STORAGE_FILE_COW] = {
-        0, "OOOM",
-        LV_BIG_ENDIAN, 4, 4, {2},
-        4+4+1024+4, 8, 1, NULL, cowGetBackingStore, NULL
-    },
-    [VIR_STORAGE_FILE_QCOW] = {
-        0, "QFI",
-        LV_BIG_ENDIAN, 4, 4, {1},
-        QCOWX_HDR_IMAGE_SIZE, 8, 1,
-        qcow1EncryptionInfo,
-        qcowXGetBackingStore, NULL
-    },
-    [VIR_STORAGE_FILE_QCOW2] = {
-        0, "QFI",
-        LV_BIG_ENDIAN, 4, 4, {2, 3},
-        QCOWX_HDR_IMAGE_SIZE, 8, 1,
-        qcow2EncryptionInfo,
-        qcowXGetBackingStore,
-        qcow2GetFeatures
-    },
-    [VIR_STORAGE_FILE_QED] = {
-        /* https://wiki.qemu.org/Features/QED */
-        0, "QED",
-        LV_LITTLE_ENDIAN, -2, 0, {0},
-        QED_HDR_IMAGE_SIZE, 8, 1, NULL, qedGetBackingStore, NULL
-    },
-    [VIR_STORAGE_FILE_VMDK] = {
-        0, "KDMV",
-        LV_LITTLE_ENDIAN, 4, 4, {1, 2, 3},
-        4+4+4, 8, 512, NULL, vmdk4GetBackingStore, NULL
-    },
-};
-G_STATIC_ASSERT(G_N_ELEMENTS(fileTypeInfo) == VIR_STORAGE_FILE_LAST);
-
-
-/* qcow2 compatible features in the order they appear on-disk */
-enum qcow2CompatibleFeature {
-    QCOW2_COMPATIBLE_FEATURE_LAZY_REFCOUNTS = 0,
-
-    QCOW2_COMPATIBLE_FEATURE_LAST
-};
-
-/* conversion to virStorageFileFeature */
-static const int qcow2CompatibleFeatureArray[] = {
-    VIR_STORAGE_FILE_FEATURE_LAZY_REFCOUNTS,
-};
-G_STATIC_ASSERT(G_N_ELEMENTS(qcow2CompatibleFeatureArray) ==
-       QCOW2_COMPATIBLE_FEATURE_LAST);
-
-static int
-cowGetBackingStore(char **res,
-                   int *format,
-                   const char *buf,
-                   size_t buf_size)
-{
-#define COW_FILENAME_MAXLEN 1024
-    *res = NULL;
-    *format = VIR_STORAGE_FILE_AUTO;
-
-    if (buf_size < 4+4+ COW_FILENAME_MAXLEN)
-        return BACKING_STORE_INVALID;
-    if (buf[4+4] == '\0') { /* cow_header_v2.backing_file[0] */
-        *format = VIR_STORAGE_FILE_NONE;
-        return BACKING_STORE_OK;
-    }
-
-    *res = g_strndup((const char *)buf + 4 + 4, COW_FILENAME_MAXLEN);
-    return BACKING_STORE_OK;
-}
-
-
-static int
-qcow2GetExtensions(const char *buf,
-                   size_t buf_size,
-                   int *backingFormat)
-{
-    size_t offset;
-    size_t extension_start;
-    size_t extension_end;
-    int version = virReadBufInt32BE(buf + QCOWX_HDR_VERSION);
-
-    if (version < 2) {
-        /* QCow1 doesn't have the extensions capability
-         * used to store backing format */
-        return 0;
-    }
-
-    if (version == 2)
-        extension_start = QCOW2_HDR_TOTAL_SIZE;
-    else
-        extension_start = virReadBufInt32BE(buf + QCOW2v3_HDR_SIZE);
-
-    /*
-     * Traditionally QCow2 files had a layout of
-     *
-     * [header]
-     * [backingStoreName]
-     *
-     * Although the backingStoreName typically followed
-     * the header immediately, this was not required by
-     * the format. By specifying a higher byte offset for
-     * the backing file offset in the header, it was
-     * possible to leave space between the header and
-     * start of backingStore.
-     *
-     * This hack is now used to store extensions to the
-     * qcow2 format:
-     *
-     * [header]
-     * [extensions]
-     * [backingStoreName]
-     *
-     * Thus the file region to search for extensions is
-     * between the end of the header (QCOW2_HDR_TOTAL_SIZE)
-     * and the start of the backingStoreName (offset)
-     *
-     * for qcow2 v3 images, the length of the header
-     * is stored at QCOW2v3_HDR_SIZE
-     */
-    extension_end = virReadBufInt64BE(buf + QCOWX_HDR_BACKING_FILE_OFFSET);
-    if (extension_end > buf_size)
-        return -1;
-
-    /*
-     * The extensions take format of
-     *
-     * int32: magic
-     * int32: length
-     * byte[length]: payload
-     *
-     * Unknown extensions can be ignored by skipping
-     * over "length" bytes in the data stream.
-     */
-    offset = extension_start;
-    while (offset < (buf_size-8) &&
-           offset < (extension_end-8)) {
-        unsigned int magic = virReadBufInt32BE(buf + offset);
-        unsigned int len = virReadBufInt32BE(buf + offset + 4);
-
-        offset += 8;
-
-        if ((offset + len) < offset)
-            break;
-
-        if ((offset + len) > buf_size)
-            break;
-
-        switch (magic) {
-        case QCOW2_HDR_EXTENSION_BACKING_FORMAT: {
-            g_autofree char *tmp = NULL;
-            if (!backingFormat)
-                break;
-
-            tmp = g_new0(char, len + 1);
-            memcpy(tmp, buf + offset, len);
-            tmp[len] = '\0';
-
-            *backingFormat = virStorageFileFormatTypeFromString(tmp);
-            if (*backingFormat <= VIR_STORAGE_FILE_NONE)
-                return -1;
-            break;
-        }
-
-        case QCOW2_HDR_EXTENSION_END:
-            return 0;
-        }
-
-        offset += len;
-    }
-
-    return 0;
-}
-
-
-static int
-qcowXGetBackingStore(char **res,
-                     int *format,
-                     const char *buf,
-                     size_t buf_size)
-{
-    unsigned long long offset;
-    unsigned int size;
-
-    *res = NULL;
-    *format = VIR_STORAGE_FILE_AUTO;
-
-    if (buf_size < QCOWX_HDR_BACKING_FILE_OFFSET+8+4)
-        return BACKING_STORE_INVALID;
-
-    offset = virReadBufInt64BE(buf + QCOWX_HDR_BACKING_FILE_OFFSET);
-    if (offset > buf_size)
-        return BACKING_STORE_INVALID;
-
-    if (offset == 0) {
-        *format = VIR_STORAGE_FILE_NONE;
-        return BACKING_STORE_OK;
-    }
-
-    size = virReadBufInt32BE(buf + QCOWX_HDR_BACKING_FILE_SIZE);
-    if (size == 0) {
-        *format = VIR_STORAGE_FILE_NONE;
-        return BACKING_STORE_OK;
-    }
-    if (size > 1023)
-        return BACKING_STORE_INVALID;
-    if (offset + size > buf_size || offset + size < offset)
-        return BACKING_STORE_INVALID;
-    *res = g_new0(char, size + 1);
-    memcpy(*res, buf + offset, size);
-    (*res)[size] = '\0';
-
-    if (qcow2GetExtensions(buf, buf_size, format) < 0)
-        return BACKING_STORE_INVALID;
-
-    return BACKING_STORE_OK;
-}
-
-
-static int
-vmdk4GetBackingStore(char **res,
-                     int *format,
-                     const char *buf,
-                     size_t buf_size)
-{
-    static const char prefix[] = "parentFileNameHint=\"";
-    char *start, *end;
-    size_t len;
-    g_autofree char *desc = NULL;
-
-    desc = g_new0(char, VIR_STORAGE_MAX_HEADER);
-
-    *res = NULL;
-    /*
-     * Technically this should have been VMDK, since
-     * VMDK spec / VMware impl only support VMDK backed
-     * by VMDK. QEMU isn't following this though and
-     * does probing on VMDK backing files, hence we set
-     * AUTO
-     */
-    *format = VIR_STORAGE_FILE_AUTO;
-
-    if (buf_size <= 0x200)
-        return BACKING_STORE_INVALID;
-
-    len = buf_size - 0x200;
-    if (len > VIR_STORAGE_MAX_HEADER)
-        len = VIR_STORAGE_MAX_HEADER;
-    memcpy(desc, buf + 0x200, len);
-    desc[len] = '\0';
-    start = strstr(desc, prefix);
-    if (start == NULL) {
-        *format = VIR_STORAGE_FILE_NONE;
-        return BACKING_STORE_OK;
-    }
-    start += strlen(prefix);
-    end = strchr(start, '"');
-    if (end == NULL)
-        return BACKING_STORE_INVALID;
-
-    if (end == start) {
-        *format = VIR_STORAGE_FILE_NONE;
-        return BACKING_STORE_OK;
-    }
-    *end = '\0';
-    *res = g_strdup(start);
-
-    return BACKING_STORE_OK;
-}
-
-static int
-qedGetBackingStore(char **res,
-                   int *format,
-                   const char *buf,
-                   size_t buf_size)
-{
-    unsigned long long flags;
-    unsigned long offset, size;
-
-    *res = NULL;
-    /* Check if this image has a backing file */
-    if (buf_size < QED_HDR_FEATURES_OFFSET+8)
-        return BACKING_STORE_INVALID;
-    flags = virReadBufInt64LE(buf + QED_HDR_FEATURES_OFFSET);
-    if (!(flags & QED_F_BACKING_FILE)) {
-        *format = VIR_STORAGE_FILE_NONE;
-        return BACKING_STORE_OK;
-    }
-
-    /* Parse the backing file */
-    if (buf_size < QED_HDR_BACKING_FILE_OFFSET+8)
-        return BACKING_STORE_INVALID;
-    offset = virReadBufInt32LE(buf + QED_HDR_BACKING_FILE_OFFSET);
-    if (offset > buf_size)
-        return BACKING_STORE_INVALID;
-    size = virReadBufInt32LE(buf + QED_HDR_BACKING_FILE_SIZE);
-    if (size == 0)
-        return BACKING_STORE_OK;
-    if (offset + size > buf_size || offset + size < offset)
-        return BACKING_STORE_INVALID;
-    *res = g_new0(char, size + 1);
-    memcpy(*res, buf + offset, size);
-    (*res)[size] = '\0';
-
-    if (flags & QED_F_BACKING_FORMAT_NO_PROBE)
-        *format = VIR_STORAGE_FILE_RAW;
-    else
-        *format = VIR_STORAGE_FILE_AUTO_SAFE;
-
-    return BACKING_STORE_OK;
-}
-
-
-static bool
-virStorageFileMatchesMagic(int magicOffset,
-                           const char *magic,
-                           char *buf,
-                           size_t buflen)
-{
-    int mlen;
-
-    if (magic == NULL)
-        return false;
-
-    /* Validate magic data */
-    mlen = strlen(magic);
-    if (magicOffset + mlen > buflen)
-        return false;
-
-    if (memcmp(buf + magicOffset, magic, mlen) != 0)
-        return false;
-
-    return true;
-}
-
-
-static bool
-virStorageFileMatchesVersion(int versionOffset,
-                             int versionSize,
-                             const int *versionNumbers,
-                             int endian,
-                             char *buf,
-                             size_t buflen)
-{
-    int version;
-    size_t i;
-
-    /* Validate version number info */
-    if (versionOffset == -1)
-        return false;
-
-    /* -2 == non-versioned file format, so trivially match */
-    if (versionOffset == -2)
-        return true;
-
-    /* A positive versionOffset, requires using a valid versionSize */
-    if (versionSize != 2 && versionSize != 4)
-        return false;
-
-    if ((versionOffset + versionSize) > buflen)
-        return false;
-
-    if (endian == LV_LITTLE_ENDIAN) {
-        if (versionSize == 4)
-            version = virReadBufInt32LE(buf +
-                                        versionOffset);
-        else
-            version = virReadBufInt16LE(buf +
-                                        versionOffset);
-    } else {
-        if (versionSize == 4)
-            version = virReadBufInt32BE(buf +
-                                        versionOffset);
-        else
-            version = virReadBufInt16BE(buf +
-                                        versionOffset);
-    }
-
-    for (i = 0;
-         i < FILE_TYPE_VERSIONS_LAST && versionNumbers[i];
-         i++) {
-        VIR_DEBUG("Compare detected version %d vs one of the expected versions %d",
-                  version, versionNumbers[i]);
-        if (version == versionNumbers[i])
-            return true;
-    }
-
-    return false;
-}
 
 bool
 virStorageIsFile(const char *backing)
@@ -792,289 +146,6 @@ virStorageIsRelative(const char *backing)
 }
 
 
-static int
-virStorageFileProbeFormatFromBuf(const char *path,
-                                 char *buf,
-                                 size_t buflen)
-{
-    int format = VIR_STORAGE_FILE_RAW;
-    size_t i;
-    int possibleFormat = VIR_STORAGE_FILE_RAW;
-    VIR_DEBUG("path=%s, buf=%p, buflen=%zu", path, buf, buflen);
-
-    /* First check file magic */
-    for (i = 0; i < VIR_STORAGE_FILE_LAST; i++) {
-        if (virStorageFileMatchesMagic(fileTypeInfo[i].magicOffset,
-                                       fileTypeInfo[i].magic,
-                                       buf, buflen)) {
-            if (!virStorageFileMatchesVersion(fileTypeInfo[i].versionOffset,
-                                              fileTypeInfo[i].versionSize,
-                                              fileTypeInfo[i].versionNumbers,
-                                              fileTypeInfo[i].endian,
-                                              buf, buflen)) {
-                possibleFormat = i;
-                continue;
-            }
-            format = i;
-            goto cleanup;
-        }
-    }
-
-    if (possibleFormat != VIR_STORAGE_FILE_RAW)
-        VIR_WARN("File %s matches %s magic, but version is wrong. "
-                 "Please report new version to libvir-list@redhat.com",
-                 path, virStorageFileFormatTypeToString(possibleFormat));
-
- cleanup:
-    VIR_DEBUG("format=%d", format);
-    return format;
-}
-
-
-static int
-qcow2GetFeatures(virBitmapPtr *features,
-                 int format,
-                 char *buf,
-                 ssize_t len)
-{
-    int version = -1;
-    virBitmapPtr feat = NULL;
-    uint64_t bits;
-    size_t i;
-
-    version = virReadBufInt32BE(buf + fileTypeInfo[format].versionOffset);
-
-    if (version == 2)
-        return 0;
-
-    if (len < QCOW2v3_HDR_SIZE)
-        return -1;
-
-    feat = virBitmapNew(VIR_STORAGE_FILE_FEATURE_LAST);
-
-    /* todo: check for incompatible or autoclear features? */
-    bits = virReadBufInt64BE(buf + QCOW2v3_HDR_FEATURES_COMPATIBLE);
-    for (i = 0; i < QCOW2_COMPATIBLE_FEATURE_LAST; i++) {
-        if (bits & ((uint64_t) 1 << i))
-            ignore_value(virBitmapSetBit(feat, qcow2CompatibleFeatureArray[i]));
-    }
-
-    *features = feat;
-    return 0;
-}
-
-
-static bool
-virStorageFileHasEncryptionFormat(const struct FileEncryptionInfo *info,
-                                  char *buf,
-                                  size_t len)
-{
-    if (!info->magic && info->modeOffset == -1)
-        return false; /* Shouldn't happen - expect at least one */
-
-    if (info->magic) {
-        if (!virStorageFileMatchesMagic(info->magicOffset,
-                                        info->magic,
-                                        buf, len))
-            return false;
-
-        if (info->versionOffset != -1 &&
-            !virStorageFileMatchesVersion(info->versionOffset,
-                                          info->versionSize,
-                                          info->versionNumbers,
-                                          info->endian,
-                                          buf, len))
-            return false;
-
-        return true;
-    } else if (info->modeOffset != -1) {
-        int crypt_format;
-
-        if (info->modeOffset >= len)
-            return false;
-
-        crypt_format = virReadBufInt32BE(buf + info->modeOffset);
-        if (crypt_format != info->modeValue)
-            return false;
-
-        return true;
-    } else {
-        return false;
-    }
-}
-
-
-static int
-virStorageFileGetEncryptionPayloadOffset(const struct FileEncryptionInfo *info,
-                                         char *buf)
-{
-    int payload_offset = -1;
-
-    if (info->payloadOffset != -1) {
-        if (info->endian == LV_LITTLE_ENDIAN)
-            payload_offset = virReadBufInt32LE(buf + info->payloadOffset);
-        else
-            payload_offset = virReadBufInt32BE(buf + info->payloadOffset);
-    }
-
-    return payload_offset;
-}
-
-
-/* Given a header in BUF with length LEN, as parsed from the storage file
- * assuming it has the given FORMAT, populate information into META
- * with information about the file and its backing store. Return format
- * of the backing store as BACKING_FORMAT. PATH and FORMAT have to be
- * pre-populated in META.
- *
- * Note that this function may be called repeatedly on @meta, so it must
- * clean up any existing allocated memory which would be overwritten.
- */
-static int
-virStorageFileGetMetadataInternal(virStorageSourcePtr meta,
-                                  char *buf,
-                                  size_t len)
-{
-    int format;
-    size_t i;
-
-    VIR_DEBUG("path=%s, buf=%p, len=%zu, meta->format=%d",
-              meta->path, buf, len, meta->format);
-
-    if (meta->format == VIR_STORAGE_FILE_AUTO)
-        meta->format = virStorageFileProbeFormatFromBuf(meta->path, buf, len);
-
-    if (meta->format <= VIR_STORAGE_FILE_NONE ||
-        meta->format >= VIR_STORAGE_FILE_LAST) {
-        virReportSystemError(EINVAL, _("unknown storage file meta->format %d"),
-                             meta->format);
-        return -1;
-    }
-
-    if (fileTypeInfo[meta->format].cryptInfo != NULL) {
-        for (i = 0; fileTypeInfo[meta->format].cryptInfo[i].format != 0; i++) {
-            if (virStorageFileHasEncryptionFormat(&fileTypeInfo[meta->format].cryptInfo[i],
-                                                  buf, len)) {
-                int expt_fmt = fileTypeInfo[meta->format].cryptInfo[i].format;
-                if (!meta->encryption) {
-                    meta->encryption = g_new0(virStorageEncryption, 1);
-                    meta->encryption->format = expt_fmt;
-                } else {
-                    if (meta->encryption->format != expt_fmt) {
-                        virReportError(VIR_ERR_XML_ERROR,
-                                       _("encryption format %d doesn't match "
-                                         "expected format %d"),
-                                       meta->encryption->format, expt_fmt);
-                        return -1;
-                    }
-                }
-                meta->encryption->payload_offset =
-                    virStorageFileGetEncryptionPayloadOffset(&fileTypeInfo[meta->format].cryptInfo[i], buf);
-            }
-        }
-    }
-
-    /* XXX we should consider moving virStorageBackendUpdateVolInfo
-     * code into this method, for non-magic files
-     */
-    if (!fileTypeInfo[meta->format].magic)
-        return 0;
-
-    /* Optionally extract capacity from file */
-    if (fileTypeInfo[meta->format].sizeOffset != -1) {
-        if ((fileTypeInfo[meta->format].sizeOffset + 8) > len)
-            return 0;
-
-        if (fileTypeInfo[meta->format].endian == LV_LITTLE_ENDIAN)
-            meta->capacity = virReadBufInt64LE(buf +
-                                               fileTypeInfo[meta->format].sizeOffset);
-        else
-            meta->capacity = virReadBufInt64BE(buf +
-                                               fileTypeInfo[meta->format].sizeOffset);
-        /* Avoid unlikely, but theoretically possible overflow */
-        if (meta->capacity > (ULLONG_MAX /
-                              fileTypeInfo[meta->format].sizeMultiplier))
-            return 0;
-        meta->capacity *= fileTypeInfo[meta->format].sizeMultiplier;
-    }
-
-    VIR_FREE(meta->backingStoreRaw);
-    if (fileTypeInfo[meta->format].getBackingStore != NULL) {
-        int store = fileTypeInfo[meta->format].getBackingStore(&meta->backingStoreRaw,
-                                                               &format,
-                                                               buf, len);
-        meta->backingStoreRawFormat = format;
-
-        if (store == BACKING_STORE_INVALID)
-            return 0;
-
-        if (store == BACKING_STORE_ERROR)
-            return -1;
-    }
-
-    virBitmapFree(meta->features);
-    meta->features = NULL;
-    if (fileTypeInfo[meta->format].getFeatures != NULL &&
-        fileTypeInfo[meta->format].getFeatures(&meta->features, meta->format, buf, len) < 0)
-        return -1;
-
-    VIR_FREE(meta->compat);
-    if (meta->format == VIR_STORAGE_FILE_QCOW2 && meta->features)
-        meta->compat = g_strdup("1.1");
-
-    return 0;
-}
-
-
-/**
- * virStorageFileProbeFormat:
- *
- * Probe for the format of 'path', returning the detected
- * disk format.
- *
- * Callers are advised never to trust the returned 'format'
- * unless it is listed as VIR_STORAGE_FILE_RAW, since a
- * malicious guest can turn a raw file into any other non-raw
- * format at will.
- *
- * Best option: Don't use this function
- */
-int
-virStorageFileProbeFormat(const char *path, uid_t uid, gid_t gid)
-{
-    struct stat sb;
-    ssize_t len = VIR_STORAGE_MAX_HEADER;
-    VIR_AUTOCLOSE fd = -1;
-    g_autofree char *header = NULL;
-
-    if ((fd = virFileOpenAs(path, O_RDONLY, 0, uid, gid, 0)) < 0) {
-        virReportSystemError(-fd, _("Failed to open file '%s'"), path);
-        return -1;
-    }
-
-    if (fstat(fd, &sb) < 0) {
-        virReportSystemError(errno, _("cannot stat file '%s'"), path);
-        return -1;
-    }
-
-    /* No header to probe for directories */
-    if (S_ISDIR(sb.st_mode))
-        return VIR_STORAGE_FILE_DIR;
-
-    if (lseek(fd, 0, SEEK_SET) == (off_t)-1) {
-        virReportSystemError(errno, _("cannot set to start of '%s'"), path);
-        return -1;
-    }
-
-    if ((len = virFileReadHeaderFD(fd, len, &header)) < 0) {
-        virReportSystemError(errno, _("cannot read header '%s'"), path);
-        return -1;
-    }
-
-    return virStorageFileProbeFormatFromBuf(path, header, len);
-}
-
-
 static virStorageSourcePtr
 virStorageFileMetadataNew(const char *path,
                           int format)
@@ -1123,7 +194,7 @@ virStorageFileGetMetadataFromBuf(const char *path,
     if (!(ret = virStorageFileMetadataNew(path, format)))
         return NULL;
 
-    if (virStorageFileGetMetadataInternal(ret, buf, len) < 0) {
+    if (virStorageFileProbeGetMetadata(ret, buf, len) < 0) {
         virObjectUnref(ret);
         return NULL;
     }
@@ -1183,7 +254,7 @@ virStorageFileGetMetadataFromFD(const char *path,
         return NULL;
     }
 
-    if (virStorageFileGetMetadataInternal(meta, buf, len) < 0)
+    if (virStorageFileProbeGetMetadata(meta, buf, len) < 0)
         return NULL;
 
     if (S_ISREG(sb.st_mode))
@@ -5149,7 +4220,7 @@ virStorageFileGetMetadataRecurse(virStorageSourcePtr src,
                                                    &buf, &headerLen, cycle) < 0)
         return -1;
 
-    if (virStorageFileGetMetadataInternal(src, buf, headerLen) < 0)
+    if (virStorageFileProbeGetMetadata(src, buf, headerLen) < 0)
         return -1;
 
     /* If we probed the format we MUST ensure that nothing else than the current
@@ -5292,7 +4363,7 @@ virStorageFileGetBackingStoreStr(virStorageSourcePtr src,
     if (!(tmp = virStorageSourceCopy(src, false)))
         return -1;
 
-    if (virStorageFileGetMetadataInternal(tmp, buf, headerLen) < 0)
+    if (virStorageFileProbeGetMetadata(tmp, buf, headerLen) < 0)
         return -1;
 
     *backing = g_steal_pointer(&tmp->backingStoreRaw);
diff --git a/src/util/virstoragefile.h b/src/util/virstoragefile.h
index 46da6a8a18..27ac6a493f 100644
--- a/src/util/virstoragefile.h
+++ b/src/util/virstoragefile.h
@@ -31,15 +31,6 @@
 #include "virenum.h"
 #include "virpci.h"
 
-/* Minimum header size required to probe all known formats with
- * virStorageFileProbeFormat, or obtain metadata from a known format.
- * Rounded to multiple of 512 (ISO has a 5-byte magic at offset
- * 32769).  Some formats can be probed with fewer bytes.  Although
- * some formats theoretically permit metadata that can rely on offsets
- * beyond this size, in practice that doesn't matter.  */
-#define VIR_STORAGE_MAX_HEADER 0x8200
-
-
 /* Types of disk backends (host resource).  Comparable to the public
  * virStorageVolType, except we have an undetermined state, don't have
  * a netdir type, and add a volume type for reference through a
@@ -401,8 +392,6 @@ G_DEFINE_AUTOPTR_CLEANUP_FUNC(virStorageSource, virObjectUnref);
 # define DEV_BSIZE 512
 #endif
 
-int virStorageFileProbeFormat(const char *path, uid_t uid, gid_t gid);
-
 virStorageSourcePtr virStorageFileGetMetadataFromFD(const char *path,
                                                     int fd,
                                                     int format);
diff --git a/src/util/virstoragefileprobe.c b/src/util/virstoragefileprobe.c
new file mode 100644
index 0000000000..bca098cd35
--- /dev/null
+++ b/src/util/virstoragefileprobe.c
@@ -0,0 +1,967 @@
+/*
+ * virstoragefileprobe.c: file utility functions for FS storage backend
+ *
+ * Copyright (C) 2007-2017 Red Hat, Inc.
+ * Copyright (C) 2007-2008 Daniel P. Berrange
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library.  If not, see
+ * <http://www.gnu.org/licenses/>.
+ */
+
+#include <config.h>
+
+#include <sys/types.h>
+#include <unistd.h>
+#include <fcntl.h>
+
+#include "internal.h"
+#include "viralloc.h"
+#include "virbitmap.h"
+#include "virendian.h"
+#include "virfile.h"
+#include "virlog.h"
+#include "virstoragefile.h"
+#include "virstoragefileprobe.h"
+
+#define VIR_FROM_THIS VIR_FROM_STORAGE
+
+VIR_LOG_INIT("util.storagefileprobe");
+
+enum lv_endian {
+    LV_LITTLE_ENDIAN = 1, /* 1234 */
+    LV_BIG_ENDIAN         /* 4321 */
+};
+
+enum {
+    BACKING_STORE_OK,
+    BACKING_STORE_INVALID,
+    BACKING_STORE_ERROR,
+};
+
+#define FILE_TYPE_VERSIONS_LAST 3
+
+struct FileEncryptionInfo {
+    int format; /* Encryption format to assign */
+
+    int magicOffset; /* Byte offset of the magic */
+    const char *magic; /* Optional string of magic */
+
+    enum lv_endian endian; /* Endianness of file format */
+
+    int versionOffset;    /* Byte offset from start of file
+                           * where we find version number,
+                           * -1 to always fail the version test,
+                           * -2 to always pass the version test */
+    int versionSize;      /* Size in bytes of version data (0, 2, or 4) */
+    int versionNumbers[FILE_TYPE_VERSIONS_LAST];
+                          /* Version numbers to validate. Zeroes are ignored. */
+
+    int modeOffset; /* Byte offset of the format native encryption mode */
+    char modeValue; /* Value expected at offset */
+
+    int payloadOffset; /* start offset of the volume data (in 512 byte sectors) */
+};
+
+struct FileTypeInfo {
+    int magicOffset;    /* Byte offset of the magic */
+    const char *magic;  /* Optional string of file magic
+                         * to check at head of file */
+    enum lv_endian endian; /* Endianness of file format */
+
+    int versionOffset;    /* Byte offset from start of file
+                           * where we find version number,
+                           * -1 to always fail the version test,
+                           * -2 to always pass the version test */
+    int versionSize;      /* Size in bytes of version data (0, 2, or 4) */
+    int versionNumbers[FILE_TYPE_VERSIONS_LAST];
+                          /* Version numbers to validate. Zeroes are ignored. */
+    int sizeOffset;       /* Byte offset from start of file
+                           * where we find capacity info,
+                           * -1 to use st_size as capacity */
+    int sizeBytes;        /* Number of bytes for size field */
+    int sizeMultiplier;   /* A scaling factor if size is not in bytes */
+                          /* Store a COW base image path (possibly relative),
+                           * or NULL if there is no COW base image, to RES;
+                           * return BACKING_STORE_* */
+    const struct FileEncryptionInfo *cryptInfo; /* Encryption info */
+    int (*getBackingStore)(char **res, int *format,
+                           const char *buf, size_t buf_size);
+    int (*getFeatures)(virBitmapPtr *features, int format,
+                       char *buf, ssize_t len);
+};
+
+
+static int cowGetBackingStore(char **, int *,
+                              const char *, size_t);
+static int qcowXGetBackingStore(char **, int *,
+                                const char *, size_t);
+static int qcow2GetFeatures(virBitmapPtr *features, int format,
+                            char *buf, ssize_t len);
+static int vmdk4GetBackingStore(char **, int *,
+                                const char *, size_t);
+static int
+qedGetBackingStore(char **, int *, const char *, size_t);
+
+#define QCOWX_HDR_VERSION (4)
+#define QCOWX_HDR_BACKING_FILE_OFFSET (QCOWX_HDR_VERSION+4)
+#define QCOWX_HDR_BACKING_FILE_SIZE (QCOWX_HDR_BACKING_FILE_OFFSET+8)
+#define QCOWX_HDR_IMAGE_SIZE (QCOWX_HDR_BACKING_FILE_SIZE+4+4)
+
+#define QCOW1_HDR_CRYPT (QCOWX_HDR_IMAGE_SIZE+8+1+1+2)
+#define QCOW2_HDR_CRYPT (QCOWX_HDR_IMAGE_SIZE+8)
+
+#define QCOW1_HDR_TOTAL_SIZE (QCOW1_HDR_CRYPT+4+8)
+#define QCOW2_HDR_TOTAL_SIZE (QCOW2_HDR_CRYPT+4+4+8+8+4+4+8)
+
+#define QCOW2_HDR_EXTENSION_END 0
+#define QCOW2_HDR_EXTENSION_BACKING_FORMAT 0xE2792ACA
+
+#define QCOW2v3_HDR_FEATURES_INCOMPATIBLE (QCOW2_HDR_TOTAL_SIZE)
+#define QCOW2v3_HDR_FEATURES_COMPATIBLE (QCOW2v3_HDR_FEATURES_INCOMPATIBLE+8)
+#define QCOW2v3_HDR_FEATURES_AUTOCLEAR (QCOW2v3_HDR_FEATURES_COMPATIBLE+8)
+
+/* The location of the header size [4 bytes] */
+#define QCOW2v3_HDR_SIZE       (QCOW2_HDR_TOTAL_SIZE+8+8+8+4)
+
+#define QED_HDR_FEATURES_OFFSET (4+4+4+4)
+#define QED_HDR_IMAGE_SIZE (QED_HDR_FEATURES_OFFSET+8+8+8+8)
+#define QED_HDR_BACKING_FILE_OFFSET (QED_HDR_IMAGE_SIZE+8)
+#define QED_HDR_BACKING_FILE_SIZE (QED_HDR_BACKING_FILE_OFFSET+4)
+#define QED_F_BACKING_FILE 0x01
+#define QED_F_BACKING_FORMAT_NO_PROBE 0x04
+
+#define PLOOP_IMAGE_SIZE_OFFSET 36
+#define PLOOP_SIZE_MULTIPLIER 512
+
+#define LUKS_HDR_MAGIC_LEN 6
+#define LUKS_HDR_VERSION_LEN 2
+#define LUKS_HDR_CIPHER_NAME_LEN 32
+#define LUKS_HDR_CIPHER_MODE_LEN 32
+#define LUKS_HDR_HASH_SPEC_LEN 32
+#define LUKS_HDR_PAYLOAD_LEN 4
+
+/* Format described by qemu commit id '3e308f20e' */
+#define LUKS_HDR_VERSION_OFFSET LUKS_HDR_MAGIC_LEN
+#define LUKS_HDR_PAYLOAD_OFFSET (LUKS_HDR_MAGIC_LEN+\
+                                 LUKS_HDR_VERSION_LEN+\
+                                 LUKS_HDR_CIPHER_NAME_LEN+\
+                                 LUKS_HDR_CIPHER_MODE_LEN+\
+                                 LUKS_HDR_HASH_SPEC_LEN)
+
+static struct FileEncryptionInfo const luksEncryptionInfo[] = {
+    {
+        .format = VIR_STORAGE_ENCRYPTION_FORMAT_LUKS,
+
+        /* Magic is 'L','U','K','S', 0xBA, 0xBE */
+        .magicOffset = 0,
+        .magic = "\x4c\x55\x4b\x53\xba\xbe",
+        .endian = LV_BIG_ENDIAN,
+
+        .versionOffset  = LUKS_HDR_VERSION_OFFSET,
+        .versionSize = LUKS_HDR_VERSION_LEN,
+        .versionNumbers = {1},
+
+        .modeOffset = -1,
+        .modeValue = -1,
+
+        .payloadOffset = LUKS_HDR_PAYLOAD_OFFSET,
+    },
+    { 0 }
+};
+
+static struct FileEncryptionInfo const qcow1EncryptionInfo[] = {
+    {
+        .format = VIR_STORAGE_ENCRYPTION_FORMAT_QCOW,
+
+        .magicOffset = 0,
+        .magic = NULL,
+        .endian = LV_BIG_ENDIAN,
+
+        .versionOffset  = -1,
+        .versionSize = 0,
+        .versionNumbers = {},
+
+        .modeOffset = QCOW1_HDR_CRYPT,
+        .modeValue = 1,
+
+        .payloadOffset = -1,
+    },
+    { 0 }
+};
+
+static struct FileEncryptionInfo const qcow2EncryptionInfo[] = {
+    {
+        .format = VIR_STORAGE_ENCRYPTION_FORMAT_QCOW,
+
+        .magicOffset = 0,
+        .magic = NULL,
+        .endian = LV_BIG_ENDIAN,
+
+        .versionOffset  = -1,
+        .versionSize = 0,
+        .versionNumbers = {},
+
+        .modeOffset = QCOW2_HDR_CRYPT,
+        .modeValue = 1,
+
+        .payloadOffset = -1,
+    },
+    {
+        .format = VIR_STORAGE_ENCRYPTION_FORMAT_LUKS,
+
+        .magicOffset = 0,
+        .magic = NULL,
+        .endian = LV_BIG_ENDIAN,
+
+        .versionOffset  = -1,
+        .versionSize = 0,
+        .versionNumbers = {},
+
+        .modeOffset = QCOW2_HDR_CRYPT,
+        .modeValue = 2,
+
+        .payloadOffset = -1,
+    },
+    { 0 }
+};
+
+static struct FileTypeInfo const fileTypeInfo[] = {
+    [VIR_STORAGE_FILE_NONE] = { 0, NULL, LV_LITTLE_ENDIAN,
+                                -1, 0, {0}, 0, 0, 0, NULL, NULL, NULL },
+    [VIR_STORAGE_FILE_RAW] = { 0, NULL, LV_LITTLE_ENDIAN,
+                               -1, 0, {0}, 0, 0, 0,
+                               luksEncryptionInfo,
+                               NULL, NULL },
+    [VIR_STORAGE_FILE_DIR] = { 0, NULL, LV_LITTLE_ENDIAN,
+                               -1, 0, {0}, 0, 0, 0, NULL, NULL, NULL },
+    [VIR_STORAGE_FILE_BOCHS] = {
+        /*"Bochs Virtual HD Image", */ /* Untested */
+        0, NULL,
+        LV_LITTLE_ENDIAN, 64, 4, {0x20000},
+        32+16+16+4+4+4+4+4, 8, 1, NULL, NULL, NULL
+    },
+    [VIR_STORAGE_FILE_CLOOP] = {
+        /* #!/bin/sh
+           #V2.0 Format
+           modprobe cloop file=$0 && mount -r -t iso9660 /dev/cloop $1
+        */ /* Untested */
+        0, NULL,
+        LV_LITTLE_ENDIAN, -1, 0, {0},
+        -1, 0, 0, NULL, NULL, NULL
+    },
+    [VIR_STORAGE_FILE_DMG] = {
+        /* XXX QEMU says there's no magic for dmg,
+         * /usr/share/misc/magic lists double magic (both offsets
+         * would have to match) but then disables that check. */
+        0, NULL,
+        0, -1, 0, {0},
+        -1, 0, 0, NULL, NULL, NULL
+    },
+    [VIR_STORAGE_FILE_ISO] = {
+        32769, "CD001",
+        LV_LITTLE_ENDIAN, -2, 0, {0},
+        -1, 0, 0, NULL, NULL, NULL
+    },
+    [VIR_STORAGE_FILE_VPC] = {
+        0, "conectix",
+        LV_BIG_ENDIAN, 12, 4, {0x10000},
+        8 + 4 + 4 + 8 + 4 + 4 + 2 + 2 + 4, 8, 1, NULL, NULL, NULL
+    },
+    /* TODO: add getBackingStore function */
+    [VIR_STORAGE_FILE_VDI] = {
+        64, "\x7f\x10\xda\xbe",
+        LV_LITTLE_ENDIAN, 68, 4, {0x00010001},
+        64 + 5 * 4 + 256 + 7 * 4, 8, 1, NULL, NULL, NULL},
+
+    /* Not direct file formats, but used for various drivers */
+    [VIR_STORAGE_FILE_FAT] = { 0, NULL, LV_LITTLE_ENDIAN,
+                               -1, 0, {0}, 0, 0, 0, NULL, NULL, NULL },
+    [VIR_STORAGE_FILE_VHD] = { 0, NULL, LV_LITTLE_ENDIAN,
+                               -1, 0, {0}, 0, 0, 0, NULL, NULL, NULL },
+    [VIR_STORAGE_FILE_PLOOP] = { 0, "WithouFreSpacExt", LV_LITTLE_ENDIAN,
+                                 -2, 0, {0}, PLOOP_IMAGE_SIZE_OFFSET, 0,
+                                 PLOOP_SIZE_MULTIPLIER, NULL, NULL, NULL },
+
+    /* All formats with a backing store probe below here */
+    [VIR_STORAGE_FILE_COW] = {
+        0, "OOOM",
+        LV_BIG_ENDIAN, 4, 4, {2},
+        4+4+1024+4, 8, 1, NULL, cowGetBackingStore, NULL
+    },
+    [VIR_STORAGE_FILE_QCOW] = {
+        0, "QFI",
+        LV_BIG_ENDIAN, 4, 4, {1},
+        QCOWX_HDR_IMAGE_SIZE, 8, 1,
+        qcow1EncryptionInfo,
+        qcowXGetBackingStore, NULL
+    },
+    [VIR_STORAGE_FILE_QCOW2] = {
+        0, "QFI",
+        LV_BIG_ENDIAN, 4, 4, {2, 3},
+        QCOWX_HDR_IMAGE_SIZE, 8, 1,
+        qcow2EncryptionInfo,
+        qcowXGetBackingStore,
+        qcow2GetFeatures
+    },
+    [VIR_STORAGE_FILE_QED] = {
+        /* https://wiki.qemu.org/Features/QED */
+        0, "QED",
+        LV_LITTLE_ENDIAN, -2, 0, {0},
+        QED_HDR_IMAGE_SIZE, 8, 1, NULL, qedGetBackingStore, NULL
+    },
+    [VIR_STORAGE_FILE_VMDK] = {
+        0, "KDMV",
+        LV_LITTLE_ENDIAN, 4, 4, {1, 2, 3},
+        4+4+4, 8, 512, NULL, vmdk4GetBackingStore, NULL
+    },
+};
+G_STATIC_ASSERT(G_N_ELEMENTS(fileTypeInfo) == VIR_STORAGE_FILE_LAST);
+
+
+/* qcow2 compatible features in the order they appear on-disk */
+enum qcow2CompatibleFeature {
+    QCOW2_COMPATIBLE_FEATURE_LAZY_REFCOUNTS = 0,
+
+    QCOW2_COMPATIBLE_FEATURE_LAST
+};
+
+/* conversion to virStorageFileFeature */
+static const int qcow2CompatibleFeatureArray[] = {
+    VIR_STORAGE_FILE_FEATURE_LAZY_REFCOUNTS,
+};
+G_STATIC_ASSERT(G_N_ELEMENTS(qcow2CompatibleFeatureArray) ==
+       QCOW2_COMPATIBLE_FEATURE_LAST);
+
+static int
+cowGetBackingStore(char **res,
+                   int *format,
+                   const char *buf,
+                   size_t buf_size)
+{
+#define COW_FILENAME_MAXLEN 1024
+    *res = NULL;
+    *format = VIR_STORAGE_FILE_AUTO;
+
+    if (buf_size < 4+4+ COW_FILENAME_MAXLEN)
+        return BACKING_STORE_INVALID;
+    if (buf[4+4] == '\0') { /* cow_header_v2.backing_file[0] */
+        *format = VIR_STORAGE_FILE_NONE;
+        return BACKING_STORE_OK;
+    }
+
+    *res = g_strndup((const char *)buf + 4 + 4, COW_FILENAME_MAXLEN);
+    return BACKING_STORE_OK;
+}
+
+
+static int
+qcow2GetExtensions(const char *buf,
+                   size_t buf_size,
+                   int *backingFormat)
+{
+    size_t offset;
+    size_t extension_start;
+    size_t extension_end;
+    int version = virReadBufInt32BE(buf + QCOWX_HDR_VERSION);
+
+    if (version < 2) {
+        /* QCow1 doesn't have the extensions capability
+         * used to store backing format */
+        return 0;
+    }
+
+    if (version == 2)
+        extension_start = QCOW2_HDR_TOTAL_SIZE;
+    else
+        extension_start = virReadBufInt32BE(buf + QCOW2v3_HDR_SIZE);
+
+    /*
+     * Traditionally QCow2 files had a layout of
+     *
+     * [header]
+     * [backingStoreName]
+     *
+     * Although the backingStoreName typically followed
+     * the header immediately, this was not required by
+     * the format. By specifying a higher byte offset for
+     * the backing file offset in the header, it was
+     * possible to leave space between the header and
+     * start of backingStore.
+     *
+     * This hack is now used to store extensions to the
+     * qcow2 format:
+     *
+     * [header]
+     * [extensions]
+     * [backingStoreName]
+     *
+     * Thus the file region to search for extensions is
+     * between the end of the header (QCOW2_HDR_TOTAL_SIZE)
+     * and the start of the backingStoreName (offset)
+     *
+     * for qcow2 v3 images, the length of the header
+     * is stored at QCOW2v3_HDR_SIZE
+     */
+    extension_end = virReadBufInt64BE(buf + QCOWX_HDR_BACKING_FILE_OFFSET);
+    if (extension_end > buf_size)
+        return -1;
+
+    /*
+     * The extensions take format of
+     *
+     * int32: magic
+     * int32: length
+     * byte[length]: payload
+     *
+     * Unknown extensions can be ignored by skipping
+     * over "length" bytes in the data stream.
+     */
+    offset = extension_start;
+    while (offset < (buf_size-8) &&
+           offset < (extension_end-8)) {
+        unsigned int magic = virReadBufInt32BE(buf + offset);
+        unsigned int len = virReadBufInt32BE(buf + offset + 4);
+
+        offset += 8;
+
+        if ((offset + len) < offset)
+            break;
+
+        if ((offset + len) > buf_size)
+            break;
+
+        switch (magic) {
+        case QCOW2_HDR_EXTENSION_BACKING_FORMAT: {
+            g_autofree char *tmp = NULL;
+            if (!backingFormat)
+                break;
+
+            tmp = g_new0(char, len + 1);
+            memcpy(tmp, buf + offset, len);
+            tmp[len] = '\0';
+
+            *backingFormat = virStorageFileFormatTypeFromString(tmp);
+            if (*backingFormat <= VIR_STORAGE_FILE_NONE)
+                return -1;
+            break;
+        }
+
+        case QCOW2_HDR_EXTENSION_END:
+            return 0;
+        }
+
+        offset += len;
+    }
+
+    return 0;
+}
+
+
+static int
+qcowXGetBackingStore(char **res,
+                     int *format,
+                     const char *buf,
+                     size_t buf_size)
+{
+    unsigned long long offset;
+    unsigned int size;
+
+    *res = NULL;
+    *format = VIR_STORAGE_FILE_AUTO;
+
+    if (buf_size < QCOWX_HDR_BACKING_FILE_OFFSET+8+4)
+        return BACKING_STORE_INVALID;
+
+    offset = virReadBufInt64BE(buf + QCOWX_HDR_BACKING_FILE_OFFSET);
+    if (offset > buf_size)
+        return BACKING_STORE_INVALID;
+
+    if (offset == 0) {
+        *format = VIR_STORAGE_FILE_NONE;
+        return BACKING_STORE_OK;
+    }
+
+    size = virReadBufInt32BE(buf + QCOWX_HDR_BACKING_FILE_SIZE);
+    if (size == 0) {
+        *format = VIR_STORAGE_FILE_NONE;
+        return BACKING_STORE_OK;
+    }
+    if (size > 1023)
+        return BACKING_STORE_INVALID;
+    if (offset + size > buf_size || offset + size < offset)
+        return BACKING_STORE_INVALID;
+    *res = g_new0(char, size + 1);
+    memcpy(*res, buf + offset, size);
+    (*res)[size] = '\0';
+
+    if (qcow2GetExtensions(buf, buf_size, format) < 0)
+        return BACKING_STORE_INVALID;
+
+    return BACKING_STORE_OK;
+}
+
+
+static int
+vmdk4GetBackingStore(char **res,
+                     int *format,
+                     const char *buf,
+                     size_t buf_size)
+{
+    static const char prefix[] = "parentFileNameHint=\"";
+    char *start, *end;
+    size_t len;
+    g_autofree char *desc = NULL;
+
+    desc = g_new0(char, VIR_STORAGE_MAX_HEADER);
+
+    *res = NULL;
+    /*
+     * Technically this should have been VMDK, since
+     * VMDK spec / VMware impl only support VMDK backed
+     * by VMDK. QEMU isn't following this though and
+     * does probing on VMDK backing files, hence we set
+     * AUTO
+     */
+    *format = VIR_STORAGE_FILE_AUTO;
+
+    if (buf_size <= 0x200)
+        return BACKING_STORE_INVALID;
+
+    len = buf_size - 0x200;
+    if (len > VIR_STORAGE_MAX_HEADER)
+        len = VIR_STORAGE_MAX_HEADER;
+    memcpy(desc, buf + 0x200, len);
+    desc[len] = '\0';
+    start = strstr(desc, prefix);
+    if (start == NULL) {
+        *format = VIR_STORAGE_FILE_NONE;
+        return BACKING_STORE_OK;
+    }
+    start += strlen(prefix);
+    end = strchr(start, '"');
+    if (end == NULL)
+        return BACKING_STORE_INVALID;
+
+    if (end == start) {
+        *format = VIR_STORAGE_FILE_NONE;
+        return BACKING_STORE_OK;
+    }
+    *end = '\0';
+    *res = g_strdup(start);
+
+    return BACKING_STORE_OK;
+}
+
+static int
+qedGetBackingStore(char **res,
+                   int *format,
+                   const char *buf,
+                   size_t buf_size)
+{
+    unsigned long long flags;
+    unsigned long offset, size;
+
+    *res = NULL;
+    /* Check if this image has a backing file */
+    if (buf_size < QED_HDR_FEATURES_OFFSET+8)
+        return BACKING_STORE_INVALID;
+    flags = virReadBufInt64LE(buf + QED_HDR_FEATURES_OFFSET);
+    if (!(flags & QED_F_BACKING_FILE)) {
+        *format = VIR_STORAGE_FILE_NONE;
+        return BACKING_STORE_OK;
+    }
+
+    /* Parse the backing file */
+    if (buf_size < QED_HDR_BACKING_FILE_OFFSET+8)
+        return BACKING_STORE_INVALID;
+    offset = virReadBufInt32LE(buf + QED_HDR_BACKING_FILE_OFFSET);
+    if (offset > buf_size)
+        return BACKING_STORE_INVALID;
+    size = virReadBufInt32LE(buf + QED_HDR_BACKING_FILE_SIZE);
+    if (size == 0)
+        return BACKING_STORE_OK;
+    if (offset + size > buf_size || offset + size < offset)
+        return BACKING_STORE_INVALID;
+    *res = g_new0(char, size + 1);
+    memcpy(*res, buf + offset, size);
+    (*res)[size] = '\0';
+
+    if (flags & QED_F_BACKING_FORMAT_NO_PROBE)
+        *format = VIR_STORAGE_FILE_RAW;
+    else
+        *format = VIR_STORAGE_FILE_AUTO_SAFE;
+
+    return BACKING_STORE_OK;
+}
+
+
+static bool
+virStorageFileMatchesMagic(int magicOffset,
+                           const char *magic,
+                           char *buf,
+                           size_t buflen)
+{
+    int mlen;
+
+    if (magic == NULL)
+        return false;
+
+    /* Validate magic data */
+    mlen = strlen(magic);
+    if (magicOffset + mlen > buflen)
+        return false;
+
+    if (memcmp(buf + magicOffset, magic, mlen) != 0)
+        return false;
+
+    return true;
+}
+
+
+static bool
+virStorageFileMatchesVersion(int versionOffset,
+                             int versionSize,
+                             const int *versionNumbers,
+                             int endian,
+                             char *buf,
+                             size_t buflen)
+{
+    int version;
+    size_t i;
+
+    /* Validate version number info */
+    if (versionOffset == -1)
+        return false;
+
+    /* -2 == non-versioned file format, so trivially match */
+    if (versionOffset == -2)
+        return true;
+
+    /* A positive versionOffset, requires using a valid versionSize */
+    if (versionSize != 2 && versionSize != 4)
+        return false;
+
+    if ((versionOffset + versionSize) > buflen)
+        return false;
+
+    if (endian == LV_LITTLE_ENDIAN) {
+        if (versionSize == 4)
+            version = virReadBufInt32LE(buf +
+                                        versionOffset);
+        else
+            version = virReadBufInt16LE(buf +
+                                        versionOffset);
+    } else {
+        if (versionSize == 4)
+            version = virReadBufInt32BE(buf +
+                                        versionOffset);
+        else
+            version = virReadBufInt16BE(buf +
+                                        versionOffset);
+    }
+
+    for (i = 0;
+         i < FILE_TYPE_VERSIONS_LAST && versionNumbers[i];
+         i++) {
+        VIR_DEBUG("Compare detected version %d vs one of the expected versions %d",
+                  version, versionNumbers[i]);
+        if (version == versionNumbers[i])
+            return true;
+    }
+
+    return false;
+}
+
+
+static int
+virStorageFileProbeFormatFromBuf(const char *path,
+                                 char *buf,
+                                 size_t buflen)
+{
+    int format = VIR_STORAGE_FILE_RAW;
+    size_t i;
+    int possibleFormat = VIR_STORAGE_FILE_RAW;
+    VIR_DEBUG("path=%s, buf=%p, buflen=%zu", path, buf, buflen);
+
+    /* First check file magic */
+    for (i = 0; i < VIR_STORAGE_FILE_LAST; i++) {
+        if (virStorageFileMatchesMagic(fileTypeInfo[i].magicOffset,
+                                       fileTypeInfo[i].magic,
+                                       buf, buflen)) {
+            if (!virStorageFileMatchesVersion(fileTypeInfo[i].versionOffset,
+                                              fileTypeInfo[i].versionSize,
+                                              fileTypeInfo[i].versionNumbers,
+                                              fileTypeInfo[i].endian,
+                                              buf, buflen)) {
+                possibleFormat = i;
+                continue;
+            }
+            format = i;
+            goto cleanup;
+        }
+    }
+
+    if (possibleFormat != VIR_STORAGE_FILE_RAW)
+        VIR_WARN("File %s matches %s magic, but version is wrong. "
+                 "Please report new version to libvir-list@redhat.com",
+                 path, virStorageFileFormatTypeToString(possibleFormat));
+
+ cleanup:
+    VIR_DEBUG("format=%d", format);
+    return format;
+}
+
+
+static int
+qcow2GetFeatures(virBitmapPtr *features,
+                 int format,
+                 char *buf,
+                 ssize_t len)
+{
+    int version = -1;
+    virBitmapPtr feat = NULL;
+    uint64_t bits;
+    size_t i;
+
+    version = virReadBufInt32BE(buf + fileTypeInfo[format].versionOffset);
+
+    if (version == 2)
+        return 0;
+
+    if (len < QCOW2v3_HDR_SIZE)
+        return -1;
+
+    feat = virBitmapNew(VIR_STORAGE_FILE_FEATURE_LAST);
+
+    /* todo: check for incompatible or autoclear features? */
+    bits = virReadBufInt64BE(buf + QCOW2v3_HDR_FEATURES_COMPATIBLE);
+    for (i = 0; i < QCOW2_COMPATIBLE_FEATURE_LAST; i++) {
+        if (bits & ((uint64_t) 1 << i))
+            ignore_value(virBitmapSetBit(feat, qcow2CompatibleFeatureArray[i]));
+    }
+
+    *features = feat;
+    return 0;
+}
+
+
+static bool
+virStorageFileHasEncryptionFormat(const struct FileEncryptionInfo *info,
+                                  char *buf,
+                                  size_t len)
+{
+    if (!info->magic && info->modeOffset == -1)
+        return false; /* Shouldn't happen - expect at least one */
+
+    if (info->magic) {
+        if (!virStorageFileMatchesMagic(info->magicOffset,
+                                        info->magic,
+                                        buf, len))
+            return false;
+
+        if (info->versionOffset != -1 &&
+            !virStorageFileMatchesVersion(info->versionOffset,
+                                          info->versionSize,
+                                          info->versionNumbers,
+                                          info->endian,
+                                          buf, len))
+            return false;
+
+        return true;
+    } else if (info->modeOffset != -1) {
+        int crypt_format;
+
+        if (info->modeOffset >= len)
+            return false;
+
+        crypt_format = virReadBufInt32BE(buf + info->modeOffset);
+        if (crypt_format != info->modeValue)
+            return false;
+
+        return true;
+    } else {
+        return false;
+    }
+}
+
+
+static int
+virStorageFileGetEncryptionPayloadOffset(const struct FileEncryptionInfo *info,
+                                         char *buf)
+{
+    int payload_offset = -1;
+
+    if (info->payloadOffset != -1) {
+        if (info->endian == LV_LITTLE_ENDIAN)
+            payload_offset = virReadBufInt32LE(buf + info->payloadOffset);
+        else
+            payload_offset = virReadBufInt32BE(buf + info->payloadOffset);
+    }
+
+    return payload_offset;
+}
+
+
+/* Given a header in BUF with length LEN, as parsed from the storage file
+ * assuming it has the given FORMAT, populate information into META
+ * with information about the file and its backing store. Return format
+ * of the backing store as BACKING_FORMAT. PATH and FORMAT have to be
+ * pre-populated in META.
+ *
+ * Note that this function may be called repeatedly on @meta, so it must
+ * clean up any existing allocated memory which would be overwritten.
+ */
+int
+virStorageFileProbeGetMetadata(virStorageSourcePtr meta,
+                               char *buf,
+                               size_t len)
+{
+    int format;
+    size_t i;
+
+    VIR_DEBUG("path=%s, buf=%p, len=%zu, meta->format=%d",
+              meta->path, buf, len, meta->format);
+
+    if (meta->format == VIR_STORAGE_FILE_AUTO)
+        meta->format = virStorageFileProbeFormatFromBuf(meta->path, buf, len);
+
+    if (meta->format <= VIR_STORAGE_FILE_NONE ||
+        meta->format >= VIR_STORAGE_FILE_LAST) {
+        virReportSystemError(EINVAL, _("unknown storage file meta->format %d"),
+                             meta->format);
+        return -1;
+    }
+
+    if (fileTypeInfo[meta->format].cryptInfo != NULL) {
+        for (i = 0; fileTypeInfo[meta->format].cryptInfo[i].format != 0; i++) {
+            if (virStorageFileHasEncryptionFormat(&fileTypeInfo[meta->format].cryptInfo[i],
+                                                  buf, len)) {
+                int expt_fmt = fileTypeInfo[meta->format].cryptInfo[i].format;
+                if (!meta->encryption) {
+                    meta->encryption = g_new0(virStorageEncryption, 1);
+                    meta->encryption->format = expt_fmt;
+                } else {
+                    if (meta->encryption->format != expt_fmt) {
+                        virReportError(VIR_ERR_XML_ERROR,
+                                       _("encryption format %d doesn't match "
+                                         "expected format %d"),
+                                       meta->encryption->format, expt_fmt);
+                        return -1;
+                    }
+                }
+                meta->encryption->payload_offset =
+                    virStorageFileGetEncryptionPayloadOffset(&fileTypeInfo[meta->format].cryptInfo[i], buf);
+            }
+        }
+    }
+
+    /* XXX we should consider moving virStorageBackendUpdateVolInfo
+     * code into this method, for non-magic files
+     */
+    if (!fileTypeInfo[meta->format].magic)
+        return 0;
+
+    /* Optionally extract capacity from file */
+    if (fileTypeInfo[meta->format].sizeOffset != -1) {
+        if ((fileTypeInfo[meta->format].sizeOffset + 8) > len)
+            return 0;
+
+        if (fileTypeInfo[meta->format].endian == LV_LITTLE_ENDIAN)
+            meta->capacity = virReadBufInt64LE(buf +
+                                               fileTypeInfo[meta->format].sizeOffset);
+        else
+            meta->capacity = virReadBufInt64BE(buf +
+                                               fileTypeInfo[meta->format].sizeOffset);
+        /* Avoid unlikely, but theoretically possible overflow */
+        if (meta->capacity > (ULLONG_MAX /
+                              fileTypeInfo[meta->format].sizeMultiplier))
+            return 0;
+        meta->capacity *= fileTypeInfo[meta->format].sizeMultiplier;
+    }
+
+    VIR_FREE(meta->backingStoreRaw);
+    if (fileTypeInfo[meta->format].getBackingStore != NULL) {
+        int store = fileTypeInfo[meta->format].getBackingStore(&meta->backingStoreRaw,
+                                                               &format,
+                                                               buf, len);
+        meta->backingStoreRawFormat = format;
+
+        if (store == BACKING_STORE_INVALID)
+            return 0;
+
+        if (store == BACKING_STORE_ERROR)
+            return -1;
+    }
+
+    virBitmapFree(meta->features);
+    meta->features = NULL;
+    if (fileTypeInfo[meta->format].getFeatures != NULL &&
+        fileTypeInfo[meta->format].getFeatures(&meta->features, meta->format, buf, len) < 0)
+        return -1;
+
+    VIR_FREE(meta->compat);
+    if (meta->format == VIR_STORAGE_FILE_QCOW2 && meta->features)
+        meta->compat = g_strdup("1.1");
+
+    return 0;
+}
+
+
+/**
+ * virStorageFileProbeFormat:
+ *
+ * Probe for the format of 'path', returning the detected
+ * disk format.
+ *
+ * Callers are advised never to trust the returned 'format'
+ * unless it is listed as VIR_STORAGE_FILE_RAW, since a
+ * malicious guest can turn a raw file into any other non-raw
+ * format at will.
+ *
+ * Best option: Don't use this function
+ */
+int
+virStorageFileProbeFormat(const char *path, uid_t uid, gid_t gid)
+{
+    struct stat sb;
+    ssize_t len = VIR_STORAGE_MAX_HEADER;
+    VIR_AUTOCLOSE fd = -1;
+    g_autofree char *header = NULL;
+
+    if ((fd = virFileOpenAs(path, O_RDONLY, 0, uid, gid, 0)) < 0) {
+        virReportSystemError(-fd, _("Failed to open file '%s'"), path);
+        return -1;
+    }
+
+    if (fstat(fd, &sb) < 0) {
+        virReportSystemError(errno, _("cannot stat file '%s'"), path);
+        return -1;
+    }
+
+    /* No header to probe for directories */
+    if (S_ISDIR(sb.st_mode))
+        return VIR_STORAGE_FILE_DIR;
+
+    if (lseek(fd, 0, SEEK_SET) == (off_t)-1) {
+        virReportSystemError(errno, _("cannot set to start of '%s'"), path);
+        return -1;
+    }
+
+    if ((len = virFileReadHeaderFD(fd, len, &header)) < 0) {
+        virReportSystemError(errno, _("cannot read header '%s'"), path);
+        return -1;
+    }
+
+    return virStorageFileProbeFormatFromBuf(path, header, len);
+}
diff --git a/src/util/virstoragefileprobe.h b/src/util/virstoragefileprobe.h
new file mode 100644
index 0000000000..2b94a4ae51
--- /dev/null
+++ b/src/util/virstoragefileprobe.h
@@ -0,0 +1,44 @@
+/*
+ * virstoragefileprobe.h: file utility functions for FS storage backend
+ *
+ * Copyright (C) 2007-2009, 2012-2016 Red Hat, Inc.
+ * Copyright (C) 2007-2008 Daniel P. Berrange
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library.  If not, see
+ * <http://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+
+#include <sys/stat.h>
+
+#include "virstoragefile.h"
+
+/* Minimum header size required to probe all known formats with
+ * virStorageFileProbeFormat, or obtain metadata from a known format.
+ * Rounded to multiple of 512 (ISO has a 5-byte magic at offset
+ * 32769).  Some formats can be probed with fewer bytes.  Although
+ * some formats theoretically permit metadata that can rely on offsets
+ * beyond this size, in practice that doesn't matter.  */
+#define VIR_STORAGE_MAX_HEADER 0x8200
+
+int
+virStorageFileProbeGetMetadata(virStorageSourcePtr meta,
+                               char *buf,
+                               size_t len);
+
+int
+virStorageFileProbeFormat(const char *path,
+                          uid_t uid,
+                          gid_t gid);
-- 
2.29.2

Re: [libvirt PATCH v2 06/13] util: extract storage file probe code into virtstoragefileprobe.c
Posted by Peter Krempa 5 years ago
On Thu, Jan 21, 2021 at 20:34:20 +0100, Pavel Hrdina wrote:
> This code is not directly relevant to virStorageSource so move it to
> separate file.
> 
> Signed-off-by: Pavel Hrdina <phrdina@redhat.com>
> ---
>  po/POTFILES.in                        |   1 +
>  src/libvirt_private.syms              |   6 +-
>  src/qemu/qemu_driver.c                |   1 +
>  src/storage/storage_backend_gluster.c |   1 +
>  src/storage/storage_util.c            |   1 +
>  src/util/meson.build                  |   1 +
>  src/util/virstoragefile.c             | 939 +------------------------
>  src/util/virstoragefile.h             |  11 -
>  src/util/virstoragefileprobe.c        | 967 ++++++++++++++++++++++++++
>  src/util/virstoragefileprobe.h        |  44 ++
>  10 files changed, 1026 insertions(+), 946 deletions(-)
>  create mode 100644 src/util/virstoragefileprobe.c
>  create mode 100644 src/util/virstoragefileprobe.h

[...]

> diff --git a/src/util/virstoragefile.c b/src/util/virstoragefile.c
> index 13a86f34e5..98a3222d09 100644
> --- a/src/util/virstoragefile.c
> +++ b/src/util/virstoragefile.c

[...]

> @@ -792,289 +146,6 @@ virStorageIsRelative(const char *backing)
>  }
>  
>  
> -static int
> -virStorageFileProbeFormatFromBuf(const char *path,
> -                                 char *buf,
> -                                 size_t buflen)

I'd prefer if this is the function exported from the new module ...

> -{
> -    int format = VIR_STORAGE_FILE_RAW;
> -    size_t i;
> -    int possibleFormat = VIR_STORAGE_FILE_RAW;
> -    VIR_DEBUG("path=%s, buf=%p, buflen=%zu", path, buf, buflen);
> -
> -    /* First check file magic */
> -    for (i = 0; i < VIR_STORAGE_FILE_LAST; i++) {
> -        if (virStorageFileMatchesMagic(fileTypeInfo[i].magicOffset,
> -                                       fileTypeInfo[i].magic,
> -                                       buf, buflen)) {
> -            if (!virStorageFileMatchesVersion(fileTypeInfo[i].versionOffset,
> -                                              fileTypeInfo[i].versionSize,
> -                                              fileTypeInfo[i].versionNumbers,
> -                                              fileTypeInfo[i].endian,
> -                                              buf, buflen)) {
> -                possibleFormat = i;
> -                continue;
> -            }
> -            format = i;
> -            goto cleanup;
> -        }
> -    }
> -
> -    if (possibleFormat != VIR_STORAGE_FILE_RAW)
> -        VIR_WARN("File %s matches %s magic, but version is wrong. "
> -                 "Please report new version to libvir-list@redhat.com",
> -                 path, virStorageFileFormatTypeToString(possibleFormat));
> -
> - cleanup:
> -    VIR_DEBUG("format=%d", format);
> -    return format;
> -}

[...]

> -/**
> - * virStorageFileProbeFormat:
> - *
> - * Probe for the format of 'path', returning the detected
> - * disk format.
> - *
> - * Callers are advised never to trust the returned 'format'
> - * unless it is listed as VIR_STORAGE_FILE_RAW, since a
> - * malicious guest can turn a raw file into any other non-raw
> - * format at will.
> - *
> - * Best option: Don't use this function
> - */
> -int
> -virStorageFileProbeFormat(const char *path, uid_t uid, gid_t gid)
> -{

... and not this.

> -    struct stat sb;
> -    ssize_t len = VIR_STORAGE_MAX_HEADER;
> -    VIR_AUTOCLOSE fd = -1;
> -    g_autofree char *header = NULL;
> -
> -    if ((fd = virFileOpenAs(path, O_RDONLY, 0, uid, gid, 0)) < 0) {

Specifically the new module should not ever touch any real storage and
should just be a self-contained prober of metadata froma buffer.

> -        virReportSystemError(-fd, _("Failed to open file '%s'"), path);
> -        return -1;
> -    }
> -
> -    if (fstat(fd, &sb) < 0) {
> -        virReportSystemError(errno, _("cannot stat file '%s'"), path);
> -        return -1;
> -    }
> -
> -    /* No header to probe for directories */
> -    if (S_ISDIR(sb.st_mode))
> -        return VIR_STORAGE_FILE_DIR;
> -
> -    if (lseek(fd, 0, SEEK_SET) == (off_t)-1) {
> -        virReportSystemError(errno, _("cannot set to start of '%s'"), path);
> -        return -1;
> -    }
> -
> -    if ((len = virFileReadHeaderFD(fd, len, &header)) < 0) {
> -        virReportSystemError(errno, _("cannot read header '%s'"), path);
> -        return -1;
> -    }
> -
> -    return virStorageFileProbeFormatFromBuf(path, header, len);
> -}

[...]

> diff --git a/src/util/virstoragefileprobe.h b/src/util/virstoragefileprobe.h
> new file mode 100644
> index 0000000000..2b94a4ae51
> --- /dev/null
> +++ b/src/util/virstoragefileprobe.h
> @@ -0,0 +1,44 @@
> +/*
> + * virstoragefileprobe.h: file utility functions for FS storage backend
> + *
> + * Copyright (C) 2007-2009, 2012-2016 Red Hat, Inc.
> + * Copyright (C) 2007-2008 Daniel P. Berrange
> + *
> + * This library is free software; you can redistribute it and/or
> + * modify it under the terms of the GNU Lesser General Public
> + * License as published by the Free Software Foundation; either
> + * version 2.1 of the License, or (at your option) any later version.
> + *
> + * This library is distributed in the hope that it will be useful,
> + * but WITHOUT ANY WARRANTY; without even the implied warranty of
> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
> + * Lesser General Public License for more details.
> + *
> + * You should have received a copy of the GNU Lesser General Public
> + * License along with this library.  If not, see
> + * <http://www.gnu.org/licenses/>.
> + */
> +
> +#pragma once
> +
> +#include <sys/stat.h>

#include <sys/types.h> for uid_t/gid_t instead, there's no use of the
stat struct in the header, but that shouldn't be necessary at all if the
format is probed fromt he buffer directly.

> +
> +#include "virstoragefile.h"
> +
> +/* Minimum header size required to probe all known formats with
> + * virStorageFileProbeFormat, or obtain metadata from a known format.
> + * Rounded to multiple of 512 (ISO has a 5-byte magic at offset
> + * 32769).  Some formats can be probed with fewer bytes.  Although
> + * some formats theoretically permit metadata that can rely on offsets
> + * beyond this size, in practice that doesn't matter.  */
> +#define VIR_STORAGE_MAX_HEADER 0x8200
> +
> +int
> +virStorageFileProbeGetMetadata(virStorageSourcePtr meta,
> +                               char *buf,
> +                               size_t len);
> +
> +int
> +virStorageFileProbeFormat(const char *path,
> +                          uid_t uid,
> +                          gid_t gid);

Reviewed-by: Peter Krempa <pkrempa@redhat.com>

If you decide that it's too much hassle to shuffle this around, you can
return the code opening the file back to the appropriate file later.

(Or I'll do it, if you don't, but virstoragefileprobe.c will not
'open()' anything in the end)

Re: [libvirt PATCH v2 06/13] util: extract storage file probe code into virtstoragefileprobe.c
Posted by Pavel Hrdina 5 years ago
On Fri, Jan 22, 2021 at 09:23:00AM +0100, Peter Krempa wrote:
> On Thu, Jan 21, 2021 at 20:34:20 +0100, Pavel Hrdina wrote:
> > This code is not directly relevant to virStorageSource so move it to
> > separate file.
> > 
> > Signed-off-by: Pavel Hrdina <phrdina@redhat.com>
> > ---
> >  po/POTFILES.in                        |   1 +
> >  src/libvirt_private.syms              |   6 +-
> >  src/qemu/qemu_driver.c                |   1 +
> >  src/storage/storage_backend_gluster.c |   1 +
> >  src/storage/storage_util.c            |   1 +
> >  src/util/meson.build                  |   1 +
> >  src/util/virstoragefile.c             | 939 +------------------------
> >  src/util/virstoragefile.h             |  11 -
> >  src/util/virstoragefileprobe.c        | 967 ++++++++++++++++++++++++++
> >  src/util/virstoragefileprobe.h        |  44 ++
> >  10 files changed, 1026 insertions(+), 946 deletions(-)
> >  create mode 100644 src/util/virstoragefileprobe.c
> >  create mode 100644 src/util/virstoragefileprobe.h
> 
> [...]
> 
> > diff --git a/src/util/virstoragefile.c b/src/util/virstoragefile.c
> > index 13a86f34e5..98a3222d09 100644
> > --- a/src/util/virstoragefile.c
> > +++ b/src/util/virstoragefile.c
> 
> [...]
> 
> > @@ -792,289 +146,6 @@ virStorageIsRelative(const char *backing)
> >  }
> >  
> >  
> > -static int
> > -virStorageFileProbeFormatFromBuf(const char *path,
> > -                                 char *buf,
> > -                                 size_t buflen)
> 
> I'd prefer if this is the function exported from the new module ...
> 
> > -{
> > -    int format = VIR_STORAGE_FILE_RAW;
> > -    size_t i;
> > -    int possibleFormat = VIR_STORAGE_FILE_RAW;
> > -    VIR_DEBUG("path=%s, buf=%p, buflen=%zu", path, buf, buflen);
> > -
> > -    /* First check file magic */
> > -    for (i = 0; i < VIR_STORAGE_FILE_LAST; i++) {
> > -        if (virStorageFileMatchesMagic(fileTypeInfo[i].magicOffset,
> > -                                       fileTypeInfo[i].magic,
> > -                                       buf, buflen)) {
> > -            if (!virStorageFileMatchesVersion(fileTypeInfo[i].versionOffset,
> > -                                              fileTypeInfo[i].versionSize,
> > -                                              fileTypeInfo[i].versionNumbers,
> > -                                              fileTypeInfo[i].endian,
> > -                                              buf, buflen)) {
> > -                possibleFormat = i;
> > -                continue;
> > -            }
> > -            format = i;
> > -            goto cleanup;
> > -        }
> > -    }
> > -
> > -    if (possibleFormat != VIR_STORAGE_FILE_RAW)
> > -        VIR_WARN("File %s matches %s magic, but version is wrong. "
> > -                 "Please report new version to libvir-list@redhat.com",
> > -                 path, virStorageFileFormatTypeToString(possibleFormat));
> > -
> > - cleanup:
> > -    VIR_DEBUG("format=%d", format);
> > -    return format;
> > -}
> 
> [...]
> 
> > -/**
> > - * virStorageFileProbeFormat:
> > - *
> > - * Probe for the format of 'path', returning the detected
> > - * disk format.
> > - *
> > - * Callers are advised never to trust the returned 'format'
> > - * unless it is listed as VIR_STORAGE_FILE_RAW, since a
> > - * malicious guest can turn a raw file into any other non-raw
> > - * format at will.
> > - *
> > - * Best option: Don't use this function
> > - */
> > -int
> > -virStorageFileProbeFormat(const char *path, uid_t uid, gid_t gid)
> > -{
> 
> ... and not this.
> 
> > -    struct stat sb;
> > -    ssize_t len = VIR_STORAGE_MAX_HEADER;
> > -    VIR_AUTOCLOSE fd = -1;
> > -    g_autofree char *header = NULL;
> > -
> > -    if ((fd = virFileOpenAs(path, O_RDONLY, 0, uid, gid, 0)) < 0) {
> 
> Specifically the new module should not ever touch any real storage and
> should just be a self-contained prober of metadata froma buffer.
> 
> > -        virReportSystemError(-fd, _("Failed to open file '%s'"), path);
> > -        return -1;
> > -    }
> > -
> > -    if (fstat(fd, &sb) < 0) {
> > -        virReportSystemError(errno, _("cannot stat file '%s'"), path);
> > -        return -1;
> > -    }
> > -
> > -    /* No header to probe for directories */
> > -    if (S_ISDIR(sb.st_mode))
> > -        return VIR_STORAGE_FILE_DIR;
> > -
> > -    if (lseek(fd, 0, SEEK_SET) == (off_t)-1) {
> > -        virReportSystemError(errno, _("cannot set to start of '%s'"), path);
> > -        return -1;
> > -    }
> > -
> > -    if ((len = virFileReadHeaderFD(fd, len, &header)) < 0) {
> > -        virReportSystemError(errno, _("cannot read header '%s'"), path);
> > -        return -1;
> > -    }
> > -
> > -    return virStorageFileProbeFormatFromBuf(path, header, len);
> > -}
> 
> [...]
> 
> > diff --git a/src/util/virstoragefileprobe.h b/src/util/virstoragefileprobe.h
> > new file mode 100644
> > index 0000000000..2b94a4ae51
> > --- /dev/null
> > +++ b/src/util/virstoragefileprobe.h
> > @@ -0,0 +1,44 @@
> > +/*
> > + * virstoragefileprobe.h: file utility functions for FS storage backend
> > + *
> > + * Copyright (C) 2007-2009, 2012-2016 Red Hat, Inc.
> > + * Copyright (C) 2007-2008 Daniel P. Berrange
> > + *
> > + * This library is free software; you can redistribute it and/or
> > + * modify it under the terms of the GNU Lesser General Public
> > + * License as published by the Free Software Foundation; either
> > + * version 2.1 of the License, or (at your option) any later version.
> > + *
> > + * This library is distributed in the hope that it will be useful,
> > + * but WITHOUT ANY WARRANTY; without even the implied warranty of
> > + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
> > + * Lesser General Public License for more details.
> > + *
> > + * You should have received a copy of the GNU Lesser General Public
> > + * License along with this library.  If not, see
> > + * <http://www.gnu.org/licenses/>.
> > + */
> > +
> > +#pragma once
> > +
> > +#include <sys/stat.h>
> 
> #include <sys/types.h> for uid_t/gid_t instead, there's no use of the
> stat struct in the header, but that shouldn't be necessary at all if the
> format is probed fromt he buffer directly.
> 
> > +
> > +#include "virstoragefile.h"
> > +
> > +/* Minimum header size required to probe all known formats with
> > + * virStorageFileProbeFormat, or obtain metadata from a known format.
> > + * Rounded to multiple of 512 (ISO has a 5-byte magic at offset
> > + * 32769).  Some formats can be probed with fewer bytes.  Although
> > + * some formats theoretically permit metadata that can rely on offsets
> > + * beyond this size, in practice that doesn't matter.  */
> > +#define VIR_STORAGE_MAX_HEADER 0x8200
> > +
> > +int
> > +virStorageFileProbeGetMetadata(virStorageSourcePtr meta,
> > +                               char *buf,
> > +                               size_t len);
> > +
> > +int
> > +virStorageFileProbeFormat(const char *path,
> > +                          uid_t uid,
> > +                          gid_t gid);
> 
> Reviewed-by: Peter Krempa <pkrempa@redhat.com>
> 
> If you decide that it's too much hassle to shuffle this around, you can
> return the code opening the file back to the appropriate file later.
> 
> (Or I'll do it, if you don't, but virstoragefileprobe.c will not
> 'open()' anything in the end)

Make sense, I'll look into it and post v3.

Thanks,

Pavel
Re: [libvirt PATCH v2 06/13] util: extract storage file probe code into virtstoragefileprobe.c
Posted by Pavel Hrdina 5 years ago
On Fri, Jan 22, 2021 at 10:09:27AM +0100, Pavel Hrdina wrote:
> On Fri, Jan 22, 2021 at 09:23:00AM +0100, Peter Krempa wrote:
> > On Thu, Jan 21, 2021 at 20:34:20 +0100, Pavel Hrdina wrote:
> > > This code is not directly relevant to virStorageSource so move it to
> > > separate file.
> > > 
> > > Signed-off-by: Pavel Hrdina <phrdina@redhat.com>
> > > ---
> > >  po/POTFILES.in                        |   1 +
> > >  src/libvirt_private.syms              |   6 +-
> > >  src/qemu/qemu_driver.c                |   1 +
> > >  src/storage/storage_backend_gluster.c |   1 +
> > >  src/storage/storage_util.c            |   1 +
> > >  src/util/meson.build                  |   1 +
> > >  src/util/virstoragefile.c             | 939 +------------------------
> > >  src/util/virstoragefile.h             |  11 -
> > >  src/util/virstoragefileprobe.c        | 967 ++++++++++++++++++++++++++
> > >  src/util/virstoragefileprobe.h        |  44 ++
> > >  10 files changed, 1026 insertions(+), 946 deletions(-)
> > >  create mode 100644 src/util/virstoragefileprobe.c
> > >  create mode 100644 src/util/virstoragefileprobe.h
> > 
> > [...]
> > 
> > > diff --git a/src/util/virstoragefile.c b/src/util/virstoragefile.c
> > > index 13a86f34e5..98a3222d09 100644
> > > --- a/src/util/virstoragefile.c
> > > +++ b/src/util/virstoragefile.c
> > 
> > [...]
> > 
> > > @@ -792,289 +146,6 @@ virStorageIsRelative(const char *backing)
> > >  }
> > >  
> > >  
> > > -static int
> > > -virStorageFileProbeFormatFromBuf(const char *path,
> > > -                                 char *buf,
> > > -                                 size_t buflen)
> > 
> > I'd prefer if this is the function exported from the new module ...
> > 
> > > -{
> > > -    int format = VIR_STORAGE_FILE_RAW;
> > > -    size_t i;
> > > -    int possibleFormat = VIR_STORAGE_FILE_RAW;
> > > -    VIR_DEBUG("path=%s, buf=%p, buflen=%zu", path, buf, buflen);
> > > -
> > > -    /* First check file magic */
> > > -    for (i = 0; i < VIR_STORAGE_FILE_LAST; i++) {
> > > -        if (virStorageFileMatchesMagic(fileTypeInfo[i].magicOffset,
> > > -                                       fileTypeInfo[i].magic,
> > > -                                       buf, buflen)) {
> > > -            if (!virStorageFileMatchesVersion(fileTypeInfo[i].versionOffset,
> > > -                                              fileTypeInfo[i].versionSize,
> > > -                                              fileTypeInfo[i].versionNumbers,
> > > -                                              fileTypeInfo[i].endian,
> > > -                                              buf, buflen)) {
> > > -                possibleFormat = i;
> > > -                continue;
> > > -            }
> > > -            format = i;
> > > -            goto cleanup;
> > > -        }
> > > -    }
> > > -
> > > -    if (possibleFormat != VIR_STORAGE_FILE_RAW)
> > > -        VIR_WARN("File %s matches %s magic, but version is wrong. "
> > > -                 "Please report new version to libvir-list@redhat.com",
> > > -                 path, virStorageFileFormatTypeToString(possibleFormat));
> > > -
> > > - cleanup:
> > > -    VIR_DEBUG("format=%d", format);
> > > -    return format;
> > > -}
> > 
> > [...]
> > 
> > > -/**
> > > - * virStorageFileProbeFormat:
> > > - *
> > > - * Probe for the format of 'path', returning the detected
> > > - * disk format.
> > > - *
> > > - * Callers are advised never to trust the returned 'format'
> > > - * unless it is listed as VIR_STORAGE_FILE_RAW, since a
> > > - * malicious guest can turn a raw file into any other non-raw
> > > - * format at will.
> > > - *
> > > - * Best option: Don't use this function
> > > - */
> > > -int
> > > -virStorageFileProbeFormat(const char *path, uid_t uid, gid_t gid)
> > > -{
> > 
> > ... and not this.
> > 
> > > -    struct stat sb;
> > > -    ssize_t len = VIR_STORAGE_MAX_HEADER;
> > > -    VIR_AUTOCLOSE fd = -1;
> > > -    g_autofree char *header = NULL;
> > > -
> > > -    if ((fd = virFileOpenAs(path, O_RDONLY, 0, uid, gid, 0)) < 0) {
> > 
> > Specifically the new module should not ever touch any real storage and
> > should just be a self-contained prober of metadata froma buffer.
> > 
> > > -        virReportSystemError(-fd, _("Failed to open file '%s'"), path);
> > > -        return -1;
> > > -    }
> > > -
> > > -    if (fstat(fd, &sb) < 0) {
> > > -        virReportSystemError(errno, _("cannot stat file '%s'"), path);
> > > -        return -1;
> > > -    }
> > > -
> > > -    /* No header to probe for directories */
> > > -    if (S_ISDIR(sb.st_mode))
> > > -        return VIR_STORAGE_FILE_DIR;
> > > -
> > > -    if (lseek(fd, 0, SEEK_SET) == (off_t)-1) {
> > > -        virReportSystemError(errno, _("cannot set to start of '%s'"), path);
> > > -        return -1;
> > > -    }
> > > -
> > > -    if ((len = virFileReadHeaderFD(fd, len, &header)) < 0) {
> > > -        virReportSystemError(errno, _("cannot read header '%s'"), path);
> > > -        return -1;
> > > -    }
> > > -
> > > -    return virStorageFileProbeFormatFromBuf(path, header, len);
> > > -}
> > 
> > [...]
> > 
> > > diff --git a/src/util/virstoragefileprobe.h b/src/util/virstoragefileprobe.h
> > > new file mode 100644
> > > index 0000000000..2b94a4ae51
> > > --- /dev/null
> > > +++ b/src/util/virstoragefileprobe.h
> > > @@ -0,0 +1,44 @@
> > > +/*
> > > + * virstoragefileprobe.h: file utility functions for FS storage backend
> > > + *
> > > + * Copyright (C) 2007-2009, 2012-2016 Red Hat, Inc.
> > > + * Copyright (C) 2007-2008 Daniel P. Berrange
> > > + *
> > > + * This library is free software; you can redistribute it and/or
> > > + * modify it under the terms of the GNU Lesser General Public
> > > + * License as published by the Free Software Foundation; either
> > > + * version 2.1 of the License, or (at your option) any later version.
> > > + *
> > > + * This library is distributed in the hope that it will be useful,
> > > + * but WITHOUT ANY WARRANTY; without even the implied warranty of
> > > + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
> > > + * Lesser General Public License for more details.
> > > + *
> > > + * You should have received a copy of the GNU Lesser General Public
> > > + * License along with this library.  If not, see
> > > + * <http://www.gnu.org/licenses/>.
> > > + */
> > > +
> > > +#pragma once
> > > +
> > > +#include <sys/stat.h>
> > 
> > #include <sys/types.h> for uid_t/gid_t instead, there's no use of the
> > stat struct in the header, but that shouldn't be necessary at all if the
> > format is probed fromt he buffer directly.
> > 
> > > +
> > > +#include "virstoragefile.h"
> > > +
> > > +/* Minimum header size required to probe all known formats with
> > > + * virStorageFileProbeFormat, or obtain metadata from a known format.
> > > + * Rounded to multiple of 512 (ISO has a 5-byte magic at offset
> > > + * 32769).  Some formats can be probed with fewer bytes.  Although
> > > + * some formats theoretically permit metadata that can rely on offsets
> > > + * beyond this size, in practice that doesn't matter.  */
> > > +#define VIR_STORAGE_MAX_HEADER 0x8200
> > > +
> > > +int
> > > +virStorageFileProbeGetMetadata(virStorageSourcePtr meta,
> > > +                               char *buf,
> > > +                               size_t len);
> > > +
> > > +int
> > > +virStorageFileProbeFormat(const char *path,
> > > +                          uid_t uid,
> > > +                          gid_t gid);
> > 
> > Reviewed-by: Peter Krempa <pkrempa@redhat.com>
> > 
> > If you decide that it's too much hassle to shuffle this around, you can
> > return the code opening the file back to the appropriate file later.
> > 
> > (Or I'll do it, if you don't, but virstoragefileprobe.c will not
> > 'open()' anything in the end)
> 
> Make sense, I'll look into it and post v3.

So I looked into it and the best what we can do is remove
virStorageFileProbeFormat completely. I've discussed it with Peter
and it will be done as followup so I'll take your RB for now.

Thanks for the review, I'll push this series shortly.

Pavel