Shows, how much data qcow2 allocates in underlying file. This should
be helpful on non-sparse file systems, when qemu-img info "disk size"
doesn't provide this information.
Signed-off-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>
---
Hi all.
Here is an allocated-size feature for qemu-img info.
block/qcow2-refcount.c | 60 ++++++++++++++++++++++++++++++++++++++++++++++++++
block/qcow2.c | 4 ++++
block/qcow2.h | 2 ++
qapi/block-core.json | 6 ++++-
4 files changed, 71 insertions(+), 1 deletion(-)
diff --git a/block/qcow2-refcount.c b/block/qcow2-refcount.c
index 7c06061aae..a2f7696a0a 100644
--- a/block/qcow2-refcount.c
+++ b/block/qcow2-refcount.c
@@ -2931,3 +2931,63 @@ done:
qemu_vfree(new_refblock);
return ret;
}
+
+typedef struct NbAllocatedClustersCo {
+ BlockDriverState *bs;
+ int64_t ret;
+} NbAllocatedClustersCo;
+
+static void coroutine_fn qcow2_nb_allocated_clusters_co_entry(void *opaque)
+{
+ NbAllocatedClustersCo *nbco = opaque;
+ BlockDriverState *bs = nbco->bs;
+ BDRVQcow2State *s = bs->opaque;
+ int64_t size, nb_clusters, nb_allocated = 0, i;
+ int ret = 0;
+
+ size = bdrv_getlength(bs->file->bs);
+ if (size < 0) {
+ nbco->ret = size;
+ return;
+ }
+
+ nb_clusters = size_to_clusters(s, size);
+
+ qemu_co_mutex_lock(&s->lock);
+
+ for (i = 0; i < nb_clusters; ++i) {
+ uint64_t refcount;
+ ret = qcow2_get_refcount(bs, i, &refcount);
+ if (ret < 0) {
+ nbco->ret = ret;
+ qemu_co_mutex_unlock(&s->lock);
+ return;
+ }
+ if (refcount > 0) {
+ nb_allocated++;
+ }
+ }
+
+ qemu_co_mutex_unlock(&s->lock);
+
+ nbco->ret = nb_allocated;
+}
+
+int64_t qcow2_nb_allocated_clusters(BlockDriverState *bs)
+{
+ NbAllocatedClustersCo nbco = {
+ .bs = bs,
+ .ret = -EINPROGRESS
+ };
+
+ if (qemu_in_coroutine()) {
+ qcow2_nb_allocated_clusters_co_entry(&nbco);
+ } else {
+ Coroutine *co =
+ qemu_coroutine_create(qcow2_nb_allocated_clusters_co_entry, &nbco);
+ qemu_coroutine_enter(co);
+ BDRV_POLL_WHILE(bs, nbco.ret == -EINPROGRESS);
+ }
+
+ return nbco.ret;
+}
diff --git a/block/qcow2.c b/block/qcow2.c
index a8d61f0981..9a64b89ba2 100644
--- a/block/qcow2.c
+++ b/block/qcow2.c
@@ -2927,6 +2927,8 @@ static ImageInfoSpecific *qcow2_get_specific_info(BlockDriverState *bs)
.refcount_bits = s->refcount_bits,
};
} else if (s->qcow_version == 3) {
+ int64_t nb_allocated = qcow2_nb_allocated_clusters(bs);
+
*spec_info->u.qcow2.data = (ImageInfoSpecificQCow2){
.compat = g_strdup("1.1"),
.lazy_refcounts = s->compatible_features &
@@ -2936,6 +2938,8 @@ static ImageInfoSpecific *qcow2_get_specific_info(BlockDriverState *bs)
QCOW2_INCOMPAT_CORRUPT,
.has_corrupt = true,
.refcount_bits = s->refcount_bits,
+ .allocated_size = MAX(0, nb_allocated) << s->cluster_bits,
+ .has_allocated_size = nb_allocated >= 0,
};
} else {
/* if this assertion fails, this probably means a new version was
diff --git a/block/qcow2.h b/block/qcow2.h
index 1801dc30dc..a0f0bf9358 100644
--- a/block/qcow2.h
+++ b/block/qcow2.h
@@ -532,6 +532,8 @@ int qcow2_change_refcount_order(BlockDriverState *bs, int refcount_order,
BlockDriverAmendStatusCB *status_cb,
void *cb_opaque, Error **errp);
+int64_t qcow2_nb_allocated_clusters(BlockDriverState *bs);
+
/* qcow2-cluster.c functions */
int qcow2_grow_l1_table(BlockDriverState *bs, uint64_t min_size,
bool exact_size);
diff --git a/qapi/block-core.json b/qapi/block-core.json
index 6b974b952f..ddaffff830 100644
--- a/qapi/block-core.json
+++ b/qapi/block-core.json
@@ -44,6 +44,9 @@
#
# @refcount-bits: width of a refcount entry in bits (since 2.3)
#
+# @allocated-size: number of allocated clusters (including metadata)
+# if available; only valid for compat >= 1.1 (since v2.10)
+#
# Since: 1.7
##
{ 'struct': 'ImageInfoSpecificQCow2',
@@ -51,7 +54,8 @@
'compat': 'str',
'*lazy-refcounts': 'bool',
'*corrupt': 'bool',
- 'refcount-bits': 'int'
+ 'refcount-bits': 'int',
+ '*allocated-size': 'uint64'
} }
##
--
2.11.1
Am 18.05.2017 um 12:09 hat Vladimir Sementsov-Ogievskiy geschrieben:
> Shows, how much data qcow2 allocates in underlying file. This should
> be helpful on non-sparse file systems, when qemu-img info "disk size"
> doesn't provide this information.
>
> Signed-off-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>
> ---
> Hi all.
>
> Here is an allocated-size feature for qemu-img info.
I'm not a fan of loading all L2 tables (can take some time) for
'qemu-img info' (which should be very quick). Why isn't the qemu-img
check output good enough?
Kevin
$ ./qemu-img check /tmp/test.qcow2
No errors were found on the image.
16164/491520 = 3.29% allocated, 11.98% fragmented, 0.00% compressed clusters
Image end offset: 1060044800
$ ./qemu-img check --output=json /tmp/test.qcow2
{
"image-end-offset": 1060044800,
"total-clusters": 491520,
"check-errors": 0,
"allocated-clusters": 16164,
"filename": "/tmp/test.qcow2",
"format": "qcow2",
"fragmented-clusters": 1937
}
On 05/18/2017 01:25 PM, Kevin Wolf wrote:
> Am 18.05.2017 um 12:09 hat Vladimir Sementsov-Ogievskiy geschrieben:
>> Shows, how much data qcow2 allocates in underlying file. This should
>> be helpful on non-sparse file systems, when qemu-img info "disk size"
>> doesn't provide this information.
>>
>> Signed-off-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>
>> ---
>> Hi all.
>>
>> Here is an allocated-size feature for qemu-img info.
> I'm not a fan of loading all L2 tables (can take some time) for
> 'qemu-img info' (which should be very quick). Why isn't the qemu-img
> check output good enough?
>
> Kevin
>
> $ ./qemu-img check /tmp/test.qcow2
> No errors were found on the image.
> 16164/491520 = 3.29% allocated, 11.98% fragmented, 0.00% compressed clusters
> Image end offset: 1060044800
> $ ./qemu-img check --output=json /tmp/test.qcow2
> {
> "image-end-offset": 1060044800,
> "total-clusters": 491520,
> "check-errors": 0,
> "allocated-clusters": 16164,
> "filename": "/tmp/test.qcow2",
> "format": "qcow2",
> "fragmented-clusters": 1937
> }
this can be done, sure. Is this considered safe
for running VM too?
Den
18.05.2017 13:25, Kevin Wolf wrote:
> Am 18.05.2017 um 12:09 hat Vladimir Sementsov-Ogievskiy geschrieben:
>> Shows, how much data qcow2 allocates in underlying file. This should
>> be helpful on non-sparse file systems, when qemu-img info "disk size"
>> doesn't provide this information.
>>
>> Signed-off-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>
>> ---
>> Hi all.
>>
>> Here is an allocated-size feature for qemu-img info.
> I'm not a fan of loading all L2 tables (can take some time) for
> 'qemu-img info' (which should be very quick). Why isn't the qemu-img
> check output good enough?
>
> Kevin
>
> $ ./qemu-img check /tmp/test.qcow2
> No errors were found on the image.
> 16164/491520 = 3.29% allocated, 11.98% fragmented, 0.00% compressed clusters
> Image end offset: 1060044800
> $ ./qemu-img check --output=json /tmp/test.qcow2
> {
> "image-end-offset": 1060044800,
> "total-clusters": 491520,
> "check-errors": 0,
> "allocated-clusters": 16164,
> "filename": "/tmp/test.qcow2",
> "format": "qcow2",
> "fragmented-clusters": 1937
> }
It is not the same, it shows guest clusters, but we need host clusters -
including all metadata, dirty bitmaps, snapshots, etc..
--
Best regards,
Vladimir
Am 18.05.2017 um 13:04 hat Vladimir Sementsov-Ogievskiy geschrieben:
> 18.05.2017 13:25, Kevin Wolf wrote:
> >Am 18.05.2017 um 12:09 hat Vladimir Sementsov-Ogievskiy geschrieben:
> >>Shows, how much data qcow2 allocates in underlying file. This should
> >>be helpful on non-sparse file systems, when qemu-img info "disk size"
> >>doesn't provide this information.
> >>
> >>Signed-off-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>
> >>---
> >>Hi all.
> >>
> >>Here is an allocated-size feature for qemu-img info.
> >I'm not a fan of loading all L2 tables (can take some time) for
> >'qemu-img info' (which should be very quick). Why isn't the qemu-img
> >check output good enough?
> >
> >Kevin
> >
> >$ ./qemu-img check /tmp/test.qcow2
> >No errors were found on the image.
> >16164/491520 = 3.29% allocated, 11.98% fragmented, 0.00% compressed clusters
> >Image end offset: 1060044800
> >$ ./qemu-img check --output=json /tmp/test.qcow2
> >{
> > "image-end-offset": 1060044800,
> > "total-clusters": 491520,
> > "check-errors": 0,
> > "allocated-clusters": 16164,
> > "filename": "/tmp/test.qcow2",
> > "format": "qcow2",
> > "fragmented-clusters": 1937
> >}
>
> It is not the same, it shows guest clusters, but we need host
> clusters - including all metadata, dirty bitmaps, snapshots, etc..
Ah, right. But isn't that exactly the "disk size" (actual-size in JSON)
from qemu-img info? Your commit message mentions non-sparse filesystems
(which one?), but why wouldn't "disk size" provide this information
there?
The one case where it doesn't work is if you store a qcow2 image on a
raw block device (this is something that oVirt does). In that case,
you can't benefit from sparseness and disk space is used for a cluster
in the middle even if its refcount is 0. oVirt uses "image-end-offset"
to get the size of the first of the block device that is actually in use
by the image.
What is your exact use case? Maybe this helps me understand the exact
kind of information that you need.
Kevin
On 05/18/2017 03:10 PM, Kevin Wolf wrote:
> Am 18.05.2017 um 13:04 hat Vladimir Sementsov-Ogievskiy geschrieben:
>> 18.05.2017 13:25, Kevin Wolf wrote:
>>> Am 18.05.2017 um 12:09 hat Vladimir Sementsov-Ogievskiy geschrieben:
>>>> Shows, how much data qcow2 allocates in underlying file. This should
>>>> be helpful on non-sparse file systems, when qemu-img info "disk size"
>>>> doesn't provide this information.
>>>>
>>>> Signed-off-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>
>>>> ---
>>>> Hi all.
>>>>
>>>> Here is an allocated-size feature for qemu-img info.
>>> I'm not a fan of loading all L2 tables (can take some time) for
>>> 'qemu-img info' (which should be very quick). Why isn't the qemu-img
>>> check output good enough?
>>>
>>> Kevin
>>>
>>> $ ./qemu-img check /tmp/test.qcow2
>>> No errors were found on the image.
>>> 16164/491520 = 3.29% allocated, 11.98% fragmented, 0.00% compressed clusters
>>> Image end offset: 1060044800
>>> $ ./qemu-img check --output=json /tmp/test.qcow2
>>> {
>>> "image-end-offset": 1060044800,
>>> "total-clusters": 491520,
>>> "check-errors": 0,
>>> "allocated-clusters": 16164,
>>> "filename": "/tmp/test.qcow2",
>>> "format": "qcow2",
>>> "fragmented-clusters": 1937
>>> }
>> It is not the same, it shows guest clusters, but we need host
>> clusters - including all metadata, dirty bitmaps, snapshots, etc..
> Ah, right. But isn't that exactly the "disk size" (actual-size in JSON)
> from qemu-img info? Your commit message mentions non-sparse filesystems
> (which one?), but why wouldn't "disk size" provide this information
> there?
>
> The one case where it doesn't work is if you store a qcow2 image on a
> raw block device (this is something that oVirt does). In that case,
> you can't benefit from sparseness and disk space is used for a cluster
> in the middle even if its refcount is 0. oVirt uses "image-end-offset"
> to get the size of the first of the block device that is actually in use
> by the image.
>
> What is your exact use case? Maybe this helps me understand the exact
> kind of information that you need.
>
> Kevin
Let us assume we have an image like the following:
[0][1][2][3][4][5][6]
Here [N] represents guest block number N, i.e. there are
7 sequential guest blocks. Let us assume that the guest
issues TRIM and says that block [1] is not needed at all.
The image becomes like
[0][.][2][3][4][5][6]
If the filesystem with this image is dumb and does not
support holes, we could not determine that we have
not used space inside the disk marked as [.]
The goal of this patch is to know amount of [.] blocks.
Den
Am 18.05.2017 um 14:22 hat Denis V. Lunev geschrieben:
> On 05/18/2017 03:10 PM, Kevin Wolf wrote:
> > Am 18.05.2017 um 13:04 hat Vladimir Sementsov-Ogievskiy geschrieben:
> >> 18.05.2017 13:25, Kevin Wolf wrote:
> >>> Am 18.05.2017 um 12:09 hat Vladimir Sementsov-Ogievskiy geschrieben:
> >>>> Shows, how much data qcow2 allocates in underlying file. This should
> >>>> be helpful on non-sparse file systems, when qemu-img info "disk size"
> >>>> doesn't provide this information.
> >>>>
> >>>> Signed-off-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>
> >>>> ---
> >>>> Hi all.
> >>>>
> >>>> Here is an allocated-size feature for qemu-img info.
> >>> I'm not a fan of loading all L2 tables (can take some time) for
> >>> 'qemu-img info' (which should be very quick). Why isn't the qemu-img
> >>> check output good enough?
> >>>
> >>> Kevin
> >>>
> >>> $ ./qemu-img check /tmp/test.qcow2
> >>> No errors were found on the image.
> >>> 16164/491520 = 3.29% allocated, 11.98% fragmented, 0.00% compressed clusters
> >>> Image end offset: 1060044800
> >>> $ ./qemu-img check --output=json /tmp/test.qcow2
> >>> {
> >>> "image-end-offset": 1060044800,
> >>> "total-clusters": 491520,
> >>> "check-errors": 0,
> >>> "allocated-clusters": 16164,
> >>> "filename": "/tmp/test.qcow2",
> >>> "format": "qcow2",
> >>> "fragmented-clusters": 1937
> >>> }
> >> It is not the same, it shows guest clusters, but we need host
> >> clusters - including all metadata, dirty bitmaps, snapshots, etc..
> > Ah, right. But isn't that exactly the "disk size" (actual-size in JSON)
> > from qemu-img info? Your commit message mentions non-sparse filesystems
> > (which one?), but why wouldn't "disk size" provide this information
> > there?
> >
> > The one case where it doesn't work is if you store a qcow2 image on a
> > raw block device (this is something that oVirt does). In that case,
> > you can't benefit from sparseness and disk space is used for a cluster
> > in the middle even if its refcount is 0. oVirt uses "image-end-offset"
> > to get the size of the first of the block device that is actually in use
> > by the image.
> >
> > What is your exact use case? Maybe this helps me understand the exact
> > kind of information that you need.
> >
> > Kevin
> Let us assume we have an image like the following:
> [0][1][2][3][4][5][6]
> Here [N] represents guest block number N, i.e. there are
> 7 sequential guest blocks. Let us assume that the guest
> issues TRIM and says that block [1] is not needed at all.
> The image becomes like
> [0][.][2][3][4][5][6]
> If the filesystem with this image is dumb and does not
> support holes, we could not determine that we have
> not used space inside the disk marked as [.]
>
> The goal of this patch is to know amount of [.] blocks.
Okay. If the existing tools can't give you the numbers that you need,
I think we could easily add this number to qemu-img check.
However, is this really what you need or do you want to know the image
size if the image was converted into a new image file? Because in this
case, some metadata (refcount blocks) might go away as well. In this
case, it sounds like Stefan's 'qemu-img measure' patches could be useful
for you (I haven't looked at them in detail yet, but that's what I
understood from the high level overview).
Kevin
On 05/18/2017 05:54 PM, Kevin Wolf wrote:
> Am 18.05.2017 um 14:22 hat Denis V. Lunev geschrieben:
>> On 05/18/2017 03:10 PM, Kevin Wolf wrote:
>>> Am 18.05.2017 um 13:04 hat Vladimir Sementsov-Ogievskiy geschrieben:
>>>> 18.05.2017 13:25, Kevin Wolf wrote:
>>>>> Am 18.05.2017 um 12:09 hat Vladimir Sementsov-Ogievskiy geschrieben:
>>>>>> Shows, how much data qcow2 allocates in underlying file. This should
>>>>>> be helpful on non-sparse file systems, when qemu-img info "disk size"
>>>>>> doesn't provide this information.
>>>>>>
>>>>>> Signed-off-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>
>>>>>> ---
>>>>>> Hi all.
>>>>>>
>>>>>> Here is an allocated-size feature for qemu-img info.
>>>>> I'm not a fan of loading all L2 tables (can take some time) for
>>>>> 'qemu-img info' (which should be very quick). Why isn't the qemu-img
>>>>> check output good enough?
>>>>>
>>>>> Kevin
>>>>>
>>>>> $ ./qemu-img check /tmp/test.qcow2
>>>>> No errors were found on the image.
>>>>> 16164/491520 = 3.29% allocated, 11.98% fragmented, 0.00% compressed clusters
>>>>> Image end offset: 1060044800
>>>>> $ ./qemu-img check --output=json /tmp/test.qcow2
>>>>> {
>>>>> "image-end-offset": 1060044800,
>>>>> "total-clusters": 491520,
>>>>> "check-errors": 0,
>>>>> "allocated-clusters": 16164,
>>>>> "filename": "/tmp/test.qcow2",
>>>>> "format": "qcow2",
>>>>> "fragmented-clusters": 1937
>>>>> }
>>>> It is not the same, it shows guest clusters, but we need host
>>>> clusters - including all metadata, dirty bitmaps, snapshots, etc..
>>> Ah, right. But isn't that exactly the "disk size" (actual-size in JSON)
>>> from qemu-img info? Your commit message mentions non-sparse filesystems
>>> (which one?), but why wouldn't "disk size" provide this information
>>> there?
>>>
>>> The one case where it doesn't work is if you store a qcow2 image on a
>>> raw block device (this is something that oVirt does). In that case,
>>> you can't benefit from sparseness and disk space is used for a cluster
>>> in the middle even if its refcount is 0. oVirt uses "image-end-offset"
>>> to get the size of the first of the block device that is actually in use
>>> by the image.
>>>
>>> What is your exact use case? Maybe this helps me understand the exact
>>> kind of information that you need.
>>>
>>> Kevin
>> Let us assume we have an image like the following:
>> [0][1][2][3][4][5][6]
>> Here [N] represents guest block number N, i.e. there are
>> 7 sequential guest blocks. Let us assume that the guest
>> issues TRIM and says that block [1] is not needed at all.
>> The image becomes like
>> [0][.][2][3][4][5][6]
>> If the filesystem with this image is dumb and does not
>> support holes, we could not determine that we have
>> not used space inside the disk marked as [.]
>>
>> The goal of this patch is to know amount of [.] blocks.
> Okay. If the existing tools can't give you the numbers that you need,
> I think we could easily add this number to qemu-img check.
>
> However, is this really what you need or do you want to know the image
> size if the image was converted into a new image file? Because in this
> case, some metadata (refcount blocks) might go away as well. In this
> case, it sounds like Stefan's 'qemu-img measure' patches could be useful
> for you (I haven't looked at them in detail yet, but that's what I
> understood from the high level overview).
>
> Kevin
OK. 'qemu-img check' is not a problem to integrated with.
I'll look into measure patches to see applicability.
This is for sure good idea.
Thanks.
18.05.2017 15:10, Kevin Wolf wrote:
> Am 18.05.2017 um 13:04 hat Vladimir Sementsov-Ogievskiy geschrieben:
>> 18.05.2017 13:25, Kevin Wolf wrote:
>>> Am 18.05.2017 um 12:09 hat Vladimir Sementsov-Ogievskiy geschrieben:
>>>> Shows, how much data qcow2 allocates in underlying file. This should
>>>> be helpful on non-sparse file systems, when qemu-img info "disk size"
>>>> doesn't provide this information.
>>>>
>>>> Signed-off-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>
>>>> ---
>>>> Hi all.
>>>>
>>>> Here is an allocated-size feature for qemu-img info.
>>> I'm not a fan of loading all L2 tables (can take some time) for
>>> 'qemu-img info' (which should be very quick). Why isn't the qemu-img
>>> check output good enough?
>>>
>>> Kevin
>>>
>>> $ ./qemu-img check /tmp/test.qcow2
>>> No errors were found on the image.
>>> 16164/491520 = 3.29% allocated, 11.98% fragmented, 0.00% compressed clusters
>>> Image end offset: 1060044800
>>> $ ./qemu-img check --output=json /tmp/test.qcow2
>>> {
>>> "image-end-offset": 1060044800,
>>> "total-clusters": 491520,
>>> "check-errors": 0,
>>> "allocated-clusters": 16164,
>>> "filename": "/tmp/test.qcow2",
>>> "format": "qcow2",
>>> "fragmented-clusters": 1937
>>> }
>> It is not the same, it shows guest clusters, but we need host
>> clusters - including all metadata, dirty bitmaps, snapshots, etc..
> Ah, right. But isn't that exactly the "disk size" (actual-size in JSON)
> from qemu-img info? Your commit message mentions non-sparse filesystems
> (which one?), but why wouldn't "disk size" provide this information
> there?
it goes through bs->file, as qcow2 doesn't have
.bdrv_get_allocated_file_size, so in case of raw
bs->file we just call fstat on bs->file.
>
> The one case where it doesn't work is if you store a qcow2 image on a
> raw block device (this is something that oVirt does). In that case,
> you can't benefit from sparseness and disk space is used for a cluster
> in the middle even if its refcount is 0. oVirt uses "image-end-offset"
> to get the size of the first of the block device that is actually in use
> by the image.
Our customer has a distributed fs not maintaining sparse files. We need
this stat, about how many holes we have in qcow2 image.
>
> What is your exact use case? Maybe this helps me understand the exact
> kind of information that you need.
>
> Kevin
--
Best regards,
Vladimir
© 2016 - 2024 Red Hat, Inc.