docs/system/arm/virt.rst | 4 ++++ hw/arm/virt.c | 38 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+)
The MMIO region size required to support virtualized environments with
large PCI BAR regions can exceed the hardcoded limit configured in QEMU.
For example, a VM with multiple NVIDIA Grace-Hopper GPUs passed through
requires more MMIO memory than the amount provided by VIRT_HIGH_PCIE_MMIO
(currently 512GB). Instead of updating VIRT_HIGH_PCIE_MMIO, introduce a
new parameter, highmem-mmio-size, that specifies the MMIO size required
to support the VM configuration.
Example usage with 1TB MMIO region size:
-machine virt,gic-version=3,highmem-mmio-size=1T
Signed-off-by: Matthew R. Ochs <mochs@nvidia.com>
---
v2: - Add unit suffix to example in commit message
- Use existing "high memory region" terminology
- Resolve minor braces nit
docs/system/arm/virt.rst | 4 ++++
hw/arm/virt.c | 38 ++++++++++++++++++++++++++++++++++++++
2 files changed, 42 insertions(+)
diff --git a/docs/system/arm/virt.rst b/docs/system/arm/virt.rst
index e67e7f0f7c50..f96cf4da2a78 100644
--- a/docs/system/arm/virt.rst
+++ b/docs/system/arm/virt.rst
@@ -138,6 +138,10 @@ highmem-mmio
Set ``on``/``off`` to enable/disable the high memory region for PCI MMIO.
The default is ``on``.
+highmem-mmio-size
+ Set the high memory region size for PCI MMIO. Must be a power-of-2 and
+ greater than the default size.
+
gic-version
Specify the version of the Generic Interrupt Controller (GIC) to provide.
Valid values are:
diff --git a/hw/arm/virt.c b/hw/arm/virt.c
index 49eb0355ef0c..d8d62df43f04 100644
--- a/hw/arm/virt.c
+++ b/hw/arm/virt.c
@@ -2773,6 +2773,36 @@ static void virt_set_highmem_mmio(Object *obj, bool value, Error **errp)
vms->highmem_mmio = value;
}
+static void virt_get_highmem_mmio_size(Object *obj, Visitor *v, const char *name,
+ void *opaque, Error **errp)
+{
+ uint64_t size = extended_memmap[VIRT_HIGH_PCIE_MMIO].size;
+
+ visit_type_size(v, name, &size, errp);
+}
+
+static void virt_set_highmem_mmio_size(Object *obj, Visitor *v, const char *name,
+ void *opaque, Error **errp)
+{
+ uint64_t size;
+
+ if (!visit_type_size(v, name, &size, errp)) {
+ return;
+ }
+
+ if (!is_power_of_2(size)) {
+ error_setg(errp, "highmem_mmio_size is not a power-of-2");
+ return;
+ }
+
+ if (size < extended_memmap[VIRT_HIGH_PCIE_MMIO].size) {
+ error_setg(errp, "highmem_mmio_size is less than the default (%lu)",
+ extended_memmap[VIRT_HIGH_PCIE_MMIO].size);
+ return;
+ }
+
+ extended_memmap[VIRT_HIGH_PCIE_MMIO].size = size;
+}
static bool virt_get_its(Object *obj, Error **errp)
{
@@ -3446,6 +3476,14 @@ static void virt_machine_class_init(ObjectClass *oc, void *data)
"Set on/off to enable/disable high "
"memory region for PCI MMIO");
+ object_class_property_add(oc, "highmem-mmio-size", "size",
+ virt_get_highmem_mmio_size,
+ virt_set_highmem_mmio_size,
+ NULL, NULL);
+ object_class_property_set_description(oc, "highmem-mmio-size",
+ "Set the high memory region size "
+ "for PCI MMIO");
+
object_class_property_add_str(oc, "gic-version", virt_get_gic_version,
virt_set_gic_version);
object_class_property_set_description(oc, "gic-version",
--
2.46.0On 2/5/25 2:19 PM, Matthew R. Ochs wrote:
> The MMIO region size required to support virtualized environments with
> large PCI BAR regions can exceed the hardcoded limit configured in QEMU.
> For example, a VM with multiple NVIDIA Grace-Hopper GPUs passed through
> requires more MMIO memory than the amount provided by VIRT_HIGH_PCIE_MMIO
> (currently 512GB). Instead of updating VIRT_HIGH_PCIE_MMIO, introduce a
> new parameter, highmem-mmio-size, that specifies the MMIO size required
> to support the VM configuration.
>
> Example usage with 1TB MMIO region size:
> -machine virt,gic-version=3,highmem-mmio-size=1T
>
> Signed-off-by: Matthew R. Ochs <mochs@nvidia.com>
> ---
> v2: - Add unit suffix to example in commit message
> - Use existing "high memory region" terminology
> - Resolve minor braces nit
>
> docs/system/arm/virt.rst | 4 ++++
> hw/arm/virt.c | 38 ++++++++++++++++++++++++++++++++++++++
> 2 files changed, 42 insertions(+)
>
With the following nitpick addressed:
Reviewed-by: Gavin Shan <gshan@redhat.com>
> diff --git a/docs/system/arm/virt.rst b/docs/system/arm/virt.rst
> index e67e7f0f7c50..f96cf4da2a78 100644
> --- a/docs/system/arm/virt.rst
> +++ b/docs/system/arm/virt.rst
> @@ -138,6 +138,10 @@ highmem-mmio
> Set ``on``/``off`` to enable/disable the high memory region for PCI MMIO.
> The default is ``on``.
>
> +highmem-mmio-size
> + Set the high memory region size for PCI MMIO. Must be a power-of-2 and
> + greater than the default size.
> +
The description doesn't match with the code changes because it's acceptable
when the specified size is equal to the default one. I think the description
needs to be improved for this.
> gic-version
> Specify the version of the Generic Interrupt Controller (GIC) to provide.
> Valid values are:
> diff --git a/hw/arm/virt.c b/hw/arm/virt.c
> index 49eb0355ef0c..d8d62df43f04 100644
> --- a/hw/arm/virt.c
> +++ b/hw/arm/virt.c
> @@ -2773,6 +2773,36 @@ static void virt_set_highmem_mmio(Object *obj, bool value, Error **errp)
> vms->highmem_mmio = value;
> }
>
> +static void virt_get_highmem_mmio_size(Object *obj, Visitor *v, const char *name,
> + void *opaque, Error **errp)
> +{
> + uint64_t size = extended_memmap[VIRT_HIGH_PCIE_MMIO].size;
> +
> + visit_type_size(v, name, &size, errp);
> +}
> +
> +static void virt_set_highmem_mmio_size(Object *obj, Visitor *v, const char *name,
> + void *opaque, Error **errp)
> +{
> + uint64_t size;
> +
> + if (!visit_type_size(v, name, &size, errp)) {
> + return;
> + }
> +
> + if (!is_power_of_2(size)) {
> + error_setg(errp, "highmem_mmio_size is not a power-of-2");
> + return;
> + }
> +
> + if (size < extended_memmap[VIRT_HIGH_PCIE_MMIO].size) {
> + error_setg(errp, "highmem_mmio_size is less than the default (%lu)",
> + extended_memmap[VIRT_HIGH_PCIE_MMIO].size);
> + return;
> + }
> +
It's acceptable when the specified size is equal to the default one, slightly
conflicting with the description in docs/system/arm/virt.rst
> + extended_memmap[VIRT_HIGH_PCIE_MMIO].size = size;
> +}
>
> static bool virt_get_its(Object *obj, Error **errp)
> {
> @@ -3446,6 +3476,14 @@ static void virt_machine_class_init(ObjectClass *oc, void *data)
> "Set on/off to enable/disable high "
> "memory region for PCI MMIO");
>
> + object_class_property_add(oc, "highmem-mmio-size", "size",
> + virt_get_highmem_mmio_size,
> + virt_set_highmem_mmio_size,
> + NULL, NULL);
> + object_class_property_set_description(oc, "highmem-mmio-size",
> + "Set the high memory region size "
> + "for PCI MMIO");
> +
> object_class_property_add_str(oc, "gic-version", virt_get_gic_version,
> virt_set_gic_version);
> object_class_property_set_description(oc, "gic-version",
Thanks,
Gavin
On Feb 4, 2025, at 11:08 PM, Gavin Shan <gshan@redhat.com> wrote:
On 2/5/25 2:19 PM, Matthew R. Ochs wrote:
The MMIO region size required to support virtualized environments with
large PCI BAR regions can exceed the hardcoded limit configured in QEMU.
For example, a VM with multiple NVIDIA Grace-Hopper GPUs passed through
requires more MMIO memory than the amount provided by VIRT_HIGH_PCIE_MMIO
(currently 512GB). Instead of updating VIRT_HIGH_PCIE_MMIO, introduce a
new parameter, highmem-mmio-size, that specifies the MMIO size required
to support the VM configuration.
Example usage with 1TB MMIO region size:
-machine virt,gic-version=3,highmem-mmio-size=1T
Signed-off-by: Matthew R. Ochs <mochs@nvidia.com>
---
v2: - Add unit suffix to example in commit message
- Use existing "high memory region" terminology
- Resolve minor braces nit
docs/system/arm/virt.rst | 4 ++++
hw/arm/virt.c | 38 ++++++++++++++++++++++++++++++++++++++
2 files changed, 42 insertions(+)
With the following nitpick addressed:
Reviewed-by: Gavin Shan <gshan@redhat.com<mailto:gshan@redhat.com>>
Thanks Gavin, I will correct the description in v3.
© 2016 - 2025 Red Hat, Inc.