Nested SMMUv3 feature requires the support/presence of host-level SMMUv3
instance(s). Add a helper to read the sysfs for the number of instances.
Log them in a vms list using a new struct VirtNestedSmmu.

This will be used by a following patch to assign a passthrough device to
corresponding nested SMMUv3 instance.

Signed-off-by: Nicolin Chen <nicolinc@nvidia.com>
---
 hw/arm/virt.c         | 35 +++++++++++++++++++++++++++++++++++
 include/hw/arm/virt.h |  8 ++++++++
 2 files changed, 43 insertions(+)

diff --git a/hw/arm/virt.c b/hw/arm/virt.c
index 71093d7c60..be3d8b0ce6 100644
--- a/hw/arm/virt.c
+++ b/hw/arm/virt.c
@@ -2668,6 +2668,37 @@ static char *virt_get_iommu(Object *obj, Error **errp)
     }
 }
 
+static int virt_get_num_nested_smmus(VirtMachineState *vms, Error **errp)
+{
+    VirtNestedSmmu *nested_smmu;
+    struct dirent *dent;
+    DIR *dir = NULL;
+    int num = 0;
+
+    dir = opendir("/sys/class/iommu");
+    if (!dir) {
+        error_setg_errno(errp, errno, "couldn't open /sys/class/iommu");
+        return 0;
+    }
+
+    while ((dent = readdir(dir))) {
+        if (!strncmp(dent->d_name, "smmu3.0x", 7)) {
+            nested_smmu = g_new0(VirtNestedSmmu, 1);
+            nested_smmu->index = num;
+            nested_smmu->smmu_node = g_strdup(dent->d_name);
+            QLIST_INSERT_HEAD(&vms->nested_smmu_list, nested_smmu, next);
+            num++;
+        }
+    }
+
+    if (num == 0) {
+        error_setg_errno(errp, errno,
+                         "couldn't find any SMMUv3 HW to setup nesting");
+    }
+
+    return num;
+}
+
 static void virt_set_iommu(Object *obj, const char *value, Error **errp)
 {
     VirtMachineState *vms = VIRT_MACHINE(obj);
@@ -2676,6 +2707,7 @@ static void virt_set_iommu(Object *obj, const char *value, Error **errp)
         vms->iommu = VIRT_IOMMU_SMMUV3;
     } else if (!strcmp(value, "nested-smmuv3")) {
         vms->iommu = VIRT_IOMMU_NESTED_SMMUV3;
+        vms->num_nested_smmus = virt_get_num_nested_smmus(vms, errp);
     } else if (!strcmp(value, "none")) {
         vms->iommu = VIRT_IOMMU_NONE;
     } else {
@@ -3232,6 +3264,9 @@ static void virt_instance_init(Object *obj)
     /* The default root bus is attached to iommu by default */
     vms->default_bus_bypass_iommu = false;
 
+    /* Default disallows nested SMMU instantiation */
+    vms->num_nested_smmus = 0;
+
     /* Default disallows RAS instantiation */
     vms->ras = false;
 
diff --git a/include/hw/arm/virt.h b/include/hw/arm/virt.h
index d5cbce1a30..e0c07527c4 100644
--- a/include/hw/arm/virt.h
+++ b/include/hw/arm/virt.h
@@ -135,6 +135,12 @@ struct VirtMachineClass {
     bool no_ns_el2_virt_timer_irq;
 };
 
+typedef struct VirtNestedSmmu {
+    int index;
+    char *smmu_node;
+    QLIST_ENTRY(VirtNestedSmmu) next;
+} VirtNestedSmmu;
+
 struct VirtMachineState {
     MachineState parent;
     Notifier machine_done;
@@ -153,6 +159,7 @@ struct VirtMachineState {
     bool ras;
     bool mte;
     bool dtb_randomness;
+    int num_nested_smmus;
     OnOffAuto acpi;
     VirtGICType gic_version;
     IOMMUFDBackend *iommufd;
@@ -178,6 +185,7 @@ struct VirtMachineState {
     char *oem_id;
     char *oem_table_id;
     bool ns_el2_virt_timer_irq;
+    QLIST_HEAD(, VirtNestedSmmu) nested_smmu_list;
 };
 
 #define VIRT_ECAM_ID(high) (high ? VIRT_HIGH_PCIE_ECAM : VIRT_PCIE_ECAM)
-- 
2.43.0

Hi Nicolin,

On 6/26/24 02:28, Nicolin Chen wrote:
> Nested SMMUv3 feature requires the support/presence of host-level SMMUv3
> instance(s). Add a helper to read the sysfs for the number of instances.
> Log them in a vms list using a new struct VirtNestedSmmu.
>
> This will be used by a following patch to assign a passthrough device to
> corresponding nested SMMUv3 instance.
Laterly the HostIOMMUDevice has been introduced to allow, among other
things, to pass information related to the physical IOMMU to the virtual
IOMMU.
I guess it would be well fitted to associate the viommu with its
underlying piommu.

I don't think we have such kind of host introspection in machine type.
Generally in can happen in the very device or in libvirt.

Thanks

Eric
>
> Signed-off-by: Nicolin Chen <nicolinc@nvidia.com>
> ---
>  hw/arm/virt.c         | 35 +++++++++++++++++++++++++++++++++++
>  include/hw/arm/virt.h |  8 ++++++++
>  2 files changed, 43 insertions(+)
>
> diff --git a/hw/arm/virt.c b/hw/arm/virt.c
> index 71093d7c60..be3d8b0ce6 100644
> --- a/hw/arm/virt.c
> +++ b/hw/arm/virt.c
> @@ -2668,6 +2668,37 @@ static char *virt_get_iommu(Object *obj, Error **errp)
>      }
>  }
>  
> +static int virt_get_num_nested_smmus(VirtMachineState *vms, Error **errp)
> +{
> +    VirtNestedSmmu *nested_smmu;
> +    struct dirent *dent;
> +    DIR *dir = NULL;
> +    int num = 0;
> +
> +    dir = opendir("/sys/class/iommu");
> +    if (!dir) {
> +        error_setg_errno(errp, errno, "couldn't open /sys/class/iommu");
> +        return 0;
> +    }
> +
> +    while ((dent = readdir(dir))) {
> +        if (!strncmp(dent->d_name, "smmu3.0x", 7)) {
> +            nested_smmu = g_new0(VirtNestedSmmu, 1);
> +            nested_smmu->index = num;
> +            nested_smmu->smmu_node = g_strdup(dent->d_name);
> +            QLIST_INSERT_HEAD(&vms->nested_smmu_list, nested_smmu, next);
> +            num++;
> +        }
> +    }
> +
> +    if (num == 0) {
> +        error_setg_errno(errp, errno,
> +                         "couldn't find any SMMUv3 HW to setup nesting");
> +    }
> +
> +    return num;
> +}
> +
>  static void virt_set_iommu(Object *obj, const char *value, Error **errp)
>  {
>      VirtMachineState *vms = VIRT_MACHINE(obj);
> @@ -2676,6 +2707,7 @@ static void virt_set_iommu(Object *obj, const char *value, Error **errp)
>          vms->iommu = VIRT_IOMMU_SMMUV3;
>      } else if (!strcmp(value, "nested-smmuv3")) {
>          vms->iommu = VIRT_IOMMU_NESTED_SMMUV3;
> +        vms->num_nested_smmus = virt_get_num_nested_smmus(vms, errp);
>      } else if (!strcmp(value, "none")) {
>          vms->iommu = VIRT_IOMMU_NONE;
>      } else {
> @@ -3232,6 +3264,9 @@ static void virt_instance_init(Object *obj)
>      /* The default root bus is attached to iommu by default */
>      vms->default_bus_bypass_iommu = false;
>  
> +    /* Default disallows nested SMMU instantiation */
> +    vms->num_nested_smmus = 0;
> +
>      /* Default disallows RAS instantiation */
>      vms->ras = false;
>  
> diff --git a/include/hw/arm/virt.h b/include/hw/arm/virt.h
> index d5cbce1a30..e0c07527c4 100644
> --- a/include/hw/arm/virt.h
> +++ b/include/hw/arm/virt.h
> @@ -135,6 +135,12 @@ struct VirtMachineClass {
>      bool no_ns_el2_virt_timer_irq;
>  };
>  
> +typedef struct VirtNestedSmmu {
> +    int index;
> +    char *smmu_node;
> +    QLIST_ENTRY(VirtNestedSmmu) next;
> +} VirtNestedSmmu;
> +
>  struct VirtMachineState {
>      MachineState parent;
>      Notifier machine_done;
> @@ -153,6 +159,7 @@ struct VirtMachineState {
>      bool ras;
>      bool mte;
>      bool dtb_randomness;
> +    int num_nested_smmus;
>      OnOffAuto acpi;
>      VirtGICType gic_version;
>      IOMMUFDBackend *iommufd;
> @@ -178,6 +185,7 @@ struct VirtMachineState {
>      char *oem_id;
>      char *oem_table_id;
>      bool ns_el2_virt_timer_irq;
> +    QLIST_HEAD(, VirtNestedSmmu) nested_smmu_list;
>  };
>  
>  #define VIRT_ECAM_ID(high) (high ? VIRT_HIGH_PCIE_ECAM : VIRT_PCIE_ECAM)

On Tue, Jul 09, 2024 at 11:20:16AM +0200, Eric Auger wrote:
> On 6/26/24 02:28, Nicolin Chen wrote:
> > Nested SMMUv3 feature requires the support/presence of host-level SMMUv3
> > instance(s). Add a helper to read the sysfs for the number of instances.
> > Log them in a vms list using a new struct VirtNestedSmmu.
> >
> > This will be used by a following patch to assign a passthrough device to
> > corresponding nested SMMUv3 instance.

> Laterly the HostIOMMUDevice has been introduced to allow, among other
> things, to pass information related to the physical IOMMU to the virtual
> IOMMU.
> I guess it would be well fitted to associate the viommu with its
> underlying piommu.

Wow, I missed that part -- backends/host_iommu_device. I will
see how I can fit these well with that.

> I don't think we have such kind of host introspection in machine type.
> Generally in can happen in the very device or in libvirt.

I think the biggest reason for having such an introspection in
the virt code is because of hotplug, (though it's not properly
implemented yet), as we don't know what new devices requiring
for nested translation would be joining later. So somebody has
to hold a full list.

Would you mind elaborating how the "device" or "libvirt" can
handle that?

Thanks!
Nicolin

On 7/9/24 19:11, Nicolin Chen wrote:
> On Tue, Jul 09, 2024 at 11:20:16AM +0200, Eric Auger wrote:
>> On 6/26/24 02:28, Nicolin Chen wrote:
>>> Nested SMMUv3 feature requires the support/presence of host-level SMMUv3
>>> instance(s). Add a helper to read the sysfs for the number of instances.
>>> Log them in a vms list using a new struct VirtNestedSmmu.
>>>
>>> This will be used by a following patch to assign a passthrough device to
>>> corresponding nested SMMUv3 instance.
>> Laterly the HostIOMMUDevice has been introduced to allow, among other
>> things, to pass information related to the physical IOMMU to the virtual
>> IOMMU.
>> I guess it would be well fitted to associate the viommu with its
>> underlying piommu.
> Wow, I missed that part -- backends/host_iommu_device. I will
> see how I can fit these well with that.
>
>> I don't think we have such kind of host introspection in machine type.
>> Generally in can happen in the very device or in libvirt.
> I think the biggest reason for having such an introspection in
> the virt code is because of hotplug, (though it's not properly
> implemented yet), as we don't know what new devices requiring
> for nested translation would be joining later. So somebody has
> to hold a full list.
>
> Would you mind elaborating how the "device" or "libvirt" can
> handle that?
If you know that on Grace you have 5 SMMU instances, can't you pre-build
a PCIe topology with 5 PXB and root ports at libvirt level.
Then when you hotplug your device you specify the corresponding slot
just as we do normally. But maybe I misunderstood the hotplug problematics.

Eric
>
> Thanks!
> Nicolin
>

On Tue, Jul 09, 2024 at 07:22:16PM +0200, Eric Auger wrote:
> On 7/9/24 19:11, Nicolin Chen wrote:
> > On Tue, Jul 09, 2024 at 11:20:16AM +0200, Eric Auger wrote:
> >> On 6/26/24 02:28, Nicolin Chen wrote:
> >>> Nested SMMUv3 feature requires the support/presence of host-level SMMUv3
> >>> instance(s). Add a helper to read the sysfs for the number of instances.
> >>> Log them in a vms list using a new struct VirtNestedSmmu.
> >>>
> >>> This will be used by a following patch to assign a passthrough device to
> >>> corresponding nested SMMUv3 instance.
> >> Laterly the HostIOMMUDevice has been introduced to allow, among other
> >> things, to pass information related to the physical IOMMU to the virtual
> >> IOMMU.
> >> I guess it would be well fitted to associate the viommu with its
> >> underlying piommu.
> > Wow, I missed that part -- backends/host_iommu_device. I will
> > see how I can fit these well with that.
> >
> >> I don't think we have such kind of host introspection in machine type.
> >> Generally in can happen in the very device or in libvirt.
> > I think the biggest reason for having such an introspection in
> > the virt code is because of hotplug, (though it's not properly
> > implemented yet), as we don't know what new devices requiring
> > for nested translation would be joining later. So somebody has
> > to hold a full list.
> >
> > Would you mind elaborating how the "device" or "libvirt" can
> > handle that?
> If you know that on Grace you have 5 SMMU instances, can't you pre-build
> a PCIe topology with 5 PXB and root ports at libvirt level.
> Then when you hotplug your device you specify the corresponding slot
> just as we do normally. But maybe I misunderstood the hotplug problematics.

I guess I got your point: basically, the introspection and sysfs
node matching for device assigning should happen in libvirt.

Thanks
Nicolin