In nested mode, we call the set_pasid_table() callback on each STE
update to pass the guest stage 1 configuration to the host and
apply it at physical level.

In the case of live migration, we need to manual call the
set_pasid_table() to load the guest stage 1 configurations to the
host. If this operation is fail, the migration is fail.

Signed-off-by: Kunkun Jiang <jiangkunkun@huawei.com>
---
 hw/arm/smmuv3.c     | 60 +++++++++++++++++++++++++++++++++++++++++++++
 hw/arm/trace-events |  1 +
 2 files changed, 61 insertions(+)

diff --git a/hw/arm/smmuv3.c b/hw/arm/smmuv3.c
index 6c6ed84e78..94ca15375c 100644
--- a/hw/arm/smmuv3.c
+++ b/hw/arm/smmuv3.c
@@ -1473,6 +1473,65 @@ static void smmu_realize(DeviceState *d, Error **errp)
     smmu_init_irq(s, dev);
 }
 
+static int smmuv3_manual_set_pci_device_pasid_table(SMMUDevice *sdev)
+{
+#ifdef __linux__
+    IOMMUMemoryRegion *mr = &(sdev->iommu);
+    int sid = smmu_get_sid(sdev);
+    SMMUEventInfo event = {.type = SMMU_EVT_NONE, .sid = sid,
+                           .inval_ste_allowed = true};
+    IOMMUConfig iommu_config = {};
+    SMMUTransCfg *cfg;
+    int ret = -1;
+
+    cfg = smmuv3_get_config(sdev, &event);
+    if (!cfg) {
+        return ret;
+    }
+
+    iommu_config.pasid_cfg.argsz = sizeof(struct iommu_pasid_table_config);
+    iommu_config.pasid_cfg.version = PASID_TABLE_CFG_VERSION_1;
+    iommu_config.pasid_cfg.format = IOMMU_PASID_FORMAT_SMMUV3;
+    iommu_config.pasid_cfg.base_ptr = cfg->s1ctxptr;
+    iommu_config.pasid_cfg.pasid_bits = 0;
+    iommu_config.pasid_cfg.vendor_data.smmuv3.version = PASID_TABLE_SMMUV3_CFG_VERSION_1;
+
+    if (cfg->disabled || cfg->bypassed) {
+        iommu_config.pasid_cfg.config = IOMMU_PASID_CONFIG_BYPASS;
+    } else if (cfg->aborted) {
+        iommu_config.pasid_cfg.config = IOMMU_PASID_CONFIG_ABORT;
+    } else {
+        iommu_config.pasid_cfg.config = IOMMU_PASID_CONFIG_TRANSLATE;
+    }
+
+    ret = pci_device_set_pasid_table(sdev->bus, sdev->devfn, &iommu_config);
+    if (ret) {
+        error_report("Failed to pass PASID table to host for iommu mr %s (%m)",
+                     mr->parent_obj.name);
+    }
+
+    return ret;
+#endif
+}
+
+static int smmuv3_post_load(void *opaque, int version_id)
+{
+    SMMUv3State *s3 = opaque;
+    SMMUState *s = &(s3->smmu_state);
+    SMMUDevice *sdev;
+    int ret = 0;
+
+    QLIST_FOREACH(sdev, &s->devices_with_notifiers, next) {
+        trace_smmuv3_post_load_sdev(sdev->devfn, sdev->iommu.parent_obj.name);
+        ret = smmuv3_manual_set_pci_device_pasid_table(sdev);
+        if (ret) {
+            break;
+        }
+    }
+
+    return ret;
+}
+
 static const VMStateDescription vmstate_smmuv3_queue = {
     .name = "smmuv3_queue",
     .version_id = 1,
@@ -1491,6 +1550,7 @@ static const VMStateDescription vmstate_smmuv3 = {
     .version_id = 1,
     .minimum_version_id = 1,
     .priority = MIG_PRI_IOMMU,
+    .post_load = smmuv3_post_load,
     .fields = (VMStateField[]) {
         VMSTATE_UINT32(features, SMMUv3State),
         VMSTATE_UINT8(sid_size, SMMUv3State),
diff --git a/hw/arm/trace-events b/hw/arm/trace-events
index 35e562ab74..caa864dd72 100644
--- a/hw/arm/trace-events
+++ b/hw/arm/trace-events
@@ -53,4 +53,5 @@ smmuv3_notify_flag_add(const char *iommu) "ADD SMMUNotifier node for iommu mr=%s
 smmuv3_notify_flag_del(const char *iommu) "DEL SMMUNotifier node for iommu mr=%s"
 smmuv3_inv_notifiers_iova(const char *name, uint16_t asid, uint64_t iova, uint8_t tg, uint64_t num_pages) "iommu mr=%s asid=%d iova=0x%"PRIx64" tg=%d num_pages=0x%"PRIx64
 smmuv3_notify_config_change(const char *name, uint8_t config, uint64_t s1ctxptr) "iommu mr=%s config=%d s1ctxptr=0x%"PRIx64
+smmuv3_post_load_sdev(int devfn, const char *name) "sdev devfn=%d iommu mr=%s"PRIx64
 
-- 
2.23.0

Hi Kunkun,
On 2/19/21 10:42 AM, Kunkun Jiang wrote:
> In nested mode, we call the set_pasid_table() callback on each STE
> update to pass the guest stage 1 configuration to the host and
> apply it at physical level.
> 
> In the case of live migration, we need to manual call the
s/manual/manually
> set_pasid_table() to load the guest stage 1 configurations to the
> host. If this operation is fail, the migration is fail.
s/If this operation is fail, the migration is fail/If this operation
fails, the migration fails.
> 
> Signed-off-by: Kunkun Jiang <jiangkunkun@huawei.com>
> ---
>  hw/arm/smmuv3.c     | 60 +++++++++++++++++++++++++++++++++++++++++++++
>  hw/arm/trace-events |  1 +
>  2 files changed, 61 insertions(+)
> 
> diff --git a/hw/arm/smmuv3.c b/hw/arm/smmuv3.c
> index 6c6ed84e78..94ca15375c 100644
> --- a/hw/arm/smmuv3.c
> +++ b/hw/arm/smmuv3.c
> @@ -1473,6 +1473,65 @@ static void smmu_realize(DeviceState *d, Error **errp)
>      smmu_init_irq(s, dev);
>  }
>  
> +static int smmuv3_manual_set_pci_device_pasid_table(SMMUDevice *sdev)
Can't you retrieve the associated sid and then call
smmuv3_notify_config_change()
> +{
> +#ifdef __linux__
> +    IOMMUMemoryRegion *mr = &(sdev->iommu);
> +    int sid = smmu_get_sid(sdev);
> +    SMMUEventInfo event = {.type = SMMU_EVT_NONE, .sid = sid,
> +                           .inval_ste_allowed = true};
> +    IOMMUConfig iommu_config = {};
> +    SMMUTransCfg *cfg;
> +    int ret = -1;
> +
> +    cfg = smmuv3_get_config(sdev, &event);
> +    if (!cfg) {
> +        return ret;
> +    }
> +
> +    iommu_config.pasid_cfg.argsz = sizeof(struct iommu_pasid_table_config);
> +    iommu_config.pasid_cfg.version = PASID_TABLE_CFG_VERSION_1;
> +    iommu_config.pasid_cfg.format = IOMMU_PASID_FORMAT_SMMUV3;
> +    iommu_config.pasid_cfg.base_ptr = cfg->s1ctxptr;
> +    iommu_config.pasid_cfg.pasid_bits = 0;
> +    iommu_config.pasid_cfg.vendor_data.smmuv3.version = PASID_TABLE_SMMUV3_CFG_VERSION_1;
> +
> +    if (cfg->disabled || cfg->bypassed) {
> +        iommu_config.pasid_cfg.config = IOMMU_PASID_CONFIG_BYPASS;
> +    } else if (cfg->aborted) {
> +        iommu_config.pasid_cfg.config = IOMMU_PASID_CONFIG_ABORT;
> +    } else {
> +        iommu_config.pasid_cfg.config = IOMMU_PASID_CONFIG_TRANSLATE;
> +    }
> +
> +    ret = pci_device_set_pasid_table(sdev->bus, sdev->devfn, &iommu_config);
> +    if (ret) {
> +        error_report("Failed to pass PASID table to host for iommu mr %s (%m)",
> +                     mr->parent_obj.name);
> +    }
> +
> +    return ret;
> +#endif
> +}
> +
> +static int smmuv3_post_load(void *opaque, int version_id)
> +{
> +    SMMUv3State *s3 = opaque;
> +    SMMUState *s = &(s3->smmu_state);
> +    SMMUDevice *sdev;
> +    int ret = 0;
> +
> +    QLIST_FOREACH(sdev, &s->devices_with_notifiers, next) {
> +        trace_smmuv3_post_load_sdev(sdev->devfn, sdev->iommu.parent_obj.name);
> +        ret = smmuv3_manual_set_pci_device_pasid_table(sdev);
> +        if (ret) {
> +            break;
> +        }
> +    }
> +
> +    return ret;
> +}
> +
>  static const VMStateDescription vmstate_smmuv3_queue = {
>      .name = "smmuv3_queue",
>      .version_id = 1,
> @@ -1491,6 +1550,7 @@ static const VMStateDescription vmstate_smmuv3 = {
>      .version_id = 1,
>      .minimum_version_id = 1,
>      .priority = MIG_PRI_IOMMU,
> +    .post_load = smmuv3_post_load,
>      .fields = (VMStateField[]) {
>          VMSTATE_UINT32(features, SMMUv3State),
>          VMSTATE_UINT8(sid_size, SMMUv3State),
> diff --git a/hw/arm/trace-events b/hw/arm/trace-events
> index 35e562ab74..caa864dd72 100644
> --- a/hw/arm/trace-events
> +++ b/hw/arm/trace-events
> @@ -53,4 +53,5 @@ smmuv3_notify_flag_add(const char *iommu) "ADD SMMUNotifier node for iommu mr=%s
>  smmuv3_notify_flag_del(const char *iommu) "DEL SMMUNotifier node for iommu mr=%s"
>  smmuv3_inv_notifiers_iova(const char *name, uint16_t asid, uint64_t iova, uint8_t tg, uint64_t num_pages) "iommu mr=%s asid=%d iova=0x%"PRIx64" tg=%d num_pages=0x%"PRIx64
>  smmuv3_notify_config_change(const char *name, uint8_t config, uint64_t s1ctxptr) "iommu mr=%s config=%d s1ctxptr=0x%"PRIx64
> +smmuv3_post_load_sdev(int devfn, const char *name) "sdev devfn=%d iommu mr=%s"PRIx64
>  
> 
Thanks

Eric

Hi Eric,

On 2021/4/12 16:34, Auger Eric wrote:
> Hi Kunkun,
> On 2/19/21 10:42 AM, Kunkun Jiang wrote:
>> In nested mode, we call the set_pasid_table() callback on each STE
>> update to pass the guest stage 1 configuration to the host and
>> apply it at physical level.
>>
>> In the case of live migration, we need to manual call the
> s/manual/manually
You are right.
>> set_pasid_table() to load the guest stage 1 configurations to the
>> host. If this operation is fail, the migration is fail.
> s/If this operation is fail, the migration is fail/If this operation
> fails, the migration fails.
Thanks for your careful review.
>> Signed-off-by: Kunkun Jiang <jiangkunkun@huawei.com>
>> ---
>>   hw/arm/smmuv3.c     | 60 +++++++++++++++++++++++++++++++++++++++++++++
>>   hw/arm/trace-events |  1 +
>>   2 files changed, 61 insertions(+)
>>
>> diff --git a/hw/arm/smmuv3.c b/hw/arm/smmuv3.c
>> index 6c6ed84e78..94ca15375c 100644
>> --- a/hw/arm/smmuv3.c
>> +++ b/hw/arm/smmuv3.c
>> @@ -1473,6 +1473,65 @@ static void smmu_realize(DeviceState *d, Error **errp)
>>       smmu_init_irq(s, dev);
>>   }
>>   
>> +static int smmuv3_manual_set_pci_device_pasid_table(SMMUDevice *sdev)
> Can't you retrieve the associated sid and then call
> smmuv3_notify_config_change()
Agreed. It can reuse the code of smmuv3_notify_config_change().
I will modify it in the next version.😁

But I need a return value to judge whether it is successful, which
may need to modify the type of smmuv3_notify_config_change().

Thanks,
Kunkun Jiang
>> +{
>> +#ifdef __linux__
>> +    IOMMUMemoryRegion *mr = &(sdev->iommu);
>> +    int sid = smmu_get_sid(sdev);
>> +    SMMUEventInfo event = {.type = SMMU_EVT_NONE, .sid = sid,
>> +                           .inval_ste_allowed = true};
>> +    IOMMUConfig iommu_config = {};
>> +    SMMUTransCfg *cfg;
>> +    int ret = -1;
>> +
>> +    cfg = smmuv3_get_config(sdev, &event);
>> +    if (!cfg) {
>> +        return ret;
>> +    }
>> +
>> +    iommu_config.pasid_cfg.argsz = sizeof(struct iommu_pasid_table_config);
>> +    iommu_config.pasid_cfg.version = PASID_TABLE_CFG_VERSION_1;
>> +    iommu_config.pasid_cfg.format = IOMMU_PASID_FORMAT_SMMUV3;
>> +    iommu_config.pasid_cfg.base_ptr = cfg->s1ctxptr;
>> +    iommu_config.pasid_cfg.pasid_bits = 0;
>> +    iommu_config.pasid_cfg.vendor_data.smmuv3.version = PASID_TABLE_SMMUV3_CFG_VERSION_1;
>> +
>> +    if (cfg->disabled || cfg->bypassed) {
>> +        iommu_config.pasid_cfg.config = IOMMU_PASID_CONFIG_BYPASS;
>> +    } else if (cfg->aborted) {
>> +        iommu_config.pasid_cfg.config = IOMMU_PASID_CONFIG_ABORT;
>> +    } else {
>> +        iommu_config.pasid_cfg.config = IOMMU_PASID_CONFIG_TRANSLATE;
>> +    }
>> +
>> +    ret = pci_device_set_pasid_table(sdev->bus, sdev->devfn, &iommu_config);
>> +    if (ret) {
>> +        error_report("Failed to pass PASID table to host for iommu mr %s (%m)",
>> +                     mr->parent_obj.name);
>> +    }
>> +
>> +    return ret;
>> +#endif
>> +}
>> +
>> +static int smmuv3_post_load(void *opaque, int version_id)
>> +{
>> +    SMMUv3State *s3 = opaque;
>> +    SMMUState *s = &(s3->smmu_state);
>> +    SMMUDevice *sdev;
>> +    int ret = 0;
>> +
>> +    QLIST_FOREACH(sdev, &s->devices_with_notifiers, next) {
>> +        trace_smmuv3_post_load_sdev(sdev->devfn, sdev->iommu.parent_obj.name);
>> +        ret = smmuv3_manual_set_pci_device_pasid_table(sdev);
>> +        if (ret) {
>> +            break;
>> +        }
>> +    }
>> +
>> +    return ret;
>> +}
>> +
>>   static const VMStateDescription vmstate_smmuv3_queue = {
>>       .name = "smmuv3_queue",
>>       .version_id = 1,
>> @@ -1491,6 +1550,7 @@ static const VMStateDescription vmstate_smmuv3 = {
>>       .version_id = 1,
>>       .minimum_version_id = 1,
>>       .priority = MIG_PRI_IOMMU,
>> +    .post_load = smmuv3_post_load,
>>       .fields = (VMStateField[]) {
>>           VMSTATE_UINT32(features, SMMUv3State),
>>           VMSTATE_UINT8(sid_size, SMMUv3State),
>> diff --git a/hw/arm/trace-events b/hw/arm/trace-events
>> index 35e562ab74..caa864dd72 100644
>> --- a/hw/arm/trace-events
>> +++ b/hw/arm/trace-events
>> @@ -53,4 +53,5 @@ smmuv3_notify_flag_add(const char *iommu) "ADD SMMUNotifier node for iommu mr=%s
>>   smmuv3_notify_flag_del(const char *iommu) "DEL SMMUNotifier node for iommu mr=%s"
>>   smmuv3_inv_notifiers_iova(const char *name, uint16_t asid, uint64_t iova, uint8_t tg, uint64_t num_pages) "iommu mr=%s asid=%d iova=0x%"PRIx64" tg=%d num_pages=0x%"PRIx64
>>   smmuv3_notify_config_change(const char *name, uint8_t config, uint64_t s1ctxptr) "iommu mr=%s config=%d s1ctxptr=0x%"PRIx64
>> +smmuv3_post_load_sdev(int devfn, const char *name) "sdev devfn=%d iommu mr=%s"PRIx64
>>   
>>
> Thanks
>
> Eric
>
> .