vdpasim_queue_ready calls vringh_init_iotlb, which resets split indexes.
But it can be called after setting a ring base with
vdpasim_set_vq_state.

Fix it by stashing them. They're still resetted in vdpasim_vq_reset.

This was discovered and tested live migrating the vdpa_sim_net device.

Fixes: 2c53d0f64c06 ("vdpasim: vDPA device simulator")
Signed-off-by: Eugenio Pérez <eperezma@redhat.com>
---
 drivers/vdpa/vdpa_sim/vdpa_sim.c | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/drivers/vdpa/vdpa_sim/vdpa_sim.c b/drivers/vdpa/vdpa_sim/vdpa_sim.c
index cb88891b44a8..8839232a3fcb 100644
--- a/drivers/vdpa/vdpa_sim/vdpa_sim.c
+++ b/drivers/vdpa/vdpa_sim/vdpa_sim.c
@@ -66,6 +66,7 @@ static void vdpasim_vq_notify(struct vringh *vring)
 static void vdpasim_queue_ready(struct vdpasim *vdpasim, unsigned int idx)
 {
 	struct vdpasim_virtqueue *vq = &vdpasim->vqs[idx];
+	uint16_t last_avail_idx = vq->vring.last_avail_idx;
 
 	vringh_init_iotlb(&vq->vring, vdpasim->features, vq->num, false,
 			  (struct vring_desc *)(uintptr_t)vq->desc_addr,
@@ -74,6 +75,7 @@ static void vdpasim_queue_ready(struct vdpasim *vdpasim, unsigned int idx)
 			  (struct vring_used *)
 			  (uintptr_t)vq->device_addr);
 
+	vq->vring.last_avail_idx = last_avail_idx;
 	vq->vring.notify = vdpasim_vq_notify;
 }
 
-- 
2.31.1

On Thu, Jan 19, 2023 at 12:44 AM Eugenio Pérez <eperezma@redhat.com> wrote:
>
> vdpasim_queue_ready calls vringh_init_iotlb, which resets split indexes.
> But it can be called after setting a ring base with
> vdpasim_set_vq_state.
>
> Fix it by stashing them. They're still resetted in vdpasim_vq_reset.
>
> This was discovered and tested live migrating the vdpa_sim_net device.
>
> Fixes: 2c53d0f64c06 ("vdpasim: vDPA device simulator")
> Signed-off-by: Eugenio Pérez <eperezma@redhat.com>
> ---
>  drivers/vdpa/vdpa_sim/vdpa_sim.c | 2 ++
>  1 file changed, 2 insertions(+)
>
> diff --git a/drivers/vdpa/vdpa_sim/vdpa_sim.c b/drivers/vdpa/vdpa_sim/vdpa_sim.c
> index cb88891b44a8..8839232a3fcb 100644
> --- a/drivers/vdpa/vdpa_sim/vdpa_sim.c
> +++ b/drivers/vdpa/vdpa_sim/vdpa_sim.c
> @@ -66,6 +66,7 @@ static void vdpasim_vq_notify(struct vringh *vring)
>  static void vdpasim_queue_ready(struct vdpasim *vdpasim, unsigned int idx)
>  {
>         struct vdpasim_virtqueue *vq = &vdpasim->vqs[idx];
> +       uint16_t last_avail_idx = vq->vring.last_avail_idx;
>
>         vringh_init_iotlb(&vq->vring, vdpasim->features, vq->num, false,
>                           (struct vring_desc *)(uintptr_t)vq->desc_addr,
> @@ -74,6 +75,7 @@ static void vdpasim_queue_ready(struct vdpasim *vdpasim, unsigned int idx)
>                           (struct vring_used *)
>                           (uintptr_t)vq->device_addr);
>
> +       vq->vring.last_avail_idx = last_avail_idx;

Does this need to be serialized with the datapath?

E.g in set_vq_state() we do:

spin_lock(&vdpasim->lock);
vrh->last_avail_idx = state->split.avail_index;
spin_unlock(&vdpasim->lock);

Thanks

>         vq->vring.notify = vdpasim_vq_notify;
>  }
>
> --
> 2.31.1
>

On Thu, Jan 19, 2023 at 4:16 AM Jason Wang <jasowang@redhat.com> wrote:
>
> On Thu, Jan 19, 2023 at 12:44 AM Eugenio Pérez <eperezma@redhat.com> wrote:
> >
> > vdpasim_queue_ready calls vringh_init_iotlb, which resets split indexes.
> > But it can be called after setting a ring base with
> > vdpasim_set_vq_state.
> >
> > Fix it by stashing them. They're still resetted in vdpasim_vq_reset.
> >
> > This was discovered and tested live migrating the vdpa_sim_net device.
> >
> > Fixes: 2c53d0f64c06 ("vdpasim: vDPA device simulator")
> > Signed-off-by: Eugenio Pérez <eperezma@redhat.com>
> > ---
> >  drivers/vdpa/vdpa_sim/vdpa_sim.c | 2 ++
> >  1 file changed, 2 insertions(+)
> >
> > diff --git a/drivers/vdpa/vdpa_sim/vdpa_sim.c b/drivers/vdpa/vdpa_sim/vdpa_sim.c
> > index cb88891b44a8..8839232a3fcb 100644
> > --- a/drivers/vdpa/vdpa_sim/vdpa_sim.c
> > +++ b/drivers/vdpa/vdpa_sim/vdpa_sim.c
> > @@ -66,6 +66,7 @@ static void vdpasim_vq_notify(struct vringh *vring)
> >  static void vdpasim_queue_ready(struct vdpasim *vdpasim, unsigned int idx)
> >  {
> >         struct vdpasim_virtqueue *vq = &vdpasim->vqs[idx];
> > +       uint16_t last_avail_idx = vq->vring.last_avail_idx;
> >
> >         vringh_init_iotlb(&vq->vring, vdpasim->features, vq->num, false,
> >                           (struct vring_desc *)(uintptr_t)vq->desc_addr,
> > @@ -74,6 +75,7 @@ static void vdpasim_queue_ready(struct vdpasim *vdpasim, unsigned int idx)
> >                           (struct vring_used *)
> >                           (uintptr_t)vq->device_addr);
> >
> > +       vq->vring.last_avail_idx = last_avail_idx;
>
> Does this need to be serialized with the datapath?
>
> E.g in set_vq_state() we do:
>
> spin_lock(&vdpasim->lock);
> vrh->last_avail_idx = state->split.avail_index;
> spin_unlock(&vdpasim->lock);
>

vdpasim_queue_ready is called from vdpasim_set_vq_ready, which holds
these locks.

Maybe it's too much indirection and to embed vdpasim_queue_ready in
vdpasim_set_vq_ready would be clearer for the future?

Thanks!

在 2023/1/19 17:14, Eugenio Perez Martin 写道:
> On Thu, Jan 19, 2023 at 4:16 AM Jason Wang <jasowang@redhat.com> wrote:
>> On Thu, Jan 19, 2023 at 12:44 AM Eugenio Pérez <eperezma@redhat.com> wrote:
>>> vdpasim_queue_ready calls vringh_init_iotlb, which resets split indexes.
>>> But it can be called after setting a ring base with
>>> vdpasim_set_vq_state.
>>>
>>> Fix it by stashing them. They're still resetted in vdpasim_vq_reset.
>>>
>>> This was discovered and tested live migrating the vdpa_sim_net device.
>>>
>>> Fixes: 2c53d0f64c06 ("vdpasim: vDPA device simulator")
>>> Signed-off-by: Eugenio Pérez <eperezma@redhat.com>
>>> ---
>>>   drivers/vdpa/vdpa_sim/vdpa_sim.c | 2 ++
>>>   1 file changed, 2 insertions(+)
>>>
>>> diff --git a/drivers/vdpa/vdpa_sim/vdpa_sim.c b/drivers/vdpa/vdpa_sim/vdpa_sim.c
>>> index cb88891b44a8..8839232a3fcb 100644
>>> --- a/drivers/vdpa/vdpa_sim/vdpa_sim.c
>>> +++ b/drivers/vdpa/vdpa_sim/vdpa_sim.c
>>> @@ -66,6 +66,7 @@ static void vdpasim_vq_notify(struct vringh *vring)
>>>   static void vdpasim_queue_ready(struct vdpasim *vdpasim, unsigned int idx)
>>>   {
>>>          struct vdpasim_virtqueue *vq = &vdpasim->vqs[idx];
>>> +       uint16_t last_avail_idx = vq->vring.last_avail_idx;
>>>
>>>          vringh_init_iotlb(&vq->vring, vdpasim->features, vq->num, false,
>>>                            (struct vring_desc *)(uintptr_t)vq->desc_addr,
>>> @@ -74,6 +75,7 @@ static void vdpasim_queue_ready(struct vdpasim *vdpasim, unsigned int idx)
>>>                            (struct vring_used *)
>>>                            (uintptr_t)vq->device_addr);
>>>
>>> +       vq->vring.last_avail_idx = last_avail_idx;
>> Does this need to be serialized with the datapath?
>>
>> E.g in set_vq_state() we do:
>>
>> spin_lock(&vdpasim->lock);
>> vrh->last_avail_idx = state->split.avail_index;
>> spin_unlock(&vdpasim->lock);
>>
> vdpasim_queue_ready is called from vdpasim_set_vq_ready, which holds
> these locks.
>
> Maybe it's too much indirection and to embed vdpasim_queue_ready in
> vdpasim_set_vq_ready would be clearer for the future?


Nope, I miss the caller.

Acked-by: Jason Wang <jasowang@redhat.com>

Thanks


>
> Thanks!
>

The patch was tested by QE in a test environment and regression tested
using vdpa_sim device with virtio_vdpa and vhost_vdpa;There are no new
issues caused by this patch.

Tested-by: Lei Yang <leiyang@redhat.com>

Jason Wang <jasowang@redhat.com> 于2023年1月29日周日 13:56写道：
>
>
> 在 2023/1/19 17:14, Eugenio Perez Martin 写道:
> > On Thu, Jan 19, 2023 at 4:16 AM Jason Wang <jasowang@redhat.com> wrote:
> >> On Thu, Jan 19, 2023 at 12:44 AM Eugenio Pérez <eperezma@redhat.com> wrote:
> >>> vdpasim_queue_ready calls vringh_init_iotlb, which resets split indexes.
> >>> But it can be called after setting a ring base with
> >>> vdpasim_set_vq_state.
> >>>
> >>> Fix it by stashing them. They're still resetted in vdpasim_vq_reset.
> >>>
> >>> This was discovered and tested live migrating the vdpa_sim_net device.
> >>>
> >>> Fixes: 2c53d0f64c06 ("vdpasim: vDPA device simulator")
> >>> Signed-off-by: Eugenio Pérez <eperezma@redhat.com>
> >>> ---
> >>>   drivers/vdpa/vdpa_sim/vdpa_sim.c | 2 ++
> >>>   1 file changed, 2 insertions(+)
> >>>
> >>> diff --git a/drivers/vdpa/vdpa_sim/vdpa_sim.c b/drivers/vdpa/vdpa_sim/vdpa_sim.c
> >>> index cb88891b44a8..8839232a3fcb 100644
> >>> --- a/drivers/vdpa/vdpa_sim/vdpa_sim.c
> >>> +++ b/drivers/vdpa/vdpa_sim/vdpa_sim.c
> >>> @@ -66,6 +66,7 @@ static void vdpasim_vq_notify(struct vringh *vring)
> >>>   static void vdpasim_queue_ready(struct vdpasim *vdpasim, unsigned int idx)
> >>>   {
> >>>          struct vdpasim_virtqueue *vq = &vdpasim->vqs[idx];
> >>> +       uint16_t last_avail_idx = vq->vring.last_avail_idx;
> >>>
> >>>          vringh_init_iotlb(&vq->vring, vdpasim->features, vq->num, false,
> >>>                            (struct vring_desc *)(uintptr_t)vq->desc_addr,
> >>> @@ -74,6 +75,7 @@ static void vdpasim_queue_ready(struct vdpasim *vdpasim, unsigned int idx)
> >>>                            (struct vring_used *)
> >>>                            (uintptr_t)vq->device_addr);
> >>>
> >>> +       vq->vring.last_avail_idx = last_avail_idx;
> >> Does this need to be serialized with the datapath?
> >>
> >> E.g in set_vq_state() we do:
> >>
> >> spin_lock(&vdpasim->lock);
> >> vrh->last_avail_idx = state->split.avail_index;
> >> spin_unlock(&vdpasim->lock);
> >>
> > vdpasim_queue_ready is called from vdpasim_set_vq_ready, which holds
> > these locks.
> >
> > Maybe it's too much indirection and to embed vdpasim_queue_ready in
> > vdpasim_set_vq_ready would be clearer for the future?
>
>
> Nope, I miss the caller.
>
> Acked-by: Jason Wang <jasowang@redhat.com>
>
> Thanks
>
>
> >
> > Thanks!
> >
>