1. Patchew
  2. QEMU
  3. [RFC for-5.1 0/4] Better handling of attempt NVLink2 unplug
  4. View patch

[RFC for-5.1 1/4] spapr: Refactor locating NVLink2 devices for device tree creation

David Gibson posted 4 patches 5 years, 10 months ago
Maintainers: David Gibson <david@gibson.dropbear.id.au>
[RFC for-5.1 1/4] spapr: Refactor locating NVLink2 devices for device tree creation
Posted by David Gibson 5 years, 10 months ago 
Currently spapr_phb_nvgpu_populate_pcidev_dt() works a little cryptically.
It steps through all the NVLink2 GPUs and NPUs and if they match the device
we're called for, we generate the relevant device tree information.

Make this a little more obvious by introducing helpers to determine it a
given PCI device is an NVLink2 GPU or NPU, returning the NVLink2 slot and
link number information as well.

Signed-off-by: David Gibson <david@gibson.dropbear.id.au>
---
 hw/ppc/spapr_pci_nvlink2.c | 115 +++++++++++++++++++++++++------------
 1 file changed, 79 insertions(+), 36 deletions(-)

diff --git a/hw/ppc/spapr_pci_nvlink2.c b/hw/ppc/spapr_pci_nvlink2.c
index 8332d5694e..7d3a685421 100644
--- a/hw/ppc/spapr_pci_nvlink2.c
+++ b/hw/ppc/spapr_pci_nvlink2.c
@@ -390,13 +390,12 @@ void spapr_phb_nvgpu_ram_populate_dt(SpaprPhbState *sphb, void *fdt)
 
 }
 
-void spapr_phb_nvgpu_populate_pcidev_dt(PCIDevice *dev, void *fdt, int offset,
-                                        SpaprPhbState *sphb)
+static bool is_nvgpu(PCIDevice *dev, SpaprPhbState *sphb, int *slot)
 {
-    int i, j;
+    int i;
 
     if (!sphb->nvgpus) {
-        return;
+        return false;
     }
 
     for (i = 0; i < sphb->nvgpus->num; ++i) {
@@ -406,47 +405,91 @@ void spapr_phb_nvgpu_populate_pcidev_dt(PCIDevice *dev, void *fdt, int offset,
         if (!nvslot->gpdev) {
             continue;
         }
+
         if (dev == nvslot->gpdev) {
-            uint32_t npus[nvslot->linknum];
+            if (slot) {
+                *slot = i;
+            }
+            return true;
+        }
+    }
 
-            for (j = 0; j < nvslot->linknum; ++j) {
-                PCIDevice *npdev = nvslot->links[j].npdev;
+    return false;
+}
 
-                npus[j] = cpu_to_be32(PHANDLE_PCIDEV(sphb, npdev));
-            }
-            _FDT(fdt_setprop(fdt, offset, "ibm,npu", npus,
-                             j * sizeof(npus[0])));
-            _FDT((fdt_setprop_cell(fdt, offset, "phandle",
-                                   PHANDLE_PCIDEV(sphb, dev))));
+static bool is_nvnpu(PCIDevice *dev, SpaprPhbState *sphb, int *slot, int *link)
+{
+    int i, j;
+
+    if (!sphb->nvgpus) {
+        return false;
+    }
+
+    for (i = 0; i < sphb->nvgpus->num; ++i) {
+        SpaprPhbPciNvGpuSlot *nvslot = &sphb->nvgpus->slots[i];
+
+        /* Skip "slot" without attached GPU */
+        if (!nvslot->gpdev) {
             continue;
         }
 
         for (j = 0; j < nvslot->linknum; ++j) {
-            if (dev != nvslot->links[j].npdev) {
-                continue;
+            if (dev == nvslot->links[j].npdev) {
+                if (slot) {
+                    *slot = i;
+                }
+                if (link) {
+                    *link = j;
+                }
+                return true;
             }
+        }
+    }
 
-            _FDT((fdt_setprop_cell(fdt, offset, "phandle",
-                                   PHANDLE_PCIDEV(sphb, dev))));
-            _FDT(fdt_setprop_cell(fdt, offset, "ibm,gpu",
-                                  PHANDLE_PCIDEV(sphb, nvslot->gpdev)));
-            _FDT((fdt_setprop_cell(fdt, offset, "ibm,nvlink",
-                                   PHANDLE_NVLINK(sphb, i, j))));
-            /*
-             * If we ever want to emulate GPU RAM at the same location as on
-             * the host - here is the encoding GPA->TGT:
-             *
-             * gta  = ((sphb->nv2_gpa >> 42) & 0x1) << 42;
-             * gta |= ((sphb->nv2_gpa >> 45) & 0x3) << 43;
-             * gta |= ((sphb->nv2_gpa >> 49) & 0x3) << 45;
-             * gta |= sphb->nv2_gpa & ((1UL << 43) - 1);
-             */
-            _FDT(fdt_setprop_cell(fdt, offset, "memory-region",
-                                  PHANDLE_GPURAM(sphb, i)));
-            _FDT(fdt_setprop_u64(fdt, offset, "ibm,device-tgt-addr",
-                                 nvslot->tgt));
-            _FDT(fdt_setprop_cell(fdt, offset, "ibm,nvlink-speed",
-                                  nvslot->links[j].link_speed));
+    return false;
+}
+
+void spapr_phb_nvgpu_populate_pcidev_dt(PCIDevice *dev, void *fdt, int offset,
+                                        SpaprPhbState *sphb)
+{
+    int slot, link;
+
+    if (is_nvgpu(dev, sphb, &slot)) {
+        SpaprPhbPciNvGpuSlot *nvslot = &sphb->nvgpus->slots[slot];
+        uint32_t npus[nvslot->linknum];
+
+        for (link = 0; link < nvslot->linknum; ++link) {
+            PCIDevice *npdev = nvslot->links[link].npdev;
+
+            npus[link] = cpu_to_be32(PHANDLE_PCIDEV(sphb, npdev));
         }
+        _FDT(fdt_setprop(fdt, offset, "ibm,npu", npus,
+                         link * sizeof(npus[0])));
+        _FDT((fdt_setprop_cell(fdt, offset, "phandle",
+                               PHANDLE_PCIDEV(sphb, dev))));
+    } else if (is_nvnpu(dev, sphb, &slot, &link)) {
+        SpaprPhbPciNvGpuSlot *nvslot = &sphb->nvgpus->slots[slot];
+
+        _FDT((fdt_setprop_cell(fdt, offset, "phandle",
+                               PHANDLE_PCIDEV(sphb, dev))));
+        _FDT(fdt_setprop_cell(fdt, offset, "ibm,gpu",
+                              PHANDLE_PCIDEV(sphb, nvslot->gpdev)));
+        _FDT((fdt_setprop_cell(fdt, offset, "ibm,nvlink",
+                               PHANDLE_NVLINK(sphb, slot, link))));
+        /*
+         * If we ever want to emulate GPU RAM at the same location as
+         * on the host - here is the encoding GPA->TGT:
+         *
+         * gta  = ((sphb->nv2_gpa >> 42) & 0x1) << 42;
+         * gta |= ((sphb->nv2_gpa >> 45) & 0x3) << 43;
+         * gta |= ((sphb->nv2_gpa >> 49) & 0x3) << 45;
+         * gta |= sphb->nv2_gpa & ((1UL << 43) - 1);
+         */
+        _FDT(fdt_setprop_cell(fdt, offset, "memory-region",
+                              PHANDLE_GPURAM(sphb, slot)));
+        _FDT(fdt_setprop_u64(fdt, offset, "ibm,device-tgt-addr",
+                             nvslot->tgt));
+        _FDT(fdt_setprop_cell(fdt, offset, "ibm,nvlink-speed",
+                              nvslot->links[link].link_speed));
     }
 }
-- 
2.25.1
Re: [RFC for-5.1 1/4] spapr: Refactor locating NVLink2 devices for device tree creation
Posted by Greg Kurz 5 years, 10 months ago 
On Thu, 26 Mar 2020 16:40:06 +1100
David Gibson <david@gibson.dropbear.id.au> wrote:

> Currently spapr_phb_nvgpu_populate_pcidev_dt() works a little cryptically.
> It steps through all the NVLink2 GPUs and NPUs and if they match the device
> we're called for, we generate the relevant device tree information.
> 
> Make this a little more obvious by introducing helpers to determine it a

... to determine if a

> given PCI device is an NVLink2 GPU or NPU, returning the NVLink2 slot and
> link number information as well.
> 
> Signed-off-by: David Gibson <david@gibson.dropbear.id.au>
> ---

LGTM

Reviewed-by: Greg Kurz <groug@kaod.org>

>  hw/ppc/spapr_pci_nvlink2.c | 115 +++++++++++++++++++++++++------------
>  1 file changed, 79 insertions(+), 36 deletions(-)
> 
> diff --git a/hw/ppc/spapr_pci_nvlink2.c b/hw/ppc/spapr_pci_nvlink2.c
> index 8332d5694e..7d3a685421 100644
> --- a/hw/ppc/spapr_pci_nvlink2.c
> +++ b/hw/ppc/spapr_pci_nvlink2.c
> @@ -390,13 +390,12 @@ void spapr_phb_nvgpu_ram_populate_dt(SpaprPhbState *sphb, void *fdt)
>  
>  }
>  
> -void spapr_phb_nvgpu_populate_pcidev_dt(PCIDevice *dev, void *fdt, int offset,
> -                                        SpaprPhbState *sphb)
> +static bool is_nvgpu(PCIDevice *dev, SpaprPhbState *sphb, int *slot)
>  {
> -    int i, j;
> +    int i;
>  
>      if (!sphb->nvgpus) {
> -        return;
> +        return false;
>      }
>  
>      for (i = 0; i < sphb->nvgpus->num; ++i) {
> @@ -406,47 +405,91 @@ void spapr_phb_nvgpu_populate_pcidev_dt(PCIDevice *dev, void *fdt, int offset,
>          if (!nvslot->gpdev) {
>              continue;
>          }
> +
>          if (dev == nvslot->gpdev) {
> -            uint32_t npus[nvslot->linknum];
> +            if (slot) {
> +                *slot = i;
> +            }
> +            return true;
> +        }
> +    }
>  
> -            for (j = 0; j < nvslot->linknum; ++j) {
> -                PCIDevice *npdev = nvslot->links[j].npdev;
> +    return false;
> +}
>  
> -                npus[j] = cpu_to_be32(PHANDLE_PCIDEV(sphb, npdev));
> -            }
> -            _FDT(fdt_setprop(fdt, offset, "ibm,npu", npus,
> -                             j * sizeof(npus[0])));
> -            _FDT((fdt_setprop_cell(fdt, offset, "phandle",
> -                                   PHANDLE_PCIDEV(sphb, dev))));
> +static bool is_nvnpu(PCIDevice *dev, SpaprPhbState *sphb, int *slot, int *link)
> +{
> +    int i, j;
> +
> +    if (!sphb->nvgpus) {
> +        return false;
> +    }
> +
> +    for (i = 0; i < sphb->nvgpus->num; ++i) {
> +        SpaprPhbPciNvGpuSlot *nvslot = &sphb->nvgpus->slots[i];
> +
> +        /* Skip "slot" without attached GPU */
> +        if (!nvslot->gpdev) {
>              continue;
>          }
>  
>          for (j = 0; j < nvslot->linknum; ++j) {
> -            if (dev != nvslot->links[j].npdev) {
> -                continue;
> +            if (dev == nvslot->links[j].npdev) {
> +                if (slot) {
> +                    *slot = i;
> +                }
> +                if (link) {
> +                    *link = j;
> +                }
> +                return true;
>              }
> +        }
> +    }
>  
> -            _FDT((fdt_setprop_cell(fdt, offset, "phandle",
> -                                   PHANDLE_PCIDEV(sphb, dev))));
> -            _FDT(fdt_setprop_cell(fdt, offset, "ibm,gpu",
> -                                  PHANDLE_PCIDEV(sphb, nvslot->gpdev)));
> -            _FDT((fdt_setprop_cell(fdt, offset, "ibm,nvlink",
> -                                   PHANDLE_NVLINK(sphb, i, j))));
> -            /*
> -             * If we ever want to emulate GPU RAM at the same location as on
> -             * the host - here is the encoding GPA->TGT:
> -             *
> -             * gta  = ((sphb->nv2_gpa >> 42) & 0x1) << 42;
> -             * gta |= ((sphb->nv2_gpa >> 45) & 0x3) << 43;
> -             * gta |= ((sphb->nv2_gpa >> 49) & 0x3) << 45;
> -             * gta |= sphb->nv2_gpa & ((1UL << 43) - 1);
> -             */
> -            _FDT(fdt_setprop_cell(fdt, offset, "memory-region",
> -                                  PHANDLE_GPURAM(sphb, i)));
> -            _FDT(fdt_setprop_u64(fdt, offset, "ibm,device-tgt-addr",
> -                                 nvslot->tgt));
> -            _FDT(fdt_setprop_cell(fdt, offset, "ibm,nvlink-speed",
> -                                  nvslot->links[j].link_speed));
> +    return false;
> +}
> +
> +void spapr_phb_nvgpu_populate_pcidev_dt(PCIDevice *dev, void *fdt, int offset,
> +                                        SpaprPhbState *sphb)
> +{
> +    int slot, link;
> +
> +    if (is_nvgpu(dev, sphb, &slot)) {
> +        SpaprPhbPciNvGpuSlot *nvslot = &sphb->nvgpus->slots[slot];
> +        uint32_t npus[nvslot->linknum];
> +
> +        for (link = 0; link < nvslot->linknum; ++link) {
> +            PCIDevice *npdev = nvslot->links[link].npdev;
> +
> +            npus[link] = cpu_to_be32(PHANDLE_PCIDEV(sphb, npdev));
>          }
> +        _FDT(fdt_setprop(fdt, offset, "ibm,npu", npus,
> +                         link * sizeof(npus[0])));
> +        _FDT((fdt_setprop_cell(fdt, offset, "phandle",
> +                               PHANDLE_PCIDEV(sphb, dev))));
> +    } else if (is_nvnpu(dev, sphb, &slot, &link)) {
> +        SpaprPhbPciNvGpuSlot *nvslot = &sphb->nvgpus->slots[slot];
> +
> +        _FDT((fdt_setprop_cell(fdt, offset, "phandle",
> +                               PHANDLE_PCIDEV(sphb, dev))));
> +        _FDT(fdt_setprop_cell(fdt, offset, "ibm,gpu",
> +                              PHANDLE_PCIDEV(sphb, nvslot->gpdev)));
> +        _FDT((fdt_setprop_cell(fdt, offset, "ibm,nvlink",
> +                               PHANDLE_NVLINK(sphb, slot, link))));
> +        /*
> +         * If we ever want to emulate GPU RAM at the same location as
> +         * on the host - here is the encoding GPA->TGT:
> +         *
> +         * gta  = ((sphb->nv2_gpa >> 42) & 0x1) << 42;
> +         * gta |= ((sphb->nv2_gpa >> 45) & 0x3) << 43;
> +         * gta |= ((sphb->nv2_gpa >> 49) & 0x3) << 45;
> +         * gta |= sphb->nv2_gpa & ((1UL << 43) - 1);
> +         */
> +        _FDT(fdt_setprop_cell(fdt, offset, "memory-region",
> +                              PHANDLE_GPURAM(sphb, slot)));
> +        _FDT(fdt_setprop_u64(fdt, offset, "ibm,device-tgt-addr",
> +                             nvslot->tgt));
> +        _FDT(fdt_setprop_cell(fdt, offset, "ibm,nvlink-speed",
> +                              nvslot->links[link].link_speed));
>      }
>  }
Re: [RFC for-5.1 1/4] spapr: Refactor locating NVLink2 devices for device tree creation
Posted by David Gibson 5 years, 10 months ago 
On Thu, Mar 26, 2020 at 12:57:38PM +0100, Greg Kurz wrote:
65;5803;1c> On Thu, 26 Mar 2020 16:40:06 +1100
> David Gibson <david@gibson.dropbear.id.au> wrote:
> 
> > Currently spapr_phb_nvgpu_populate_pcidev_dt() works a little cryptically.
> > It steps through all the NVLink2 GPUs and NPUs and if they match the device
> > we're called for, we generate the relevant device tree information.
> > 
> > Make this a little more obvious by introducing helpers to determine it a
> 
> ... to determine if a

Fixed, thanks.

> 
> > given PCI device is an NVLink2 GPU or NPU, returning the NVLink2 slot and
> > link number information as well.
> > 
> > Signed-off-by: David Gibson <david@gibson.dropbear.id.au>
> > ---
> 
> LGTM
> 
> Reviewed-by: Greg Kurz <groug@kaod.org>
> 
> >  hw/ppc/spapr_pci_nvlink2.c | 115 +++++++++++++++++++++++++------------
> >  1 file changed, 79 insertions(+), 36 deletions(-)
> > 
> > diff --git a/hw/ppc/spapr_pci_nvlink2.c b/hw/ppc/spapr_pci_nvlink2.c
> > index 8332d5694e..7d3a685421 100644
> > --- a/hw/ppc/spapr_pci_nvlink2.c
> > +++ b/hw/ppc/spapr_pci_nvlink2.c
> > @@ -390,13 +390,12 @@ void spapr_phb_nvgpu_ram_populate_dt(SpaprPhbState *sphb, void *fdt)
> >  
> >  }
> >  
> > -void spapr_phb_nvgpu_populate_pcidev_dt(PCIDevice *dev, void *fdt, int offset,
> > -                                        SpaprPhbState *sphb)
> > +static bool is_nvgpu(PCIDevice *dev, SpaprPhbState *sphb, int *slot)
> >  {
> > -    int i, j;
> > +    int i;
> >  
> >      if (!sphb->nvgpus) {
> > -        return;
> > +        return false;
> >      }
> >  
> >      for (i = 0; i < sphb->nvgpus->num; ++i) {
> > @@ -406,47 +405,91 @@ void spapr_phb_nvgpu_populate_pcidev_dt(PCIDevice *dev, void *fdt, int offset,
> >          if (!nvslot->gpdev) {
> >              continue;
> >          }
> > +
> >          if (dev == nvslot->gpdev) {
> > -            uint32_t npus[nvslot->linknum];
> > +            if (slot) {
> > +                *slot = i;
> > +            }
> > +            return true;
> > +        }
> > +    }
> >  
> > -            for (j = 0; j < nvslot->linknum; ++j) {
> > -                PCIDevice *npdev = nvslot->links[j].npdev;
> > +    return false;
> > +}
> >  
> > -                npus[j] = cpu_to_be32(PHANDLE_PCIDEV(sphb, npdev));
> > -            }
> > -            _FDT(fdt_setprop(fdt, offset, "ibm,npu", npus,
> > -                             j * sizeof(npus[0])));
> > -            _FDT((fdt_setprop_cell(fdt, offset, "phandle",
> > -                                   PHANDLE_PCIDEV(sphb, dev))));
> > +static bool is_nvnpu(PCIDevice *dev, SpaprPhbState *sphb, int *slot, int *link)
> > +{
> > +    int i, j;
> > +
> > +    if (!sphb->nvgpus) {
> > +        return false;
> > +    }
> > +
> > +    for (i = 0; i < sphb->nvgpus->num; ++i) {
> > +        SpaprPhbPciNvGpuSlot *nvslot = &sphb->nvgpus->slots[i];
> > +
> > +        /* Skip "slot" without attached GPU */
> > +        if (!nvslot->gpdev) {
> >              continue;
> >          }
> >  
> >          for (j = 0; j < nvslot->linknum; ++j) {
> > -            if (dev != nvslot->links[j].npdev) {
> > -                continue;
> > +            if (dev == nvslot->links[j].npdev) {
> > +                if (slot) {
> > +                    *slot = i;
> > +                }
> > +                if (link) {
> > +                    *link = j;
> > +                }
> > +                return true;
> >              }
> > +        }
> > +    }
> >  
> > -            _FDT((fdt_setprop_cell(fdt, offset, "phandle",
> > -                                   PHANDLE_PCIDEV(sphb, dev))));
> > -            _FDT(fdt_setprop_cell(fdt, offset, "ibm,gpu",
> > -                                  PHANDLE_PCIDEV(sphb, nvslot->gpdev)));
> > -            _FDT((fdt_setprop_cell(fdt, offset, "ibm,nvlink",
> > -                                   PHANDLE_NVLINK(sphb, i, j))));
> > -            /*
> > -             * If we ever want to emulate GPU RAM at the same location as on
> > -             * the host - here is the encoding GPA->TGT:
> > -             *
> > -             * gta  = ((sphb->nv2_gpa >> 42) & 0x1) << 42;
> > -             * gta |= ((sphb->nv2_gpa >> 45) & 0x3) << 43;
> > -             * gta |= ((sphb->nv2_gpa >> 49) & 0x3) << 45;
> > -             * gta |= sphb->nv2_gpa & ((1UL << 43) - 1);
> > -             */
> > -            _FDT(fdt_setprop_cell(fdt, offset, "memory-region",
> > -                                  PHANDLE_GPURAM(sphb, i)));
> > -            _FDT(fdt_setprop_u64(fdt, offset, "ibm,device-tgt-addr",
> > -                                 nvslot->tgt));
> > -            _FDT(fdt_setprop_cell(fdt, offset, "ibm,nvlink-speed",
> > -                                  nvslot->links[j].link_speed));
> > +    return false;
> > +}
> > +
> > +void spapr_phb_nvgpu_populate_pcidev_dt(PCIDevice *dev, void *fdt, int offset,
> > +                                        SpaprPhbState *sphb)
> > +{
> > +    int slot, link;
> > +
> > +    if (is_nvgpu(dev, sphb, &slot)) {
> > +        SpaprPhbPciNvGpuSlot *nvslot = &sphb->nvgpus->slots[slot];
> > +        uint32_t npus[nvslot->linknum];
> > +
> > +        for (link = 0; link < nvslot->linknum; ++link) {
> > +            PCIDevice *npdev = nvslot->links[link].npdev;
> > +
> > +            npus[link] = cpu_to_be32(PHANDLE_PCIDEV(sphb, npdev));
> >          }
> > +        _FDT(fdt_setprop(fdt, offset, "ibm,npu", npus,
> > +                         link * sizeof(npus[0])));
> > +        _FDT((fdt_setprop_cell(fdt, offset, "phandle",
> > +                               PHANDLE_PCIDEV(sphb, dev))));
> > +    } else if (is_nvnpu(dev, sphb, &slot, &link)) {
> > +        SpaprPhbPciNvGpuSlot *nvslot = &sphb->nvgpus->slots[slot];
> > +
> > +        _FDT((fdt_setprop_cell(fdt, offset, "phandle",
> > +                               PHANDLE_PCIDEV(sphb, dev))));
> > +        _FDT(fdt_setprop_cell(fdt, offset, "ibm,gpu",
> > +                              PHANDLE_PCIDEV(sphb, nvslot->gpdev)));
> > +        _FDT((fdt_setprop_cell(fdt, offset, "ibm,nvlink",
> > +                               PHANDLE_NVLINK(sphb, slot, link))));
> > +        /*
> > +         * If we ever want to emulate GPU RAM at the same location as
> > +         * on the host - here is the encoding GPA->TGT:
> > +         *
> > +         * gta  = ((sphb->nv2_gpa >> 42) & 0x1) << 42;
> > +         * gta |= ((sphb->nv2_gpa >> 45) & 0x3) << 43;
> > +         * gta |= ((sphb->nv2_gpa >> 49) & 0x3) << 45;
> > +         * gta |= sphb->nv2_gpa & ((1UL << 43) - 1);
> > +         */
> > +        _FDT(fdt_setprop_cell(fdt, offset, "memory-region",
> > +                              PHANDLE_GPURAM(sphb, slot)));
> > +        _FDT(fdt_setprop_u64(fdt, offset, "ibm,device-tgt-addr",
> > +                             nvslot->tgt));
> > +        _FDT(fdt_setprop_cell(fdt, offset, "ibm,nvlink-speed",
> > +                              nvslot->links[link].link_speed));
> >      }
> >  }
> 

-- 
David Gibson			| I'll have my music baroque, and my code
david AT gibson.dropbear.id.au	| minimalist, thank you.  NOT _the_ _other_
				| _way_ _around_!
http://www.ozlabs.org/~dgibson

Patchew 2.1-devel-b67e4c2

© 2016 - 2026 Red Hat, Inc.