Similar to regular resizable BAR, VF BAR can also be resized, e.g. by

the system firmware or the PCI subsystem itself.

The capability layout is the same as PCI_EXT_CAP_ID_REBAR.

Add the capability ID and restore it as a part of IOV state.

See PCIe r6.2, sec 7.8.7.

Signed-off-by: Michał Winiarski <michal.winiarski@intel.com>

Reviewed-by: Ilpo Järvinen <ilpo.jarvinen@linux.intel.com>

Reviewed-by: Christian König <christian.koenig@amd.com>

---

drivers/pci/iov.c | 30 +++++++++++++++++++++++++++++-

drivers/pci/pci.h | 1 +

include/uapi/linux/pci_regs.h | 9 +++++++++

3 files changed, 39 insertions(+), 1 deletion(-)

diff --git a/drivers/pci/iov.c b/drivers/pci/iov.c

index XXXXXXX..XXXXXXX 100644

--- a/drivers/pci/iov.c

+++ b/drivers/pci/iov.c

@@ -XXX,XX +XXX,XX @@

* Copyright (C) 2009 Intel Corporation, Yu Zhao <yu.zhao@intel.com>

*/

+#include <linux/bitfield.h>

#include <linux/pci.h>

#include <linux/slab.h>

#include <linux/export.h>

@@ -XXX,XX +XXX,XX @@ static int sriov_init(struct pci_dev *dev, int pos)

    pci_read_config_byte(dev, pos + PCI_SRIOV_FUNC_LINK, &iov->link);

    if (pci_pcie_type(dev) == PCI_EXP_TYPE_RC_END)

        iov->link = PCI_DEVFN(PCI_SLOT(dev->devfn), iov->link);

+    iov->vf_rebar_cap = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_VF_REBAR);

    if (pdev)

        iov->dev = pci_dev_get(pdev);

@@ -XXX,XX +XXX,XX @@ static void sriov_release(struct pci_dev *dev)

    dev->sriov = NULL;

}

+static void sriov_restore_vf_rebar_state(struct pci_dev *dev)

+{

+    unsigned int pos, nbars, i;

+    u32 ctrl;

+

+    pos = dev->sriov->vf_rebar_cap;

+    if (!pos)

+        return;

+

+    pci_read_config_dword(dev, pos + PCI_VF_REBAR_CTRL, &ctrl);

+    nbars = FIELD_GET(PCI_VF_REBAR_CTRL_NBAR_MASK, ctrl);

+

+    for (i = 0; i < nbars; i++, pos += 8) {

+        int bar_idx, size;

+

+        pci_read_config_dword(dev, pos + PCI_VF_REBAR_CTRL, &ctrl);

+        bar_idx = FIELD_GET(PCI_VF_REBAR_CTRL_BAR_IDX, ctrl);

+        size = pci_rebar_bytes_to_size(dev->sriov->barsz[bar_idx]);

+        ctrl &= ~PCI_VF_REBAR_CTRL_BAR_SIZE;

+        ctrl |= FIELD_PREP(PCI_VF_REBAR_CTRL_BAR_SIZE, size);

+        pci_write_config_dword(dev, pos + PCI_VF_REBAR_CTRL, ctrl);

+    }

+}

+

static void sriov_restore_state(struct pci_dev *dev)

{

    int i;

@@ -XXX,XX +XXX,XX @@ resource_size_t pci_sriov_resource_alignment(struct pci_dev *dev, int resno)

*/

void pci_restore_iov_state(struct pci_dev *dev)

{

-    if (dev->is_physfn)

+    if (dev->is_physfn) {

+        sriov_restore_vf_rebar_state(dev);

        sriov_restore_state(dev);

+    }

}

/**

diff --git a/drivers/pci/pci.h b/drivers/pci/pci.h

index XXXXXXX..XXXXXXX 100644

--- a/drivers/pci/pci.h

+++ b/drivers/pci/pci.h

@@ -XXX,XX +XXX,XX @@ struct pci_sriov {

    u16        subsystem_vendor; /* VF subsystem vendor */

    u16        subsystem_device; /* VF subsystem device */

    resource_size_t    barsz[PCI_SRIOV_NUM_BARS];    /* VF BAR size */

+    u16        vf_rebar_cap;    /* VF Resizable BAR capability offset */

    bool        drivers_autoprobe; /* Auto probing of VFs by driver */

};

diff --git a/include/uapi/linux/pci_regs.h b/include/uapi/linux/pci_regs.h

index XXXXXXX..XXXXXXX 100644

--- a/include/uapi/linux/pci_regs.h

+++ b/include/uapi/linux/pci_regs.h

@@ -XXX,XX +XXX,XX @@

#define PCI_EXT_CAP_ID_L1SS    0x1E    /* L1 PM Substates */

#define PCI_EXT_CAP_ID_PTM    0x1F    /* Precision Time Measurement */

#define PCI_EXT_CAP_ID_DVSEC    0x23    /* Designated Vendor-Specific */

+#define PCI_EXT_CAP_ID_VF_REBAR 0x24    /* VF Resizable BAR */

#define PCI_EXT_CAP_ID_DLF    0x25    /* Data Link Feature */

#define PCI_EXT_CAP_ID_PL_16GT    0x26    /* Physical Layer 16.0 GT/s */

#define PCI_EXT_CAP_ID_NPEM    0x29    /* Native PCIe Enclosure Management */

@@ -XXX,XX +XXX,XX @@

#define PCI_DVSEC_HEADER2        0x8 /* Designated Vendor-Specific Header2 */

#define PCI_DVSEC_HEADER2_ID(x)        ((x) & 0xffff)

+/* VF Resizable BARs, same layout as PCI_REBAR */

+#define PCI_VF_REBAR_CAP    PCI_REBAR_CAP

+#define PCI_VF_REBAR_CAP_SIZES        PCI_REBAR_CAP_SIZES

+#define PCI_VF_REBAR_CTRL    PCI_REBAR_CTRL

+#define PCI_VF_REBAR_CTRL_BAR_IDX    PCI_REBAR_CTRL_BAR_IDX

+#define PCI_VF_REBAR_CTRL_NBAR_MASK    PCI_REBAR_CTRL_NBAR_MASK

+#define PCI_VF_REBAR_CTRL_BAR_SIZE    PCI_REBAR_CTRL_BAR_SIZE

+

/* Data Link Feature */

#define PCI_DLF_CAP        0x04    /* Capabilities Register */

#define PCI_DLF_EXCHANGE_ENABLE    0x80000000 /* Data Link Feature Exchange Enable */

--

2.49.0

There are multiple places where conversions between IOV resources and

corresponding VF BAR numbers are done.

Extract the logic to pci_resource_num_from_vf_bar() and

pci_resource_num_to_vf_bar() helpers.

Suggested-by: Ilpo Järvinen <ilpo.jarvinen@linux.intel.com>

Signed-off-by: Michał Winiarski <michal.winiarski@intel.com>

Acked-by: Christian König <christian.koenig@amd.com>

---

drivers/pci/iov.c | 26 ++++++++++++++++----------

drivers/pci/pci.h | 19 +++++++++++++++++++

drivers/pci/setup-bus.c | 3 ++-

3 files changed, 37 insertions(+), 11 deletions(-)

diff --git a/drivers/pci/iov.c b/drivers/pci/iov.c

index XXXXXXX..XXXXXXX 100644

--- a/drivers/pci/iov.c

+++ b/drivers/pci/iov.c

@@ -XXX,XX +XXX,XX @@ resource_size_t pci_iov_resource_size(struct pci_dev *dev, int resno)

    if (!dev->is_physfn)

        return 0;

-    return dev->sriov->barsz[resno - PCI_IOV_RESOURCES];

+    return dev->sriov->barsz[pci_resource_num_to_vf_bar(resno)];

}

static void pci_read_vf_config_common(struct pci_dev *virtfn)

@@ -XXX,XX +XXX,XX @@ int pci_iov_add_virtfn(struct pci_dev *dev, int id)

    virtfn->multifunction = 0;

    for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) {

-        res = &dev->resource[i + PCI_IOV_RESOURCES];

+        int idx = pci_resource_num_from_vf_bar(i);

+

+        res = &dev->resource[idx];

        if (!res->parent)

            continue;

        virtfn->resource[i].name = pci_name(virtfn);

        virtfn->resource[i].flags = res->flags;

-        size = pci_iov_resource_size(dev, i + PCI_IOV_RESOURCES);

+        size = pci_iov_resource_size(dev, idx);

        resource_set_range(&virtfn->resource[i],

                 res->start + size * id, size);

        rc = request_resource(res, &virtfn->resource[i]);

@@ -XXX,XX +XXX,XX @@ static int sriov_enable(struct pci_dev *dev, int nr_virtfn)

    nres = 0;

    for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) {

-        bars |= (1 << (i + PCI_IOV_RESOURCES));

-        res = &dev->resource[i + PCI_IOV_RESOURCES];

+        int idx = pci_resource_num_from_vf_bar(i);

+

+        bars |= (1 << idx);

+        res = &dev->resource[idx];

        if (res->parent)

            nres++;

    }

@@ -XXX,XX +XXX,XX @@ static int sriov_init(struct pci_dev *dev, int pos)

    nres = 0;

    for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) {

-        res = &dev->resource[i + PCI_IOV_RESOURCES];

-        res_name = pci_resource_name(dev, i + PCI_IOV_RESOURCES);

+        int idx = pci_resource_num_from_vf_bar(i);

+

+        res = &dev->resource[idx];

+        res_name = pci_resource_name(dev, idx);

        /*

         * If it is already FIXED, don't change it, something

@@ -XXX,XX +XXX,XX @@ static int sriov_init(struct pci_dev *dev, int pos)

    dev->is_physfn = 0;

failed:

    for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) {

-        res = &dev->resource[i + PCI_IOV_RESOURCES];

+        res = &dev->resource[pci_resource_num_from_vf_bar(i)];

        res->flags = 0;

    }

@@ -XXX,XX +XXX,XX @@ static void sriov_restore_state(struct pci_dev *dev)

    pci_write_config_word(dev, iov->pos + PCI_SRIOV_CTRL, ctrl);

    for (i = 0; i < PCI_SRIOV_NUM_BARS; i++)

-        pci_update_resource(dev, i + PCI_IOV_RESOURCES);

+        pci_update_resource(dev, pci_resource_num_from_vf_bar(i));

    pci_write_config_dword(dev, iov->pos + PCI_SRIOV_SYS_PGSIZE, iov->pgsz);

    pci_iov_set_numvfs(dev, iov->num_VFs);

@@ -XXX,XX +XXX,XX @@ void pci_iov_update_resource(struct pci_dev *dev, int resno)

{

    struct pci_sriov *iov = dev->is_physfn ? dev->sriov : NULL;

    struct resource *res = pci_resource_n(dev, resno);

-    int vf_bar = resno - PCI_IOV_RESOURCES;

+    int vf_bar = pci_resource_num_to_vf_bar(resno);

    struct pci_bus_region region;

    u16 cmd;

    u32 new;

diff --git a/drivers/pci/pci.h b/drivers/pci/pci.h

index XXXXXXX..XXXXXXX 100644

--- a/drivers/pci/pci.h

+++ b/drivers/pci/pci.h

@@ -XXX,XX +XXX,XX @@ static inline bool pci_resource_is_iov(int resno)

{

    return resno >= PCI_IOV_RESOURCES && resno <= PCI_IOV_RESOURCE_END;

}

+static inline int pci_resource_num_from_vf_bar(int resno)

+{

+    return resno + PCI_IOV_RESOURCES;

+}

+

+static inline int pci_resource_num_to_vf_bar(int resno)

+{

+    return resno - PCI_IOV_RESOURCES;

+}

extern const struct attribute_group sriov_pf_dev_attr_group;

extern const struct attribute_group sriov_vf_dev_attr_group;

#else

@@ -XXX,XX +XXX,XX @@ static inline bool pci_resource_is_iov(int resno)

{

    return false;

}

+static inline int pci_resource_num_from_vf_bar(int resno)

+{

+    WARN_ON_ONCE(1);

+    return -ENODEV;

+}

+static inline int pci_resource_num_to_vf_bar(int resno)

+{

+    WARN_ON_ONCE(1);

+    return -ENODEV;

+}

#endif /* CONFIG_PCI_IOV */

#ifdef CONFIG_PCIE_TPH

diff --git a/drivers/pci/setup-bus.c b/drivers/pci/setup-bus.c

index XXXXXXX..XXXXXXX 100644

--- a/drivers/pci/setup-bus.c

+++ b/drivers/pci/setup-bus.c

@@ -XXX,XX +XXX,XX @@ static int iov_resources_unassigned(struct pci_dev *dev, void *data)

    bool *unassigned = data;

    for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) {

-        struct resource *r = &dev->resource[i + PCI_IOV_RESOURCES];

+        int idx = pci_resource_num_from_vf_bar(i);

+        struct resource *r = &dev->resource[idx];

        struct pci_bus_region region;

        /* Not assigned or rejected by kernel? */

--

2.49.0

    return resno >= PCI_IOV_RESOURCES && resno <= PCI_IOV_RESOURCE_END;

+

+    if (pci_resource_is_iov(resno))

+        return pci_iov_is_memory_decoding_enabled(dev);

When the resource representing VF MMIO BAR reservation is created, its

size is always large enough to accommodate the BAR of all SR-IOV Virtual

Functions that can potentially be created (total VFs). If for whatever

reason it's not possible to accommodate all VFs - the resource is not

assigned and no VFs can be created.

An upcoming change will allow VF BAR size to be modified by drivers at

a later point in time, which means that the check for resource

assignment is no longer sufficient.

Add an additional check that verifies that VF BAR for all enabled VFs

fits within the underlying reservation resource.

Signed-off-by: Michał Winiarski <michal.winiarski@intel.com>

---

drivers/pci/iov.c | 3 +++

1 file changed, 3 insertions(+)

diff --git a/drivers/pci/iov.c b/drivers/pci/iov.c

index XXXXXXX..XXXXXXX 100644

--- a/drivers/pci/iov.c

+++ b/drivers/pci/iov.c

@@ -XXX,XX +XXX,XX @@ static int sriov_enable(struct pci_dev *dev, int nr_virtfn)

    nres = 0;

    for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) {

        int idx = pci_resource_num_from_vf_bar(i);

+        resource_size_t vf_bar_sz = pci_iov_resource_size(dev, idx);

        bars |= (1 << idx);

        res = &dev->resource[idx];

+        if (vf_bar_sz * nr_virtfn > resource_size(res))

+            continue;

        if (res->parent)

            nres++;

    }

--

2.49.0

+static inline u32 pci_iov_vf_bar_get_sizes(struct pci_dev *dev, int resno, int num_vfs)

+{ return 0; }

Always try to set the largest possible BAR size that allows to fit the

number of enabled VFs and inform the user in case the resize attempt is

not successful.

Reviewed-by: Ilpo Järvinen <ilpo.jarvinen@linux.intel.com>

* Copyright © 2023-2024 Intel Corporation