1
Similar to regular BAR, drivers can use pci_resize_resource() to resize
1
Hi,
2
an IOV BAR to the desired size provided that is supported by the
3
hardware, which can be queried using pci_rebar_get_possible_sizes().
4
2
5
This feature is based on the fact that (default VF BAR size) * (supported
3
Another revision after the feedback from Ilpo.
6
VF number) covers all possible resource/address ranges (rounded up to the
4
Few tweaks here and there, the biggest change is the removal of the
7
power of 2 size). For example, the total size of the resource behind the
5
loop in pci_iov_vf_bar_get_sizes().
8
BAR is 256GB, the supported maximum VF number is 4, the default VF BAR
9
size should then be set to 64GB. When the enabled vf_num changes, the VF
10
BAR size will adjust accordingly as
11
- For 1 VF, VF BAR size is 256GB
12
- For 2 VFs, VF BAR size is 128GB
13
- For 4 VFs, VF BAR size is 64GB
14
6
15
This feature is necessary to accommodate the limited address per PCI port.
7
v6 can be found here:
8
https://lore.kernel.org/linux-pci/20250320110854.3866284-1-michal.winiarski@intel.com/
16
9
17
Lianjie Shi (1):
10
For regular BAR, drivers can use pci_resize_resource to resize it to the
18
PCI: Support VF resizable BAR
11
desired size provided that it is supported by the hardware, which the
12
driver can query using pci_rebar_get_possible_sizes.
13
This series expands the API to work with IOV BAR as well.
14
It also adds the additional API for drivers to change the VF BAR size
15
without resizing the entire underlying reservation (within the original
16
resource boundary).
19
17
20
drivers/pci/pci.c | 47 ++++++++++++++++++++++++++++++++++-
18
Thanks,
21
drivers/pci/setup-res.c | 45 +++++++++++++++++++++++++++------
19
-Michał
22
include/uapi/linux/pci_regs.h | 1 +
23
3 files changed, 85 insertions(+), 8 deletions(-)
24
20
21
v6 -> v7:
22
- Eliminate the loop in pci_iov_vf_bar_get_sizes() (Ilpo)
23
- Use helper variable for indexes (Ilpo)
24
- Kerneldoc formatting (Ilpo)
25
- Refer to latest PCI spec (Ilpo)
26
- Commit message wording (Ilpo)
25
27
26
base-commit: cf87f46fd34d6c19283d9625a7822f20d90b64a4
28
v5 -> v6:
29
- Rebased on latest pci/next
30
- Cache the VF resizable BAR capability position to avoid multiple
31
lookups (Ilpo)
32
- Use pci_resource_n helper (Ilpo)
33
34
v4 -> v5:
35
- Rename pci_resource_to/from_iov helpers and add WARN if called without
36
CONFIG_PCI_IOV (Ilpo)
37
- Reword kerneldoc for pci_iov_vf_bar_get_sizes (Bjorn)
38
- Reword commit message for VF BAR size check, extract the additional
39
size check to separate conditional (Bjorn)
40
41
v3 -> v4:
42
- Change the approach to extending the BAR (Christian)
43
- Tidy the commit messages, use 80 line limit where necessary (Bjorn)
44
- Add kerneldocs to exported functions (Bjorn)
45
- Add pci_resource_to_iov() / pci_resource_from_iov() helpers (Ilpo)
46
- Use FIELD_GET(), tidy whitespace (Ilpo)
47
48
v2 -> v3:
49
- Extract introducing pci_resource_is_iov to separate commit and
50
use it elsewhere in PCI subsystem (Christian)
51
- Extract restoring VF rebar state to separate commit (Christian)
52
- Reorganize memory decoding check (Christian)
53
- Don't use dev_WARN (Ilpo)
54
- Fix build without CONFIG_PCI_IOV (CI)
55
56
v1 -> v2:
57
- Add pci_iov_resource_extend() and usage in Xe driver
58
- Reduce the number of ifdefs (Christian)
59
- Drop patch 2/2 from v1 (Christian)
60
- Add a helper to avoid upsetting static analysis tools (Krzysztof)
61
62
Michał Winiarski (6):
63
PCI/IOV: Restore VF resizable BAR state after reset
64
PCI: Add a helper to convert between VF BAR number and IOV resource
65
PCI: Allow IOV resources to be resized in pci_resize_resource()
66
PCI/IOV: Check that VF BAR fits within the reservation
67
PCI: Allow drivers to control VF BAR size
68
drm/xe/pf: Set VF LMEM BAR size
69
70
drivers/gpu/drm/xe/regs/xe_bars.h | 1 +
71
drivers/gpu/drm/xe/xe_pci_sriov.c | 22 +++++
72
drivers/pci/iov.c | 149 +++++++++++++++++++++++++++---
73
drivers/pci/pci.c | 10 +-
74
drivers/pci/pci.h | 29 ++++++
75
drivers/pci/setup-bus.c | 3 +-
76
drivers/pci/setup-res.c | 35 ++++++-
77
include/linux/pci.h | 6 ++
78
include/uapi/linux/pci_regs.h | 9 ++
79
9 files changed, 246 insertions(+), 18 deletions(-)
80
27
--
81
--
28
2.34.1
82
2.49.0
83
diff view generated by jsdifflib
New patch
1
Similar to regular resizable BAR, VF BAR can also be resized, e.g. by
2
the system firmware or the PCI subsystem itself.
1
3
4
The capability layout is the same as PCI_EXT_CAP_ID_REBAR.
5
6
Add the capability ID and restore it as a part of IOV state.
7
8
See PCIe r6.2, sec 7.8.7.
9
10
Signed-off-by: Michał Winiarski <michal.winiarski@intel.com>
11
Reviewed-by: Ilpo Järvinen <ilpo.jarvinen@linux.intel.com>
12
Reviewed-by: Christian König <christian.koenig@amd.com>
13
---
14
drivers/pci/iov.c | 30 +++++++++++++++++++++++++++++-
15
drivers/pci/pci.h | 1 +
16
include/uapi/linux/pci_regs.h | 9 +++++++++
17
3 files changed, 39 insertions(+), 1 deletion(-)
18
19
diff --git a/drivers/pci/iov.c b/drivers/pci/iov.c
20
index XXXXXXX..XXXXXXX 100644
21
--- a/drivers/pci/iov.c
22
+++ b/drivers/pci/iov.c
23
@@ -XXX,XX +XXX,XX @@
24
* Copyright (C) 2009 Intel Corporation, Yu Zhao <yu.zhao@intel.com>
25
*/
26
27
+#include <linux/bitfield.h>
28
#include <linux/pci.h>
29
#include <linux/slab.h>
30
#include <linux/export.h>
31
@@ -XXX,XX +XXX,XX @@ static int sriov_init(struct pci_dev *dev, int pos)
32
    pci_read_config_byte(dev, pos + PCI_SRIOV_FUNC_LINK, &iov->link);
33
    if (pci_pcie_type(dev) == PCI_EXP_TYPE_RC_END)
34
        iov->link = PCI_DEVFN(PCI_SLOT(dev->devfn), iov->link);
35
+    iov->vf_rebar_cap = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_VF_REBAR);
36
37
    if (pdev)
38
        iov->dev = pci_dev_get(pdev);
39
@@ -XXX,XX +XXX,XX @@ static void sriov_release(struct pci_dev *dev)
40
    dev->sriov = NULL;
41
}
42
43
+static void sriov_restore_vf_rebar_state(struct pci_dev *dev)
44
+{
45
+    unsigned int pos, nbars, i;
46
+    u32 ctrl;
47
+
48
+    pos = dev->sriov->vf_rebar_cap;
49
+    if (!pos)
50
+        return;
51
+
52
+    pci_read_config_dword(dev, pos + PCI_VF_REBAR_CTRL, &ctrl);
53
+    nbars = FIELD_GET(PCI_VF_REBAR_CTRL_NBAR_MASK, ctrl);
54
+
55
+    for (i = 0; i < nbars; i++, pos += 8) {
56
+        int bar_idx, size;
57
+
58
+        pci_read_config_dword(dev, pos + PCI_VF_REBAR_CTRL, &ctrl);
59
+        bar_idx = FIELD_GET(PCI_VF_REBAR_CTRL_BAR_IDX, ctrl);
60
+        size = pci_rebar_bytes_to_size(dev->sriov->barsz[bar_idx]);
61
+        ctrl &= ~PCI_VF_REBAR_CTRL_BAR_SIZE;
62
+        ctrl |= FIELD_PREP(PCI_VF_REBAR_CTRL_BAR_SIZE, size);
63
+        pci_write_config_dword(dev, pos + PCI_VF_REBAR_CTRL, ctrl);
64
+    }
65
+}
66
+
67
static void sriov_restore_state(struct pci_dev *dev)
68
{
69
    int i;
70
@@ -XXX,XX +XXX,XX @@ resource_size_t pci_sriov_resource_alignment(struct pci_dev *dev, int resno)
71
*/
72
void pci_restore_iov_state(struct pci_dev *dev)
73
{
74
-    if (dev->is_physfn)
75
+    if (dev->is_physfn) {
76
+        sriov_restore_vf_rebar_state(dev);
77
        sriov_restore_state(dev);
78
+    }
79
}
80
81
/**
82
diff --git a/drivers/pci/pci.h b/drivers/pci/pci.h
83
index XXXXXXX..XXXXXXX 100644
84
--- a/drivers/pci/pci.h
85
+++ b/drivers/pci/pci.h
86
@@ -XXX,XX +XXX,XX @@ struct pci_sriov {
87
    u16        subsystem_vendor; /* VF subsystem vendor */
88
    u16        subsystem_device; /* VF subsystem device */
89
    resource_size_t    barsz[PCI_SRIOV_NUM_BARS];    /* VF BAR size */
90
+    u16        vf_rebar_cap;    /* VF Resizable BAR capability offset */
91
    bool        drivers_autoprobe; /* Auto probing of VFs by driver */
92
};
93
94
diff --git a/include/uapi/linux/pci_regs.h b/include/uapi/linux/pci_regs.h
95
index XXXXXXX..XXXXXXX 100644
96
--- a/include/uapi/linux/pci_regs.h
97
+++ b/include/uapi/linux/pci_regs.h
98
@@ -XXX,XX +XXX,XX @@
99
#define PCI_EXT_CAP_ID_L1SS    0x1E    /* L1 PM Substates */
100
#define PCI_EXT_CAP_ID_PTM    0x1F    /* Precision Time Measurement */
101
#define PCI_EXT_CAP_ID_DVSEC    0x23    /* Designated Vendor-Specific */
102
+#define PCI_EXT_CAP_ID_VF_REBAR 0x24    /* VF Resizable BAR */
103
#define PCI_EXT_CAP_ID_DLF    0x25    /* Data Link Feature */
104
#define PCI_EXT_CAP_ID_PL_16GT    0x26    /* Physical Layer 16.0 GT/s */
105
#define PCI_EXT_CAP_ID_NPEM    0x29    /* Native PCIe Enclosure Management */
106
@@ -XXX,XX +XXX,XX @@
107
#define PCI_DVSEC_HEADER2        0x8 /* Designated Vendor-Specific Header2 */
108
#define PCI_DVSEC_HEADER2_ID(x)        ((x) & 0xffff)
109
110
+/* VF Resizable BARs, same layout as PCI_REBAR */
111
+#define PCI_VF_REBAR_CAP    PCI_REBAR_CAP
112
+#define PCI_VF_REBAR_CAP_SIZES        PCI_REBAR_CAP_SIZES
113
+#define PCI_VF_REBAR_CTRL    PCI_REBAR_CTRL
114
+#define PCI_VF_REBAR_CTRL_BAR_IDX    PCI_REBAR_CTRL_BAR_IDX
115
+#define PCI_VF_REBAR_CTRL_NBAR_MASK    PCI_REBAR_CTRL_NBAR_MASK
116
+#define PCI_VF_REBAR_CTRL_BAR_SIZE    PCI_REBAR_CTRL_BAR_SIZE
117
+
118
/* Data Link Feature */
119
#define PCI_DLF_CAP        0x04    /* Capabilities Register */
120
#define PCI_DLF_EXCHANGE_ENABLE    0x80000000 /* Data Link Feature Exchange Enable */
121
--
122
2.49.0
123
diff view generated by jsdifflib
New patch
1
There are multiple places where conversions between IOV resources and
2
corresponding VF BAR numbers are done.
1
3
4
Extract the logic to pci_resource_num_from_vf_bar() and
5
pci_resource_num_to_vf_bar() helpers.
6
7
Suggested-by: Ilpo Järvinen <ilpo.jarvinen@linux.intel.com>
8
Signed-off-by: Michał Winiarski <michal.winiarski@intel.com>
9
Acked-by: Christian König <christian.koenig@amd.com>
10
---
11
drivers/pci/iov.c | 26 ++++++++++++++++----------
12
drivers/pci/pci.h | 19 +++++++++++++++++++
13
drivers/pci/setup-bus.c | 3 ++-
14
3 files changed, 37 insertions(+), 11 deletions(-)
15
16
diff --git a/drivers/pci/iov.c b/drivers/pci/iov.c
17
index XXXXXXX..XXXXXXX 100644
18
--- a/drivers/pci/iov.c
19
+++ b/drivers/pci/iov.c
20
@@ -XXX,XX +XXX,XX @@ resource_size_t pci_iov_resource_size(struct pci_dev *dev, int resno)
21
    if (!dev->is_physfn)
22
        return 0;
23
24
-    return dev->sriov->barsz[resno - PCI_IOV_RESOURCES];
25
+    return dev->sriov->barsz[pci_resource_num_to_vf_bar(resno)];
26
}
27
28
static void pci_read_vf_config_common(struct pci_dev *virtfn)
29
@@ -XXX,XX +XXX,XX @@ int pci_iov_add_virtfn(struct pci_dev *dev, int id)
30
    virtfn->multifunction = 0;
31
32
    for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) {
33
-        res = &dev->resource[i + PCI_IOV_RESOURCES];
34
+        int idx = pci_resource_num_from_vf_bar(i);
35
+
36
+        res = &dev->resource[idx];
37
        if (!res->parent)
38
            continue;
39
        virtfn->resource[i].name = pci_name(virtfn);
40
        virtfn->resource[i].flags = res->flags;
41
-        size = pci_iov_resource_size(dev, i + PCI_IOV_RESOURCES);
42
+        size = pci_iov_resource_size(dev, idx);
43
        resource_set_range(&virtfn->resource[i],
44
                 res->start + size * id, size);
45
        rc = request_resource(res, &virtfn->resource[i]);
46
@@ -XXX,XX +XXX,XX @@ static int sriov_enable(struct pci_dev *dev, int nr_virtfn)
47
48
    nres = 0;
49
    for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) {
50
-        bars |= (1 << (i + PCI_IOV_RESOURCES));
51
-        res = &dev->resource[i + PCI_IOV_RESOURCES];
52
+        int idx = pci_resource_num_from_vf_bar(i);
53
+
54
+        bars |= (1 << idx);
55
+        res = &dev->resource[idx];
56
        if (res->parent)
57
            nres++;
58
    }
59
@@ -XXX,XX +XXX,XX @@ static int sriov_init(struct pci_dev *dev, int pos)
60
61
    nres = 0;
62
    for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) {
63
-        res = &dev->resource[i + PCI_IOV_RESOURCES];
64
-        res_name = pci_resource_name(dev, i + PCI_IOV_RESOURCES);
65
+        int idx = pci_resource_num_from_vf_bar(i);
66
+
67
+        res = &dev->resource[idx];
68
+        res_name = pci_resource_name(dev, idx);
69
70
        /*
71
         * If it is already FIXED, don't change it, something
72
@@ -XXX,XX +XXX,XX @@ static int sriov_init(struct pci_dev *dev, int pos)
73
    dev->is_physfn = 0;
74
failed:
75
    for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) {
76
-        res = &dev->resource[i + PCI_IOV_RESOURCES];
77
+        res = &dev->resource[pci_resource_num_from_vf_bar(i)];
78
        res->flags = 0;
79
    }
80
81
@@ -XXX,XX +XXX,XX @@ static void sriov_restore_state(struct pci_dev *dev)
82
    pci_write_config_word(dev, iov->pos + PCI_SRIOV_CTRL, ctrl);
83
84
    for (i = 0; i < PCI_SRIOV_NUM_BARS; i++)
85
-        pci_update_resource(dev, i + PCI_IOV_RESOURCES);
86
+        pci_update_resource(dev, pci_resource_num_from_vf_bar(i));
87
88
    pci_write_config_dword(dev, iov->pos + PCI_SRIOV_SYS_PGSIZE, iov->pgsz);
89
    pci_iov_set_numvfs(dev, iov->num_VFs);
90
@@ -XXX,XX +XXX,XX @@ void pci_iov_update_resource(struct pci_dev *dev, int resno)
91
{
92
    struct pci_sriov *iov = dev->is_physfn ? dev->sriov : NULL;
93
    struct resource *res = pci_resource_n(dev, resno);
94
-    int vf_bar = resno - PCI_IOV_RESOURCES;
95
+    int vf_bar = pci_resource_num_to_vf_bar(resno);
96
    struct pci_bus_region region;
97
    u16 cmd;
98
    u32 new;
99
diff --git a/drivers/pci/pci.h b/drivers/pci/pci.h
100
index XXXXXXX..XXXXXXX 100644
101
--- a/drivers/pci/pci.h
102
+++ b/drivers/pci/pci.h
103
@@ -XXX,XX +XXX,XX @@ static inline bool pci_resource_is_iov(int resno)
104
{
105
    return resno >= PCI_IOV_RESOURCES && resno <= PCI_IOV_RESOURCE_END;
106
}
107
+static inline int pci_resource_num_from_vf_bar(int resno)
108
+{
109
+    return resno + PCI_IOV_RESOURCES;
110
+}
111
+
112
+static inline int pci_resource_num_to_vf_bar(int resno)
113
+{
114
+    return resno - PCI_IOV_RESOURCES;
115
+}
116
extern const struct attribute_group sriov_pf_dev_attr_group;
117
extern const struct attribute_group sriov_vf_dev_attr_group;
118
#else
119
@@ -XXX,XX +XXX,XX @@ static inline bool pci_resource_is_iov(int resno)
120
{
121
    return false;
122
}
123
+static inline int pci_resource_num_from_vf_bar(int resno)
124
+{
125
+    WARN_ON_ONCE(1);
126
+    return -ENODEV;
127
+}
128
+static inline int pci_resource_num_to_vf_bar(int resno)
129
+{
130
+    WARN_ON_ONCE(1);
131
+    return -ENODEV;
132
+}
133
#endif /* CONFIG_PCI_IOV */
134
135
#ifdef CONFIG_PCIE_TPH
136
diff --git a/drivers/pci/setup-bus.c b/drivers/pci/setup-bus.c
137
index XXXXXXX..XXXXXXX 100644
138
--- a/drivers/pci/setup-bus.c
139
+++ b/drivers/pci/setup-bus.c
140
@@ -XXX,XX +XXX,XX @@ static int iov_resources_unassigned(struct pci_dev *dev, void *data)
141
    bool *unassigned = data;
142
143
    for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) {
144
-        struct resource *r = &dev->resource[i + PCI_IOV_RESOURCES];
145
+        int idx = pci_resource_num_from_vf_bar(i);
146
+        struct resource *r = &dev->resource[idx];
147
        struct pci_bus_region region;
148
149
        /* Not assigned or rejected by kernel? */
150
--
151
2.49.0
152
diff view generated by jsdifflib
1
Add support for VF resizable BAR PCI extended cap.
1
Similar to regular resizable BAR, VF BAR can also be resized.
2
Similar to regular BAR, drivers can use pci_resize_resource() to
3
resize an IOV BAR. For each VF, dev->sriov->barsz of the IOV BAR is
4
resized, but the total resource size of the IOV resource should not
5
exceed its original size upon init.
6
2
7
Based on following patch series:
3
The capability layout is the same as PCI_EXT_CAP_ID_REBAR, which means
8
Link: https://lore.kernel.org/lkml/YbqGplTKl5i%2F1%2FkY@rocinante/T/
4
we can reuse most of the implementation, the only difference being
5
resource size calculation (which is multiplied by total VFs) and memory
6
decoding (which is controlled by a separate VF MSE field in SR-IOV cap).
9
7
10
Signed-off-by: Lianjie Shi <Lianjie.Shi@amd.com>
8
Extend the pci_resize_resource() function to accept IOV resources.
9
10
See PCIe r6.2, sec 7.8.7.
11
12
Signed-off-by: Michał Winiarski <michal.winiarski@intel.com>
11
---
13
---
12
drivers/pci/pci.c | 47 ++++++++++++++++++++++++++++++++++-
14
drivers/pci/iov.c | 21 +++++++++++++++++++++
13
drivers/pci/setup-res.c | 45 +++++++++++++++++++++++++++------
15
drivers/pci/pci.c | 10 +++++++++-
14
include/uapi/linux/pci_regs.h | 1 +
16
drivers/pci/pci.h | 9 +++++++++
15
3 files changed, 85 insertions(+), 8 deletions(-)
17
drivers/pci/setup-res.c | 35 ++++++++++++++++++++++++++++++-----
18
4 files changed, 69 insertions(+), 6 deletions(-)
16
19
20
diff --git a/drivers/pci/iov.c b/drivers/pci/iov.c
21
index XXXXXXX..XXXXXXX 100644
22
--- a/drivers/pci/iov.c
23
+++ b/drivers/pci/iov.c
24
@@ -XXX,XX +XXX,XX @@ resource_size_t pci_iov_resource_size(struct pci_dev *dev, int resno)
25
    return dev->sriov->barsz[pci_resource_num_to_vf_bar(resno)];
26
}
27
28
+void pci_iov_resource_set_size(struct pci_dev *dev, int resno,
29
+             resource_size_t size)
30
+{
31
+    if (!pci_resource_is_iov(resno)) {
32
+        pci_warn(dev, "%s is not an IOV resource\n",
33
+             pci_resource_name(dev, resno));
34
+        return;
35
+    }
36
+
37
+    dev->sriov->barsz[pci_resource_num_to_vf_bar(resno)] = size;
38
+}
39
+
40
+bool pci_iov_is_memory_decoding_enabled(struct pci_dev *dev)
41
+{
42
+    u16 cmd;
43
+
44
+    pci_read_config_word(dev, dev->sriov->pos + PCI_SRIOV_CTRL, &cmd);
45
+
46
+    return cmd & PCI_SRIOV_CTRL_MSE;
47
+}
48
+
49
static void pci_read_vf_config_common(struct pci_dev *virtfn)
50
{
51
    struct pci_dev *physfn = virtfn->physfn;
17
diff --git a/drivers/pci/pci.c b/drivers/pci/pci.c
52
diff --git a/drivers/pci/pci.c b/drivers/pci/pci.c
18
index XXXXXXX..XXXXXXX 100644
53
index XXXXXXX..XXXXXXX 100644
19
--- a/drivers/pci/pci.c
54
--- a/drivers/pci/pci.c
20
+++ b/drivers/pci/pci.c
55
+++ b/drivers/pci/pci.c
21
@@ -XXX,XX +XXX,XX @@ static void pci_restore_rebar_state(struct pci_dev *pdev)
56
@@ -XXX,XX +XXX,XX @@ static int pci_rebar_find_pos(struct pci_dev *pdev, int bar)
22
    }
23
}
24
25
+static void pci_restore_vf_rebar_state(struct pci_dev *pdev)
26
+{
27
+#ifdef CONFIG_PCI_IOV
28
+    unsigned int pos, nbars, i;
29
+    u32 ctrl;
30
+    u16 total;
31
+
32
+    pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_VF_REBAR);
33
+    if (!pos)
34
+        return;
35
+
36
+    pci_read_config_dword(pdev, pos + PCI_REBAR_CTRL, &ctrl);
37
+    nbars = FIELD_GET(PCI_REBAR_CTRL_NBAR_MASK, ctrl);
38
+
39
+    for (i = 0; i < nbars; i++, pos += 8) {
40
+        struct resource *res;
41
+        int bar_idx, size;
42
+        u64 tmp;
43
+
44
+        pci_read_config_dword(pdev, pos + PCI_REBAR_CTRL, &ctrl);
45
+        bar_idx = ctrl & PCI_REBAR_CTRL_BAR_IDX;
46
+        total = pdev->sriov->total_VFs;
47
+        if (!total)
48
+            return;
49
+
50
+        res = pdev->resource + bar_idx + PCI_IOV_RESOURCES;
51
+        tmp = resource_size(res);
52
+        do_div(tmp, total);
53
+        size = pci_rebar_bytes_to_size(tmp);
54
+        ctrl &= ~PCI_REBAR_CTRL_BAR_SIZE;
55
+        ctrl |= FIELD_PREP(PCI_REBAR_CTRL_BAR_SIZE, size);
56
+        pci_write_config_dword(pdev, pos + PCI_REBAR_CTRL, ctrl);
57
+    }
58
+#endif
59
+}
60
+
61
/**
62
* pci_restore_state - Restore the saved state of a PCI device
63
* @dev: PCI device that we're dealing with
64
@@ -XXX,XX +XXX,XX @@ void pci_restore_state(struct pci_dev *dev)
65
    pci_restore_ats_state(dev);
66
    pci_restore_vc_state(dev);
67
    pci_restore_rebar_state(dev);
68
+    pci_restore_vf_rebar_state(dev);
69
    pci_restore_dpc_state(dev);
70
    pci_restore_ptm_state(dev);
71
72
@@ -XXX,XX +XXX,XX @@ void pci_acs_init(struct pci_dev *dev)
73
*/
74
static int pci_rebar_find_pos(struct pci_dev *pdev, int bar)
75
{
76
+    int cap = PCI_EXT_CAP_ID_REBAR;
77
    unsigned int pos, nbars, i;
57
    unsigned int pos, nbars, i;
78
    u32 ctrl;
58
    u32 ctrl;
79
59
80
-    pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_REBAR);
60
-    pos = pdev->rebar_cap;
81
+#ifdef CONFIG_PCI_IOV
61
+    if (pci_resource_is_iov(bar)) {
82
+    if (bar >= PCI_IOV_RESOURCES) {
62
+        if (!pdev->physfn)
83
+        cap = PCI_EXT_CAP_ID_VF_REBAR;
63
+            return -ENOTSUPP;
84
+        bar -= PCI_IOV_RESOURCES;
64
+        pos = pdev->sriov->vf_rebar_cap;
65
+        bar = pci_resource_num_to_vf_bar(bar);
66
+    } else {
67
+        pos = pdev->rebar_cap;
85
+    }
68
+    }
86
+#endif
87
+
69
+
88
+    pos = pci_find_ext_capability(pdev, cap);
89
    if (!pos)
70
    if (!pos)
90
        return -ENOTSUPP;
71
        return -ENOTSUPP;
91
72
73
diff --git a/drivers/pci/pci.h b/drivers/pci/pci.h
74
index XXXXXXX..XXXXXXX 100644
75
--- a/drivers/pci/pci.h
76
+++ b/drivers/pci/pci.h
77
@@ -XXX,XX +XXX,XX @@ void pci_iov_update_resource(struct pci_dev *dev, int resno);
78
resource_size_t pci_sriov_resource_alignment(struct pci_dev *dev, int resno);
79
void pci_restore_iov_state(struct pci_dev *dev);
80
int pci_iov_bus_range(struct pci_bus *bus);
81
+void pci_iov_resource_set_size(struct pci_dev *dev, int resno,
82
+             resource_size_t size);
83
+bool pci_iov_is_memory_decoding_enabled(struct pci_dev *dev);
84
static inline bool pci_resource_is_iov(int resno)
85
{
86
    return resno >= PCI_IOV_RESOURCES && resno <= PCI_IOV_RESOURCE_END;
87
@@ -XXX,XX +XXX,XX @@ static inline int pci_iov_bus_range(struct pci_bus *bus)
88
{
89
    return 0;
90
}
91
+static inline void pci_iov_resource_set_size(struct pci_dev *dev, int resno,
92
+                     resource_size_t size) { }
93
+static inline bool pci_iov_is_memory_decoding_enabled(struct pci_dev *dev)
94
+{
95
+    return false;
96
+}
97
static inline bool pci_resource_is_iov(int resno)
98
{
99
    return false;
92
diff --git a/drivers/pci/setup-res.c b/drivers/pci/setup-res.c
100
diff --git a/drivers/pci/setup-res.c b/drivers/pci/setup-res.c
93
index XXXXXXX..XXXXXXX 100644
101
index XXXXXXX..XXXXXXX 100644
94
--- a/drivers/pci/setup-res.c
102
--- a/drivers/pci/setup-res.c
95
+++ b/drivers/pci/setup-res.c
103
+++ b/drivers/pci/setup-res.c
96
@@ -XXX,XX +XXX,XX @@ void pci_release_resource(struct pci_dev *dev, int resno)
104
@@ -XXX,XX +XXX,XX @@ void pci_release_resource(struct pci_dev *dev, int resno)
97
}
105
}
98
EXPORT_SYMBOL(pci_release_resource);
106
EXPORT_SYMBOL(pci_release_resource);
99
107
100
+static int pci_memory_decoding(struct pci_dev *dev, int resno)
108
+static bool pci_resize_is_memory_decoding_enabled(struct pci_dev *dev,
109
+                         int resno)
101
+{
110
+{
102
+    u16 cmd;
111
+    u16 cmd;
103
+
112
+
104
+#ifdef CONFIG_PCI_IOV
113
+    if (pci_resource_is_iov(resno))
105
+    if (resno >= PCI_IOV_RESOURCES) {
114
+        return pci_iov_is_memory_decoding_enabled(dev);
106
+        pci_read_config_word(dev, dev->sriov->pos + PCI_SRIOV_CTRL, &cmd);
115
+
107
+        if (cmd & PCI_SRIOV_CTRL_MSE)
116
+    pci_read_config_word(dev, PCI_COMMAND, &cmd);
108
+            return -EBUSY;
117
+
109
+        else
118
+    return cmd & PCI_COMMAND_MEMORY;
110
+            return 0;
119
+}
120
+
121
+static void pci_resize_resource_set_size(struct pci_dev *dev, int resno,
122
+                     int size)
123
+{
124
+    resource_size_t res_size = pci_rebar_size_to_bytes(size);
125
+    struct resource *res = pci_resource_n(dev, resno);
126
+
127
+    if (!pci_resource_is_iov(resno)) {
128
+        resource_set_size(res, res_size);
129
+    } else {
130
+        resource_set_size(res, res_size * pci_sriov_get_totalvfs(dev));
131
+        pci_iov_resource_set_size(dev, resno, res_size);
111
+    }
132
+    }
112
+#endif
113
+    pci_read_config_word(dev, PCI_COMMAND, &cmd);
114
+    if (cmd & PCI_COMMAND_MEMORY)
115
+        return -EBUSY;
116
+
117
+    return 0;
118
+}
133
+}
119
+
134
+
120
int pci_resize_resource(struct pci_dev *dev, int resno, int size)
135
int pci_resize_resource(struct pci_dev *dev, int resno, int size)
121
{
136
{
122
    struct resource *res = dev->resource + resno;
137
    struct resource *res = pci_resource_n(dev, resno);
123
    struct pci_host_bridge *host;
138
    struct pci_host_bridge *host;
124
    int old, ret;
139
    int old, ret;
125
    u32 sizes;
140
    u32 sizes;
126
-    u16 cmd;
141
-    u16 cmd;
127
142
...
...
131
    if (!(res->flags & IORESOURCE_UNSET))
146
    if (!(res->flags & IORESOURCE_UNSET))
132
        return -EBUSY;
147
        return -EBUSY;
133
148
134
-    pci_read_config_word(dev, PCI_COMMAND, &cmd);
149
-    pci_read_config_word(dev, PCI_COMMAND, &cmd);
135
-    if (cmd & PCI_COMMAND_MEMORY)
150
-    if (cmd & PCI_COMMAND_MEMORY)
136
-        return -EBUSY;
151
+    if (pci_resize_is_memory_decoding_enabled(dev, resno))
137
+    ret = pci_memory_decoding(dev, resno);
152
        return -EBUSY;
138
+    if (ret)
139
+        return ret;
140
153
141
    sizes = pci_rebar_get_possible_sizes(dev, resno);
154
    sizes = pci_rebar_get_possible_sizes(dev, resno);
142
    if (!sizes)
143
@@ -XXX,XX +XXX,XX @@ int pci_resize_resource(struct pci_dev *dev, int resno, int size)
155
@@ -XXX,XX +XXX,XX @@ int pci_resize_resource(struct pci_dev *dev, int resno, int size)
144
    if (ret)
156
    if (ret)
145
        return ret;
157
        return ret;
146
158
147
-    res->end = res->start + pci_rebar_size_to_bytes(size) - 1;
159
-    resource_set_size(res, pci_rebar_size_to_bytes(size));
148
+#ifdef CONFIG_PCI_IOV
160
+    pci_resize_resource_set_size(dev, resno, size);
149
+    if (resno >= PCI_IOV_RESOURCES)
150
+        dev->sriov->barsz[resno - PCI_IOV_RESOURCES] = pci_rebar_size_to_bytes(size);
151
+    else
152
+#endif
153
+        res->end = res->start + pci_rebar_size_to_bytes(size) - 1;
154
161
155
    /* Check if the new config works by trying to assign everything. */
162
    /* Check if the new config works by trying to assign everything. */
156
    if (dev->bus->self) {
163
    if (dev->bus->self) {
157
        ret = pci_reassign_bridge_resources(dev->bus->self, res->flags);
164
@@ -XXX,XX +XXX,XX @@ int pci_resize_resource(struct pci_dev *dev, int resno, int size)
158
-        if (ret)
159
+        if (ret && ret != -ENOENT)
160
            goto error_resize;
161
    }
162
    return 0;
163
165
164
error_resize:
166
error_resize:
165
    pci_rebar_set_size(dev, resno, old);
167
    pci_rebar_set_size(dev, resno, old);
166
-    res->end = res->start + pci_rebar_size_to_bytes(old) - 1;
168
-    resource_set_size(res, pci_rebar_size_to_bytes(old));
167
+
169
+    pci_resize_resource_set_size(dev, resno, old);
168
+#ifdef CONFIG_PCI_IOV
169
+    if (resno >= PCI_IOV_RESOURCES)
170
+        dev->sriov->barsz[resno - PCI_IOV_RESOURCES] = pci_rebar_size_to_bytes(old);
171
+    else
172
+#endif
173
+        res->end = res->start + pci_rebar_size_to_bytes(old) - 1;
174
+
175
    return ret;
170
    return ret;
176
}
171
}
177
EXPORT_SYMBOL(pci_resize_resource);
172
EXPORT_SYMBOL(pci_resize_resource);
178
diff --git a/include/uapi/linux/pci_regs.h b/include/uapi/linux/pci_regs.h
179
index XXXXXXX..XXXXXXX 100644
180
--- a/include/uapi/linux/pci_regs.h
181
+++ b/include/uapi/linux/pci_regs.h
182
@@ -XXX,XX +XXX,XX @@
183
#define PCI_EXT_CAP_ID_L1SS    0x1E    /* L1 PM Substates */
184
#define PCI_EXT_CAP_ID_PTM    0x1F    /* Precision Time Measurement */
185
#define PCI_EXT_CAP_ID_DVSEC    0x23    /* Designated Vendor-Specific */
186
+#define PCI_EXT_CAP_ID_VF_REBAR    0x24    /* VF Resizable BAR */
187
#define PCI_EXT_CAP_ID_DLF    0x25    /* Data Link Feature */
188
#define PCI_EXT_CAP_ID_PL_16GT    0x26    /* Physical Layer 16.0 GT/s */
189
#define PCI_EXT_CAP_ID_PL_32GT 0x2A /* Physical Layer 32.0 GT/s */
190
--
173
--
191
2.34.1
174
2.49.0
175
diff view generated by jsdifflib
New patch
1
When the resource representing VF MMIO BAR reservation is created, its
2
size is always large enough to accommodate the BAR of all SR-IOV Virtual
3
Functions that can potentially be created (total VFs). If for whatever
4
reason it's not possible to accommodate all VFs - the resource is not
5
assigned and no VFs can be created.
1
6
7
An upcoming change will allow VF BAR size to be modified by drivers at
8
a later point in time, which means that the check for resource
9
assignment is no longer sufficient.
10
11
Add an additional check that verifies that VF BAR for all enabled VFs
12
fits within the underlying reservation resource.
13
14
Signed-off-by: Michał Winiarski <michal.winiarski@intel.com>
15
---
16
drivers/pci/iov.c | 3 +++
17
1 file changed, 3 insertions(+)
18
19
diff --git a/drivers/pci/iov.c b/drivers/pci/iov.c
20
index XXXXXXX..XXXXXXX 100644
21
--- a/drivers/pci/iov.c
22
+++ b/drivers/pci/iov.c
23
@@ -XXX,XX +XXX,XX @@ static int sriov_enable(struct pci_dev *dev, int nr_virtfn)
24
    nres = 0;
25
    for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) {
26
        int idx = pci_resource_num_from_vf_bar(i);
27
+        resource_size_t vf_bar_sz = pci_iov_resource_size(dev, idx);
28
29
        bars |= (1 << idx);
30
        res = &dev->resource[idx];
31
+        if (vf_bar_sz * nr_virtfn > resource_size(res))
32
+            continue;
33
        if (res->parent)
34
            nres++;
35
    }
36
--
37
2.49.0
38
diff view generated by jsdifflib
New patch
1
Drivers could leverage the fact that the VF BAR MMIO reservation is
2
created for total number of VFs supported by the device by resizing the
3
BAR to larger size when smaller number of VFs is enabled.
1
4
5
Add a pci_iov_vf_bar_set_size() function to control the size and a
6
pci_iov_vf_bar_get_sizes() helper to get the VF BAR sizes that will
7
allow up to num_vfs to be successfully enabled with the current
8
underlying reservation size.
9
10
Signed-off-by: Michał Winiarski <michal.winiarski@intel.com>
11
---
12
drivers/pci/iov.c | 69 +++++++++++++++++++++++++++++++++++++++++++++
13
include/linux/pci.h | 6 ++++
14
2 files changed, 75 insertions(+)
15
16
diff --git a/drivers/pci/iov.c b/drivers/pci/iov.c
17
index XXXXXXX..XXXXXXX 100644
18
--- a/drivers/pci/iov.c
19
+++ b/drivers/pci/iov.c
20
@@ -XXX,XX +XXX,XX @@ int pci_sriov_configure_simple(struct pci_dev *dev, int nr_virtfn)
21
    return nr_virtfn;
22
}
23
EXPORT_SYMBOL_GPL(pci_sriov_configure_simple);
24
+
25
+/**
26
+ * pci_iov_vf_bar_set_size - set a new size for a VF BAR
27
+ * @dev: the PCI device
28
+ * @resno: the resource number
29
+ * @size: new size as defined in the spec (0=1MB, 31=128TB)
30
+ *
31
+ * Set the new size of a VF BAR that supports VF resizable BAR capability.
32
+ * Unlike pci_resize_resource(), this does not cause the resource that
33
+ * reserves the MMIO space (originally up to total_VFs) to be resized, which
34
+ * means that following calls to pci_enable_sriov() can fail if the resources
35
+ * no longer fit.
36
+ *
37
+ * Return: 0 on success, or negative on failure.
38
+ */
39
+int pci_iov_vf_bar_set_size(struct pci_dev *dev, int resno, int size)
40
+{
41
+    int ret;
42
+    u32 sizes;
43
+
44
+    if (!pci_resource_is_iov(resno))
45
+        return -EINVAL;
46
+
47
+    if (pci_iov_is_memory_decoding_enabled(dev))
48
+        return -EBUSY;
49
+
50
+    sizes = pci_rebar_get_possible_sizes(dev, resno);
51
+    if (!sizes)
52
+        return -ENOTSUPP;
53
+
54
+    if (!(sizes & BIT(size)))
55
+        return -EINVAL;
56
+
57
+    ret = pci_rebar_set_size(dev, resno, size);
58
+    if (ret)
59
+        return ret;
60
+
61
+    pci_iov_resource_set_size(dev, resno, pci_rebar_size_to_bytes(size));
62
+
63
+    return 0;
64
+}
65
+EXPORT_SYMBOL_GPL(pci_iov_vf_bar_set_size);
66
+
67
+/**
68
+ * pci_iov_vf_bar_get_sizes - get VF BAR sizes allowing to create up to num_vfs
69
+ * @dev: the PCI device
70
+ * @resno: the resource number
71
+ * @num_vfs: number of VFs
72
+ *
73
+ * Get the sizes of a VF resizable BAR that can accommodate @num_vfs within
74
+ * the currently assigned size of the resource @resno.
75
+ *
76
+ * Return: A bitmask of sizes in format defined in the spec (bit 0=1MB,
77
+ * bit 31=128TB).
78
+ */
79
+u32 pci_iov_vf_bar_get_sizes(struct pci_dev *dev, int resno, int num_vfs)
80
+{
81
+    resource_size_t vf_len = pci_resource_len(dev, resno);
82
+    u32 sizes;
83
+
84
+    if (!num_vfs)
85
+        return 0;
86
+
87
+    do_div(vf_len, num_vfs);
88
+    sizes = (roundup_pow_of_two(vf_len + 1) - 1) >> ilog2(SZ_1M);
89
+
90
+    return sizes & pci_rebar_get_possible_sizes(dev, resno);
91
+}
92
+EXPORT_SYMBOL_GPL(pci_iov_vf_bar_get_sizes);
93
diff --git a/include/linux/pci.h b/include/linux/pci.h
94
index XXXXXXX..XXXXXXX 100644
95
--- a/include/linux/pci.h
96
+++ b/include/linux/pci.h
97
@@ -XXX,XX +XXX,XX @@ int pci_sriov_set_totalvfs(struct pci_dev *dev, u16 numvfs);
98
int pci_sriov_get_totalvfs(struct pci_dev *dev);
99
int pci_sriov_configure_simple(struct pci_dev *dev, int nr_virtfn);
100
resource_size_t pci_iov_resource_size(struct pci_dev *dev, int resno);
101
+int pci_iov_vf_bar_set_size(struct pci_dev *dev, int resno, int size);
102
+u32 pci_iov_vf_bar_get_sizes(struct pci_dev *dev, int resno, int num_vfs);
103
void pci_vf_drivers_autoprobe(struct pci_dev *dev, bool probe);
104
105
/* Arch may override these (weak) */
106
@@ -XXX,XX +XXX,XX @@ static inline int pci_sriov_get_totalvfs(struct pci_dev *dev)
107
#define pci_sriov_configure_simple    NULL
108
static inline resource_size_t pci_iov_resource_size(struct pci_dev *dev, int resno)
109
{ return 0; }
110
+static inline int pci_iov_vf_bar_set_size(struct pci_dev *dev, int resno, int size)
111
+{ return -ENODEV; }
112
+static inline u32 pci_iov_vf_bar_get_sizes(struct pci_dev *dev, int resno, int num_vfs)
113
+{ return 0; }
114
static inline void pci_vf_drivers_autoprobe(struct pci_dev *dev, bool probe) { }
115
#endif
116
117
--
118
2.49.0
119
diff view generated by jsdifflib
New patch
1
LMEM is partitioned between multiple VFs and we expect that the more
2
VFs we have, the less LMEM is assigned to each VF.
3
This means that we can achieve full LMEM BAR access without the need to
4
attempt full VF LMEM BAR resize via pci_resize_resource().
1
5
6
Always try to set the largest possible BAR size that allows to fit the
7
number of enabled VFs and inform the user in case the resize attempt is
8
not successful.
9
10
Signed-off-by: Michał Winiarski <michal.winiarski@intel.com>
11
Reviewed-by: Ilpo Järvinen <ilpo.jarvinen@linux.intel.com>
12
---
13
drivers/gpu/drm/xe/regs/xe_bars.h | 1 +
14
drivers/gpu/drm/xe/xe_pci_sriov.c | 22 ++++++++++++++++++++++
15
2 files changed, 23 insertions(+)
16
17
diff --git a/drivers/gpu/drm/xe/regs/xe_bars.h b/drivers/gpu/drm/xe/regs/xe_bars.h
18
index XXXXXXX..XXXXXXX 100644
19
--- a/drivers/gpu/drm/xe/regs/xe_bars.h
20
+++ b/drivers/gpu/drm/xe/regs/xe_bars.h
21
@@ -XXX,XX +XXX,XX @@
22
23
#define GTTMMADR_BAR            0 /* MMIO + GTT */
24
#define LMEM_BAR            2 /* VRAM */
25
+#define VF_LMEM_BAR            9 /* VF VRAM */
26
27
#endif
28
diff --git a/drivers/gpu/drm/xe/xe_pci_sriov.c b/drivers/gpu/drm/xe/xe_pci_sriov.c
29
index XXXXXXX..XXXXXXX 100644
30
--- a/drivers/gpu/drm/xe/xe_pci_sriov.c
31
+++ b/drivers/gpu/drm/xe/xe_pci_sriov.c
32
@@ -XXX,XX +XXX,XX @@
33
* Copyright © 2023-2024 Intel Corporation
34
*/
35
36
+#include <linux/bitops.h>
37
+#include <linux/pci.h>
38
+
39
+#include "regs/xe_bars.h"
40
#include "xe_assert.h"
41
#include "xe_device.h"
42
#include "xe_gt_sriov_pf_config.h"
43
@@ -XXX,XX +XXX,XX @@ static void pf_link_vfs(struct xe_device *xe, int num_vfs)
44
    }
45
}
46
47
+static int resize_vf_vram_bar(struct xe_device *xe, int num_vfs)
48
+{
49
+    struct pci_dev *pdev = to_pci_dev(xe->drm.dev);
50
+    u32 sizes;
51
+
52
+    sizes = pci_iov_vf_bar_get_sizes(pdev, VF_LMEM_BAR, num_vfs);
53
+    if (!sizes)
54
+        return 0;
55
+
56
+    return pci_iov_vf_bar_set_size(pdev, VF_LMEM_BAR, __fls(sizes));
57
+}
58
+
59
static int pf_enable_vfs(struct xe_device *xe, int num_vfs)
60
{
61
    struct pci_dev *pdev = to_pci_dev(xe->drm.dev);
62
@@ -XXX,XX +XXX,XX @@ static int pf_enable_vfs(struct xe_device *xe, int num_vfs)
63
    if (err < 0)
64
        goto failed;
65
66
+    if (IS_DGFX(xe)) {
67
+        err = resize_vf_vram_bar(xe, num_vfs);
68
+        if (err)
69
+            xe_sriov_info(xe, "Failed to set VF LMEM BAR size: %d\n", err);
70
+    }
71
+
72
    err = pci_enable_sriov(pdev, num_vfs);
73
    if (err < 0)
74
        goto failed;
75
--
76
2.49.0
77
diff view generated by jsdifflib