docs/formatdomain.html.in | 9 +- docs/news.xml | 11 + docs/schemas/basictypes.rng | 31 ++ docs/schemas/domaincommon.rng | 1 + src/conf/device_conf.c | 73 +++++ src/conf/device_conf.h | 1 + src/conf/domain_addr.c | 346 +++++++++++++++++++++ src/conf/domain_addr.h | 29 ++ src/conf/domain_conf.c | 6 + src/libvirt_private.syms | 4 + src/qemu/qemu_capabilities.c | 6 + src/qemu/qemu_capabilities.h | 1 + src/qemu/qemu_command.c | 115 +++++++ src/qemu/qemu_command.h | 4 + src/qemu/qemu_domain.c | 1 + src/qemu/qemu_domain_address.c | 181 ++++++++++- src/qemu/qemu_hotplug.c | 182 ++++++++++- src/util/virpci.h | 13 + tests/qemucapabilitiesdata/caps_2.10.0.s390x.xml | 1 + tests/qemucapabilitiesdata/caps_2.11.0.s390x.xml | 1 + tests/qemucapabilitiesdata/caps_2.12.0.s390x.xml | 1 + tests/qemucapabilitiesdata/caps_2.7.0.s390x.xml | 1 + tests/qemucapabilitiesdata/caps_2.8.0.s390x.xml | 1 + tests/qemucapabilitiesdata/caps_2.9.0.s390x.xml | 1 + tests/qemuxml2argvdata/disk-virtio-s390-zpci.args | 27 ++ tests/qemuxml2argvdata/disk-virtio-s390-zpci.xml | 17 + .../hostdev-vfio-zpci-autogenerate.args | 26 ++ .../hostdev-vfio-zpci-autogenerate.xml | 18 ++ .../hostdev-vfio-zpci-boundaries.args | 30 ++ .../hostdev-vfio-zpci-boundaries.xml | 26 ++ .../hostdev-vfio-zpci-multidomain-many.args | 40 +++ .../hostdev-vfio-zpci-multidomain-many.xml | 67 ++++ tests/qemuxml2argvdata/hostdev-vfio-zpci.args | 26 ++ tests/qemuxml2argvdata/hostdev-vfio-zpci.xml | 19 ++ tests/qemuxml2argvtest.c | 14 + tests/qemuxml2xmloutdata/disk-virtio-s390-zpci.xml | 29 ++ tests/qemuxml2xmloutdata/hostdev-vfio-zpci.xml | 30 ++ tests/qemuxml2xmltest.c | 3 + 38 files changed, 1377 insertions(+), 15 deletions(-) create mode 100644 tests/qemuxml2argvdata/disk-virtio-s390-zpci.args create mode 100644 tests/qemuxml2argvdata/disk-virtio-s390-zpci.xml create mode 100644 tests/qemuxml2argvdata/hostdev-vfio-zpci-autogenerate.args create mode 100644 tests/qemuxml2argvdata/hostdev-vfio-zpci-autogenerate.xml create mode 100644 tests/qemuxml2argvdata/hostdev-vfio-zpci-boundaries.args create mode 100644 tests/qemuxml2argvdata/hostdev-vfio-zpci-boundaries.xml create mode 100644 tests/qemuxml2argvdata/hostdev-vfio-zpci-multidomain-many.args create mode 100644 tests/qemuxml2argvdata/hostdev-vfio-zpci-multidomain-many.xml create mode 100644 tests/qemuxml2argvdata/hostdev-vfio-zpci.args create mode 100644 tests/qemuxml2argvdata/hostdev-vfio-zpci.xml create mode 100644 tests/qemuxml2xmloutdata/disk-virtio-s390-zpci.xml create mode 100644 tests/qemuxml2xmloutdata/hostdev-vfio-zpci.xml
Abstract ======== The PCI representation in QEMU has recently been extended for S390 allowing configuration of zPCI attributes like uid (user-defined identifier) and fid (PCI function identifier). The details can be found here: https://lists.gnu.org/archive/html/qemu-devel/2016-06/msg07262.html To support the new zPCI feature of the S390 platform, two new XML attributes, @uid and @fid, are introduced for device addresses of type 'pci', i.e.: <hostdev mode='subsystem' type='pci'> <driver name='vfio'/> <source> <address domain='0x0001' bus='0x00' slot='0x00' function='0x0'/> </source> <address type='pci' domain='0x0000' bus='0x00' slot='0x01' function='0x0' uid='0x0003' fid='0x00000027'/> </hostdev> uid and fid are optional attributes. If they are defined by the user, unique values within the guest domain must be used. If they are not specified and the architecture requires them, they are automatically generated with non-conflicting values. Current implementation is the most seamless one for the user as it unites the address specific data of a PCI device on one XML element. It could accommodate both specifying our special parameters (uid and fid) and re-using standard statements (domain, bus, slot and function) for PCI devices. User can still specify bus/slot/function for the virtualized PCI devices in the XML. Thus uid/fid act as an extension to the PCI address and are stored in a new structure 'virZPCIDeviceAddress' which is a member of common PCI Address structure. Additionally, two hashtables are used for assignment and reservation of uid/fid. In support of extending the PCI address, a new PCI address extension flag is introduced. This extension flag allows is not only dedicated for the S390 platform but also other architectures needing certain extensions to PCI address space. Code Base ========= commit in master: 767f9e1449b1a36111532847f0c62dc758263c42 qemu: validate: Enforce compile time switch type checking for videos Change Log ========== v1->v2: 1. Separate test commit and merge testcases into corresponding commits that introduce the functionalities firstly. 2. Spare some checks for zpci device. 3. Add vsock and controller support. 4. Add uin32 type schema. 5. Rename zpciuid and zpcifid to zpci_uid and zpci_fid. 6. Always return multibus support on S390. Yi Min Zhao (12): conf: Add definitions for 'uid' and 'fid' PCI address attributes qemu: Introduce zPCI capability conf: Introduce a new PCI address extension flag qemu: Enable PCI multi bus for S390 guests qemu: Auto add pci-root for s390/s390x guests conf: Introduce address caching for PCI extensions conf: Introduce parser, formatter for uid and fid conf: Allocate/release 'uid' and 'fid' in PCI address qemu: Generate and use zPCI device in QEMU command line qemu: Add hotpluging support for PCI devices on S390 guests docs: Add 'uid' and 'fid' information news: Update news for PCI address extension attributes docs/formatdomain.html.in | 9 +- docs/news.xml | 11 + docs/schemas/basictypes.rng | 31 ++ docs/schemas/domaincommon.rng | 1 + src/conf/device_conf.c | 73 +++++ src/conf/device_conf.h | 1 + src/conf/domain_addr.c | 346 +++++++++++++++++++++ src/conf/domain_addr.h | 29 ++ src/conf/domain_conf.c | 6 + src/libvirt_private.syms | 4 + src/qemu/qemu_capabilities.c | 6 + src/qemu/qemu_capabilities.h | 1 + src/qemu/qemu_command.c | 115 +++++++ src/qemu/qemu_command.h | 4 + src/qemu/qemu_domain.c | 1 + src/qemu/qemu_domain_address.c | 181 ++++++++++- src/qemu/qemu_hotplug.c | 182 ++++++++++- src/util/virpci.h | 13 + tests/qemucapabilitiesdata/caps_2.10.0.s390x.xml | 1 + tests/qemucapabilitiesdata/caps_2.11.0.s390x.xml | 1 + tests/qemucapabilitiesdata/caps_2.12.0.s390x.xml | 1 + tests/qemucapabilitiesdata/caps_2.7.0.s390x.xml | 1 + tests/qemucapabilitiesdata/caps_2.8.0.s390x.xml | 1 + tests/qemucapabilitiesdata/caps_2.9.0.s390x.xml | 1 + tests/qemuxml2argvdata/disk-virtio-s390-zpci.args | 27 ++ tests/qemuxml2argvdata/disk-virtio-s390-zpci.xml | 17 + .../hostdev-vfio-zpci-autogenerate.args | 26 ++ .../hostdev-vfio-zpci-autogenerate.xml | 18 ++ .../hostdev-vfio-zpci-boundaries.args | 30 ++ .../hostdev-vfio-zpci-boundaries.xml | 26 ++ .../hostdev-vfio-zpci-multidomain-many.args | 40 +++ .../hostdev-vfio-zpci-multidomain-many.xml | 67 ++++ tests/qemuxml2argvdata/hostdev-vfio-zpci.args | 26 ++ tests/qemuxml2argvdata/hostdev-vfio-zpci.xml | 19 ++ tests/qemuxml2argvtest.c | 14 + tests/qemuxml2xmloutdata/disk-virtio-s390-zpci.xml | 29 ++ tests/qemuxml2xmloutdata/hostdev-vfio-zpci.xml | 30 ++ tests/qemuxml2xmltest.c | 3 + 38 files changed, 1377 insertions(+), 15 deletions(-) create mode 100644 tests/qemuxml2argvdata/disk-virtio-s390-zpci.args create mode 100644 tests/qemuxml2argvdata/disk-virtio-s390-zpci.xml create mode 100644 tests/qemuxml2argvdata/hostdev-vfio-zpci-autogenerate.args create mode 100644 tests/qemuxml2argvdata/hostdev-vfio-zpci-autogenerate.xml create mode 100644 tests/qemuxml2argvdata/hostdev-vfio-zpci-boundaries.args create mode 100644 tests/qemuxml2argvdata/hostdev-vfio-zpci-boundaries.xml create mode 100644 tests/qemuxml2argvdata/hostdev-vfio-zpci-multidomain-many.args create mode 100644 tests/qemuxml2argvdata/hostdev-vfio-zpci-multidomain-many.xml create mode 100644 tests/qemuxml2argvdata/hostdev-vfio-zpci.args create mode 100644 tests/qemuxml2argvdata/hostdev-vfio-zpci.xml create mode 100644 tests/qemuxml2xmloutdata/disk-virtio-s390-zpci.xml create mode 100644 tests/qemuxml2xmloutdata/hostdev-vfio-zpci.xml -- Yi Min -- libvir-list mailing list libvir-list@redhat.com https://www.redhat.com/mailman/listinfo/libvir-list
On Tue, 10 Jul 2018 16:02:14 +0800 Yi Min Zhao <zyimin@linux.ibm.com> wrote: > Abstract > ======== > The PCI representation in QEMU has recently been extended for S390 > allowing configuration of zPCI attributes like uid (user-defined > identifier) and fid (PCI function identifier). > The details can be found here: > https://lists.gnu.org/archive/html/qemu-devel/2016-06/msg07262.html > > To support the new zPCI feature of the S390 platform, two new XML > attributes, @uid and @fid, are introduced for device addresses of type > 'pci', i.e.: > <hostdev mode='subsystem' type='pci'> > <driver name='vfio'/> > <source> > <address domain='0x0001' bus='0x00' slot='0x00' function='0x0'/> > </source> > <address type='pci' domain='0x0000' bus='0x00' slot='0x01' function='0x0' > uid='0x0003' fid='0x00000027'/> > </hostdev> > > uid and fid are optional attributes. If they are defined by the user, > unique values within the guest domain must be used. If they are not > specified and the architecture requires them, they are automatically > generated with non-conflicting values. > > Current implementation is the most seamless one for the user as it > unites the address specific data of a PCI device on one XML element. > It could accommodate both specifying our special parameters (uid and fid) > and re-using standard statements (domain, bus, slot and function) for > PCI devices. User can still specify bus/slot/function for the virtualized > PCI devices in the XML. > > Thus uid/fid act as an extension to the PCI address and are stored in > a new structure 'virZPCIDeviceAddress' which is a member of common PCI > Address structure. Additionally, two hashtables are used for assignment > and reservation of uid/fid. > > In support of extending the PCI address, a new PCI address extension flag is > introduced. This extension flag allows is not only dedicated for the S390 > platform but also other architectures needing certain extensions to PCI > address space. FWIW, from my QEMU POV there's nothing obviously wrong in here, but I'm not familiar with the libvirt code base. So I'll leave this to the libvirt developers. -- libvir-list mailing list libvir-list@redhat.com https://www.redhat.com/mailman/listinfo/libvir-list
在 2018/7/17 下午8:35, Cornelia Huck 写道: > On Tue, 10 Jul 2018 16:02:14 +0800 > Yi Min Zhao <zyimin@linux.ibm.com> wrote: > >> Abstract >> ======== >> The PCI representation in QEMU has recently been extended for S390 >> allowing configuration of zPCI attributes like uid (user-defined >> identifier) and fid (PCI function identifier). >> The details can be found here: >> https://lists.gnu.org/archive/html/qemu-devel/2016-06/msg07262.html >> >> To support the new zPCI feature of the S390 platform, two new XML >> attributes, @uid and @fid, are introduced for device addresses of type >> 'pci', i.e.: >> <hostdev mode='subsystem' type='pci'> >> <driver name='vfio'/> >> <source> >> <address domain='0x0001' bus='0x00' slot='0x00' function='0x0'/> >> </source> >> <address type='pci' domain='0x0000' bus='0x00' slot='0x01' function='0x0' >> uid='0x0003' fid='0x00000027'/> >> </hostdev> >> >> uid and fid are optional attributes. If they are defined by the user, >> unique values within the guest domain must be used. If they are not >> specified and the architecture requires them, they are automatically >> generated with non-conflicting values. >> >> Current implementation is the most seamless one for the user as it >> unites the address specific data of a PCI device on one XML element. >> It could accommodate both specifying our special parameters (uid and fid) >> and re-using standard statements (domain, bus, slot and function) for >> PCI devices. User can still specify bus/slot/function for the virtualized >> PCI devices in the XML. >> >> Thus uid/fid act as an extension to the PCI address and are stored in >> a new structure 'virZPCIDeviceAddress' which is a member of common PCI >> Address structure. Additionally, two hashtables are used for assignment >> and reservation of uid/fid. >> >> In support of extending the PCI address, a new PCI address extension flag is >> introduced. This extension flag allows is not only dedicated for the S390 >> platform but also other architectures needing certain extensions to PCI >> address space. > FWIW, from my QEMU POV there's nothing obviously wrong in here, but I'm > not familiar with the libvirt code base. So I'll leave this to the > libvirt developers. > > Thanks! Libvirt developers have not given any comment on v2 until now. I'm afraid the end of this month is coming soon. -- libvir-list mailing list libvir-list@redhat.com https://www.redhat.com/mailman/listinfo/libvir-list
On Tue, Jul 10, 2018 at 04:02:14PM +0800, Yi Min Zhao wrote: >Abstract >======== >The PCI representation in QEMU has recently been extended for S390 >allowing configuration of zPCI attributes like uid (user-defined >identifier) and fid (PCI function identifier). >The details can be found here: >https://lists.gnu.org/archive/html/qemu-devel/2016-06/msg07262.html > >To support the new zPCI feature of the S390 platform, two new XML >attributes, @uid and @fid, are introduced for device addresses of type >'pci', i.e.: > <hostdev mode='subsystem' type='pci'> > <driver name='vfio'/> > <source> > <address domain='0x0001' bus='0x00' slot='0x00' function='0x0'/> > </source> > <address type='pci' domain='0x0000' bus='0x00' slot='0x01' function='0x0' > uid='0x0003' fid='0x00000027'/> > </hostdev> > >uid and fid are optional attributes. If they are defined by the user, >unique values within the guest domain must be used. If they are not >specified and the architecture requires them, they are automatically >generated with non-conflicting values. > >Current implementation is the most seamless one for the user as it >unites the address specific data of a PCI device on one XML element. >It could accommodate both specifying our special parameters (uid and fid) >and re-using standard statements (domain, bus, slot and function) for >PCI devices. User can still specify bus/slot/function for the virtualized >PCI devices in the XML. > >Thus uid/fid act as an extension to the PCI address and are stored in >a new structure 'virZPCIDeviceAddress' which is a member of common PCI >Address structure. Additionally, two hashtables are used for assignment >and reservation of uid/fid. > >In support of extending the PCI address, a new PCI address extension flag is >introduced. This extension flag allows is not only dedicated for the S390 >platform but also other architectures needing certain extensions to PCI >address space. > >Code Base >========= >commit in master: >767f9e1449b1a36111532847f0c62dc758263c42 >qemu: validate: Enforce compile time switch type checking for videos > >Change Log >========== >v1->v2: >1. Separate test commit and merge testcases into corresponding commits that > introduce the functionalities firstly. >2. Spare some checks for zpci device. >3. Add vsock and controller support. >4. Add uin32 type schema. >5. Rename zpciuid and zpcifid to zpci_uid and zpci_fid. >6. Always return multibus support on S390. > >Yi Min Zhao (12): > conf: Add definitions for 'uid' and 'fid' PCI address attributes > qemu: Introduce zPCI capability > conf: Introduce a new PCI address extension flag > qemu: Enable PCI multi bus for S390 guests > qemu: Auto add pci-root for s390/s390x guests > conf: Introduce address caching for PCI extensions > conf: Introduce parser, formatter for uid and fid > conf: Allocate/release 'uid' and 'fid' in PCI address > qemu: Generate and use zPCI device in QEMU command line > qemu: Add hotpluging support for PCI devices on S390 guests > docs: Add 'uid' and 'fid' information > news: Update news for PCI address extension attributes > The patches look OK to me, therefore: Reviewed-by: Ján Tomko <jtomko@redhat.com> But I'd like to hear from some other developer who touched the PCI code last (Laine? Andrea?) Jano -- libvir-list mailing list libvir-list@redhat.com https://www.redhat.com/mailman/listinfo/libvir-list
On Mon, Jul 23, 2018, 2:37 PM Ján Tomko <jtomko@redhat.com> wrote: > > But I'd like to hear from some other developer who touched the PCI code > last (Laine? Andrea?) > I'm looking at this on the screen of my phone as I stare out the hotel window at an afternoon rainstorm over Razgrad, Bulgaria, and won't be back to my office for another week, so I'm afraid I'm not in much of a position to make any useful comment :-) > -- libvir-list mailing list libvir-list@redhat.com https://www.redhat.com/mailman/listinfo/libvir-list
On Mon, 2018-07-23 at 13:37 +0200, Ján Tomko wrote: > On Tue, Jul 10, 2018 at 04:02:14PM +0800, Yi Min Zhao wrote: > > Abstract > > ======== > > The PCI representation in QEMU has recently been extended for S390 > > allowing configuration of zPCI attributes like uid (user-defined > > identifier) and fid (PCI function identifier). > > The details can be found here: > > https://lists.gnu.org/archive/html/qemu-devel/2016-06/msg07262.html > > > > To support the new zPCI feature of the S390 platform, two new XML > > attributes, @uid and @fid, are introduced for device addresses of type > > 'pci', i.e.: > > <hostdev mode='subsystem' type='pci'> > > <driver name='vfio'/> > > <source> > > <address domain='0x0001' bus='0x00' slot='0x00' function='0x0'/> > > </source> > > <address type='pci' domain='0x0000' bus='0x00' slot='0x01' function='0x0' > > uid='0x0003' fid='0x00000027'/> > > </hostdev> > > > > uid and fid are optional attributes. If they are defined by the user, > > unique values within the guest domain must be used. If they are not > > specified and the architecture requires them, they are automatically > > generated with non-conflicting values. > > > > Current implementation is the most seamless one for the user as it > > unites the address specific data of a PCI device on one XML element. > > It could accommodate both specifying our special parameters (uid and fid) > > and re-using standard statements (domain, bus, slot and function) for > > PCI devices. User can still specify bus/slot/function for the virtualized > > PCI devices in the XML. > > > > Thus uid/fid act as an extension to the PCI address and are stored in > > a new structure 'virZPCIDeviceAddress' which is a member of common PCI > > Address structure. Additionally, two hashtables are used for assignment > > and reservation of uid/fid. > > > > In support of extending the PCI address, a new PCI address extension flag is > > introduced. This extension flag allows is not only dedicated for the S390 > > platform but also other architectures needing certain extensions to PCI > > address space. [...] > > The patches look OK to me, therefore: > > Reviewed-by: Ján Tomko <jtomko@redhat.com> > > But I'd like to hear from some other developer who touched the PCI code > last (Laine? Andrea?) Thanks for bringing this to my attention - it had somehow managed to escape my radar so far :) I've quickly gone through the patches and spotted some minor code style issues that I'd like to see addressed (I'll point them out separately); first, though, I have a couple of questions about the general idea behind the feature. Looking at both the generated QEMU command line and the qemu-devel message linked above, I seem to understand the zpci device is basically some sort of adapter that sits between a regular PCI device (emulated or otherwise) and an s390 guest and presents an ID-based interface to the latter; so IIUC the s390 guest doesn't actually see a PCI device, but a couple of IDs that can (somehow) be used to access the underlying PCI device. >From whatever little s390 knowledge I have, I recall the architecture using peculiar ways to address and access devices, so PCI not being usable wouldn't surprise me that much; what I find odd, however, is that regular PCI seems to be available at least on the host side, because otherwise stuff like -device zpci,uid=1,fid=1,target=vpci0 -device vfio-pci,host=0000:00:00.0,id=vpci0 wouldn't be possible. So, would anyone with s390 knowledge please spend a few words explaining how the various pieces fit together? Assuming the above is a correct reading of the situation, it would seem the IDs are the only guest-visible part that we need to make sure doesn't change during the lifetime of the guest, while the usual bus/slot/function triplet assigned to the underlying PCI device doesn't actually matter. And if that's the case, wouldn't something like <address type='zpci' uid='1' fid='1'/> be a better representation of the arrangement? We could leave it up to QEMU to assign addresses to the PCI devices in this case... But maybe they still matter for things such as migration? Or maybe I've just gotten it wrong altogether? :) -- Andrea Bolognani / Red Hat / Virtualization -- libvir-list mailing list libvir-list@redhat.com https://www.redhat.com/mailman/listinfo/libvir-list
On Tue, 24 Jul 2018 10:20:48 +0200 Andrea Bolognani <abologna@redhat.com> wrote: > Looking at both the generated QEMU command line and the qemu-devel > message linked above, I seem to understand the zpci device is > basically some sort of adapter that sits between a regular PCI > device (emulated or otherwise) and an s390 guest and presents an > ID-based interface to the latter; so IIUC the s390 guest doesn't > actually see a PCI device, but a couple of IDs that can (somehow) > be used to access the underlying PCI device. > > From whatever little s390 knowledge I have, I recall the > architecture using peculiar ways to address and access devices, so > PCI not being usable wouldn't surprise me that much; what I find > odd, however, is that regular PCI seems to be available at least > on the host side, because otherwise stuff like > > -device zpci,uid=1,fid=1,target=vpci0 > -device vfio-pci,host=0000:00:00.0,id=vpci0 > > wouldn't be possible. So, would anyone with s390 knowledge please > spend a few words explaining how the various pieces fit together? For some hints, let me point to https://virtualpenguins.blogspot.de/2018/02/notes-on-pci-on-s390x.html -- for implementation details, I'll point to the IBM folks :) > > Assuming the above is a correct reading of the situation, it > would seem the IDs are the only guest-visible part that we need > to make sure doesn't change during the lifetime of the guest, > while the usual bus/slot/function triplet assigned to the > underlying PCI device doesn't actually matter. And if that's the > case, wouldn't something like > > <address type='zpci' uid='1' fid='1'/> > > be a better representation of the arrangement? We could leave it > up to QEMU to assign addresses to the PCI devices in this case... > But maybe they still matter for things such as migration? Or maybe > I've just gotten it wrong altogether? :) > -- libvir-list mailing list libvir-list@redhat.com https://www.redhat.com/mailman/listinfo/libvir-list
On 07/24/2018 10:20 AM, Andrea Bolognani wrote: > On Mon, 2018-07-23 at 13:37 +0200, Ján Tomko wrote: >> On Tue, Jul 10, 2018 at 04:02:14PM +0800, Yi Min Zhao wrote: >>> Abstract >>> ======== >>> The PCI representation in QEMU has recently been extended for S390 >>> allowing configuration of zPCI attributes like uid (user-defined >>> identifier) and fid (PCI function identifier). >>> The details can be found here: >>> https://lists.gnu.org/archive/html/qemu-devel/2016-06/msg07262.html >>> >>> To support the new zPCI feature of the S390 platform, two new XML >>> attributes, @uid and @fid, are introduced for device addresses of type >>> 'pci', i.e.: >>> <hostdev mode='subsystem' type='pci'> >>> <driver name='vfio'/> >>> <source> >>> <address domain='0x0001' bus='0x00' slot='0x00' function='0x0'/> >>> </source> >>> <address type='pci' domain='0x0000' bus='0x00' slot='0x01' function='0x0' >>> uid='0x0003' fid='0x00000027'/> >>> </hostdev> >>> >>> uid and fid are optional attributes. If they are defined by the user, >>> unique values within the guest domain must be used. If they are not >>> specified and the architecture requires them, they are automatically >>> generated with non-conflicting values. >>> >>> Current implementation is the most seamless one for the user as it >>> unites the address specific data of a PCI device on one XML element. >>> It could accommodate both specifying our special parameters (uid and fid) >>> and re-using standard statements (domain, bus, slot and function) for >>> PCI devices. User can still specify bus/slot/function for the virtualized >>> PCI devices in the XML. >>> >>> Thus uid/fid act as an extension to the PCI address and are stored in >>> a new structure 'virZPCIDeviceAddress' which is a member of common PCI >>> Address structure. Additionally, two hashtables are used for assignment >>> and reservation of uid/fid. >>> >>> In support of extending the PCI address, a new PCI address extension flag is >>> introduced. This extension flag allows is not only dedicated for the S390 >>> platform but also other architectures needing certain extensions to PCI >>> address space. > [...] >> >> The patches look OK to me, therefore: >> >> Reviewed-by: Ján Tomko <jtomko@redhat.com> >> >> But I'd like to hear from some other developer who touched the PCI code >> last (Laine? Andrea?) > > Thanks for bringing this to my attention - it had somehow managed > to escape my radar so far :) > > I've quickly gone through the patches and spotted some minor code > style issues that I'd like to see addressed (I'll point them out > separately); first, though, I have a couple of questions about the > general idea behind the feature. > > Looking at both the generated QEMU command line and the qemu-devel > message linked above, I seem to understand the zpci device is > basically some sort of adapter that sits between a regular PCI > device (emulated or otherwise) and an s390 guest and presents an > ID-based interface to the latter; so IIUC the s390 guest doesn't > actually see a PCI device, but a couple of IDs that can (somehow) > be used to access the underlying PCI device. > >>From whatever little s390 knowledge I have, I recall the > architecture using peculiar ways to address and access devices, so > PCI not being usable wouldn't surprise me that much; what I find > odd, however, is that regular PCI seems to be available at least > on the host side, because otherwise stuff like > > -device zpci,uid=1,fid=1,target=vpci0 > -device vfio-pci,host=0000:00:00.0,id=vpci0 > > wouldn't be possible. So, would anyone with s390 knowledge please > spend a few words explaining how the various pieces fit together? > > Assuming the above is a correct reading of the situation, it > would seem the IDs are the only guest-visible part that we need > to make sure doesn't change during the lifetime of the guest, > while the usual bus/slot/function triplet assigned to the > underlying PCI device doesn't actually matter. And if that's the > case, wouldn't something like > > <address type='zpci' uid='1' fid='1'/> Then a pci device on s390 would need a special address type zpci in libvirt and the design idea for libvirt is to stay compatible with pci. Therefore uid and fid are optional attributes and if not specified on s390 they are generated. The patch series also allows on s390 to specify the pci address type just with attributes uid and/or fid causing the rest of the pci address attributes to be generated. That means <address type='pci' uid='1' fid='1'/> is a valid pci address on s390 but would get expanded during definition into e.g. <address type='pci' domain='0x0000' bus='0x00' slot='0x01' function='0x0' uid='1' fid='1'/> Also <address type='pci' domain='0x0000' bus='0x00' slot='0x01' function='0x0'> is a valid pci address on s390 and is would be expanded during definition into e.g. <address type='pci' domain='0x0000' bus='0x00' slot='0x01' function='0x0' uid='1' fid='1'/> > > be a better representation of the arrangement? We could leave it > up to QEMU to assign addresses to the PCI devices in this case... > But maybe they still matter for things such as migration? Or maybe > I've just gotten it wrong altogether? :) > -- Mit freundlichen Grüßen/Kind regards Boris Fiuczynski IBM Deutschland Research & Development GmbH Vorsitzender des Aufsichtsrats: Martina Köderitz Geschäftsführung: Dirk Wittkopp Sitz der Gesellschaft: Böblingen Registergericht: Amtsgericht Stuttgart, HRB 243294 -- libvir-list mailing list libvir-list@redhat.com https://www.redhat.com/mailman/listinfo/libvir-list
On Tue, 2018-07-24 at 11:31 +0200, Boris Fiuczynski wrote: > On 07/24/2018 10:20 AM, Andrea Bolognani wrote: > > Assuming the above is a correct reading of the situation, it > > would seem the IDs are the only guest-visible part that we need > > to make sure doesn't change during the lifetime of the guest, > > while the usual bus/slot/function triplet assigned to the > > underlying PCI device doesn't actually matter. And if that's the > > case, wouldn't something like > > > > <address type='zpci' uid='1' fid='1'/> > > Then a pci device on s390 would need a special address type zpci in > libvirt and the design idea for libvirt is to stay compatible with pci. Being compatible with the existing PCI machinery is certainly a good idea when it makes sense to do so, but I'm not quite convinced that is the case here. According to Cornelia's blog post on the subject, the PCI topology inside the guest will be determined entirely by the IDs. Is there even a way to eg. use bridges to create a non-flat PCI hierarchy? Or to have several PCI devices share the same bus or slot? If none of the above applies, then that doesn't look a whole lot like PCI to me :) Moreover, we already have several address types in addition to PCI such as USB, virtio-mmio, spapr-vio, ccw... Adding yet another one is not a problem if it makes the interface more sensible. More concrete questions: one of the zPCI test cases includes <controller type='pci' index='1' model='pci-bridge'/> <hostdev mode='subsystem' type='pci' managed='no'> <driver name='vfio'/> <source> <address domain='0xffff' bus='0x00' slot='0x00' function='0x0'/> </source> <address type='pci' domain='0x0000' bus='0x01' slot='0x1f' function='0x0' uid='0xffff' fid='0xffffffff'/> </hostdev> which translates to -device zpci,uid=3,fid=2,target=pci.1,id=zpci3 \ -device pci-bridge,chassis_nr=1,id=pci.1,bus=pci.0,addr=0x1 \ -device zpci,uid=65535,fid=4294967295,target=hostdev0,id=zpci65535 \ -device vfio-pci,host=ffff:00:00.0,id=hostdev0,bus=pci.1,addr=0x1f \ How does the pci-bridge controller show up in the guest, if at all? Do the bus= and addr= attributes of vfio-pci and pci-bridge in the example above matter eg. for migration purposes? > Therefore uid and fid are optional attributes and if not specified on > s390 they are generated. The patch series also allows on s390 to specify > the pci address type just with attributes uid and/or fid causing the > rest of the pci address attributes to be generated. This goes without saying: users should not have to worry about addresses at all unless they have very specific needs. -- Andrea Bolognani / Red Hat / Virtualization -- libvir-list mailing list libvir-list@redhat.com https://www.redhat.com/mailman/listinfo/libvir-list
On 07/24/2018 03:41 PM, Andrea Bolognani wrote: > On Tue, 2018-07-24 at 11:31 +0200, Boris Fiuczynski wrote: >> On 07/24/2018 10:20 AM, Andrea Bolognani wrote: >>> Assuming the above is a correct reading of the situation, it >>> would seem the IDs are the only guest-visible part that we need >>> to make sure doesn't change during the lifetime of the guest, >>> while the usual bus/slot/function triplet assigned to the >>> underlying PCI device doesn't actually matter. And if that's the >>> case, wouldn't something like >>> >>> <address type='zpci' uid='1' fid='1'/> >> >> Then a pci device on s390 would need a special address type zpci in >> libvirt and the design idea for libvirt is to stay compatible with pci. > > Being compatible with the existing PCI machinery is certainly a > good idea when it makes sense to do so, but I'm not quite convinced > that is the case here. From a user perspective: I take your example below and reduce it to pci only like this: <controller type='pci' index='1' model='pci-bridge'/> <hostdev mode='subsystem' type='pci' managed='no'> <driver name='vfio'/> <source> <address domain='0xffff' bus='0x00' slot='0x00' function='0x0'/> </source> <address type='pci' domain='0x0000' bus='0x01' slot='0x1f' function='0x0'/> </hostdev> This works on x86 as well as on s390 where as your suggested zpci address type would not allow this. This is what I wanted to express with the word "compatible". As I wrote before: It would also be valid for a user to not care about the attributes domain, bus, slot and function and reduce the specified set of attributes to e.g. <address type='pci' uid='0xffff'/> > > According to Cornelia's blog post on the subject, the PCI topology > inside the guest will be determined entirely by the IDs. Is there > even a way to eg. use bridges to create a non-flat PCI hierarchy? > Or to have several PCI devices share the same bus or slot? > > If none of the above applies, then that doesn't look a whole lot > like PCI to me :) > > Moreover, we already have several address types in addition to PCI > such as USB, virtio-mmio, spapr-vio, ccw... Adding yet another one > is not a problem if it makes the interface more sensible. Sure you can add one more but wouldn't you end up with e.g. a hostdev model vfio-pci with an address type of pci on all pci supporting architectures but s390 where you need to use zpci? What would be the benefit for the user or higher level management software? Actually I would not like to introduce special handling unless required. > > More concrete questions: one of the zPCI test cases includes > > <controller type='pci' index='1' model='pci-bridge'/> > <hostdev mode='subsystem' type='pci' managed='no'> > <driver name='vfio'/> > <source> > <address domain='0xffff' bus='0x00' slot='0x00' function='0x0'/> > </source> > <address type='pci' domain='0x0000' bus='0x01' slot='0x1f' > function='0x0' uid='0xffff' fid='0xffffffff'/> > </hostdev> > > which translates to > > -device zpci,uid=3,fid=2,target=pci.1,id=zpci3 \ > -device pci-bridge,chassis_nr=1,id=pci.1,bus=pci.0,addr=0x1 \ > -device zpci,uid=65535,fid=4294967295,target=hostdev0,id=zpci65535 \ > -device vfio-pci,host=ffff:00:00.0,id=hostdev0,bus=pci.1,addr=0x1f \ > > How does the pci-bridge controller show up in the guest, if at all? > > Do the bus= and addr= attributes of vfio-pci and pci-bridge in the > example above matter eg. for migration purposes? > >> Therefore uid and fid are optional attributes and if not specified on >> s390 they are generated. The patch series also allows on s390 to specify >> the pci address type just with attributes uid and/or fid causing the >> rest of the pci address attributes to be generated. > > This goes without saying: users should not have to worry about > addresses at all unless they have very specific needs. > -- Mit freundlichen Grüßen/Kind regards Boris Fiuczynski IBM Deutschland Research & Development GmbH Vorsitzender des Aufsichtsrats: Martina Köderitz Geschäftsführung: Dirk Wittkopp Sitz der Gesellschaft: Böblingen Registergericht: Amtsgericht Stuttgart, HRB 243294 -- libvir-list mailing list libvir-list@redhat.com https://www.redhat.com/mailman/listinfo/libvir-list
On Tue, 2018-07-24 at 17:15 +0200, Boris Fiuczynski wrote: > On 07/24/2018 03:41 PM, Andrea Bolognani wrote: > > Being compatible with the existing PCI machinery is certainly a > > good idea when it makes sense to do so, but I'm not quite convinced > > that is the case here. > > From a user perspective: > I take your example below and reduce it to pci only like this: > > <controller type='pci' index='1' model='pci-bridge'/> > <hostdev mode='subsystem' type='pci' managed='no'> > <driver name='vfio'/> > <source> > <address domain='0xffff' bus='0x00' slot='0x00' function='0x0'/> > </source> > <address type='pci' domain='0x0000' bus='0x01' slot='0x1f' > function='0x0'/> > </hostdev> > > This works on x86 as well as on s390 where as your suggested zpci > address type would not allow this. This is what I wanted to express with > the word "compatible". > As I wrote before: It would also be valid for a user to not care about > the attributes domain, bus, slot and function and reduce the specified > set of attributes to e.g. <address type='pci' uid='0xffff'/> That's not really what users and management applications pass to libvirt, though: a more realistic example would be <hostdev mode='subsystem' type='pci'> <driver name='vfio'/> <source> <address domain='0xffff' bus='0x00' slot='0x00' function='0x0'/> </source> </hostdev> eg. you specify the host address and leave coming up with a suitable guest address entirely up to libvirt, in which case whether the resulting address is type=pci or type=zpci hardly matters. If you want to take device address assignment upon yourself, then you're gonna have to assign addresses to controllers as well, not to mention specify the entire PCI topology with all which that entails... Not exactly a common scenario. > > According to Cornelia's blog post on the subject, the PCI topology > > inside the guest will be determined entirely by the IDs. Is there > > even a way to eg. use bridges to create a non-flat PCI hierarchy? > > Or to have several PCI devices share the same bus or slot? > > > > If none of the above applies, then that doesn't look a whole lot > > like PCI to me :) > > > > Moreover, we already have several address types in addition to PCI > > such as USB, virtio-mmio, spapr-vio, ccw... Adding yet another one > > is not a problem if it makes the interface more sensible. > > Sure you can add one more but wouldn't you end up with e.g. a hostdev > model vfio-pci with an address type of pci on all pci supporting > architectures but s390 where you need to use zpci? What would be the > benefit for the user or higher level management software? Actually I > would not like to introduce special handling unless required. I'm all for offering users an interface that abstracts as many platform-specific quirks as possible, but there's a balance to be found and we should be careful not to lean too much the opposite way. With my current understanding, it doesn't look to me like zPCI behaves similarly enough to how PCI behaves on other platforms for us to sensibly describe both using the same interface, and the fact that QEMU had to come up with a specific middleware device seems to confirm my suspicion... In any case, would you mind answering the questions below? That would certainly help me gain a better understanding of the whole issue. > > More concrete questions: one of the zPCI test cases includes > > > > <controller type='pci' index='1' model='pci-bridge'/> > > <hostdev mode='subsystem' type='pci' managed='no'> > > <driver name='vfio'/> > > <source> > > <address domain='0xffff' bus='0x00' slot='0x00' function='0x0'/> > > </source> > > <address type='pci' domain='0x0000' bus='0x01' slot='0x1f' > > function='0x0' uid='0xffff' fid='0xffffffff'/> > > </hostdev> > > > > which translates to > > > > -device zpci,uid=3,fid=2,target=pci.1,id=zpci3 \ > > -device pci-bridge,chassis_nr=1,id=pci.1,bus=pci.0,addr=0x1 \ > > -device zpci,uid=65535,fid=4294967295,target=hostdev0,id=zpci65535 \ > > -device vfio-pci,host=ffff:00:00.0,id=hostdev0,bus=pci.1,addr=0x1f \ > > > > How does the pci-bridge controller show up in the guest, if at all? > > > > Do the bus= and addr= attributes of vfio-pci and pci-bridge in the > > example above matter eg. for migration purposes? -- Andrea Bolognani / Red Hat / Virtualization -- libvir-list mailing list libvir-list@redhat.com https://www.redhat.com/mailman/listinfo/libvir-list
在 2018/7/24 下午11:43, Andrea Bolognani 写道: > On Tue, 2018-07-24 at 17:15 +0200, Boris Fiuczynski wrote: >> On 07/24/2018 03:41 PM, Andrea Bolognani wrote: >>> Being compatible with the existing PCI machinery is certainly a >>> good idea when it makes sense to do so, but I'm not quite convinced >>> that is the case here. >> From a user perspective: >> I take your example below and reduce it to pci only like this: >> >> <controller type='pci' index='1' model='pci-bridge'/> >> <hostdev mode='subsystem' type='pci' managed='no'> >> <driver name='vfio'/> >> <source> >> <address domain='0xffff' bus='0x00' slot='0x00' function='0x0'/> >> </source> >> <address type='pci' domain='0x0000' bus='0x01' slot='0x1f' >> function='0x0'/> >> </hostdev> >> >> This works on x86 as well as on s390 where as your suggested zpci >> address type would not allow this. This is what I wanted to express with >> the word "compatible". >> As I wrote before: It would also be valid for a user to not care about >> the attributes domain, bus, slot and function and reduce the specified >> set of attributes to e.g. <address type='pci' uid='0xffff'/> > That's not really what users and management applications pass to > libvirt, though: a more realistic example would be > > <hostdev mode='subsystem' type='pci'> > <driver name='vfio'/> > <source> > <address domain='0xffff' bus='0x00' slot='0x00' function='0x0'/> > </source> > </hostdev> > > eg. you specify the host address and leave coming up with a > suitable guest address entirely up to libvirt, in which case > whether the resulting address is type=pci or type=zpci hardly > matters. > > If you want to take device address assignment upon yourself, then > you're gonna have to assign addresses to controllers as well, not > to mention specify the entire PCI topology with all which that > entails... Not exactly a common scenario. > >>> According to Cornelia's blog post on the subject, the PCI topology >>> inside the guest will be determined entirely by the IDs. Is there >>> even a way to eg. use bridges to create a non-flat PCI hierarchy? >>> Or to have several PCI devices share the same bus or slot? >>> >>> If none of the above applies, then that doesn't look a whole lot >>> like PCI to me :) >>> >>> Moreover, we already have several address types in addition to PCI >>> such as USB, virtio-mmio, spapr-vio, ccw... Adding yet another one >>> is not a problem if it makes the interface more sensible. >> Sure you can add one more but wouldn't you end up with e.g. a hostdev >> model vfio-pci with an address type of pci on all pci supporting >> architectures but s390 where you need to use zpci? What would be the >> benefit for the user or higher level management software? Actually I >> would not like to introduce special handling unless required. > I'm all for offering users an interface that abstracts as many > platform-specific quirks as possible, but there's a balance to be > found and we should be careful not to lean too much the opposite > way. > > With my current understanding, it doesn't look to me like zPCI > behaves similarly enough to how PCI behaves on other platforms > for us to sensibly describe both using the same interface, and > the fact that QEMU had to come up with a specific middleware > device seems to confirm my suspicion... > > In any case, would you mind answering the questions below? That > would certainly help me gain a better understanding of the whole > issue. > >>> More concrete questions: one of the zPCI test cases includes >>> >>> <controller type='pci' index='1' model='pci-bridge'/> >>> <hostdev mode='subsystem' type='pci' managed='no'> >>> <driver name='vfio'/> >>> <source> >>> <address domain='0xffff' bus='0x00' slot='0x00' function='0x0'/> >>> </source> >>> <address type='pci' domain='0x0000' bus='0x01' slot='0x1f' >>> function='0x0' uid='0xffff' fid='0xffffffff'/> >>> </hostdev> >>> >>> which translates to >>> >>> -device zpci,uid=3,fid=2,target=pci.1,id=zpci3 \ >>> -device pci-bridge,chassis_nr=1,id=pci.1,bus=pci.0,addr=0x1 \ >>> -device zpci,uid=65535,fid=4294967295,target=hostdev0,id=zpci65535 \ >>> -device vfio-pci,host=ffff:00:00.0,id=hostdev0,bus=pci.1,addr=0x1f \ >>> >>> How does the pci-bridge controller show up in the guest, if at all? Qemu hides pci-bridge devices and just exposes pci devices to the guest. In above example, indeed, qemu will generate a pci-bridge device and it will be existing in pci topology. But the guest can't see it. This is very special. >>> >>> Do the bus= and addr= attributes of vfio-pci and pci-bridge in the >>> example above matter eg. for migration purposes? Do you mean we leave attribute generation for bus and addr to qemu? -- libvir-list mailing list libvir-list@redhat.com https://www.redhat.com/mailman/listinfo/libvir-list
On Wed, 2018-07-25 at 16:34 +0800, Yi Min Zhao wrote: > 在 2018/7/24 下午11:43, Andrea Bolognani 写道: > > > > More concrete questions: one of the zPCI test cases includes > > > > > > > > <controller type='pci' index='1' model='pci-bridge'/> > > > > <hostdev mode='subsystem' type='pci' managed='no'> > > > > <driver name='vfio'/> > > > > <source> > > > > <address domain='0xffff' bus='0x00' slot='0x00' function='0x0'/> > > > > </source> > > > > <address type='pci' domain='0x0000' bus='0x01' slot='0x1f' > > > > function='0x0' uid='0xffff' fid='0xffffffff'/> > > > > </hostdev> > > > > > > > > which translates to > > > > > > > > -device zpci,uid=3,fid=2,target=pci.1,id=zpci3 \ > > > > -device pci-bridge,chassis_nr=1,id=pci.1,bus=pci.0,addr=0x1 \ > > > > -device zpci,uid=65535,fid=4294967295,target=hostdev0,id=zpci65535 \ > > > > -device vfio-pci,host=ffff:00:00.0,id=hostdev0,bus=pci.1,addr=0x1f \ > > > > > > > > How does the pci-bridge controller show up in the guest, if at all? > > Qemu hides pci-bridge devices and just exposes pci devices to the guest. > In above example, indeed, qemu will generate a pci-bridge device and it will > be existing in pci topology. But the guest can't see it. This is very > special. Yeah, that's kinda problematic as it violates the principle of least surprise... If s390 guests can only see a flat PCI topology, then we should find a way to reject bridges altogether instead of allowing the user to create them (or even create them automatically) only for them not to show up in the guest. > > > > Do the bus= and addr= attributes of vfio-pci and pci-bridge in the > > > > example above matter eg. for migration purposes? > > Do you mean we leave attribute generation for bus and addr to qemu? That would be the idea, but of course it can only work if the address of the underlying PCI device can change without affecting the guest in any way, including migration. If that's not the case, and the PCI address needs to be as stable as the IDs, then I don't think there's a way to avoid storing it in the guest XML, no matter how confusing that will end up looking. -- Andrea Bolognani / Red Hat / Virtualization -- libvir-list mailing list libvir-list@redhat.com https://www.redhat.com/mailman/listinfo/libvir-list
在 2018/7/25 下午11:44, Andrea Bolognani 写道: > On Wed, 2018-07-25 at 16:34 +0800, Yi Min Zhao wrote: >> 在 2018/7/24 下午11:43, Andrea Bolognani 写道: >>>>> More concrete questions: one of the zPCI test cases includes >>>>> >>>>> <controller type='pci' index='1' model='pci-bridge'/> >>>>> <hostdev mode='subsystem' type='pci' managed='no'> >>>>> <driver name='vfio'/> >>>>> <source> >>>>> <address domain='0xffff' bus='0x00' slot='0x00' function='0x0'/> >>>>> </source> >>>>> <address type='pci' domain='0x0000' bus='0x01' slot='0x1f' >>>>> function='0x0' uid='0xffff' fid='0xffffffff'/> >>>>> </hostdev> >>>>> >>>>> which translates to >>>>> >>>>> -device zpci,uid=3,fid=2,target=pci.1,id=zpci3 \ >>>>> -device pci-bridge,chassis_nr=1,id=pci.1,bus=pci.0,addr=0x1 \ >>>>> -device zpci,uid=65535,fid=4294967295,target=hostdev0,id=zpci65535 \ >>>>> -device vfio-pci,host=ffff:00:00.0,id=hostdev0,bus=pci.1,addr=0x1f \ >>>>> >>>>> How does the pci-bridge controller show up in the guest, if at all? >> Qemu hides pci-bridge devices and just exposes pci devices to the guest. >> In above example, indeed, qemu will generate a pci-bridge device and it will >> be existing in pci topology. But the guest can't see it. This is very >> special. > Yeah, that's kinda problematic as it violates the principle of least > surprise... If s390 guests can only see a flat PCI topology, then we > should find a way to reject bridges altogether instead of allowing > the user to create them (or even create them automatically) only for > them not to show up in the guest. If we reject bridges, there would be only one pci bus that maximum 32 pci devices could be plugged to. This kind of limitation is more problematic IMO. >>>>> Do the bus= and addr= attributes of vfio-pci and pci-bridge in the >>>>> example above matter eg. for migration purposes? >> Do you mean we leave attribute generation for bus and addr to qemu? > That would be the idea, but of course it can only work if the > address of the underlying PCI device can change without affecting > the guest in any way, including migration. If that's not the case, > and the PCI address needs to be as stable as the IDs, then I don't > think there's a way to avoid storing it in the guest XML, no matter > how confusing that will end up looking. > I think it relies on pci base code. Although we don't expose pci address to the guest, any pci device still exists PCI tree tolopogy in qemu. IIUC, this has effect on qemu process itself. For example, if we hotplug a pci device permanently, it will be dynamically assigned with a pci address, and this address might change after shutdown and start again the guest and also might change in destination during migration. -- libvir-list mailing list libvir-list@redhat.com https://www.redhat.com/mailman/listinfo/libvir-list
On Thu, 2018-07-26 at 13:43 +0800, Yi Min Zhao wrote: > 在 2018/7/25 下午11:44, Andrea Bolognani 写道: > > > > > > How does the pci-bridge controller show up in the guest, if at all? > > > > > > Qemu hides pci-bridge devices and just exposes pci devices to the guest. > > > In above example, indeed, qemu will generate a pci-bridge device and it will > > > be existing in pci topology. But the guest can't see it. This is very > > > special. > > > > Yeah, that's kinda problematic as it violates the principle of least > > surprise... If s390 guests can only see a flat PCI topology, then we > > should find a way to reject bridges altogether instead of allowing > > the user to create them (or even create them automatically) only for > > them not to show up in the guest. > > If we reject bridges, there would be only one pci bus that maximum > 32 pci devices could be plugged to. This kind of limitation is more > problematic IMO. I see how that would be pretty limiting. >From the test cases I see a zpci devices, with its own uid and fid, is created for the pci-bridge as well... Is that intentional? > > > > > > Do the bus= and addr= attributes of vfio-pci and pci-bridge in the > > > > > > example above matter eg. for migration purposes? > > > > > > Do you mean we leave attribute generation for bus and addr to qemu? > > > > That would be the idea, but of course it can only work if the > > address of the underlying PCI device can change without affecting > > the guest in any way, including migration. If that's not the case, > > and the PCI address needs to be as stable as the IDs, then I don't > > think there's a way to avoid storing it in the guest XML, no matter > > how confusing that will end up looking. > > I think it relies on pci base code. Although we don't expose pci address > to the guest, any > pci device still exists PCI tree tolopogy in qemu. IIUC, this has effect > on qemu process itself. > For example, if we hotplug a pci device permanently, it will be > dynamically assigned with a > pci address, and this address might change after shutdown and start > again the guest and also > might change in destination during migration. Okay, if that's the case we'll definitely have to store the PCI address in the guest XML :( -- Andrea Bolognani / Red Hat / Virtualization -- libvir-list mailing list libvir-list@redhat.com https://www.redhat.com/mailman/listinfo/libvir-list
在 2018/7/26 下午7:00, Andrea Bolognani 写道: >>>>>>> How does the pci-bridge controller show up in the guest, if at all? >>>> Qemu hides pci-bridge devices and just exposes pci devices to the guest. >>>> In above example, indeed, qemu will generate a pci-bridge device and it will >>>> be existing in pci topology. But the guest can't see it. This is very >>>> special. >>> Yeah, that's kinda problematic as it violates the principle of least >>> surprise... If s390 guests can only see a flat PCI topology, then we >>> should find a way to reject bridges altogether instead of allowing >>> the user to create them (or even create them automatically) only for >>> them not to show up in the guest. >> If we reject bridges, there would be only one pci bus that maximum >> 32 pci devices could be plugged to. This kind of limitation is more >> problematic IMO. > I see how that would be pretty limiting. > > >From the test cases I see a zpci devices, with its own uid and fid, > is created for the pci-bridge as well... Is that intentional? > Firstly pci bridge can be auto-generated if a pci device is to be plugged to non-existing pci bus. IIUC, pci-bridge is treated as a controller device in libvirt. So I think, it's pretty readable not only in libvirt xml but also in qtree, if we assign zpci device for it. Otherwise address type of pci-bridge is pci type but has no uid and fid. Isn't it odd? Of course, we could avoid assigning a zpci device to pci-bridge. Actually my old patch did in this way. -- libvir-list mailing list libvir-list@redhat.com https://www.redhat.com/mailman/listinfo/libvir-list
On Thu, Jul 26, 2018 at 07:17:03PM +0800, Yi Min Zhao wrote: > > > 在 2018/7/26 下午7:00, Andrea Bolognani 写道: > > > > > > > > How does the pci-bridge controller show up in the guest, if at all? > > > > > Qemu hides pci-bridge devices and just exposes pci devices to the guest. > > > > > In above example, indeed, qemu will generate a pci-bridge device and it will > > > > > be existing in pci topology. But the guest can't see it. This is very > > > > > special. > > > > Yeah, that's kinda problematic as it violates the principle of least > > > > surprise... If s390 guests can only see a flat PCI topology, then we > > > > should find a way to reject bridges altogether instead of allowing > > > > the user to create them (or even create them automatically) only for > > > > them not to show up in the guest. > > > If we reject bridges, there would be only one pci bus that maximum > > > 32 pci devices could be plugged to. This kind of limitation is more > > > problematic IMO. > > I see how that would be pretty limiting. > > > > >From the test cases I see a zpci devices, with its own uid and fid, > > is created for the pci-bridge as well... Is that intentional? > > > Firstly pci bridge can be auto-generated if a pci device is to be plugged to > non-existing pci bus. > IIUC, pci-bridge is treated as a controller device in libvirt. So I think, > it's pretty readable not only > in libvirt xml but also in qtree, if we assign zpci device for it. Otherwise > address type of pci-bridge > is pci type but has no uid and fid. Isn't it odd? >From the libvirt side we must avoid any scenario where QEMU auto-adds devices behind our back. If adding a device requires adding a controller libvirt must do this explicitly and record it in the XML. Regards, Daniel -- |: https://berrange.com -o- https://www.flickr.com/photos/dberrange :| |: https://libvirt.org -o- https://fstop138.berrange.com :| |: https://entangle-photo.org -o- https://www.instagram.com/dberrange :| -- libvir-list mailing list libvir-list@redhat.com https://www.redhat.com/mailman/listinfo/libvir-list
On Thu, 2018-07-26 at 12:22 +0100, Daniel P. Berrangé wrote: > On Thu, Jul 26, 2018 at 07:17:03PM +0800, Yi Min Zhao wrote: > > 在 2018/7/26 下午7:00, Andrea Bolognani 写道: > > > From the test cases I see a zpci devices, with its own uid and fid, > > > is created for the pci-bridge as well... Is that intentional? > > > > Firstly pci bridge can be auto-generated if a pci device is to be plugged to > > non-existing pci bus. > > IIUC, pci-bridge is treated as a controller device in libvirt. So I think, > > it's pretty readable not only > > in libvirt xml but also in qtree, if we assign zpci device for it. Otherwise > > address type of pci-bridge > > is pci type but has no uid and fid. Isn't it odd? Everything about zPCI is odd ;) I guess there's no harm in creating an additional zpci device, and as you say it will keep things a bit more consistent, which is good. > From the libvirt side we must avoid any scenario where QEMU auto-adds > devices behind our back. If adding a device requires adding a controller > libvirt must do this explicitly and record it in the XML. Definitely. My question was whether the corresponding zpci device should be created as well... -- Andrea Bolognani / Red Hat / Virtualization -- libvir-list mailing list libvir-list@redhat.com https://www.redhat.com/mailman/listinfo/libvir-list
On Thu, Jul 26, 2018 at 01:47:52PM +0200, Andrea Bolognani wrote: > On Thu, 2018-07-26 at 12:22 +0100, Daniel P. Berrangé wrote: > > On Thu, Jul 26, 2018 at 07:17:03PM +0800, Yi Min Zhao wrote: > > > 在 2018/7/26 下午7:00, Andrea Bolognani 写道: > > > > From the test cases I see a zpci devices, with its own uid and fid, > > > > is created for the pci-bridge as well... Is that intentional? > > > > > > Firstly pci bridge can be auto-generated if a pci device is to be plugged to > > > non-existing pci bus. > > > IIUC, pci-bridge is treated as a controller device in libvirt. So I think, > > > it's pretty readable not only > > > in libvirt xml but also in qtree, if we assign zpci device for it. Otherwise > > > address type of pci-bridge > > > is pci type but has no uid and fid. Isn't it odd? > > Everything about zPCI is odd ;) > > I guess there's no harm in creating an additional zpci device, > and as you say it will keep things a bit more consistent, which > is good. > > > From the libvirt side we must avoid any scenario where QEMU auto-adds > > devices behind our back. If adding a device requires adding a controller > > libvirt must do this explicitly and record it in the XML. > > Definitely. My question was whether the corresponding zpci device > should be created as well... I'm not sure I understand it fully, but it sounds like zpci devices are providing info that is guest ABI sensitive, which would mean libvirt must control and record it. So from that POV we should create zpci devices Regards, Daniel -- |: https://berrange.com -o- https://www.flickr.com/photos/dberrange :| |: https://libvirt.org -o- https://fstop138.berrange.com :| |: https://entangle-photo.org -o- https://www.instagram.com/dberrange :| -- libvir-list mailing list libvir-list@redhat.com https://www.redhat.com/mailman/listinfo/libvir-list
On Thu, 2018-07-26 at 12:52 +0100, Daniel P. Berrangé wrote: > On Thu, Jul 26, 2018 at 01:47:52PM +0200, Andrea Bolognani wrote: > > On Thu, 2018-07-26 at 12:22 +0100, Daniel P. Berrangé wrote: > > > From the libvirt side we must avoid any scenario where QEMU auto-adds > > > devices behind our back. If adding a device requires adding a controller > > > libvirt must do this explicitly and record it in the XML. > > > > Definitely. My question was whether the corresponding zpci device > > should be created as well... > > I'm not sure I understand it fully, but it sounds like zpci devices are > providing info that is guest ABI sensitive, which would mean libvirt must > control and record it. So from that POV we should create zpci devices Making sure the guest-visible address is stable is taken care of by the additional PCI address attributes uid and fid, which are then used when creating the zpci device. PCI controllers such as pci-bridge are, however, not visible to the guest and as such it would be possible to skip the corresponding zpci devices; it's been argued, though, that it's better to create them regardless in order to maintain consistency. -- Andrea Bolognani / Red Hat / Virtualization -- libvir-list mailing list libvir-list@redhat.com https://www.redhat.com/mailman/listinfo/libvir-list
Hi all, The next release is coming soon. I'm really expecting the comments from all of you. Hope you could consider a review before the next review. Thank you very much! Yi Min 在 2018/7/10 下午4:02, Yi Min Zhao 写道: > Abstract > ======== > The PCI representation in QEMU has recently been extended for S390 > allowing configuration of zPCI attributes like uid (user-defined > identifier) and fid (PCI function identifier). > The details can be found here: > https://lists.gnu.org/archive/html/qemu-devel/2016-06/msg07262.html > > To support the new zPCI feature of the S390 platform, two new XML > attributes, @uid and @fid, are introduced for device addresses of type > 'pci', i.e.: > <hostdev mode='subsystem' type='pci'> > <driver name='vfio'/> > <source> > <address domain='0x0001' bus='0x00' slot='0x00' function='0x0'/> > </source> > <address type='pci' domain='0x0000' bus='0x00' slot='0x01' function='0x0' > uid='0x0003' fid='0x00000027'/> > </hostdev> > > uid and fid are optional attributes. If they are defined by the user, > unique values within the guest domain must be used. If they are not > specified and the architecture requires them, they are automatically > generated with non-conflicting values. > > Current implementation is the most seamless one for the user as it > unites the address specific data of a PCI device on one XML element. > It could accommodate both specifying our special parameters (uid and fid) > and re-using standard statements (domain, bus, slot and function) for > PCI devices. User can still specify bus/slot/function for the virtualized > PCI devices in the XML. > > Thus uid/fid act as an extension to the PCI address and are stored in > a new structure 'virZPCIDeviceAddress' which is a member of common PCI > Address structure. Additionally, two hashtables are used for assignment > and reservation of uid/fid. > > In support of extending the PCI address, a new PCI address extension flag is > introduced. This extension flag allows is not only dedicated for the S390 > platform but also other architectures needing certain extensions to PCI > address space. > > Code Base > ========= > commit in master: > 767f9e1449b1a36111532847f0c62dc758263c42 > qemu: validate: Enforce compile time switch type checking for videos > > Change Log > ========== > v1->v2: > 1. Separate test commit and merge testcases into corresponding commits that > introduce the functionalities firstly. > 2. Spare some checks for zpci device. > 3. Add vsock and controller support. > 4. Add uin32 type schema. > 5. Rename zpciuid and zpcifid to zpci_uid and zpci_fid. > 6. Always return multibus support on S390. > > Yi Min Zhao (12): > conf: Add definitions for 'uid' and 'fid' PCI address attributes > qemu: Introduce zPCI capability > conf: Introduce a new PCI address extension flag > qemu: Enable PCI multi bus for S390 guests > qemu: Auto add pci-root for s390/s390x guests > conf: Introduce address caching for PCI extensions > conf: Introduce parser, formatter for uid and fid > conf: Allocate/release 'uid' and 'fid' in PCI address > qemu: Generate and use zPCI device in QEMU command line > qemu: Add hotpluging support for PCI devices on S390 guests > docs: Add 'uid' and 'fid' information > news: Update news for PCI address extension attributes > > docs/formatdomain.html.in | 9 +- > docs/news.xml | 11 + > docs/schemas/basictypes.rng | 31 ++ > docs/schemas/domaincommon.rng | 1 + > src/conf/device_conf.c | 73 +++++ > src/conf/device_conf.h | 1 + > src/conf/domain_addr.c | 346 +++++++++++++++++++++ > src/conf/domain_addr.h | 29 ++ > src/conf/domain_conf.c | 6 + > src/libvirt_private.syms | 4 + > src/qemu/qemu_capabilities.c | 6 + > src/qemu/qemu_capabilities.h | 1 + > src/qemu/qemu_command.c | 115 +++++++ > src/qemu/qemu_command.h | 4 + > src/qemu/qemu_domain.c | 1 + > src/qemu/qemu_domain_address.c | 181 ++++++++++- > src/qemu/qemu_hotplug.c | 182 ++++++++++- > src/util/virpci.h | 13 + > tests/qemucapabilitiesdata/caps_2.10.0.s390x.xml | 1 + > tests/qemucapabilitiesdata/caps_2.11.0.s390x.xml | 1 + > tests/qemucapabilitiesdata/caps_2.12.0.s390x.xml | 1 + > tests/qemucapabilitiesdata/caps_2.7.0.s390x.xml | 1 + > tests/qemucapabilitiesdata/caps_2.8.0.s390x.xml | 1 + > tests/qemucapabilitiesdata/caps_2.9.0.s390x.xml | 1 + > tests/qemuxml2argvdata/disk-virtio-s390-zpci.args | 27 ++ > tests/qemuxml2argvdata/disk-virtio-s390-zpci.xml | 17 + > .../hostdev-vfio-zpci-autogenerate.args | 26 ++ > .../hostdev-vfio-zpci-autogenerate.xml | 18 ++ > .../hostdev-vfio-zpci-boundaries.args | 30 ++ > .../hostdev-vfio-zpci-boundaries.xml | 26 ++ > .../hostdev-vfio-zpci-multidomain-many.args | 40 +++ > .../hostdev-vfio-zpci-multidomain-many.xml | 67 ++++ > tests/qemuxml2argvdata/hostdev-vfio-zpci.args | 26 ++ > tests/qemuxml2argvdata/hostdev-vfio-zpci.xml | 19 ++ > tests/qemuxml2argvtest.c | 14 + > tests/qemuxml2xmloutdata/disk-virtio-s390-zpci.xml | 29 ++ > tests/qemuxml2xmloutdata/hostdev-vfio-zpci.xml | 30 ++ > tests/qemuxml2xmltest.c | 3 + > 38 files changed, 1377 insertions(+), 15 deletions(-) > create mode 100644 tests/qemuxml2argvdata/disk-virtio-s390-zpci.args > create mode 100644 tests/qemuxml2argvdata/disk-virtio-s390-zpci.xml > create mode 100644 tests/qemuxml2argvdata/hostdev-vfio-zpci-autogenerate.args > create mode 100644 tests/qemuxml2argvdata/hostdev-vfio-zpci-autogenerate.xml > create mode 100644 tests/qemuxml2argvdata/hostdev-vfio-zpci-boundaries.args > create mode 100644 tests/qemuxml2argvdata/hostdev-vfio-zpci-boundaries.xml > create mode 100644 tests/qemuxml2argvdata/hostdev-vfio-zpci-multidomain-many.args > create mode 100644 tests/qemuxml2argvdata/hostdev-vfio-zpci-multidomain-many.xml > create mode 100644 tests/qemuxml2argvdata/hostdev-vfio-zpci.args > create mode 100644 tests/qemuxml2argvdata/hostdev-vfio-zpci.xml > create mode 100644 tests/qemuxml2xmloutdata/disk-virtio-s390-zpci.xml > create mode 100644 tests/qemuxml2xmloutdata/hostdev-vfio-zpci.xml > -- libvir-list mailing list libvir-list@redhat.com https://www.redhat.com/mailman/listinfo/libvir-list
© 2016 - 2024 Red Hat, Inc.