Create host_bridge_info struct describing PCI host bridges' resources
so that one or more host bridges can be specified by the hypervisor
and their information propagated through fw-cfg. The host is able to
determine the bus number, the bus number range, the NUMA proximity and
all MMIO resources each root bridge uses (low and high memory both
prefetchable and non-prefetchable).

With the increased control we give for the host to define the resources
each root bridge can use, the case may arise in which one of the
ranges, particularly the 64-bit MMIO window, exceeds the usable range.
Taking into account that PcdPciMmio64Size token determines, together with
PcdPciMmio64Base, the usable 64-bit range, we verify if the MMIO windows
set in the specification fit inside the valid range. If one of the root
bridges is set to use a higher memory address space, PcdPciMmio64Size
token is increased so that the intended window lies inside the valid range.

For example, using PcdPciMmio64Size of 0x800000000 and a runtime calculated
base address also of 0x800000000, the high memory usable addresses are
restricted to 0x1000000000. With the increased flexibility to specify
host bridge resources, a feasible example would be:

struct host_bridge_info pci_hb_spec[] = {
        {
                .set_pxm                                    = true,
                .mark_prefetchable_resources_as_cacheable   = true,

                .root_bus_nr            = 0x00,
                .pxm                    = 1,
                .num_hot_plug_slots     = 32,

                .root_bur_nr_start      = 0x00,
                .root_bus_nr_end        = 0x7F,

                .lowmem_start           = 0xC0000000,
                .lowmem_end             = 0xD0000000 - 1,
                .lowmem_pref_start      = 0xFFFFFFFF,
                .lowmem_pref_end        = 0xFFFFFFFF,

                .highmem_start          = 0xFFFFFFFFFFFFFFFF,
                .highmem_end            = 0xFFFFFFFFFFFFFFFF,
                .highmem_pref_start     = 0x800000000,
                .highmem_pref_end       = 0x1000000000 - 1
        },
        {
                .set_pxm                                        = true,
                .mark_prefetchable_resources_as_cacheable       = true,

                .root_bus_nr            = 0x80,
                .pxm                    = 1,
                .num_hot_plug_slots     = 32,

                .root_bur_nr_start      = 0x80,
                .root_bus_nr_end        = 0xFF,

                .lowmem_start           = 0xD0000000,
                .lowmem_end             = 0xE0000000 - 1,
                .lowmem_pref_start      = 0xFFFFFFFF,
                .lowmem_pref_end        = 0xFFFFFFFF,

                .highmem_start          = 0xFFFFFFFFFFFFFFFF,
                .highmem_end            = 0xFFFFFFFFFFFFFFFF,
                .highmem_pref_start     = 0x1000000000,
                .highmem_pref_end       = 0x1800000000 - 1
        }
};

The first host bridge already uses all the available space. Consequently,
the 64-bit PCI size needs to be extended. For that, the resource spec
exemplified is read from fw-cfg and PcdPciMmio64Size extended to
0x1000000000 so that both host bridges fit inside the valid window.

Signed-off-by: Nicolas Ojeda Leon <ncoleon@amazon.com>
Cc: Alexander Graf <graf@amazon.de>
---
 .../Include/Library/PciHostBridgeInfoLib.h    | 43 ++++++++++
 OvmfPkg/PlatformPei/MemDetect.c               | 83 +++++++++++++++++++
 2 files changed, 126 insertions(+)
 create mode 100644 OvmfPkg/Include/Library/PciHostBridgeInfoLib.h

diff --git a/OvmfPkg/Include/Library/PciHostBridgeInfoLib.h b/OvmfPkg/Include/Library/PciHostBridgeInfoLib.h
new file mode 100644
index 0000000000..e09e6c496e
--- /dev/null
+++ b/OvmfPkg/Include/Library/PciHostBridgeInfoLib.h
@@ -0,0 +1,43 @@
+/*
+ * Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * This program and the accompanying materials
+ * are licensed and made available under the terms and conditions of the BSD License
+ * which accompanies this distribution.  The full text of the license may be found at
+ * http://opensource.org/licenses/bsd-license.php
+ *
+ * THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
+ */
+
+#ifndef __PCI_HOST_BRIDGE_INFO_LIB_H__
+#define __PCI_HOST_BRIDGE_INFO_LIB_H__
+
+//
+// Root bridge resource information for parsing fw-cfg data
+//
+#pragma pack(1)
+typedef struct {
+  BOOLEAN set_pxm;
+  BOOLEAN mark_prefetchable_resources_as_cacheable;
+
+  UINT8 root_bus_nr;
+  UINT8 pxm;
+  UINT8 num_hot_plug_slots;
+
+  UINT8 root_bur_nr_start;
+  UINT8 root_bus_nr_end;
+
+  UINT32 lowmem_start;
+  UINT32 lowmem_end;
+  UINT32 lowmem_pref_start;
+  UINT32 lowmem_pref_end;
+
+  UINT64 highmem_start;
+  UINT64 highmem_end;
+  UINT64 highmem_pref_start;
+  UINT64 highmem_pref_end;
+} HOST_BRIDGE_INFO;
+#pragma pack()
+
+#endif
diff --git a/OvmfPkg/PlatformPei/MemDetect.c b/OvmfPkg/PlatformPei/MemDetect.c
index 2c2c4641ec..2872e805a4 100644
--- a/OvmfPkg/PlatformPei/MemDetect.c
+++ b/OvmfPkg/PlatformPei/MemDetect.c
@@ -30,6 +30,7 @@ Module Name:
 #include <Library/MemEncryptSevLib.h>
 #include <Library/PcdLib.h>
 #include <Library/PciLib.h>
+#include <Library/PciHostBridgeInfoLib.h>
 #include <Library/PeimEntryPoint.h>
 #include <Library/ResourcePublicationLib.h>
 #include <Library/MtrrLib.h>
@@ -323,6 +324,83 @@ GetSystemMemorySizeAbove4gb (
 }
 
 
+/**
+  Iterate over the PCI host bridges resources information optionally provided
+  in fw-cfg.
+
+  Find the highest address used by the PCI bridges in 64-bit MMIO space to
+  calculate and modify the PCI aperture size accordingly (PciMmio64Size)
+
+  @param[in] PciMmio64Base    Base address (start) of the 64-bit PCI MMIO
+                              address space.
+
+  @param[inout] PciMmio64Size Size of the PCI 64-bit MMIO aperture provided
+                              as input and modified (output) if the resources
+                              indicated by fw_cfg require a larger address
+                              space.
+
+  @retval EFI_SUCCESS         The fw_cfg host-bridges-info was found and
+                              processed.
+
+  @retval EFI_PROTOCOL_ERROR  The host bridges information file was found,
+                              but its size wasn't a whole multiple of
+                              sizeof(HOST_BRIDGE_INFO). No entry was processed.
+
+  @retval EFI_NOT_FOUND       fw-cfg file with host bridges information was not
+                              found. Does not constitute an errro since the file
+                              is optional and used in special cases.
+
+  @retval EFI_UNSUPPORTED     fw-cfg is unavailable
+
+**/
+STATIC
+EFI_STATUS
+ScanPci64BitApertureSize (
+  IN UINT64         PciMmio64Base,
+  IN OUT UINT64     *PciMmio64Size
+  )
+{
+  EFI_STATUS           Status;
+  FIRMWARE_CONFIG_ITEM FwCfgItem;
+  HOST_BRIDGE_INFO     HostBridge;
+  UINTN                FwCfgSize;
+  UINTN                Processed;
+  UINT64               PciEnd;
+  UINT64               PcdPciEnd;
+
+  Status = QemuFwCfgFindFile ("etc/host-bridge-info", &FwCfgItem, &FwCfgSize);
+  if (EFI_ERROR (Status)) {
+    return Status;
+  }
+  if (FwCfgSize % sizeof HostBridge != 0) {
+    return EFI_PROTOCOL_ERROR;
+  }
+
+  PciEnd = 0;
+  QemuFwCfgSelectItem (FwCfgItem);
+  for (Processed = 0; Processed < FwCfgSize; Processed += sizeof HostBridge) {
+    QemuFwCfgReadBytes (sizeof HostBridge, &HostBridge);
+
+    if (HostBridge.highmem_end != MAX_UINT64 &&
+        HostBridge.highmem_end > PciEnd) {
+      PciEnd = HostBridge.highmem_end;
+    }
+    if (HostBridge.highmem_pref_end != MAX_UINT64 &&
+        HostBridge.highmem_pref_end > PciEnd) {
+      PciEnd = HostBridge.highmem_pref_end;
+    }
+  }
+
+  PcdPciEnd = PciMmio64Base + *PciMmio64Size;
+  if (PciEnd > PcdPciEnd) {
+    *PciMmio64Size = PciEnd - PciMmio64Base;
+    *PciMmio64Size = ALIGN_VALUE (*PciMmio64Size, (UINT64)SIZE_1GB);
+  }
+
+  return EFI_SUCCESS;
+}
+
+
 /**
   Return the highest address that DXE could possibly use, plus one.
 **/
@@ -452,6 +530,11 @@ GetFirstNonAddress (
   //
   Pci64Base = ALIGN_VALUE (Pci64Base, GetPowerOfTwo64 (Pci64Size));
 
+  //
+  // Extend Pci64Size if fw_cfg Host bridges specification requires it
+  //
+  ScanPci64BitApertureSize (Pci64Base, &Pci64Size);
+
   if (mBootMode != BOOT_ON_S3_RESUME) {
     //
     // The core PciHostBridgeDxe driver will automatically add this range to
-- 
2.17.1




Amazon Development Center Germany GmbH
Krausenstr. 38
10117 Berlin
Geschaeftsfuehrung: Christian Schlaeger, Jonathan Weiss
Eingetragen am Amtsgericht Charlottenburg unter HRB 149173 B
Sitz: Berlin
Ust-ID: DE 289 237 879





-=-=-=-=-=-=-=-=-=-=-=-
Groups.io Links: You receive all messages sent to this group.
View/Reply Online (#80299): https://edk2.groups.io/g/devel/message/80299
Mute This Topic: https://groups.io/mt/85437209/1787277
Group Owner: devel+owner@edk2.groups.io
Unsubscribe: https://edk2.groups.io/g/devel/unsub [importer@patchew.org]
-=-=-=-=-=-=-=-=-=-=-=-

> +//
> +// Root bridge resource information for parsing fw-cfg data
> +//
> +#pragma pack(1)
> +typedef struct {
> +  BOOLEAN set_pxm;
> +  BOOLEAN mark_prefetchable_resources_as_cacheable;
> +
> +  UINT8 root_bus_nr;
> +  UINT8 pxm;
> +  UINT8 num_hot_plug_slots;
> +
> +  UINT8 root_bur_nr_start;
> +  UINT8 root_bus_nr_end;
> +
> +  UINT32 lowmem_start;
> +  UINT32 lowmem_end;
> +  UINT32 lowmem_pref_start;
> +  UINT32 lowmem_pref_end;
> +
> +  UINT64 highmem_start;
> +  UINT64 highmem_end;
> +  UINT64 highmem_pref_start;
> +  UINT64 highmem_pref_end;
> +} HOST_BRIDGE_INFO;
> +#pragma pack()

Hmm.  A number of fields is not used by this patch series, pxm for
example.  On the other hand it is incomplete, ioports are not there.
It looks like you are simply throwing some internal data structure
into fw_cfg.  This is *not* how things should be done.

Is the host-side code for this open source btw?  If so, do you
have a pointer?

I see two reasonable options going forward:

  (1) Create a format which contains *everything* needed to describe
      a pci(e) host bridge, not only those parts which you happen to
      need for your use case.  Basically all information needed by
      PciHostBridgeUtilityInitRootBridge(), and the pcie config space
      location.  Drop fields not needed.  And add comments documenting
      the struct.
  (2) Use an existing format.  Obvious choice coming to mind is
      device-tree.  We already have a parser for that because the
      arm platform (and riscv too soon) needs it to find the pcie
      host bridge.

take care,
  Gerd



-=-=-=-=-=-=-=-=-=-=-=-
Groups.io Links: You receive all messages sent to this group.
View/Reply Online (#80323): https://edk2.groups.io/g/devel/message/80323
Mute This Topic: https://groups.io/mt/85437209/1787277
Group Owner: devel+owner@edk2.groups.io
Unsubscribe: https://edk2.groups.io/g/devel/unsub [importer@patchew.org]
-=-=-=-=-=-=-=-=-=-=-=-