Refactor setup_frametable_mappings() into init_frametable(), modeled after x86's implementation. Instead of mapping one contiguous frametable covering ram_start to ram_end (including holes), iterate the pdx_group_valid bitmap to allocate and map frametable memory only for valid PDX groups, skipping gaps in the physical address space. At the moment we don't really take into account pdx_group_valid bitmap. This reduces memory consumption on systems with sparse RAM layouts by not allocating frametable entries for non-existent memory regions. A file-local pdx_to_page() override is needed because the generic macro in xen/include/xen/pdx.h does not account for ARM's non-zero frametable_base_pdx. Update the MPU implementation to match the new init_frametable() signature. Since MPU has no virtual address translation (ma == va), hole-skipping is not possible and the frametable remains a single contiguous allocation. Signed-off-by: Michal Orzel <michal.orzel@amd.com> --- We've been using this approach at AMD for a while now. Without this we would not be able to boot some of our boards that have huge holes in the PA space, so I consider this patch a great improvement. Two things to consider as a follow-up in the future: - change generic pdx_to_page, page_to_pdx to take into account offset that on x86 is zero but on other arches it is not. The page list code is for now unaffected because the offset cancels out, - use the same on RISCV. --- xen/arch/arm/arm32/mmu/mm.c | 3 +- xen/arch/arm/include/asm/mm.h | 4 +- xen/arch/arm/mm.c | 2 +- xen/arch/arm/mmu/mm.c | 77 ++++++++++++++++++++++++----------- xen/arch/arm/mpu/mm.c | 23 ++++++----- 5 files changed, 70 insertions(+), 39 deletions(-) diff --git a/xen/arch/arm/arm32/mmu/mm.c b/xen/arch/arm/arm32/mmu/mm.c index XXXXXXX..XXXXXXX 100644 --- a/xen/arch/arm/arm32/mmu/mm.c +++ b/xen/arch/arm/arm32/mmu/mm.c @@ -XXX,XX +XXX,XX @@ void __init setup_mm(void) setup_directmap_mappings(mfn_x(directmap_mfn_start), xenheap_pages); - /* Frame table covers all of RAM region, including holes */ - setup_frametable_mappings(ram_start, ram_end); + init_frametable(ram_start); /* * The allocators may need to use map_domain_page() (such as for diff --git a/xen/arch/arm/include/asm/mm.h b/xen/arch/arm/include/asm/mm.h index XXXXXXX..XXXXXXX 100644 --- a/xen/arch/arm/include/asm/mm.h +++ b/xen/arch/arm/include/asm/mm.h @@ -XXX,XX +XXX,XX @@ extern void *early_fdt_map(paddr_t fdt_paddr); extern void remove_early_mappings(void); /* Prepare the memory subystem to bring-up the given secondary CPU */ extern int prepare_secondary_mm(int cpu); -/* Map a frame table to cover physical addresses ps through pe */ -extern void setup_frametable_mappings(paddr_t ps, paddr_t pe); +/* Map a frame table */ +void init_frametable(paddr_t ram_start); /* Helper function to setup memory management */ void setup_mm_helper(void); /* map a physical range in virtual memory */ diff --git a/xen/arch/arm/mm.c b/xen/arch/arm/mm.c index XXXXXXX..XXXXXXX 100644 --- a/xen/arch/arm/mm.c +++ b/xen/arch/arm/mm.c @@ -XXX,XX +XXX,XX @@ void __init setup_mm(void) setup_mm_helper(); - setup_frametable_mappings(ram_start, ram_end); + init_frametable(ram_start); init_staticmem_pages(); init_sharedmem_pages(); diff --git a/xen/arch/arm/mmu/mm.c b/xen/arch/arm/mmu/mm.c index XXXXXXX..XXXXXXX 100644 --- a/xen/arch/arm/mmu/mm.c +++ b/xen/arch/arm/mmu/mm.c @@ -XXX,XX +XXX,XX @@ #include <xen/pdx.h> #include <xen/string.h> -/* Map a frame table to cover physical addresses ps through pe */ -void __init setup_frametable_mappings(paddr_t ps, paddr_t pe) +#undef pdx_to_page +#define pdx_to_page(pdx) gcc11_wrap(frame_table + ((pdx) - frametable_base_pdx)) + +static void __init +init_frametable_chunk(unsigned long pdx_s, unsigned long pdx_e) { - unsigned long nr_pdxs = mfn_to_pdx(mfn_add(maddr_to_mfn(pe), -1)) - - mfn_to_pdx(maddr_to_mfn(ps)) + 1; - unsigned long frametable_size = nr_pdxs * sizeof(struct page_info); - mfn_t base_mfn; - const unsigned long mapping_size = frametable_size < MB(32) ? MB(2) - : MB(32); + unsigned long nr_pdxs = pdx_e - pdx_s; + unsigned long chunk_size = nr_pdxs * sizeof(struct page_info); + const unsigned long mapping_size = chunk_size < MB(32) ? MB(2) : MB(32); + unsigned long virt; int rc; + mfn_t base_mfn; + + /* Round up to 2M or 32M boundary, as appropriate. */ + chunk_size = ROUNDUP(chunk_size, mapping_size); + base_mfn = alloc_boot_pages(chunk_size >> PAGE_SHIFT, 32 << (20 - 12)); + + virt = (unsigned long)pdx_to_page(pdx_s); + rc = map_pages_to_xen(virt, base_mfn, chunk_size >> PAGE_SHIFT, + PAGE_HYPERVISOR_RW | _PAGE_BLOCK); + if ( rc ) + panic("Unable to setup the frametable mappings\n"); + + memset(pdx_to_page(pdx_s), 0, nr_pdxs * sizeof(struct page_info)); + memset(pdx_to_page(pdx_e), -1, + chunk_size - nr_pdxs * sizeof(struct page_info)); +} + +void __init init_frametable(paddr_t ram_start) +{ + unsigned int sidx, nidx, max_idx; /* * The size of paddr_t should be sufficient for the complete range of @@ -XXX,XX +XXX,XX @@ void __init setup_frametable_mappings(paddr_t ps, paddr_t pe) BUILD_BUG_ON((sizeof(paddr_t) * BITS_PER_BYTE) < PADDR_BITS); BUILD_BUG_ON(sizeof(struct page_info) != PAGE_INFO_SIZE); - if ( frametable_size > FRAMETABLE_SIZE ) - panic("The frametable cannot cover the physical region %#"PRIpaddr" - %#"PRIpaddr"\n", - ps, pe); + max_idx = DIV_ROUND_UP(max_pdx, PDX_GROUP_COUNT); + frametable_base_pdx = mfn_to_pdx(maddr_to_mfn(ram_start)); - frametable_base_pdx = mfn_to_pdx(maddr_to_mfn(ps)); - /* Round up to 2M or 32M boundary, as appropriate. */ - frametable_size = ROUNDUP(frametable_size, mapping_size); - base_mfn = alloc_boot_pages(frametable_size >> PAGE_SHIFT, 32<<(20-12)); + /* + * pdx_to_page(pdx_s) in init_frametable_chunk must be page-aligned + * for map_pages_to_xen(). Aligning to PDX_GROUP_COUNT guarantees this + * because PDX_GROUP_COUNT * sizeof(page_info) is always a multiple of + * PAGE_SIZE by construction. + */ + frametable_base_pdx = ROUNDDOWN(frametable_base_pdx, PDX_GROUP_COUNT); - rc = map_pages_to_xen(FRAMETABLE_VIRT_START, base_mfn, - frametable_size >> PAGE_SHIFT, - PAGE_HYPERVISOR_RW | _PAGE_BLOCK); - if ( rc ) - panic("Unable to setup the frametable mappings.\n"); + if ( (max_pdx - frametable_base_pdx) > FRAMETABLE_NR ) + panic("Frametable too small\n"); + + for ( sidx = (frametable_base_pdx / PDX_GROUP_COUNT); ; sidx = nidx ) + { + unsigned int eidx; + + eidx = find_next_zero_bit(pdx_group_valid, max_idx, sidx); + nidx = find_next_bit(pdx_group_valid, max_idx, eidx); + + if ( nidx >= max_idx ) + break; + + init_frametable_chunk(sidx * PDX_GROUP_COUNT, eidx * PDX_GROUP_COUNT); + } - memset(&frame_table[0], 0, nr_pdxs * sizeof(struct page_info)); - memset(&frame_table[nr_pdxs], -1, - frametable_size - (nr_pdxs * sizeof(struct page_info))); + init_frametable_chunk(sidx * PDX_GROUP_COUNT, max_pdx); } /* diff --git a/xen/arch/arm/mpu/mm.c b/xen/arch/arm/mpu/mm.c index XXXXXXX..XXXXXXX 100644 --- a/xen/arch/arm/mpu/mm.c +++ b/xen/arch/arm/mpu/mm.c @@ -XXX,XX +XXX,XX @@ static int is_mm_attr_match(pr_t *region, unsigned int attributes) return 0; } -/* Map a frame table to cover physical addresses ps through pe */ -void __init setup_frametable_mappings(paddr_t ps, paddr_t pe) +/* + * Allocate a contiguous frame table covering ram_start through max_pdx. + * Unlike the MMU version, MPU cannot skip holes because there is no virtual + * address translation (ma == va). + */ +void __init init_frametable(paddr_t ram_start) { + unsigned long nr_pdxs, frametable_size; mfn_t base_mfn; - paddr_t aligned_ps = ROUNDUP(ps, PAGE_SIZE); - paddr_t aligned_pe = ROUNDDOWN(pe, PAGE_SIZE); - - unsigned long nr_pdxs = mfn_to_pdx(mfn_add(maddr_to_mfn(aligned_pe), -1)) - - mfn_to_pdx(maddr_to_mfn(aligned_ps)) + 1; - unsigned long frametable_size = nr_pdxs * sizeof(struct page_info); /* * The size of paddr_t should be sufficient for the complete range of @@ -XXX,XX +XXX,XX @@ void __init setup_frametable_mappings(paddr_t ps, paddr_t pe) BUILD_BUG_ON((sizeof(paddr_t) * BITS_PER_BYTE) < PADDR_BITS); BUILD_BUG_ON(sizeof(struct page_info) != PAGE_INFO_SIZE); + frametable_base_pdx = mfn_to_pdx(maddr_to_mfn(ram_start)); + nr_pdxs = max_pdx - frametable_base_pdx; + frametable_size = nr_pdxs * sizeof(struct page_info); + if ( frametable_size > FRAMETABLE_SIZE ) - panic("The frametable cannot cover the physical region %#"PRIpaddr" - %#"PRIpaddr"\n", - ps, pe); + panic("Frametable too small\n"); - frametable_base_pdx = paddr_to_pdx(aligned_ps); frametable_size = ROUNDUP(frametable_size, PAGE_SIZE); base_mfn = alloc_boot_pages(frametable_size >> PAGE_SHIFT, 1); -- 2.43.0 Refactor setup_frametable_mappings() into init_frametable(), modeled after x86's implementation. Instead of mapping one contiguous frametable covering ram_start to ram_end (including holes), iterate the pdx_group_valid bitmap to allocate and map frametable memory only for valid PDX groups, skipping gaps in the physical address space. This reduces memory consumption on systems with sparse RAM layouts by not allocating frametable entries for non-existent memory regions. The chunk allocator rounds chunk_size up to PAGE_SIZE only, rather than to a larger mapping granularity, to avoid overshooting past chunk boundaries into subsequent gaps or valid regions. This rounding has no impact for in-loop chunks given that chunk size is a multiple of 14MB on Arm64 and 2MB on Arm32. The rounding matters only for the last out-of-loop chunk. Physical allocations prefer 32MB alignment so that map_pages_to_xen() can use the contiguous bit for larger TLB entries where virtual alignment also permits. Fall back to 2MB if the chunk is smaller than 32MB. Add a comment explaining why we don't use pdx_to_page(). For complete discussion see [1]. As ram_end is no longer needed by init_frametable(), drop the now-dead ram_end/bank_end computation from setup_mm(). Update the MPU implementation to match the new init_frametable() signature. Since MPU has no virtual address translation (ma == va), hole-skipping is not possible and the frametable remains a single contiguous allocation. [1] https://lore.kernel.org/xen-devel/20260430125103.401811-1-michal.orzel@amd.com/T/#m803025eb6720a1425443dd0f8e72be93ef02f344 Signed-off-by: Michal Orzel <michal.orzel@amd.com> --- Changes in v3: - don't generalize pdx_to_page with frametable_base_pdx - use different alignment depending on chunk size - drop tags due to the above changes Changes in v2: - fix overshoot problem with 32MB rounding --- xen/arch/arm/arm32/mmu/mm.c | 3 +- xen/arch/arm/include/asm/mm.h | 4 +- xen/arch/arm/mm.c | 5 +- xen/arch/arm/mmu/mm.c | 99 +++++++++++++++++++++++++++-------- xen/arch/arm/mpu/mm.c | 23 ++++---- 5 files changed, 92 insertions(+), 42 deletions(-) diff --git a/xen/arch/arm/arm32/mmu/mm.c b/xen/arch/arm/arm32/mmu/mm.c index XXXXXXX..XXXXXXX 100644 --- a/xen/arch/arm/arm32/mmu/mm.c +++ b/xen/arch/arm/arm32/mmu/mm.c @@ -XXX,XX +XXX,XX @@ void __init setup_mm(void) setup_directmap_mappings(mfn_x(directmap_mfn_start), xenheap_pages); - /* Frame table covers all of RAM region, including holes */ - setup_frametable_mappings(ram_start, ram_end); + init_frametable(ram_start); /* * The allocators may need to use map_domain_page() (such as for diff --git a/xen/arch/arm/include/asm/mm.h b/xen/arch/arm/include/asm/mm.h index XXXXXXX..XXXXXXX 100644 --- a/xen/arch/arm/include/asm/mm.h +++ b/xen/arch/arm/include/asm/mm.h @@ -XXX,XX +XXX,XX @@ extern void *early_fdt_map(paddr_t fdt_paddr); extern void remove_early_mappings(void); /* Prepare the memory subystem to bring-up the given secondary CPU */ extern int prepare_secondary_mm(int cpu); -/* Map a frame table to cover physical addresses ps through pe */ -extern void setup_frametable_mappings(paddr_t ps, paddr_t pe); +/* Map a frame table */ +void init_frametable(paddr_t ram_start); /* Helper function to setup memory management */ void setup_mm_helper(void); /* map a physical range in virtual memory */ diff --git a/xen/arch/arm/mm.c b/xen/arch/arm/mm.c index XXXXXXX..XXXXXXX 100644 --- a/xen/arch/arm/mm.c +++ b/xen/arch/arm/mm.c @@ -XXX,XX +XXX,XX @@ void __init setup_mm(void) { const struct membanks *banks = bootinfo_get_mem(); paddr_t ram_start = INVALID_PADDR; - paddr_t ram_end = 0; paddr_t ram_size = 0; unsigned int i; @@ -XXX,XX +XXX,XX @@ void __init setup_mm(void) for ( i = 0; i < banks->nr_banks; i++ ) { const struct membank *bank = &banks->bank[i]; - paddr_t bank_end = bank->start + bank->size; ram_size = ram_size + bank->size; ram_start = min(ram_start, bank->start); - ram_end = max(ram_end, bank_end); } total_pages = ram_size >> PAGE_SHIFT; @@ -XXX,XX +XXX,XX @@ void __init setup_mm(void) setup_mm_helper(); - setup_frametable_mappings(ram_start, ram_end); + init_frametable(ram_start); init_staticmem_pages(); init_sharedmem_pages(); diff --git a/xen/arch/arm/mmu/mm.c b/xen/arch/arm/mmu/mm.c index XXXXXXX..XXXXXXX 100644 --- a/xen/arch/arm/mmu/mm.c +++ b/xen/arch/arm/mmu/mm.c @@ -XXX,XX +XXX,XX @@ #include <xen/mm.h> #include <xen/mm-frame.h> #include <xen/pdx.h> +#include <xen/sizes.h> #include <xen/string.h> -/* Map a frame table to cover physical addresses ps through pe */ -void __init setup_frametable_mappings(paddr_t ps, paddr_t pe) +static void __init init_frametable_chunk(unsigned long pdx_s, + unsigned long pdx_e) { - unsigned long nr_pdxs = mfn_to_pdx(mfn_add(maddr_to_mfn(pe), -1)) - - mfn_to_pdx(maddr_to_mfn(ps)) + 1; - unsigned long frametable_size = nr_pdxs * sizeof(struct page_info); - mfn_t base_mfn; - const unsigned long mapping_size = frametable_size < MB(32) ? MB(2) - : MB(32); + unsigned long nr_pdxs = pdx_e - pdx_s; + unsigned long chunk_size = nr_pdxs * sizeof(struct page_info); + unsigned long pfn_align; + struct page_info *pg; int rc; + mfn_t base_mfn; + + /* + * In-loop chunks span whole PDX groups, which are always page-size + * aligned. The last chunk ending at max_pdx may not be, so round up. + */ + chunk_size = ROUNDUP(chunk_size, PAGE_SIZE); + + /* + * Try to align the allocation to the contiguous mapping size so that + * map_pages_to_xen() can use the contiguous bit. + */ + pfn_align = ((chunk_size >= MB(32)) ? MB(32) : MB(2)) >> PAGE_SHIFT; + + base_mfn = alloc_boot_pages(chunk_size >> PAGE_SHIFT, pfn_align); + + /* + * Resolve the frametable VA via mfn_to_page(pdx_to_mfn(...)) rather + * than pdx_to_page() because the generic pdx_to_page() does not subtract + * frametable_base_pdx. There's more work to be done to make it generic, so + * for now route through mfn_to_page(), which on Arm applies the + * frametable_base_pdx offset and yields the correct VA. + */ + pg = mfn_to_page(pdx_to_mfn(pdx_s)); + rc = map_pages_to_xen((unsigned long)pg, base_mfn, + chunk_size >> PAGE_SHIFT, + PAGE_HYPERVISOR_RW | _PAGE_BLOCK); + if ( rc ) + panic("Unable to setup the frametable mappings\n"); + + memset(pg, 0, nr_pdxs * sizeof(struct page_info)); + memset(pg + nr_pdxs, -1, + chunk_size - nr_pdxs * sizeof(struct page_info)); +} + +void __init init_frametable(paddr_t ram_start) +{ + unsigned int sidx, nidx, max_idx; /* * The size of paddr_t should be sufficient for the complete range of @@ -XXX,XX +XXX,XX @@ void __init setup_frametable_mappings(paddr_t ps, paddr_t pe) BUILD_BUG_ON((sizeof(paddr_t) * BITS_PER_BYTE) < PADDR_BITS); BUILD_BUG_ON(sizeof(struct page_info) != PAGE_INFO_SIZE); - if ( frametable_size > FRAMETABLE_SIZE ) - panic("The frametable cannot cover the physical region %#"PRIpaddr" - %#"PRIpaddr"\n", - ps, pe); + /* init_frametable_chunk() allocation alignment assumes 4KB granule */ + BUILD_BUG_ON(PAGE_SIZE != SZ_4K); - frametable_base_pdx = mfn_to_pdx(maddr_to_mfn(ps)); - /* Round up to 2M or 32M boundary, as appropriate. */ - frametable_size = ROUNDUP(frametable_size, mapping_size); - base_mfn = alloc_boot_pages(frametable_size >> PAGE_SHIFT, 32<<(20-12)); + /* In-loop chunks must produce page-aligned frametable regions */ + BUILD_BUG_ON((PDX_GROUP_COUNT * sizeof(struct page_info)) % PAGE_SIZE); - rc = map_pages_to_xen(FRAMETABLE_VIRT_START, base_mfn, - frametable_size >> PAGE_SHIFT, - PAGE_HYPERVISOR_RW | _PAGE_BLOCK); - if ( rc ) - panic("Unable to setup the frametable mappings.\n"); + max_idx = DIV_ROUND_UP(max_pdx, PDX_GROUP_COUNT); + frametable_base_pdx = mfn_to_pdx(maddr_to_mfn(ram_start)); + + /* + * Mapping address in init_frametable_chunk must be page-aligned + * for map_pages_to_xen(). Aligning to PDX_GROUP_COUNT guarantees this + * because PDX_GROUP_COUNT * sizeof(page_info) is always a multiple of + * PAGE_SIZE by construction. + */ + frametable_base_pdx = ROUNDDOWN(frametable_base_pdx, PDX_GROUP_COUNT); + + if ( (max_pdx - frametable_base_pdx) > FRAMETABLE_NR ) + panic("Frametable too small\n"); + + for ( sidx = (frametable_base_pdx / PDX_GROUP_COUNT); ; sidx = nidx ) + { + unsigned int eidx; + + eidx = find_next_zero_bit(pdx_group_valid, max_idx, sidx); + nidx = find_next_bit(pdx_group_valid, max_idx, eidx); + + if ( nidx >= max_idx ) + break; + + init_frametable_chunk(sidx * PDX_GROUP_COUNT, eidx * PDX_GROUP_COUNT); + } - memset(&frame_table[0], 0, nr_pdxs * sizeof(struct page_info)); - memset(&frame_table[nr_pdxs], -1, - frametable_size - (nr_pdxs * sizeof(struct page_info))); + init_frametable_chunk(sidx * PDX_GROUP_COUNT, max_pdx); } /* diff --git a/xen/arch/arm/mpu/mm.c b/xen/arch/arm/mpu/mm.c index XXXXXXX..XXXXXXX 100644 --- a/xen/arch/arm/mpu/mm.c +++ b/xen/arch/arm/mpu/mm.c @@ -XXX,XX +XXX,XX @@ static int is_mm_attr_match(pr_t *region, unsigned int attributes) return 0; } -/* Map a frame table to cover physical addresses ps through pe */ -void __init setup_frametable_mappings(paddr_t ps, paddr_t pe) +/* + * Allocate a contiguous frame table covering ram_start through max_pdx. + * Unlike the MMU version, MPU cannot skip holes because there is no virtual + * address translation (ma == va). + */ +void __init init_frametable(paddr_t ram_start) { + unsigned long nr_pdxs, frametable_size; mfn_t base_mfn; - paddr_t aligned_ps = ROUNDUP(ps, PAGE_SIZE); - paddr_t aligned_pe = ROUNDDOWN(pe, PAGE_SIZE); - - unsigned long nr_pdxs = mfn_to_pdx(mfn_add(maddr_to_mfn(aligned_pe), -1)) - - mfn_to_pdx(maddr_to_mfn(aligned_ps)) + 1; - unsigned long frametable_size = nr_pdxs * sizeof(struct page_info); /* * The size of paddr_t should be sufficient for the complete range of @@ -XXX,XX +XXX,XX @@ void __init setup_frametable_mappings(paddr_t ps, paddr_t pe) BUILD_BUG_ON((sizeof(paddr_t) * BITS_PER_BYTE) < PADDR_BITS); BUILD_BUG_ON(sizeof(struct page_info) != PAGE_INFO_SIZE); + frametable_base_pdx = mfn_to_pdx(maddr_to_mfn(ram_start)); + nr_pdxs = max_pdx - frametable_base_pdx; + frametable_size = nr_pdxs * sizeof(struct page_info); + if ( frametable_size > FRAMETABLE_SIZE ) - panic("The frametable cannot cover the physical region %#"PRIpaddr" - %#"PRIpaddr"\n", - ps, pe); + panic("Frametable too small\n"); - frametable_base_pdx = paddr_to_pdx(aligned_ps); frametable_size = ROUNDUP(frametable_size, PAGE_SIZE); base_mfn = alloc_boot_pages(frametable_size >> PAGE_SHIFT, 1); -- 2.43.0