The patch series introduces the following things: 1. Functionality to build the page tables for Xen that map the following: * The physical location of Xen (where the bootloader loaded it) * The link-time location of Xen (where the linker expected Xen's addresses to be. 2. Load the built page table into the SATP 3. Enables MMU. 4. Updates smoke test to grep message which should be printed after MMU is enabled. Oleksii Kurochko (3): xen/riscv: introduce setup_initial_pages xen/riscv: setup initial pagetables automation: update RISC-V smoke test automation/scripts/qemu-smoke-riscv64.sh | 2 +- xen/arch/riscv/Makefile | 1 + xen/arch/riscv/include/asm/mm.h | 9 + xen/arch/riscv/include/asm/page.h | 90 +++++++++ xen/arch/riscv/mm.c | 223 +++++++++++++++++++++++ xen/arch/riscv/setup.c | 11 ++ 6 files changed, 335 insertions(+), 1 deletion(-) create mode 100644 xen/arch/riscv/include/asm/mm.h create mode 100644 xen/arch/riscv/include/asm/page.h create mode 100644 xen/arch/riscv/mm.c -- 2.39.0 Mostly the code for setup_initial_pages was taken from Bobby's repo except for the following changes: * Use only a minimal part of the code enough to enable MMU * rename {_}setup_initial_pagetables functions * add writable argument for _setup_initial_pagetables to have an opportunity to make some sections read-only * update setup_initial_pagetables function to make some sections read-only * change the order of _setup_inital_pagetables() in setup_initial_pagetable(): * first it is called for text, init, rodata sections * after call it for ranges [link_addr_start : link_addr_end] and [load_addr_start : load_addr_end] Before it was done first for the ranges and after for sections but in that case read-only status will be equal to 'true' and as sections' addresses can/are inside the ranges the read-only status won't be updated for them as it was set up before. Origin: https://gitlab.com/xen-on-risc-v/xen/-/tree/riscv-rebase 4af165b468af Signed-off-by: Oleksii Kurochko <oleksii.kurochko@gmail.com> --- xen/arch/riscv/Makefile | 1 + xen/arch/riscv/include/asm/mm.h | 9 ++ xen/arch/riscv/include/asm/page.h | 90 ++++++++++++ xen/arch/riscv/mm.c | 223 ++++++++++++++++++++++++++++++ 4 files changed, 323 insertions(+) create mode 100644 xen/arch/riscv/include/asm/mm.h create mode 100644 xen/arch/riscv/include/asm/page.h create mode 100644 xen/arch/riscv/mm.c diff --git a/xen/arch/riscv/Makefile b/xen/arch/riscv/Makefile index XXXXXXX..XXXXXXX 100644 --- a/xen/arch/riscv/Makefile +++ b/xen/arch/riscv/Makefile @@ -XXX,XX +XXX,XX @@ obj-$(CONFIG_EARLY_PRINTK) += early_printk.o obj-y += entry.o +obj-y += mm.o obj-$(CONFIG_RISCV_64) += riscv64/ obj-y += sbi.o obj-y += setup.o diff --git a/xen/arch/riscv/include/asm/mm.h b/xen/arch/riscv/include/asm/mm.h new file mode 100644 index XXXXXXX..XXXXXXX --- /dev/null +++ b/xen/arch/riscv/include/asm/mm.h @@ -XXX,XX +XXX,XX @@ +#ifndef _ASM_RISCV_MM_H +#define _ASM_RISCV_MM_H + +void setup_initial_pagetables(unsigned long load_addr_start, + unsigned long load_addr_end, + unsigned long linker_addr_start, + unsigned long linker_addr_end); + +#endif /* _ASM_RISCV_MM_H */ diff --git a/xen/arch/riscv/include/asm/page.h b/xen/arch/riscv/include/asm/page.h new file mode 100644 index XXXXXXX..XXXXXXX --- /dev/null +++ b/xen/arch/riscv/include/asm/page.h @@ -XXX,XX +XXX,XX @@ +#ifndef _ASM_RISCV_PAGE_H +#define _ASM_RISCV_PAGE_H + +#include <xen/const.h> +#include <xen/types.h> + +#define PAGE_ENTRIES 512 +#define VPN_BITS (9) +#define VPN_MASK ((unsigned long)((1 << VPN_BITS) - 1)) + +#ifdef CONFIG_RISCV_64 +/* L3 index Bit[47:39] */ +#define THIRD_SHIFT (39) +#define THIRD_MASK (VPN_MASK << THIRD_SHIFT) +/* L2 index Bit[38:30] */ +#define SECOND_SHIFT (30) +#define SECOND_MASK (VPN_MASK << SECOND_SHIFT) +/* L1 index Bit[29:21] */ +#define FIRST_SHIFT (21) +#define FIRST_MASK (VPN_MASK << FIRST_SHIFT) +/* L0 index Bit[20:12] */ +#define ZEROETH_SHIFT (12) +#define ZEROETH_MASK (VPN_MASK << ZEROETH_SHIFT) + +#else // CONFIG_RISCV_32 + +/* L1 index Bit[31:22] */ +#define FIRST_SHIFT (22) +#define FIRST_MASK (VPN_MASK << FIRST_SHIFT) + +/* L0 index Bit[21:12] */ +#define ZEROETH_SHIFT (12) +#define ZEROETH_MASK (VPN_MASK << ZEROETH_SHIFT) +#endif + +#define THIRD_SIZE (1 << THIRD_SHIFT) +#define THIRD_MAP_MASK (~(THIRD_SIZE - 1)) +#define SECOND_SIZE (1 << SECOND_SHIFT) +#define SECOND_MAP_MASK (~(SECOND_SIZE - 1)) +#define FIRST_SIZE (1 << FIRST_SHIFT) +#define FIRST_MAP_MASK (~(FIRST_SIZE - 1)) +#define ZEROETH_SIZE (1 << ZEROETH_SHIFT) +#define ZEROETH_MAP_MASK (~(ZEROETH_SIZE - 1)) + +#define PTE_SHIFT 10 + +#define PTE_VALID BIT(0, UL) +#define PTE_READABLE BIT(1, UL) +#define PTE_WRITABLE BIT(2, UL) +#define PTE_EXECUTABLE BIT(3, UL) +#define PTE_USER BIT(4, UL) +#define PTE_GLOBAL BIT(5, UL) +#define PTE_ACCESSED BIT(6, UL) +#define PTE_DIRTY BIT(7, UL) +#define PTE_RSW (BIT(8, UL) | BIT(9, UL)) + +#define PTE_LEAF_DEFAULT (PTE_VALID | PTE_READABLE | PTE_WRITABLE | PTE_EXECUTABLE) +#define PTE_TABLE (PTE_VALID) + +/* Calculate the offsets into the pagetables for a given VA */ +#define zeroeth_linear_offset(va) ((va) >> ZEROETH_SHIFT) +#define first_linear_offset(va) ((va) >> FIRST_SHIFT) +#define second_linear_offset(va) ((va) >> SECOND_SHIFT) +#define third_linear_offset(va) ((va) >> THIRD_SHIFT) + +#define pagetable_zeroeth_index(va) zeroeth_linear_offset((va) & ZEROETH_MASK) +#define pagetable_first_index(va) first_linear_offset((va) & FIRST_MASK) +#define pagetable_second_index(va) second_linear_offset((va) & SECOND_MASK) +#define pagetable_third_index(va) third_linear_offset((va) & THIRD_MASK) + +/* Page Table entry */ +typedef struct { + uint64_t pte; +} pte_t; + +/* Shift the VPN[x] or PPN[x] fields of a virtual or physical address + * to become the shifted PPN[x] fields of a page table entry */ +#define addr_to_ppn(x) (((x) >> PAGE_SHIFT) << PTE_SHIFT) + +static inline pte_t paddr_to_pte(unsigned long paddr) +{ + return (pte_t) { .pte = addr_to_ppn(paddr) }; +} + +static inline bool pte_is_valid(pte_t *p) +{ + return p->pte & PTE_VALID; +} + +#endif /* _ASM_RISCV_PAGE_H */ diff --git a/xen/arch/riscv/mm.c b/xen/arch/riscv/mm.c new file mode 100644 index XXXXXXX..XXXXXXX --- /dev/null +++ b/xen/arch/riscv/mm.c @@ -XXX,XX +XXX,XX @@ +#include <xen/init.h> +#include <xen/lib.h> + +#include <asm/csr.h> +#include <asm/mm.h> +#include <asm/page.h> + +/* + * xen_second_pagetable is indexed with the VPN[2] page table entry field + * xen_first_pagetable is accessed from the VPN[1] page table entry field + * xen_zeroeth_pagetable is accessed from the VPN[0] page table entry field + */ +pte_t xen_second_pagetable[PAGE_ENTRIES] __attribute__((__aligned__(PAGE_SIZE))); +static pte_t xen_first_pagetable[PAGE_ENTRIES] + __attribute__((__aligned__(PAGE_SIZE))); +static pte_t xen_zeroeth_pagetable[PAGE_ENTRIES] + __attribute__((__aligned__(PAGE_SIZE))); + +extern unsigned long _stext; +extern unsigned long _etext; +extern unsigned long __init_begin; +extern unsigned long __init_end; +extern unsigned long _srodata; +extern unsigned long _erodata; + +paddr_t phys_offset; + +#define resolve_early_addr(x) \ + ({ \ + unsigned long * __##x; \ + if ( load_addr_start <= x && x < load_addr_end ) \ + __##x = (unsigned long *)x; \ + else \ + __##x = (unsigned long *)(x + load_addr_start - linker_addr_start); \ + __##x; \ + }) + +static void __init clear_pagetables(unsigned long load_addr_start, + unsigned long load_addr_end, + unsigned long linker_addr_start, + unsigned long linker_addr_end) +{ + unsigned long *p; + unsigned long page; + unsigned long i; + + page = (unsigned long)&xen_second_pagetable[0]; + + p = resolve_early_addr(page); + for ( i = 0; i < ARRAY_SIZE(xen_second_pagetable); i++ ) + { + p[i] = 0ULL; + } + + page = (unsigned long)&xen_first_pagetable[0]; + p = resolve_early_addr(page); + for ( i = 0; i < ARRAY_SIZE(xen_first_pagetable); i++ ) + { + p[i] = 0ULL; + } + + page = (unsigned long)&xen_zeroeth_pagetable[0]; + p = resolve_early_addr(page); + for ( i = 0; i < ARRAY_SIZE(xen_zeroeth_pagetable); i++ ) + { + p[i] = 0ULL; + } +} + +/* + * WARNING: load_addr() and linker_addr() are to be called only when the MMU is + * disabled and only when executed by the primary CPU. They cannot refer to + * any global variable or functions. + */ + +/* + * Convert an addressed layed out at link time to the address where it was loaded + * by the bootloader. + */ +#define load_addr(linker_address) \ + ({ \ + unsigned long __linker_address = (unsigned long)(linker_address); \ + if ( linker_addr_start <= __linker_address && \ + __linker_address < linker_addr_end ) \ + { \ + __linker_address = \ + __linker_address - linker_addr_start + load_addr_start; \ + } \ + __linker_address; \ + }) + +/* Convert boot-time Xen address from where it was loaded by the boot loader to the address it was layed out + * at link-time. + */ +#define linker_addr(load_address) \ + ({ \ + unsigned long __load_address = (unsigned long)(load_address); \ + if ( load_addr_start <= __load_address && \ + __load_address < load_addr_end ) \ + { \ + __load_address = \ + __load_address - load_addr_start + linker_addr_start; \ + } \ + __load_address; \ + }) + +static void __attribute__((section(".entry"))) +_setup_initial_pagetables(pte_t *second, pte_t *first, pte_t *zeroeth, + unsigned long map_start, + unsigned long map_end, + unsigned long pa_start, + bool writable) +{ + unsigned long page_addr; + unsigned long index2; + unsigned long index1; + unsigned long index0; + + /* align start addresses */ + map_start &= ZEROETH_MAP_MASK; + pa_start &= ZEROETH_MAP_MASK; + + page_addr = map_start; + while ( page_addr < map_end ) + { + index2 = pagetable_second_index(page_addr); + index1 = pagetable_first_index(page_addr); + index0 = pagetable_zeroeth_index(page_addr); + + /* Setup level2 table */ + second[index2] = paddr_to_pte((unsigned long)first); + second[index2].pte |= PTE_TABLE; + + /* Setup level1 table */ + first[index1] = paddr_to_pte((unsigned long)zeroeth); + first[index1].pte |= PTE_TABLE; + + /* Setup level0 table */ + if ( !pte_is_valid(&zeroeth[index0]) ) + { + /* Update level0 table */ + zeroeth[index0] = paddr_to_pte((page_addr - map_start) + pa_start); + zeroeth[index0].pte |= PTE_LEAF_DEFAULT; + zeroeth[index0].pte &= ~((!writable) ? PTE_WRITABLE : 0); + } + + /* Point to next page */ + page_addr += ZEROETH_SIZE; + } +} + +/* + * setup_initial_pagetables: + * + * 1) Build the page tables for Xen that map the following: + * 1.1) The physical location of Xen (where the bootloader loaded it) + * 1.2) The link-time location of Xen (where the linker expected Xen's + * addresses to be) + * 2) Load the page table into the SATP and enable the MMU + */ +void __attribute__((section(".entry"))) +setup_initial_pagetables(unsigned long load_addr_start, + unsigned long load_addr_end, + unsigned long linker_addr_start, + unsigned long linker_addr_end) +{ + pte_t *second; + pte_t *first; + pte_t *zeroeth; + + clear_pagetables(load_addr_start, load_addr_end, + linker_addr_start, linker_addr_end); + + /* Get the addresses where the page tables were loaded */ + second = (pte_t *)load_addr(&xen_second_pagetable); + first = (pte_t *)load_addr(&xen_first_pagetable); + zeroeth = (pte_t *)load_addr(&xen_zeroeth_pagetable); + + /* + * Create a mapping from Xen's link-time addresses to where they were actually loaded. + */ + _setup_initial_pagetables(second, first, zeroeth, + linker_addr(&_stext), + linker_addr(&_etext), + load_addr(&_stext), + false); + _setup_initial_pagetables(second, first, zeroeth, + linker_addr(&__init_begin), + linker_addr(&__init_end), + load_addr(&__init_begin), + true); + _setup_initial_pagetables(second, first, zeroeth, + linker_addr(&_srodata), + linker_addr(&_erodata), + load_addr(&_srodata), + false); + _setup_initial_pagetables(second, first, zeroeth, + linker_addr_start, + linker_addr_end, + load_addr_start, + true); + + /* + * Create a mapping of the load time address range to... the load time address range. + * This mapping is used at boot time only. + */ + _setup_initial_pagetables(second, first, zeroeth, + load_addr_start, + load_addr_end, + load_addr_start, + true); + + /* Ensure page table writes precede loading the SATP */ + asm volatile("sfence.vma"); + + /* Enable the MMU and load the new pagetable for Xen */ + csr_write(CSR_SATP, + (load_addr(xen_second_pagetable) >> PAGE_SHIFT) | SATP_MODE_SV39 << SATP_MODE_SHIFT); + + phys_offset = load_addr_start - linker_addr_start; + + return; +} -- 2.39.0 Calculate load and linker linker image addresses and setup initial pagetables. Signed-off-by: Oleksii Kurochko <oleksii.kurochko@gmail.com> --- xen/arch/riscv/setup.c | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/xen/arch/riscv/setup.c b/xen/arch/riscv/setup.c index XXXXXXX..XXXXXXX 100644 --- a/xen/arch/riscv/setup.c +++ b/xen/arch/riscv/setup.c @@ -XXX,XX +XXX,XX @@ #include <xen/bug.h> #include <xen/compile.h> #include <xen/init.h> +#include <xen/kernel.h> #include <asm/csr.h> #include <asm/early_printk.h> +#include <asm/mm.h> #include <asm/traps.h> /* Xen stack for bringing up the first CPU. */ @@ -XXX,XX +XXX,XX @@ static void __init disable_fpu(void) void __init noreturn start_xen(void) { + unsigned long load_start = (unsigned long)start; + unsigned long load_end = load_start + (unsigned long)(_end - _start); + unsigned long linker_start = (unsigned long)_start; + unsigned long linker_end = (unsigned long)_end; + /* * The following things are passed by bootloader: * a0 -> hart_id @@ -XXX,XX +XXX,XX @@ void __init noreturn start_xen(void) test_macros_from_bug_h(); + setup_initial_pagetables(load_start, load_end, linker_start, linker_end); + + early_printk("MMU has been enabled\n"); + for ( ;; ) asm volatile ("wfi"); -- 2.39.0 The smoke test was updated to verify that MMU has been enabled. Signed-off-by: Oleksii Kurochko <oleksii.kurochko@gmail.com> --- automation/scripts/qemu-smoke-riscv64.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/automation/scripts/qemu-smoke-riscv64.sh b/automation/scripts/qemu-smoke-riscv64.sh index XXXXXXX..XXXXXXX 100755 --- a/automation/scripts/qemu-smoke-riscv64.sh +++ b/automation/scripts/qemu-smoke-riscv64.sh @@ -XXX,XX +XXX,XX @@ qemu-system-riscv64 \ |& tee smoke.serial set -e -(grep -q "WARN is most likely working" smoke.serial) || exit 1 +(grep -q "MMU has been enabled" smoke.serial) || exit 1 exit 0 -- 2.39.0 The patch series is based on top of 'RISCV basic exception handling implementation' patch series. [1] The patch series introduces the following things: 1. Functionality to build the page tables for Xen that map link-time to physical-time location. 2. Check that Xen is less then page size. 3. Check that load addresses don't overlap with linker addresses. 4. Prepare things for proper switch to virtual memory world. 5. Load the built page table into the SATP 6. Enable MMU. [1] https://lore.kernel.org/xen-devel/cover.1678976127.git.oleksii.kurochko@gmail.com/T/#t --- Changes in V2: * Remove {ZEROETH,FIRST,...}_{SHIFT,MASK, SIZE,...} and introduce instead of them XEN_PT_LEVEL_*() and LEVEL_* * Rework pt_linear_offset() and pt_index based on XEN_PT_LEVEL_*() * Remove clear_pagetables() functions as pagetables were zeroed during .bss initialization * Rename _setup_initial_pagetables() to setup_initial_mapping() * Make PTE_DEFAULT equal to RX. * Update prototype of setup_initial_mapping(..., bool writable) -> setup_initial_mapping(..., UL flags) * Update calls of setup_initial_mapping according to new prototype * Remove unnecessary call of: _setup_initial_pagetables(..., load_addr_start, load_addr_end, load_addr_start, ...) * Define index* in the loop of setup_initial_mapping * Remove attribute "__attribute__((section(".entry")))" for setup_initial_pagetables() as we don't have such section * make arguments of paddr_to_pte() and pte_is_valid() as const. * use <xen/kernel.h> instead of declaring extern unsigned long _stext, 0etext, _srodata, _erodata * update 'extern unsigned long __init_begin' to 'extern unsigned long __init_begin[]' * use aligned() instead of "__attribute__((__aligned__(PAGE_SIZE)))" * set __section(".bss.page_aligned") for page tables arrays * fix identatations * Change '__attribute__((section(".entry")))' to '__init' * Remove alignment of {map, pa}_start &= XEN_PT_LEVEL_MAP_MASK(0); in setup_inital_mapping() as they should be already aligned. * Remove clear_pagetables() as initial pagetables will be zeroed during bss initialization * Remove __attribute__((section(".entry")) for setup_initial_pagetables() as there is no such section in xen.lds.S * Update the argument of pte_is_valid() to "const pte_t *p" * Remove patch "[PATCH v1 3/3] automation: update RISC-V smoke test" from the patch series as it was introduced simplified approach for RISC-V smoke test by Andrew Cooper * Add patch [[xen/riscv: remove dummy_bss variable] as there is no any sense in dummy_bss variable after introduction of inittial page tables. --- Oleksii Kurochko (3): xen/riscv: introduce setup_initial_pages xen/riscv: setup initial pagetables xen/riscv: remove dummy_bss variable xen/arch/riscv/Makefile | 1 + xen/arch/riscv/include/asm/mm.h | 8 ++ xen/arch/riscv/include/asm/page.h | 67 +++++++++++++++++ xen/arch/riscv/mm.c | 121 ++++++++++++++++++++++++++++++ xen/arch/riscv/riscv64/head.S | 65 ++++++++++++++++ xen/arch/riscv/setup.c | 13 ++-- xen/arch/riscv/xen.lds.S | 2 + 7 files changed, 269 insertions(+), 8 deletions(-) create mode 100644 xen/arch/riscv/include/asm/mm.h create mode 100644 xen/arch/riscv/include/asm/page.h create mode 100644 xen/arch/riscv/mm.c -- 2.39.2 Mostly the code for setup_initial_pages was taken from Bobby's repo except for the following changes: * Use only a minimal part of the code enough to enable MMU * rename {_}setup_initial_pagetables functions * add an argument for setup_initial_mapping to have an opportunity to make set PTE flags. * update setup_initial_pagetables function to map sections with correct PTE flags. * introduce separate enable_mmu() to be able for proper handling the case when load start address isn't equal to linker start address. * map linker addresses range to load addresses range without 1:1 mapping. * add safety checks such as: * Xen size is less than page size * linker addresses range doesn't overlap load addresses range * Rework macros {THIRD,SECOND,FIRST,ZEROETH}_{SHIFT,MASK} * change PTE_LEAF_DEFAULT to RX instead of RWX. * Remove phys_offset as it isn't used now. * Remove alignment of {map, pa}_start &= XEN_PT_LEVEL_MAP_MASK(0); in setup_inital_mapping() as they should be already aligned. * Remove clear_pagetables() as initial pagetables will be zeroed during bss initialization * Remove __attribute__((section(".entry")) for setup_initial_pagetables() as there is no such section in xen.lds.S * Update the argument of pte_is_valid() to "const pte_t *p" Origin: https://gitlab.com/xen-on-risc-v/xen/-/tree/riscv-rebase 4af165b468af Signed-off-by: Oleksii Kurochko <oleksii.kurochko@gmail.com> --- Changes in V2: * update the commit message: * Remove {ZEROETH,FIRST,...}_{SHIFT,MASK, SIZE,...} and introduce instead of them XEN_PT_LEVEL_*() and LEVEL_* * Rework pt_linear_offset() and pt_index based on XEN_PT_LEVEL_*() * Remove clear_pagetables() functions as pagetables were zeroed during .bss initialization * Rename _setup_initial_pagetables() to setup_initial_mapping() * Make PTE_DEFAULT equal to RX. * Update prototype of setup_initial_mapping(..., bool writable) -> setup_initial_mapping(..., UL flags) * Update calls of setup_initial_mapping according to new prototype * Remove unnecessary call of: _setup_initial_pagetables(..., load_addr_start, load_addr_end, load_addr_start, ...) * Define index* in the loop of setup_initial_mapping * Remove attribute "__attribute__((section(".entry")))" for setup_initial_pagetables() as we don't have such section * make arguments of paddr_to_pte() and pte_is_valid() as const. * make xen_second_pagetable static. * use <xen/kernel.h> instead of declaring extern unsigned long _stext, 0etext, _srodata, _erodata * update 'extern unsigned long __init_begin' to 'extern unsigned long __init_begin[]' * use aligned() instead of "__attribute__((__aligned__(PAGE_SIZE)))" * set __section(".bss.page_aligned") for page tables arrays * fix identatations * Change '__attribute__((section(".entry")))' to '__init' * Remove phys_offset as it isn't used now. * Remove alignment of {map, pa}_start &= XEN_PT_LEVEL_MAP_MASK(0); in setup_inital_mapping() as they should be already aligned. * Remove clear_pagetables() as initial pagetables will be zeroed during bss initialization * Remove __attribute__((section(".entry")) for setup_initial_pagetables() as there is no such section in xen.lds.S * Update the argument of pte_is_valid() to "const pte_t *p" --- xen/arch/riscv/Makefile | 1 + xen/arch/riscv/include/asm/mm.h | 8 ++ xen/arch/riscv/include/asm/page.h | 67 +++++++++++++++++ xen/arch/riscv/mm.c | 121 ++++++++++++++++++++++++++++++ xen/arch/riscv/riscv64/head.S | 65 ++++++++++++++++ xen/arch/riscv/xen.lds.S | 2 + 6 files changed, 264 insertions(+) create mode 100644 xen/arch/riscv/include/asm/mm.h create mode 100644 xen/arch/riscv/include/asm/page.h create mode 100644 xen/arch/riscv/mm.c diff --git a/xen/arch/riscv/Makefile b/xen/arch/riscv/Makefile index XXXXXXX..XXXXXXX 100644 --- a/xen/arch/riscv/Makefile +++ b/xen/arch/riscv/Makefile @@ -XXX,XX +XXX,XX @@ obj-$(CONFIG_EARLY_PRINTK) += early_printk.o obj-y += entry.o +obj-y += mm.o obj-$(CONFIG_RISCV_64) += riscv64/ obj-y += sbi.o obj-y += setup.o diff --git a/xen/arch/riscv/include/asm/mm.h b/xen/arch/riscv/include/asm/mm.h new file mode 100644 index XXXXXXX..XXXXXXX --- /dev/null +++ b/xen/arch/riscv/include/asm/mm.h @@ -XXX,XX +XXX,XX @@ +#ifndef _ASM_RISCV_MM_H +#define _ASM_RISCV_MM_H + +void setup_initial_pagetables(void); + +extern void enable_mmu(void); + +#endif /* _ASM_RISCV_MM_H */ diff --git a/xen/arch/riscv/include/asm/page.h b/xen/arch/riscv/include/asm/page.h new file mode 100644 index XXXXXXX..XXXXXXX --- /dev/null +++ b/xen/arch/riscv/include/asm/page.h @@ -XXX,XX +XXX,XX @@ +#ifndef _ASM_RISCV_PAGE_H +#define _ASM_RISCV_PAGE_H + +#include <xen/const.h> +#include <xen/types.h> + +#define PAGE_ENTRIES (1 << PAGETABLE_ORDER) +#define VPN_MASK ((unsigned long)(PAGE_ENTRIES - 1)) + +#define PAGE_ORDER (12) + +#ifdef CONFIG_RISCV_64 +#define PAGETABLE_ORDER (9) +#else /* CONFIG_RISCV_32 */ +#define PAGETABLE_ORDER (10) +#endif + +#define LEVEL_ORDER(lvl) (lvl * PAGETABLE_ORDER) +#define LEVEL_SHIFT(lvl) (LEVEL_ORDER(lvl) + PAGE_ORDER) +#define LEVEL_SIZE(lvl) (_AT(paddr_t, 1) << LEVEL_SHIFT(lvl)) + +#define XEN_PT_LEVEL_SHIFT(lvl) LEVEL_SHIFT(lvl) +#define XEN_PT_LEVEL_ORDER(lvl) LEVEL_ORDER(lvl) +#define XEN_PT_LEVEL_SIZE(lvl) LEVEL_SIZE(lvl) +#define XEN_PT_LEVEL_MAP_MASK(lvl) (~(XEN_PT_LEVEL_SIZE(lvl) - 1)) +#define XEN_PT_LEVEL_MASK(lvl) (VPN_MASK << XEN_PT_LEVEL_SHIFT(lvl)) + +#define PTE_SHIFT 10 + +#define PTE_VALID BIT(0, UL) +#define PTE_READABLE BIT(1, UL) +#define PTE_WRITABLE BIT(2, UL) +#define PTE_EXECUTABLE BIT(3, UL) +#define PTE_USER BIT(4, UL) +#define PTE_GLOBAL BIT(5, UL) +#define PTE_ACCESSED BIT(6, UL) +#define PTE_DIRTY BIT(7, UL) +#define PTE_RSW (BIT(8, UL) | BIT(9, UL)) + +#define PTE_LEAF_DEFAULT (PTE_VALID | PTE_READABLE | PTE_EXECUTABLE) +#define PTE_TABLE (PTE_VALID) + +/* Calculate the offsets into the pagetables for a given VA */ +#define pt_linear_offset(lvl, va) ((va) >> XEN_PT_LEVEL_SHIFT(lvl)) + +#define pt_index(lvl, va) pt_linear_offset(lvl, (va) & XEN_PT_LEVEL_MASK(lvl)) + +/* Page Table entry */ +typedef struct { + uint64_t pte; +} pte_t; + +/* Shift the VPN[x] or PPN[x] fields of a virtual or physical address + * to become the shifted PPN[x] fields of a page table entry */ +#define addr_to_ppn(x) (((x) >> PAGE_SHIFT) << PTE_SHIFT) + +static inline pte_t paddr_to_pte(const unsigned long paddr) +{ + return (pte_t) { .pte = addr_to_ppn(paddr) }; +} + +static inline bool pte_is_valid(const pte_t *p) +{ + return p->pte & PTE_VALID; +} + +#endif /* _ASM_RISCV_PAGE_H */ diff --git a/xen/arch/riscv/mm.c b/xen/arch/riscv/mm.c new file mode 100644 index XXXXXXX..XXXXXXX --- /dev/null +++ b/xen/arch/riscv/mm.c @@ -XXX,XX +XXX,XX @@ +#include <xen/compiler.h> +#include <xen/init.h> +#include <xen/kernel.h> +#include <xen/lib.h> +#include <xen/page-size.h> + +#include <asm/boot-info.h> +#include <asm/config.h> +#include <asm/csr.h> +#include <asm/mm.h> +#include <asm/page.h> +#include <asm/traps.h> + +/* + * xen_second_pagetable is indexed with the VPN[2] page table entry field + * xen_first_pagetable is accessed from the VPN[1] page table entry field + * xen_zeroeth_pagetable is accessed from the VPN[0] page table entry field + */ +pte_t __section(".bss.page_aligned") __aligned(PAGE_SIZE) + xen_second_pagetable[PAGE_ENTRIES]; +static pte_t __section(".bss.page_aligned") __aligned(PAGE_SIZE) + xen_first_pagetable[PAGE_ENTRIES]; +static pte_t __section(".bss.page_aligned") __aligned(PAGE_SIZE) + xen_zeroeth_pagetable[PAGE_ENTRIES]; + +extern unsigned long __init_begin[]; +extern unsigned long __init_end[]; +extern unsigned char cpu0_boot_stack[STACK_SIZE]; + +static void __init +setup_initial_mapping(pte_t *second, pte_t *first, pte_t *zeroeth, + unsigned long map_start, + unsigned long map_end, + unsigned long pa_start, + unsigned long flags) +{ + unsigned long page_addr; + + // /* align start addresses */ + // map_start &= XEN_PT_LEVEL_MAP_MASK(0); + // pa_start &= XEN_PT_LEVEL_MAP_MASK(0); + + page_addr = map_start; + while ( page_addr < map_end ) + { + unsigned long index2 = pt_index(2, page_addr); + unsigned long index1 = pt_index(1, page_addr); + unsigned long index0 = pt_index(0, page_addr); + + /* Setup level2 table */ + second[index2] = paddr_to_pte((unsigned long)first); + second[index2].pte |= PTE_TABLE; + + /* Setup level1 table */ + first[index1] = paddr_to_pte((unsigned long)zeroeth); + first[index1].pte |= PTE_TABLE; + + + /* Setup level0 table */ + if ( !pte_is_valid(&zeroeth[index0]) ) + { + /* Update level0 table */ + zeroeth[index0] = paddr_to_pte((page_addr - map_start) + pa_start); + zeroeth[index0].pte |= flags; + } + + /* Point to next page */ + page_addr += XEN_PT_LEVEL_SIZE(0); + } +} + +/* + * setup_initial_pagetables: + * + * Build the page tables for Xen that map the following: + * load addresses to linker addresses + */ +void __init setup_initial_pagetables(void) +{ + pte_t *second; + pte_t *first; + pte_t *zeroeth; + + unsigned long load_addr_start = boot_info.load_start; + unsigned long load_addr_end = boot_info.load_end; + unsigned long linker_addr_start = boot_info.linker_start; + unsigned long linker_addr_end = boot_info.linker_end; + + BUG_ON(load_addr_start % (PAGE_ENTRIES * PAGE_SIZE) != 0); + if (load_addr_start != linker_addr_start) + BUG_ON((linker_addr_end > load_addr_start && load_addr_end > linker_addr_start)); + + /* Get the addresses where the page tables were loaded */ + second = (pte_t *)(&xen_second_pagetable); + first = (pte_t *)(&xen_first_pagetable); + zeroeth = (pte_t *)(&xen_zeroeth_pagetable); + + setup_initial_mapping(second, first, zeroeth, + LOAD_TO_LINK((unsigned long)&_stext), + LOAD_TO_LINK((unsigned long)&_etext), + (unsigned long)&_stext, + PTE_LEAF_DEFAULT); + + setup_initial_mapping(second, first, zeroeth, + LOAD_TO_LINK((unsigned long)&__init_begin), + LOAD_TO_LINK((unsigned long)&__init_end), + (unsigned long)&__init_begin, + PTE_LEAF_DEFAULT | PTE_WRITABLE); + + setup_initial_mapping(second, first, zeroeth, + LOAD_TO_LINK((unsigned long)&_srodata), + LOAD_TO_LINK((unsigned long)&_erodata), + (unsigned long)(&_srodata), + PTE_VALID | PTE_READABLE); + + setup_initial_mapping(second, first, zeroeth, + linker_addr_start, + linker_addr_end, + load_addr_start, + PTE_LEAF_DEFAULT | PTE_READABLE); +} diff --git a/xen/arch/riscv/riscv64/head.S b/xen/arch/riscv/riscv64/head.S index XXXXXXX..XXXXXXX 100644 --- a/xen/arch/riscv/riscv64/head.S +++ b/xen/arch/riscv/riscv64/head.S @@ -XXX,XX +XXX,XX @@ #include <asm/asm.h> +#include <asm/asm-offsets.h> #include <asm/riscv_encoding.h> .section .text.header, "ax", %progbits @@ -XXX,XX +XXX,XX @@ ENTRY(start) add sp, sp, t0 tail start_xen + +ENTRY(enable_mmu) + /* Calculate physical offset between linker and load addresses */ + la t0, boot_info + REG_L t1, BI_LINKER_START(t0) + REG_L t2, BI_LOAD_START(t0) + sub t1, t1, t2 + + /* + * Calculate and update a linker time address of the .L_mmu_is_enabled + * label and update CSR_STVEC with it. + * MMU is configured in a way where linker addresses are mapped + * on load addresses so case when linker addresses are not equal to + * load addresses, and thereby, after MMU is enabled, it will cause + * an exception and jump to linker time addresses + */ + la t3, .L_mmu_is_enabled + add t3, t3, t1 + csrw CSR_STVEC, t3 + + /* Calculate a value for SATP register */ + li t5, SATP_MODE_SV39 + li t6, SATP_MODE_SHIFT + sll t5, t5, t6 + + la t4, xen_second_pagetable + srl t4, t4, PAGE_SHIFT + or t4, t4, t5 + sfence.vma + csrw CSR_SATP, t4 + + .align 2 +.L_mmu_is_enabled: + /* + * Stack should be re-inited as: + * 1. Right now an address of the stack is relative to load time + * addresses what will cause an issue in case of load start address + * isn't equal to linker start address. + * 2. Addresses in stack are all load time relative which can be an + * issue in case when load start address isn't equal to linker + * start address. + */ + la sp, cpu0_boot_stack + li t0, STACK_SIZE + add sp, sp, t0 + + /* + * Re-init an address of exception handler as it was overwritten with + * the address of the .L_mmu_is_enabled label. + * Before jump to trap_init save return address of enable_mmu() to + * know where we should back after enable_mmu() will be finished. + */ + mv s0, ra + lla t0, trap_init + jalr ra, t0 + + /* + * Re-calculate the return address of enable_mmu() function for case + * when linker start address isn't equal to load start address + */ + add s0, s0, t1 + mv ra, s0 + + ret diff --git a/xen/arch/riscv/xen.lds.S b/xen/arch/riscv/xen.lds.S index XXXXXXX..XXXXXXX 100644 --- a/xen/arch/riscv/xen.lds.S +++ b/xen/arch/riscv/xen.lds.S @@ -XXX,XX +XXX,XX @@ SECTIONS ASSERT(!SIZEOF(.got), ".got non-empty") ASSERT(!SIZEOF(.got.plt), ".got.plt non-empty") + +ASSERT(_end - _start <= MB(2), "Xen too large for early-boot assumptions") -- 2.39.2 Signed-off-by: Oleksii Kurochko <oleksii.kurochko@gmail.com> --- Changes in V2: * Update the commit message --- xen/arch/riscv/setup.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/xen/arch/riscv/setup.c b/xen/arch/riscv/setup.c index XXXXXXX..XXXXXXX 100644 --- a/xen/arch/riscv/setup.c +++ b/xen/arch/riscv/setup.c @@ -XXX,XX +XXX,XX @@ #include <asm/boot-info.h> #include <asm/csr.h> #include <asm/early_printk.h> +#include <asm/mm.h> #include <asm/traps.h> /* Xen stack for bringing up the first CPU. */ @@ -XXX,XX +XXX,XX @@ void __init noreturn start_xen(unsigned long bootcpu_id, test_macros_from_bug_h(); + setup_initial_pagetables(); + + enable_mmu(); + early_printk("All set up\n"); for ( ;; ) -- 2.39.2 After introduction of initial pagetables there is no any sense in dummy_bss variable as bss section will not be empty anymore. Signed-off-by: Oleksii Kurochko <oleksii.kurochko@gmail.com> --- Changes in V2: * patch was introduced in the current one patch series (v2). --- xen/arch/riscv/setup.c | 8 -------- 1 file changed, 8 deletions(-) diff --git a/xen/arch/riscv/setup.c b/xen/arch/riscv/setup.c index XXXXXXX..XXXXXXX 100644 --- a/xen/arch/riscv/setup.c +++ b/xen/arch/riscv/setup.c @@ -XXX,XX +XXX,XX @@ unsigned char __initdata cpu0_boot_stack[STACK_SIZE] struct boot_info boot_info = { (unsigned long)&_start, (unsigned long)&_end, 0UL, 0UL }; -/* - * To be sure that .bss isn't zero. It will simplify code of - * .bss initialization. - * TODO: - * To be deleted when the first real .bss user appears - */ -int dummy_bss __attribute__((unused)); - static void fill_boot_info(void) { boot_info.load_start = (unsigned long)_start; -- 2.39.2