Introduce exception table handling for RISC-V so faults from selected
instructions can be recovered via fixup handlers instead of being
treated as fatal.

Add the RISC-V exception table format, sorting at boot to allow binary
search used furthuer, and lookup from the trap handler. Update the
linker script to emit the .ex_table section using introduced common
EX_TABLE macro shared with other architectures.

Also, reduce __start___ex_table alignment from 8 to 4 bytes to
match the natural alignment of struct exception_table_entry,
which contains two int32_t fields.

Add inclusion of asm/extable.h to asm/bug.h to deal with compilation
issue of common/virtual_region.c, which require declaration of
__start___ex_table and __stop___ex_table.

This implementation is based on Linux 6.16.

Signed-off-by: Oleksii Kurochko <oleksii.kurochko@gmail.com>
---
Changes in v2:
 - Corrected the name for __start_ex_table identifier in the commit message.
 - Droped plural where extables is used.
 - Added inclusuion of <asm/extable.h> to deal with compilation (nothing
   declares __start___ex_table and __stop___ex_table) of common/virtual_region.c.
 - Use long for delta variable inside swap_ex in extable.c.
 - To take into acoount live-patching code:
   - s/sort_extable/sort_exception_tables.
   - Introduce sort_exception_table() as liveaptch code requires it and
     re-use it inside sort_exception_tables().
- Drop cmp_ex_search() and rename cmp_ex_sort() to cmp_ex().
  Rename local variable l and r inside cmp_ex().
- Identation fixes.
- prefer sizeof(<expression>) over sizeof(<type>) in calls of bsearch() and
  sort().
- Return back defintion of asm_extable() for __ASSEMBLER__ case.
- Correct the comment above declaration of struct exception_table_entry.
- Drop else in do_trap() before "if ( fixup_exception() )" to visually separate
  the set of checks.
- Align start of exception table section by 4-bytes as exception table struct
  contains two 4 bytes integers.
- Make extable.o compile unconditionally.
- Drop ifdef HAS_EX_TABLE in extable.h as extable.o is always compiled.
- Drop ifdef around defintion of EX_TABLE.
- Drop __init for cmp_ex as it is now used in fixup_exception() which isn't
  marked as __init.
- Return void instead of bool for ex_handler_fixup() as this function always
  returns true.
- Update the comment above defintion of struct exception_table_entry() to be
  more accurate.
- Add inclusion of asm/extable.h to asm/bug.h to deal with compilation issue
  of common/virtual_region.c, which require declaration of __start___ex_table
  and __stop___ex_table.
---
 xen/arch/riscv/Kconfig               |  1 +
 xen/arch/riscv/Makefile              |  1 +
 xen/arch/riscv/extable.c             | 85 ++++++++++++++++++++++++++++
 xen/arch/riscv/include/asm/bug.h     |  2 +
 xen/arch/riscv/include/asm/extable.h | 58 +++++++++++++++++++
 xen/arch/riscv/setup.c               |  3 +
 xen/arch/riscv/traps.c               |  5 ++
 xen/arch/riscv/xen.lds.S             |  3 +
 xen/arch/x86/xen.lds.S               |  6 +-
 xen/include/xen/xen.lds.h            |  6 ++
 10 files changed, 165 insertions(+), 5 deletions(-)
 create mode 100644 xen/arch/riscv/extable.c
 create mode 100644 xen/arch/riscv/include/asm/extable.h

diff --git a/xen/arch/riscv/Kconfig b/xen/arch/riscv/Kconfig
index 89876b32175d..a5e87c1757f7 100644
--- a/xen/arch/riscv/Kconfig
+++ b/xen/arch/riscv/Kconfig
@@ -4,6 +4,7 @@ config RISCV
 	select GENERIC_BUG_FRAME
 	select GENERIC_UART_INIT
 	select HAS_DEVICE_TREE_DISCOVERY
+	select HAS_EX_TABLE
 	select HAS_PMAP
 	select HAS_UBSAN
 	select HAS_VMAP
diff --git a/xen/arch/riscv/Makefile b/xen/arch/riscv/Makefile
index ffbd7062e214..04f02ad89cba 100644
--- a/xen/arch/riscv/Makefile
+++ b/xen/arch/riscv/Makefile
@@ -3,6 +3,7 @@ obj-y += cpufeature.o
 obj-y += domain.o
 obj-$(CONFIG_EARLY_PRINTK) += early_printk.o
 obj-y += entry.o
+obj-y += extable.o
 obj-y += imsic.o
 obj-y += intc.o
 obj-y += irq.o
diff --git a/xen/arch/riscv/extable.c b/xen/arch/riscv/extable.c
new file mode 100644
index 000000000000..882ae9508d19
--- /dev/null
+++ b/xen/arch/riscv/extable.c
@@ -0,0 +1,85 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+
+#include <xen/init.h>
+#include <xen/bsearch.h>
+#include <xen/lib.h>
+#include <xen/livepatch.h>
+#include <xen/sort.h>
+#include <xen/virtual_region.h>
+
+#include <asm/extable.h>
+#include <asm/processor.h>
+
+#define EX_FIELD(ptr, field) ((unsigned long)&(ptr)->field + (ptr)->field)
+
+static inline unsigned long ex_insn(const struct exception_table_entry *ex)
+{
+    return EX_FIELD(ex, insn);
+}
+
+static inline unsigned long ex_fixup(const struct exception_table_entry *ex)
+{
+    return EX_FIELD(ex, fixup);
+}
+
+static void __init cf_check swap_ex(void *a, void *b)
+{
+    struct exception_table_entry *x = a, *y = b, tmp;
+    long delta = b - a;
+
+    tmp = *x;
+    x->insn = y->insn + delta;
+    y->insn = tmp.insn - delta;
+
+    x->fixup = y->fixup + delta;
+    y->fixup = tmp.fixup - delta;
+}
+
+static int cf_check cmp_ex(const void *a, const void *b)
+{
+    const unsigned long insn_a = ex_insn(a);
+    const unsigned long insn_b = ex_insn(b);
+
+    /* avoid overflow */
+    return (insn_a > insn_b) - (insn_a < insn_b);
+}
+
+void init_or_livepatch sort_exception_table(struct exception_table_entry *start,
+                                 const struct exception_table_entry *stop)
+{
+    sort(start, stop - start, sizeof(*start), cmp_ex, swap_ex);
+}
+
+void __init sort_exception_tables(void)
+{
+    sort_exception_table(__start___ex_table, __stop___ex_table);
+}
+
+static void ex_handler_fixup(const struct exception_table_entry *ex,
+                             struct cpu_user_regs *regs)
+{
+    regs->sepc = ex_fixup(ex);
+}
+
+bool fixup_exception(struct cpu_user_regs *regs)
+{
+    unsigned long pc = regs->sepc;
+    const struct virtual_region *region = find_text_region(pc);
+    const struct exception_table_entry *ex;
+    struct exception_table_entry key;
+
+    if ( !region || !region->ex )
+        return false;
+
+    key.insn = pc - (unsigned long)&key.insn;
+
+    ex = bsearch(&key, region->ex, region->ex_end - region->ex, sizeof(key),
+                 cmp_ex);
+
+    if ( !ex )
+        return false;
+
+    ex_handler_fixup(ex, regs);
+
+    return true;
+}
diff --git a/xen/arch/riscv/include/asm/bug.h b/xen/arch/riscv/include/asm/bug.h
index 6ec8adc528a9..e6f286881662 100644
--- a/xen/arch/riscv/include/asm/bug.h
+++ b/xen/arch/riscv/include/asm/bug.h
@@ -9,6 +9,8 @@
 
 #ifndef __ASSEMBLER__
 
+#include <asm/extable.h>
+
 #define BUG_INSTR "unimp"
 
 /*
diff --git a/xen/arch/riscv/include/asm/extable.h b/xen/arch/riscv/include/asm/extable.h
new file mode 100644
index 000000000000..4f50f84e69f2
--- /dev/null
+++ b/xen/arch/riscv/include/asm/extable.h
@@ -0,0 +1,58 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+
+#ifndef ASM__RISCV__ASM_EXTABLE_H
+#define ASM__RISCV__ASM_EXTABLE_H
+
+#ifdef __ASSEMBLER__
+
+#define ASM_EXTABLE(insn, fixup)    \
+    .pushsection .ex_table, "a";    \
+    .balign     4;                  \
+    .long       ((insn) - .);       \
+    .long       ((fixup) - .);      \
+    .popsection;
+
+.macro asm_extable, insn, fixup
+    ASM_EXTABLE(\insn, \fixup)
+.endm
+
+#else /* __ASSEMBLER__ */
+
+#include <xen/stringify.h>
+#include <xen/types.h>
+
+struct cpu_user_regs;
+
+#define ASM_EXTABLE(insn, fixup)        \
+    ".pushsection .ex_table, \"a\"\n"   \
+    ".balign    4\n"                    \
+    ".long      ((" #insn ") - .)\n"    \
+    ".long      ((" #fixup ") - .)\n"   \
+    ".popsection\n"
+
+/*
+ * The exception table consists of pairs of relative offsets: the first
+ * is the relative offset to an instruction that is allowed to fault,
+ * and the second is the relative offset at which the program should
+ * continue. No general-purpose registers are modified by the exception
+ * handling mechanism itself, so it is up to the fixup code to handle
+ * any necessary state cleanup.
+ *
+ * The exception table and fixup code live out of line with the main
+ * instruction path. This means when everything is well, we don't even
+ * have to jump over them. Further, they do not intrude on our cache or
+ * tlb entries.
+ */
+struct exception_table_entry {
+    int32_t insn, fixup;
+};
+
+extern struct exception_table_entry __start___ex_table[];
+extern struct exception_table_entry __stop___ex_table[];
+
+void sort_exception_tables(void);
+bool fixup_exception(struct cpu_user_regs *regs);
+
+#endif /* __ASSEMBLY__ */
+
+#endif /* ASM__RISCV__ASM_EXTABLE_H */
diff --git a/xen/arch/riscv/setup.c b/xen/arch/riscv/setup.c
index cae49bb29626..56a0907a855f 100644
--- a/xen/arch/riscv/setup.c
+++ b/xen/arch/riscv/setup.c
@@ -19,6 +19,7 @@
 
 #include <public/version.h>
 
+#include <asm/extable.h>
 #include <asm/cpufeature.h>
 #include <asm/early_printk.h>
 #include <asm/fixmap.h>
@@ -81,6 +82,8 @@ void __init noreturn start_xen(unsigned long bootcpu_id,
 
     smp_prepare_boot_cpu();
 
+    sort_exception_tables();
+
     set_cpuid_to_hartid(0, bootcpu_id);
 
     trap_init();
diff --git a/xen/arch/riscv/traps.c b/xen/arch/riscv/traps.c
index 326f2be62823..d35c013e1399 100644
--- a/xen/arch/riscv/traps.c
+++ b/xen/arch/riscv/traps.c
@@ -12,6 +12,7 @@
 #include <xen/sched.h>
 #include <xen/softirq.h>
 
+#include <asm/extable.h>
 #include <asm/cpufeature.h>
 #include <asm/intc.h>
 #include <asm/processor.h>
@@ -217,6 +218,10 @@ void do_trap(struct cpu_user_regs *cpu_regs)
 
             break;
         }
+
+        if ( fixup_exception(cpu_regs) )
+            break;
+
         fallthrough;
     default:
         if ( cause & CAUSE_IRQ_FLAG )
diff --git a/xen/arch/riscv/xen.lds.S b/xen/arch/riscv/xen.lds.S
index 331a7d63d3c9..65f136dce9f7 100644
--- a/xen/arch/riscv/xen.lds.S
+++ b/xen/arch/riscv/xen.lds.S
@@ -74,6 +74,9 @@ SECTIONS
     .data.ro_after_init : {
         __ro_after_init_start = .;
         *(.data.ro_after_init)
+
+        EX_TABLE
+
         . = ALIGN(PAGE_SIZE);
         __ro_after_init_end = .;
     } : text
diff --git a/xen/arch/x86/xen.lds.S b/xen/arch/x86/xen.lds.S
index c326538ebbb2..b9e888e5962f 100644
--- a/xen/arch/x86/xen.lds.S
+++ b/xen/arch/x86/xen.lds.S
@@ -113,11 +113,7 @@ SECTIONS
        __ro_after_init_start = .;
        *(.data.ro_after_init)
 
-       . = ALIGN(8);
-       /* Exception table */
-       __start___ex_table = .;
-       *(.ex_table)
-       __stop___ex_table = .;
+       EX_TABLE
 
        . = ALIGN(PAGE_SIZE);
        __ro_after_init_end = .;
diff --git a/xen/include/xen/xen.lds.h b/xen/include/xen/xen.lds.h
index 136849ecd515..ea11e3fb6213 100644
--- a/xen/include/xen/xen.lds.h
+++ b/xen/include/xen/xen.lds.h
@@ -219,4 +219,10 @@
 #define VPCI_ARRAY
 #endif
 
+#define EX_TABLE                  \
+        . = ALIGN(4);             \
+        __start___ex_table = .;   \
+        *(.ex_table)              \
+        __stop___ex_table = .;
+
 #endif /* __XEN_LDS_H__ */
-- 
2.53.0