Very few files need the stubs.  Move the infrastructure out of
processor.h and config.h into a new stubs.h, and adjust the includes
accordingly.

Make the per-cpu struct stubs be read mostly; they're unmodified
during the uptime of the CPU, and move them into smpboot.c seeing as
that's where they're allocated and freed.

No functional change.

Signed-off-by: Andrew Cooper <andrew.cooper3@citrix.com>
---
CC: Jan Beulich <JBeulich@suse.com>
CC: Roger Pau Monné <roger.pau@citrix.com>

This is from one of my several failed attempts to deal with the stubs
more nicely in XSA-469.  Still, getting the infrastructure out of
processor.h is a good cleanup.

The FRED series is probably going to result in getting rid of
x86_64/traps.c, given that most of the content gets dropped.
---
 xen/arch/x86/alternative.c           |  1 +
 xen/arch/x86/extable.c               |  8 +++---
 xen/arch/x86/hvm/vmx/vmx.c           |  1 +
 xen/arch/x86/include/asm/config.h    |  5 ----
 xen/arch/x86/include/asm/processor.h | 11 ---------
 xen/arch/x86/include/asm/stubs.h     | 37 ++++++++++++++++++++++++++++
 xen/arch/x86/pv/emul-priv-op.c       |  1 +
 xen/arch/x86/setup.c                 |  1 +
 xen/arch/x86/smpboot.c               |  3 +++
 xen/arch/x86/x86_64/traps.c          |  3 +--
 xen/arch/x86/x86_emulate/private.h   |  2 ++
 11 files changed, 52 insertions(+), 21 deletions(-)
 create mode 100644 xen/arch/x86/include/asm/stubs.h

diff --git a/xen/arch/x86/alternative.c b/xen/arch/x86/alternative.c
index ecc56964bd9c..d4fe56b3dae8 100644
--- a/xen/arch/x86/alternative.c
+++ b/xen/arch/x86/alternative.c
@@ -11,6 +11,7 @@
 #include <asm/alternative.h>
 #include <xen/init.h>
 #include <asm/setup.h>
+#include <asm/stubs.h>
 #include <asm/system.h>
 #include <asm/traps.h>
 #include <asm/nmi.h>
diff --git a/xen/arch/x86/extable.c b/xen/arch/x86/extable.c
index 1572efa69a00..8b78c75d3374 100644
--- a/xen/arch/x86/extable.c
+++ b/xen/arch/x86/extable.c
@@ -1,16 +1,18 @@
 
+#include <xen/domain_page.h>
 #include <xen/init.h>
 #include <xen/list.h>
+#include <xen/livepatch.h>
 #include <xen/perfc.h>
 #include <xen/rcupdate.h>
 #include <xen/sort.h>
 #include <xen/spinlock.h>
-#include <asm/uaccess.h>
-#include <xen/domain_page.h>
 #include <xen/virtual_region.h>
-#include <xen/livepatch.h>
 #include <xen/warning.h>
 
+#include <asm/stubs.h>
+#include <asm/uaccess.h>
+
 #define EX_FIELD(ptr, field) ((unsigned long)&(ptr)->field + (ptr)->field)
 
 static inline unsigned long ex_addr(const struct exception_table_entry *x)
diff --git a/xen/arch/x86/hvm/vmx/vmx.c b/xen/arch/x86/hvm/vmx/vmx.c
index 827db6bdd807..c2262c584822 100644
--- a/xen/arch/x86/hvm/vmx/vmx.c
+++ b/xen/arch/x86/hvm/vmx/vmx.c
@@ -48,6 +48,7 @@
 #include <asm/monitor.h>
 #include <asm/prot-key.h>
 #include <asm/spec_ctrl.h>
+#include <asm/stubs.h>
 #include <public/arch-x86/cpuid.h>
 
 static bool __initdata opt_force_ept;
diff --git a/xen/arch/x86/include/asm/config.h b/xen/arch/x86/include/asm/config.h
index f0123a7de983..3553bf89dc97 100644
--- a/xen/arch/x86/include/asm/config.h
+++ b/xen/arch/x86/include/asm/config.h
@@ -49,11 +49,6 @@
 /* Primary shadow stack is slot 5 of 8, immediately under the primary stack. */
 #define PRIMARY_SHSTK_SLOT 5
 
-/* Total size of syscall and emulation stubs. */
-#define STUB_BUF_SHIFT (L1_CACHE_SHIFT > 7 ? L1_CACHE_SHIFT : 7)
-#define STUB_BUF_SIZE  (1 << STUB_BUF_SHIFT)
-#define STUBS_PER_PAGE (PAGE_SIZE / STUB_BUF_SIZE)
-
 /* Return value for zero-size _xmalloc(), distinguished from NULL. */
 #define ZERO_BLOCK_PTR ((void *)0xBAD0BAD0BAD0BAD0UL)
 
diff --git a/xen/arch/x86/include/asm/processor.h b/xen/arch/x86/include/asm/processor.h
index eacd425c5350..1820e04a32f9 100644
--- a/xen/arch/x86/include/asm/processor.h
+++ b/xen/arch/x86/include/asm/processor.h
@@ -424,17 +424,6 @@ static inline void enable_nmis(void)
 
 void nocall sysenter_entry(void);
 
-struct stubs {
-    union {
-        void(*func)(void);
-        unsigned long addr;
-    };
-    unsigned long mfn;
-};
-
-DECLARE_PER_CPU(struct stubs, stubs);
-unsigned long alloc_stub_page(unsigned int cpu, unsigned long *mfn);
-
 static inline uint8_t get_cpu_family(uint32_t raw, uint8_t *model,
                                      uint8_t *stepping)
 {
diff --git a/xen/arch/x86/include/asm/stubs.h b/xen/arch/x86/include/asm/stubs.h
new file mode 100644
index 000000000000..a520928e9a50
--- /dev/null
+++ b/xen/arch/x86/include/asm/stubs.h
@@ -0,0 +1,37 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+#ifndef X86_ASM_STUBS_H
+#define X86_ASM_STUBS_H
+
+/*
+ * Xen has several per-cpu executable stubs which are written dynamically.
+ * These are:
+ *
+ * - The SYSCALL entry stubs, LSTAR and CSTAR.  These are written on boot, and
+ *   are responsible for moving back onto Xen's stack.
+ *
+ * - The emulation stub.  This is used to replay an instruction or sequence
+ *   which trapped for emulation.
+ *
+ * The stubs have an executable alias in l2_xenmap[] (i.e. within 1G of the
+ * rest of .text), and are written via map_domain_page().
+ */
+
+#include <xen/percpu.h>
+
+/* Total size of syscall and emulation stubs. */
+#define STUB_BUF_SHIFT (L1_CACHE_SHIFT > 7 ? L1_CACHE_SHIFT : 7)
+#define STUB_BUF_SIZE  (1 << STUB_BUF_SHIFT)
+#define STUBS_PER_PAGE (PAGE_SIZE / STUB_BUF_SIZE)
+
+struct stubs {
+    union {
+        void (*func)(void);
+        unsigned long addr;
+    };
+    unsigned long mfn;
+};
+
+DECLARE_PER_CPU(struct stubs, stubs);
+unsigned long alloc_stub_page(unsigned int cpu, unsigned long *mfn);
+
+#endif /* X86_ASM_STUBS_H */
diff --git a/xen/arch/x86/pv/emul-priv-op.c b/xen/arch/x86/pv/emul-priv-op.c
index 295d847ea24c..4da3f379ef3e 100644
--- a/xen/arch/x86/pv/emul-priv-op.c
+++ b/xen/arch/x86/pv/emul-priv-op.c
@@ -21,6 +21,7 @@
 #include <asm/pv/domain.h>
 #include <asm/pv/trace.h>
 #include <asm/shared.h>
+#include <asm/stubs.h>
 
 #include <xsm/xsm.h>
 
diff --git a/xen/arch/x86/setup.c b/xen/arch/x86/setup.c
index 25189541244d..1f5cb67bd0ee 100644
--- a/xen/arch/x86/setup.c
+++ b/xen/arch/x86/setup.c
@@ -55,6 +55,7 @@
 #include <asm/setup.h>
 #include <asm/smp.h>
 #include <asm/spec_ctrl.h>
+#include <asm/stubs.h>
 #include <asm/tboot.h>
 #include <asm/trampoline.h>
 #include <asm/traps.h>
diff --git a/xen/arch/x86/smpboot.c b/xen/arch/x86/smpboot.c
index 0189d6c332a4..41fe67d43c94 100644
--- a/xen/arch/x86/smpboot.c
+++ b/xen/arch/x86/smpboot.c
@@ -38,6 +38,7 @@
 #include <asm/prot-key.h>
 #include <asm/setup.h>
 #include <asm/spec_ctrl.h>
+#include <asm/stubs.h>
 #include <asm/tboot.h>
 #include <asm/time.h>
 #include <asm/trampoline.h>
@@ -57,6 +58,8 @@ static cpumask_t scratch_cpu0mask;
 DEFINE_PER_CPU_READ_MOSTLY(cpumask_var_t, send_ipi_cpumask);
 static cpumask_t send_ipi_cpu0mask;
 
+DEFINE_PER_CPU_READ_MOSTLY(struct stubs, stubs);
+
 cpumask_t cpu_online_map __read_mostly;
 EXPORT_SYMBOL(cpu_online_map);
 
diff --git a/xen/arch/x86/x86_64/traps.c b/xen/arch/x86/x86_64/traps.c
index ac0fafd72d31..b10bd0becafb 100644
--- a/xen/arch/x86/x86_64/traps.c
+++ b/xen/arch/x86/x86_64/traps.c
@@ -21,6 +21,7 @@
 #include <asm/nmi.h>
 #include <asm/page.h>
 #include <asm/shared.h>
+#include <asm/stubs.h>
 #include <asm/traps.h>
 
 
@@ -330,8 +331,6 @@ static unsigned int write_stub_trampoline(
     return ROUNDUP(p - stub, 16);
 }
 
-DEFINE_PER_CPU(struct stubs, stubs);
-
 void nocall lstar_enter(void);
 void nocall cstar_enter(void);
 
diff --git a/xen/arch/x86/x86_emulate/private.h b/xen/arch/x86/x86_emulate/private.h
index 30be59547032..c4138afe1db5 100644
--- a/xen/arch/x86/x86_emulate/private.h
+++ b/xen/arch/x86/x86_emulate/private.h
@@ -10,8 +10,10 @@
 
 # include <xen/bug.h>
 # include <xen/kernel.h>
+
 # include <asm/endbr.h>
 # include <asm/msr-index.h>
+# include <asm/stubs.h>
 # include <asm/x86-vendors.h>
 # include <asm/x86_emulate.h>
 
-- 
2.34.1

On 16.05.2025 15:33, Andrew Cooper wrote:
> Very few files need the stubs.  Move the infrastructure out of
> processor.h and config.h into a new stubs.h, and adjust the includes
> accordingly.
> 
> Make the per-cpu struct stubs be read mostly; they're unmodified
> during the uptime of the CPU, and move them into smpboot.c seeing as
> that's where they're allocated and freed.
> 
> No functional change.
> 
> Signed-off-by: Andrew Cooper <andrew.cooper3@citrix.com>

Acked-by: Jan Beulich <jbeulich@suse.com>
with one possible suggestion:

> --- /dev/null
> +++ b/xen/arch/x86/include/asm/stubs.h
> @@ -0,0 +1,37 @@
> +/* SPDX-License-Identifier: GPL-2.0-only */
> +#ifndef X86_ASM_STUBS_H
> +#define X86_ASM_STUBS_H
> +
> +/*
> + * Xen has several per-cpu executable stubs which are written dynamically.

This puts it pretty well. Yet in principle there may be further, perhaps
entirely different stubs in the future. Hence stubs.h feels a little
generic. What about exec-stubs.h?

Jan

On 16/05/2025 2:41 pm, Jan Beulich wrote:
> On 16.05.2025 15:33, Andrew Cooper wrote:
>> Very few files need the stubs.  Move the infrastructure out of
>> processor.h and config.h into a new stubs.h, and adjust the includes
>> accordingly.
>>
>> Make the per-cpu struct stubs be read mostly; they're unmodified
>> during the uptime of the CPU, and move them into smpboot.c seeing as
>> that's where they're allocated and freed.
>>
>> No functional change.
>>
>> Signed-off-by: Andrew Cooper <andrew.cooper3@citrix.com>
> Acked-by: Jan Beulich <jbeulich@suse.com>

Thanks.

> with one possible suggestion:
>
>> --- /dev/null
>> +++ b/xen/arch/x86/include/asm/stubs.h
>> @@ -0,0 +1,37 @@
>> +/* SPDX-License-Identifier: GPL-2.0-only */
>> +#ifndef X86_ASM_STUBS_H
>> +#define X86_ASM_STUBS_H
>> +
>> +/*
>> + * Xen has several per-cpu executable stubs which are written dynamically.
> This puts it pretty well. Yet in principle there may be further, perhaps
> entirely different stubs in the future. Hence stubs.h feels a little
> generic. What about exec-stubs.h?

stubs is quite generic; in fact, that was my feedback for struct stubs.

There is something to be said for the header file to be the same as the
struct you want from it.

What did you have in mind for "different stubs"?  The only thing that
makes these special (i.e. not regular per-cpu data) is that we need an
executable mapping of them.  So, while I think it's reasonably likely
that we'll gain other uses (although, we're losing LSTAR/CSTAR when FRED
is enabled), I'm less certain what non-executable stubs would look like.

~Andrew

On 16.05.2025 15:49, Andrew Cooper wrote:
> On 16/05/2025 2:41 pm, Jan Beulich wrote:
>> On 16.05.2025 15:33, Andrew Cooper wrote:
>>> --- /dev/null
>>> +++ b/xen/arch/x86/include/asm/stubs.h
>>> @@ -0,0 +1,37 @@
>>> +/* SPDX-License-Identifier: GPL-2.0-only */
>>> +#ifndef X86_ASM_STUBS_H
>>> +#define X86_ASM_STUBS_H
>>> +
>>> +/*
>>> + * Xen has several per-cpu executable stubs which are written dynamically.
>> This puts it pretty well. Yet in principle there may be further, perhaps
>> entirely different stubs in the future. Hence stubs.h feels a little
>> generic. What about exec-stubs.h?
> 
> stubs is quite generic; in fact, that was my feedback for struct stubs.
> 
> There is something to be said for the header file to be the same as the
> struct you want from it.
> 
> What did you have in mind for "different stubs"?  The only thing that
> makes these special (i.e. not regular per-cpu data) is that we need an
> executable mapping of them.  So, while I think it's reasonably likely
> that we'll gain other uses (although, we're losing LSTAR/CSTAR when FRED
> is enabled), I'm less certain what non-executable stubs would look like.

A while after having written my reply, I started wondering myself. Likely
we wouldn't normally call anything non-executable a "stub", so please
disregard my suggestion.

Jan