Some cleanups and reorg of paravirt code and headers: - The first 2 patches should be not controversial at all, as they remove just some no longer needed #include and struct forward declarations. - The 3rd patch is removing CONFIG_PARAVIRT_DEBUG, which IMO has no real value, as it just changes a crash to a BUG() (the stack trace will basically be the same). As the maintainer of the main paravirt user (Xen) I have never seen this crash/BUG() to happen. - The 4th patch is just a movement of code. - I don't know for what reason asm/paravirt_api_clock.h was added, as all archs supporting it do it exactly in the same way. Patch 5 is removing it. - Patches 6-14 are streamlining the paravirt clock interfaces by using a common implementation across architectures where possible and by moving the related code into common sched code, as this is where it should live. - Patches 15-20 are more like RFC material preparing the paravirt infrastructure to support multiple pv_ops function arrays. As a prerequisite for that it makes life in objtool much easier with dropping the Xen static initializers of the pv_ops sub- structures, which is done in patches 15-17. Patches 18-20 are doing the real preparations for multiple pv_ops arrays and using those arrays in multiple headers. - Patch 21 is an example how the new scheme can look like using the PV-spinlocks. Changes in V2: - new patches 13-18 and 20 - complete rework of patch 21 Changes in V3: - fixed 2 issues detected by kernel test robot Juergen Gross (21): x86/paravirt: Remove not needed includes of paravirt.h x86/paravirt: Remove some unneeded struct declarations x86/paravirt: Remove PARAVIRT_DEBUG config option x86/paravirt: Move thunk macros to paravirt_types.h paravirt: Remove asm/paravirt_api_clock.h sched: Move clock related paravirt code to kernel/sched arm/paravirt: Use common code for paravirt_steal_clock() arm64/paravirt: Use common code for paravirt_steal_clock() loongarch/paravirt: Use common code for paravirt_steal_clock() riscv/paravirt: Use common code for paravirt_steal_clock() x86/paravirt: Use common code for paravirt_steal_clock() x86/paravirt: Move paravirt_sched_clock() related code into tsc.c x86/paravirt: Introduce new paravirt-base.h header x86/paravirt: Move pv_native_*() prototypes to paravirt.c x86/xen: Drop xen_irq_ops x86/xen: Drop xen_cpu_ops x86/xen: Drop xen_mmu_ops objtool: Allow multiple pv_ops arrays x86/paravirt: Allow pv-calls outside paravirt.h x86/paravirt: Specify pv_ops array in paravirt macros x86/pvlocks: Move paravirt spinlock functions into own header arch/Kconfig | 3 + arch/arm/Kconfig | 1 + arch/arm/include/asm/paravirt.h | 22 -- arch/arm/include/asm/paravirt_api_clock.h | 1 - arch/arm/kernel/Makefile | 1 - arch/arm/kernel/paravirt.c | 23 -- arch/arm64/Kconfig | 1 + arch/arm64/include/asm/paravirt.h | 14 - arch/arm64/include/asm/paravirt_api_clock.h | 1 - arch/arm64/kernel/paravirt.c | 11 +- arch/loongarch/Kconfig | 1 + arch/loongarch/include/asm/paravirt.h | 13 - .../include/asm/paravirt_api_clock.h | 1 - arch/loongarch/kernel/paravirt.c | 10 +- arch/powerpc/include/asm/paravirt.h | 3 - arch/powerpc/include/asm/paravirt_api_clock.h | 2 - arch/powerpc/platforms/pseries/setup.c | 4 +- arch/riscv/Kconfig | 1 + arch/riscv/include/asm/paravirt.h | 14 - arch/riscv/include/asm/paravirt_api_clock.h | 1 - arch/riscv/kernel/paravirt.c | 11 +- arch/x86/Kconfig | 8 +- arch/x86/entry/entry_64.S | 1 - arch/x86/entry/vsyscall/vsyscall_64.c | 1 - arch/x86/hyperv/hv_spinlock.c | 11 +- arch/x86/include/asm/apic.h | 4 - arch/x86/include/asm/highmem.h | 1 - arch/x86/include/asm/mshyperv.h | 1 - arch/x86/include/asm/paravirt-base.h | 29 ++ arch/x86/include/asm/paravirt-spinlock.h | 146 ++++++++ arch/x86/include/asm/paravirt.h | 331 +++++------------- arch/x86/include/asm/paravirt_api_clock.h | 1 - arch/x86/include/asm/paravirt_types.h | 269 +++++++------- arch/x86/include/asm/pgtable_32.h | 1 - arch/x86/include/asm/ptrace.h | 2 +- arch/x86/include/asm/qspinlock.h | 89 +---- arch/x86/include/asm/spinlock.h | 1 - arch/x86/include/asm/timer.h | 1 + arch/x86/include/asm/tlbflush.h | 4 - arch/x86/kernel/Makefile | 2 +- arch/x86/kernel/apm_32.c | 1 - arch/x86/kernel/callthunks.c | 1 - arch/x86/kernel/cpu/bugs.c | 1 - arch/x86/kernel/cpu/vmware.c | 1 + arch/x86/kernel/kvm.c | 11 +- arch/x86/kernel/kvmclock.c | 1 + arch/x86/kernel/paravirt-spinlocks.c | 26 +- arch/x86/kernel/paravirt.c | 42 +-- arch/x86/kernel/tsc.c | 10 +- arch/x86/kernel/vsmp_64.c | 1 - arch/x86/kernel/x86_init.c | 1 - arch/x86/lib/cache-smp.c | 1 - arch/x86/mm/init.c | 1 - arch/x86/xen/enlighten_pv.c | 82 ++--- arch/x86/xen/irq.c | 20 +- arch/x86/xen/mmu_pv.c | 100 ++---- arch/x86/xen/spinlock.c | 11 +- arch/x86/xen/time.c | 2 + drivers/clocksource/hyperv_timer.c | 2 + drivers/xen/time.c | 2 +- include/linux/sched/cputime.h | 18 + kernel/sched/core.c | 5 + kernel/sched/cputime.c | 13 + kernel/sched/sched.h | 3 +- tools/objtool/arch/x86/decode.c | 8 +- tools/objtool/check.c | 78 ++++- tools/objtool/include/objtool/check.h | 2 + 67 files changed, 659 insertions(+), 827 deletions(-) delete mode 100644 arch/arm/include/asm/paravirt.h delete mode 100644 arch/arm/include/asm/paravirt_api_clock.h delete mode 100644 arch/arm/kernel/paravirt.c delete mode 100644 arch/arm64/include/asm/paravirt_api_clock.h delete mode 100644 arch/loongarch/include/asm/paravirt_api_clock.h delete mode 100644 arch/powerpc/include/asm/paravirt_api_clock.h delete mode 100644 arch/riscv/include/asm/paravirt_api_clock.h create mode 100644 arch/x86/include/asm/paravirt-base.h create mode 100644 arch/x86/include/asm/paravirt-spinlock.h delete mode 100644 arch/x86/include/asm/paravirt_api_clock.h -- 2.51.0 In some places asm/paravirt.h is included without really being needed. Remove the related #include statements. Signed-off-by: Juergen Gross <jgross@suse.com> Acked-by: Peter Zijlstra (Intel) <peterz@infradead.org> --- V3: - reinstate the include in mmu_context.h (kernel test robot) --- arch/x86/entry/entry_64.S | 1 - arch/x86/entry/vsyscall/vsyscall_64.c | 1 - arch/x86/hyperv/hv_spinlock.c | 1 - arch/x86/include/asm/apic.h | 4 ---- arch/x86/include/asm/highmem.h | 1 - arch/x86/include/asm/mshyperv.h | 1 - arch/x86/include/asm/pgtable_32.h | 1 - arch/x86/include/asm/spinlock.h | 1 - arch/x86/include/asm/tlbflush.h | 4 ---- arch/x86/kernel/apm_32.c | 1 - arch/x86/kernel/callthunks.c | 1 - arch/x86/kernel/cpu/bugs.c | 1 - arch/x86/kernel/vsmp_64.c | 1 - arch/x86/kernel/x86_init.c | 1 - arch/x86/lib/cache-smp.c | 1 - arch/x86/mm/init.c | 1 - arch/x86/xen/spinlock.c | 1 - 17 files changed, 23 deletions(-) diff --git a/arch/x86/entry/entry_64.S b/arch/x86/entry/entry_64.S index XXXXXXX..XXXXXXX 100644 --- a/arch/x86/entry/entry_64.S +++ b/arch/x86/entry/entry_64.S @@ -XXX,XX +XXX,XX @@ #include <asm/hw_irq.h> #include <asm/page_types.h> #include <asm/irqflags.h> -#include <asm/paravirt.h> #include <asm/percpu.h> #include <asm/asm.h> #include <asm/smap.h> diff --git a/arch/x86/entry/vsyscall/vsyscall_64.c b/arch/x86/entry/vsyscall/vsyscall_64.c index XXXXXXX..XXXXXXX 100644 --- a/arch/x86/entry/vsyscall/vsyscall_64.c +++ b/arch/x86/entry/vsyscall/vsyscall_64.c @@ -XXX,XX +XXX,XX @@ #include <asm/unistd.h> #include <asm/fixmap.h> #include <asm/traps.h> -#include <asm/paravirt.h> #define CREATE_TRACE_POINTS #include "vsyscall_trace.h" diff --git a/arch/x86/hyperv/hv_spinlock.c b/arch/x86/hyperv/hv_spinlock.c index XXXXXXX..XXXXXXX 100644 --- a/arch/x86/hyperv/hv_spinlock.c +++ b/arch/x86/hyperv/hv_spinlock.c @@ -XXX,XX +XXX,XX @@ #include <linux/spinlock.h> #include <asm/mshyperv.h> -#include <asm/paravirt.h> #include <asm/apic.h> #include <asm/msr.h> diff --git a/arch/x86/include/asm/apic.h b/arch/x86/include/asm/apic.h index XXXXXXX..XXXXXXX 100644 --- a/arch/x86/include/asm/apic.h +++ b/arch/x86/include/asm/apic.h @@ -XXX,XX +XXX,XX @@ static inline bool apic_from_smp_config(void) /* * Basic functions accessing APICs. */ -#ifdef CONFIG_PARAVIRT -#include <asm/paravirt.h> -#endif - static inline void native_apic_mem_write(u32 reg, u32 v) { volatile u32 *addr = (volatile u32 *)(APIC_BASE + reg); diff --git a/arch/x86/include/asm/highmem.h b/arch/x86/include/asm/highmem.h index XXXXXXX..XXXXXXX 100644 --- a/arch/x86/include/asm/highmem.h +++ b/arch/x86/include/asm/highmem.h @@ -XXX,XX +XXX,XX @@ #include <linux/interrupt.h> #include <linux/threads.h> #include <asm/tlbflush.h> -#include <asm/paravirt.h> #include <asm/fixmap.h> #include <asm/pgtable_areas.h> diff --git a/arch/x86/include/asm/mshyperv.h b/arch/x86/include/asm/mshyperv.h index XXXXXXX..XXXXXXX 100644 --- a/arch/x86/include/asm/mshyperv.h +++ b/arch/x86/include/asm/mshyperv.h @@ -XXX,XX +XXX,XX @@ #include <linux/msi.h> #include <linux/io.h> #include <asm/nospec-branch.h> -#include <asm/paravirt.h> #include <asm/msr.h> #include <hyperv/hvhdk.h> diff --git a/arch/x86/include/asm/pgtable_32.h b/arch/x86/include/asm/pgtable_32.h index XXXXXXX..XXXXXXX 100644 --- a/arch/x86/include/asm/pgtable_32.h +++ b/arch/x86/include/asm/pgtable_32.h @@ -XXX,XX +XXX,XX @@ #ifndef __ASSEMBLER__ #include <asm/processor.h> #include <linux/threads.h> -#include <asm/paravirt.h> #include <linux/bitops.h> #include <linux/list.h> diff --git a/arch/x86/include/asm/spinlock.h b/arch/x86/include/asm/spinlock.h index XXXXXXX..XXXXXXX 100644 --- a/arch/x86/include/asm/spinlock.h +++ b/arch/x86/include/asm/spinlock.h @@ -XXX,XX +XXX,XX @@ #include <asm/page.h> #include <asm/processor.h> #include <linux/compiler.h> -#include <asm/paravirt.h> #include <asm/bitops.h> /* diff --git a/arch/x86/include/asm/tlbflush.h b/arch/x86/include/asm/tlbflush.h index XXXXXXX..XXXXXXX 100644 --- a/arch/x86/include/asm/tlbflush.h +++ b/arch/x86/include/asm/tlbflush.h @@ -XXX,XX +XXX,XX @@ static inline void mm_clear_asid_transition(struct mm_struct *mm) { } static inline bool mm_in_asid_transition(struct mm_struct *mm) { return false; } #endif /* CONFIG_BROADCAST_TLB_FLUSH */ -#ifdef CONFIG_PARAVIRT -#include <asm/paravirt.h> -#endif - #define flush_tlb_mm(mm) \ flush_tlb_mm_range(mm, 0UL, TLB_FLUSH_ALL, 0UL, true) diff --git a/arch/x86/kernel/apm_32.c b/arch/x86/kernel/apm_32.c index XXXXXXX..XXXXXXX 100644 --- a/arch/x86/kernel/apm_32.c +++ b/arch/x86/kernel/apm_32.c @@ -XXX,XX +XXX,XX @@ #include <linux/uaccess.h> #include <asm/desc.h> #include <asm/olpc.h> -#include <asm/paravirt.h> #include <asm/reboot.h> #include <asm/nospec-branch.h> #include <asm/ibt.h> diff --git a/arch/x86/kernel/callthunks.c b/arch/x86/kernel/callthunks.c index XXXXXXX..XXXXXXX 100644 --- a/arch/x86/kernel/callthunks.c +++ b/arch/x86/kernel/callthunks.c @@ -XXX,XX +XXX,XX @@ #include <asm/insn.h> #include <asm/kexec.h> #include <asm/nospec-branch.h> -#include <asm/paravirt.h> #include <asm/sections.h> #include <asm/switch_to.h> #include <asm/sync_core.h> diff --git a/arch/x86/kernel/cpu/bugs.c b/arch/x86/kernel/cpu/bugs.c index XXXXXXX..XXXXXXX 100644 --- a/arch/x86/kernel/cpu/bugs.c +++ b/arch/x86/kernel/cpu/bugs.c @@ -XXX,XX +XXX,XX @@ #include <asm/fpu/api.h> #include <asm/msr.h> #include <asm/vmx.h> -#include <asm/paravirt.h> #include <asm/cpu_device_id.h> #include <asm/e820/api.h> #include <asm/hypervisor.h> diff --git a/arch/x86/kernel/vsmp_64.c b/arch/x86/kernel/vsmp_64.c index XXXXXXX..XXXXXXX 100644 --- a/arch/x86/kernel/vsmp_64.c +++ b/arch/x86/kernel/vsmp_64.c @@ -XXX,XX +XXX,XX @@ #include <asm/apic.h> #include <asm/pci-direct.h> #include <asm/io.h> -#include <asm/paravirt.h> #include <asm/setup.h> #define TOPOLOGY_REGISTER_OFFSET 0x10 diff --git a/arch/x86/kernel/x86_init.c b/arch/x86/kernel/x86_init.c index XXXXXXX..XXXXXXX 100644 --- a/arch/x86/kernel/x86_init.c +++ b/arch/x86/kernel/x86_init.c @@ -XXX,XX +XXX,XX @@ #include <asm/acpi.h> #include <asm/bios_ebda.h> -#include <asm/paravirt.h> #include <asm/pci_x86.h> #include <asm/mpspec.h> #include <asm/setup.h> diff --git a/arch/x86/lib/cache-smp.c b/arch/x86/lib/cache-smp.c index XXXXXXX..XXXXXXX 100644 --- a/arch/x86/lib/cache-smp.c +++ b/arch/x86/lib/cache-smp.c @@ -XXX,XX +XXX,XX @@ // SPDX-License-Identifier: GPL-2.0 -#include <asm/paravirt.h> #include <linux/smp.h> #include <linux/export.h> diff --git a/arch/x86/mm/init.c b/arch/x86/mm/init.c index XXXXXXX..XXXXXXX 100644 --- a/arch/x86/mm/init.c +++ b/arch/x86/mm/init.c @@ -XXX,XX +XXX,XX @@ #include <asm/pti.h> #include <asm/text-patching.h> #include <asm/memtype.h> -#include <asm/paravirt.h> #include <asm/mmu_context.h> /* diff --git a/arch/x86/xen/spinlock.c b/arch/x86/xen/spinlock.c index XXXXXXX..XXXXXXX 100644 --- a/arch/x86/xen/spinlock.c +++ b/arch/x86/xen/spinlock.c @@ -XXX,XX +XXX,XX @@ #include <linux/slab.h> #include <linux/atomic.h> -#include <asm/paravirt.h> #include <asm/qspinlock.h> #include <xen/events.h> -- 2.51.0 Paravirt clock related functions are available in multiple archs. In order to share the common parts, move the common static keys to kernel/sched/ and remove them from the arch specific files. Make a common paravirt_steal_clock() implementation available in kernel/sched/cputime.c, guarding it with a new config option CONFIG_HAVE_PV_STEAL_CLOCK_GEN, which can be selected by an arch in case it wants to use that common variant. Signed-off-by: Juergen Gross <jgross@suse.com> Acked-by: Peter Zijlstra (Intel) <peterz@infradead.org> --- arch/Kconfig | 3 +++ arch/arm/include/asm/paravirt.h | 4 ---- arch/arm/kernel/paravirt.c | 3 --- arch/arm64/include/asm/paravirt.h | 4 ---- arch/arm64/kernel/paravirt.c | 4 +--- arch/loongarch/include/asm/paravirt.h | 3 --- arch/loongarch/kernel/paravirt.c | 3 +-- arch/powerpc/include/asm/paravirt.h | 3 --- arch/powerpc/platforms/pseries/setup.c | 4 +--- arch/riscv/include/asm/paravirt.h | 4 ---- arch/riscv/kernel/paravirt.c | 4 +--- arch/x86/include/asm/paravirt.h | 4 ---- arch/x86/kernel/cpu/vmware.c | 1 + arch/x86/kernel/kvm.c | 1 + arch/x86/kernel/paravirt.c | 3 --- drivers/xen/time.c | 1 + include/linux/sched/cputime.h | 18 ++++++++++++++++++ kernel/sched/core.c | 5 +++++ kernel/sched/cputime.c | 13 +++++++++++++ kernel/sched/sched.h | 2 +- 20 files changed, 47 insertions(+), 40 deletions(-) diff --git a/arch/Kconfig b/arch/Kconfig index XXXXXXX..XXXXXXX 100644 --- a/arch/Kconfig +++ b/arch/Kconfig @@ -XXX,XX +XXX,XX @@ config HAVE_IRQ_TIME_ACCOUNTING Archs need to ensure they use a high enough resolution clock to support irq time accounting and then call enable_sched_clock_irqtime(). +config HAVE_PV_STEAL_CLOCK_GEN + bool + config HAVE_MOVE_PUD bool help diff --git a/arch/arm/include/asm/paravirt.h b/arch/arm/include/asm/paravirt.h index XXXXXXX..XXXXXXX 100644 --- a/arch/arm/include/asm/paravirt.h +++ b/arch/arm/include/asm/paravirt.h @@ -XXX,XX +XXX,XX @@ #ifdef CONFIG_PARAVIRT #include <linux/static_call_types.h> -struct static_key; -extern struct static_key paravirt_steal_enabled; -extern struct static_key paravirt_steal_rq_enabled; - u64 dummy_steal_clock(int cpu); DECLARE_STATIC_CALL(pv_steal_clock, dummy_steal_clock); diff --git a/arch/arm/kernel/paravirt.c b/arch/arm/kernel/paravirt.c index XXXXXXX..XXXXXXX 100644 --- a/arch/arm/kernel/paravirt.c +++ b/arch/arm/kernel/paravirt.c @@ -XXX,XX +XXX,XX @@ #include <linux/static_call.h> #include <asm/paravirt.h> -struct static_key paravirt_steal_enabled; -struct static_key paravirt_steal_rq_enabled; - static u64 native_steal_clock(int cpu) { return 0; diff --git a/arch/arm64/include/asm/paravirt.h b/arch/arm64/include/asm/paravirt.h index XXXXXXX..XXXXXXX 100644 --- a/arch/arm64/include/asm/paravirt.h +++ b/arch/arm64/include/asm/paravirt.h @@ -XXX,XX +XXX,XX @@ #ifdef CONFIG_PARAVIRT #include <linux/static_call_types.h> -struct static_key; -extern struct static_key paravirt_steal_enabled; -extern struct static_key paravirt_steal_rq_enabled; - u64 dummy_steal_clock(int cpu); DECLARE_STATIC_CALL(pv_steal_clock, dummy_steal_clock); diff --git a/arch/arm64/kernel/paravirt.c b/arch/arm64/kernel/paravirt.c index XXXXXXX..XXXXXXX 100644 --- a/arch/arm64/kernel/paravirt.c +++ b/arch/arm64/kernel/paravirt.c @@ -XXX,XX +XXX,XX @@ #include <linux/slab.h> #include <linux/types.h> #include <linux/static_call.h> +#include <linux/sched/cputime.h> #include <asm/paravirt.h> #include <asm/pvclock-abi.h> #include <asm/smp_plat.h> -struct static_key paravirt_steal_enabled; -struct static_key paravirt_steal_rq_enabled; - static u64 native_steal_clock(int cpu) { return 0; diff --git a/arch/loongarch/include/asm/paravirt.h b/arch/loongarch/include/asm/paravirt.h index XXXXXXX..XXXXXXX 100644 --- a/arch/loongarch/include/asm/paravirt.h +++ b/arch/loongarch/include/asm/paravirt.h @@ -XXX,XX +XXX,XX @@ #ifdef CONFIG_PARAVIRT #include <linux/static_call_types.h> -struct static_key; -extern struct static_key paravirt_steal_enabled; -extern struct static_key paravirt_steal_rq_enabled; u64 dummy_steal_clock(int cpu); DECLARE_STATIC_CALL(pv_steal_clock, dummy_steal_clock); diff --git a/arch/loongarch/kernel/paravirt.c b/arch/loongarch/kernel/paravirt.c index XXXXXXX..XXXXXXX 100644 --- a/arch/loongarch/kernel/paravirt.c +++ b/arch/loongarch/kernel/paravirt.c @@ -XXX,XX +XXX,XX @@ #include <linux/kvm_para.h> #include <linux/reboot.h> #include <linux/static_call.h> +#include <linux/sched/cputime.h> #include <asm/paravirt.h> static int has_steal_clock; -struct static_key paravirt_steal_enabled; -struct static_key paravirt_steal_rq_enabled; static DEFINE_PER_CPU(struct kvm_steal_time, steal_time) __aligned(64); DEFINE_STATIC_KEY_FALSE(virt_spin_lock_key); diff --git a/arch/powerpc/include/asm/paravirt.h b/arch/powerpc/include/asm/paravirt.h index XXXXXXX..XXXXXXX 100644 --- a/arch/powerpc/include/asm/paravirt.h +++ b/arch/powerpc/include/asm/paravirt.h @@ -XXX,XX +XXX,XX @@ static inline bool is_shared_processor(void) } #ifdef CONFIG_PARAVIRT_TIME_ACCOUNTING -extern struct static_key paravirt_steal_enabled; -extern struct static_key paravirt_steal_rq_enabled; - u64 pseries_paravirt_steal_clock(int cpu); static inline u64 paravirt_steal_clock(int cpu) diff --git a/arch/powerpc/platforms/pseries/setup.c b/arch/powerpc/platforms/pseries/setup.c index XXXXXXX..XXXXXXX 100644 --- a/arch/powerpc/platforms/pseries/setup.c +++ b/arch/powerpc/platforms/pseries/setup.c @@ -XXX,XX +XXX,XX @@ #include <linux/memblock.h> #include <linux/swiotlb.h> #include <linux/seq_buf.h> +#include <linux/sched/cputime.h> #include <asm/mmu.h> #include <asm/processor.h> @@ -XXX,XX +XXX,XX @@ DEFINE_STATIC_KEY_FALSE(shared_processor); EXPORT_SYMBOL(shared_processor); #ifdef CONFIG_PARAVIRT_TIME_ACCOUNTING -struct static_key paravirt_steal_enabled; -struct static_key paravirt_steal_rq_enabled; - static bool steal_acc = true; static int __init parse_no_stealacc(char *arg) { diff --git a/arch/riscv/include/asm/paravirt.h b/arch/riscv/include/asm/paravirt.h index XXXXXXX..XXXXXXX 100644 --- a/arch/riscv/include/asm/paravirt.h +++ b/arch/riscv/include/asm/paravirt.h @@ -XXX,XX +XXX,XX @@ #ifdef CONFIG_PARAVIRT #include <linux/static_call_types.h> -struct static_key; -extern struct static_key paravirt_steal_enabled; -extern struct static_key paravirt_steal_rq_enabled; - u64 dummy_steal_clock(int cpu); DECLARE_STATIC_CALL(pv_steal_clock, dummy_steal_clock); diff --git a/arch/riscv/kernel/paravirt.c b/arch/riscv/kernel/paravirt.c index XXXXXXX..XXXXXXX 100644 --- a/arch/riscv/kernel/paravirt.c +++ b/arch/riscv/kernel/paravirt.c @@ -XXX,XX +XXX,XX @@ #include <linux/printk.h> #include <linux/static_call.h> #include <linux/types.h> +#include <linux/sched/cputime.h> #include <asm/barrier.h> #include <asm/page.h> #include <asm/paravirt.h> #include <asm/sbi.h> -struct static_key paravirt_steal_enabled; -struct static_key paravirt_steal_rq_enabled; - static u64 native_steal_clock(int cpu) { return 0; diff --git a/arch/x86/include/asm/paravirt.h b/arch/x86/include/asm/paravirt.h index XXXXXXX..XXXXXXX 100644 --- a/arch/x86/include/asm/paravirt.h +++ b/arch/x86/include/asm/paravirt.h @@ -XXX,XX +XXX,XX @@ static __always_inline u64 paravirt_sched_clock(void) return static_call(pv_sched_clock)(); } -struct static_key; -extern struct static_key paravirt_steal_enabled; -extern struct static_key paravirt_steal_rq_enabled; - __visible void __native_queued_spin_unlock(struct qspinlock *lock); bool pv_is_native_spin_unlock(void); __visible bool __native_vcpu_is_preempted(long cpu); diff --git a/arch/x86/kernel/cpu/vmware.c b/arch/x86/kernel/cpu/vmware.c index XXXXXXX..XXXXXXX 100644 --- a/arch/x86/kernel/cpu/vmware.c +++ b/arch/x86/kernel/cpu/vmware.c @@ -XXX,XX +XXX,XX @@ #include <linux/efi.h> #include <linux/reboot.h> #include <linux/static_call.h> +#include <linux/sched/cputime.h> #include <asm/div64.h> #include <asm/x86_init.h> #include <asm/hypervisor.h> diff --git a/arch/x86/kernel/kvm.c b/arch/x86/kernel/kvm.c index XXXXXXX..XXXXXXX 100644 --- a/arch/x86/kernel/kvm.c +++ b/arch/x86/kernel/kvm.c @@ -XXX,XX +XXX,XX @@ #include <linux/syscore_ops.h> #include <linux/cc_platform.h> #include <linux/efi.h> +#include <linux/sched/cputime.h> #include <asm/timer.h> #include <asm/cpu.h> #include <asm/traps.h> diff --git a/arch/x86/kernel/paravirt.c b/arch/x86/kernel/paravirt.c index XXXXXXX..XXXXXXX 100644 --- a/arch/x86/kernel/paravirt.c +++ b/arch/x86/kernel/paravirt.c @@ -XXX,XX +XXX,XX @@ void __init native_pv_lock_init(void) static_branch_enable(&virt_spin_lock_key); } -struct static_key paravirt_steal_enabled; -struct static_key paravirt_steal_rq_enabled; - static u64 native_steal_clock(int cpu) { return 0; diff --git a/drivers/xen/time.c b/drivers/xen/time.c index XXXXXXX..XXXXXXX 100644 --- a/drivers/xen/time.c +++ b/drivers/xen/time.c @@ -XXX,XX +XXX,XX @@ #include <linux/gfp.h> #include <linux/slab.h> #include <linux/static_call.h> +#include <linux/sched/cputime.h> #include <asm/paravirt.h> #include <asm/xen/hypervisor.h> diff --git a/include/linux/sched/cputime.h b/include/linux/sched/cputime.h index XXXXXXX..XXXXXXX 100644 --- a/include/linux/sched/cputime.h +++ b/include/linux/sched/cputime.h @@ -XXX,XX +XXX,XX @@ #ifndef _LINUX_SCHED_CPUTIME_H #define _LINUX_SCHED_CPUTIME_H +#include <linux/static_call_types.h> #include <linux/sched/signal.h> /* @@ -XXX,XX +XXX,XX @@ static inline void prev_cputime_init(struct prev_cputime *prev) extern unsigned long long task_sched_runtime(struct task_struct *task); +#ifdef CONFIG_PARAVIRT +struct static_key; +extern struct static_key paravirt_steal_enabled; +extern struct static_key paravirt_steal_rq_enabled; + +#ifdef CONFIG_HAVE_PV_STEAL_CLOCK_GEN +u64 dummy_steal_clock(int cpu); + +DECLARE_STATIC_CALL(pv_steal_clock, dummy_steal_clock); + +static inline u64 paravirt_steal_clock(int cpu) +{ + return static_call(pv_steal_clock)(cpu); +} +#endif +#endif + #endif /* _LINUX_SCHED_CPUTIME_H */ diff --git a/kernel/sched/core.c b/kernel/sched/core.c index XXXXXXX..XXXXXXX 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -XXX,XX +XXX,XX @@ struct rq *task_rq_lock(struct task_struct *p, struct rq_flags *rf) * RQ-clock updating methods: */ +/* Use CONFIG_PARAVIRT as this will avoid more #ifdef in arch code. */ +#ifdef CONFIG_PARAVIRT +struct static_key paravirt_steal_rq_enabled; +#endif + static void update_rq_clock_task(struct rq *rq, s64 delta) { /* diff --git a/kernel/sched/cputime.c b/kernel/sched/cputime.c index XXXXXXX..XXXXXXX 100644 --- a/kernel/sched/cputime.c +++ b/kernel/sched/cputime.c @@ -XXX,XX +XXX,XX @@ void __account_forceidle_time(struct task_struct *p, u64 delta) * ticks are not redelivered later. Due to that, this function may on * occasion account more time than the calling functions think elapsed. */ +#ifdef CONFIG_PARAVIRT +struct static_key paravirt_steal_enabled; + +#ifdef CONFIG_HAVE_PV_STEAL_CLOCK_GEN +static u64 native_steal_clock(int cpu) +{ + return 0; +} + +DEFINE_STATIC_CALL(pv_steal_clock, native_steal_clock); +#endif +#endif + static __always_inline u64 steal_account_process_time(u64 maxtime) { #ifdef CONFIG_PARAVIRT diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index XXXXXXX..XXXXXXX 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -XXX,XX +XXX,XX @@ struct rt_rq; struct sched_group; struct cpuidle_state; -#ifdef CONFIG_PARAVIRT +#if defined(CONFIG_PARAVIRT) && !defined(CONFIG_HAVE_PV_STEAL_CLOCK_GEN) # include <asm/paravirt.h> #endif -- 2.51.0 Remove the arch specific variant of paravirt_steal_clock() and use the common one instead. This allows to remove paravirt.c and paravirt.h from arch/arm. Until all archs supporting Xen have been switched to the common code of paravirt_steal_clock(), drivers/xen/time.c needs to include asm/paravirt.h for those archs, while this is not necessary for arm any longer. Signed-off-by: Juergen Gross <jgross@suse.com> Acked-by: Peter Zijlstra (Intel) <peterz@infradead.org> --- arch/arm/Kconfig | 1 + arch/arm/include/asm/paravirt.h | 18 ------------------ arch/arm/kernel/Makefile | 1 - arch/arm/kernel/paravirt.c | 20 -------------------- drivers/xen/time.c | 2 ++ 5 files changed, 3 insertions(+), 39 deletions(-) delete mode 100644 arch/arm/include/asm/paravirt.h delete mode 100644 arch/arm/kernel/paravirt.c diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig index XXXXXXX..XXXXXXX 100644 --- a/arch/arm/Kconfig +++ b/arch/arm/Kconfig @@ -XXX,XX +XXX,XX @@ config UACCESS_WITH_MEMCPY config PARAVIRT bool "Enable paravirtualization code" + select HAVE_PV_STEAL_CLOCK_GEN help This changes the kernel so it can modify itself when it is run under a hypervisor, potentially improving performance significantly diff --git a/arch/arm/include/asm/paravirt.h b/arch/arm/include/asm/paravirt.h deleted file mode 100644 index XXXXXXX..XXXXXXX --- a/arch/arm/include/asm/paravirt.h +++ /dev/null @@ -XXX,XX +XXX,XX @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef _ASM_ARM_PARAVIRT_H -#define _ASM_ARM_PARAVIRT_H - -#ifdef CONFIG_PARAVIRT -#include <linux/static_call_types.h> - -u64 dummy_steal_clock(int cpu); - -DECLARE_STATIC_CALL(pv_steal_clock, dummy_steal_clock); - -static inline u64 paravirt_steal_clock(int cpu) -{ - return static_call(pv_steal_clock)(cpu); -} -#endif - -#endif diff --git a/arch/arm/kernel/Makefile b/arch/arm/kernel/Makefile index XXXXXXX..XXXXXXX 100644 --- a/arch/arm/kernel/Makefile +++ b/arch/arm/kernel/Makefile @@ -XXX,XX +XXX,XX @@ AFLAGS_iwmmxt.o := -Wa,-mcpu=iwmmxt obj-$(CONFIG_ARM_CPU_TOPOLOGY) += topology.o obj-$(CONFIG_VDSO) += vdso.o obj-$(CONFIG_EFI) += efi.o -obj-$(CONFIG_PARAVIRT) += paravirt.o obj-y += head$(MMUEXT).o obj-$(CONFIG_DEBUG_LL) += debug.o diff --git a/arch/arm/kernel/paravirt.c b/arch/arm/kernel/paravirt.c deleted file mode 100644 index XXXXXXX..XXXXXXX --- a/arch/arm/kernel/paravirt.c +++ /dev/null @@ -XXX,XX +XXX,XX @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * - * Copyright (C) 2013 Citrix Systems - * - * Author: Stefano Stabellini <stefano.stabellini@eu.citrix.com> - */ - -#include <linux/export.h> -#include <linux/jump_label.h> -#include <linux/types.h> -#include <linux/static_call.h> -#include <asm/paravirt.h> - -static u64 native_steal_clock(int cpu) -{ - return 0; -} - -DEFINE_STATIC_CALL(pv_steal_clock, native_steal_clock); diff --git a/drivers/xen/time.c b/drivers/xen/time.c index XXXXXXX..XXXXXXX 100644 --- a/drivers/xen/time.c +++ b/drivers/xen/time.c @@ -XXX,XX +XXX,XX @@ #include <linux/static_call.h> #include <linux/sched/cputime.h> +#ifndef CONFIG_HAVE_PV_STEAL_CLOCK_GEN #include <asm/paravirt.h> +#endif #include <asm/xen/hypervisor.h> #include <asm/xen/hypercall.h> -- 2.51.0 Remove the arch specific variant of paravirt_steal_clock() and use the common one instead. With all archs supporting Xen now having been switched to the common variant, including paravirt.h can be dropped from drivers/xen/time.c. Signed-off-by: Juergen Gross <jgross@suse.com> Acked-by: Peter Zijlstra (Intel) <peterz@infradead.org> --- arch/x86/Kconfig | 1 + arch/x86/include/asm/paravirt.h | 7 ------- arch/x86/kernel/paravirt.c | 6 ------ arch/x86/xen/time.c | 1 + drivers/xen/time.c | 3 --- 5 files changed, 2 insertions(+), 16 deletions(-) diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index XXXXXXX..XXXXXXX 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -XXX,XX +XXX,XX @@ if HYPERVISOR_GUEST config PARAVIRT bool "Enable paravirtualization code" depends on HAVE_STATIC_CALL + select HAVE_PV_STEAL_CLOCK_GEN help This changes the kernel so it can modify itself when it is run under a hypervisor, potentially improving performance significantly diff --git a/arch/x86/include/asm/paravirt.h b/arch/x86/include/asm/paravirt.h index XXXXXXX..XXXXXXX 100644 --- a/arch/x86/include/asm/paravirt.h +++ b/arch/x86/include/asm/paravirt.h @@ -XXX,XX +XXX,XX @@ #include <linux/static_call_types.h> #include <asm/frame.h> -u64 dummy_steal_clock(int cpu); u64 dummy_sched_clock(void); -DECLARE_STATIC_CALL(pv_steal_clock, dummy_steal_clock); DECLARE_STATIC_CALL(pv_sched_clock, dummy_sched_clock); void paravirt_set_sched_clock(u64 (*func)(void)); @@ -XXX,XX +XXX,XX @@ bool pv_is_native_spin_unlock(void); __visible bool __native_vcpu_is_preempted(long cpu); bool pv_is_native_vcpu_is_preempted(void); -static inline u64 paravirt_steal_clock(int cpu) -{ - return static_call(pv_steal_clock)(cpu); -} - #ifdef CONFIG_PARAVIRT_SPINLOCKS void __init paravirt_set_cap(void); #endif diff --git a/arch/x86/kernel/paravirt.c b/arch/x86/kernel/paravirt.c index XXXXXXX..XXXXXXX 100644 --- a/arch/x86/kernel/paravirt.c +++ b/arch/x86/kernel/paravirt.c @@ -XXX,XX +XXX,XX @@ void __init native_pv_lock_init(void) static_branch_enable(&virt_spin_lock_key); } -static u64 native_steal_clock(int cpu) -{ - return 0; -} - -DEFINE_STATIC_CALL(pv_steal_clock, native_steal_clock); DEFINE_STATIC_CALL(pv_sched_clock, native_sched_clock); void paravirt_set_sched_clock(u64 (*func)(void)) diff --git a/arch/x86/xen/time.c b/arch/x86/xen/time.c index XXXXXXX..XXXXXXX 100644 --- a/arch/x86/xen/time.c +++ b/arch/x86/xen/time.c @@ -XXX,XX +XXX,XX @@ #include <linux/slab.h> #include <linux/pvclock_gtod.h> #include <linux/timekeeper_internal.h> +#include <linux/sched/cputime.h> #include <asm/pvclock.h> #include <asm/xen/hypervisor.h> diff --git a/drivers/xen/time.c b/drivers/xen/time.c index XXXXXXX..XXXXXXX 100644 --- a/drivers/xen/time.c +++ b/drivers/xen/time.c @@ -XXX,XX +XXX,XX @@ #include <linux/static_call.h> #include <linux/sched/cputime.h> -#ifndef CONFIG_HAVE_PV_STEAL_CLOCK_GEN -#include <asm/paravirt.h> -#endif #include <asm/xen/hypervisor.h> #include <asm/xen/hypercall.h> -- 2.51.0 The only user of paravirt_sched_clock() is in tsc.c, so move the code from paravirt.c and paravirt.h to tsc.c. Signed-off-by: Juergen Gross <jgross@suse.com> Acked-by: Peter Zijlstra (Intel) <peterz@infradead.org> --- arch/x86/include/asm/paravirt.h | 12 ------------ arch/x86/include/asm/timer.h | 1 + arch/x86/kernel/kvmclock.c | 1 + arch/x86/kernel/paravirt.c | 7 ------- arch/x86/kernel/tsc.c | 10 +++++++++- arch/x86/xen/time.c | 1 + drivers/clocksource/hyperv_timer.c | 2 ++ 7 files changed, 14 insertions(+), 20 deletions(-) diff --git a/arch/x86/include/asm/paravirt.h b/arch/x86/include/asm/paravirt.h index XXXXXXX..XXXXXXX 100644 --- a/arch/x86/include/asm/paravirt.h +++ b/arch/x86/include/asm/paravirt.h @@ -XXX,XX +XXX,XX @@ #ifndef __ASSEMBLER__ #include <linux/types.h> #include <linux/cpumask.h> -#include <linux/static_call_types.h> #include <asm/frame.h> -u64 dummy_sched_clock(void); - -DECLARE_STATIC_CALL(pv_sched_clock, dummy_sched_clock); - -void paravirt_set_sched_clock(u64 (*func)(void)); - -static __always_inline u64 paravirt_sched_clock(void) -{ - return static_call(pv_sched_clock)(); -} - __visible void __native_queued_spin_unlock(struct qspinlock *lock); bool pv_is_native_spin_unlock(void); __visible bool __native_vcpu_is_preempted(long cpu); diff --git a/arch/x86/include/asm/timer.h b/arch/x86/include/asm/timer.h index XXXXXXX..XXXXXXX 100644 --- a/arch/x86/include/asm/timer.h +++ b/arch/x86/include/asm/timer.h @@ -XXX,XX +XXX,XX @@ extern void recalibrate_cpu_khz(void); extern int no_timer_check; extern bool using_native_sched_clock(void); +void paravirt_set_sched_clock(u64 (*func)(void)); /* * We use the full linear equation: f(x) = a + b*x, in order to allow diff --git a/arch/x86/kernel/kvmclock.c b/arch/x86/kernel/kvmclock.c index XXXXXXX..XXXXXXX 100644 --- a/arch/x86/kernel/kvmclock.c +++ b/arch/x86/kernel/kvmclock.c @@ -XXX,XX +XXX,XX @@ #include <linux/cc_platform.h> #include <asm/hypervisor.h> +#include <asm/timer.h> #include <asm/x86_init.h> #include <asm/kvmclock.h> diff --git a/arch/x86/kernel/paravirt.c b/arch/x86/kernel/paravirt.c index XXXXXXX..XXXXXXX 100644 --- a/arch/x86/kernel/paravirt.c +++ b/arch/x86/kernel/paravirt.c @@ -XXX,XX +XXX,XX @@ void __init native_pv_lock_init(void) static_branch_enable(&virt_spin_lock_key); } -DEFINE_STATIC_CALL(pv_sched_clock, native_sched_clock); - -void paravirt_set_sched_clock(u64 (*func)(void)) -{ - static_call_update(pv_sched_clock, func); -} - static noinstr void pv_native_safe_halt(void) { native_safe_halt(); diff --git a/arch/x86/kernel/tsc.c b/arch/x86/kernel/tsc.c index XXXXXXX..XXXXXXX 100644 --- a/arch/x86/kernel/tsc.c +++ b/arch/x86/kernel/tsc.c @@ -XXX,XX +XXX,XX @@ u64 native_sched_clock_from_tsc(u64 tsc) /* We need to define a real function for sched_clock, to override the weak default version */ #ifdef CONFIG_PARAVIRT +DEFINE_STATIC_CALL(pv_sched_clock, native_sched_clock); + noinstr u64 sched_clock_noinstr(void) { - return paravirt_sched_clock(); + return static_call(pv_sched_clock)(); } bool using_native_sched_clock(void) { return static_call_query(pv_sched_clock) == native_sched_clock; } + +void paravirt_set_sched_clock(u64 (*func)(void)) +{ + static_call_update(pv_sched_clock, func); +} #else u64 sched_clock_noinstr(void) __attribute__((alias("native_sched_clock"))); bool using_native_sched_clock(void) { return true; } +void paravirt_set_sched_clock(u64 (*func)(void)) { } #endif notrace u64 sched_clock(void) diff --git a/arch/x86/xen/time.c b/arch/x86/xen/time.c index XXXXXXX..XXXXXXX 100644 --- a/arch/x86/xen/time.c +++ b/arch/x86/xen/time.c @@ -XXX,XX +XXX,XX @@ #include <linux/sched/cputime.h> #include <asm/pvclock.h> +#include <asm/timer.h> #include <asm/xen/hypervisor.h> #include <asm/xen/hypercall.h> #include <asm/xen/cpuid.h> diff --git a/drivers/clocksource/hyperv_timer.c b/drivers/clocksource/hyperv_timer.c index XXXXXXX..XXXXXXX 100644 --- a/drivers/clocksource/hyperv_timer.c +++ b/drivers/clocksource/hyperv_timer.c @@ -XXX,XX +XXX,XX @@ static __always_inline void hv_setup_sched_clock(void *sched_clock) sched_clock_register(sched_clock, 64, NSEC_PER_SEC); } #elif defined CONFIG_PARAVIRT +#include <asm/timer.h> + static __always_inline void hv_setup_sched_clock(void *sched_clock) { /* We're on x86/x64 *and* using PV ops */ -- 2.51.0 Instead of having a pre-filled array xen_irq_ops for Xen PV paravirt functions, drop the array and assign each element individually. Signed-off-by: Juergen Gross <jgross@suse.com> --- V2: - new patch --- arch/x86/xen/irq.c | 20 +++++++------------- tools/objtool/check.c | 1 - 2 files changed, 7 insertions(+), 14 deletions(-) diff --git a/arch/x86/xen/irq.c b/arch/x86/xen/irq.c index XXXXXXX..XXXXXXX 100644 --- a/arch/x86/xen/irq.c +++ b/arch/x86/xen/irq.c @@ -XXX,XX +XXX,XX @@ static void xen_halt(void) xen_safe_halt(); } -static const typeof(pv_ops) xen_irq_ops __initconst = { - .irq = { - /* Initial interrupt flag handling only called while interrupts off. */ - .save_fl = __PV_IS_CALLEE_SAVE(paravirt_ret0), - .irq_disable = __PV_IS_CALLEE_SAVE(paravirt_nop), - .irq_enable = __PV_IS_CALLEE_SAVE(BUG_func), - - .safe_halt = xen_safe_halt, - .halt = xen_halt, - }, -}; - void __init xen_init_irq_ops(void) { - pv_ops.irq = xen_irq_ops.irq; + /* Initial interrupt flag handling only called while interrupts off. */ + pv_ops.irq.save_fl = __PV_IS_CALLEE_SAVE(paravirt_ret0); + pv_ops.irq.irq_disable = __PV_IS_CALLEE_SAVE(paravirt_nop); + pv_ops.irq.irq_enable = __PV_IS_CALLEE_SAVE(BUG_func); + pv_ops.irq.safe_halt = xen_safe_halt; + pv_ops.irq.halt = xen_halt; + x86_init.irqs.intr_init = xen_init_IRQ; } diff --git a/tools/objtool/check.c b/tools/objtool/check.c index XXXXXXX..XXXXXXX 100644 --- a/tools/objtool/check.c +++ b/tools/objtool/check.c @@ -XXX,XX +XXX,XX @@ static int init_pv_ops(struct objtool_file *file) static const char *pv_ops_tables[] = { "pv_ops", "xen_cpu_ops", - "xen_irq_ops", "xen_mmu_ops", NULL, }; -- 2.51.0 Instead of having a pre-filled array xen_cpu_ops for Xen PV paravirt functions, drop the array and assign each element individually. Signed-off-by: Juergen Gross <jgross@suse.com> --- V2: - new patch --- arch/x86/xen/enlighten_pv.c | 82 +++++++++++++++---------------------- tools/objtool/check.c | 1 - 2 files changed, 33 insertions(+), 50 deletions(-) diff --git a/arch/x86/xen/enlighten_pv.c b/arch/x86/xen/enlighten_pv.c index XXXXXXX..XXXXXXX 100644 --- a/arch/x86/xen/enlighten_pv.c +++ b/arch/x86/xen/enlighten_pv.c @@ -XXX,XX +XXX,XX @@ static const struct pv_info xen_info __initconst = { .name = "Xen", }; -static const typeof(pv_ops) xen_cpu_ops __initconst = { - .cpu = { - .cpuid = xen_cpuid, - - .set_debugreg = xen_set_debugreg, - .get_debugreg = xen_get_debugreg, - - .read_cr0 = xen_read_cr0, - .write_cr0 = xen_write_cr0, - - .write_cr4 = xen_write_cr4, - - .read_msr = xen_read_msr, - .write_msr = xen_write_msr, - - .read_msr_safe = xen_read_msr_safe, - .write_msr_safe = xen_write_msr_safe, - - .read_pmc = xen_read_pmc, - - .load_tr_desc = paravirt_nop, - .set_ldt = xen_set_ldt, - .load_gdt = xen_load_gdt, - .load_idt = xen_load_idt, - .load_tls = xen_load_tls, - .load_gs_index = xen_load_gs_index, - - .alloc_ldt = xen_alloc_ldt, - .free_ldt = xen_free_ldt, - - .store_tr = xen_store_tr, - - .write_ldt_entry = xen_write_ldt_entry, - .write_gdt_entry = xen_write_gdt_entry, - .write_idt_entry = xen_write_idt_entry, - .load_sp0 = xen_load_sp0, - -#ifdef CONFIG_X86_IOPL_IOPERM - .invalidate_io_bitmap = xen_invalidate_io_bitmap, - .update_io_bitmap = xen_update_io_bitmap, -#endif - .io_delay = xen_io_delay, - - .start_context_switch = xen_start_context_switch, - .end_context_switch = xen_end_context_switch, - }, -}; - static void xen_restart(char *msg) { xen_reboot(SHUTDOWN_reboot); @@ -XXX,XX +XXX,XX @@ asmlinkage __visible void __init xen_start_kernel(struct start_info *si) /* Install Xen paravirt ops */ pv_info = xen_info; - pv_ops.cpu = xen_cpu_ops.cpu; + + pv_ops.cpu.cpuid = xen_cpuid; + pv_ops.cpu.set_debugreg = xen_set_debugreg; + pv_ops.cpu.get_debugreg = xen_get_debugreg; + pv_ops.cpu.read_cr0 = xen_read_cr0; + pv_ops.cpu.write_cr0 = xen_write_cr0; + pv_ops.cpu.write_cr4 = xen_write_cr4; + pv_ops.cpu.read_msr = xen_read_msr; + pv_ops.cpu.write_msr = xen_write_msr; + pv_ops.cpu.read_msr_safe = xen_read_msr_safe; + pv_ops.cpu.write_msr_safe = xen_write_msr_safe; + pv_ops.cpu.read_pmc = xen_read_pmc; + pv_ops.cpu.load_tr_desc = paravirt_nop; + pv_ops.cpu.set_ldt = xen_set_ldt; + pv_ops.cpu.load_gdt = xen_load_gdt; + pv_ops.cpu.load_idt = xen_load_idt; + pv_ops.cpu.load_tls = xen_load_tls; + pv_ops.cpu.load_gs_index = xen_load_gs_index; + pv_ops.cpu.alloc_ldt = xen_alloc_ldt; + pv_ops.cpu.free_ldt = xen_free_ldt; + pv_ops.cpu.store_tr = xen_store_tr; + pv_ops.cpu.write_ldt_entry = xen_write_ldt_entry; + pv_ops.cpu.write_gdt_entry = xen_write_gdt_entry; + pv_ops.cpu.write_idt_entry = xen_write_idt_entry; + pv_ops.cpu.load_sp0 = xen_load_sp0; +#ifdef CONFIG_X86_IOPL_IOPERM + pv_ops.cpu.invalidate_io_bitmap = xen_invalidate_io_bitmap; + pv_ops.cpu.update_io_bitmap = xen_update_io_bitmap; +#endif + pv_ops.cpu.io_delay = xen_io_delay; + pv_ops.cpu.start_context_switch = xen_start_context_switch; + pv_ops.cpu.end_context_switch = xen_end_context_switch; + xen_init_irq_ops(); /* diff --git a/tools/objtool/check.c b/tools/objtool/check.c index XXXXXXX..XXXXXXX 100644 --- a/tools/objtool/check.c +++ b/tools/objtool/check.c @@ -XXX,XX +XXX,XX @@ static int init_pv_ops(struct objtool_file *file) { static const char *pv_ops_tables[] = { "pv_ops", - "xen_cpu_ops", "xen_mmu_ops", NULL, }; -- 2.51.0 Instead of having a pre-filled array xen_mmu_ops for Xen PV paravirt functions, drop the array and assign each element individually. Signed-off-by: Juergen Gross <jgross@suse.com> --- V2: - new patch --- arch/x86/xen/mmu_pv.c | 100 ++++++++++++++++-------------------------- tools/objtool/check.c | 1 - 2 files changed, 38 insertions(+), 63 deletions(-) diff --git a/arch/x86/xen/mmu_pv.c b/arch/x86/xen/mmu_pv.c index XXXXXXX..XXXXXXX 100644 --- a/arch/x86/xen/mmu_pv.c +++ b/arch/x86/xen/mmu_pv.c @@ -XXX,XX +XXX,XX @@ static void xen_leave_lazy_mmu(void) preempt_enable(); } -static const typeof(pv_ops) xen_mmu_ops __initconst = { - .mmu = { - .read_cr2 = __PV_IS_CALLEE_SAVE(xen_read_cr2), - .write_cr2 = xen_write_cr2, - - .read_cr3 = xen_read_cr3, - .write_cr3 = xen_write_cr3_init, - - .flush_tlb_user = xen_flush_tlb, - .flush_tlb_kernel = xen_flush_tlb, - .flush_tlb_one_user = xen_flush_tlb_one_user, - .flush_tlb_multi = xen_flush_tlb_multi, - - .pgd_alloc = xen_pgd_alloc, - .pgd_free = xen_pgd_free, - - .alloc_pte = xen_alloc_pte_init, - .release_pte = xen_release_pte_init, - .alloc_pmd = xen_alloc_pmd_init, - .release_pmd = xen_release_pmd_init, - - .set_pte = xen_set_pte_init, - .set_pmd = xen_set_pmd_hyper, - - .ptep_modify_prot_start = xen_ptep_modify_prot_start, - .ptep_modify_prot_commit = xen_ptep_modify_prot_commit, - - .pte_val = PV_CALLEE_SAVE(xen_pte_val), - .pgd_val = PV_CALLEE_SAVE(xen_pgd_val), - - .make_pte = PV_CALLEE_SAVE(xen_make_pte_init), - .make_pgd = PV_CALLEE_SAVE(xen_make_pgd), - - .set_pud = xen_set_pud_hyper, - - .make_pmd = PV_CALLEE_SAVE(xen_make_pmd), - .pmd_val = PV_CALLEE_SAVE(xen_pmd_val), - - .pud_val = PV_CALLEE_SAVE(xen_pud_val), - .make_pud = PV_CALLEE_SAVE(xen_make_pud), - .set_p4d = xen_set_p4d_hyper, - - .alloc_pud = xen_alloc_pmd_init, - .release_pud = xen_release_pmd_init, - - .p4d_val = PV_CALLEE_SAVE(xen_p4d_val), - .make_p4d = PV_CALLEE_SAVE(xen_make_p4d), - - .enter_mmap = xen_enter_mmap, - .exit_mmap = xen_exit_mmap, - - .lazy_mode = { - .enter = xen_enter_lazy_mmu, - .leave = xen_leave_lazy_mmu, - .flush = xen_flush_lazy_mmu, - }, - - .set_fixmap = xen_set_fixmap, - }, -}; - void __init xen_init_mmu_ops(void) { x86_init.paging.pagetable_init = xen_pagetable_init; x86_init.hyper.init_after_bootmem = xen_after_bootmem; - pv_ops.mmu = xen_mmu_ops.mmu; + pv_ops.mmu.read_cr2 = __PV_IS_CALLEE_SAVE(xen_read_cr2); + pv_ops.mmu.write_cr2 = xen_write_cr2; + pv_ops.mmu.read_cr3 = xen_read_cr3; + pv_ops.mmu.write_cr3 = xen_write_cr3_init; + pv_ops.mmu.flush_tlb_user = xen_flush_tlb; + pv_ops.mmu.flush_tlb_kernel = xen_flush_tlb; + pv_ops.mmu.flush_tlb_one_user = xen_flush_tlb_one_user; + pv_ops.mmu.flush_tlb_multi = xen_flush_tlb_multi; + pv_ops.mmu.pgd_alloc = xen_pgd_alloc; + pv_ops.mmu.pgd_free = xen_pgd_free; + pv_ops.mmu.alloc_pte = xen_alloc_pte_init; + pv_ops.mmu.release_pte = xen_release_pte_init; + pv_ops.mmu.alloc_pmd = xen_alloc_pmd_init; + pv_ops.mmu.release_pmd = xen_release_pmd_init; + pv_ops.mmu.set_pte = xen_set_pte_init; + pv_ops.mmu.set_pmd = xen_set_pmd_hyper; + pv_ops.mmu.ptep_modify_prot_start = xen_ptep_modify_prot_start; + pv_ops.mmu.ptep_modify_prot_commit = xen_ptep_modify_prot_commit; + pv_ops.mmu.pte_val = PV_CALLEE_SAVE(xen_pte_val); + pv_ops.mmu.pgd_val = PV_CALLEE_SAVE(xen_pgd_val); + pv_ops.mmu.make_pte = PV_CALLEE_SAVE(xen_make_pte_init); + pv_ops.mmu.make_pgd = PV_CALLEE_SAVE(xen_make_pgd); + pv_ops.mmu.set_pud = xen_set_pud_hyper; + pv_ops.mmu.make_pmd = PV_CALLEE_SAVE(xen_make_pmd); + pv_ops.mmu.pmd_val = PV_CALLEE_SAVE(xen_pmd_val); + pv_ops.mmu.pud_val = PV_CALLEE_SAVE(xen_pud_val); + pv_ops.mmu.make_pud = PV_CALLEE_SAVE(xen_make_pud); + pv_ops.mmu.set_p4d = xen_set_p4d_hyper; + pv_ops.mmu.alloc_pud = xen_alloc_pmd_init; + pv_ops.mmu.release_pud = xen_release_pmd_init; + pv_ops.mmu.p4d_val = PV_CALLEE_SAVE(xen_p4d_val); + pv_ops.mmu.make_p4d = PV_CALLEE_SAVE(xen_make_p4d); + pv_ops.mmu.enter_mmap = xen_enter_mmap; + pv_ops.mmu.exit_mmap = xen_exit_mmap; + pv_ops.mmu.lazy_mode.enter = xen_enter_lazy_mmu; + pv_ops.mmu.lazy_mode.leave = xen_leave_lazy_mmu; + pv_ops.mmu.lazy_mode.flush = xen_flush_lazy_mmu; + pv_ops.mmu.set_fixmap = xen_set_fixmap; memset(dummy_mapping, 0xff, PAGE_SIZE); } diff --git a/tools/objtool/check.c b/tools/objtool/check.c index XXXXXXX..XXXXXXX 100644 --- a/tools/objtool/check.c +++ b/tools/objtool/check.c @@ -XXX,XX +XXX,XX @@ static int init_pv_ops(struct objtool_file *file) { static const char *pv_ops_tables[] = { "pv_ops", - "xen_mmu_ops", NULL, }; const char *pv_ops; -- 2.51.0 Instead of having the pv spinlock function definitions in paravirt.h, move them into the new header paravirt-spinlock.h. Signed-off-by: Juergen Gross <jgross@suse.com> --- V2: - use new header instead of qspinlock.h - use dedicated pv_ops_lock array - move more paravirt related lock code V3: - hide native_pv_lock_init() with CONFIG_SMP (kernel test robot) --- arch/x86/hyperv/hv_spinlock.c | 10 +- arch/x86/include/asm/paravirt-spinlock.h | 146 +++++++++++++++++++++++ arch/x86/include/asm/paravirt.h | 61 ---------- arch/x86/include/asm/paravirt_types.h | 17 --- arch/x86/include/asm/qspinlock.h | 89 ++------------ arch/x86/kernel/Makefile | 2 +- arch/x86/kernel/kvm.c | 10 +- arch/x86/kernel/paravirt-spinlocks.c | 26 +++- arch/x86/kernel/paravirt.c | 21 ---- arch/x86/xen/spinlock.c | 10 +- tools/objtool/check.c | 1 + 11 files changed, 194 insertions(+), 199 deletions(-) create mode 100644 arch/x86/include/asm/paravirt-spinlock.h diff --git a/arch/x86/hyperv/hv_spinlock.c b/arch/x86/hyperv/hv_spinlock.c index XXXXXXX..XXXXXXX 100644 --- a/arch/x86/hyperv/hv_spinlock.c +++ b/arch/x86/hyperv/hv_spinlock.c @@ -XXX,XX +XXX,XX @@ void __init hv_init_spinlocks(void) pr_info("PV spinlocks enabled\n"); __pv_init_lock_hash(); - pv_ops.lock.queued_spin_lock_slowpath = __pv_queued_spin_lock_slowpath; - pv_ops.lock.queued_spin_unlock = PV_CALLEE_SAVE(__pv_queued_spin_unlock); - pv_ops.lock.wait = hv_qlock_wait; - pv_ops.lock.kick = hv_qlock_kick; - pv_ops.lock.vcpu_is_preempted = PV_CALLEE_SAVE(hv_vcpu_is_preempted); + pv_ops_lock.queued_spin_lock_slowpath = __pv_queued_spin_lock_slowpath; + pv_ops_lock.queued_spin_unlock = PV_CALLEE_SAVE(__pv_queued_spin_unlock); + pv_ops_lock.wait = hv_qlock_wait; + pv_ops_lock.kick = hv_qlock_kick; + pv_ops_lock.vcpu_is_preempted = PV_CALLEE_SAVE(hv_vcpu_is_preempted); } static __init int hv_parse_nopvspin(char *arg) diff --git a/arch/x86/include/asm/paravirt-spinlock.h b/arch/x86/include/asm/paravirt-spinlock.h new file mode 100644 index XXXXXXX..XXXXXXX --- /dev/null +++ b/arch/x86/include/asm/paravirt-spinlock.h @@ -XXX,XX +XXX,XX @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef _ASM_X86_PARAVIRT_SPINLOCK_H +#define _ASM_X86_PARAVIRT_SPINLOCK_H + +#include <asm/paravirt_types.h> + +#ifdef CONFIG_SMP +#include <asm/spinlock_types.h> +#endif + +struct qspinlock; + +struct pv_lock_ops { + void (*queued_spin_lock_slowpath)(struct qspinlock *lock, u32 val); + struct paravirt_callee_save queued_spin_unlock; + + void (*wait)(u8 *ptr, u8 val); + void (*kick)(int cpu); + + struct paravirt_callee_save vcpu_is_preempted; +} __no_randomize_layout; + +extern struct pv_lock_ops pv_ops_lock; + +#ifdef CONFIG_PARAVIRT_SPINLOCKS +void __init paravirt_set_cap(void); +extern void native_queued_spin_lock_slowpath(struct qspinlock *lock, u32 val); +extern void __pv_init_lock_hash(void); +extern void __pv_queued_spin_lock_slowpath(struct qspinlock *lock, u32 val); +extern void __raw_callee_save___pv_queued_spin_unlock(struct qspinlock *lock); +extern bool nopvspin; + +static __always_inline void pv_queued_spin_lock_slowpath(struct qspinlock *lock, + u32 val) +{ + PVOP_VCALL2(pv_ops_lock, queued_spin_lock_slowpath, lock, val); +} + +static __always_inline void pv_queued_spin_unlock(struct qspinlock *lock) +{ + PVOP_ALT_VCALLEE1(pv_ops_lock, queued_spin_unlock, lock, + "movb $0, (%%" _ASM_ARG1 ");", + ALT_NOT(X86_FEATURE_PVUNLOCK)); +} + +static __always_inline bool pv_vcpu_is_preempted(long cpu) +{ + return PVOP_ALT_CALLEE1(bool, pv_ops_lock, vcpu_is_preempted, cpu, + "xor %%" _ASM_AX ", %%" _ASM_AX ";", + ALT_NOT(X86_FEATURE_VCPUPREEMPT)); +} + +#define queued_spin_unlock queued_spin_unlock +/** + * queued_spin_unlock - release a queued spinlock + * @lock : Pointer to queued spinlock structure + * + * A smp_store_release() on the least-significant byte. + */ +static inline void native_queued_spin_unlock(struct qspinlock *lock) +{ + smp_store_release(&lock->locked, 0); +} + +static inline void queued_spin_lock_slowpath(struct qspinlock *lock, u32 val) +{ + pv_queued_spin_lock_slowpath(lock, val); +} + +static inline void queued_spin_unlock(struct qspinlock *lock) +{ + kcsan_release(); + pv_queued_spin_unlock(lock); +} + +#define vcpu_is_preempted vcpu_is_preempted +static inline bool vcpu_is_preempted(long cpu) +{ + return pv_vcpu_is_preempted(cpu); +} + +static __always_inline void pv_wait(u8 *ptr, u8 val) +{ + PVOP_VCALL2(pv_ops_lock, wait, ptr, val); +} + +static __always_inline void pv_kick(int cpu) +{ + PVOP_VCALL1(pv_ops_lock, kick, cpu); +} + +void __raw_callee_save___native_queued_spin_unlock(struct qspinlock *lock); +bool __raw_callee_save___native_vcpu_is_preempted(long cpu); +#endif /* CONFIG_PARAVIRT_SPINLOCKS */ + +void __init native_pv_lock_init(void); +__visible void __native_queued_spin_unlock(struct qspinlock *lock); +bool pv_is_native_spin_unlock(void); +__visible bool __native_vcpu_is_preempted(long cpu); +bool pv_is_native_vcpu_is_preempted(void); + +/* + * virt_spin_lock_key - disables by default the virt_spin_lock() hijack. + * + * Native (and PV wanting native due to vCPU pinning) should keep this key + * disabled. Native does not touch the key. + * + * When in a guest then native_pv_lock_init() enables the key first and + * KVM/XEN might conditionally disable it later in the boot process again. + */ +DECLARE_STATIC_KEY_FALSE(virt_spin_lock_key); + +/* + * Shortcut for the queued_spin_lock_slowpath() function that allows + * virt to hijack it. + * + * Returns: + * true - lock has been negotiated, all done; + * false - queued_spin_lock_slowpath() will do its thing. + */ +#define virt_spin_lock virt_spin_lock +static inline bool virt_spin_lock(struct qspinlock *lock) +{ + int val; + + if (!static_branch_likely(&virt_spin_lock_key)) + return false; + + /* + * On hypervisors without PARAVIRT_SPINLOCKS support we fall + * back to a Test-and-Set spinlock, because fair locks have + * horrible lock 'holder' preemption issues. + */ + + __retry: + val = atomic_read(&lock->val); + + if (val || !atomic_try_cmpxchg(&lock->val, &val, _Q_LOCKED_VAL)) { + cpu_relax(); + goto __retry; + } + + return true; +} + +#endif /* _ASM_X86_PARAVIRT_SPINLOCK_H */ diff --git a/arch/x86/include/asm/paravirt.h b/arch/x86/include/asm/paravirt.h index XXXXXXX..XXXXXXX 100644 --- a/arch/x86/include/asm/paravirt.h +++ b/arch/x86/include/asm/paravirt.h @@ -XXX,XX +XXX,XX @@ #include <linux/cpumask.h> #include <asm/frame.h> -__visible void __native_queued_spin_unlock(struct qspinlock *lock); -bool pv_is_native_spin_unlock(void); -__visible bool __native_vcpu_is_preempted(long cpu); -bool pv_is_native_vcpu_is_preempted(void); - -#ifdef CONFIG_PARAVIRT_SPINLOCKS -void __init paravirt_set_cap(void); -#endif - /* The paravirtualized I/O functions */ static inline void slow_down_io(void) { @@ -XXX,XX +XXX,XX @@ static inline void __set_fixmap(unsigned /* enum fixed_addresses */ idx, { pv_ops.mmu.set_fixmap(idx, phys, flags); } -#endif - -#if defined(CONFIG_SMP) && defined(CONFIG_PARAVIRT_SPINLOCKS) - -static __always_inline void pv_queued_spin_lock_slowpath(struct qspinlock *lock, - u32 val) -{ - PVOP_VCALL2(pv_ops, lock.queued_spin_lock_slowpath, lock, val); -} - -static __always_inline void pv_queued_spin_unlock(struct qspinlock *lock) -{ - PVOP_ALT_VCALLEE1(pv_ops, lock.queued_spin_unlock, lock, - "movb $0, (%%" _ASM_ARG1 ");", - ALT_NOT(X86_FEATURE_PVUNLOCK)); -} - -static __always_inline void pv_wait(u8 *ptr, u8 val) -{ - PVOP_VCALL2(pv_ops, lock.wait, ptr, val); -} - -static __always_inline void pv_kick(int cpu) -{ - PVOP_VCALL1(pv_ops, lock.kick, cpu); -} - -static __always_inline bool pv_vcpu_is_preempted(long cpu) -{ - return PVOP_ALT_CALLEE1(bool, pv_ops, lock.vcpu_is_preempted, cpu, - "xor %%" _ASM_AX ", %%" _ASM_AX ";", - ALT_NOT(X86_FEATURE_VCPUPREEMPT)); -} -void __raw_callee_save___native_queued_spin_unlock(struct qspinlock *lock); -bool __raw_callee_save___native_vcpu_is_preempted(long cpu); - -#endif /* SMP && PARAVIRT_SPINLOCKS */ - -#ifdef CONFIG_PARAVIRT_XXL static __always_inline unsigned long arch_local_save_flags(void) { return PVOP_ALT_CALLEE0(unsigned long, pv_ops, irq.save_fl, "pushf; pop %%rax;", @@ -XXX,XX +XXX,XX @@ static __always_inline unsigned long arch_local_irq_save(void) } #endif -void native_pv_lock_init(void) __init; - #else /* __ASSEMBLER__ */ #ifdef CONFIG_X86_64 @@ -XXX,XX +XXX,XX @@ void native_pv_lock_init(void) __init; #endif /* __ASSEMBLER__ */ #else /* CONFIG_PARAVIRT */ # define default_banner x86_init_noop - -#ifndef __ASSEMBLER__ -static inline void native_pv_lock_init(void) -{ -} -#endif #endif /* !CONFIG_PARAVIRT */ #ifndef __ASSEMBLER__ @@ -XXX,XX +XXX,XX @@ static inline void paravirt_arch_exit_mmap(struct mm_struct *mm) } #endif -#ifndef CONFIG_PARAVIRT_SPINLOCKS -static inline void paravirt_set_cap(void) -{ -} -#endif #endif /* __ASSEMBLER__ */ #endif /* _ASM_X86_PARAVIRT_H */ diff --git a/arch/x86/include/asm/paravirt_types.h b/arch/x86/include/asm/paravirt_types.h index XXXXXXX..XXXXXXX 100644 --- a/arch/x86/include/asm/paravirt_types.h +++ b/arch/x86/include/asm/paravirt_types.h @@ -XXX,XX +XXX,XX @@ struct pv_mmu_ops { #endif } __no_randomize_layout; -#ifdef CONFIG_SMP -#include <asm/spinlock_types.h> -#endif - -struct qspinlock; - -struct pv_lock_ops { - void (*queued_spin_lock_slowpath)(struct qspinlock *lock, u32 val); - struct paravirt_callee_save queued_spin_unlock; - - void (*wait)(u8 *ptr, u8 val); - void (*kick)(int cpu); - - struct paravirt_callee_save vcpu_is_preempted; -} __no_randomize_layout; - /* This contains all the paravirt structures: we get a convenient * number for each function using the offset which we use to indicate * what to patch. */ @@ -XXX,XX +XXX,XX @@ struct paravirt_patch_template { struct pv_cpu_ops cpu; struct pv_irq_ops irq; struct pv_mmu_ops mmu; - struct pv_lock_ops lock; } __no_randomize_layout; extern struct paravirt_patch_template pv_ops; diff --git a/arch/x86/include/asm/qspinlock.h b/arch/x86/include/asm/qspinlock.h index XXXXXXX..XXXXXXX 100644 --- a/arch/x86/include/asm/qspinlock.h +++ b/arch/x86/include/asm/qspinlock.h @@ -XXX,XX +XXX,XX @@ #include <asm-generic/qspinlock_types.h> #include <asm/paravirt.h> #include <asm/rmwcc.h> +#ifdef CONFIG_PARAVIRT +#include <asm/paravirt-spinlock.h> +#endif #define _Q_PENDING_LOOPS (1 << 9) @@ -XXX,XX +XXX,XX @@ static __always_inline u32 queued_fetch_set_pending_acquire(struct qspinlock *lo return val; } -#ifdef CONFIG_PARAVIRT_SPINLOCKS -extern void native_queued_spin_lock_slowpath(struct qspinlock *lock, u32 val); -extern void __pv_init_lock_hash(void); -extern void __pv_queued_spin_lock_slowpath(struct qspinlock *lock, u32 val); -extern void __raw_callee_save___pv_queued_spin_unlock(struct qspinlock *lock); -extern bool nopvspin; - -#define queued_spin_unlock queued_spin_unlock -/** - * queued_spin_unlock - release a queued spinlock - * @lock : Pointer to queued spinlock structure - * - * A smp_store_release() on the least-significant byte. - */ -static inline void native_queued_spin_unlock(struct qspinlock *lock) -{ - smp_store_release(&lock->locked, 0); -} - -static inline void queued_spin_lock_slowpath(struct qspinlock *lock, u32 val) -{ - pv_queued_spin_lock_slowpath(lock, val); -} - -static inline void queued_spin_unlock(struct qspinlock *lock) -{ - kcsan_release(); - pv_queued_spin_unlock(lock); -} - -#define vcpu_is_preempted vcpu_is_preempted -static inline bool vcpu_is_preempted(long cpu) -{ - return pv_vcpu_is_preempted(cpu); -} +#ifndef CONFIG_PARAVIRT_SPINLOCKS +static inline void paravirt_set_cap(void) { } #endif -#ifdef CONFIG_PARAVIRT -/* - * virt_spin_lock_key - disables by default the virt_spin_lock() hijack. - * - * Native (and PV wanting native due to vCPU pinning) should keep this key - * disabled. Native does not touch the key. - * - * When in a guest then native_pv_lock_init() enables the key first and - * KVM/XEN might conditionally disable it later in the boot process again. - */ -DECLARE_STATIC_KEY_FALSE(virt_spin_lock_key); - -/* - * Shortcut for the queued_spin_lock_slowpath() function that allows - * virt to hijack it. - * - * Returns: - * true - lock has been negotiated, all done; - * false - queued_spin_lock_slowpath() will do its thing. - */ -#define virt_spin_lock virt_spin_lock -static inline bool virt_spin_lock(struct qspinlock *lock) -{ - int val; - - if (!static_branch_likely(&virt_spin_lock_key)) - return false; - - /* - * On hypervisors without PARAVIRT_SPINLOCKS support we fall - * back to a Test-and-Set spinlock, because fair locks have - * horrible lock 'holder' preemption issues. - */ - - __retry: - val = atomic_read(&lock->val); - - if (val || !atomic_try_cmpxchg(&lock->val, &val, _Q_LOCKED_VAL)) { - cpu_relax(); - goto __retry; - } - - return true; -} - -#endif /* CONFIG_PARAVIRT */ +#ifndef CONFIG_PARAVIRT +static inline void native_pv_lock_init(void) { } +#endif #include <asm-generic/qspinlock.h> diff --git a/arch/x86/kernel/Makefile b/arch/x86/kernel/Makefile index XXXXXXX..XXXXXXX 100644 --- a/arch/x86/kernel/Makefile +++ b/arch/x86/kernel/Makefile @@ -XXX,XX +XXX,XX @@ obj-$(CONFIG_DEBUG_NMI_SELFTEST) += nmi_selftest.o obj-$(CONFIG_KVM_GUEST) += kvm.o kvmclock.o obj-$(CONFIG_PARAVIRT) += paravirt.o -obj-$(CONFIG_PARAVIRT_SPINLOCKS)+= paravirt-spinlocks.o +obj-$(CONFIG_PARAVIRT) += paravirt-spinlocks.o obj-$(CONFIG_PARAVIRT_CLOCK) += pvclock.o obj-$(CONFIG_X86_PMEM_LEGACY_DEVICE) += pmem.o diff --git a/arch/x86/kernel/kvm.c b/arch/x86/kernel/kvm.c index XXXXXXX..XXXXXXX 100644 --- a/arch/x86/kernel/kvm.c +++ b/arch/x86/kernel/kvm.c @@ -XXX,XX +XXX,XX @@ static void __init kvm_guest_init(void) has_steal_clock = 1; static_call_update(pv_steal_clock, kvm_steal_clock); - pv_ops.lock.vcpu_is_preempted = + pv_ops_lock.vcpu_is_preempted = PV_CALLEE_SAVE(__kvm_vcpu_is_preempted); } @@ -XXX,XX +XXX,XX @@ void __init kvm_spinlock_init(void) pr_info("PV spinlocks enabled\n"); __pv_init_lock_hash(); - pv_ops.lock.queued_spin_lock_slowpath = __pv_queued_spin_lock_slowpath; - pv_ops.lock.queued_spin_unlock = + pv_ops_lock.queued_spin_lock_slowpath = __pv_queued_spin_lock_slowpath; + pv_ops_lock.queued_spin_unlock = PV_CALLEE_SAVE(__pv_queued_spin_unlock); - pv_ops.lock.wait = kvm_wait; - pv_ops.lock.kick = kvm_kick_cpu; + pv_ops_lock.wait = kvm_wait; + pv_ops_lock.kick = kvm_kick_cpu; /* * When PV spinlock is enabled which is preferred over diff --git a/arch/x86/kernel/paravirt-spinlocks.c b/arch/x86/kernel/paravirt-spinlocks.c index XXXXXXX..XXXXXXX 100644 --- a/arch/x86/kernel/paravirt-spinlocks.c +++ b/arch/x86/kernel/paravirt-spinlocks.c @@ -XXX,XX +XXX,XX @@ * Split spinlock implementation out into its own file, so it can be * compiled in a FTRACE-compatible way. */ +#include <linux/static_call.h> #include <linux/spinlock.h> #include <linux/export.h> #include <linux/jump_label.h> -#include <asm/paravirt.h> +DEFINE_STATIC_KEY_FALSE(virt_spin_lock_key); +#ifdef CONFIG_SMP +void __init native_pv_lock_init(void) +{ + if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) + static_branch_enable(&virt_spin_lock_key); +} +#endif + +#ifdef CONFIG_PARAVIRT_SPINLOCKS __visible void __native_queued_spin_unlock(struct qspinlock *lock) { native_queued_spin_unlock(lock); @@ -XXX,XX +XXX,XX @@ PV_CALLEE_SAVE_REGS_THUNK(__native_queued_spin_unlock); bool pv_is_native_spin_unlock(void) { - return pv_ops.lock.queued_spin_unlock.func == + return pv_ops_lock.queued_spin_unlock.func == __raw_callee_save___native_queued_spin_unlock; } @@ -XXX,XX +XXX,XX @@ PV_CALLEE_SAVE_REGS_THUNK(__native_vcpu_is_preempted); bool pv_is_native_vcpu_is_preempted(void) { - return pv_ops.lock.vcpu_is_preempted.func == + return pv_ops_lock.vcpu_is_preempted.func == __raw_callee_save___native_vcpu_is_preempted; } @@ -XXX,XX +XXX,XX @@ void __init paravirt_set_cap(void) if (!pv_is_native_vcpu_is_preempted()) setup_force_cpu_cap(X86_FEATURE_VCPUPREEMPT); } + +struct pv_lock_ops pv_ops_lock = { + .queued_spin_lock_slowpath = native_queued_spin_lock_slowpath, + .queued_spin_unlock = PV_CALLEE_SAVE(__native_queued_spin_unlock), + .wait = paravirt_nop, + .kick = paravirt_nop, + .vcpu_is_preempted = PV_CALLEE_SAVE(__native_vcpu_is_preempted), +}; +EXPORT_SYMBOL(pv_ops_lock); +#endif diff --git a/arch/x86/kernel/paravirt.c b/arch/x86/kernel/paravirt.c index XXXXXXX..XXXXXXX 100644 --- a/arch/x86/kernel/paravirt.c +++ b/arch/x86/kernel/paravirt.c @@ -XXX,XX +XXX,XX @@ DEFINE_ASM_FUNC(pv_native_irq_enable, "sti", .noinstr.text); DEFINE_ASM_FUNC(pv_native_read_cr2, "mov %cr2, %rax", .noinstr.text); #endif -DEFINE_STATIC_KEY_FALSE(virt_spin_lock_key); - -void __init native_pv_lock_init(void) -{ - if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) - static_branch_enable(&virt_spin_lock_key); -} - static noinstr void pv_native_safe_halt(void) { native_safe_halt(); @@ -XXX,XX +XXX,XX @@ struct paravirt_patch_template pv_ops = { .mmu.set_fixmap = native_set_fixmap, #endif /* CONFIG_PARAVIRT_XXL */ - -#if defined(CONFIG_PARAVIRT_SPINLOCKS) - /* Lock ops. */ -#ifdef CONFIG_SMP - .lock.queued_spin_lock_slowpath = native_queued_spin_lock_slowpath, - .lock.queued_spin_unlock = - PV_CALLEE_SAVE(__native_queued_spin_unlock), - .lock.wait = paravirt_nop, - .lock.kick = paravirt_nop, - .lock.vcpu_is_preempted = - PV_CALLEE_SAVE(__native_vcpu_is_preempted), -#endif /* SMP */ -#endif }; #ifdef CONFIG_PARAVIRT_XXL diff --git a/arch/x86/xen/spinlock.c b/arch/x86/xen/spinlock.c index XXXXXXX..XXXXXXX 100644 --- a/arch/x86/xen/spinlock.c +++ b/arch/x86/xen/spinlock.c @@ -XXX,XX +XXX,XX @@ void __init xen_init_spinlocks(void) printk(KERN_DEBUG "xen: PV spinlocks enabled\n"); __pv_init_lock_hash(); - pv_ops.lock.queued_spin_lock_slowpath = __pv_queued_spin_lock_slowpath; - pv_ops.lock.queued_spin_unlock = + pv_ops_lock.queued_spin_lock_slowpath = __pv_queued_spin_lock_slowpath; + pv_ops_lock.queued_spin_unlock = PV_CALLEE_SAVE(__pv_queued_spin_unlock); - pv_ops.lock.wait = xen_qlock_wait; - pv_ops.lock.kick = xen_qlock_kick; - pv_ops.lock.vcpu_is_preempted = PV_CALLEE_SAVE(xen_vcpu_stolen); + pv_ops_lock.wait = xen_qlock_wait; + pv_ops_lock.kick = xen_qlock_kick; + pv_ops_lock.vcpu_is_preempted = PV_CALLEE_SAVE(xen_vcpu_stolen); } diff --git a/tools/objtool/check.c b/tools/objtool/check.c index XXXXXXX..XXXXXXX 100644 --- a/tools/objtool/check.c +++ b/tools/objtool/check.c @@ -XXX,XX +XXX,XX @@ static struct { int idx_off; } pv_ops_tables[] = { { .name = "pv_ops", }, + { .name = "pv_ops_lock", }, { .name = NULL, .idx_off = -1 } }; -- 2.51.0 Some cleanups and reorg of paravirt code and headers: - The first 2 patches should be not controversial at all, as they remove just some no longer needed #include and struct forward declarations. - The 3rd patch is removing CONFIG_PARAVIRT_DEBUG, which IMO has no real value, as it just changes a crash to a BUG() (the stack trace will basically be the same). As the maintainer of the main paravirt user (Xen) I have never seen this crash/BUG() to happen. - The 4th patch is just a movement of code. - I don't know for what reason asm/paravirt_api_clock.h was added, as all archs supporting it do it exactly in the same way. Patch 5 is removing it. - Patches 6-14 are streamlining the paravirt clock interfaces by using a common implementation across architectures where possible and by moving the related code into common sched code, as this is where it should live. - Patches 15-20 are more like RFC material preparing the paravirt infrastructure to support multiple pv_ops function arrays. As a prerequisite for that it makes life in objtool much easier with dropping the Xen static initializers of the pv_ops sub- structures, which is done in patches 15-17. Patches 18-20 are doing the real preparations for multiple pv_ops arrays and using those arrays in multiple headers. - Patch 21 is an example how the new scheme can look like using the PV-spinlocks. Changes in V2: - new patches 13-18 and 20 - complete rework of patch 21 Changes in V3: - fixed 2 issues detected by kernel test robot Changes in V4: - fixed one build issue Changes in V5: - fixed another build issue - rebase Juergen Gross (21): x86/paravirt: Remove not needed includes of paravirt.h x86/paravirt: Remove some unneeded struct declarations x86/paravirt: Remove PARAVIRT_DEBUG config option x86/paravirt: Move thunk macros to paravirt_types.h paravirt: Remove asm/paravirt_api_clock.h sched: Move clock related paravirt code to kernel/sched arm/paravirt: Use common code for paravirt_steal_clock() arm64/paravirt: Use common code for paravirt_steal_clock() loongarch/paravirt: Use common code for paravirt_steal_clock() riscv/paravirt: Use common code for paravirt_steal_clock() x86/paravirt: Use common code for paravirt_steal_clock() x86/paravirt: Move paravirt_sched_clock() related code into tsc.c x86/paravirt: Introduce new paravirt-base.h header x86/paravirt: Move pv_native_*() prototypes to paravirt.c x86/xen: Drop xen_irq_ops x86/xen: Drop xen_cpu_ops x86/xen: Drop xen_mmu_ops objtool: Allow multiple pv_ops arrays x86/paravirt: Allow pv-calls outside paravirt.h x86/paravirt: Specify pv_ops array in paravirt macros x86/pvlocks: Move paravirt spinlock functions into own header arch/Kconfig | 3 + arch/arm/Kconfig | 1 + arch/arm/include/asm/paravirt.h | 22 -- arch/arm/include/asm/paravirt_api_clock.h | 1 - arch/arm/kernel/Makefile | 1 - arch/arm/kernel/paravirt.c | 23 -- arch/arm64/Kconfig | 1 + arch/arm64/include/asm/paravirt.h | 14 - arch/arm64/include/asm/paravirt_api_clock.h | 1 - arch/arm64/kernel/paravirt.c | 11 +- arch/loongarch/Kconfig | 1 + arch/loongarch/include/asm/paravirt.h | 13 - .../include/asm/paravirt_api_clock.h | 1 - arch/loongarch/kernel/paravirt.c | 10 +- arch/powerpc/include/asm/paravirt.h | 3 - arch/powerpc/include/asm/paravirt_api_clock.h | 2 - arch/powerpc/platforms/pseries/setup.c | 4 +- arch/riscv/Kconfig | 1 + arch/riscv/include/asm/paravirt.h | 14 - arch/riscv/include/asm/paravirt_api_clock.h | 1 - arch/riscv/kernel/paravirt.c | 11 +- arch/x86/Kconfig | 8 +- arch/x86/entry/entry_64.S | 1 - arch/x86/entry/vsyscall/vsyscall_64.c | 1 - arch/x86/hyperv/hv_spinlock.c | 11 +- arch/x86/include/asm/apic.h | 4 - arch/x86/include/asm/highmem.h | 1 - arch/x86/include/asm/mshyperv.h | 1 - arch/x86/include/asm/paravirt-base.h | 35 ++ arch/x86/include/asm/paravirt-spinlock.h | 145 ++++++++ arch/x86/include/asm/paravirt.h | 331 +++++------------- arch/x86/include/asm/paravirt_api_clock.h | 1 - arch/x86/include/asm/paravirt_types.h | 269 +++++++------- arch/x86/include/asm/pgtable_32.h | 1 - arch/x86/include/asm/ptrace.h | 2 +- arch/x86/include/asm/qspinlock.h | 87 +---- arch/x86/include/asm/spinlock.h | 1 - arch/x86/include/asm/timer.h | 1 + arch/x86/include/asm/tlbflush.h | 4 - arch/x86/kernel/Makefile | 2 +- arch/x86/kernel/apm_32.c | 1 - arch/x86/kernel/callthunks.c | 1 - arch/x86/kernel/cpu/bugs.c | 1 - arch/x86/kernel/cpu/vmware.c | 1 + arch/x86/kernel/kvm.c | 13 +- arch/x86/kernel/kvmclock.c | 1 + arch/x86/kernel/paravirt-spinlocks.c | 26 +- arch/x86/kernel/paravirt.c | 42 +-- arch/x86/kernel/tsc.c | 10 +- arch/x86/kernel/vsmp_64.c | 1 - arch/x86/lib/cache-smp.c | 1 - arch/x86/mm/init.c | 1 - arch/x86/xen/enlighten_pv.c | 82 ++--- arch/x86/xen/irq.c | 20 +- arch/x86/xen/mmu_pv.c | 100 ++---- arch/x86/xen/spinlock.c | 11 +- arch/x86/xen/time.c | 2 + drivers/clocksource/hyperv_timer.c | 2 + drivers/xen/time.c | 2 +- include/linux/sched/cputime.h | 18 + kernel/sched/core.c | 5 + kernel/sched/cputime.c | 13 + kernel/sched/sched.h | 3 +- tools/objtool/arch/x86/decode.c | 8 +- tools/objtool/check.c | 78 ++++- tools/objtool/include/objtool/check.h | 1 + 66 files changed, 662 insertions(+), 827 deletions(-) delete mode 100644 arch/arm/include/asm/paravirt.h delete mode 100644 arch/arm/include/asm/paravirt_api_clock.h delete mode 100644 arch/arm/kernel/paravirt.c delete mode 100644 arch/arm64/include/asm/paravirt_api_clock.h delete mode 100644 arch/loongarch/include/asm/paravirt_api_clock.h delete mode 100644 arch/powerpc/include/asm/paravirt_api_clock.h delete mode 100644 arch/riscv/include/asm/paravirt_api_clock.h create mode 100644 arch/x86/include/asm/paravirt-base.h create mode 100644 arch/x86/include/asm/paravirt-spinlock.h delete mode 100644 arch/x86/include/asm/paravirt_api_clock.h -- 2.51.0 In some places asm/paravirt.h is included without really being needed. Remove the related #include statements. Signed-off-by: Juergen Gross <jgross@suse.com> Acked-by: Peter Zijlstra (Intel) <peterz@infradead.org> --- V3: - reinstate the include in mmu_context.h (kernel test robot) V4: - reinstate the include in arch/x86/kernel/x86_init.c (Boris Petkov) --- arch/x86/entry/entry_64.S | 1 - arch/x86/entry/vsyscall/vsyscall_64.c | 1 - arch/x86/hyperv/hv_spinlock.c | 1 - arch/x86/include/asm/apic.h | 4 ---- arch/x86/include/asm/highmem.h | 1 - arch/x86/include/asm/mshyperv.h | 1 - arch/x86/include/asm/pgtable_32.h | 1 - arch/x86/include/asm/spinlock.h | 1 - arch/x86/include/asm/tlbflush.h | 4 ---- arch/x86/kernel/apm_32.c | 1 - arch/x86/kernel/callthunks.c | 1 - arch/x86/kernel/cpu/bugs.c | 1 - arch/x86/kernel/vsmp_64.c | 1 - arch/x86/lib/cache-smp.c | 1 - arch/x86/mm/init.c | 1 - arch/x86/xen/spinlock.c | 1 - 16 files changed, 22 deletions(-) diff --git a/arch/x86/entry/entry_64.S b/arch/x86/entry/entry_64.S index XXXXXXX..XXXXXXX 100644 --- a/arch/x86/entry/entry_64.S +++ b/arch/x86/entry/entry_64.S @@ -XXX,XX +XXX,XX @@ #include <asm/hw_irq.h> #include <asm/page_types.h> #include <asm/irqflags.h> -#include <asm/paravirt.h> #include <asm/percpu.h> #include <asm/asm.h> #include <asm/smap.h> diff --git a/arch/x86/entry/vsyscall/vsyscall_64.c b/arch/x86/entry/vsyscall/vsyscall_64.c index XXXXXXX..XXXXXXX 100644 --- a/arch/x86/entry/vsyscall/vsyscall_64.c +++ b/arch/x86/entry/vsyscall/vsyscall_64.c @@ -XXX,XX +XXX,XX @@ #include <asm/unistd.h> #include <asm/fixmap.h> #include <asm/traps.h> -#include <asm/paravirt.h> #define CREATE_TRACE_POINTS #include "vsyscall_trace.h" diff --git a/arch/x86/hyperv/hv_spinlock.c b/arch/x86/hyperv/hv_spinlock.c index XXXXXXX..XXXXXXX 100644 --- a/arch/x86/hyperv/hv_spinlock.c +++ b/arch/x86/hyperv/hv_spinlock.c @@ -XXX,XX +XXX,XX @@ #include <linux/spinlock.h> #include <asm/mshyperv.h> -#include <asm/paravirt.h> #include <asm/apic.h> #include <asm/msr.h> diff --git a/arch/x86/include/asm/apic.h b/arch/x86/include/asm/apic.h index XXXXXXX..XXXXXXX 100644 --- a/arch/x86/include/asm/apic.h +++ b/arch/x86/include/asm/apic.h @@ -XXX,XX +XXX,XX @@ static inline bool apic_from_smp_config(void) /* * Basic functions accessing APICs. */ -#ifdef CONFIG_PARAVIRT -#include <asm/paravirt.h> -#endif - static inline void native_apic_mem_write(u32 reg, u32 v) { volatile u32 *addr = (volatile u32 *)(APIC_BASE + reg); diff --git a/arch/x86/include/asm/highmem.h b/arch/x86/include/asm/highmem.h index XXXXXXX..XXXXXXX 100644 --- a/arch/x86/include/asm/highmem.h +++ b/arch/x86/include/asm/highmem.h @@ -XXX,XX +XXX,XX @@ #include <linux/interrupt.h> #include <linux/threads.h> #include <asm/tlbflush.h> -#include <asm/paravirt.h> #include <asm/fixmap.h> #include <asm/pgtable_areas.h> diff --git a/arch/x86/include/asm/mshyperv.h b/arch/x86/include/asm/mshyperv.h index XXXXXXX..XXXXXXX 100644 --- a/arch/x86/include/asm/mshyperv.h +++ b/arch/x86/include/asm/mshyperv.h @@ -XXX,XX +XXX,XX @@ #include <linux/io.h> #include <linux/static_call.h> #include <asm/nospec-branch.h> -#include <asm/paravirt.h> #include <asm/msr.h> #include <hyperv/hvhdk.h> #include <asm/fpu/types.h> diff --git a/arch/x86/include/asm/pgtable_32.h b/arch/x86/include/asm/pgtable_32.h index XXXXXXX..XXXXXXX 100644 --- a/arch/x86/include/asm/pgtable_32.h +++ b/arch/x86/include/asm/pgtable_32.h @@ -XXX,XX +XXX,XX @@ #ifndef __ASSEMBLER__ #include <asm/processor.h> #include <linux/threads.h> -#include <asm/paravirt.h> #include <linux/bitops.h> #include <linux/list.h> diff --git a/arch/x86/include/asm/spinlock.h b/arch/x86/include/asm/spinlock.h index XXXXXXX..XXXXXXX 100644 --- a/arch/x86/include/asm/spinlock.h +++ b/arch/x86/include/asm/spinlock.h @@ -XXX,XX +XXX,XX @@ #include <asm/page.h> #include <asm/processor.h> #include <linux/compiler.h> -#include <asm/paravirt.h> #include <asm/bitops.h> /* diff --git a/arch/x86/include/asm/tlbflush.h b/arch/x86/include/asm/tlbflush.h index XXXXXXX..XXXXXXX 100644 --- a/arch/x86/include/asm/tlbflush.h +++ b/arch/x86/include/asm/tlbflush.h @@ -XXX,XX +XXX,XX @@ static inline void mm_clear_asid_transition(struct mm_struct *mm) { } static inline bool mm_in_asid_transition(struct mm_struct *mm) { return false; } #endif /* CONFIG_BROADCAST_TLB_FLUSH */ -#ifdef CONFIG_PARAVIRT -#include <asm/paravirt.h> -#endif - #define flush_tlb_mm(mm) \ flush_tlb_mm_range(mm, 0UL, TLB_FLUSH_ALL, 0UL, true) diff --git a/arch/x86/kernel/apm_32.c b/arch/x86/kernel/apm_32.c index XXXXXXX..XXXXXXX 100644 --- a/arch/x86/kernel/apm_32.c +++ b/arch/x86/kernel/apm_32.c @@ -XXX,XX +XXX,XX @@ #include <linux/uaccess.h> #include <asm/desc.h> #include <asm/olpc.h> -#include <asm/paravirt.h> #include <asm/reboot.h> #include <asm/nospec-branch.h> #include <asm/ibt.h> diff --git a/arch/x86/kernel/callthunks.c b/arch/x86/kernel/callthunks.c index XXXXXXX..XXXXXXX 100644 --- a/arch/x86/kernel/callthunks.c +++ b/arch/x86/kernel/callthunks.c @@ -XXX,XX +XXX,XX @@ #include <asm/insn.h> #include <asm/kexec.h> #include <asm/nospec-branch.h> -#include <asm/paravirt.h> #include <asm/sections.h> #include <asm/switch_to.h> #include <asm/sync_core.h> diff --git a/arch/x86/kernel/cpu/bugs.c b/arch/x86/kernel/cpu/bugs.c index XXXXXXX..XXXXXXX 100644 --- a/arch/x86/kernel/cpu/bugs.c +++ b/arch/x86/kernel/cpu/bugs.c @@ -XXX,XX +XXX,XX @@ #include <asm/fpu/api.h> #include <asm/msr.h> #include <asm/vmx.h> -#include <asm/paravirt.h> #include <asm/cpu_device_id.h> #include <asm/e820/api.h> #include <asm/hypervisor.h> diff --git a/arch/x86/kernel/vsmp_64.c b/arch/x86/kernel/vsmp_64.c index XXXXXXX..XXXXXXX 100644 --- a/arch/x86/kernel/vsmp_64.c +++ b/arch/x86/kernel/vsmp_64.c @@ -XXX,XX +XXX,XX @@ #include <asm/apic.h> #include <asm/pci-direct.h> #include <asm/io.h> -#include <asm/paravirt.h> #include <asm/setup.h> #define TOPOLOGY_REGISTER_OFFSET 0x10 diff --git a/arch/x86/lib/cache-smp.c b/arch/x86/lib/cache-smp.c index XXXXXXX..XXXXXXX 100644 --- a/arch/x86/lib/cache-smp.c +++ b/arch/x86/lib/cache-smp.c @@ -XXX,XX +XXX,XX @@ // SPDX-License-Identifier: GPL-2.0 -#include <asm/paravirt.h> #include <linux/smp.h> #include <linux/export.h> #include <linux/kvm_types.h> diff --git a/arch/x86/mm/init.c b/arch/x86/mm/init.c index XXXXXXX..XXXXXXX 100644 --- a/arch/x86/mm/init.c +++ b/arch/x86/mm/init.c @@ -XXX,XX +XXX,XX @@ #include <asm/pti.h> #include <asm/text-patching.h> #include <asm/memtype.h> -#include <asm/paravirt.h> #include <asm/mmu_context.h> /* diff --git a/arch/x86/xen/spinlock.c b/arch/x86/xen/spinlock.c index XXXXXXX..XXXXXXX 100644 --- a/arch/x86/xen/spinlock.c +++ b/arch/x86/xen/spinlock.c @@ -XXX,XX +XXX,XX @@ #include <linux/slab.h> #include <linux/atomic.h> -#include <asm/paravirt.h> #include <asm/qspinlock.h> #include <xen/events.h> -- 2.51.0 Paravirt clock related functions are available in multiple archs. In order to share the common parts, move the common static keys to kernel/sched/ and remove them from the arch specific files. Make a common paravirt_steal_clock() implementation available in kernel/sched/cputime.c, guarding it with a new config option CONFIG_HAVE_PV_STEAL_CLOCK_GEN, which can be selected by an arch in case it wants to use that common variant. Signed-off-by: Juergen Gross <jgross@suse.com> Acked-by: Peter Zijlstra (Intel) <peterz@infradead.org> --- arch/Kconfig | 3 +++ arch/arm/include/asm/paravirt.h | 4 ---- arch/arm/kernel/paravirt.c | 3 --- arch/arm64/include/asm/paravirt.h | 4 ---- arch/arm64/kernel/paravirt.c | 4 +--- arch/loongarch/include/asm/paravirt.h | 3 --- arch/loongarch/kernel/paravirt.c | 3 +-- arch/powerpc/include/asm/paravirt.h | 3 --- arch/powerpc/platforms/pseries/setup.c | 4 +--- arch/riscv/include/asm/paravirt.h | 4 ---- arch/riscv/kernel/paravirt.c | 4 +--- arch/x86/include/asm/paravirt.h | 4 ---- arch/x86/kernel/cpu/vmware.c | 1 + arch/x86/kernel/kvm.c | 1 + arch/x86/kernel/paravirt.c | 3 --- drivers/xen/time.c | 1 + include/linux/sched/cputime.h | 18 ++++++++++++++++++ kernel/sched/core.c | 5 +++++ kernel/sched/cputime.c | 13 +++++++++++++ kernel/sched/sched.h | 2 +- 20 files changed, 47 insertions(+), 40 deletions(-) diff --git a/arch/Kconfig b/arch/Kconfig index XXXXXXX..XXXXXXX 100644 --- a/arch/Kconfig +++ b/arch/Kconfig @@ -XXX,XX +XXX,XX @@ config HAVE_IRQ_TIME_ACCOUNTING Archs need to ensure they use a high enough resolution clock to support irq time accounting and then call enable_sched_clock_irqtime(). +config HAVE_PV_STEAL_CLOCK_GEN + bool + config HAVE_MOVE_PUD bool help diff --git a/arch/arm/include/asm/paravirt.h b/arch/arm/include/asm/paravirt.h index XXXXXXX..XXXXXXX 100644 --- a/arch/arm/include/asm/paravirt.h +++ b/arch/arm/include/asm/paravirt.h @@ -XXX,XX +XXX,XX @@ #ifdef CONFIG_PARAVIRT #include <linux/static_call_types.h> -struct static_key; -extern struct static_key paravirt_steal_enabled; -extern struct static_key paravirt_steal_rq_enabled; - u64 dummy_steal_clock(int cpu); DECLARE_STATIC_CALL(pv_steal_clock, dummy_steal_clock); diff --git a/arch/arm/kernel/paravirt.c b/arch/arm/kernel/paravirt.c index XXXXXXX..XXXXXXX 100644 --- a/arch/arm/kernel/paravirt.c +++ b/arch/arm/kernel/paravirt.c @@ -XXX,XX +XXX,XX @@ #include <linux/static_call.h> #include <asm/paravirt.h> -struct static_key paravirt_steal_enabled; -struct static_key paravirt_steal_rq_enabled; - static u64 native_steal_clock(int cpu) { return 0; diff --git a/arch/arm64/include/asm/paravirt.h b/arch/arm64/include/asm/paravirt.h index XXXXXXX..XXXXXXX 100644 --- a/arch/arm64/include/asm/paravirt.h +++ b/arch/arm64/include/asm/paravirt.h @@ -XXX,XX +XXX,XX @@ #ifdef CONFIG_PARAVIRT #include <linux/static_call_types.h> -struct static_key; -extern struct static_key paravirt_steal_enabled; -extern struct static_key paravirt_steal_rq_enabled; - u64 dummy_steal_clock(int cpu); DECLARE_STATIC_CALL(pv_steal_clock, dummy_steal_clock); diff --git a/arch/arm64/kernel/paravirt.c b/arch/arm64/kernel/paravirt.c index XXXXXXX..XXXXXXX 100644 --- a/arch/arm64/kernel/paravirt.c +++ b/arch/arm64/kernel/paravirt.c @@ -XXX,XX +XXX,XX @@ #include <linux/slab.h> #include <linux/types.h> #include <linux/static_call.h> +#include <linux/sched/cputime.h> #include <asm/paravirt.h> #include <asm/pvclock-abi.h> #include <asm/smp_plat.h> -struct static_key paravirt_steal_enabled; -struct static_key paravirt_steal_rq_enabled; - static u64 native_steal_clock(int cpu) { return 0; diff --git a/arch/loongarch/include/asm/paravirt.h b/arch/loongarch/include/asm/paravirt.h index XXXXXXX..XXXXXXX 100644 --- a/arch/loongarch/include/asm/paravirt.h +++ b/arch/loongarch/include/asm/paravirt.h @@ -XXX,XX +XXX,XX @@ #ifdef CONFIG_PARAVIRT #include <linux/static_call_types.h> -struct static_key; -extern struct static_key paravirt_steal_enabled; -extern struct static_key paravirt_steal_rq_enabled; u64 dummy_steal_clock(int cpu); DECLARE_STATIC_CALL(pv_steal_clock, dummy_steal_clock); diff --git a/arch/loongarch/kernel/paravirt.c b/arch/loongarch/kernel/paravirt.c index XXXXXXX..XXXXXXX 100644 --- a/arch/loongarch/kernel/paravirt.c +++ b/arch/loongarch/kernel/paravirt.c @@ -XXX,XX +XXX,XX @@ #include <linux/kvm_para.h> #include <linux/reboot.h> #include <linux/static_call.h> +#include <linux/sched/cputime.h> #include <asm/paravirt.h> static int has_steal_clock; -struct static_key paravirt_steal_enabled; -struct static_key paravirt_steal_rq_enabled; static DEFINE_PER_CPU(struct kvm_steal_time, steal_time) __aligned(64); DEFINE_STATIC_KEY_FALSE(virt_spin_lock_key); diff --git a/arch/powerpc/include/asm/paravirt.h b/arch/powerpc/include/asm/paravirt.h index XXXXXXX..XXXXXXX 100644 --- a/arch/powerpc/include/asm/paravirt.h +++ b/arch/powerpc/include/asm/paravirt.h @@ -XXX,XX +XXX,XX @@ static inline bool is_shared_processor(void) } #ifdef CONFIG_PARAVIRT_TIME_ACCOUNTING -extern struct static_key paravirt_steal_enabled; -extern struct static_key paravirt_steal_rq_enabled; - u64 pseries_paravirt_steal_clock(int cpu); static inline u64 paravirt_steal_clock(int cpu) diff --git a/arch/powerpc/platforms/pseries/setup.c b/arch/powerpc/platforms/pseries/setup.c index XXXXXXX..XXXXXXX 100644 --- a/arch/powerpc/platforms/pseries/setup.c +++ b/arch/powerpc/platforms/pseries/setup.c @@ -XXX,XX +XXX,XX @@ #include <linux/memblock.h> #include <linux/swiotlb.h> #include <linux/seq_buf.h> +#include <linux/sched/cputime.h> #include <asm/mmu.h> #include <asm/processor.h> @@ -XXX,XX +XXX,XX @@ DEFINE_STATIC_KEY_FALSE(shared_processor); EXPORT_SYMBOL(shared_processor); #ifdef CONFIG_PARAVIRT_TIME_ACCOUNTING -struct static_key paravirt_steal_enabled; -struct static_key paravirt_steal_rq_enabled; - static bool steal_acc = true; static int __init parse_no_stealacc(char *arg) { diff --git a/arch/riscv/include/asm/paravirt.h b/arch/riscv/include/asm/paravirt.h index XXXXXXX..XXXXXXX 100644 --- a/arch/riscv/include/asm/paravirt.h +++ b/arch/riscv/include/asm/paravirt.h @@ -XXX,XX +XXX,XX @@ #ifdef CONFIG_PARAVIRT #include <linux/static_call_types.h> -struct static_key; -extern struct static_key paravirt_steal_enabled; -extern struct static_key paravirt_steal_rq_enabled; - u64 dummy_steal_clock(int cpu); DECLARE_STATIC_CALL(pv_steal_clock, dummy_steal_clock); diff --git a/arch/riscv/kernel/paravirt.c b/arch/riscv/kernel/paravirt.c index XXXXXXX..XXXXXXX 100644 --- a/arch/riscv/kernel/paravirt.c +++ b/arch/riscv/kernel/paravirt.c @@ -XXX,XX +XXX,XX @@ #include <linux/printk.h> #include <linux/static_call.h> #include <linux/types.h> +#include <linux/sched/cputime.h> #include <asm/barrier.h> #include <asm/page.h> #include <asm/paravirt.h> #include <asm/sbi.h> -struct static_key paravirt_steal_enabled; -struct static_key paravirt_steal_rq_enabled; - static u64 native_steal_clock(int cpu) { return 0; diff --git a/arch/x86/include/asm/paravirt.h b/arch/x86/include/asm/paravirt.h index XXXXXXX..XXXXXXX 100644 --- a/arch/x86/include/asm/paravirt.h +++ b/arch/x86/include/asm/paravirt.h @@ -XXX,XX +XXX,XX @@ static __always_inline u64 paravirt_sched_clock(void) return static_call(pv_sched_clock)(); } -struct static_key; -extern struct static_key paravirt_steal_enabled; -extern struct static_key paravirt_steal_rq_enabled; - __visible void __native_queued_spin_unlock(struct qspinlock *lock); bool pv_is_native_spin_unlock(void); __visible bool __native_vcpu_is_preempted(long cpu); diff --git a/arch/x86/kernel/cpu/vmware.c b/arch/x86/kernel/cpu/vmware.c index XXXXXXX..XXXXXXX 100644 --- a/arch/x86/kernel/cpu/vmware.c +++ b/arch/x86/kernel/cpu/vmware.c @@ -XXX,XX +XXX,XX @@ #include <linux/efi.h> #include <linux/reboot.h> #include <linux/static_call.h> +#include <linux/sched/cputime.h> #include <asm/div64.h> #include <asm/x86_init.h> #include <asm/hypervisor.h> diff --git a/arch/x86/kernel/kvm.c b/arch/x86/kernel/kvm.c index XXXXXXX..XXXXXXX 100644 --- a/arch/x86/kernel/kvm.c +++ b/arch/x86/kernel/kvm.c @@ -XXX,XX +XXX,XX @@ #include <linux/cc_platform.h> #include <linux/efi.h> #include <linux/kvm_types.h> +#include <linux/sched/cputime.h> #include <asm/timer.h> #include <asm/cpu.h> #include <asm/traps.h> diff --git a/arch/x86/kernel/paravirt.c b/arch/x86/kernel/paravirt.c index XXXXXXX..XXXXXXX 100644 --- a/arch/x86/kernel/paravirt.c +++ b/arch/x86/kernel/paravirt.c @@ -XXX,XX +XXX,XX @@ void __init native_pv_lock_init(void) static_branch_enable(&virt_spin_lock_key); } -struct static_key paravirt_steal_enabled; -struct static_key paravirt_steal_rq_enabled; - static u64 native_steal_clock(int cpu) { return 0; diff --git a/drivers/xen/time.c b/drivers/xen/time.c index XXXXXXX..XXXXXXX 100644 --- a/drivers/xen/time.c +++ b/drivers/xen/time.c @@ -XXX,XX +XXX,XX @@ #include <linux/gfp.h> #include <linux/slab.h> #include <linux/static_call.h> +#include <linux/sched/cputime.h> #include <asm/paravirt.h> #include <asm/xen/hypervisor.h> diff --git a/include/linux/sched/cputime.h b/include/linux/sched/cputime.h index XXXXXXX..XXXXXXX 100644 --- a/include/linux/sched/cputime.h +++ b/include/linux/sched/cputime.h @@ -XXX,XX +XXX,XX @@ #ifndef _LINUX_SCHED_CPUTIME_H #define _LINUX_SCHED_CPUTIME_H +#include <linux/static_call_types.h> #include <linux/sched/signal.h> /* @@ -XXX,XX +XXX,XX @@ static inline void prev_cputime_init(struct prev_cputime *prev) extern unsigned long long task_sched_runtime(struct task_struct *task); +#ifdef CONFIG_PARAVIRT +struct static_key; +extern struct static_key paravirt_steal_enabled; +extern struct static_key paravirt_steal_rq_enabled; + +#ifdef CONFIG_HAVE_PV_STEAL_CLOCK_GEN +u64 dummy_steal_clock(int cpu); + +DECLARE_STATIC_CALL(pv_steal_clock, dummy_steal_clock); + +static inline u64 paravirt_steal_clock(int cpu) +{ + return static_call(pv_steal_clock)(cpu); +} +#endif +#endif + #endif /* _LINUX_SCHED_CPUTIME_H */ diff --git a/kernel/sched/core.c b/kernel/sched/core.c index XXXXXXX..XXXXXXX 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -XXX,XX +XXX,XX @@ struct rq *task_rq_lock(struct task_struct *p, struct rq_flags *rf) * RQ-clock updating methods: */ +/* Use CONFIG_PARAVIRT as this will avoid more #ifdef in arch code. */ +#ifdef CONFIG_PARAVIRT +struct static_key paravirt_steal_rq_enabled; +#endif + static void update_rq_clock_task(struct rq *rq, s64 delta) { /* diff --git a/kernel/sched/cputime.c b/kernel/sched/cputime.c index XXXXXXX..XXXXXXX 100644 --- a/kernel/sched/cputime.c +++ b/kernel/sched/cputime.c @@ -XXX,XX +XXX,XX @@ void __account_forceidle_time(struct task_struct *p, u64 delta) * ticks are not redelivered later. Due to that, this function may on * occasion account more time than the calling functions think elapsed. */ +#ifdef CONFIG_PARAVIRT +struct static_key paravirt_steal_enabled; + +#ifdef CONFIG_HAVE_PV_STEAL_CLOCK_GEN +static u64 native_steal_clock(int cpu) +{ + return 0; +} + +DEFINE_STATIC_CALL(pv_steal_clock, native_steal_clock); +#endif +#endif + static __always_inline u64 steal_account_process_time(u64 maxtime) { #ifdef CONFIG_PARAVIRT diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index XXXXXXX..XXXXXXX 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -XXX,XX +XXX,XX @@ struct rt_rq; struct sched_group; struct cpuidle_state; -#ifdef CONFIG_PARAVIRT +#if defined(CONFIG_PARAVIRT) && !defined(CONFIG_HAVE_PV_STEAL_CLOCK_GEN) # include <asm/paravirt.h> #endif -- 2.51.0 Remove the arch specific variant of paravirt_steal_clock() and use the common one instead. This allows to remove paravirt.c and paravirt.h from arch/arm. Until all archs supporting Xen have been switched to the common code of paravirt_steal_clock(), drivers/xen/time.c needs to include asm/paravirt.h for those archs, while this is not necessary for arm any longer. Signed-off-by: Juergen Gross <jgross@suse.com> Acked-by: Peter Zijlstra (Intel) <peterz@infradead.org> --- arch/arm/Kconfig | 1 + arch/arm/include/asm/paravirt.h | 18 ------------------ arch/arm/kernel/Makefile | 1 - arch/arm/kernel/paravirt.c | 20 -------------------- drivers/xen/time.c | 2 ++ 5 files changed, 3 insertions(+), 39 deletions(-) delete mode 100644 arch/arm/include/asm/paravirt.h delete mode 100644 arch/arm/kernel/paravirt.c diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig index XXXXXXX..XXXXXXX 100644 --- a/arch/arm/Kconfig +++ b/arch/arm/Kconfig @@ -XXX,XX +XXX,XX @@ config UACCESS_WITH_MEMCPY config PARAVIRT bool "Enable paravirtualization code" + select HAVE_PV_STEAL_CLOCK_GEN help This changes the kernel so it can modify itself when it is run under a hypervisor, potentially improving performance significantly diff --git a/arch/arm/include/asm/paravirt.h b/arch/arm/include/asm/paravirt.h deleted file mode 100644 index XXXXXXX..XXXXXXX --- a/arch/arm/include/asm/paravirt.h +++ /dev/null @@ -XXX,XX +XXX,XX @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef _ASM_ARM_PARAVIRT_H -#define _ASM_ARM_PARAVIRT_H - -#ifdef CONFIG_PARAVIRT -#include <linux/static_call_types.h> - -u64 dummy_steal_clock(int cpu); - -DECLARE_STATIC_CALL(pv_steal_clock, dummy_steal_clock); - -static inline u64 paravirt_steal_clock(int cpu) -{ - return static_call(pv_steal_clock)(cpu); -} -#endif - -#endif diff --git a/arch/arm/kernel/Makefile b/arch/arm/kernel/Makefile index XXXXXXX..XXXXXXX 100644 --- a/arch/arm/kernel/Makefile +++ b/arch/arm/kernel/Makefile @@ -XXX,XX +XXX,XX @@ AFLAGS_iwmmxt.o := -Wa,-mcpu=iwmmxt obj-$(CONFIG_ARM_CPU_TOPOLOGY) += topology.o obj-$(CONFIG_VDSO) += vdso.o obj-$(CONFIG_EFI) += efi.o -obj-$(CONFIG_PARAVIRT) += paravirt.o obj-y += head$(MMUEXT).o obj-$(CONFIG_DEBUG_LL) += debug.o diff --git a/arch/arm/kernel/paravirt.c b/arch/arm/kernel/paravirt.c deleted file mode 100644 index XXXXXXX..XXXXXXX --- a/arch/arm/kernel/paravirt.c +++ /dev/null @@ -XXX,XX +XXX,XX @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * - * Copyright (C) 2013 Citrix Systems - * - * Author: Stefano Stabellini <stefano.stabellini@eu.citrix.com> - */ - -#include <linux/export.h> -#include <linux/jump_label.h> -#include <linux/types.h> -#include <linux/static_call.h> -#include <asm/paravirt.h> - -static u64 native_steal_clock(int cpu) -{ - return 0; -} - -DEFINE_STATIC_CALL(pv_steal_clock, native_steal_clock); diff --git a/drivers/xen/time.c b/drivers/xen/time.c index XXXXXXX..XXXXXXX 100644 --- a/drivers/xen/time.c +++ b/drivers/xen/time.c @@ -XXX,XX +XXX,XX @@ #include <linux/static_call.h> #include <linux/sched/cputime.h> +#ifndef CONFIG_HAVE_PV_STEAL_CLOCK_GEN #include <asm/paravirt.h> +#endif #include <asm/xen/hypervisor.h> #include <asm/xen/hypercall.h> -- 2.51.0 Remove the arch specific variant of paravirt_steal_clock() and use the common one instead. With all archs supporting Xen now having been switched to the common variant, including paravirt.h can be dropped from drivers/xen/time.c. Signed-off-by: Juergen Gross <jgross@suse.com> Acked-by: Peter Zijlstra (Intel) <peterz@infradead.org> --- arch/x86/Kconfig | 1 + arch/x86/include/asm/paravirt.h | 7 ------- arch/x86/kernel/paravirt.c | 6 ------ arch/x86/xen/time.c | 1 + drivers/xen/time.c | 3 --- 5 files changed, 2 insertions(+), 16 deletions(-) diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index XXXXXXX..XXXXXXX 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -XXX,XX +XXX,XX @@ if HYPERVISOR_GUEST config PARAVIRT bool "Enable paravirtualization code" depends on HAVE_STATIC_CALL + select HAVE_PV_STEAL_CLOCK_GEN help This changes the kernel so it can modify itself when it is run under a hypervisor, potentially improving performance significantly diff --git a/arch/x86/include/asm/paravirt.h b/arch/x86/include/asm/paravirt.h index XXXXXXX..XXXXXXX 100644 --- a/arch/x86/include/asm/paravirt.h +++ b/arch/x86/include/asm/paravirt.h @@ -XXX,XX +XXX,XX @@ #include <linux/static_call_types.h> #include <asm/frame.h> -u64 dummy_steal_clock(int cpu); u64 dummy_sched_clock(void); -DECLARE_STATIC_CALL(pv_steal_clock, dummy_steal_clock); DECLARE_STATIC_CALL(pv_sched_clock, dummy_sched_clock); void paravirt_set_sched_clock(u64 (*func)(void)); @@ -XXX,XX +XXX,XX @@ bool pv_is_native_spin_unlock(void); __visible bool __native_vcpu_is_preempted(long cpu); bool pv_is_native_vcpu_is_preempted(void); -static inline u64 paravirt_steal_clock(int cpu) -{ - return static_call(pv_steal_clock)(cpu); -} - #ifdef CONFIG_PARAVIRT_SPINLOCKS void __init paravirt_set_cap(void); #endif diff --git a/arch/x86/kernel/paravirt.c b/arch/x86/kernel/paravirt.c index XXXXXXX..XXXXXXX 100644 --- a/arch/x86/kernel/paravirt.c +++ b/arch/x86/kernel/paravirt.c @@ -XXX,XX +XXX,XX @@ void __init native_pv_lock_init(void) static_branch_enable(&virt_spin_lock_key); } -static u64 native_steal_clock(int cpu) -{ - return 0; -} - -DEFINE_STATIC_CALL(pv_steal_clock, native_steal_clock); DEFINE_STATIC_CALL(pv_sched_clock, native_sched_clock); void paravirt_set_sched_clock(u64 (*func)(void)) diff --git a/arch/x86/xen/time.c b/arch/x86/xen/time.c index XXXXXXX..XXXXXXX 100644 --- a/arch/x86/xen/time.c +++ b/arch/x86/xen/time.c @@ -XXX,XX +XXX,XX @@ #include <linux/slab.h> #include <linux/pvclock_gtod.h> #include <linux/timekeeper_internal.h> +#include <linux/sched/cputime.h> #include <asm/pvclock.h> #include <asm/xen/hypervisor.h> diff --git a/drivers/xen/time.c b/drivers/xen/time.c index XXXXXXX..XXXXXXX 100644 --- a/drivers/xen/time.c +++ b/drivers/xen/time.c @@ -XXX,XX +XXX,XX @@ #include <linux/static_call.h> #include <linux/sched/cputime.h> -#ifndef CONFIG_HAVE_PV_STEAL_CLOCK_GEN -#include <asm/paravirt.h> -#endif #include <asm/xen/hypervisor.h> #include <asm/xen/hypercall.h> -- 2.51.0 The only user of paravirt_sched_clock() is in tsc.c, so move the code from paravirt.c and paravirt.h to tsc.c. Signed-off-by: Juergen Gross <jgross@suse.com> Acked-by: Peter Zijlstra (Intel) <peterz@infradead.org> --- arch/x86/include/asm/paravirt.h | 12 ------------ arch/x86/include/asm/timer.h | 1 + arch/x86/kernel/kvmclock.c | 1 + arch/x86/kernel/paravirt.c | 7 ------- arch/x86/kernel/tsc.c | 10 +++++++++- arch/x86/xen/time.c | 1 + drivers/clocksource/hyperv_timer.c | 2 ++ 7 files changed, 14 insertions(+), 20 deletions(-) diff --git a/arch/x86/include/asm/paravirt.h b/arch/x86/include/asm/paravirt.h index XXXXXXX..XXXXXXX 100644 --- a/arch/x86/include/asm/paravirt.h +++ b/arch/x86/include/asm/paravirt.h @@ -XXX,XX +XXX,XX @@ #ifndef __ASSEMBLER__ #include <linux/types.h> #include <linux/cpumask.h> -#include <linux/static_call_types.h> #include <asm/frame.h> -u64 dummy_sched_clock(void); - -DECLARE_STATIC_CALL(pv_sched_clock, dummy_sched_clock); - -void paravirt_set_sched_clock(u64 (*func)(void)); - -static __always_inline u64 paravirt_sched_clock(void) -{ - return static_call(pv_sched_clock)(); -} - __visible void __native_queued_spin_unlock(struct qspinlock *lock); bool pv_is_native_spin_unlock(void); __visible bool __native_vcpu_is_preempted(long cpu); diff --git a/arch/x86/include/asm/timer.h b/arch/x86/include/asm/timer.h index XXXXXXX..XXXXXXX 100644 --- a/arch/x86/include/asm/timer.h +++ b/arch/x86/include/asm/timer.h @@ -XXX,XX +XXX,XX @@ extern void recalibrate_cpu_khz(void); extern int no_timer_check; extern bool using_native_sched_clock(void); +void paravirt_set_sched_clock(u64 (*func)(void)); /* * We use the full linear equation: f(x) = a + b*x, in order to allow diff --git a/arch/x86/kernel/kvmclock.c b/arch/x86/kernel/kvmclock.c index XXXXXXX..XXXXXXX 100644 --- a/arch/x86/kernel/kvmclock.c +++ b/arch/x86/kernel/kvmclock.c @@ -XXX,XX +XXX,XX @@ #include <linux/cc_platform.h> #include <asm/hypervisor.h> +#include <asm/timer.h> #include <asm/x86_init.h> #include <asm/kvmclock.h> diff --git a/arch/x86/kernel/paravirt.c b/arch/x86/kernel/paravirt.c index XXXXXXX..XXXXXXX 100644 --- a/arch/x86/kernel/paravirt.c +++ b/arch/x86/kernel/paravirt.c @@ -XXX,XX +XXX,XX @@ void __init native_pv_lock_init(void) static_branch_enable(&virt_spin_lock_key); } -DEFINE_STATIC_CALL(pv_sched_clock, native_sched_clock); - -void paravirt_set_sched_clock(u64 (*func)(void)) -{ - static_call_update(pv_sched_clock, func); -} - static noinstr void pv_native_safe_halt(void) { native_safe_halt(); diff --git a/arch/x86/kernel/tsc.c b/arch/x86/kernel/tsc.c index XXXXXXX..XXXXXXX 100644 --- a/arch/x86/kernel/tsc.c +++ b/arch/x86/kernel/tsc.c @@ -XXX,XX +XXX,XX @@ u64 native_sched_clock_from_tsc(u64 tsc) /* We need to define a real function for sched_clock, to override the weak default version */ #ifdef CONFIG_PARAVIRT +DEFINE_STATIC_CALL(pv_sched_clock, native_sched_clock); + noinstr u64 sched_clock_noinstr(void) { - return paravirt_sched_clock(); + return static_call(pv_sched_clock)(); } bool using_native_sched_clock(void) { return static_call_query(pv_sched_clock) == native_sched_clock; } + +void paravirt_set_sched_clock(u64 (*func)(void)) +{ + static_call_update(pv_sched_clock, func); +} #else u64 sched_clock_noinstr(void) __attribute__((alias("native_sched_clock"))); bool using_native_sched_clock(void) { return true; } +void paravirt_set_sched_clock(u64 (*func)(void)) { } #endif notrace u64 sched_clock(void) diff --git a/arch/x86/xen/time.c b/arch/x86/xen/time.c index XXXXXXX..XXXXXXX 100644 --- a/arch/x86/xen/time.c +++ b/arch/x86/xen/time.c @@ -XXX,XX +XXX,XX @@ #include <linux/sched/cputime.h> #include <asm/pvclock.h> +#include <asm/timer.h> #include <asm/xen/hypervisor.h> #include <asm/xen/hypercall.h> #include <asm/xen/cpuid.h> diff --git a/drivers/clocksource/hyperv_timer.c b/drivers/clocksource/hyperv_timer.c index XXXXXXX..XXXXXXX 100644 --- a/drivers/clocksource/hyperv_timer.c +++ b/drivers/clocksource/hyperv_timer.c @@ -XXX,XX +XXX,XX @@ static __always_inline void hv_setup_sched_clock(void *sched_clock) sched_clock_register(sched_clock, 64, NSEC_PER_SEC); } #elif defined CONFIG_PARAVIRT +#include <asm/timer.h> + static __always_inline void hv_setup_sched_clock(void *sched_clock) { /* We're on x86/x64 *and* using PV ops */ -- 2.51.0 Instead of having a pre-filled array xen_irq_ops for Xen PV paravirt functions, drop the array and assign each element individually. This is in preparation of reducing the paravirt include hell by splitting paravirt.h into multiple more fine grained header files, which will in turn require to split up the pv_ops vector as well. Dropping the pre-filled array makes life easier for objtool to detect missing initializers in multiple pv_ops_ arrays. Signed-off-by: Juergen Gross <jgross@suse.com> Reviewed-by: Boris Ostrovsky <boris.ostrovsky@oracle.com> --- V2: - new patch --- arch/x86/xen/irq.c | 20 +++++++------------- tools/objtool/check.c | 1 - 2 files changed, 7 insertions(+), 14 deletions(-) diff --git a/arch/x86/xen/irq.c b/arch/x86/xen/irq.c index XXXXXXX..XXXXXXX 100644 --- a/arch/x86/xen/irq.c +++ b/arch/x86/xen/irq.c @@ -XXX,XX +XXX,XX @@ static void xen_halt(void) xen_safe_halt(); } -static const typeof(pv_ops) xen_irq_ops __initconst = { - .irq = { - /* Initial interrupt flag handling only called while interrupts off. */ - .save_fl = __PV_IS_CALLEE_SAVE(paravirt_ret0), - .irq_disable = __PV_IS_CALLEE_SAVE(paravirt_nop), - .irq_enable = __PV_IS_CALLEE_SAVE(BUG_func), - - .safe_halt = xen_safe_halt, - .halt = xen_halt, - }, -}; - void __init xen_init_irq_ops(void) { - pv_ops.irq = xen_irq_ops.irq; + /* Initial interrupt flag handling only called while interrupts off. */ + pv_ops.irq.save_fl = __PV_IS_CALLEE_SAVE(paravirt_ret0); + pv_ops.irq.irq_disable = __PV_IS_CALLEE_SAVE(paravirt_nop); + pv_ops.irq.irq_enable = __PV_IS_CALLEE_SAVE(BUG_func); + pv_ops.irq.safe_halt = xen_safe_halt; + pv_ops.irq.halt = xen_halt; + x86_init.irqs.intr_init = xen_init_IRQ; } diff --git a/tools/objtool/check.c b/tools/objtool/check.c index XXXXXXX..XXXXXXX 100644 --- a/tools/objtool/check.c +++ b/tools/objtool/check.c @@ -XXX,XX +XXX,XX @@ static int init_pv_ops(struct objtool_file *file) static const char *pv_ops_tables[] = { "pv_ops", "xen_cpu_ops", - "xen_irq_ops", "xen_mmu_ops", NULL, }; -- 2.51.0 Instead of having a pre-filled array xen_cpu_ops for Xen PV paravirt functions, drop the array and assign each element individually. This is in preparation of reducing the paravirt include hell by splitting paravirt.h into multiple more fine grained header files, which will in turn require to split up the pv_ops vector as well. Dropping the pre-filled array makes life easier for objtool to detect missing initializers in multiple pv_ops_ arrays. Signed-off-by: Juergen Gross <jgross@suse.com> Reviewed-by: Boris Ostrovsky <boris.ostrovsky@oracle.com> --- V2: - new patch --- arch/x86/xen/enlighten_pv.c | 82 +++++++++++++++---------------------- tools/objtool/check.c | 1 - 2 files changed, 33 insertions(+), 50 deletions(-) diff --git a/arch/x86/xen/enlighten_pv.c b/arch/x86/xen/enlighten_pv.c index XXXXXXX..XXXXXXX 100644 --- a/arch/x86/xen/enlighten_pv.c +++ b/arch/x86/xen/enlighten_pv.c @@ -XXX,XX +XXX,XX @@ static const struct pv_info xen_info __initconst = { .name = "Xen", }; -static const typeof(pv_ops) xen_cpu_ops __initconst = { - .cpu = { - .cpuid = xen_cpuid, - - .set_debugreg = xen_set_debugreg, - .get_debugreg = xen_get_debugreg, - - .read_cr0 = xen_read_cr0, - .write_cr0 = xen_write_cr0, - - .write_cr4 = xen_write_cr4, - - .read_msr = xen_read_msr, - .write_msr = xen_write_msr, - - .read_msr_safe = xen_read_msr_safe, - .write_msr_safe = xen_write_msr_safe, - - .read_pmc = xen_read_pmc, - - .load_tr_desc = paravirt_nop, - .set_ldt = xen_set_ldt, - .load_gdt = xen_load_gdt, - .load_idt = xen_load_idt, - .load_tls = xen_load_tls, - .load_gs_index = xen_load_gs_index, - - .alloc_ldt = xen_alloc_ldt, - .free_ldt = xen_free_ldt, - - .store_tr = xen_store_tr, - - .write_ldt_entry = xen_write_ldt_entry, - .write_gdt_entry = xen_write_gdt_entry, - .write_idt_entry = xen_write_idt_entry, - .load_sp0 = xen_load_sp0, - -#ifdef CONFIG_X86_IOPL_IOPERM - .invalidate_io_bitmap = xen_invalidate_io_bitmap, - .update_io_bitmap = xen_update_io_bitmap, -#endif - .io_delay = xen_io_delay, - - .start_context_switch = xen_start_context_switch, - .end_context_switch = xen_end_context_switch, - }, -}; - static void xen_restart(char *msg) { xen_reboot(SHUTDOWN_reboot); @@ -XXX,XX +XXX,XX @@ asmlinkage __visible void __init xen_start_kernel(struct start_info *si) /* Install Xen paravirt ops */ pv_info = xen_info; - pv_ops.cpu = xen_cpu_ops.cpu; + + pv_ops.cpu.cpuid = xen_cpuid; + pv_ops.cpu.set_debugreg = xen_set_debugreg; + pv_ops.cpu.get_debugreg = xen_get_debugreg; + pv_ops.cpu.read_cr0 = xen_read_cr0; + pv_ops.cpu.write_cr0 = xen_write_cr0; + pv_ops.cpu.write_cr4 = xen_write_cr4; + pv_ops.cpu.read_msr = xen_read_msr; + pv_ops.cpu.write_msr = xen_write_msr; + pv_ops.cpu.read_msr_safe = xen_read_msr_safe; + pv_ops.cpu.write_msr_safe = xen_write_msr_safe; + pv_ops.cpu.read_pmc = xen_read_pmc; + pv_ops.cpu.load_tr_desc = paravirt_nop; + pv_ops.cpu.set_ldt = xen_set_ldt; + pv_ops.cpu.load_gdt = xen_load_gdt; + pv_ops.cpu.load_idt = xen_load_idt; + pv_ops.cpu.load_tls = xen_load_tls; + pv_ops.cpu.load_gs_index = xen_load_gs_index; + pv_ops.cpu.alloc_ldt = xen_alloc_ldt; + pv_ops.cpu.free_ldt = xen_free_ldt; + pv_ops.cpu.store_tr = xen_store_tr; + pv_ops.cpu.write_ldt_entry = xen_write_ldt_entry; + pv_ops.cpu.write_gdt_entry = xen_write_gdt_entry; + pv_ops.cpu.write_idt_entry = xen_write_idt_entry; + pv_ops.cpu.load_sp0 = xen_load_sp0; +#ifdef CONFIG_X86_IOPL_IOPERM + pv_ops.cpu.invalidate_io_bitmap = xen_invalidate_io_bitmap; + pv_ops.cpu.update_io_bitmap = xen_update_io_bitmap; +#endif + pv_ops.cpu.io_delay = xen_io_delay; + pv_ops.cpu.start_context_switch = xen_start_context_switch; + pv_ops.cpu.end_context_switch = xen_end_context_switch; + xen_init_irq_ops(); /* diff --git a/tools/objtool/check.c b/tools/objtool/check.c index XXXXXXX..XXXXXXX 100644 --- a/tools/objtool/check.c +++ b/tools/objtool/check.c @@ -XXX,XX +XXX,XX @@ static int init_pv_ops(struct objtool_file *file) { static const char *pv_ops_tables[] = { "pv_ops", - "xen_cpu_ops", "xen_mmu_ops", NULL, }; -- 2.51.0 Instead of having a pre-filled array xen_mmu_ops for Xen PV paravirt functions, drop the array and assign each element individually. This is in preparation of reducing the paravirt include hell by splitting paravirt.h into multiple more fine grained header files, which will in turn require to split up the pv_ops vector as well. Dropping the pre-filled array makes life easier for objtool to detect missing initializers in multiple pv_ops_ arrays. Signed-off-by: Juergen Gross <jgross@suse.com> Reviewed-by: Boris Ostrovsky <boris.ostrovsky@oracle.com> --- V2: - new patch --- arch/x86/xen/mmu_pv.c | 100 ++++++++++++++++-------------------------- tools/objtool/check.c | 1 - 2 files changed, 38 insertions(+), 63 deletions(-) diff --git a/arch/x86/xen/mmu_pv.c b/arch/x86/xen/mmu_pv.c index XXXXXXX..XXXXXXX 100644 --- a/arch/x86/xen/mmu_pv.c +++ b/arch/x86/xen/mmu_pv.c @@ -XXX,XX +XXX,XX @@ static void xen_leave_lazy_mmu(void) preempt_enable(); } -static const typeof(pv_ops) xen_mmu_ops __initconst = { - .mmu = { - .read_cr2 = __PV_IS_CALLEE_SAVE(xen_read_cr2), - .write_cr2 = xen_write_cr2, - - .read_cr3 = xen_read_cr3, - .write_cr3 = xen_write_cr3_init, - - .flush_tlb_user = xen_flush_tlb, - .flush_tlb_kernel = xen_flush_tlb, - .flush_tlb_one_user = xen_flush_tlb_one_user, - .flush_tlb_multi = xen_flush_tlb_multi, - - .pgd_alloc = xen_pgd_alloc, - .pgd_free = xen_pgd_free, - - .alloc_pte = xen_alloc_pte_init, - .release_pte = xen_release_pte_init, - .alloc_pmd = xen_alloc_pmd_init, - .release_pmd = xen_release_pmd_init, - - .set_pte = xen_set_pte_init, - .set_pmd = xen_set_pmd_hyper, - - .ptep_modify_prot_start = xen_ptep_modify_prot_start, - .ptep_modify_prot_commit = xen_ptep_modify_prot_commit, - - .pte_val = PV_CALLEE_SAVE(xen_pte_val), - .pgd_val = PV_CALLEE_SAVE(xen_pgd_val), - - .make_pte = PV_CALLEE_SAVE(xen_make_pte_init), - .make_pgd = PV_CALLEE_SAVE(xen_make_pgd), - - .set_pud = xen_set_pud_hyper, - - .make_pmd = PV_CALLEE_SAVE(xen_make_pmd), - .pmd_val = PV_CALLEE_SAVE(xen_pmd_val), - - .pud_val = PV_CALLEE_SAVE(xen_pud_val), - .make_pud = PV_CALLEE_SAVE(xen_make_pud), - .set_p4d = xen_set_p4d_hyper, - - .alloc_pud = xen_alloc_pmd_init, - .release_pud = xen_release_pmd_init, - - .p4d_val = PV_CALLEE_SAVE(xen_p4d_val), - .make_p4d = PV_CALLEE_SAVE(xen_make_p4d), - - .enter_mmap = xen_enter_mmap, - .exit_mmap = xen_exit_mmap, - - .lazy_mode = { - .enter = xen_enter_lazy_mmu, - .leave = xen_leave_lazy_mmu, - .flush = xen_flush_lazy_mmu, - }, - - .set_fixmap = xen_set_fixmap, - }, -}; - void __init xen_init_mmu_ops(void) { x86_init.paging.pagetable_init = xen_pagetable_init; x86_init.hyper.init_after_bootmem = xen_after_bootmem; - pv_ops.mmu = xen_mmu_ops.mmu; + pv_ops.mmu.read_cr2 = __PV_IS_CALLEE_SAVE(xen_read_cr2); + pv_ops.mmu.write_cr2 = xen_write_cr2; + pv_ops.mmu.read_cr3 = xen_read_cr3; + pv_ops.mmu.write_cr3 = xen_write_cr3_init; + pv_ops.mmu.flush_tlb_user = xen_flush_tlb; + pv_ops.mmu.flush_tlb_kernel = xen_flush_tlb; + pv_ops.mmu.flush_tlb_one_user = xen_flush_tlb_one_user; + pv_ops.mmu.flush_tlb_multi = xen_flush_tlb_multi; + pv_ops.mmu.pgd_alloc = xen_pgd_alloc; + pv_ops.mmu.pgd_free = xen_pgd_free; + pv_ops.mmu.alloc_pte = xen_alloc_pte_init; + pv_ops.mmu.release_pte = xen_release_pte_init; + pv_ops.mmu.alloc_pmd = xen_alloc_pmd_init; + pv_ops.mmu.release_pmd = xen_release_pmd_init; + pv_ops.mmu.set_pte = xen_set_pte_init; + pv_ops.mmu.set_pmd = xen_set_pmd_hyper; + pv_ops.mmu.ptep_modify_prot_start = xen_ptep_modify_prot_start; + pv_ops.mmu.ptep_modify_prot_commit = xen_ptep_modify_prot_commit; + pv_ops.mmu.pte_val = PV_CALLEE_SAVE(xen_pte_val); + pv_ops.mmu.pgd_val = PV_CALLEE_SAVE(xen_pgd_val); + pv_ops.mmu.make_pte = PV_CALLEE_SAVE(xen_make_pte_init); + pv_ops.mmu.make_pgd = PV_CALLEE_SAVE(xen_make_pgd); + pv_ops.mmu.set_pud = xen_set_pud_hyper; + pv_ops.mmu.make_pmd = PV_CALLEE_SAVE(xen_make_pmd); + pv_ops.mmu.pmd_val = PV_CALLEE_SAVE(xen_pmd_val); + pv_ops.mmu.pud_val = PV_CALLEE_SAVE(xen_pud_val); + pv_ops.mmu.make_pud = PV_CALLEE_SAVE(xen_make_pud); + pv_ops.mmu.set_p4d = xen_set_p4d_hyper; + pv_ops.mmu.alloc_pud = xen_alloc_pmd_init; + pv_ops.mmu.release_pud = xen_release_pmd_init; + pv_ops.mmu.p4d_val = PV_CALLEE_SAVE(xen_p4d_val); + pv_ops.mmu.make_p4d = PV_CALLEE_SAVE(xen_make_p4d); + pv_ops.mmu.enter_mmap = xen_enter_mmap; + pv_ops.mmu.exit_mmap = xen_exit_mmap; + pv_ops.mmu.lazy_mode.enter = xen_enter_lazy_mmu; + pv_ops.mmu.lazy_mode.leave = xen_leave_lazy_mmu; + pv_ops.mmu.lazy_mode.flush = xen_flush_lazy_mmu; + pv_ops.mmu.set_fixmap = xen_set_fixmap; memset(dummy_mapping, 0xff, PAGE_SIZE); } diff --git a/tools/objtool/check.c b/tools/objtool/check.c index XXXXXXX..XXXXXXX 100644 --- a/tools/objtool/check.c +++ b/tools/objtool/check.c @@ -XXX,XX +XXX,XX @@ static int init_pv_ops(struct objtool_file *file) { static const char *pv_ops_tables[] = { "pv_ops", - "xen_mmu_ops", NULL, }; const char *pv_ops; -- 2.51.0 Instead of having the pv spinlock function definitions in paravirt.h, move them into the new header paravirt-spinlock.h. Signed-off-by: Juergen Gross <jgross@suse.com> --- V2: - use new header instead of qspinlock.h - use dedicated pv_ops_lock array - move more paravirt related lock code V3: - hide native_pv_lock_init() with CONFIG_SMP (kernel test robot) V4: - don't reference pv_ops_lock without CONFIG_PARAVIRT_SPINLOCKS (kernel test robot) V5: - move paravirt_set_cap() declaration into paravirt-base.h (kernel test robot) --- arch/x86/hyperv/hv_spinlock.c | 10 +- arch/x86/include/asm/paravirt-base.h | 6 + arch/x86/include/asm/paravirt-spinlock.h | 145 +++++++++++++++++++++++ arch/x86/include/asm/paravirt.h | 61 ---------- arch/x86/include/asm/paravirt_types.h | 17 --- arch/x86/include/asm/qspinlock.h | 87 +------------- arch/x86/kernel/Makefile | 2 +- arch/x86/kernel/kvm.c | 12 +- arch/x86/kernel/paravirt-spinlocks.c | 26 +++- arch/x86/kernel/paravirt.c | 21 ---- arch/x86/xen/spinlock.c | 10 +- tools/objtool/check.c | 1 + 12 files changed, 198 insertions(+), 200 deletions(-) create mode 100644 arch/x86/include/asm/paravirt-spinlock.h diff --git a/arch/x86/hyperv/hv_spinlock.c b/arch/x86/hyperv/hv_spinlock.c index XXXXXXX..XXXXXXX 100644 --- a/arch/x86/hyperv/hv_spinlock.c +++ b/arch/x86/hyperv/hv_spinlock.c @@ -XXX,XX +XXX,XX @@ void __init hv_init_spinlocks(void) pr_info("PV spinlocks enabled\n"); __pv_init_lock_hash(); - pv_ops.lock.queued_spin_lock_slowpath = __pv_queued_spin_lock_slowpath; - pv_ops.lock.queued_spin_unlock = PV_CALLEE_SAVE(__pv_queued_spin_unlock); - pv_ops.lock.wait = hv_qlock_wait; - pv_ops.lock.kick = hv_qlock_kick; - pv_ops.lock.vcpu_is_preempted = PV_CALLEE_SAVE(hv_vcpu_is_preempted); + pv_ops_lock.queued_spin_lock_slowpath = __pv_queued_spin_lock_slowpath; + pv_ops_lock.queued_spin_unlock = PV_CALLEE_SAVE(__pv_queued_spin_unlock); + pv_ops_lock.wait = hv_qlock_wait; + pv_ops_lock.kick = hv_qlock_kick; + pv_ops_lock.vcpu_is_preempted = PV_CALLEE_SAVE(hv_vcpu_is_preempted); } static __init int hv_parse_nopvspin(char *arg) diff --git a/arch/x86/include/asm/paravirt-base.h b/arch/x86/include/asm/paravirt-base.h index XXXXXXX..XXXXXXX 100644 --- a/arch/x86/include/asm/paravirt-base.h +++ b/arch/x86/include/asm/paravirt-base.h @@ -XXX,XX +XXX,XX @@ u64 _paravirt_ident_64(u64); #endif #define paravirt_nop ((void *)nop_func) +#ifdef CONFIG_PARAVIRT_SPINLOCKS +void paravirt_set_cap(void); +#else +static inline void paravirt_set_cap(void) { } +#endif + #endif /* _ASM_X86_PARAVIRT_BASE_H */ diff --git a/arch/x86/include/asm/paravirt-spinlock.h b/arch/x86/include/asm/paravirt-spinlock.h new file mode 100644 index XXXXXXX..XXXXXXX --- /dev/null +++ b/arch/x86/include/asm/paravirt-spinlock.h @@ -XXX,XX +XXX,XX @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef _ASM_X86_PARAVIRT_SPINLOCK_H +#define _ASM_X86_PARAVIRT_SPINLOCK_H + +#include <asm/paravirt_types.h> + +#ifdef CONFIG_SMP +#include <asm/spinlock_types.h> +#endif + +struct qspinlock; + +struct pv_lock_ops { + void (*queued_spin_lock_slowpath)(struct qspinlock *lock, u32 val); + struct paravirt_callee_save queued_spin_unlock; + + void (*wait)(u8 *ptr, u8 val); + void (*kick)(int cpu); + + struct paravirt_callee_save vcpu_is_preempted; +} __no_randomize_layout; + +extern struct pv_lock_ops pv_ops_lock; + +#ifdef CONFIG_PARAVIRT_SPINLOCKS +extern void native_queued_spin_lock_slowpath(struct qspinlock *lock, u32 val); +extern void __pv_init_lock_hash(void); +extern void __pv_queued_spin_lock_slowpath(struct qspinlock *lock, u32 val); +extern void __raw_callee_save___pv_queued_spin_unlock(struct qspinlock *lock); +extern bool nopvspin; + +static __always_inline void pv_queued_spin_lock_slowpath(struct qspinlock *lock, + u32 val) +{ + PVOP_VCALL2(pv_ops_lock, queued_spin_lock_slowpath, lock, val); +} + +static __always_inline void pv_queued_spin_unlock(struct qspinlock *lock) +{ + PVOP_ALT_VCALLEE1(pv_ops_lock, queued_spin_unlock, lock, + "movb $0, (%%" _ASM_ARG1 ");", + ALT_NOT(X86_FEATURE_PVUNLOCK)); +} + +static __always_inline bool pv_vcpu_is_preempted(long cpu) +{ + return PVOP_ALT_CALLEE1(bool, pv_ops_lock, vcpu_is_preempted, cpu, + "xor %%" _ASM_AX ", %%" _ASM_AX ";", + ALT_NOT(X86_FEATURE_VCPUPREEMPT)); +} + +#define queued_spin_unlock queued_spin_unlock +/** + * queued_spin_unlock - release a queued spinlock + * @lock : Pointer to queued spinlock structure + * + * A smp_store_release() on the least-significant byte. + */ +static inline void native_queued_spin_unlock(struct qspinlock *lock) +{ + smp_store_release(&lock->locked, 0); +} + +static inline void queued_spin_lock_slowpath(struct qspinlock *lock, u32 val) +{ + pv_queued_spin_lock_slowpath(lock, val); +} + +static inline void queued_spin_unlock(struct qspinlock *lock) +{ + kcsan_release(); + pv_queued_spin_unlock(lock); +} + +#define vcpu_is_preempted vcpu_is_preempted +static inline bool vcpu_is_preempted(long cpu) +{ + return pv_vcpu_is_preempted(cpu); +} + +static __always_inline void pv_wait(u8 *ptr, u8 val) +{ + PVOP_VCALL2(pv_ops_lock, wait, ptr, val); +} + +static __always_inline void pv_kick(int cpu) +{ + PVOP_VCALL1(pv_ops_lock, kick, cpu); +} + +void __raw_callee_save___native_queued_spin_unlock(struct qspinlock *lock); +bool __raw_callee_save___native_vcpu_is_preempted(long cpu); +#endif /* CONFIG_PARAVIRT_SPINLOCKS */ + +void __init native_pv_lock_init(void); +__visible void __native_queued_spin_unlock(struct qspinlock *lock); +bool pv_is_native_spin_unlock(void); +__visible bool __native_vcpu_is_preempted(long cpu); +bool pv_is_native_vcpu_is_preempted(void); + +/* + * virt_spin_lock_key - disables by default the virt_spin_lock() hijack. + * + * Native (and PV wanting native due to vCPU pinning) should keep this key + * disabled. Native does not touch the key. + * + * When in a guest then native_pv_lock_init() enables the key first and + * KVM/XEN might conditionally disable it later in the boot process again. + */ +DECLARE_STATIC_KEY_FALSE(virt_spin_lock_key); + +/* + * Shortcut for the queued_spin_lock_slowpath() function that allows + * virt to hijack it. + * + * Returns: + * true - lock has been negotiated, all done; + * false - queued_spin_lock_slowpath() will do its thing. + */ +#define virt_spin_lock virt_spin_lock +static inline bool virt_spin_lock(struct qspinlock *lock) +{ + int val; + + if (!static_branch_likely(&virt_spin_lock_key)) + return false; + + /* + * On hypervisors without PARAVIRT_SPINLOCKS support we fall + * back to a Test-and-Set spinlock, because fair locks have + * horrible lock 'holder' preemption issues. + */ + + __retry: + val = atomic_read(&lock->val); + + if (val || !atomic_try_cmpxchg(&lock->val, &val, _Q_LOCKED_VAL)) { + cpu_relax(); + goto __retry; + } + + return true; +} + +#endif /* _ASM_X86_PARAVIRT_SPINLOCK_H */ diff --git a/arch/x86/include/asm/paravirt.h b/arch/x86/include/asm/paravirt.h index XXXXXXX..XXXXXXX 100644 --- a/arch/x86/include/asm/paravirt.h +++ b/arch/x86/include/asm/paravirt.h @@ -XXX,XX +XXX,XX @@ #include <linux/cpumask.h> #include <asm/frame.h> -__visible void __native_queued_spin_unlock(struct qspinlock *lock); -bool pv_is_native_spin_unlock(void); -__visible bool __native_vcpu_is_preempted(long cpu); -bool pv_is_native_vcpu_is_preempted(void); - -#ifdef CONFIG_PARAVIRT_SPINLOCKS -void __init paravirt_set_cap(void); -#endif - /* The paravirtualized I/O functions */ static inline void slow_down_io(void) { @@ -XXX,XX +XXX,XX @@ static inline void __set_fixmap(unsigned /* enum fixed_addresses */ idx, { pv_ops.mmu.set_fixmap(idx, phys, flags); } -#endif - -#if defined(CONFIG_SMP) && defined(CONFIG_PARAVIRT_SPINLOCKS) - -static __always_inline void pv_queued_spin_lock_slowpath(struct qspinlock *lock, - u32 val) -{ - PVOP_VCALL2(pv_ops, lock.queued_spin_lock_slowpath, lock, val); -} - -static __always_inline void pv_queued_spin_unlock(struct qspinlock *lock) -{ - PVOP_ALT_VCALLEE1(pv_ops, lock.queued_spin_unlock, lock, - "movb $0, (%%" _ASM_ARG1 ");", - ALT_NOT(X86_FEATURE_PVUNLOCK)); -} - -static __always_inline void pv_wait(u8 *ptr, u8 val) -{ - PVOP_VCALL2(pv_ops, lock.wait, ptr, val); -} - -static __always_inline void pv_kick(int cpu) -{ - PVOP_VCALL1(pv_ops, lock.kick, cpu); -} - -static __always_inline bool pv_vcpu_is_preempted(long cpu) -{ - return PVOP_ALT_CALLEE1(bool, pv_ops, lock.vcpu_is_preempted, cpu, - "xor %%" _ASM_AX ", %%" _ASM_AX ";", - ALT_NOT(X86_FEATURE_VCPUPREEMPT)); -} -void __raw_callee_save___native_queued_spin_unlock(struct qspinlock *lock); -bool __raw_callee_save___native_vcpu_is_preempted(long cpu); - -#endif /* SMP && PARAVIRT_SPINLOCKS */ - -#ifdef CONFIG_PARAVIRT_XXL static __always_inline unsigned long arch_local_save_flags(void) { return PVOP_ALT_CALLEE0(unsigned long, pv_ops, irq.save_fl, "pushf; pop %%rax;", @@ -XXX,XX +XXX,XX @@ static __always_inline unsigned long arch_local_irq_save(void) } #endif -void native_pv_lock_init(void) __init; - #else /* __ASSEMBLER__ */ #ifdef CONFIG_X86_64 @@ -XXX,XX +XXX,XX @@ void native_pv_lock_init(void) __init; #endif /* __ASSEMBLER__ */ #else /* CONFIG_PARAVIRT */ # define default_banner x86_init_noop - -#ifndef __ASSEMBLER__ -static inline void native_pv_lock_init(void) -{ -} -#endif #endif /* !CONFIG_PARAVIRT */ #ifndef __ASSEMBLER__ @@ -XXX,XX +XXX,XX @@ static inline void paravirt_arch_exit_mmap(struct mm_struct *mm) } #endif -#ifndef CONFIG_PARAVIRT_SPINLOCKS -static inline void paravirt_set_cap(void) -{ -} -#endif #endif /* __ASSEMBLER__ */ #endif /* _ASM_X86_PARAVIRT_H */ diff --git a/arch/x86/include/asm/paravirt_types.h b/arch/x86/include/asm/paravirt_types.h index XXXXXXX..XXXXXXX 100644 --- a/arch/x86/include/asm/paravirt_types.h +++ b/arch/x86/include/asm/paravirt_types.h @@ -XXX,XX +XXX,XX @@ struct pv_mmu_ops { #endif } __no_randomize_layout; -#ifdef CONFIG_SMP -#include <asm/spinlock_types.h> -#endif - -struct qspinlock; - -struct pv_lock_ops { - void (*queued_spin_lock_slowpath)(struct qspinlock *lock, u32 val); - struct paravirt_callee_save queued_spin_unlock; - - void (*wait)(u8 *ptr, u8 val); - void (*kick)(int cpu); - - struct paravirt_callee_save vcpu_is_preempted; -} __no_randomize_layout; - /* This contains all the paravirt structures: we get a convenient * number for each function using the offset which we use to indicate * what to patch. */ @@ -XXX,XX +XXX,XX @@ struct paravirt_patch_template { struct pv_cpu_ops cpu; struct pv_irq_ops irq; struct pv_mmu_ops mmu; - struct pv_lock_ops lock; } __no_randomize_layout; extern struct paravirt_patch_template pv_ops; diff --git a/arch/x86/include/asm/qspinlock.h b/arch/x86/include/asm/qspinlock.h index XXXXXXX..XXXXXXX 100644 --- a/arch/x86/include/asm/qspinlock.h +++ b/arch/x86/include/asm/qspinlock.h @@ -XXX,XX +XXX,XX @@ #include <asm-generic/qspinlock_types.h> #include <asm/paravirt.h> #include <asm/rmwcc.h> +#ifdef CONFIG_PARAVIRT +#include <asm/paravirt-spinlock.h> +#endif #define _Q_PENDING_LOOPS (1 << 9) @@ -XXX,XX +XXX,XX @@ static __always_inline u32 queued_fetch_set_pending_acquire(struct qspinlock *lo return val; } -#ifdef CONFIG_PARAVIRT_SPINLOCKS -extern void native_queued_spin_lock_slowpath(struct qspinlock *lock, u32 val); -extern void __pv_init_lock_hash(void); -extern void __pv_queued_spin_lock_slowpath(struct qspinlock *lock, u32 val); -extern void __raw_callee_save___pv_queued_spin_unlock(struct qspinlock *lock); -extern bool nopvspin; - -#define queued_spin_unlock queued_spin_unlock -/** - * queued_spin_unlock - release a queued spinlock - * @lock : Pointer to queued spinlock structure - * - * A smp_store_release() on the least-significant byte. - */ -static inline void native_queued_spin_unlock(struct qspinlock *lock) -{ - smp_store_release(&lock->locked, 0); -} - -static inline void queued_spin_lock_slowpath(struct qspinlock *lock, u32 val) -{ - pv_queued_spin_lock_slowpath(lock, val); -} - -static inline void queued_spin_unlock(struct qspinlock *lock) -{ - kcsan_release(); - pv_queued_spin_unlock(lock); -} - -#define vcpu_is_preempted vcpu_is_preempted -static inline bool vcpu_is_preempted(long cpu) -{ - return pv_vcpu_is_preempted(cpu); -} +#ifndef CONFIG_PARAVIRT +static inline void native_pv_lock_init(void) { } #endif -#ifdef CONFIG_PARAVIRT -/* - * virt_spin_lock_key - disables by default the virt_spin_lock() hijack. - * - * Native (and PV wanting native due to vCPU pinning) should keep this key - * disabled. Native does not touch the key. - * - * When in a guest then native_pv_lock_init() enables the key first and - * KVM/XEN might conditionally disable it later in the boot process again. - */ -DECLARE_STATIC_KEY_FALSE(virt_spin_lock_key); - -/* - * Shortcut for the queued_spin_lock_slowpath() function that allows - * virt to hijack it. - * - * Returns: - * true - lock has been negotiated, all done; - * false - queued_spin_lock_slowpath() will do its thing. - */ -#define virt_spin_lock virt_spin_lock -static inline bool virt_spin_lock(struct qspinlock *lock) -{ - int val; - - if (!static_branch_likely(&virt_spin_lock_key)) - return false; - - /* - * On hypervisors without PARAVIRT_SPINLOCKS support we fall - * back to a Test-and-Set spinlock, because fair locks have - * horrible lock 'holder' preemption issues. - */ - - __retry: - val = atomic_read(&lock->val); - - if (val || !atomic_try_cmpxchg(&lock->val, &val, _Q_LOCKED_VAL)) { - cpu_relax(); - goto __retry; - } - - return true; -} - -#endif /* CONFIG_PARAVIRT */ - #include <asm-generic/qspinlock.h> #endif /* _ASM_X86_QSPINLOCK_H */ diff --git a/arch/x86/kernel/Makefile b/arch/x86/kernel/Makefile index XXXXXXX..XXXXXXX 100644 --- a/arch/x86/kernel/Makefile +++ b/arch/x86/kernel/Makefile @@ -XXX,XX +XXX,XX @@ obj-$(CONFIG_DEBUG_NMI_SELFTEST) += nmi_selftest.o obj-$(CONFIG_KVM_GUEST) += kvm.o kvmclock.o obj-$(CONFIG_PARAVIRT) += paravirt.o -obj-$(CONFIG_PARAVIRT_SPINLOCKS)+= paravirt-spinlocks.o +obj-$(CONFIG_PARAVIRT) += paravirt-spinlocks.o obj-$(CONFIG_PARAVIRT_CLOCK) += pvclock.o obj-$(CONFIG_X86_PMEM_LEGACY_DEVICE) += pmem.o diff --git a/arch/x86/kernel/kvm.c b/arch/x86/kernel/kvm.c index XXXXXXX..XXXXXXX 100644 --- a/arch/x86/kernel/kvm.c +++ b/arch/x86/kernel/kvm.c @@ -XXX,XX +XXX,XX @@ static void __init kvm_guest_init(void) has_steal_clock = 1; static_call_update(pv_steal_clock, kvm_steal_clock); - pv_ops.lock.vcpu_is_preempted = +#ifdef CONFIG_PARAVIRT_SPINLOCKS + pv_ops_lock.vcpu_is_preempted = PV_CALLEE_SAVE(__kvm_vcpu_is_preempted); +#endif } if (kvm_para_has_feature(KVM_FEATURE_PV_EOI)) @@ -XXX,XX +XXX,XX @@ void __init kvm_spinlock_init(void) pr_info("PV spinlocks enabled\n"); __pv_init_lock_hash(); - pv_ops.lock.queued_spin_lock_slowpath = __pv_queued_spin_lock_slowpath; - pv_ops.lock.queued_spin_unlock = + pv_ops_lock.queued_spin_lock_slowpath = __pv_queued_spin_lock_slowpath; + pv_ops_lock.queued_spin_unlock = PV_CALLEE_SAVE(__pv_queued_spin_unlock); - pv_ops.lock.wait = kvm_wait; - pv_ops.lock.kick = kvm_kick_cpu; + pv_ops_lock.wait = kvm_wait; + pv_ops_lock.kick = kvm_kick_cpu; /* * When PV spinlock is enabled which is preferred over diff --git a/arch/x86/kernel/paravirt-spinlocks.c b/arch/x86/kernel/paravirt-spinlocks.c index XXXXXXX..XXXXXXX 100644 --- a/arch/x86/kernel/paravirt-spinlocks.c +++ b/arch/x86/kernel/paravirt-spinlocks.c @@ -XXX,XX +XXX,XX @@ * Split spinlock implementation out into its own file, so it can be * compiled in a FTRACE-compatible way. */ +#include <linux/static_call.h> #include <linux/spinlock.h> #include <linux/export.h> #include <linux/jump_label.h> -#include <asm/paravirt.h> +DEFINE_STATIC_KEY_FALSE(virt_spin_lock_key); +#ifdef CONFIG_SMP +void __init native_pv_lock_init(void) +{ + if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) + static_branch_enable(&virt_spin_lock_key); +} +#endif + +#ifdef CONFIG_PARAVIRT_SPINLOCKS __visible void __native_queued_spin_unlock(struct qspinlock *lock) { native_queued_spin_unlock(lock); @@ -XXX,XX +XXX,XX @@ PV_CALLEE_SAVE_REGS_THUNK(__native_queued_spin_unlock); bool pv_is_native_spin_unlock(void) { - return pv_ops.lock.queued_spin_unlock.func == + return pv_ops_lock.queued_spin_unlock.func == __raw_callee_save___native_queued_spin_unlock; } @@ -XXX,XX +XXX,XX @@ PV_CALLEE_SAVE_REGS_THUNK(__native_vcpu_is_preempted); bool pv_is_native_vcpu_is_preempted(void) { - return pv_ops.lock.vcpu_is_preempted.func == + return pv_ops_lock.vcpu_is_preempted.func == __raw_callee_save___native_vcpu_is_preempted; } @@ -XXX,XX +XXX,XX @@ void __init paravirt_set_cap(void) if (!pv_is_native_vcpu_is_preempted()) setup_force_cpu_cap(X86_FEATURE_VCPUPREEMPT); } + +struct pv_lock_ops pv_ops_lock = { + .queued_spin_lock_slowpath = native_queued_spin_lock_slowpath, + .queued_spin_unlock = PV_CALLEE_SAVE(__native_queued_spin_unlock), + .wait = paravirt_nop, + .kick = paravirt_nop, + .vcpu_is_preempted = PV_CALLEE_SAVE(__native_vcpu_is_preempted), +}; +EXPORT_SYMBOL(pv_ops_lock); +#endif diff --git a/arch/x86/kernel/paravirt.c b/arch/x86/kernel/paravirt.c index XXXXXXX..XXXXXXX 100644 --- a/arch/x86/kernel/paravirt.c +++ b/arch/x86/kernel/paravirt.c @@ -XXX,XX +XXX,XX @@ DEFINE_ASM_FUNC(pv_native_irq_enable, "sti", .noinstr.text); DEFINE_ASM_FUNC(pv_native_read_cr2, "mov %cr2, %rax", .noinstr.text); #endif -DEFINE_STATIC_KEY_FALSE(virt_spin_lock_key); - -void __init native_pv_lock_init(void) -{ - if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) - static_branch_enable(&virt_spin_lock_key); -} - static noinstr void pv_native_safe_halt(void) { native_safe_halt(); @@ -XXX,XX +XXX,XX @@ struct paravirt_patch_template pv_ops = { .mmu.set_fixmap = native_set_fixmap, #endif /* CONFIG_PARAVIRT_XXL */ - -#if defined(CONFIG_PARAVIRT_SPINLOCKS) - /* Lock ops. */ -#ifdef CONFIG_SMP - .lock.queued_spin_lock_slowpath = native_queued_spin_lock_slowpath, - .lock.queued_spin_unlock = - PV_CALLEE_SAVE(__native_queued_spin_unlock), - .lock.wait = paravirt_nop, - .lock.kick = paravirt_nop, - .lock.vcpu_is_preempted = - PV_CALLEE_SAVE(__native_vcpu_is_preempted), -#endif /* SMP */ -#endif }; #ifdef CONFIG_PARAVIRT_XXL diff --git a/arch/x86/xen/spinlock.c b/arch/x86/xen/spinlock.c index XXXXXXX..XXXXXXX 100644 --- a/arch/x86/xen/spinlock.c +++ b/arch/x86/xen/spinlock.c @@ -XXX,XX +XXX,XX @@ void __init xen_init_spinlocks(void) printk(KERN_DEBUG "xen: PV spinlocks enabled\n"); __pv_init_lock_hash(); - pv_ops.lock.queued_spin_lock_slowpath = __pv_queued_spin_lock_slowpath; - pv_ops.lock.queued_spin_unlock = + pv_ops_lock.queued_spin_lock_slowpath = __pv_queued_spin_lock_slowpath; + pv_ops_lock.queued_spin_unlock = PV_CALLEE_SAVE(__pv_queued_spin_unlock); - pv_ops.lock.wait = xen_qlock_wait; - pv_ops.lock.kick = xen_qlock_kick; - pv_ops.lock.vcpu_is_preempted = PV_CALLEE_SAVE(xen_vcpu_stolen); + pv_ops_lock.wait = xen_qlock_wait; + pv_ops_lock.kick = xen_qlock_kick; + pv_ops_lock.vcpu_is_preempted = PV_CALLEE_SAVE(xen_vcpu_stolen); } diff --git a/tools/objtool/check.c b/tools/objtool/check.c index XXXXXXX..XXXXXXX 100644 --- a/tools/objtool/check.c +++ b/tools/objtool/check.c @@ -XXX,XX +XXX,XX @@ static struct { int idx_off; } pv_ops_tables[] = { { .name = "pv_ops", }, + { .name = "pv_ops_lock", }, { .name = NULL, .idx_off = -1 } }; -- 2.51.0