Merge tag 'x86_paravirt_for_v7.0_rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

+3

arch/Kconfig

··· 1056 1056 Archs need to ensure they use a high enough resolution clock to 1057 1057 support irq time accounting and then call enable_sched_clock_irqtime(). 1058 1058 1059 + config HAVE_PV_STEAL_CLOCK_GEN 1060 + bool 1061 + 1059 1062 config HAVE_MOVE_PUD 1060 1063 bool 1061 1064 help

+1

arch/arm/Kconfig

··· 1320 1320 1321 1321 config PARAVIRT 1322 1322 bool "Enable paravirtualization code" 1323 + select HAVE_PV_STEAL_CLOCK_GEN 1323 1324 help 1324 1325 This changes the kernel so it can modify itself when it is run 1325 1326 under a hypervisor, potentially improving performance significantly

-22

arch/arm/include/asm/paravirt.h

··· 1 - /* SPDX-License-Identifier: GPL-2.0 */ 2 - #ifndef _ASM_ARM_PARAVIRT_H 3 - #define _ASM_ARM_PARAVIRT_H 4 - 5 - #ifdef CONFIG_PARAVIRT 6 - #include <linux/static_call_types.h> 7 - 8 - struct static_key; 9 - extern struct static_key paravirt_steal_enabled; 10 - extern struct static_key paravirt_steal_rq_enabled; 11 - 12 - u64 dummy_steal_clock(int cpu); 13 - 14 - DECLARE_STATIC_CALL(pv_steal_clock, dummy_steal_clock); 15 - 16 - static inline u64 paravirt_steal_clock(int cpu) 17 - { 18 - return static_call(pv_steal_clock)(cpu); 19 - } 20 - #endif 21 - 22 - #endif

-1

arch/arm/include/asm/paravirt_api_clock.h

··· 1 - #include <asm/paravirt.h>

-1

arch/arm/kernel/Makefile

··· 83 83 obj-$(CONFIG_ARM_CPU_TOPOLOGY) += topology.o 84 84 obj-$(CONFIG_VDSO) += vdso.o 85 85 obj-$(CONFIG_EFI) += efi.o 86 - obj-$(CONFIG_PARAVIRT) += paravirt.o 87 86 88 87 obj-y += head$(MMUEXT).o 89 88 obj-$(CONFIG_DEBUG_LL) += debug.o

-23

arch/arm/kernel/paravirt.c

··· 1 - // SPDX-License-Identifier: GPL-2.0-only 2 - /* 3 - * 4 - * Copyright (C) 2013 Citrix Systems 5 - * 6 - * Author: Stefano Stabellini <stefano.stabellini@eu.citrix.com> 7 - */ 8 - 9 - #include <linux/export.h> 10 - #include <linux/jump_label.h> 11 - #include <linux/types.h> 12 - #include <linux/static_call.h> 13 - #include <asm/paravirt.h> 14 - 15 - struct static_key paravirt_steal_enabled; 16 - struct static_key paravirt_steal_rq_enabled; 17 - 18 - static u64 native_steal_clock(int cpu) 19 - { 20 - return 0; 21 - } 22 - 23 - DEFINE_STATIC_CALL(pv_steal_clock, native_steal_clock);

+1

arch/arm64/Kconfig

··· 1580 1580 1581 1581 config PARAVIRT 1582 1582 bool "Enable paravirtualization code" 1583 + select HAVE_PV_STEAL_CLOCK_GEN 1583 1584 help 1584 1585 This changes the kernel so it can modify itself when it is run 1585 1586 under a hypervisor, potentially improving performance significantly

-14

arch/arm64/include/asm/paravirt.h

··· 3 3 #define _ASM_ARM64_PARAVIRT_H 4 4 5 5 #ifdef CONFIG_PARAVIRT 6 - #include <linux/static_call_types.h> 7 - 8 - struct static_key; 9 - extern struct static_key paravirt_steal_enabled; 10 - extern struct static_key paravirt_steal_rq_enabled; 11 - 12 - u64 dummy_steal_clock(int cpu); 13 - 14 - DECLARE_STATIC_CALL(pv_steal_clock, dummy_steal_clock); 15 - 16 - static inline u64 paravirt_steal_clock(int cpu) 17 - { 18 - return static_call(pv_steal_clock)(cpu); 19 - } 20 6 21 7 int __init pv_time_init(void); 22 8

-1

arch/arm64/include/asm/paravirt_api_clock.h

··· 1 - #include <asm/paravirt.h>

+1 -10

arch/arm64/kernel/paravirt.c

··· 19 19 #include <linux/slab.h> 20 20 #include <linux/types.h> 21 21 #include <linux/static_call.h> 22 + #include <linux/sched/cputime.h> 22 23 23 24 #include <asm/paravirt.h> 24 25 #include <asm/pvclock-abi.h> 25 26 #include <asm/smp_plat.h> 26 - 27 - struct static_key paravirt_steal_enabled; 28 - struct static_key paravirt_steal_rq_enabled; 29 - 30 - static u64 native_steal_clock(int cpu) 31 - { 32 - return 0; 33 - } 34 - 35 - DEFINE_STATIC_CALL(pv_steal_clock, native_steal_clock); 36 27 37 28 struct pv_time_stolen_time_region { 38 29 struct pvclock_vcpu_stolen_time __rcu *kaddr;

+1

arch/loongarch/Kconfig

··· 687 687 config PARAVIRT 688 688 bool "Enable paravirtualization code" 689 689 depends on AS_HAS_LVZ_EXTENSION 690 + select HAVE_PV_STEAL_CLOCK_GEN 690 691 help 691 692 This changes the kernel so it can modify itself when it is run 692 693 under a hypervisor, potentially improving performance significantly

-13

arch/loongarch/include/asm/paravirt.h

··· 4 4 5 5 #ifdef CONFIG_PARAVIRT 6 6 7 - #include <linux/static_call_types.h> 8 - struct static_key; 9 - extern struct static_key paravirt_steal_enabled; 10 - extern struct static_key paravirt_steal_rq_enabled; 11 - 12 - u64 dummy_steal_clock(int cpu); 13 - DECLARE_STATIC_CALL(pv_steal_clock, dummy_steal_clock); 14 - 15 - static inline u64 paravirt_steal_clock(int cpu) 16 - { 17 - return static_call(pv_steal_clock)(cpu); 18 - } 19 - 20 7 int __init pv_ipi_init(void); 21 8 int __init pv_time_init(void); 22 9 int __init pv_spinlock_init(void);

-1

arch/loongarch/include/asm/paravirt_api_clock.h

··· 1 - #include <asm/paravirt.h>

+1 -9

arch/loongarch/kernel/paravirt.c

··· 6 6 #include <linux/kvm_para.h> 7 7 #include <linux/reboot.h> 8 8 #include <linux/static_call.h> 9 + #include <linux/sched/cputime.h> 9 10 #include <asm/paravirt.h> 10 11 11 12 static int has_steal_clock; 12 - struct static_key paravirt_steal_enabled; 13 - struct static_key paravirt_steal_rq_enabled; 14 13 static DEFINE_PER_CPU(struct kvm_steal_time, steal_time) __aligned(64); 15 14 DEFINE_STATIC_KEY_FALSE(virt_spin_lock_key); 16 - 17 - static u64 native_steal_clock(int cpu) 18 - { 19 - return 0; 20 - } 21 - 22 - DEFINE_STATIC_CALL(pv_steal_clock, native_steal_clock); 23 15 24 16 static bool steal_acc = true; 25 17

-3

arch/powerpc/include/asm/paravirt.h

··· 23 23 } 24 24 25 25 #ifdef CONFIG_PARAVIRT_TIME_ACCOUNTING 26 - extern struct static_key paravirt_steal_enabled; 27 - extern struct static_key paravirt_steal_rq_enabled; 28 - 29 26 u64 pseries_paravirt_steal_clock(int cpu); 30 27 31 28 static inline u64 paravirt_steal_clock(int cpu)

-2

arch/powerpc/include/asm/paravirt_api_clock.h

··· 1 - /* SPDX-License-Identifier: GPL-2.0 */ 2 - #include <asm/paravirt.h>

+1 -3

arch/powerpc/platforms/pseries/setup.c

··· 42 42 #include <linux/memblock.h> 43 43 #include <linux/swiotlb.h> 44 44 #include <linux/seq_buf.h> 45 + #include <linux/sched/cputime.h> 45 46 46 47 #include <asm/mmu.h> 47 48 #include <asm/processor.h> ··· 84 83 EXPORT_SYMBOL(shared_processor); 85 84 86 85 #ifdef CONFIG_PARAVIRT_TIME_ACCOUNTING 87 - struct static_key paravirt_steal_enabled; 88 - struct static_key paravirt_steal_rq_enabled; 89 - 90 86 static bool steal_acc = true; 91 87 static int __init parse_no_stealacc(char *arg) 92 88 {

+1

arch/riscv/Kconfig

··· 1111 1111 config PARAVIRT 1112 1112 bool "Enable paravirtualization code" 1113 1113 depends on RISCV_SBI 1114 + select HAVE_PV_STEAL_CLOCK_GEN 1114 1115 help 1115 1116 This changes the kernel so it can modify itself when it is run 1116 1117 under a hypervisor, potentially improving performance significantly

-14

arch/riscv/include/asm/paravirt.h

··· 3 3 #define _ASM_RISCV_PARAVIRT_H 4 4 5 5 #ifdef CONFIG_PARAVIRT 6 - #include <linux/static_call_types.h> 7 - 8 - struct static_key; 9 - extern struct static_key paravirt_steal_enabled; 10 - extern struct static_key paravirt_steal_rq_enabled; 11 - 12 - u64 dummy_steal_clock(int cpu); 13 - 14 - DECLARE_STATIC_CALL(pv_steal_clock, dummy_steal_clock); 15 - 16 - static inline u64 paravirt_steal_clock(int cpu) 17 - { 18 - return static_call(pv_steal_clock)(cpu); 19 - } 20 6 21 7 int __init pv_time_init(void); 22 8

-1

arch/riscv/include/asm/paravirt_api_clock.h

··· 1 - #include <asm/paravirt.h>

+1 -10

arch/riscv/kernel/paravirt.c

··· 16 16 #include <linux/printk.h> 17 17 #include <linux/static_call.h> 18 18 #include <linux/types.h> 19 + #include <linux/sched/cputime.h> 19 20 20 21 #include <asm/barrier.h> 21 22 #include <asm/page.h> 22 23 #include <asm/paravirt.h> 23 24 #include <asm/sbi.h> 24 - 25 - struct static_key paravirt_steal_enabled; 26 - struct static_key paravirt_steal_rq_enabled; 27 - 28 - static u64 native_steal_clock(int cpu) 29 - { 30 - return 0; 31 - } 32 - 33 - DEFINE_STATIC_CALL(pv_steal_clock, native_steal_clock); 34 25 35 26 static bool steal_acc = true; 36 27 static int __init parse_no_stealacc(char *arg)

+1 -7

arch/x86/Kconfig

··· 813 813 config PARAVIRT 814 814 bool "Enable paravirtualization code" 815 815 depends on HAVE_STATIC_CALL 816 + select HAVE_PV_STEAL_CLOCK_GEN 816 817 help 817 818 This changes the kernel so it can modify itself when it is run 818 819 under a hypervisor, potentially improving performance significantly ··· 823 822 config PARAVIRT_XXL 824 823 bool 825 824 depends on X86_64 826 - 827 - config PARAVIRT_DEBUG 828 - bool "paravirt-ops debugging" 829 - depends on PARAVIRT && DEBUG_KERNEL 830 - help 831 - Enable to debug paravirt_ops internals. Specifically, BUG if 832 - a paravirt_op is missing when it is called. 833 825 834 826 config PARAVIRT_SPINLOCKS 835 827 bool "Paravirtualization layer for spinlocks"

-1

arch/x86/entry/entry_64.S

··· 31 31 #include <asm/hw_irq.h> 32 32 #include <asm/page_types.h> 33 33 #include <asm/irqflags.h> 34 - #include <asm/paravirt.h> 35 34 #include <asm/percpu.h> 36 35 #include <asm/asm.h> 37 36 #include <asm/smap.h>

-1

arch/x86/entry/vsyscall/vsyscall_64.c

··· 37 37 #include <asm/unistd.h> 38 38 #include <asm/fixmap.h> 39 39 #include <asm/traps.h> 40 - #include <asm/paravirt.h> 41 40 42 41 #define CREATE_TRACE_POINTS 43 42 #include "vsyscall_trace.h"

+5 -6

arch/x86/hyperv/hv_spinlock.c

··· 13 13 #include <linux/spinlock.h> 14 14 15 15 #include <asm/mshyperv.h> 16 - #include <asm/paravirt.h> 17 16 #include <asm/apic.h> 18 17 #include <asm/msr.h> 19 18 ··· 78 79 pr_info("PV spinlocks enabled\n"); 79 80 80 81 __pv_init_lock_hash(); 81 - pv_ops.lock.queued_spin_lock_slowpath = __pv_queued_spin_lock_slowpath; 82 - pv_ops.lock.queued_spin_unlock = PV_CALLEE_SAVE(__pv_queued_spin_unlock); 83 - pv_ops.lock.wait = hv_qlock_wait; 84 - pv_ops.lock.kick = hv_qlock_kick; 85 - pv_ops.lock.vcpu_is_preempted = PV_CALLEE_SAVE(hv_vcpu_is_preempted); 82 + pv_ops_lock.queued_spin_lock_slowpath = __pv_queued_spin_lock_slowpath; 83 + pv_ops_lock.queued_spin_unlock = PV_CALLEE_SAVE(__pv_queued_spin_unlock); 84 + pv_ops_lock.wait = hv_qlock_wait; 85 + pv_ops_lock.kick = hv_qlock_kick; 86 + pv_ops_lock.vcpu_is_preempted = PV_CALLEE_SAVE(hv_vcpu_is_preempted); 86 87 } 87 88 88 89 static __init int hv_parse_nopvspin(char *arg)

-4

arch/x86/include/asm/apic.h

··· 90 90 /* 91 91 * Basic functions accessing APICs. 92 92 */ 93 - #ifdef CONFIG_PARAVIRT 94 - #include <asm/paravirt.h> 95 - #endif 96 - 97 93 static inline void native_apic_mem_write(u32 reg, u32 v) 98 94 { 99 95 volatile u32 *addr = (volatile u32 *)(APIC_BASE + reg);

-1

arch/x86/include/asm/highmem.h

··· 24 24 #include <linux/interrupt.h> 25 25 #include <linux/threads.h> 26 26 #include <asm/tlbflush.h> 27 - #include <asm/paravirt.h> 28 27 #include <asm/fixmap.h> 29 28 #include <asm/pgtable_areas.h> 30 29

-1

arch/x86/include/asm/mshyperv.h

··· 8 8 #include <linux/io.h> 9 9 #include <linux/static_call.h> 10 10 #include <asm/nospec-branch.h> 11 - #include <asm/paravirt.h> 12 11 #include <asm/msr.h> 13 12 #include <hyperv/hvhdk.h> 14 13 #include <asm/fpu/types.h>

+35

arch/x86/include/asm/paravirt-base.h

··· 1 + /* SPDX-License-Identifier: GPL-2.0-only */ 2 + 3 + #ifndef _ASM_X86_PARAVIRT_BASE_H 4 + #define _ASM_X86_PARAVIRT_BASE_H 5 + 6 + /* 7 + * Wrapper type for pointers to code which uses the non-standard 8 + * calling convention. See PV_CALL_SAVE_REGS_THUNK below. 9 + */ 10 + struct paravirt_callee_save { 11 + void *func; 12 + }; 13 + 14 + struct pv_info { 15 + #ifdef CONFIG_PARAVIRT_XXL 16 + u16 extra_user_64bit_cs; /* __USER_CS if none */ 17 + #endif 18 + const char *name; 19 + }; 20 + 21 + void default_banner(void); 22 + extern struct pv_info pv_info; 23 + unsigned long paravirt_ret0(void); 24 + #ifdef CONFIG_PARAVIRT_XXL 25 + u64 _paravirt_ident_64(u64); 26 + #endif 27 + #define paravirt_nop ((void *)nop_func) 28 + 29 + #ifdef CONFIG_PARAVIRT_SPINLOCKS 30 + void paravirt_set_cap(void); 31 + #else 32 + static inline void paravirt_set_cap(void) { } 33 + #endif 34 + 35 + #endif /* _ASM_X86_PARAVIRT_BASE_H */

+145

arch/x86/include/asm/paravirt-spinlock.h

··· 1 + /* SPDX-License-Identifier: GPL-2.0-only */ 2 + #ifndef _ASM_X86_PARAVIRT_SPINLOCK_H 3 + #define _ASM_X86_PARAVIRT_SPINLOCK_H 4 + 5 + #include <asm/paravirt_types.h> 6 + 7 + #ifdef CONFIG_SMP 8 + #include <asm/spinlock_types.h> 9 + #endif 10 + 11 + struct qspinlock; 12 + 13 + struct pv_lock_ops { 14 + void (*queued_spin_lock_slowpath)(struct qspinlock *lock, u32 val); 15 + struct paravirt_callee_save queued_spin_unlock; 16 + 17 + void (*wait)(u8 *ptr, u8 val); 18 + void (*kick)(int cpu); 19 + 20 + struct paravirt_callee_save vcpu_is_preempted; 21 + } __no_randomize_layout; 22 + 23 + extern struct pv_lock_ops pv_ops_lock; 24 + 25 + #ifdef CONFIG_PARAVIRT_SPINLOCKS 26 + extern void native_queued_spin_lock_slowpath(struct qspinlock *lock, u32 val); 27 + extern void __pv_init_lock_hash(void); 28 + extern void __pv_queued_spin_lock_slowpath(struct qspinlock *lock, u32 val); 29 + extern void __raw_callee_save___pv_queued_spin_unlock(struct qspinlock *lock); 30 + extern bool nopvspin; 31 + 32 + static __always_inline void pv_queued_spin_lock_slowpath(struct qspinlock *lock, 33 + u32 val) 34 + { 35 + PVOP_VCALL2(pv_ops_lock, queued_spin_lock_slowpath, lock, val); 36 + } 37 + 38 + static __always_inline void pv_queued_spin_unlock(struct qspinlock *lock) 39 + { 40 + PVOP_ALT_VCALLEE1(pv_ops_lock, queued_spin_unlock, lock, 41 + "movb $0, (%%" _ASM_ARG1 ")", 42 + ALT_NOT(X86_FEATURE_PVUNLOCK)); 43 + } 44 + 45 + static __always_inline bool pv_vcpu_is_preempted(long cpu) 46 + { 47 + return PVOP_ALT_CALLEE1(bool, pv_ops_lock, vcpu_is_preempted, cpu, 48 + "xor %%eax, %%eax", 49 + ALT_NOT(X86_FEATURE_VCPUPREEMPT)); 50 + } 51 + 52 + #define queued_spin_unlock queued_spin_unlock 53 + /** 54 + * queued_spin_unlock - release a queued spinlock 55 + * @lock : Pointer to queued spinlock structure 56 + * 57 + * A smp_store_release() on the least-significant byte. 58 + */ 59 + static inline void native_queued_spin_unlock(struct qspinlock *lock) 60 + { 61 + smp_store_release(&lock->locked, 0); 62 + } 63 + 64 + static inline void queued_spin_lock_slowpath(struct qspinlock *lock, u32 val) 65 + { 66 + pv_queued_spin_lock_slowpath(lock, val); 67 + } 68 + 69 + static inline void queued_spin_unlock(struct qspinlock *lock) 70 + { 71 + kcsan_release(); 72 + pv_queued_spin_unlock(lock); 73 + } 74 + 75 + #define vcpu_is_preempted vcpu_is_preempted 76 + static inline bool vcpu_is_preempted(long cpu) 77 + { 78 + return pv_vcpu_is_preempted(cpu); 79 + } 80 + 81 + static __always_inline void pv_wait(u8 *ptr, u8 val) 82 + { 83 + PVOP_VCALL2(pv_ops_lock, wait, ptr, val); 84 + } 85 + 86 + static __always_inline void pv_kick(int cpu) 87 + { 88 + PVOP_VCALL1(pv_ops_lock, kick, cpu); 89 + } 90 + 91 + void __raw_callee_save___native_queued_spin_unlock(struct qspinlock *lock); 92 + bool __raw_callee_save___native_vcpu_is_preempted(long cpu); 93 + #endif /* CONFIG_PARAVIRT_SPINLOCKS */ 94 + 95 + void __init native_pv_lock_init(void); 96 + __visible void __native_queued_spin_unlock(struct qspinlock *lock); 97 + bool pv_is_native_spin_unlock(void); 98 + __visible bool __native_vcpu_is_preempted(long cpu); 99 + bool pv_is_native_vcpu_is_preempted(void); 100 + 101 + /* 102 + * virt_spin_lock_key - disables by default the virt_spin_lock() hijack. 103 + * 104 + * Native (and PV wanting native due to vCPU pinning) should keep this key 105 + * disabled. Native does not touch the key. 106 + * 107 + * When in a guest then native_pv_lock_init() enables the key first and 108 + * KVM/XEN might conditionally disable it later in the boot process again. 109 + */ 110 + DECLARE_STATIC_KEY_FALSE(virt_spin_lock_key); 111 + 112 + /* 113 + * Shortcut for the queued_spin_lock_slowpath() function that allows 114 + * virt to hijack it. 115 + * 116 + * Returns: 117 + * true - lock has been negotiated, all done; 118 + * false - queued_spin_lock_slowpath() will do its thing. 119 + */ 120 + #define virt_spin_lock virt_spin_lock 121 + static inline bool virt_spin_lock(struct qspinlock *lock) 122 + { 123 + int val; 124 + 125 + if (!static_branch_likely(&virt_spin_lock_key)) 126 + return false; 127 + 128 + /* 129 + * On hypervisors without PARAVIRT_SPINLOCKS support we fall 130 + * back to a Test-and-Set spinlock, because fair locks have 131 + * horrible lock 'holder' preemption issues. 132 + */ 133 + 134 + __retry: 135 + val = atomic_read(&lock->val); 136 + 137 + if (val || !atomic_try_cmpxchg(&lock->val, &val, _Q_LOCKED_VAL)) { 138 + cpu_relax(); 139 + goto __retry; 140 + } 141 + 142 + return true; 143 + } 144 + 145 + #endif /* _ASM_X86_PARAVIRT_SPINLOCK_H */

+85 -256

arch/x86/include/asm/paravirt.h

··· 4 4 /* Various instructions on x86 need to be replaced for 5 5 * para-virtualization: those hooks are defined here. */ 6 6 7 - #include <asm/paravirt_types.h> 8 - 9 7 #ifndef __ASSEMBLER__ 10 - struct mm_struct; 8 + #include <asm/paravirt-base.h> 11 9 #endif 10 + #include <asm/paravirt_types.h> 12 11 13 12 #ifdef CONFIG_PARAVIRT 14 13 #include <asm/pgtable_types.h> ··· 15 16 #include <asm/nospec-branch.h> 16 17 17 18 #ifndef __ASSEMBLER__ 18 - #include <linux/bug.h> 19 19 #include <linux/types.h> 20 20 #include <linux/cpumask.h> 21 - #include <linux/static_call_types.h> 22 21 #include <asm/frame.h> 23 - 24 - u64 dummy_steal_clock(int cpu); 25 - u64 dummy_sched_clock(void); 26 - 27 - DECLARE_STATIC_CALL(pv_steal_clock, dummy_steal_clock); 28 - DECLARE_STATIC_CALL(pv_sched_clock, dummy_sched_clock); 29 - 30 - void paravirt_set_sched_clock(u64 (*func)(void)); 31 - 32 - static __always_inline u64 paravirt_sched_clock(void) 33 - { 34 - return static_call(pv_sched_clock)(); 35 - } 36 - 37 - struct static_key; 38 - extern struct static_key paravirt_steal_enabled; 39 - extern struct static_key paravirt_steal_rq_enabled; 40 - 41 - __visible void __native_queued_spin_unlock(struct qspinlock *lock); 42 - bool pv_is_native_spin_unlock(void); 43 - __visible bool __native_vcpu_is_preempted(long cpu); 44 - bool pv_is_native_vcpu_is_preempted(void); 45 - 46 - static inline u64 paravirt_steal_clock(int cpu) 47 - { 48 - return static_call(pv_steal_clock)(cpu); 49 - } 50 - 51 - #ifdef CONFIG_PARAVIRT_SPINLOCKS 52 - void __init paravirt_set_cap(void); 53 - #endif 54 22 55 23 /* The paravirtualized I/O functions */ 56 24 static inline void slow_down_io(void) 57 25 { 58 - PVOP_VCALL0(cpu.io_delay); 26 + PVOP_VCALL0(pv_ops, cpu.io_delay); 59 27 #ifdef REALLY_SLOW_IO 60 - PVOP_VCALL0(cpu.io_delay); 61 - PVOP_VCALL0(cpu.io_delay); 62 - PVOP_VCALL0(cpu.io_delay); 28 + PVOP_VCALL0(pv_ops, cpu.io_delay); 29 + PVOP_VCALL0(pv_ops, cpu.io_delay); 30 + PVOP_VCALL0(pv_ops, cpu.io_delay); 63 31 #endif 64 32 } 65 33 ··· 38 72 39 73 static inline void __flush_tlb_local(void) 40 74 { 41 - PVOP_VCALL0(mmu.flush_tlb_user); 75 + PVOP_VCALL0(pv_ops, mmu.flush_tlb_user); 42 76 } 43 77 44 78 static inline void __flush_tlb_global(void) 45 79 { 46 - PVOP_VCALL0(mmu.flush_tlb_kernel); 80 + PVOP_VCALL0(pv_ops, mmu.flush_tlb_kernel); 47 81 } 48 82 49 83 static inline void __flush_tlb_one_user(unsigned long addr) 50 84 { 51 - PVOP_VCALL1(mmu.flush_tlb_one_user, addr); 85 + PVOP_VCALL1(pv_ops, mmu.flush_tlb_one_user, addr); 52 86 } 53 87 54 88 static inline void __flush_tlb_multi(const struct cpumask *cpumask, 55 89 const struct flush_tlb_info *info) 56 90 { 57 - PVOP_VCALL2(mmu.flush_tlb_multi, cpumask, info); 91 + PVOP_VCALL2(pv_ops, mmu.flush_tlb_multi, cpumask, info); 58 92 } 59 93 60 94 static inline void paravirt_arch_exit_mmap(struct mm_struct *mm) 61 95 { 62 - PVOP_VCALL1(mmu.exit_mmap, mm); 96 + PVOP_VCALL1(pv_ops, mmu.exit_mmap, mm); 63 97 } 64 98 65 99 static inline void notify_page_enc_status_changed(unsigned long pfn, 66 100 int npages, bool enc) 67 101 { 68 - PVOP_VCALL3(mmu.notify_page_enc_status_changed, pfn, npages, enc); 102 + PVOP_VCALL3(pv_ops, mmu.notify_page_enc_status_changed, pfn, npages, enc); 69 103 } 70 104 71 105 static __always_inline void arch_safe_halt(void) 72 106 { 73 - PVOP_VCALL0(irq.safe_halt); 107 + PVOP_VCALL0(pv_ops, irq.safe_halt); 74 108 } 75 109 76 110 static inline void halt(void) 77 111 { 78 - PVOP_VCALL0(irq.halt); 112 + PVOP_VCALL0(pv_ops, irq.halt); 79 113 } 80 114 81 115 #ifdef CONFIG_PARAVIRT_XXL 82 116 static inline void load_sp0(unsigned long sp0) 83 117 { 84 - PVOP_VCALL1(cpu.load_sp0, sp0); 118 + PVOP_VCALL1(pv_ops, cpu.load_sp0, sp0); 85 119 } 86 120 87 121 /* The paravirtualized CPUID instruction. */ 88 122 static inline void __cpuid(unsigned int *eax, unsigned int *ebx, 89 123 unsigned int *ecx, unsigned int *edx) 90 124 { 91 - PVOP_VCALL4(cpu.cpuid, eax, ebx, ecx, edx); 125 + PVOP_VCALL4(pv_ops, cpu.cpuid, eax, ebx, ecx, edx); 92 126 } 93 127 94 128 /* ··· 96 130 */ 97 131 static __always_inline unsigned long paravirt_get_debugreg(int reg) 98 132 { 99 - return PVOP_CALL1(unsigned long, cpu.get_debugreg, reg); 133 + return PVOP_CALL1(unsigned long, pv_ops, cpu.get_debugreg, reg); 100 134 } 101 135 #define get_debugreg(var, reg) var = paravirt_get_debugreg(reg) 102 136 static __always_inline void set_debugreg(unsigned long val, int reg) 103 137 { 104 - PVOP_VCALL2(cpu.set_debugreg, reg, val); 138 + PVOP_VCALL2(pv_ops, cpu.set_debugreg, reg, val); 105 139 } 106 140 107 141 static inline unsigned long read_cr0(void) 108 142 { 109 - return PVOP_CALL0(unsigned long, cpu.read_cr0); 143 + return PVOP_CALL0(unsigned long, pv_ops, cpu.read_cr0); 110 144 } 111 145 112 146 static inline void write_cr0(unsigned long x) 113 147 { 114 - PVOP_VCALL1(cpu.write_cr0, x); 148 + PVOP_VCALL1(pv_ops, cpu.write_cr0, x); 115 149 } 116 150 117 151 static __always_inline unsigned long read_cr2(void) 118 152 { 119 - return PVOP_ALT_CALLEE0(unsigned long, mmu.read_cr2, 120 - "mov %%cr2, %%rax;", ALT_NOT_XEN); 153 + return PVOP_ALT_CALLEE0(unsigned long, pv_ops, mmu.read_cr2, 154 + "mov %%cr2, %%rax", ALT_NOT_XEN); 121 155 } 122 156 123 157 static __always_inline void write_cr2(unsigned long x) 124 158 { 125 - PVOP_VCALL1(mmu.write_cr2, x); 159 + PVOP_VCALL1(pv_ops, mmu.write_cr2, x); 126 160 } 127 161 128 162 static inline unsigned long __read_cr3(void) 129 163 { 130 - return PVOP_ALT_CALL0(unsigned long, mmu.read_cr3, 131 - "mov %%cr3, %%rax;", ALT_NOT_XEN); 164 + return PVOP_ALT_CALL0(unsigned long, pv_ops, mmu.read_cr3, 165 + "mov %%cr3, %%rax", ALT_NOT_XEN); 132 166 } 133 167 134 168 static inline void write_cr3(unsigned long x) 135 169 { 136 - PVOP_ALT_VCALL1(mmu.write_cr3, x, "mov %%rdi, %%cr3", ALT_NOT_XEN); 170 + PVOP_ALT_VCALL1(pv_ops, mmu.write_cr3, x, "mov %%rdi, %%cr3", ALT_NOT_XEN); 137 171 } 138 172 139 173 static inline void __write_cr4(unsigned long x) 140 174 { 141 - PVOP_VCALL1(cpu.write_cr4, x); 175 + PVOP_VCALL1(pv_ops, cpu.write_cr4, x); 142 176 } 143 177 144 178 static inline u64 paravirt_read_msr(u32 msr) 145 179 { 146 - return PVOP_CALL1(u64, cpu.read_msr, msr); 180 + return PVOP_CALL1(u64, pv_ops, cpu.read_msr, msr); 147 181 } 148 182 149 183 static inline void paravirt_write_msr(u32 msr, u64 val) 150 184 { 151 - PVOP_VCALL2(cpu.write_msr, msr, val); 185 + PVOP_VCALL2(pv_ops, cpu.write_msr, msr, val); 152 186 } 153 187 154 188 static inline int paravirt_read_msr_safe(u32 msr, u64 *val) 155 189 { 156 - return PVOP_CALL2(int, cpu.read_msr_safe, msr, val); 190 + return PVOP_CALL2(int, pv_ops, cpu.read_msr_safe, msr, val); 157 191 } 158 192 159 193 static inline int paravirt_write_msr_safe(u32 msr, u64 val) 160 194 { 161 - return PVOP_CALL2(int, cpu.write_msr_safe, msr, val); 195 + return PVOP_CALL2(int, pv_ops, cpu.write_msr_safe, msr, val); 162 196 } 163 197 164 198 #define rdmsr(msr, val1, val2) \ ··· 205 239 206 240 static __always_inline u64 rdpmc(int counter) 207 241 { 208 - return PVOP_CALL1(u64, cpu.read_pmc, counter); 242 + return PVOP_CALL1(u64, pv_ops, cpu.read_pmc, counter); 209 243 } 210 244 211 245 static inline void paravirt_alloc_ldt(struct desc_struct *ldt, unsigned entries) 212 246 { 213 - PVOP_VCALL2(cpu.alloc_ldt, ldt, entries); 247 + PVOP_VCALL2(pv_ops, cpu.alloc_ldt, ldt, entries); 214 248 } 215 249 216 250 static inline void paravirt_free_ldt(struct desc_struct *ldt, unsigned entries) 217 251 { 218 - PVOP_VCALL2(cpu.free_ldt, ldt, entries); 252 + PVOP_VCALL2(pv_ops, cpu.free_ldt, ldt, entries); 219 253 } 220 254 221 255 static inline void load_TR_desc(void) 222 256 { 223 - PVOP_VCALL0(cpu.load_tr_desc); 257 + PVOP_VCALL0(pv_ops, cpu.load_tr_desc); 224 258 } 225 259 static inline void load_gdt(const struct desc_ptr *dtr) 226 260 { 227 - PVOP_VCALL1(cpu.load_gdt, dtr); 261 + PVOP_VCALL1(pv_ops, cpu.load_gdt, dtr); 228 262 } 229 263 static inline void load_idt(const struct desc_ptr *dtr) 230 264 { 231 - PVOP_VCALL1(cpu.load_idt, dtr); 265 + PVOP_VCALL1(pv_ops, cpu.load_idt, dtr); 232 266 } 233 267 static inline void set_ldt(const void *addr, unsigned entries) 234 268 { 235 - PVOP_VCALL2(cpu.set_ldt, addr, entries); 269 + PVOP_VCALL2(pv_ops, cpu.set_ldt, addr, entries); 236 270 } 237 271 static inline unsigned long paravirt_store_tr(void) 238 272 { 239 - return PVOP_CALL0(unsigned long, cpu.store_tr); 273 + return PVOP_CALL0(unsigned long, pv_ops, cpu.store_tr); 240 274 } 241 275 242 276 #define store_tr(tr) ((tr) = paravirt_store_tr()) 243 277 static inline void load_TLS(struct thread_struct *t, unsigned cpu) 244 278 { 245 - PVOP_VCALL2(cpu.load_tls, t, cpu); 279 + PVOP_VCALL2(pv_ops, cpu.load_tls, t, cpu); 246 280 } 247 281 248 282 static inline void load_gs_index(unsigned int gs) 249 283 { 250 - PVOP_VCALL1(cpu.load_gs_index, gs); 284 + PVOP_VCALL1(pv_ops, cpu.load_gs_index, gs); 251 285 } 252 286 253 287 static inline void write_ldt_entry(struct desc_struct *dt, int entry, 254 288 const void *desc) 255 289 { 256 - PVOP_VCALL3(cpu.write_ldt_entry, dt, entry, desc); 290 + PVOP_VCALL3(pv_ops, cpu.write_ldt_entry, dt, entry, desc); 257 291 } 258 292 259 293 static inline void write_gdt_entry(struct desc_struct *dt, int entry, 260 294 void *desc, int type) 261 295 { 262 - PVOP_VCALL4(cpu.write_gdt_entry, dt, entry, desc, type); 296 + PVOP_VCALL4(pv_ops, cpu.write_gdt_entry, dt, entry, desc, type); 263 297 } 264 298 265 299 static inline void write_idt_entry(gate_desc *dt, int entry, const gate_desc *g) 266 300 { 267 - PVOP_VCALL3(cpu.write_idt_entry, dt, entry, g); 301 + PVOP_VCALL3(pv_ops, cpu.write_idt_entry, dt, entry, g); 268 302 } 269 303 270 304 #ifdef CONFIG_X86_IOPL_IOPERM 271 305 static inline void tss_invalidate_io_bitmap(void) 272 306 { 273 - PVOP_VCALL0(cpu.invalidate_io_bitmap); 307 + PVOP_VCALL0(pv_ops, cpu.invalidate_io_bitmap); 274 308 } 275 309 276 310 static inline void tss_update_io_bitmap(void) 277 311 { 278 - PVOP_VCALL0(cpu.update_io_bitmap); 312 + PVOP_VCALL0(pv_ops, cpu.update_io_bitmap); 279 313 } 280 314 #endif 281 315 282 316 static inline void paravirt_enter_mmap(struct mm_struct *next) 283 317 { 284 - PVOP_VCALL1(mmu.enter_mmap, next); 318 + PVOP_VCALL1(pv_ops, mmu.enter_mmap, next); 285 319 } 286 320 287 321 static inline int paravirt_pgd_alloc(struct mm_struct *mm) 288 322 { 289 - return PVOP_CALL1(int, mmu.pgd_alloc, mm); 323 + return PVOP_CALL1(int, pv_ops, mmu.pgd_alloc, mm); 290 324 } 291 325 292 326 static inline void paravirt_pgd_free(struct mm_struct *mm, pgd_t *pgd) 293 327 { 294 - PVOP_VCALL2(mmu.pgd_free, mm, pgd); 328 + PVOP_VCALL2(pv_ops, mmu.pgd_free, mm, pgd); 295 329 } 296 330 297 331 static inline void paravirt_alloc_pte(struct mm_struct *mm, unsigned long pfn) 298 332 { 299 - PVOP_VCALL2(mmu.alloc_pte, mm, pfn); 333 + PVOP_VCALL2(pv_ops, mmu.alloc_pte, mm, pfn); 300 334 } 301 335 static inline void paravirt_release_pte(unsigned long pfn) 302 336 { 303 - PVOP_VCALL1(mmu.release_pte, pfn); 337 + PVOP_VCALL1(pv_ops, mmu.release_pte, pfn); 304 338 } 305 339 306 340 static inline void paravirt_alloc_pmd(struct mm_struct *mm, unsigned long pfn) 307 341 { 308 - PVOP_VCALL2(mmu.alloc_pmd, mm, pfn); 342 + PVOP_VCALL2(pv_ops, mmu.alloc_pmd, mm, pfn); 309 343 } 310 344 311 345 static inline void paravirt_release_pmd(unsigned long pfn) 312 346 { 313 - PVOP_VCALL1(mmu.release_pmd, pfn); 347 + PVOP_VCALL1(pv_ops, mmu.release_pmd, pfn); 314 348 } 315 349 316 350 static inline void paravirt_alloc_pud(struct mm_struct *mm, unsigned long pfn) 317 351 { 318 - PVOP_VCALL2(mmu.alloc_pud, mm, pfn); 352 + PVOP_VCALL2(pv_ops, mmu.alloc_pud, mm, pfn); 319 353 } 320 354 static inline void paravirt_release_pud(unsigned long pfn) 321 355 { 322 - PVOP_VCALL1(mmu.release_pud, pfn); 356 + PVOP_VCALL1(pv_ops, mmu.release_pud, pfn); 323 357 } 324 358 325 359 static inline void paravirt_alloc_p4d(struct mm_struct *mm, unsigned long pfn) 326 360 { 327 - PVOP_VCALL2(mmu.alloc_p4d, mm, pfn); 361 + PVOP_VCALL2(pv_ops, mmu.alloc_p4d, mm, pfn); 328 362 } 329 363 330 364 static inline void paravirt_release_p4d(unsigned long pfn) 331 365 { 332 - PVOP_VCALL1(mmu.release_p4d, pfn); 366 + PVOP_VCALL1(pv_ops, mmu.release_p4d, pfn); 333 367 } 334 368 335 369 static inline pte_t __pte(pteval_t val) 336 370 { 337 - return (pte_t) { PVOP_ALT_CALLEE1(pteval_t, mmu.make_pte, val, 371 + return (pte_t) { PVOP_ALT_CALLEE1(pteval_t, pv_ops, mmu.make_pte, val, 338 372 "mov %%rdi, %%rax", ALT_NOT_XEN) }; 339 373 } 340 374 341 375 static inline pteval_t pte_val(pte_t pte) 342 376 { 343 - return PVOP_ALT_CALLEE1(pteval_t, mmu.pte_val, pte.pte, 377 + return PVOP_ALT_CALLEE1(pteval_t, pv_ops, mmu.pte_val, pte.pte, 344 378 "mov %%rdi, %%rax", ALT_NOT_XEN); 345 379 } 346 380 347 381 static inline pgd_t __pgd(pgdval_t val) 348 382 { 349 - return (pgd_t) { PVOP_ALT_CALLEE1(pgdval_t, mmu.make_pgd, val, 383 + return (pgd_t) { PVOP_ALT_CALLEE1(pgdval_t, pv_ops, mmu.make_pgd, val, 350 384 "mov %%rdi, %%rax", ALT_NOT_XEN) }; 351 385 } 352 386 353 387 static inline pgdval_t pgd_val(pgd_t pgd) 354 388 { 355 - return PVOP_ALT_CALLEE1(pgdval_t, mmu.pgd_val, pgd.pgd, 389 + return PVOP_ALT_CALLEE1(pgdval_t, pv_ops, mmu.pgd_val, pgd.pgd, 356 390 "mov %%rdi, %%rax", ALT_NOT_XEN); 357 391 } 358 392 ··· 362 396 { 363 397 pteval_t ret; 364 398 365 - ret = PVOP_CALL3(pteval_t, mmu.ptep_modify_prot_start, vma, addr, ptep); 399 + ret = PVOP_CALL3(pteval_t, pv_ops, mmu.ptep_modify_prot_start, vma, addr, ptep); 366 400 367 401 return (pte_t) { .pte = ret }; 368 402 } ··· 371 405 pte_t *ptep, pte_t old_pte, pte_t pte) 372 406 { 373 407 374 - PVOP_VCALL4(mmu.ptep_modify_prot_commit, vma, addr, ptep, pte.pte); 408 + PVOP_VCALL4(pv_ops, mmu.ptep_modify_prot_commit, vma, addr, ptep, pte.pte); 375 409 } 376 410 377 411 static inline void set_pte(pte_t *ptep, pte_t pte) 378 412 { 379 - PVOP_VCALL2(mmu.set_pte, ptep, pte.pte); 413 + PVOP_VCALL2(pv_ops, mmu.set_pte, ptep, pte.pte); 380 414 } 381 415 382 416 static inline void set_pmd(pmd_t *pmdp, pmd_t pmd) 383 417 { 384 - PVOP_VCALL2(mmu.set_pmd, pmdp, native_pmd_val(pmd)); 418 + PVOP_VCALL2(pv_ops, mmu.set_pmd, pmdp, native_pmd_val(pmd)); 385 419 } 386 420 387 421 static inline pmd_t __pmd(pmdval_t val) 388 422 { 389 - return (pmd_t) { PVOP_ALT_CALLEE1(pmdval_t, mmu.make_pmd, val, 423 + return (pmd_t) { PVOP_ALT_CALLEE1(pmdval_t, pv_ops, mmu.make_pmd, val, 390 424 "mov %%rdi, %%rax", ALT_NOT_XEN) }; 391 425 } 392 426 393 427 static inline pmdval_t pmd_val(pmd_t pmd) 394 428 { 395 - return PVOP_ALT_CALLEE1(pmdval_t, mmu.pmd_val, pmd.pmd, 429 + return PVOP_ALT_CALLEE1(pmdval_t, pv_ops, mmu.pmd_val, pmd.pmd, 396 430 "mov %%rdi, %%rax", ALT_NOT_XEN); 397 431 } 398 432 399 433 static inline void set_pud(pud_t *pudp, pud_t pud) 400 434 { 401 - PVOP_VCALL2(mmu.set_pud, pudp, native_pud_val(pud)); 435 + PVOP_VCALL2(pv_ops, mmu.set_pud, pudp, native_pud_val(pud)); 402 436 } 403 437 404 438 static inline pud_t __pud(pudval_t val) 405 439 { 406 440 pudval_t ret; 407 441 408 - ret = PVOP_ALT_CALLEE1(pudval_t, mmu.make_pud, val, 442 + ret = PVOP_ALT_CALLEE1(pudval_t, pv_ops, mmu.make_pud, val, 409 443 "mov %%rdi, %%rax", ALT_NOT_XEN); 410 444 411 445 return (pud_t) { ret }; ··· 413 447 414 448 static inline pudval_t pud_val(pud_t pud) 415 449 { 416 - return PVOP_ALT_CALLEE1(pudval_t, mmu.pud_val, pud.pud, 450 + return PVOP_ALT_CALLEE1(pudval_t, pv_ops, mmu.pud_val, pud.pud, 417 451 "mov %%rdi, %%rax", ALT_NOT_XEN); 418 452 } 419 453 ··· 426 460 { 427 461 p4dval_t val = native_p4d_val(p4d); 428 462 429 - PVOP_VCALL2(mmu.set_p4d, p4dp, val); 463 + PVOP_VCALL2(pv_ops, mmu.set_p4d, p4dp, val); 430 464 } 431 465 432 466 static inline p4d_t __p4d(p4dval_t val) 433 467 { 434 - p4dval_t ret = PVOP_ALT_CALLEE1(p4dval_t, mmu.make_p4d, val, 468 + p4dval_t ret = PVOP_ALT_CALLEE1(p4dval_t, pv_ops, mmu.make_p4d, val, 435 469 "mov %%rdi, %%rax", ALT_NOT_XEN); 436 470 437 471 return (p4d_t) { ret }; ··· 439 473 440 474 static inline p4dval_t p4d_val(p4d_t p4d) 441 475 { 442 - return PVOP_ALT_CALLEE1(p4dval_t, mmu.p4d_val, p4d.p4d, 476 + return PVOP_ALT_CALLEE1(p4dval_t, pv_ops, mmu.p4d_val, p4d.p4d, 443 477 "mov %%rdi, %%rax", ALT_NOT_XEN); 444 478 } 445 479 446 480 static inline void __set_pgd(pgd_t *pgdp, pgd_t pgd) 447 481 { 448 - PVOP_VCALL2(mmu.set_pgd, pgdp, native_pgd_val(pgd)); 482 + PVOP_VCALL2(pv_ops, mmu.set_pgd, pgdp, native_pgd_val(pgd)); 449 483 } 450 484 451 485 #define set_pgd(pgdp, pgdval) do { \ ··· 484 518 #define __HAVE_ARCH_START_CONTEXT_SWITCH 485 519 static inline void arch_start_context_switch(struct task_struct *prev) 486 520 { 487 - PVOP_VCALL1(cpu.start_context_switch, prev); 521 + PVOP_VCALL1(pv_ops, cpu.start_context_switch, prev); 488 522 } 489 523 490 524 static inline void arch_end_context_switch(struct task_struct *next) 491 525 { 492 - PVOP_VCALL1(cpu.end_context_switch, next); 526 + PVOP_VCALL1(pv_ops, cpu.end_context_switch, next); 493 527 } 494 528 495 529 #define __HAVE_ARCH_ENTER_LAZY_MMU_MODE 496 530 static inline void arch_enter_lazy_mmu_mode(void) 497 531 { 498 - PVOP_VCALL0(mmu.lazy_mode.enter); 532 + PVOP_VCALL0(pv_ops, mmu.lazy_mode.enter); 499 533 } 500 534 501 535 static inline void arch_leave_lazy_mmu_mode(void) 502 536 { 503 - PVOP_VCALL0(mmu.lazy_mode.leave); 537 + PVOP_VCALL0(pv_ops, mmu.lazy_mode.leave); 504 538 } 505 539 506 540 static inline void arch_flush_lazy_mmu_mode(void) 507 541 { 508 - PVOP_VCALL0(mmu.lazy_mode.flush); 542 + PVOP_VCALL0(pv_ops, mmu.lazy_mode.flush); 509 543 } 510 544 511 545 static inline void __set_fixmap(unsigned /* enum fixed_addresses */ idx, ··· 513 547 { 514 548 pv_ops.mmu.set_fixmap(idx, phys, flags); 515 549 } 516 - #endif 517 550 518 - #if defined(CONFIG_SMP) && defined(CONFIG_PARAVIRT_SPINLOCKS) 519 - 520 - static __always_inline void pv_queued_spin_lock_slowpath(struct qspinlock *lock, 521 - u32 val) 522 - { 523 - PVOP_VCALL2(lock.queued_spin_lock_slowpath, lock, val); 524 - } 525 - 526 - static __always_inline void pv_queued_spin_unlock(struct qspinlock *lock) 527 - { 528 - PVOP_ALT_VCALLEE1(lock.queued_spin_unlock, lock, 529 - "movb $0, (%%" _ASM_ARG1 ");", 530 - ALT_NOT(X86_FEATURE_PVUNLOCK)); 531 - } 532 - 533 - static __always_inline void pv_wait(u8 *ptr, u8 val) 534 - { 535 - PVOP_VCALL2(lock.wait, ptr, val); 536 - } 537 - 538 - static __always_inline void pv_kick(int cpu) 539 - { 540 - PVOP_VCALL1(lock.kick, cpu); 541 - } 542 - 543 - static __always_inline bool pv_vcpu_is_preempted(long cpu) 544 - { 545 - return PVOP_ALT_CALLEE1(bool, lock.vcpu_is_preempted, cpu, 546 - "xor %%" _ASM_AX ", %%" _ASM_AX ";", 547 - ALT_NOT(X86_FEATURE_VCPUPREEMPT)); 548 - } 549 - 550 - void __raw_callee_save___native_queued_spin_unlock(struct qspinlock *lock); 551 - bool __raw_callee_save___native_vcpu_is_preempted(long cpu); 552 - 553 - #endif /* SMP && PARAVIRT_SPINLOCKS */ 554 - 555 - #ifdef CONFIG_X86_32 556 - /* save and restore all caller-save registers, except return value */ 557 - #define PV_SAVE_ALL_CALLER_REGS "pushl %ecx;" 558 - #define PV_RESTORE_ALL_CALLER_REGS "popl %ecx;" 559 - #else 560 - /* save and restore all caller-save registers, except return value */ 561 - #define PV_SAVE_ALL_CALLER_REGS \ 562 - "push %rcx;" \ 563 - "push %rdx;" \ 564 - "push %rsi;" \ 565 - "push %rdi;" \ 566 - "push %r8;" \ 567 - "push %r9;" \ 568 - "push %r10;" \ 569 - "push %r11;" 570 - #define PV_RESTORE_ALL_CALLER_REGS \ 571 - "pop %r11;" \ 572 - "pop %r10;" \ 573 - "pop %r9;" \ 574 - "pop %r8;" \ 575 - "pop %rdi;" \ 576 - "pop %rsi;" \ 577 - "pop %rdx;" \ 578 - "pop %rcx;" 579 - #endif 580 - 581 - /* 582 - * Generate a thunk around a function which saves all caller-save 583 - * registers except for the return value. This allows C functions to 584 - * be called from assembler code where fewer than normal registers are 585 - * available. It may also help code generation around calls from C 586 - * code if the common case doesn't use many registers. 587 - * 588 - * When a callee is wrapped in a thunk, the caller can assume that all 589 - * arg regs and all scratch registers are preserved across the 590 - * call. The return value in rax/eax will not be saved, even for void 591 - * functions. 592 - */ 593 - #define PV_THUNK_NAME(func) "__raw_callee_save_" #func 594 - #define __PV_CALLEE_SAVE_REGS_THUNK(func, section) \ 595 - extern typeof(func) __raw_callee_save_##func; \ 596 - \ 597 - asm(".pushsection " section ", \"ax\";" \ 598 - ".globl " PV_THUNK_NAME(func) ";" \ 599 - ".type " PV_THUNK_NAME(func) ", @function;" \ 600 - ASM_FUNC_ALIGN \ 601 - PV_THUNK_NAME(func) ":" \ 602 - ASM_ENDBR \ 603 - FRAME_BEGIN \ 604 - PV_SAVE_ALL_CALLER_REGS \ 605 - "call " #func ";" \ 606 - PV_RESTORE_ALL_CALLER_REGS \ 607 - FRAME_END \ 608 - ASM_RET \ 609 - ".size " PV_THUNK_NAME(func) ", .-" PV_THUNK_NAME(func) ";" \ 610 - ".popsection") 611 - 612 - #define PV_CALLEE_SAVE_REGS_THUNK(func) \ 613 - __PV_CALLEE_SAVE_REGS_THUNK(func, ".text") 614 - 615 - /* Get a reference to a callee-save function */ 616 - #define PV_CALLEE_SAVE(func) \ 617 - ((struct paravirt_callee_save) { __raw_callee_save_##func }) 618 - 619 - /* Promise that "func" already uses the right calling convention */ 620 - #define __PV_IS_CALLEE_SAVE(func) \ 621 - ((struct paravirt_callee_save) { func }) 622 - 623 - #ifdef CONFIG_PARAVIRT_XXL 624 551 static __always_inline unsigned long arch_local_save_flags(void) 625 552 { 626 - return PVOP_ALT_CALLEE0(unsigned long, irq.save_fl, "pushf; pop %%rax;", 553 + return PVOP_ALT_CALLEE0(unsigned long, pv_ops, irq.save_fl, "pushf; pop %%rax", 627 554 ALT_NOT_XEN); 628 555 } 629 556 630 557 static __always_inline void arch_local_irq_disable(void) 631 558 { 632 - PVOP_ALT_VCALLEE0(irq.irq_disable, "cli;", ALT_NOT_XEN); 559 + PVOP_ALT_VCALLEE0(pv_ops, irq.irq_disable, "cli", ALT_NOT_XEN); 633 560 } 634 561 635 562 static __always_inline void arch_local_irq_enable(void) 636 563 { 637 - PVOP_ALT_VCALLEE0(irq.irq_enable, "sti;", ALT_NOT_XEN); 564 + PVOP_ALT_VCALLEE0(pv_ops, irq.irq_enable, "sti", ALT_NOT_XEN); 638 565 } 639 566 640 567 static __always_inline unsigned long arch_local_irq_save(void) ··· 539 680 return f; 540 681 } 541 682 #endif 542 - 543 - 544 - /* Make sure as little as possible of this mess escapes. */ 545 - #undef PARAVIRT_CALL 546 - #undef __PVOP_CALL 547 - #undef __PVOP_VCALL 548 - #undef PVOP_VCALL0 549 - #undef PVOP_CALL0 550 - #undef PVOP_VCALL1 551 - #undef PVOP_CALL1 552 - #undef PVOP_VCALL2 553 - #undef PVOP_CALL2 554 - #undef PVOP_VCALL3 555 - #undef PVOP_CALL3 556 - #undef PVOP_VCALL4 557 - #undef PVOP_CALL4 558 - 559 - extern void default_banner(void); 560 - void native_pv_lock_init(void) __init; 561 683 562 684 #else /* __ASSEMBLER__ */ 563 685 ··· 553 713 call PARA_INDIRECT(pv_ops+PV_IRQ_save_fl); 554 714 .endm 555 715 556 - #define SAVE_FLAGS ALTERNATIVE_2 "PARA_IRQ_save_fl;", \ 557 - "ALT_CALL_INSTR;", ALT_CALL_ALWAYS, \ 558 - "pushf; pop %rax;", ALT_NOT_XEN 716 + #define SAVE_FLAGS ALTERNATIVE_2 "PARA_IRQ_save_fl", \ 717 + "ALT_CALL_INSTR", ALT_CALL_ALWAYS, \ 718 + "pushf; pop %rax", ALT_NOT_XEN 559 719 #endif 560 720 #endif /* CONFIG_PARAVIRT_XXL */ 561 721 #endif /* CONFIG_X86_64 */ ··· 563 723 #endif /* __ASSEMBLER__ */ 564 724 #else /* CONFIG_PARAVIRT */ 565 725 # define default_banner x86_init_noop 566 - 567 - #ifndef __ASSEMBLER__ 568 - static inline void native_pv_lock_init(void) 569 - { 570 - } 571 - #endif 572 726 #endif /* !CONFIG_PARAVIRT */ 573 727 574 728 #ifndef __ASSEMBLER__ ··· 578 744 } 579 745 #endif 580 746 581 - #ifndef CONFIG_PARAVIRT_SPINLOCKS 582 - static inline void paravirt_set_cap(void) 583 - { 584 - } 585 - #endif 586 747 #endif /* __ASSEMBLER__ */ 587 748 #endif /* _ASM_X86_PARAVIRT_H */

-1

arch/x86/include/asm/paravirt_api_clock.h

··· 1 - #include <asm/paravirt.h>

+136 -127

arch/x86/include/asm/paravirt_types.h

··· 7 7 #ifndef __ASSEMBLER__ 8 8 #include <linux/types.h> 9 9 10 + #include <asm/paravirt-base.h> 10 11 #include <asm/desc_defs.h> 11 12 #include <asm/pgtable_types.h> 12 13 #include <asm/nospec-branch.h> 13 14 14 - struct page; 15 15 struct thread_struct; 16 - struct desc_ptr; 17 - struct tss_struct; 18 16 struct mm_struct; 19 - struct desc_struct; 20 17 struct task_struct; 21 18 struct cpumask; 22 19 struct flush_tlb_info; 23 - struct mmu_gather; 24 20 struct vm_area_struct; 25 - 26 - /* 27 - * Wrapper type for pointers to code which uses the non-standard 28 - * calling convention. See PV_CALL_SAVE_REGS_THUNK below. 29 - */ 30 - struct paravirt_callee_save { 31 - void *func; 32 - }; 33 - 34 - /* general info */ 35 - struct pv_info { 36 - #ifdef CONFIG_PARAVIRT_XXL 37 - u16 extra_user_64bit_cs; /* __USER_CS if none */ 38 - #endif 39 - 40 - const char *name; 41 - }; 42 21 43 22 #ifdef CONFIG_PARAVIRT_XXL 44 23 struct pv_lazy_ops { ··· 184 205 #endif 185 206 } __no_randomize_layout; 186 207 187 - struct arch_spinlock; 188 - #ifdef CONFIG_SMP 189 - #include <asm/spinlock_types.h> 190 - #endif 191 - 192 - struct qspinlock; 193 - 194 - struct pv_lock_ops { 195 - void (*queued_spin_lock_slowpath)(struct qspinlock *lock, u32 val); 196 - struct paravirt_callee_save queued_spin_unlock; 197 - 198 - void (*wait)(u8 *ptr, u8 val); 199 - void (*kick)(int cpu); 200 - 201 - struct paravirt_callee_save vcpu_is_preempted; 202 - } __no_randomize_layout; 203 - 204 208 /* This contains all the paravirt structures: we get a convenient 205 209 * number for each function using the offset which we use to indicate 206 210 * what to patch. */ ··· 191 229 struct pv_cpu_ops cpu; 192 230 struct pv_irq_ops irq; 193 231 struct pv_mmu_ops mmu; 194 - struct pv_lock_ops lock; 195 232 } __no_randomize_layout; 196 233 197 - extern struct pv_info pv_info; 198 234 extern struct paravirt_patch_template pv_ops; 199 235 200 - #define paravirt_ptr(op) [paravirt_opptr] "m" (pv_ops.op) 236 + #define paravirt_ptr(array, op) [paravirt_opptr] "m" (array.op) 201 237 202 238 /* 203 239 * This generates an indirect call based on the operation type number. ··· 210 250 */ 211 251 #define PARAVIRT_CALL \ 212 252 ANNOTATE_RETPOLINE_SAFE "\n\t" \ 213 - "call *%[paravirt_opptr];" 253 + "call *%[paravirt_opptr]" 214 254 215 255 /* 216 256 * These macros are intended to wrap calls through one of the paravirt ··· 320 360 #define VEXTRA_CLOBBERS , "rax", "r8", "r9", "r10", "r11" 321 361 #endif /* CONFIG_X86_32 */ 322 362 323 - #ifdef CONFIG_PARAVIRT_DEBUG 324 - #define PVOP_TEST_NULL(op) BUG_ON(pv_ops.op == NULL) 325 - #else 326 - #define PVOP_TEST_NULL(op) ((void)pv_ops.op) 327 - #endif 328 - 329 363 #define PVOP_RETVAL(rettype) \ 330 364 ({ unsigned long __mask = ~0UL; \ 331 365 BUILD_BUG_ON(sizeof(rettype) > sizeof(unsigned long)); \ ··· 345 391 * feature is not active, the direct call is used as above via the 346 392 * ALT_FLAG_DIRECT_CALL special case and the "always on" feature. 347 393 */ 348 - #define ____PVOP_CALL(ret, op, call_clbr, extra_clbr, ...) \ 394 + #define ____PVOP_CALL(ret, array, op, call_clbr, extra_clbr, ...) \ 349 395 ({ \ 350 396 PVOP_CALL_ARGS; \ 351 - PVOP_TEST_NULL(op); \ 352 397 asm volatile(ALTERNATIVE(PARAVIRT_CALL, ALT_CALL_INSTR, \ 353 398 ALT_CALL_ALWAYS) \ 354 399 : call_clbr, ASM_CALL_CONSTRAINT \ 355 - : paravirt_ptr(op), \ 400 + : paravirt_ptr(array, op), \ 356 401 ##__VA_ARGS__ \ 357 402 : "memory", "cc" extra_clbr); \ 358 403 ret; \ 359 404 }) 360 405 361 - #define ____PVOP_ALT_CALL(ret, op, alt, cond, call_clbr, \ 406 + #define ____PVOP_ALT_CALL(ret, array, op, alt, cond, call_clbr, \ 362 407 extra_clbr, ...) \ 363 408 ({ \ 364 409 PVOP_CALL_ARGS; \ 365 - PVOP_TEST_NULL(op); \ 366 410 asm volatile(ALTERNATIVE_2(PARAVIRT_CALL, \ 367 411 ALT_CALL_INSTR, ALT_CALL_ALWAYS, \ 368 412 alt, cond) \ 369 413 : call_clbr, ASM_CALL_CONSTRAINT \ 370 - : paravirt_ptr(op), \ 414 + : paravirt_ptr(array, op), \ 371 415 ##__VA_ARGS__ \ 372 416 : "memory", "cc" extra_clbr); \ 373 417 ret; \ 374 418 }) 375 419 376 - #define __PVOP_CALL(rettype, op, ...) \ 377 - ____PVOP_CALL(PVOP_RETVAL(rettype), op, \ 420 + #define __PVOP_CALL(rettype, array, op, ...) \ 421 + ____PVOP_CALL(PVOP_RETVAL(rettype), array, op, \ 378 422 PVOP_CALL_CLOBBERS, EXTRA_CLOBBERS, ##__VA_ARGS__) 379 423 380 - #define __PVOP_ALT_CALL(rettype, op, alt, cond, ...) \ 381 - ____PVOP_ALT_CALL(PVOP_RETVAL(rettype), op, alt, cond, \ 424 + #define __PVOP_ALT_CALL(rettype, array, op, alt, cond, ...) \ 425 + ____PVOP_ALT_CALL(PVOP_RETVAL(rettype), array, op, alt, cond, \ 382 426 PVOP_CALL_CLOBBERS, EXTRA_CLOBBERS, \ 383 427 ##__VA_ARGS__) 384 428 385 - #define __PVOP_CALLEESAVE(rettype, op, ...) \ 386 - ____PVOP_CALL(PVOP_RETVAL(rettype), op.func, \ 429 + #define __PVOP_CALLEESAVE(rettype, array, op, ...) \ 430 + ____PVOP_CALL(PVOP_RETVAL(rettype), array, op.func, \ 387 431 PVOP_CALLEE_CLOBBERS, , ##__VA_ARGS__) 388 432 389 - #define __PVOP_ALT_CALLEESAVE(rettype, op, alt, cond, ...) \ 390 - ____PVOP_ALT_CALL(PVOP_RETVAL(rettype), op.func, alt, cond, \ 433 + #define __PVOP_ALT_CALLEESAVE(rettype, array, op, alt, cond, ...) \ 434 + ____PVOP_ALT_CALL(PVOP_RETVAL(rettype), array, op.func, alt, cond, \ 391 435 PVOP_CALLEE_CLOBBERS, , ##__VA_ARGS__) 392 436 393 437 394 - #define __PVOP_VCALL(op, ...) \ 395 - (void)____PVOP_CALL(, op, PVOP_VCALL_CLOBBERS, \ 438 + #define __PVOP_VCALL(array, op, ...) \ 439 + (void)____PVOP_CALL(, array, op, PVOP_VCALL_CLOBBERS, \ 396 440 VEXTRA_CLOBBERS, ##__VA_ARGS__) 397 441 398 - #define __PVOP_ALT_VCALL(op, alt, cond, ...) \ 399 - (void)____PVOP_ALT_CALL(, op, alt, cond, \ 442 + #define __PVOP_ALT_VCALL(array, op, alt, cond, ...) \ 443 + (void)____PVOP_ALT_CALL(, array, op, alt, cond, \ 400 444 PVOP_VCALL_CLOBBERS, VEXTRA_CLOBBERS, \ 401 445 ##__VA_ARGS__) 402 446 403 - #define __PVOP_VCALLEESAVE(op, ...) \ 404 - (void)____PVOP_CALL(, op.func, \ 447 + #define __PVOP_VCALLEESAVE(array, op, ...) \ 448 + (void)____PVOP_CALL(, array, op.func, \ 405 449 PVOP_VCALLEE_CLOBBERS, , ##__VA_ARGS__) 406 450 407 - #define __PVOP_ALT_VCALLEESAVE(op, alt, cond, ...) \ 408 - (void)____PVOP_ALT_CALL(, op.func, alt, cond, \ 451 + #define __PVOP_ALT_VCALLEESAVE(array, op, alt, cond, ...) \ 452 + (void)____PVOP_ALT_CALL(, array, op.func, alt, cond, \ 409 453 PVOP_VCALLEE_CLOBBERS, , ##__VA_ARGS__) 410 454 411 455 412 - #define PVOP_CALL0(rettype, op) \ 413 - __PVOP_CALL(rettype, op) 414 - #define PVOP_VCALL0(op) \ 415 - __PVOP_VCALL(op) 416 - #define PVOP_ALT_CALL0(rettype, op, alt, cond) \ 417 - __PVOP_ALT_CALL(rettype, op, alt, cond) 418 - #define PVOP_ALT_VCALL0(op, alt, cond) \ 419 - __PVOP_ALT_VCALL(op, alt, cond) 456 + #define PVOP_CALL0(rettype, array, op) \ 457 + __PVOP_CALL(rettype, array, op) 458 + #define PVOP_VCALL0(array, op) \ 459 + __PVOP_VCALL(array, op) 460 + #define PVOP_ALT_CALL0(rettype, array, op, alt, cond) \ 461 + __PVOP_ALT_CALL(rettype, array, op, alt, cond) 462 + #define PVOP_ALT_VCALL0(array, op, alt, cond) \ 463 + __PVOP_ALT_VCALL(array, op, alt, cond) 420 464 421 - #define PVOP_CALLEE0(rettype, op) \ 422 - __PVOP_CALLEESAVE(rettype, op) 423 - #define PVOP_VCALLEE0(op) \ 424 - __PVOP_VCALLEESAVE(op) 425 - #define PVOP_ALT_CALLEE0(rettype, op, alt, cond) \ 426 - __PVOP_ALT_CALLEESAVE(rettype, op, alt, cond) 427 - #define PVOP_ALT_VCALLEE0(op, alt, cond) \ 428 - __PVOP_ALT_VCALLEESAVE(op, alt, cond) 429 - 430 - 431 - #define PVOP_CALL1(rettype, op, arg1) \ 432 - __PVOP_CALL(rettype, op, PVOP_CALL_ARG1(arg1)) 433 - #define PVOP_VCALL1(op, arg1) \ 434 - __PVOP_VCALL(op, PVOP_CALL_ARG1(arg1)) 435 - #define PVOP_ALT_VCALL1(op, arg1, alt, cond) \ 436 - __PVOP_ALT_VCALL(op, alt, cond, PVOP_CALL_ARG1(arg1)) 437 - 438 - #define PVOP_CALLEE1(rettype, op, arg1) \ 439 - __PVOP_CALLEESAVE(rettype, op, PVOP_CALL_ARG1(arg1)) 440 - #define PVOP_VCALLEE1(op, arg1) \ 441 - __PVOP_VCALLEESAVE(op, PVOP_CALL_ARG1(arg1)) 442 - #define PVOP_ALT_CALLEE1(rettype, op, arg1, alt, cond) \ 443 - __PVOP_ALT_CALLEESAVE(rettype, op, alt, cond, PVOP_CALL_ARG1(arg1)) 444 - #define PVOP_ALT_VCALLEE1(op, arg1, alt, cond) \ 445 - __PVOP_ALT_VCALLEESAVE(op, alt, cond, PVOP_CALL_ARG1(arg1)) 465 + #define PVOP_CALLEE0(rettype, array, op) \ 466 + __PVOP_CALLEESAVE(rettype, array, op) 467 + #define PVOP_VCALLEE0(array, op) \ 468 + __PVOP_VCALLEESAVE(array, op) 469 + #define PVOP_ALT_CALLEE0(rettype, array, op, alt, cond) \ 470 + __PVOP_ALT_CALLEESAVE(rettype, array, op, alt, cond) 471 + #define PVOP_ALT_VCALLEE0(array, op, alt, cond) \ 472 + __PVOP_ALT_VCALLEESAVE(array, op, alt, cond) 446 473 447 474 448 - #define PVOP_CALL2(rettype, op, arg1, arg2) \ 449 - __PVOP_CALL(rettype, op, PVOP_CALL_ARG1(arg1), PVOP_CALL_ARG2(arg2)) 450 - #define PVOP_VCALL2(op, arg1, arg2) \ 451 - __PVOP_VCALL(op, PVOP_CALL_ARG1(arg1), PVOP_CALL_ARG2(arg2)) 475 + #define PVOP_CALL1(rettype, array, op, arg1) \ 476 + __PVOP_CALL(rettype, array, op, PVOP_CALL_ARG1(arg1)) 477 + #define PVOP_VCALL1(array, op, arg1) \ 478 + __PVOP_VCALL(array, op, PVOP_CALL_ARG1(arg1)) 479 + #define PVOP_ALT_VCALL1(array, op, arg1, alt, cond) \ 480 + __PVOP_ALT_VCALL(array, op, alt, cond, PVOP_CALL_ARG1(arg1)) 452 481 453 - #define PVOP_CALL3(rettype, op, arg1, arg2, arg3) \ 454 - __PVOP_CALL(rettype, op, PVOP_CALL_ARG1(arg1), \ 482 + #define PVOP_CALLEE1(rettype, array, op, arg1) \ 483 + __PVOP_CALLEESAVE(rettype, array, op, PVOP_CALL_ARG1(arg1)) 484 + #define PVOP_VCALLEE1(array, op, arg1) \ 485 + __PVOP_VCALLEESAVE(array, op, PVOP_CALL_ARG1(arg1)) 486 + #define PVOP_ALT_CALLEE1(rettype, array, op, arg1, alt, cond) \ 487 + __PVOP_ALT_CALLEESAVE(rettype, array, op, alt, cond, PVOP_CALL_ARG1(arg1)) 488 + #define PVOP_ALT_VCALLEE1(array, op, arg1, alt, cond) \ 489 + __PVOP_ALT_VCALLEESAVE(array, op, alt, cond, PVOP_CALL_ARG1(arg1)) 490 + 491 + 492 + #define PVOP_CALL2(rettype, array, op, arg1, arg2) \ 493 + __PVOP_CALL(rettype, array, op, PVOP_CALL_ARG1(arg1), PVOP_CALL_ARG2(arg2)) 494 + #define PVOP_VCALL2(array, op, arg1, arg2) \ 495 + __PVOP_VCALL(array, op, PVOP_CALL_ARG1(arg1), PVOP_CALL_ARG2(arg2)) 496 + 497 + #define PVOP_CALL3(rettype, array, op, arg1, arg2, arg3) \ 498 + __PVOP_CALL(rettype, array, op, PVOP_CALL_ARG1(arg1), \ 455 499 PVOP_CALL_ARG2(arg2), PVOP_CALL_ARG3(arg3)) 456 - #define PVOP_VCALL3(op, arg1, arg2, arg3) \ 457 - __PVOP_VCALL(op, PVOP_CALL_ARG1(arg1), \ 500 + #define PVOP_VCALL3(array, op, arg1, arg2, arg3) \ 501 + __PVOP_VCALL(array, op, PVOP_CALL_ARG1(arg1), \ 458 502 PVOP_CALL_ARG2(arg2), PVOP_CALL_ARG3(arg3)) 459 503 460 - #define PVOP_CALL4(rettype, op, arg1, arg2, arg3, arg4) \ 461 - __PVOP_CALL(rettype, op, \ 504 + #define PVOP_CALL4(rettype, array, op, arg1, arg2, arg3, arg4) \ 505 + __PVOP_CALL(rettype, array, op, \ 462 506 PVOP_CALL_ARG1(arg1), PVOP_CALL_ARG2(arg2), \ 463 507 PVOP_CALL_ARG3(arg3), PVOP_CALL_ARG4(arg4)) 464 - #define PVOP_VCALL4(op, arg1, arg2, arg3, arg4) \ 465 - __PVOP_VCALL(op, PVOP_CALL_ARG1(arg1), PVOP_CALL_ARG2(arg2), \ 508 + #define PVOP_VCALL4(array, op, arg1, arg2, arg3, arg4) \ 509 + __PVOP_VCALL(array, op, PVOP_CALL_ARG1(arg1), PVOP_CALL_ARG2(arg2), \ 466 510 PVOP_CALL_ARG3(arg3), PVOP_CALL_ARG4(arg4)) 467 - 468 - unsigned long paravirt_ret0(void); 469 - #ifdef CONFIG_PARAVIRT_XXL 470 - u64 _paravirt_ident_64(u64); 471 - unsigned long pv_native_save_fl(void); 472 - void pv_native_irq_disable(void); 473 - void pv_native_irq_enable(void); 474 - unsigned long pv_native_read_cr2(void); 475 - #endif 476 - 477 - #define paravirt_nop ((void *)nop_func) 478 511 479 512 #endif /* __ASSEMBLER__ */ 480 513 481 514 #define ALT_NOT_XEN ALT_NOT(X86_FEATURE_XENPV) 515 + 516 + #ifdef CONFIG_X86_32 517 + /* save and restore all caller-save registers, except return value */ 518 + #define PV_SAVE_ALL_CALLER_REGS "pushl %ecx;" 519 + #define PV_RESTORE_ALL_CALLER_REGS "popl %ecx;" 520 + #else 521 + /* save and restore all caller-save registers, except return value */ 522 + #define PV_SAVE_ALL_CALLER_REGS \ 523 + "push %rcx;" \ 524 + "push %rdx;" \ 525 + "push %rsi;" \ 526 + "push %rdi;" \ 527 + "push %r8;" \ 528 + "push %r9;" \ 529 + "push %r10;" \ 530 + "push %r11;" 531 + #define PV_RESTORE_ALL_CALLER_REGS \ 532 + "pop %r11;" \ 533 + "pop %r10;" \ 534 + "pop %r9;" \ 535 + "pop %r8;" \ 536 + "pop %rdi;" \ 537 + "pop %rsi;" \ 538 + "pop %rdx;" \ 539 + "pop %rcx;" 540 + #endif 541 + 542 + /* 543 + * Generate a thunk around a function which saves all caller-save 544 + * registers except for the return value. This allows C functions to 545 + * be called from assembler code where fewer than normal registers are 546 + * available. It may also help code generation around calls from C 547 + * code if the common case doesn't use many registers. 548 + * 549 + * When a callee is wrapped in a thunk, the caller can assume that all 550 + * arg regs and all scratch registers are preserved across the 551 + * call. The return value in rax/eax will not be saved, even for void 552 + * functions. 553 + */ 554 + #define PV_THUNK_NAME(func) "__raw_callee_save_" #func 555 + #define __PV_CALLEE_SAVE_REGS_THUNK(func, section) \ 556 + extern typeof(func) __raw_callee_save_##func; \ 557 + \ 558 + asm(".pushsection " section ", \"ax\";" \ 559 + ".globl " PV_THUNK_NAME(func) ";" \ 560 + ".type " PV_THUNK_NAME(func) ", @function;" \ 561 + ASM_FUNC_ALIGN \ 562 + PV_THUNK_NAME(func) ":" \ 563 + ASM_ENDBR \ 564 + FRAME_BEGIN \ 565 + PV_SAVE_ALL_CALLER_REGS \ 566 + "call " #func ";" \ 567 + PV_RESTORE_ALL_CALLER_REGS \ 568 + FRAME_END \ 569 + ASM_RET \ 570 + ".size " PV_THUNK_NAME(func) ", .-" PV_THUNK_NAME(func) ";" \ 571 + ".popsection") 572 + 573 + #define PV_CALLEE_SAVE_REGS_THUNK(func) \ 574 + __PV_CALLEE_SAVE_REGS_THUNK(func, ".text") 575 + 576 + /* Get a reference to a callee-save function */ 577 + #define PV_CALLEE_SAVE(func) \ 578 + ((struct paravirt_callee_save) { __raw_callee_save_##func }) 579 + 580 + /* Promise that "func" already uses the right calling convention */ 581 + #define __PV_IS_CALLEE_SAVE(func) \ 582 + ((struct paravirt_callee_save) { func }) 482 583 483 584 #endif /* CONFIG_PARAVIRT */ 484 585 #endif /* _ASM_X86_PARAVIRT_TYPES_H */

-1

arch/x86/include/asm/pgtable_32.h

··· 16 16 #ifndef __ASSEMBLER__ 17 17 #include <asm/processor.h> 18 18 #include <linux/threads.h> 19 - #include <asm/paravirt.h> 20 19 21 20 #include <linux/bitops.h> 22 21 #include <linux/list.h>

+1 -1

arch/x86/include/asm/ptrace.h

··· 172 172 #endif /* !__i386__ */ 173 173 174 174 #ifdef CONFIG_PARAVIRT 175 - #include <asm/paravirt_types.h> 175 + #include <asm/paravirt-base.h> 176 176 #endif 177 177 178 178 #include <asm/proto.h>

+5 -82

arch/x86/include/asm/qspinlock.h

··· 7 7 #include <asm-generic/qspinlock_types.h> 8 8 #include <asm/paravirt.h> 9 9 #include <asm/rmwcc.h> 10 + #ifdef CONFIG_PARAVIRT 11 + #include <asm/paravirt-spinlock.h> 12 + #endif 10 13 11 14 #define _Q_PENDING_LOOPS (1 << 9) 12 15 ··· 30 27 return val; 31 28 } 32 29 33 - #ifdef CONFIG_PARAVIRT_SPINLOCKS 34 - extern void native_queued_spin_lock_slowpath(struct qspinlock *lock, u32 val); 35 - extern void __pv_init_lock_hash(void); 36 - extern void __pv_queued_spin_lock_slowpath(struct qspinlock *lock, u32 val); 37 - extern void __raw_callee_save___pv_queued_spin_unlock(struct qspinlock *lock); 38 - extern bool nopvspin; 39 - 40 - #define queued_spin_unlock queued_spin_unlock 41 - /** 42 - * queued_spin_unlock - release a queued spinlock 43 - * @lock : Pointer to queued spinlock structure 44 - * 45 - * A smp_store_release() on the least-significant byte. 46 - */ 47 - static inline void native_queued_spin_unlock(struct qspinlock *lock) 48 - { 49 - smp_store_release(&lock->locked, 0); 50 - } 51 - 52 - static inline void queued_spin_lock_slowpath(struct qspinlock *lock, u32 val) 53 - { 54 - pv_queued_spin_lock_slowpath(lock, val); 55 - } 56 - 57 - static inline void queued_spin_unlock(struct qspinlock *lock) 58 - { 59 - kcsan_release(); 60 - pv_queued_spin_unlock(lock); 61 - } 62 - 63 - #define vcpu_is_preempted vcpu_is_preempted 64 - static inline bool vcpu_is_preempted(long cpu) 65 - { 66 - return pv_vcpu_is_preempted(cpu); 67 - } 30 + #ifndef CONFIG_PARAVIRT 31 + static inline void native_pv_lock_init(void) { } 68 32 #endif 69 - 70 - #ifdef CONFIG_PARAVIRT 71 - /* 72 - * virt_spin_lock_key - disables by default the virt_spin_lock() hijack. 73 - * 74 - * Native (and PV wanting native due to vCPU pinning) should keep this key 75 - * disabled. Native does not touch the key. 76 - * 77 - * When in a guest then native_pv_lock_init() enables the key first and 78 - * KVM/XEN might conditionally disable it later in the boot process again. 79 - */ 80 - DECLARE_STATIC_KEY_FALSE(virt_spin_lock_key); 81 - 82 - /* 83 - * Shortcut for the queued_spin_lock_slowpath() function that allows 84 - * virt to hijack it. 85 - * 86 - * Returns: 87 - * true - lock has been negotiated, all done; 88 - * false - queued_spin_lock_slowpath() will do its thing. 89 - */ 90 - #define virt_spin_lock virt_spin_lock 91 - static inline bool virt_spin_lock(struct qspinlock *lock) 92 - { 93 - int val; 94 - 95 - if (!static_branch_likely(&virt_spin_lock_key)) 96 - return false; 97 - 98 - /* 99 - * On hypervisors without PARAVIRT_SPINLOCKS support we fall 100 - * back to a Test-and-Set spinlock, because fair locks have 101 - * horrible lock 'holder' preemption issues. 102 - */ 103 - 104 - __retry: 105 - val = atomic_read(&lock->val); 106 - 107 - if (val || !atomic_try_cmpxchg(&lock->val, &val, _Q_LOCKED_VAL)) { 108 - cpu_relax(); 109 - goto __retry; 110 - } 111 - 112 - return true; 113 - } 114 - 115 - #endif /* CONFIG_PARAVIRT */ 116 33 117 34 #include <asm-generic/qspinlock.h> 118 35

-1

arch/x86/include/asm/spinlock.h

··· 7 7 #include <asm/page.h> 8 8 #include <asm/processor.h> 9 9 #include <linux/compiler.h> 10 - #include <asm/paravirt.h> 11 10 #include <asm/bitops.h> 12 11 13 12 /*

+1

arch/x86/include/asm/timer.h

··· 12 12 extern int no_timer_check; 13 13 14 14 extern bool using_native_sched_clock(void); 15 + void paravirt_set_sched_clock(u64 (*func)(void)); 15 16 16 17 /* 17 18 * We use the full linear equation: f(x) = a + b*x, in order to allow

-4

arch/x86/include/asm/tlbflush.h

··· 303 303 static inline bool mm_in_asid_transition(struct mm_struct *mm) { return false; } 304 304 #endif /* CONFIG_BROADCAST_TLB_FLUSH */ 305 305 306 - #ifdef CONFIG_PARAVIRT 307 - #include <asm/paravirt.h> 308 - #endif 309 - 310 306 #define flush_tlb_mm(mm) \ 311 307 flush_tlb_mm_range(mm, 0UL, TLB_FLUSH_ALL, 0UL, true) 312 308

+1 -1

arch/x86/kernel/Makefile

··· 126 126 127 127 obj-$(CONFIG_KVM_GUEST) += kvm.o kvmclock.o 128 128 obj-$(CONFIG_PARAVIRT) += paravirt.o 129 - obj-$(CONFIG_PARAVIRT_SPINLOCKS)+= paravirt-spinlocks.o 129 + obj-$(CONFIG_PARAVIRT) += paravirt-spinlocks.o 130 130 obj-$(CONFIG_PARAVIRT_CLOCK) += pvclock.o 131 131 obj-$(CONFIG_X86_PMEM_LEGACY_DEVICE) += pmem.o 132 132

-1

arch/x86/kernel/apm_32.c

··· 229 229 #include <linux/uaccess.h> 230 230 #include <asm/desc.h> 231 231 #include <asm/olpc.h> 232 - #include <asm/paravirt.h> 233 232 #include <asm/reboot.h> 234 233 #include <asm/nospec-branch.h> 235 234 #include <asm/ibt.h>

-1

arch/x86/kernel/callthunks.c

··· 15 15 #include <asm/insn.h> 16 16 #include <asm/kexec.h> 17 17 #include <asm/nospec-branch.h> 18 - #include <asm/paravirt.h> 19 18 #include <asm/sections.h> 20 19 #include <asm/switch_to.h> 21 20 #include <asm/sync_core.h>

-1

arch/x86/kernel/cpu/bugs.c

··· 26 26 #include <asm/fpu/api.h> 27 27 #include <asm/msr.h> 28 28 #include <asm/vmx.h> 29 - #include <asm/paravirt.h> 30 29 #include <asm/cpu_device_id.h> 31 30 #include <asm/e820/api.h> 32 31 #include <asm/hypervisor.h>

+1

arch/x86/kernel/cpu/vmware.c

··· 29 29 #include <linux/efi.h> 30 30 #include <linux/reboot.h> 31 31 #include <linux/static_call.h> 32 + #include <linux/sched/cputime.h> 32 33 #include <asm/div64.h> 33 34 #include <asm/x86_init.h> 34 35 #include <asm/hypervisor.h>

+8 -5

arch/x86/kernel/kvm.c

··· 30 30 #include <linux/cc_platform.h> 31 31 #include <linux/efi.h> 32 32 #include <linux/kvm_types.h> 33 + #include <linux/sched/cputime.h> 33 34 #include <asm/timer.h> 34 35 #include <asm/cpu.h> 35 36 #include <asm/traps.h> ··· 842 841 has_steal_clock = 1; 843 842 static_call_update(pv_steal_clock, kvm_steal_clock); 844 843 845 - pv_ops.lock.vcpu_is_preempted = 844 + #ifdef CONFIG_PARAVIRT_SPINLOCKS 845 + pv_ops_lock.vcpu_is_preempted = 846 846 PV_CALLEE_SAVE(__kvm_vcpu_is_preempted); 847 + #endif 847 848 } 848 849 849 850 if (kvm_para_has_feature(KVM_FEATURE_PV_EOI)) ··· 1141 1138 pr_info("PV spinlocks enabled\n"); 1142 1139 1143 1140 __pv_init_lock_hash(); 1144 - pv_ops.lock.queued_spin_lock_slowpath = __pv_queued_spin_lock_slowpath; 1145 - pv_ops.lock.queued_spin_unlock = 1141 + pv_ops_lock.queued_spin_lock_slowpath = __pv_queued_spin_lock_slowpath; 1142 + pv_ops_lock.queued_spin_unlock = 1146 1143 PV_CALLEE_SAVE(__pv_queued_spin_unlock); 1147 - pv_ops.lock.wait = kvm_wait; 1148 - pv_ops.lock.kick = kvm_kick_cpu; 1144 + pv_ops_lock.wait = kvm_wait; 1145 + pv_ops_lock.kick = kvm_kick_cpu; 1149 1146 1150 1147 /* 1151 1148 * When PV spinlock is enabled which is preferred over

+1

arch/x86/kernel/kvmclock.c

··· 19 19 #include <linux/cc_platform.h> 20 20 21 21 #include <asm/hypervisor.h> 22 + #include <asm/timer.h> 22 23 #include <asm/x86_init.h> 23 24 #include <asm/kvmclock.h> 24 25

+23 -3

arch/x86/kernel/paravirt-spinlocks.c

··· 3 3 * Split spinlock implementation out into its own file, so it can be 4 4 * compiled in a FTRACE-compatible way. 5 5 */ 6 + #include <linux/static_call.h> 6 7 #include <linux/spinlock.h> 7 8 #include <linux/export.h> 8 9 #include <linux/jump_label.h> 9 10 10 - #include <asm/paravirt.h> 11 + DEFINE_STATIC_KEY_FALSE(virt_spin_lock_key); 11 12 13 + #ifdef CONFIG_SMP 14 + void __init native_pv_lock_init(void) 15 + { 16 + if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) 17 + static_branch_enable(&virt_spin_lock_key); 18 + } 19 + #endif 20 + 21 + #ifdef CONFIG_PARAVIRT_SPINLOCKS 12 22 __visible void __native_queued_spin_unlock(struct qspinlock *lock) 13 23 { 14 24 native_queued_spin_unlock(lock); ··· 27 17 28 18 bool pv_is_native_spin_unlock(void) 29 19 { 30 - return pv_ops.lock.queued_spin_unlock.func == 20 + return pv_ops_lock.queued_spin_unlock.func == 31 21 __raw_callee_save___native_queued_spin_unlock; 32 22 } 33 23 ··· 39 29 40 30 bool pv_is_native_vcpu_is_preempted(void) 41 31 { 42 - return pv_ops.lock.vcpu_is_preempted.func == 32 + return pv_ops_lock.vcpu_is_preempted.func == 43 33 __raw_callee_save___native_vcpu_is_preempted; 44 34 } 45 35 ··· 51 41 if (!pv_is_native_vcpu_is_preempted()) 52 42 setup_force_cpu_cap(X86_FEATURE_VCPUPREEMPT); 53 43 } 44 + 45 + struct pv_lock_ops pv_ops_lock = { 46 + .queued_spin_lock_slowpath = native_queued_spin_lock_slowpath, 47 + .queued_spin_unlock = PV_CALLEE_SAVE(__native_queued_spin_unlock), 48 + .wait = paravirt_nop, 49 + .kick = paravirt_nop, 50 + .vcpu_is_preempted = PV_CALLEE_SAVE(__native_vcpu_is_preempted), 51 + }; 52 + EXPORT_SYMBOL(pv_ops_lock); 53 + #endif

+5 -37

arch/x86/kernel/paravirt.c

··· 45 45 } 46 46 47 47 #ifdef CONFIG_PARAVIRT_XXL 48 + unsigned long pv_native_save_fl(void); 49 + void pv_native_irq_disable(void); 50 + void pv_native_irq_enable(void); 51 + unsigned long pv_native_read_cr2(void); 52 + 48 53 DEFINE_ASM_FUNC(_paravirt_ident_64, "mov %rdi, %rax", .text); 49 54 DEFINE_ASM_FUNC(pv_native_save_fl, "pushf; pop %rax", .noinstr.text); 50 55 DEFINE_ASM_FUNC(pv_native_irq_disable, "cli", .noinstr.text); 51 56 DEFINE_ASM_FUNC(pv_native_irq_enable, "sti", .noinstr.text); 52 57 DEFINE_ASM_FUNC(pv_native_read_cr2, "mov %cr2, %rax", .noinstr.text); 53 58 #endif 54 - 55 - DEFINE_STATIC_KEY_FALSE(virt_spin_lock_key); 56 - 57 - void __init native_pv_lock_init(void) 58 - { 59 - if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) 60 - static_branch_enable(&virt_spin_lock_key); 61 - } 62 - 63 - struct static_key paravirt_steal_enabled; 64 - struct static_key paravirt_steal_rq_enabled; 65 - 66 - static u64 native_steal_clock(int cpu) 67 - { 68 - return 0; 69 - } 70 - 71 - DEFINE_STATIC_CALL(pv_steal_clock, native_steal_clock); 72 - DEFINE_STATIC_CALL(pv_sched_clock, native_sched_clock); 73 - 74 - void paravirt_set_sched_clock(u64 (*func)(void)) 75 - { 76 - static_call_update(pv_sched_clock, func); 77 - } 78 59 79 60 static noinstr void pv_native_safe_halt(void) 80 61 { ··· 213 232 214 233 .mmu.set_fixmap = native_set_fixmap, 215 234 #endif /* CONFIG_PARAVIRT_XXL */ 216 - 217 - #if defined(CONFIG_PARAVIRT_SPINLOCKS) 218 - /* Lock ops. */ 219 - #ifdef CONFIG_SMP 220 - .lock.queued_spin_lock_slowpath = native_queued_spin_lock_slowpath, 221 - .lock.queued_spin_unlock = 222 - PV_CALLEE_SAVE(__native_queued_spin_unlock), 223 - .lock.wait = paravirt_nop, 224 - .lock.kick = paravirt_nop, 225 - .lock.vcpu_is_preempted = 226 - PV_CALLEE_SAVE(__native_vcpu_is_preempted), 227 - #endif /* SMP */ 228 - #endif 229 235 }; 230 236 231 237 #ifdef CONFIG_PARAVIRT_XXL

+9 -1

arch/x86/kernel/tsc.c

··· 267 267 /* We need to define a real function for sched_clock, to override the 268 268 weak default version */ 269 269 #ifdef CONFIG_PARAVIRT 270 + DEFINE_STATIC_CALL(pv_sched_clock, native_sched_clock); 271 + 270 272 noinstr u64 sched_clock_noinstr(void) 271 273 { 272 - return paravirt_sched_clock(); 274 + return static_call(pv_sched_clock)(); 273 275 } 274 276 275 277 bool using_native_sched_clock(void) 276 278 { 277 279 return static_call_query(pv_sched_clock) == native_sched_clock; 278 280 } 281 + 282 + void paravirt_set_sched_clock(u64 (*func)(void)) 283 + { 284 + static_call_update(pv_sched_clock, func); 285 + } 279 286 #else 280 287 u64 sched_clock_noinstr(void) __attribute__((alias("native_sched_clock"))); 281 288 282 289 bool using_native_sched_clock(void) { return true; } 290 + void paravirt_set_sched_clock(u64 (*func)(void)) { } 283 291 #endif 284 292 285 293 notrace u64 sched_clock(void)

-1

arch/x86/kernel/vsmp_64.c

··· 18 18 #include <asm/apic.h> 19 19 #include <asm/pci-direct.h> 20 20 #include <asm/io.h> 21 - #include <asm/paravirt.h> 22 21 #include <asm/setup.h> 23 22 24 23 #define TOPOLOGY_REGISTER_OFFSET 0x10

-1

arch/x86/lib/cache-smp.c

··· 1 1 // SPDX-License-Identifier: GPL-2.0 2 - #include <asm/paravirt.h> 3 2 #include <linux/smp.h> 4 3 #include <linux/export.h> 5 4 #include <linux/kvm_types.h>

-1

arch/x86/mm/init.c

··· 27 27 #include <asm/pti.h> 28 28 #include <asm/text-patching.h> 29 29 #include <asm/memtype.h> 30 - #include <asm/paravirt.h> 31 30 #include <asm/mmu_context.h> 32 31 33 32 /*

+33 -49

arch/x86/xen/enlighten_pv.c

··· 1212 1212 .name = "Xen", 1213 1213 }; 1214 1214 1215 - static const typeof(pv_ops) xen_cpu_ops __initconst = { 1216 - .cpu = { 1217 - .cpuid = xen_cpuid, 1218 - 1219 - .set_debugreg = xen_set_debugreg, 1220 - .get_debugreg = xen_get_debugreg, 1221 - 1222 - .read_cr0 = xen_read_cr0, 1223 - .write_cr0 = xen_write_cr0, 1224 - 1225 - .write_cr4 = xen_write_cr4, 1226 - 1227 - .read_msr = xen_read_msr, 1228 - .write_msr = xen_write_msr, 1229 - 1230 - .read_msr_safe = xen_read_msr_safe, 1231 - .write_msr_safe = xen_write_msr_safe, 1232 - 1233 - .read_pmc = xen_read_pmc, 1234 - 1235 - .load_tr_desc = paravirt_nop, 1236 - .set_ldt = xen_set_ldt, 1237 - .load_gdt = xen_load_gdt, 1238 - .load_idt = xen_load_idt, 1239 - .load_tls = xen_load_tls, 1240 - .load_gs_index = xen_load_gs_index, 1241 - 1242 - .alloc_ldt = xen_alloc_ldt, 1243 - .free_ldt = xen_free_ldt, 1244 - 1245 - .store_tr = xen_store_tr, 1246 - 1247 - .write_ldt_entry = xen_write_ldt_entry, 1248 - .write_gdt_entry = xen_write_gdt_entry, 1249 - .write_idt_entry = xen_write_idt_entry, 1250 - .load_sp0 = xen_load_sp0, 1251 - 1252 - #ifdef CONFIG_X86_IOPL_IOPERM 1253 - .invalidate_io_bitmap = xen_invalidate_io_bitmap, 1254 - .update_io_bitmap = xen_update_io_bitmap, 1255 - #endif 1256 - .io_delay = xen_io_delay, 1257 - 1258 - .start_context_switch = xen_start_context_switch, 1259 - .end_context_switch = xen_end_context_switch, 1260 - }, 1261 - }; 1262 - 1263 1215 static void xen_restart(char *msg) 1264 1216 { 1265 1217 xen_reboot(SHUTDOWN_reboot); ··· 1363 1411 1364 1412 /* Install Xen paravirt ops */ 1365 1413 pv_info = xen_info; 1366 - pv_ops.cpu = xen_cpu_ops.cpu; 1414 + 1415 + pv_ops.cpu.cpuid = xen_cpuid; 1416 + pv_ops.cpu.set_debugreg = xen_set_debugreg; 1417 + pv_ops.cpu.get_debugreg = xen_get_debugreg; 1418 + pv_ops.cpu.read_cr0 = xen_read_cr0; 1419 + pv_ops.cpu.write_cr0 = xen_write_cr0; 1420 + pv_ops.cpu.write_cr4 = xen_write_cr4; 1421 + pv_ops.cpu.read_msr = xen_read_msr; 1422 + pv_ops.cpu.write_msr = xen_write_msr; 1423 + pv_ops.cpu.read_msr_safe = xen_read_msr_safe; 1424 + pv_ops.cpu.write_msr_safe = xen_write_msr_safe; 1425 + pv_ops.cpu.read_pmc = xen_read_pmc; 1426 + pv_ops.cpu.load_tr_desc = paravirt_nop; 1427 + pv_ops.cpu.set_ldt = xen_set_ldt; 1428 + pv_ops.cpu.load_gdt = xen_load_gdt; 1429 + pv_ops.cpu.load_idt = xen_load_idt; 1430 + pv_ops.cpu.load_tls = xen_load_tls; 1431 + pv_ops.cpu.load_gs_index = xen_load_gs_index; 1432 + pv_ops.cpu.alloc_ldt = xen_alloc_ldt; 1433 + pv_ops.cpu.free_ldt = xen_free_ldt; 1434 + pv_ops.cpu.store_tr = xen_store_tr; 1435 + pv_ops.cpu.write_ldt_entry = xen_write_ldt_entry; 1436 + pv_ops.cpu.write_gdt_entry = xen_write_gdt_entry; 1437 + pv_ops.cpu.write_idt_entry = xen_write_idt_entry; 1438 + pv_ops.cpu.load_sp0 = xen_load_sp0; 1439 + #ifdef CONFIG_X86_IOPL_IOPERM 1440 + pv_ops.cpu.invalidate_io_bitmap = xen_invalidate_io_bitmap; 1441 + pv_ops.cpu.update_io_bitmap = xen_update_io_bitmap; 1442 + #endif 1443 + pv_ops.cpu.io_delay = xen_io_delay; 1444 + pv_ops.cpu.start_context_switch = xen_start_context_switch; 1445 + pv_ops.cpu.end_context_switch = xen_end_context_switch; 1446 + 1367 1447 xen_init_irq_ops(); 1368 1448 1369 1449 /*

+7 -13

arch/x86/xen/irq.c

··· 40 40 xen_safe_halt(); 41 41 } 42 42 43 - static const typeof(pv_ops) xen_irq_ops __initconst = { 44 - .irq = { 45 - /* Initial interrupt flag handling only called while interrupts off. */ 46 - .save_fl = __PV_IS_CALLEE_SAVE(paravirt_ret0), 47 - .irq_disable = __PV_IS_CALLEE_SAVE(paravirt_nop), 48 - .irq_enable = __PV_IS_CALLEE_SAVE(BUG_func), 49 - 50 - .safe_halt = xen_safe_halt, 51 - .halt = xen_halt, 52 - }, 53 - }; 54 - 55 43 void __init xen_init_irq_ops(void) 56 44 { 57 - pv_ops.irq = xen_irq_ops.irq; 45 + /* Initial interrupt flag handling only called while interrupts off. */ 46 + pv_ops.irq.save_fl = __PV_IS_CALLEE_SAVE(paravirt_ret0); 47 + pv_ops.irq.irq_disable = __PV_IS_CALLEE_SAVE(paravirt_nop); 48 + pv_ops.irq.irq_enable = __PV_IS_CALLEE_SAVE(BUG_func); 49 + pv_ops.irq.safe_halt = xen_safe_halt; 50 + pv_ops.irq.halt = xen_halt; 51 + 58 52 x86_init.irqs.intr_init = xen_init_IRQ; 59 53 }

+38 -62

arch/x86/xen/mmu_pv.c

··· 2175 2175 preempt_enable(); 2176 2176 } 2177 2177 2178 - static const typeof(pv_ops) xen_mmu_ops __initconst = { 2179 - .mmu = { 2180 - .read_cr2 = __PV_IS_CALLEE_SAVE(xen_read_cr2), 2181 - .write_cr2 = xen_write_cr2, 2182 - 2183 - .read_cr3 = xen_read_cr3, 2184 - .write_cr3 = xen_write_cr3_init, 2185 - 2186 - .flush_tlb_user = xen_flush_tlb, 2187 - .flush_tlb_kernel = xen_flush_tlb, 2188 - .flush_tlb_one_user = xen_flush_tlb_one_user, 2189 - .flush_tlb_multi = xen_flush_tlb_multi, 2190 - 2191 - .pgd_alloc = xen_pgd_alloc, 2192 - .pgd_free = xen_pgd_free, 2193 - 2194 - .alloc_pte = xen_alloc_pte_init, 2195 - .release_pte = xen_release_pte_init, 2196 - .alloc_pmd = xen_alloc_pmd_init, 2197 - .release_pmd = xen_release_pmd_init, 2198 - 2199 - .set_pte = xen_set_pte_init, 2200 - .set_pmd = xen_set_pmd_hyper, 2201 - 2202 - .ptep_modify_prot_start = xen_ptep_modify_prot_start, 2203 - .ptep_modify_prot_commit = xen_ptep_modify_prot_commit, 2204 - 2205 - .pte_val = PV_CALLEE_SAVE(xen_pte_val), 2206 - .pgd_val = PV_CALLEE_SAVE(xen_pgd_val), 2207 - 2208 - .make_pte = PV_CALLEE_SAVE(xen_make_pte_init), 2209 - .make_pgd = PV_CALLEE_SAVE(xen_make_pgd), 2210 - 2211 - .set_pud = xen_set_pud_hyper, 2212 - 2213 - .make_pmd = PV_CALLEE_SAVE(xen_make_pmd), 2214 - .pmd_val = PV_CALLEE_SAVE(xen_pmd_val), 2215 - 2216 - .pud_val = PV_CALLEE_SAVE(xen_pud_val), 2217 - .make_pud = PV_CALLEE_SAVE(xen_make_pud), 2218 - .set_p4d = xen_set_p4d_hyper, 2219 - 2220 - .alloc_pud = xen_alloc_pmd_init, 2221 - .release_pud = xen_release_pmd_init, 2222 - 2223 - .p4d_val = PV_CALLEE_SAVE(xen_p4d_val), 2224 - .make_p4d = PV_CALLEE_SAVE(xen_make_p4d), 2225 - 2226 - .enter_mmap = xen_enter_mmap, 2227 - .exit_mmap = xen_exit_mmap, 2228 - 2229 - .lazy_mode = { 2230 - .enter = xen_enter_lazy_mmu, 2231 - .leave = xen_leave_lazy_mmu, 2232 - .flush = xen_flush_lazy_mmu, 2233 - }, 2234 - 2235 - .set_fixmap = xen_set_fixmap, 2236 - }, 2237 - }; 2238 - 2239 2178 void __init xen_init_mmu_ops(void) 2240 2179 { 2241 2180 x86_init.paging.pagetable_init = xen_pagetable_init; 2242 2181 x86_init.hyper.init_after_bootmem = xen_after_bootmem; 2243 2182 2244 - pv_ops.mmu = xen_mmu_ops.mmu; 2183 + pv_ops.mmu.read_cr2 = __PV_IS_CALLEE_SAVE(xen_read_cr2); 2184 + pv_ops.mmu.write_cr2 = xen_write_cr2; 2185 + pv_ops.mmu.read_cr3 = xen_read_cr3; 2186 + pv_ops.mmu.write_cr3 = xen_write_cr3_init; 2187 + pv_ops.mmu.flush_tlb_user = xen_flush_tlb; 2188 + pv_ops.mmu.flush_tlb_kernel = xen_flush_tlb; 2189 + pv_ops.mmu.flush_tlb_one_user = xen_flush_tlb_one_user; 2190 + pv_ops.mmu.flush_tlb_multi = xen_flush_tlb_multi; 2191 + pv_ops.mmu.pgd_alloc = xen_pgd_alloc; 2192 + pv_ops.mmu.pgd_free = xen_pgd_free; 2193 + pv_ops.mmu.alloc_pte = xen_alloc_pte_init; 2194 + pv_ops.mmu.release_pte = xen_release_pte_init; 2195 + pv_ops.mmu.alloc_pmd = xen_alloc_pmd_init; 2196 + pv_ops.mmu.release_pmd = xen_release_pmd_init; 2197 + pv_ops.mmu.set_pte = xen_set_pte_init; 2198 + pv_ops.mmu.set_pmd = xen_set_pmd_hyper; 2199 + pv_ops.mmu.ptep_modify_prot_start = xen_ptep_modify_prot_start; 2200 + pv_ops.mmu.ptep_modify_prot_commit = xen_ptep_modify_prot_commit; 2201 + pv_ops.mmu.pte_val = PV_CALLEE_SAVE(xen_pte_val); 2202 + pv_ops.mmu.pgd_val = PV_CALLEE_SAVE(xen_pgd_val); 2203 + pv_ops.mmu.make_pte = PV_CALLEE_SAVE(xen_make_pte_init); 2204 + pv_ops.mmu.make_pgd = PV_CALLEE_SAVE(xen_make_pgd); 2205 + pv_ops.mmu.set_pud = xen_set_pud_hyper; 2206 + pv_ops.mmu.make_pmd = PV_CALLEE_SAVE(xen_make_pmd); 2207 + pv_ops.mmu.pmd_val = PV_CALLEE_SAVE(xen_pmd_val); 2208 + pv_ops.mmu.pud_val = PV_CALLEE_SAVE(xen_pud_val); 2209 + pv_ops.mmu.make_pud = PV_CALLEE_SAVE(xen_make_pud); 2210 + pv_ops.mmu.set_p4d = xen_set_p4d_hyper; 2211 + pv_ops.mmu.alloc_pud = xen_alloc_pmd_init; 2212 + pv_ops.mmu.release_pud = xen_release_pmd_init; 2213 + pv_ops.mmu.p4d_val = PV_CALLEE_SAVE(xen_p4d_val); 2214 + pv_ops.mmu.make_p4d = PV_CALLEE_SAVE(xen_make_p4d); 2215 + pv_ops.mmu.enter_mmap = xen_enter_mmap; 2216 + pv_ops.mmu.exit_mmap = xen_exit_mmap; 2217 + pv_ops.mmu.lazy_mode.enter = xen_enter_lazy_mmu; 2218 + pv_ops.mmu.lazy_mode.leave = xen_leave_lazy_mmu; 2219 + pv_ops.mmu.lazy_mode.flush = xen_flush_lazy_mmu; 2220 + pv_ops.mmu.set_fixmap = xen_set_fixmap; 2245 2221 2246 2222 memset(dummy_mapping, 0xff, PAGE_SIZE); 2247 2223 }

+5 -6

arch/x86/xen/spinlock.c

··· 8 8 #include <linux/slab.h> 9 9 #include <linux/atomic.h> 10 10 11 - #include <asm/paravirt.h> 12 11 #include <asm/qspinlock.h> 13 12 14 13 #include <xen/events.h> ··· 134 135 printk(KERN_DEBUG "xen: PV spinlocks enabled\n"); 135 136 136 137 __pv_init_lock_hash(); 137 - pv_ops.lock.queued_spin_lock_slowpath = __pv_queued_spin_lock_slowpath; 138 - pv_ops.lock.queued_spin_unlock = 138 + pv_ops_lock.queued_spin_lock_slowpath = __pv_queued_spin_lock_slowpath; 139 + pv_ops_lock.queued_spin_unlock = 139 140 PV_CALLEE_SAVE(__pv_queued_spin_unlock); 140 - pv_ops.lock.wait = xen_qlock_wait; 141 - pv_ops.lock.kick = xen_qlock_kick; 142 - pv_ops.lock.vcpu_is_preempted = PV_CALLEE_SAVE(xen_vcpu_stolen); 141 + pv_ops_lock.wait = xen_qlock_wait; 142 + pv_ops_lock.kick = xen_qlock_kick; 143 + pv_ops_lock.vcpu_is_preempted = PV_CALLEE_SAVE(xen_vcpu_stolen); 143 144 }

+2

arch/x86/xen/time.c

··· 16 16 #include <linux/slab.h> 17 17 #include <linux/pvclock_gtod.h> 18 18 #include <linux/timekeeper_internal.h> 19 + #include <linux/sched/cputime.h> 19 20 20 21 #include <asm/pvclock.h> 22 + #include <asm/timer.h> 21 23 #include <asm/xen/hypervisor.h> 22 24 #include <asm/xen/hypercall.h> 23 25 #include <asm/xen/cpuid.h>

+2

drivers/clocksource/hyperv_timer.c

··· 535 535 sched_clock_register(sched_clock, 64, NSEC_PER_SEC); 536 536 } 537 537 #elif defined CONFIG_PARAVIRT 538 + #include <asm/timer.h> 539 + 538 540 static __always_inline void hv_setup_sched_clock(void *sched_clock) 539 541 { 540 542 /* We're on x86/x64 *and* using PV ops */

+1 -1

drivers/xen/time.c

··· 8 8 #include <linux/gfp.h> 9 9 #include <linux/slab.h> 10 10 #include <linux/static_call.h> 11 + #include <linux/sched/cputime.h> 11 12 12 - #include <asm/paravirt.h> 13 13 #include <asm/xen/hypervisor.h> 14 14 #include <asm/xen/hypercall.h> 15 15

+18

include/linux/sched/cputime.h

··· 2 2 #ifndef _LINUX_SCHED_CPUTIME_H 3 3 #define _LINUX_SCHED_CPUTIME_H 4 4 5 + #include <linux/static_call_types.h> 5 6 #include <linux/sched/signal.h> 6 7 7 8 /* ··· 180 179 181 180 extern unsigned long long 182 181 task_sched_runtime(struct task_struct *task); 182 + 183 + #ifdef CONFIG_PARAVIRT 184 + struct static_key; 185 + extern struct static_key paravirt_steal_enabled; 186 + extern struct static_key paravirt_steal_rq_enabled; 187 + 188 + #ifdef CONFIG_HAVE_PV_STEAL_CLOCK_GEN 189 + u64 dummy_steal_clock(int cpu); 190 + 191 + DECLARE_STATIC_CALL(pv_steal_clock, dummy_steal_clock); 192 + 193 + static inline u64 paravirt_steal_clock(int cpu) 194 + { 195 + return static_call(pv_steal_clock)(cpu); 196 + } 197 + #endif 198 + #endif 183 199 184 200 #endif /* _LINUX_SCHED_CPUTIME_H */

+5

kernel/sched/core.c

··· 778 778 * RQ-clock updating methods: 779 779 */ 780 780 781 + /* Use CONFIG_PARAVIRT as this will avoid more #ifdef in arch code. */ 782 + #ifdef CONFIG_PARAVIRT 783 + struct static_key paravirt_steal_rq_enabled; 784 + #endif 785 + 781 786 static void update_rq_clock_task(struct rq *rq, s64 delta) 782 787 { 783 788 /*

+13

kernel/sched/cputime.c

··· 252 252 * ticks are not redelivered later. Due to that, this function may on 253 253 * occasion account more time than the calling functions think elapsed. 254 254 */ 255 + #ifdef CONFIG_PARAVIRT 256 + struct static_key paravirt_steal_enabled; 257 + 258 + #ifdef CONFIG_HAVE_PV_STEAL_CLOCK_GEN 259 + static u64 native_steal_clock(int cpu) 260 + { 261 + return 0; 262 + } 263 + 264 + DEFINE_STATIC_CALL(pv_steal_clock, native_steal_clock); 265 + #endif 266 + #endif 267 + 255 268 static __always_inline u64 steal_account_process_time(u64 maxtime) 256 269 { 257 270 #ifdef CONFIG_PARAVIRT

+1 -2

kernel/sched/sched.h

··· 86 86 struct sched_group; 87 87 struct cpuidle_state; 88 88 89 - #ifdef CONFIG_PARAVIRT 89 + #if defined(CONFIG_PARAVIRT) && !defined(CONFIG_HAVE_PV_STEAL_CLOCK_GEN) 90 90 # include <asm/paravirt.h> 91 - # include <asm/paravirt_api_clock.h> 92 91 #endif 93 92 94 93 #include <asm/barrier.h>

+6 -2

tools/objtool/arch/x86/decode.c

··· 711 711 immr = find_reloc_by_dest(elf, (void *)sec, offset+3); 712 712 disp = find_reloc_by_dest(elf, (void *)sec, offset+7); 713 713 714 - if (!immr || strcmp(immr->sym->name, "pv_ops")) 714 + if (!immr || strncmp(immr->sym->name, "pv_ops", 6)) 715 715 break; 716 716 717 - idx = (reloc_addend(immr) + 8) / sizeof(void *); 717 + idx = pv_ops_idx_off(immr->sym->name); 718 + if (idx < 0) 719 + break; 720 + 721 + idx += (reloc_addend(immr) + 8) / sizeof(void *); 718 722 719 723 func = disp->sym; 720 724 if (disp->sym->type == STT_SECTION)

+59 -19

tools/objtool/check.c

··· 521 521 } 522 522 523 523 /* 524 - * Read the pv_ops[] .data table to find the static initialized values. 524 + * Known pv_ops*[] arrays. 525 525 */ 526 - static int add_pv_ops(struct objtool_file *file, const char *symname) 526 + static struct { 527 + const char *name; 528 + int idx_off; 529 + } pv_ops_tables[] = { 530 + { .name = "pv_ops", }, 531 + { .name = "pv_ops_lock", }, 532 + { .name = NULL, .idx_off = -1 } 533 + }; 534 + 535 + /* 536 + * Get index offset for a pv_ops* array. 537 + */ 538 + int pv_ops_idx_off(const char *symname) 539 + { 540 + int idx; 541 + 542 + for (idx = 0; pv_ops_tables[idx].name; idx++) { 543 + if (!strcmp(symname, pv_ops_tables[idx].name)) 544 + break; 545 + } 546 + 547 + return pv_ops_tables[idx].idx_off; 548 + } 549 + 550 + /* 551 + * Read a pv_ops*[] .data table to find the static initialized values. 552 + */ 553 + static int add_pv_ops(struct objtool_file *file, int pv_ops_idx) 527 554 { 528 555 struct symbol *sym, *func; 529 556 unsigned long off, end; 530 557 struct reloc *reloc; 531 - int idx; 558 + int idx, idx_off; 559 + const char *symname; 532 560 561 + symname = pv_ops_tables[pv_ops_idx].name; 533 562 sym = find_symbol_by_name(file->elf, symname); 534 - if (!sym) 535 - return 0; 563 + if (!sym) { 564 + ERROR("Unknown pv_ops array %s", symname); 565 + return -1; 566 + } 536 567 537 568 off = sym->offset; 538 569 end = off + sym->len; 570 + idx_off = pv_ops_tables[pv_ops_idx].idx_off; 571 + if (idx_off < 0) { 572 + ERROR("pv_ops array %s has unknown index offset", symname); 573 + return -1; 574 + } 575 + 539 576 for (;;) { 540 577 reloc = find_reloc_by_dest_range(file->elf, sym->sec, off, end - off); 541 578 if (!reloc) ··· 590 553 return -1; 591 554 } 592 555 593 - if (objtool_pv_add(file, idx, func)) 556 + if (objtool_pv_add(file, idx + idx_off, func)) 594 557 return -1; 595 558 596 559 off = reloc_offset(reloc) + 1; ··· 606 569 */ 607 570 static int init_pv_ops(struct objtool_file *file) 608 571 { 609 - static const char *pv_ops_tables[] = { 610 - "pv_ops", 611 - "xen_cpu_ops", 612 - "xen_irq_ops", 613 - "xen_mmu_ops", 614 - NULL, 615 - }; 616 - const char *pv_ops; 617 572 struct symbol *sym; 618 573 int idx, nr; 619 574 ··· 614 585 615 586 file->pv_ops = NULL; 616 587 617 - sym = find_symbol_by_name(file->elf, "pv_ops"); 618 - if (!sym) 588 + nr = 0; 589 + for (idx = 0; pv_ops_tables[idx].name; idx++) { 590 + sym = find_symbol_by_name(file->elf, pv_ops_tables[idx].name); 591 + if (!sym) { 592 + pv_ops_tables[idx].idx_off = -1; 593 + continue; 594 + } 595 + pv_ops_tables[idx].idx_off = nr; 596 + nr += sym->len / sizeof(unsigned long); 597 + } 598 + 599 + if (nr == 0) 619 600 return 0; 620 601 621 - nr = sym->len / sizeof(unsigned long); 622 602 file->pv_ops = calloc(nr, sizeof(struct pv_state)); 623 603 if (!file->pv_ops) { 624 604 ERROR_GLIBC("calloc"); ··· 637 599 for (idx = 0; idx < nr; idx++) 638 600 INIT_LIST_HEAD(&file->pv_ops[idx].targets); 639 601 640 - for (idx = 0; (pv_ops = pv_ops_tables[idx]); idx++) { 641 - if (add_pv_ops(file, pv_ops)) 602 + for (idx = 0; pv_ops_tables[idx].name; idx++) { 603 + if (pv_ops_tables[idx].idx_off < 0) 604 + continue; 605 + if (add_pv_ops(file, idx)) 642 606 return -1; 643 607 } 644 608

+1

tools/objtool/include/objtool/check.h

··· 159 159 160 160 extern size_t sym_name_max_len; 161 161 extern struct disas_context *objtool_disas_ctx; 162 + int pv_ops_idx_off(const char *symname); 162 163 163 164 #endif /* _CHECK_H */

Configure Feed

Configure Feed