From nobody Fri Jun 19 20:13:14 2026 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id 4B5B8C433EF for ; Tue, 29 Mar 2022 23:51:23 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S241183AbiC2Xww (ORCPT ); Tue, 29 Mar 2022 19:52:52 -0400 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:43244 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S239629AbiC2Xwt (ORCPT ); Tue, 29 Mar 2022 19:52:49 -0400 Received: from mail-pg1-x54a.google.com (mail-pg1-x54a.google.com [IPv6:2607:f8b0:4864:20::54a]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id 00DA01CAF14 for ; Tue, 29 Mar 2022 16:51:05 -0700 (PDT) Received: by mail-pg1-x54a.google.com with SMTP id p9-20020a63f449000000b0035ec8c16f0bso1266013pgk.11 for ; Tue, 29 Mar 2022 16:51:05 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=google.com; s=20210112; h=reply-to:date:in-reply-to:message-id:mime-version:references :subject:from:to:cc; bh=xn6dD0NDRkWndk+qkMADvSMilfrLo/OBZLvoP8lMgT4=; b=Zv6ZvG4i+G6qdX4EyMXuTo4rE1/1buf4+IA8KvqZucAn6jB+7rx1ho6dvdO/bZWd+y 9kEN1uShJTlpKhNHX4Zu/v910yYCZS8JI86sfUKOfiJU7KgBiEca7WkJ3MXzUeq1qZly klS4V9lj/zuBmWxAgjvvRzDTGmxq4A0y9KlO2P/QG6fxNlIkXXaXboA/fi7lUSQiRoFd +Bmsf/UjchaUHOoi/YxbWJqOhu0IBRf8YSgVfqc6T5usvIUdbWPdSYh08Qt7KYwu8UGB kxu1zlaWPYgVNdZY6VYFMUR6N1oJJNnQitrdP4c4xmZC4dseEBrcDtGV5+Q3APUio0vV 1AyA== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20210112; h=x-gm-message-state:reply-to:date:in-reply-to:message-id :mime-version:references:subject:from:to:cc; bh=xn6dD0NDRkWndk+qkMADvSMilfrLo/OBZLvoP8lMgT4=; b=oEpfPbDLcjgr32ewT2hYjDygm5xJCI58vdOcB+mcW/crxG5Xinudp2k/ymUz0ajDIF Pyrm2prf5X393CpBzRz2DpxPy/+b6AeMgnmb7BTWyccChUa5u9a1IvrGvT5d09YfeaNu i0J4MbdXQPghpyG1ytttUPX7NZlxL7qNGOy2Lyi2BOvqctkIpvs/Z7HaoHm/wz9nWV4n CwtIfNIe6Ztmho6vSpTMyYmzFLIL0qTN9Tere8sY8DiEVDUfgfVqn6XkBPcFIQeUbvE7 KRglu8CoGYtXEXj0UVKHaSViY5Sp0F0CwD7IhmmfG2uRlm1QgQc93eC6veYFe4Pf9vaY yb3Q== X-Gm-Message-State: AOAM532nBuD6htMF7J59b2bOsyuYZz7bwFhI4STAMHF/huDoA8qvxohA WKgN/lEHs4XxaFytjJNyRHf/DmrbBd8= X-Google-Smtp-Source: ABdhPJz8DKtXUDzVuwe4lXRByg3SXE/K90HbdG1Lp6s5S6ujIgdHPcCZ3v2xBfAmrxf+cEch6FwfDIdQ4o8= X-Received: from seanjc.c.googlers.com ([fda3:e722:ac3:cc00:7f:e700:c0a8:3e5]) (user=seanjc job=sendgmr) by 2002:a17:90b:3a81:b0:1c6:834e:cd61 with SMTP id om1-20020a17090b3a8100b001c6834ecd61mr1699386pjb.149.1648597865466; Tue, 29 Mar 2022 16:51:05 -0700 (PDT) Reply-To: Sean Christopherson Date: Tue, 29 Mar 2022 23:50:51 +0000 In-Reply-To: <20220329235054.3534728-1-seanjc@google.com> Message-Id: <20220329235054.3534728-2-seanjc@google.com> Mime-Version: 1.0 References: <20220329235054.3534728-1-seanjc@google.com> X-Mailer: git-send-email 2.35.1.1021.g381101b075-goog Subject: [PATCH v3.1 1/4] KVM: x86: Move kvm_ops_static_call_update() to x86.c From: Sean Christopherson To: Paolo Bonzini Cc: Sean Christopherson , Vitaly Kuznetsov , Wanpeng Li , Jim Mattson , Joerg Roedel , kvm@vger.kernel.org, linux-kernel@vger.kernel.org, Like Xu Precedence: bulk List-ID: X-Mailing-List: linux-kernel@vger.kernel.org Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" From: Like Xu The kvm_ops_static_call_update() is defined in kvm_host.h. That's completely unnecessary, it should have exactly one caller, kvm_arch_hardware_setup(). Move the helper to x86.c and have it do the actual memcpy() of the ops in addition to the static call updates. This will also allow for cleanly giving kvm_pmu_ops static_call treatment. Suggested-by: Sean Christopherson Signed-off-by: Like Xu [sean: Move memcpy() into the helper and rename accordingly] Signed-off-by: Sean Christopherson --- arch/x86/include/asm/kvm_host.h | 14 -------------- arch/x86/kvm/x86.c | 19 +++++++++++++++++-- 2 files changed, 17 insertions(+), 16 deletions(-) diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_hos= t.h index 9694dd5e6ccc..df4e057b0417 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -1557,20 +1557,6 @@ extern struct kvm_x86_ops kvm_x86_ops; #define KVM_X86_OP_OPTIONAL_RET0 KVM_X86_OP #include =20 -static inline void kvm_ops_static_call_update(void) -{ -#define __KVM_X86_OP(func) \ - static_call_update(kvm_x86_##func, kvm_x86_ops.func); -#define KVM_X86_OP(func) \ - WARN_ON(!kvm_x86_ops.func); __KVM_X86_OP(func) -#define KVM_X86_OP_OPTIONAL __KVM_X86_OP -#define KVM_X86_OP_OPTIONAL_RET0(func) \ - static_call_update(kvm_x86_##func, (void *)kvm_x86_ops.func ? : \ - (void *)__static_call_return0); -#include -#undef __KVM_X86_OP -} - #define __KVM_HAVE_ARCH_VM_ALLOC static inline struct kvm *kvm_arch_alloc_vm(void) { diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index d3a9ce07a565..99aa2d16845a 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -11595,6 +11595,22 @@ void kvm_arch_hardware_disable(void) drop_user_return_notifiers(); } =20 +static inline void kvm_ops_update(struct kvm_x86_init_ops *ops) +{ + memcpy(&kvm_x86_ops, ops->runtime_ops, sizeof(kvm_x86_ops)); + +#define __KVM_X86_OP(func) \ + static_call_update(kvm_x86_##func, kvm_x86_ops.func); +#define KVM_X86_OP(func) \ + WARN_ON(!kvm_x86_ops.func); __KVM_X86_OP(func) +#define KVM_X86_OP_OPTIONAL __KVM_X86_OP +#define KVM_X86_OP_OPTIONAL_RET0(func) \ + static_call_update(kvm_x86_##func, (void *)kvm_x86_ops.func ? : \ + (void *)__static_call_return0); +#include +#undef __KVM_X86_OP +} + int kvm_arch_hardware_setup(void *opaque) { struct kvm_x86_init_ops *ops =3D opaque; @@ -11609,8 +11625,7 @@ int kvm_arch_hardware_setup(void *opaque) if (r !=3D 0) return r; =20 - memcpy(&kvm_x86_ops, ops->runtime_ops, sizeof(kvm_x86_ops)); - kvm_ops_static_call_update(); + kvm_ops_update(ops); =20 kvm_register_perf_callbacks(ops->handle_intel_pt_intr); =20 --=20 2.35.1.1021.g381101b075-goog From nobody Fri Jun 19 20:13:14 2026 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id 738A7C43217 for ; Tue, 29 Mar 2022 23:51:27 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S241255AbiC2XxJ (ORCPT ); Tue, 29 Mar 2022 19:53:09 -0400 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:43444 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S241182AbiC2Xww (ORCPT ); Tue, 29 Mar 2022 19:52:52 -0400 Received: from mail-pl1-x649.google.com (mail-pl1-x649.google.com [IPv6:2607:f8b0:4864:20::649]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id 1EA7B205BD0 for ; Tue, 29 Mar 2022 16:51:08 -0700 (PDT) Received: by mail-pl1-x649.google.com with SMTP id g1-20020a170902fe0100b00153f95629f7so8039785plj.12 for ; Tue, 29 Mar 2022 16:51:08 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=google.com; s=20210112; h=reply-to:date:in-reply-to:message-id:mime-version:references :subject:from:to:cc; bh=rUm8X861zRhcP7IH2r62TLmqODWxaakk5bqj/4FKees=; b=lp+161YCBaksKfEsnop3WkW2sEbYQsYlWsIIuCpFR565hGaN4ctyrwOY9c+qBVVxN7 u398mF45ljWlgcx64gdh62+mdMl2nuwyT8FHRa4HvIccIqkdeqgLqb/PV620/wm5eb2J JiyBcN6V/RGFa/m4VwCCpJ7LFCyCWrxfd1Nc63OKDa6jTTlMz5X+rK3yuiDhnSacBNpO eH8LXepxAiEpd0i1nc7SRogXfkLYeIjCfVeJ+wShhqWXthXGdEQ3+yF8g5bs0HfPEAwy 18RRAypPvc17L9Hb6tP74IZ9KHhlitfn9yRfgjJUAVp9YH9Ad9+DNMPxjvLAjM1FvZg7 HBkw== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20210112; h=x-gm-message-state:reply-to:date:in-reply-to:message-id :mime-version:references:subject:from:to:cc; bh=rUm8X861zRhcP7IH2r62TLmqODWxaakk5bqj/4FKees=; b=4Cc2zT2WaX+NunX2Vm6Xw531M19H0AO1zNFRWRDuB0GalxIIGeLxZAA1XBBjveJiWk HLEGt+heN9mXLF+Hd4DGu+oCaB/q8IRoRKp/gY97RYoifmKTx3tS4e6oaNFRgCZV0CH/ aaa02VdTvTNZm0+K5RvVybdWSDhMlfgiUqNq9fcthlHndnYK5jW7SFjDU/DYK9FDdrG/ Lj44Jlh2LcQ27GQ8KdSIZwb81QlqX0RFCGucO+nfi7opJQXTFVO5MGnHONDjLLTaZTRJ hsxSJU86YBhjXEdSjZH1dVbgd6wBL4gVCtcEKtMotu2SYEvlqsesnXXclkXcPnZgyzAn 96FA== X-Gm-Message-State: AOAM530ymuY8oWAiG8iumGExa3VIfnCvw6vrPEgj9cV3o8GuW5/Adj9F QkACA7hY9JicApsV1doQFKBSdxlAvu4= X-Google-Smtp-Source: ABdhPJx+bv7YCHgDCbGLE+StVp2R1OFLwih633YA0wO8603nhVXNZi9ZdvZc6kSwNCXUPR7imoqSBzZpdTY= X-Received: from seanjc.c.googlers.com ([fda3:e722:ac3:cc00:7f:e700:c0a8:3e5]) (user=seanjc job=sendgmr) by 2002:a17:90a:858b:b0:1c6:5bc8:781a with SMTP id m11-20020a17090a858b00b001c65bc8781amr166639pjn.0.1648597867298; Tue, 29 Mar 2022 16:51:07 -0700 (PDT) Reply-To: Sean Christopherson Date: Tue, 29 Mar 2022 23:50:52 +0000 In-Reply-To: <20220329235054.3534728-1-seanjc@google.com> Message-Id: <20220329235054.3534728-3-seanjc@google.com> Mime-Version: 1.0 References: <20220329235054.3534728-1-seanjc@google.com> X-Mailer: git-send-email 2.35.1.1021.g381101b075-goog Subject: [PATCH v3.1 2/4] KVM: x86: Copy kvm_pmu_ops by value to eliminate layer of indirection From: Sean Christopherson To: Paolo Bonzini Cc: Sean Christopherson , Vitaly Kuznetsov , Wanpeng Li , Jim Mattson , Joerg Roedel , kvm@vger.kernel.org, linux-kernel@vger.kernel.org, Like Xu Precedence: bulk List-ID: X-Mailing-List: linux-kernel@vger.kernel.org Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" From: Like Xu Replace the kvm_pmu_ops pointer in common x86 with an instance of the struct to save one pointer dereference when invoking functions. Copy the struct by value to set the ops during kvm_init(). Signed-off-by: Like Xu [sean: Move pmc_is_enabled(), make kvm_pmu_ops static] Signed-off-by: Sean Christopherson --- arch/x86/kvm/pmu.c | 54 ++++++++++++++++++++++++++++------------------ arch/x86/kvm/pmu.h | 7 ++---- arch/x86/kvm/x86.c | 2 ++ 3 files changed, 37 insertions(+), 26 deletions(-) diff --git a/arch/x86/kvm/pmu.c b/arch/x86/kvm/pmu.c index eca39f56c231..bb0b1ad0fda5 100644 --- a/arch/x86/kvm/pmu.c +++ b/arch/x86/kvm/pmu.c @@ -49,6 +49,18 @@ * * AMD: [0 .. AMD64_NUM_COUNTERS-1] <=3D> gp counters */ =20 +static struct kvm_pmu_ops kvm_pmu_ops __read_mostly; + +void kvm_pmu_ops_update(const struct kvm_pmu_ops *pmu_ops) +{ + memcpy(&kvm_pmu_ops, pmu_ops, sizeof(kvm_pmu_ops)); +} + +static inline bool pmc_is_enabled(struct kvm_pmc *pmc) +{ + return kvm_pmu_ops.pmc_is_enabled(pmc); +} + static void kvm_pmi_trigger_fn(struct irq_work *irq_work) { struct kvm_pmu *pmu =3D container_of(irq_work, struct kvm_pmu, irq_work); @@ -213,7 +225,7 @@ void reprogram_gp_counter(struct kvm_pmc *pmc, u64 even= tsel) ARCH_PERFMON_EVENTSEL_CMASK | HSW_IN_TX | HSW_IN_TX_CHECKPOINTED))) { - config =3D kvm_x86_ops.pmu_ops->pmc_perf_hw_id(pmc); + config =3D kvm_pmu_ops.pmc_perf_hw_id(pmc); if (config !=3D PERF_COUNT_HW_MAX) type =3D PERF_TYPE_HARDWARE; } @@ -263,7 +275,7 @@ void reprogram_fixed_counter(struct kvm_pmc *pmc, u8 ct= rl, int idx) =20 pmc->current_config =3D (u64)ctrl; pmc_reprogram_counter(pmc, PERF_TYPE_HARDWARE, - kvm_x86_ops.pmu_ops->pmc_perf_hw_id(pmc), + kvm_pmu_ops.pmc_perf_hw_id(pmc), !(en_field & 0x2), /* exclude user */ !(en_field & 0x1), /* exclude kernel */ pmi); @@ -272,7 +284,7 @@ EXPORT_SYMBOL_GPL(reprogram_fixed_counter); =20 void reprogram_counter(struct kvm_pmu *pmu, int pmc_idx) { - struct kvm_pmc *pmc =3D kvm_x86_ops.pmu_ops->pmc_idx_to_pmc(pmu, pmc_idx); + struct kvm_pmc *pmc =3D kvm_pmu_ops.pmc_idx_to_pmc(pmu, pmc_idx); =20 if (!pmc) return; @@ -294,7 +306,7 @@ void kvm_pmu_handle_event(struct kvm_vcpu *vcpu) int bit; =20 for_each_set_bit(bit, pmu->reprogram_pmi, X86_PMC_IDX_MAX) { - struct kvm_pmc *pmc =3D kvm_x86_ops.pmu_ops->pmc_idx_to_pmc(pmu, bit); + struct kvm_pmc *pmc =3D kvm_pmu_ops.pmc_idx_to_pmc(pmu, bit); =20 if (unlikely(!pmc || !pmc->perf_event)) { clear_bit(bit, pmu->reprogram_pmi); @@ -316,7 +328,7 @@ void kvm_pmu_handle_event(struct kvm_vcpu *vcpu) /* check if idx is a valid index to access PMU */ bool kvm_pmu_is_valid_rdpmc_ecx(struct kvm_vcpu *vcpu, unsigned int idx) { - return kvm_x86_ops.pmu_ops->is_valid_rdpmc_ecx(vcpu, idx); + return kvm_pmu_ops.is_valid_rdpmc_ecx(vcpu, idx); } =20 bool is_vmware_backdoor_pmc(u32 pmc_idx) @@ -366,7 +378,7 @@ int kvm_pmu_rdpmc(struct kvm_vcpu *vcpu, unsigned idx, = u64 *data) if (is_vmware_backdoor_pmc(idx)) return kvm_pmu_rdpmc_vmware(vcpu, idx, data); =20 - pmc =3D kvm_x86_ops.pmu_ops->rdpmc_ecx_to_pmc(vcpu, idx, &mask); + pmc =3D kvm_pmu_ops.rdpmc_ecx_to_pmc(vcpu, idx, &mask); if (!pmc) return 1; =20 @@ -382,22 +394,22 @@ int kvm_pmu_rdpmc(struct kvm_vcpu *vcpu, unsigned idx= , u64 *data) void kvm_pmu_deliver_pmi(struct kvm_vcpu *vcpu) { if (lapic_in_kernel(vcpu)) { - if (kvm_x86_ops.pmu_ops->deliver_pmi) - kvm_x86_ops.pmu_ops->deliver_pmi(vcpu); + if (kvm_pmu_ops.deliver_pmi) + kvm_pmu_ops.deliver_pmi(vcpu); kvm_apic_local_deliver(vcpu->arch.apic, APIC_LVTPC); } } =20 bool kvm_pmu_is_valid_msr(struct kvm_vcpu *vcpu, u32 msr) { - return kvm_x86_ops.pmu_ops->msr_idx_to_pmc(vcpu, msr) || - kvm_x86_ops.pmu_ops->is_valid_msr(vcpu, msr); + return kvm_pmu_ops.msr_idx_to_pmc(vcpu, msr) || + kvm_pmu_ops.is_valid_msr(vcpu, msr); } =20 static void kvm_pmu_mark_pmc_in_use(struct kvm_vcpu *vcpu, u32 msr) { struct kvm_pmu *pmu =3D vcpu_to_pmu(vcpu); - struct kvm_pmc *pmc =3D kvm_x86_ops.pmu_ops->msr_idx_to_pmc(vcpu, msr); + struct kvm_pmc *pmc =3D kvm_pmu_ops.msr_idx_to_pmc(vcpu, msr); =20 if (pmc) __set_bit(pmc->idx, pmu->pmc_in_use); @@ -405,13 +417,13 @@ static void kvm_pmu_mark_pmc_in_use(struct kvm_vcpu *= vcpu, u32 msr) =20 int kvm_pmu_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) { - return kvm_x86_ops.pmu_ops->get_msr(vcpu, msr_info); + return kvm_pmu_ops.get_msr(vcpu, msr_info); } =20 int kvm_pmu_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) { kvm_pmu_mark_pmc_in_use(vcpu, msr_info->index); - return kvm_x86_ops.pmu_ops->set_msr(vcpu, msr_info); + return kvm_pmu_ops.set_msr(vcpu, msr_info); } =20 /* refresh PMU settings. This function generally is called when underlying @@ -420,7 +432,7 @@ int kvm_pmu_set_msr(struct kvm_vcpu *vcpu, struct msr_d= ata *msr_info) */ void kvm_pmu_refresh(struct kvm_vcpu *vcpu) { - kvm_x86_ops.pmu_ops->refresh(vcpu); + kvm_pmu_ops.refresh(vcpu); } =20 void kvm_pmu_reset(struct kvm_vcpu *vcpu) @@ -428,7 +440,7 @@ void kvm_pmu_reset(struct kvm_vcpu *vcpu) struct kvm_pmu *pmu =3D vcpu_to_pmu(vcpu); =20 irq_work_sync(&pmu->irq_work); - kvm_x86_ops.pmu_ops->reset(vcpu); + kvm_pmu_ops.reset(vcpu); } =20 void kvm_pmu_init(struct kvm_vcpu *vcpu) @@ -436,7 +448,7 @@ void kvm_pmu_init(struct kvm_vcpu *vcpu) struct kvm_pmu *pmu =3D vcpu_to_pmu(vcpu); =20 memset(pmu, 0, sizeof(*pmu)); - kvm_x86_ops.pmu_ops->init(vcpu); + kvm_pmu_ops.init(vcpu); init_irq_work(&pmu->irq_work, kvm_pmi_trigger_fn); pmu->event_count =3D 0; pmu->need_cleanup =3D false; @@ -468,14 +480,14 @@ void kvm_pmu_cleanup(struct kvm_vcpu *vcpu) pmu->pmc_in_use, X86_PMC_IDX_MAX); =20 for_each_set_bit(i, bitmask, X86_PMC_IDX_MAX) { - pmc =3D kvm_x86_ops.pmu_ops->pmc_idx_to_pmc(pmu, i); + pmc =3D kvm_pmu_ops.pmc_idx_to_pmc(pmu, i); =20 if (pmc && pmc->perf_event && !pmc_speculative_in_use(pmc)) pmc_stop_counter(pmc); } =20 - if (kvm_x86_ops.pmu_ops->cleanup) - kvm_x86_ops.pmu_ops->cleanup(vcpu); + if (kvm_pmu_ops.cleanup) + kvm_pmu_ops.cleanup(vcpu); =20 bitmap_zero(pmu->pmc_in_use, X86_PMC_IDX_MAX); } @@ -505,7 +517,7 @@ static inline bool eventsel_match_perf_hw_id(struct kvm= _pmc *pmc, unsigned int config; =20 pmc->eventsel &=3D (ARCH_PERFMON_EVENTSEL_EVENT | ARCH_PERFMON_EVENTSEL_U= MASK); - config =3D kvm_x86_ops.pmu_ops->pmc_perf_hw_id(pmc); + config =3D kvm_pmu_ops.pmc_perf_hw_id(pmc); pmc->eventsel =3D old_eventsel; return config =3D=3D perf_hw_id; } @@ -533,7 +545,7 @@ void kvm_pmu_trigger_event(struct kvm_vcpu *vcpu, u64 p= erf_hw_id) int i; =20 for_each_set_bit(i, pmu->all_valid_pmc_idx, X86_PMC_IDX_MAX) { - pmc =3D kvm_x86_ops.pmu_ops->pmc_idx_to_pmc(pmu, i); + pmc =3D kvm_pmu_ops.pmc_idx_to_pmc(pmu, i); =20 if (!pmc || !pmc_is_enabled(pmc) || !pmc_speculative_in_use(pmc)) continue; diff --git a/arch/x86/kvm/pmu.h b/arch/x86/kvm/pmu.h index 7a7b8d5b775e..cbe9987f9cba 100644 --- a/arch/x86/kvm/pmu.h +++ b/arch/x86/kvm/pmu.h @@ -41,6 +41,8 @@ struct kvm_pmu_ops { void (*cleanup)(struct kvm_vcpu *vcpu); }; =20 +void kvm_pmu_ops_update(const struct kvm_pmu_ops *pmu_ops); + static inline u64 pmc_bitmask(struct kvm_pmc *pmc) { struct kvm_pmu *pmu =3D pmc_to_pmu(pmc); @@ -88,11 +90,6 @@ static inline bool pmc_is_fixed(struct kvm_pmc *pmc) return pmc->type =3D=3D KVM_PMC_FIXED; } =20 -static inline bool pmc_is_enabled(struct kvm_pmc *pmc) -{ - return kvm_x86_ops.pmu_ops->pmc_is_enabled(pmc); -} - static inline bool kvm_valid_perf_global_ctrl(struct kvm_pmu *pmu, u64 data) { diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 99aa2d16845a..6f1676fab6c5 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -11609,6 +11609,8 @@ static inline void kvm_ops_update(struct kvm_x86_in= it_ops *ops) (void *)__static_call_return0); #include #undef __KVM_X86_OP + + kvm_pmu_ops_update(ops->runtime_ops->pmu_ops); } =20 int kvm_arch_hardware_setup(void *opaque) --=20 2.35.1.1021.g381101b075-goog From nobody Fri Jun 19 20:13:14 2026 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id 0FF3DC433FE for ; Tue, 29 Mar 2022 23:51:30 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S241266AbiC2XxM (ORCPT ); Tue, 29 Mar 2022 19:53:12 -0400 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:43490 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S241189AbiC2Xwx (ORCPT ); Tue, 29 Mar 2022 19:52:53 -0400 Received: from mail-pj1-x1049.google.com (mail-pj1-x1049.google.com [IPv6:2607:f8b0:4864:20::1049]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id 7BC67204C8E for ; Tue, 29 Mar 2022 16:51:09 -0700 (PDT) Received: by mail-pj1-x1049.google.com with SMTP id om8-20020a17090b3a8800b001c68e7ccd5fso292220pjb.9 for ; Tue, 29 Mar 2022 16:51:09 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=google.com; s=20210112; h=reply-to:date:in-reply-to:message-id:mime-version:references :subject:from:to:cc; bh=oCYui+K+d0TSC1DG5I9ne/xMFV78SC7XMDQtja8p6Lk=; b=osZZVurOhJxG1NLStWVQcssxNu2a5OVDaZvh9gOBHWIhWM0ku9Yz9uvqF5cqkYlJQi 0NiI5LqRaioj+nicjews9aurISSC30CoI61Cs/C7sqEXjWaqAZdJMDPAZXke4jRCHrNy Bz0eUsP76Oob4fqShCSh9vogUbMkfXTGQLcNxEAvkhbScluZH6AFBxt/dFtJQ5ydsO65 LFwiwqzvjOm+AlwNMHdfHEycfoNddpPqg5opYWVCEQhRxGLsMVoe5KiWdY3b/eUwulAD P1K1exLKD6fCPHeqYI/gctUlCE3YO8WtTrzH+CNNOv8crZk3Cl2LbgbU5VVDugkrO8Yi Eq3Q== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20210112; h=x-gm-message-state:reply-to:date:in-reply-to:message-id :mime-version:references:subject:from:to:cc; bh=oCYui+K+d0TSC1DG5I9ne/xMFV78SC7XMDQtja8p6Lk=; b=WjO7WdzC5KkDhqDanaTdBcndUrWNraxIFPPVfo2XTG8j7iTyirDZ6pvp1L7tDF0gJJ uYlqiCGKECLF5U220PAKe0YdObBfNP+NG6qu3mevr5vKm2tk0IL0NyPdryV5Rhqd9ikp ROivEVqurcgYmW29vBwf0ZAe6/3TY2Vr6W/0YvMMB1XnfKWEfFKMkg3ZteD+YyyDWIzo c4x5Hvigy/Gnjdpq8pYFYip71nelx0aLwupPqgMoAn36zASZAtVGwVNR3rXR28PcK4d+ GO+40lmn855CK4P42gj2n+2eZRk5coMY6hKyNsiDknDRrudcr0cZyhTB8phNJ+FwS0HP zJ4g== X-Gm-Message-State: AOAM5301/PXJiOUj8OXPDEDojH3wUTSWPJllzaSqikZBUozyqwUQWfqv 3EMwy43qm4nesGaVEvQk+49t6/xUYdE= X-Google-Smtp-Source: ABdhPJzdAGuJKJuW0xfREciH6xAeAYULBy5N6X6BzKXaQe7BO12MoNSDYd2DUW0kGq66425lPOZ1v214ohs= X-Received: from seanjc.c.googlers.com ([fda3:e722:ac3:cc00:7f:e700:c0a8:3e5]) (user=seanjc job=sendgmr) by 2002:a17:90b:1a89:b0:1c6:4398:673 with SMTP id ng9-20020a17090b1a8900b001c643980673mr1777594pjb.40.1648597869060; Tue, 29 Mar 2022 16:51:09 -0700 (PDT) Reply-To: Sean Christopherson Date: Tue, 29 Mar 2022 23:50:53 +0000 In-Reply-To: <20220329235054.3534728-1-seanjc@google.com> Message-Id: <20220329235054.3534728-4-seanjc@google.com> Mime-Version: 1.0 References: <20220329235054.3534728-1-seanjc@google.com> X-Mailer: git-send-email 2.35.1.1021.g381101b075-goog Subject: [PATCH v3.1 3/4] KVM: x86: Move .pmu_ops to kvm_x86_init_ops and tag as __initdata From: Sean Christopherson To: Paolo Bonzini Cc: Sean Christopherson , Vitaly Kuznetsov , Wanpeng Li , Jim Mattson , Joerg Roedel , kvm@vger.kernel.org, linux-kernel@vger.kernel.org, Like Xu Precedence: bulk List-ID: X-Mailing-List: linux-kernel@vger.kernel.org Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" From: Like Xu The pmu_ops should be moved to kvm_x86_init_ops and tagged as __initdata. That'll save those precious few bytes, and more importantly make the original ops unreachable, i.e. make it harder to sneak in post-init modification bugs. Suggested-by: Sean Christopherson Signed-off-by: Like Xu Reviewed-by: Sean Christopherson Signed-off-by: Sean Christopherson --- arch/x86/include/asm/kvm_host.h | 3 +-- arch/x86/kvm/svm/pmu.c | 2 +- arch/x86/kvm/svm/svm.c | 2 +- arch/x86/kvm/vmx/pmu_intel.c | 2 +- arch/x86/kvm/vmx/vmx.c | 2 +- arch/x86/kvm/x86.c | 2 +- 6 files changed, 6 insertions(+), 7 deletions(-) diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_hos= t.h index df4e057b0417..5bb5843d460a 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -1464,8 +1464,6 @@ struct kvm_x86_ops { int cpu_dirty_log_size; void (*update_cpu_dirty_logging)(struct kvm_vcpu *vcpu); =20 - /* pmu operations of sub-arch */ - const struct kvm_pmu_ops *pmu_ops; const struct kvm_x86_nested_ops *nested_ops; =20 void (*vcpu_blocking)(struct kvm_vcpu *vcpu); @@ -1536,6 +1534,7 @@ struct kvm_x86_init_ops { unsigned int (*handle_intel_pt_intr)(void); =20 struct kvm_x86_ops *runtime_ops; + struct kvm_pmu_ops *pmu_ops; }; =20 struct kvm_arch_async_pf { diff --git a/arch/x86/kvm/svm/pmu.c b/arch/x86/kvm/svm/pmu.c index 24eb935b6f85..57ab4739eb19 100644 --- a/arch/x86/kvm/svm/pmu.c +++ b/arch/x86/kvm/svm/pmu.c @@ -319,7 +319,7 @@ static void amd_pmu_reset(struct kvm_vcpu *vcpu) } } =20 -struct kvm_pmu_ops amd_pmu_ops =3D { +struct kvm_pmu_ops amd_pmu_ops __initdata =3D { .pmc_perf_hw_id =3D amd_pmc_perf_hw_id, .pmc_is_enabled =3D amd_pmc_is_enabled, .pmc_idx_to_pmc =3D amd_pmc_idx_to_pmc, diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index 6535adee3e9c..f3c9aa8624d8 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -4625,7 +4625,6 @@ static struct kvm_x86_ops svm_x86_ops __initdata =3D { =20 .sched_in =3D svm_sched_in, =20 - .pmu_ops =3D &amd_pmu_ops, .nested_ops =3D &svm_nested_ops, =20 .deliver_interrupt =3D svm_deliver_interrupt, @@ -4906,6 +4905,7 @@ static struct kvm_x86_init_ops svm_init_ops __initdat= a =3D { .check_processor_compatibility =3D svm_check_processor_compat, =20 .runtime_ops =3D &svm_x86_ops, + .pmu_ops =3D &amd_pmu_ops, }; =20 static int __init svm_init(void) diff --git a/arch/x86/kvm/vmx/pmu_intel.c b/arch/x86/kvm/vmx/pmu_intel.c index efa172a7278e..c766c9b3083a 100644 --- a/arch/x86/kvm/vmx/pmu_intel.c +++ b/arch/x86/kvm/vmx/pmu_intel.c @@ -723,7 +723,7 @@ static void intel_pmu_cleanup(struct kvm_vcpu *vcpu) intel_pmu_release_guest_lbr_event(vcpu); } =20 -struct kvm_pmu_ops intel_pmu_ops =3D { +struct kvm_pmu_ops intel_pmu_ops __initdata =3D { .pmc_perf_hw_id =3D intel_pmc_perf_hw_id, .pmc_is_enabled =3D intel_pmc_is_enabled, .pmc_idx_to_pmc =3D intel_pmc_idx_to_pmc, diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index 84a7500cd80c..77ba0cfe588f 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -7823,7 +7823,6 @@ static struct kvm_x86_ops vmx_x86_ops __initdata =3D { .cpu_dirty_log_size =3D PML_ENTITY_NUM, .update_cpu_dirty_logging =3D vmx_update_cpu_dirty_logging, =20 - .pmu_ops =3D &intel_pmu_ops, .nested_ops =3D &vmx_nested_ops, =20 .pi_update_irte =3D vmx_pi_update_irte, @@ -8078,6 +8077,7 @@ static struct kvm_x86_init_ops vmx_init_ops __initdat= a =3D { .handle_intel_pt_intr =3D NULL, =20 .runtime_ops =3D &vmx_x86_ops, + .pmu_ops =3D &intel_pmu_ops, }; =20 static void vmx_cleanup_l1d_flush(void) diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 6f1676fab6c5..ca48cf705917 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -11610,7 +11610,7 @@ static inline void kvm_ops_update(struct kvm_x86_in= it_ops *ops) #include #undef __KVM_X86_OP =20 - kvm_pmu_ops_update(ops->runtime_ops->pmu_ops); + kvm_pmu_ops_update(ops->pmu_ops); } =20 int kvm_arch_hardware_setup(void *opaque) --=20 2.35.1.1021.g381101b075-goog From nobody Fri Jun 19 20:13:14 2026 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id 7DCE6C433EF for ; Tue, 29 Mar 2022 23:51:35 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S241275AbiC2XxQ (ORCPT ); Tue, 29 Mar 2022 19:53:16 -0400 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:43618 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S241197AbiC2Xwz (ORCPT ); Tue, 29 Mar 2022 19:52:55 -0400 Received: from mail-pl1-x649.google.com (mail-pl1-x649.google.com [IPv6:2607:f8b0:4864:20::649]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id 2AE67204C8E for ; Tue, 29 Mar 2022 16:51:11 -0700 (PDT) Received: by mail-pl1-x649.google.com with SMTP id b4-20020a170902e94400b0015309b5c481so8040777pll.6 for ; Tue, 29 Mar 2022 16:51:11 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=google.com; s=20210112; h=reply-to:date:in-reply-to:message-id:mime-version:references :subject:from:to:cc; bh=poq6HhoJ5C86Jvzu6FLzn1emdPhg3i74xPs1BHgN3lo=; b=FbYxYTHdbMKLV6p+W5MYMbinAsdWrgodXVcFSyNwOB3OYZ60lth5g55jSeiP4byD40 ZQD/wuclgwcIes+Nt5TB7SOGuxXUcnbQDrBOeArU6Ws3ASIb7+nS3DjzyBtj1z35NPXt oqngHr4/M1Wy6JhfMB4ZxEVi/S4EMUK17NtUHmkPOB1L0QsirotMxoGbzSCPLMNf1gOo 4FVe+6sCGrzeJhRx6BmKhxdlXjYSdIs+O3HI5HoroTu22rr9YzGGDU+5pEnf+7UB5mLE ABfOUgDNQ0n+rtBQOqgyK6BddM3lFT2ZiV3Vp4aIORJlZH/4Wrvdj3tch99Qe3ywX+1v xFGA== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20210112; h=x-gm-message-state:reply-to:date:in-reply-to:message-id :mime-version:references:subject:from:to:cc; bh=poq6HhoJ5C86Jvzu6FLzn1emdPhg3i74xPs1BHgN3lo=; b=8LLrGH6jpmsSWGBwAHfEnQyK7/ENHgd85cs68NiwV5j5gm0VLGFIAIFeLQAGBhwzcF Nr7dKqQPtN13QXMTZ6oQn0GpW8lKXu3q+sb7p4N82bYea9/d4wRUObtIkv7RgELChiUY 1LhkS3I+gibO9CB5ZHC3u2K5mu2HUMXjVnx30Dd5Bu4SL4WVz5Rqm8uPhMoXFZhPl5ge yLjpi1svqtJGwhJXK8NtRRccgKdla8upXS9lP2Sm1N/UFzAuMlH3KrWbBluAGKQpasws Bygf8l3UcVOy/lf1Thv1WBTp1plgNI1gq1D6VfK/1YTQanTtwXXB0QdRcYYBiqMo46W7 0wJw== X-Gm-Message-State: AOAM530tufzlG3GUpdU/F1SM51w8XGTvu9tnG6cPjyNv1HfwGcigshj0 nFSZsrKdYRdwSD/G93DHTs7thkKL4r4= X-Google-Smtp-Source: ABdhPJxNozzSCrFK/1c3W05ocHmHw/K4lDUBdSTbJsIHmX2gNG9K/ClvnDKF2ubC0OEkmnDRLVnh26RR+RQ= X-Received: from seanjc.c.googlers.com ([fda3:e722:ac3:cc00:7f:e700:c0a8:3e5]) (user=seanjc job=sendgmr) by 2002:a05:6a00:24c6:b0:4fd:9038:74f0 with SMTP id d6-20020a056a0024c600b004fd903874f0mr3483855pfv.63.1648597870721; Tue, 29 Mar 2022 16:51:10 -0700 (PDT) Reply-To: Sean Christopherson Date: Tue, 29 Mar 2022 23:50:54 +0000 In-Reply-To: <20220329235054.3534728-1-seanjc@google.com> Message-Id: <20220329235054.3534728-5-seanjc@google.com> Mime-Version: 1.0 References: <20220329235054.3534728-1-seanjc@google.com> X-Mailer: git-send-email 2.35.1.1021.g381101b075-goog Subject: [PATCH v3.1 4/4] KVM: x86: Use static calls to reduce kvm_pmu_ops overhead From: Sean Christopherson To: Paolo Bonzini Cc: Sean Christopherson , Vitaly Kuznetsov , Wanpeng Li , Jim Mattson , Joerg Roedel , kvm@vger.kernel.org, linux-kernel@vger.kernel.org, Like Xu Precedence: bulk List-ID: X-Mailing-List: linux-kernel@vger.kernel.org Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" From: Like Xu Use static calls to improve kvm_pmu_ops performance, following the same pattern and naming scheme used by kvm-x86-ops.h. Here are the worst fenced_rdtsc() cycles numbers for the kvm_pmu_ops functions that is most often called (up to 7 digits of calls) when running a single perf test case in a guest on an ICX 2.70GHz host (mitigations=3Don= ): | legacy | static call ------------------------------------------------------------ .pmc_idx_to_pmc | 1304840 | 994872 (+23%) .pmc_is_enabled | 978670 | 1011750 (-3%) .msr_idx_to_pmc | 47828 | 41690 (+12%) .is_valid_msr | 28786 | 30108 (-4%) Signed-off-by: Like Xu [sean: Handle static call updates in pmu.c, tweak changelog] Signed-off-by: Sean Christopherson --- arch/x86/include/asm/kvm-x86-pmu-ops.h | 31 ++++++++++++++ arch/x86/kvm/pmu.c | 56 ++++++++++++++++---------- 2 files changed, 65 insertions(+), 22 deletions(-) create mode 100644 arch/x86/include/asm/kvm-x86-pmu-ops.h diff --git a/arch/x86/include/asm/kvm-x86-pmu-ops.h b/arch/x86/include/asm/= kvm-x86-pmu-ops.h new file mode 100644 index 000000000000..fdfd8e06fee6 --- /dev/null +++ b/arch/x86/include/asm/kvm-x86-pmu-ops.h @@ -0,0 +1,31 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#if !defined(KVM_X86_PMU_OP) || !defined(KVM_X86_PMU_OP_OPTIONAL) +BUILD_BUG_ON(1) +#endif + +/* + * KVM_X86_PMU_OP() and KVM_X86_PMU_OP_OPTIONAL() are used to help generate + * both DECLARE/DEFINE_STATIC_CALL() invocations and + * "static_call_update()" calls. + * + * KVM_X86_PMU_OP_OPTIONAL() can be used for those functions that can have + * a NULL definition, for example if "static_call_cond()" will be used + * at the call sites. + */ +KVM_X86_PMU_OP(pmc_perf_hw_id) +KVM_X86_PMU_OP(pmc_is_enabled) +KVM_X86_PMU_OP(pmc_idx_to_pmc) +KVM_X86_PMU_OP(rdpmc_ecx_to_pmc) +KVM_X86_PMU_OP(msr_idx_to_pmc) +KVM_X86_PMU_OP(is_valid_rdpmc_ecx) +KVM_X86_PMU_OP(is_valid_msr) +KVM_X86_PMU_OP(get_msr) +KVM_X86_PMU_OP(set_msr) +KVM_X86_PMU_OP(refresh) +KVM_X86_PMU_OP(init) +KVM_X86_PMU_OP(reset) +KVM_X86_PMU_OP_OPTIONAL(deliver_pmi) +KVM_X86_PMU_OP_OPTIONAL(cleanup) + +#undef KVM_X86_PMU_OP +#undef KVM_X86_PMU_OP_OPTIONAL diff --git a/arch/x86/kvm/pmu.c b/arch/x86/kvm/pmu.c index bb0b1ad0fda5..618f529f1c4d 100644 --- a/arch/x86/kvm/pmu.c +++ b/arch/x86/kvm/pmu.c @@ -51,14 +51,28 @@ =20 static struct kvm_pmu_ops kvm_pmu_ops __read_mostly; =20 +#define KVM_X86_PMU_OP(func) \ + DEFINE_STATIC_CALL_NULL(kvm_x86_pmu_##func, \ + *(((struct kvm_pmu_ops *)0)->func)); +#define KVM_X86_PMU_OP_OPTIONAL KVM_X86_PMU_OP +#include + void kvm_pmu_ops_update(const struct kvm_pmu_ops *pmu_ops) { memcpy(&kvm_pmu_ops, pmu_ops, sizeof(kvm_pmu_ops)); + +#define __KVM_X86_PMU_OP(func) \ + static_call_update(kvm_x86_pmu_##func, kvm_pmu_ops.func); +#define KVM_X86_PMU_OP(func) \ + WARN_ON(!kvm_pmu_ops.func); __KVM_X86_PMU_OP(func) +#define KVM_X86_PMU_OP_OPTIONAL __KVM_X86_PMU_OP +#include +#undef __KVM_X86_PMU_OP } =20 static inline bool pmc_is_enabled(struct kvm_pmc *pmc) { - return kvm_pmu_ops.pmc_is_enabled(pmc); + return static_call(kvm_x86_pmu_pmc_is_enabled)(pmc); } =20 static void kvm_pmi_trigger_fn(struct irq_work *irq_work) @@ -225,7 +239,7 @@ void reprogram_gp_counter(struct kvm_pmc *pmc, u64 even= tsel) ARCH_PERFMON_EVENTSEL_CMASK | HSW_IN_TX | HSW_IN_TX_CHECKPOINTED))) { - config =3D kvm_pmu_ops.pmc_perf_hw_id(pmc); + config =3D static_call(kvm_x86_pmu_pmc_perf_hw_id)(pmc); if (config !=3D PERF_COUNT_HW_MAX) type =3D PERF_TYPE_HARDWARE; } @@ -275,7 +289,7 @@ void reprogram_fixed_counter(struct kvm_pmc *pmc, u8 ct= rl, int idx) =20 pmc->current_config =3D (u64)ctrl; pmc_reprogram_counter(pmc, PERF_TYPE_HARDWARE, - kvm_pmu_ops.pmc_perf_hw_id(pmc), + static_call(kvm_x86_pmu_pmc_perf_hw_id)(pmc), !(en_field & 0x2), /* exclude user */ !(en_field & 0x1), /* exclude kernel */ pmi); @@ -284,7 +298,7 @@ EXPORT_SYMBOL_GPL(reprogram_fixed_counter); =20 void reprogram_counter(struct kvm_pmu *pmu, int pmc_idx) { - struct kvm_pmc *pmc =3D kvm_pmu_ops.pmc_idx_to_pmc(pmu, pmc_idx); + struct kvm_pmc *pmc =3D static_call(kvm_x86_pmu_pmc_idx_to_pmc)(pmu, pmc_= idx); =20 if (!pmc) return; @@ -306,7 +320,7 @@ void kvm_pmu_handle_event(struct kvm_vcpu *vcpu) int bit; =20 for_each_set_bit(bit, pmu->reprogram_pmi, X86_PMC_IDX_MAX) { - struct kvm_pmc *pmc =3D kvm_pmu_ops.pmc_idx_to_pmc(pmu, bit); + struct kvm_pmc *pmc =3D static_call(kvm_x86_pmu_pmc_idx_to_pmc)(pmu, bit= ); =20 if (unlikely(!pmc || !pmc->perf_event)) { clear_bit(bit, pmu->reprogram_pmi); @@ -328,7 +342,7 @@ void kvm_pmu_handle_event(struct kvm_vcpu *vcpu) /* check if idx is a valid index to access PMU */ bool kvm_pmu_is_valid_rdpmc_ecx(struct kvm_vcpu *vcpu, unsigned int idx) { - return kvm_pmu_ops.is_valid_rdpmc_ecx(vcpu, idx); + return static_call(kvm_x86_pmu_is_valid_rdpmc_ecx)(vcpu, idx); } =20 bool is_vmware_backdoor_pmc(u32 pmc_idx) @@ -378,7 +392,7 @@ int kvm_pmu_rdpmc(struct kvm_vcpu *vcpu, unsigned idx, = u64 *data) if (is_vmware_backdoor_pmc(idx)) return kvm_pmu_rdpmc_vmware(vcpu, idx, data); =20 - pmc =3D kvm_pmu_ops.rdpmc_ecx_to_pmc(vcpu, idx, &mask); + pmc =3D static_call(kvm_x86_pmu_rdpmc_ecx_to_pmc)(vcpu, idx, &mask); if (!pmc) return 1; =20 @@ -394,22 +408,21 @@ int kvm_pmu_rdpmc(struct kvm_vcpu *vcpu, unsigned idx= , u64 *data) void kvm_pmu_deliver_pmi(struct kvm_vcpu *vcpu) { if (lapic_in_kernel(vcpu)) { - if (kvm_pmu_ops.deliver_pmi) - kvm_pmu_ops.deliver_pmi(vcpu); + static_call_cond(kvm_x86_pmu_deliver_pmi)(vcpu); kvm_apic_local_deliver(vcpu->arch.apic, APIC_LVTPC); } } =20 bool kvm_pmu_is_valid_msr(struct kvm_vcpu *vcpu, u32 msr) { - return kvm_pmu_ops.msr_idx_to_pmc(vcpu, msr) || - kvm_pmu_ops.is_valid_msr(vcpu, msr); + return static_call(kvm_x86_pmu_msr_idx_to_pmc)(vcpu, msr) || + static_call(kvm_x86_pmu_is_valid_msr)(vcpu, msr); } =20 static void kvm_pmu_mark_pmc_in_use(struct kvm_vcpu *vcpu, u32 msr) { struct kvm_pmu *pmu =3D vcpu_to_pmu(vcpu); - struct kvm_pmc *pmc =3D kvm_pmu_ops.msr_idx_to_pmc(vcpu, msr); + struct kvm_pmc *pmc =3D static_call(kvm_x86_pmu_msr_idx_to_pmc)(vcpu, msr= ); =20 if (pmc) __set_bit(pmc->idx, pmu->pmc_in_use); @@ -417,13 +430,13 @@ static void kvm_pmu_mark_pmc_in_use(struct kvm_vcpu *= vcpu, u32 msr) =20 int kvm_pmu_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) { - return kvm_pmu_ops.get_msr(vcpu, msr_info); + return static_call(kvm_x86_pmu_get_msr)(vcpu, msr_info); } =20 int kvm_pmu_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) { kvm_pmu_mark_pmc_in_use(vcpu, msr_info->index); - return kvm_pmu_ops.set_msr(vcpu, msr_info); + return static_call(kvm_x86_pmu_set_msr)(vcpu, msr_info); } =20 /* refresh PMU settings. This function generally is called when underlying @@ -432,7 +445,7 @@ int kvm_pmu_set_msr(struct kvm_vcpu *vcpu, struct msr_d= ata *msr_info) */ void kvm_pmu_refresh(struct kvm_vcpu *vcpu) { - kvm_pmu_ops.refresh(vcpu); + static_call(kvm_x86_pmu_refresh)(vcpu); } =20 void kvm_pmu_reset(struct kvm_vcpu *vcpu) @@ -440,7 +453,7 @@ void kvm_pmu_reset(struct kvm_vcpu *vcpu) struct kvm_pmu *pmu =3D vcpu_to_pmu(vcpu); =20 irq_work_sync(&pmu->irq_work); - kvm_pmu_ops.reset(vcpu); + static_call(kvm_x86_pmu_reset)(vcpu); } =20 void kvm_pmu_init(struct kvm_vcpu *vcpu) @@ -448,7 +461,7 @@ void kvm_pmu_init(struct kvm_vcpu *vcpu) struct kvm_pmu *pmu =3D vcpu_to_pmu(vcpu); =20 memset(pmu, 0, sizeof(*pmu)); - kvm_pmu_ops.init(vcpu); + static_call(kvm_x86_pmu_init)(vcpu); init_irq_work(&pmu->irq_work, kvm_pmi_trigger_fn); pmu->event_count =3D 0; pmu->need_cleanup =3D false; @@ -480,14 +493,13 @@ void kvm_pmu_cleanup(struct kvm_vcpu *vcpu) pmu->pmc_in_use, X86_PMC_IDX_MAX); =20 for_each_set_bit(i, bitmask, X86_PMC_IDX_MAX) { - pmc =3D kvm_pmu_ops.pmc_idx_to_pmc(pmu, i); + pmc =3D static_call(kvm_x86_pmu_pmc_idx_to_pmc)(pmu, i); =20 if (pmc && pmc->perf_event && !pmc_speculative_in_use(pmc)) pmc_stop_counter(pmc); } =20 - if (kvm_pmu_ops.cleanup) - kvm_pmu_ops.cleanup(vcpu); + static_call_cond(kvm_x86_pmu_cleanup)(vcpu); =20 bitmap_zero(pmu->pmc_in_use, X86_PMC_IDX_MAX); } @@ -517,7 +529,7 @@ static inline bool eventsel_match_perf_hw_id(struct kvm= _pmc *pmc, unsigned int config; =20 pmc->eventsel &=3D (ARCH_PERFMON_EVENTSEL_EVENT | ARCH_PERFMON_EVENTSEL_U= MASK); - config =3D kvm_pmu_ops.pmc_perf_hw_id(pmc); + config =3D static_call(kvm_x86_pmu_pmc_perf_hw_id)(pmc); pmc->eventsel =3D old_eventsel; return config =3D=3D perf_hw_id; } @@ -545,7 +557,7 @@ void kvm_pmu_trigger_event(struct kvm_vcpu *vcpu, u64 p= erf_hw_id) int i; =20 for_each_set_bit(i, pmu->all_valid_pmc_idx, X86_PMC_IDX_MAX) { - pmc =3D kvm_pmu_ops.pmc_idx_to_pmc(pmu, i); + pmc =3D static_call(kvm_x86_pmu_pmc_idx_to_pmc)(pmu, i); =20 if (!pmc || !pmc_is_enabled(pmc) || !pmc_speculative_in_use(pmc)) continue; --=20 2.35.1.1021.g381101b075-goog