From nobody Mon Feb 9 08:00:05 2026 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id C4B3DEB64DD for ; Mon, 31 Jul 2023 01:25:53 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S229716AbjGaBZw (ORCPT ); Sun, 30 Jul 2023 21:25:52 -0400 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:48320 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S229437AbjGaBZt (ORCPT ); Sun, 30 Jul 2023 21:25:49 -0400 Received: from mailgw.kylinos.cn (mailgw.kylinos.cn [124.126.103.232]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id 7C32D1A4 for ; Sun, 30 Jul 2023 18:25:47 -0700 (PDT) X-UUID: a81924b758d144878e1e8fcdaa4d5fe0-20230731 X-CID-O-RULE: Release_Ham X-CID-RULE: Release_Ham X-CID-O-INFO: VERSION:1.1.28,REQID:3b788908-5e65-4ed2-8e8c-ea46e7604ad0,IP:5,U RL:0,TC:0,Content:-5,EDM:25,RT:0,SF:-15,FILE:0,BULK:0,RULE:Release_Ham,ACT ION:release,TS:10 X-CID-INFO: VERSION:1.1.28,REQID:3b788908-5e65-4ed2-8e8c-ea46e7604ad0,IP:5,URL :0,TC:0,Content:-5,EDM:25,RT:0,SF:-15,FILE:0,BULK:0,RULE:Release_Ham,ACTIO N:release,TS:10 X-CID-META: VersionHash:176cd25,CLOUDID:94237ed2-cd77-4e67-bbfd-aa4eaace762f,B ulkID:230731090635B7ZKYZ86,BulkQuantity:2,Recheck:0,SF:24|17|19|44|102,TC: nil,Content:0,EDM:5,IP:-2,URL:1,File:nil,Bulk:40,QS:nil,BEC:nil,COL:0,OSI: 0,OSA:0,AV:0,LES:1,SPR:NO,DKR:0,DKP:0 X-CID-BVR: 0,NGT X-CID-BAS: 0,NGT,0,_ X-CID-FACTOR: TF_CID_SPAM_FSI,TF_CID_SPAM_ULS,TF_CID_SPAM_SNR,TF_CID_SPAM_FAS, TF_CID_SPAM_FSD X-UUID: a81924b758d144878e1e8fcdaa4d5fe0-20230731 X-User: zhanghao1@kylinos.cn Received: from localhost.localdomain [(111.48.58.12)] by mailgw (envelope-from ) (Generic MTA) with ESMTP id 1861378066; Mon, 31 Jul 2023 09:25:40 +0800 From: zhanghao1 To: virtualization@lists.linux-foundation.org Cc: mst@redhat.com, jasowang@redhat.com, xuanzhuo@linux.alibaba.com, linux-kernel@vger.kernel.org, zhanghao1 Subject: [PATCH] virtio: a new vcpu watchdog driver Date: Mon, 31 Jul 2023 09:25:12 +0800 Message-Id: <20230731012512.235085-1-zhanghao1@kylinos.cn> X-Mailer: git-send-email 2.25.1 MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Precedence: bulk List-ID: X-Mailing-List: linux-kernel@vger.kernel.org Content-Type: text/plain; charset="utf-8" A new virtio pci driver is added for listening to vcpus inside guest. Each vcpu creates a corresponding thread to periodically send data to qemu's back-end watchdog device. If a vCPU is in the stall state, data cannot be sent to back-end virtio device. As a result, the back-end device can detect that the guest is in the stall state. The driver is mainly used with the back-end watchdog device of qemu. The qemu backend watchdog device is implemented as follow: https://lore.kernel.org/qemu-devel/20230705081813.411526-1-zhanghao1@kylino= s.cn/ Signed-off-by: zhanghao1 --- drivers/virtio/Kconfig | 9 + drivers/virtio/Makefile | 1 + drivers/virtio/virtio_vcpu_stall_detector.c | 299 ++++++++++++++++++++ 3 files changed, 309 insertions(+) create mode 100644 drivers/virtio/virtio_vcpu_stall_detector.c diff --git a/drivers/virtio/Kconfig b/drivers/virtio/Kconfig index 0a53a61231c2..869323e345a1 100644 --- a/drivers/virtio/Kconfig +++ b/drivers/virtio/Kconfig @@ -173,4 +173,13 @@ config VIRTIO_DMA_SHARED_BUFFER This option adds a flavor of dma buffers that are backed by virtio resources. =20 +config VIRTIO_VCPU_WATCHDOG + tristate "Virtio vcpu watchdog driver" + depends on VIRTIO_PCI + help + When this driver is bound inside a KVM guest, it will + periodically "pet" an PCI virtio watchdog device from each vCPU + and allow the host to detect vCPU stalls. + + If you do not intend to run this kernel as a guest, say N. endif # VIRTIO_MENU diff --git a/drivers/virtio/Makefile b/drivers/virtio/Makefile index 8e98d24917cc..c7341f078a34 100644 --- a/drivers/virtio/Makefile +++ b/drivers/virtio/Makefile @@ -12,3 +12,4 @@ obj-$(CONFIG_VIRTIO_INPUT) +=3D virtio_input.o obj-$(CONFIG_VIRTIO_VDPA) +=3D virtio_vdpa.o obj-$(CONFIG_VIRTIO_MEM) +=3D virtio_mem.o obj-$(CONFIG_VIRTIO_DMA_SHARED_BUFFER) +=3D virtio_dma_buf.o +obj-$(CONFIG_VIRTIO_VCPU_WATCHDOG) +=3D virtio_vcpu_stall_detector.o diff --git a/drivers/virtio/virtio_vcpu_stall_detector.c b/drivers/virtio/v= irtio_vcpu_stall_detector.c new file mode 100644 index 000000000000..58344ca528be --- /dev/null +++ b/drivers/virtio/virtio_vcpu_stall_detector.c @@ -0,0 +1,299 @@ +// SPDX-License-Identifier: GPL-2.0-only +// +// VCPU stall detector. +// Copyright (C) Kylin Software, 2023 + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define VCPU_STALL_REG_STATUS (0x00) +#define VCPU_STALL_REG_LOAD_CNT (0x04) +#define VCPU_STALL_REG_CURRENT_CNT (0x08) +#define VCPU_STALL_REG_CLOCK_FREQ_HZ (0x0C) +#define VCPU_STALL_REG_LEN (0x10) +#define VCPU_STALL_REG_TIMEOUT_SEC (0x14) + +#define VCPU_STALL_DEFAULT_CLOCK_HZ (10) +#define VCPU_STALL_MAX_CLOCK_HZ (100) +#define VCPU_STALL_DEFAULT_TIMEOUT_SEC (8) +#define VCPU_STALL_MAX_TIMEOUT_SEC (600) + +struct vcpu_stall_detect_config { + u32 clock_freq_hz; + u32 stall_timeout_sec; + + enum cpuhp_state hp_online; +}; + +struct vcpu_stall_priv { + struct hrtimer vcpu_hrtimer; + struct virtio_device *vdev; + u32 cpu_id; +}; + +struct vcpu_stall { + struct vcpu_stall_priv *priv; + struct virtqueue *vq; + spinlock_t lock; + struct pet_event { + u32 cpu_id; + bool is_initialized; + u32 ticks; + } pet_event; +}; + +static const struct virtio_device_id vcpu_stall_id_table[] =3D { + { VIRTIO_ID_WATCHDOG, VIRTIO_DEV_ANY_ID }, + { 0, }, +}; + +/* The vcpu stall configuration structure which applies to all the CPUs */ +static struct vcpu_stall_detect_config vcpu_stall_config; +static struct vcpu_stall *vcpu_stall; + +static struct vcpu_stall_priv __percpu *vcpu_stall_detectors; + +static enum hrtimer_restart +vcpu_stall_detect_timer_fn(struct hrtimer *hrtimer) +{ + u32 ticks, ping_timeout_ms; + struct scatterlist sg; + int unused, err =3D 0; + + struct vcpu_stall_priv *vcpu_stall_detector =3D + this_cpu_ptr(vcpu_stall->priv); + + /* Reload the stall detector counter register every + * `ping_timeout_ms` to prevent the virtual device + * from decrementing it to 0. The virtual device decrements this + * register at 'clock_freq_hz' frequency. + */ + ticks =3D vcpu_stall_config.clock_freq_hz * + vcpu_stall_config.stall_timeout_sec; + + spin_lock(&vcpu_stall->lock); + while (virtqueue_get_buf(vcpu_stall->vq, &unused)) + ; + vcpu_stall->pet_event.ticks =3D cpu_to_virtio32(vcpu_stall_detector->vdev= , ticks); + vcpu_stall->pet_event.is_initialized =3D true; + vcpu_stall->pet_event.cpu_id =3D vcpu_stall_detector->cpu_id; + + sg_init_one(&sg, &vcpu_stall->pet_event, sizeof(vcpu_stall->pet_event)); + err =3D virtqueue_add_outbuf(vcpu_stall->vq, &sg, 1, vcpu_stall, GFP_ATOM= IC); + if (!err) + virtqueue_kick(vcpu_stall->vq); + else + pr_err("cpu:%d failed to add outbuf, err:%d\n", vcpu_stall_detector->cpu= _id, err); + + spin_unlock(&vcpu_stall->lock); + + ping_timeout_ms =3D vcpu_stall_config.stall_timeout_sec * + MSEC_PER_SEC / 2; + hrtimer_forward_now(hrtimer, + ms_to_ktime(ping_timeout_ms)); + return HRTIMER_RESTART; +} + +static int start_stall_detector_cpu(unsigned int cpu) +{ + u32 ticks, ping_timeout_ms; + struct scatterlist sg; + struct hrtimer *vcpu_hrtimer; + int err =3D 0; + + struct vcpu_stall_priv *vcpu_stall_detector =3D + this_cpu_ptr(vcpu_stall->priv); + + vcpu_stall_detector->cpu_id =3D cpu; + + vcpu_hrtimer =3D &vcpu_stall_detector->vcpu_hrtimer; + + /* Compute the number of ticks required for the stall detector + * counter register based on the internal clock frequency and the + * timeout value given from the device tree. + */ + ticks =3D vcpu_stall_config.clock_freq_hz * + vcpu_stall_config.stall_timeout_sec; + vcpu_stall->pet_event.ticks =3D cpu_to_virtio32(vcpu_stall_detector->vdev= , ticks); + + /* Pet the stall detector at half of its expiration timeout + * to prevent spurious resets. + */ + ping_timeout_ms =3D vcpu_stall_config.stall_timeout_sec * + MSEC_PER_SEC / 2; + + hrtimer_init(vcpu_hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); + vcpu_hrtimer->function =3D vcpu_stall_detect_timer_fn; + + vcpu_stall->pet_event.is_initialized =3D true; + + spin_lock(&vcpu_stall->lock); + vcpu_stall->pet_event.cpu_id =3D cpu; + sg_init_one(&sg, &vcpu_stall->pet_event, sizeof(vcpu_stall->pet_event)); + err =3D virtqueue_add_outbuf(vcpu_stall->vq, &sg, 1, vcpu_stall, GFP_ATOM= IC); + if (!err) + virtqueue_kick(vcpu_stall->vq); + + spin_unlock(&vcpu_stall->lock); + + hrtimer_start(vcpu_hrtimer, ms_to_ktime(ping_timeout_ms), + HRTIMER_MODE_REL_PINNED); + return err; +} + +static int stop_stall_detector_cpu(unsigned int cpu) +{ + int err =3D 0; + struct scatterlist sg; + + struct vcpu_stall_priv *vcpu_stall_detector =3D + per_cpu_ptr(vcpu_stall_detectors, cpu); + + /* Disable the stall detector for the current CPU */ + hrtimer_cancel(&vcpu_stall_detector->vcpu_hrtimer); + vcpu_stall->pet_event.is_initialized =3D false; + vcpu_stall->pet_event.cpu_id =3D cpu; + + spin_lock(&vcpu_stall->lock); + sg_init_one(&sg, &vcpu_stall->pet_event, sizeof(vcpu_stall->pet_event)); + err =3D virtqueue_add_outbuf(vcpu_stall->vq, &sg, 1, vcpu_stall, GFP_ATOM= IC); + if (!err) + virtqueue_kick(vcpu_stall->vq); + + spin_unlock(&vcpu_stall->lock); + + return err; +} + +static int vcpu_stall_detect_probe(struct virtio_device *vdev) +{ + int ret, cpu; + u32 clock_freq_hz =3D VCPU_STALL_DEFAULT_CLOCK_HZ; + u32 stall_timeout_sec =3D VCPU_STALL_DEFAULT_TIMEOUT_SEC; + + vcpu_stall =3D kzalloc(sizeof(struct vcpu_stall), GFP_KERNEL); + if (!vcpu_stall) { + ret =3D -ENOMEM; + goto err; + } + vdev->priv =3D vcpu_stall; + + vcpu_stall->priv =3D devm_alloc_percpu(&vdev->dev, + typeof(struct vcpu_stall_priv)); + if (!vcpu_stall->priv) { + ret =3D -ENOMEM; + goto failed_priv; + } + + for_each_possible_cpu(cpu) { + struct vcpu_stall_priv *priv; + + priv =3D per_cpu_ptr(vcpu_stall->priv, cpu); + priv->vdev =3D vdev; + } + + ret =3D virtio_cread_feature(vdev, VCPU_STALL_REG_CLOCK_FREQ_HZ, + struct vcpu_stall_detect_config, clock_freq_hz, + &clock_freq_hz); + if (ret || !clock_freq_hz) { + if (!(clock_freq_hz > 0 && + clock_freq_hz < VCPU_STALL_MAX_CLOCK_HZ)) { + dev_warn(&vdev->dev, "clk out of range\n"); + clock_freq_hz =3D VCPU_STALL_DEFAULT_CLOCK_HZ; + } + } + ret =3D virtio_cread_feature(vdev, VCPU_STALL_REG_TIMEOUT_SEC, + struct vcpu_stall_detect_config, stall_timeout_sec, + &stall_timeout_sec); + if (ret || !stall_timeout_sec) { + if (!(stall_timeout_sec > 0 && + stall_timeout_sec < VCPU_STALL_MAX_TIMEOUT_SEC)) { + dev_warn(&vdev->dev, "stall timeout out of range\n"); + stall_timeout_sec =3D VCPU_STALL_DEFAULT_TIMEOUT_SEC; + } + } + + vcpu_stall_config =3D (struct vcpu_stall_detect_config) { + .clock_freq_hz =3D clock_freq_hz, + .stall_timeout_sec =3D stall_timeout_sec + }; + + /* find virtqueue for guest to send pet event to host */ + vcpu_stall->vq =3D virtio_find_single_vq(vdev, NULL, "pet-event"); + if (IS_ERR(vcpu_stall->vq)) { + dev_err(&vdev->dev, "failed to find vq\n"); + goto failed_priv; + } + + spin_lock_init(&vcpu_stall->lock); + + ret =3D cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, + "virt/vcpu_stall_detector:online", + start_stall_detector_cpu, + stop_stall_detector_cpu); + if (ret < 0) { + dev_err(&vdev->dev, "failed to install cpu hotplug\n"); + goto failed_priv; + } + + vcpu_stall_config.hp_online =3D ret; + return 0; + + +failed_priv: + kfree(vcpu_stall); +err: + return ret; +} + +static void vcpu_stall_detect_remove(struct virtio_device *vdev) +{ + int cpu; + + cpuhp_remove_state(vcpu_stall_config.hp_online); + + for_each_possible_cpu(cpu) + stop_stall_detector_cpu(cpu); +} + +static unsigned int features_legacy[] =3D { + VCPU_STALL_REG_STATUS, VCPU_STALL_REG_LOAD_CNT, VCPU_STALL_REG_CURRENT_CN= T, + VCPU_STALL_REG_CLOCK_FREQ_HZ, VCPU_STALL_REG_LEN, VCPU_STALL_REG_TIMEOUT_= SEC +}; + + +static unsigned int features[] =3D { + VCPU_STALL_REG_STATUS, VCPU_STALL_REG_LOAD_CNT, VCPU_STALL_REG_CURRENT_CN= T, + VCPU_STALL_REG_CLOCK_FREQ_HZ, VCPU_STALL_REG_LEN, VCPU_STALL_REG_TIMEOUT_= SEC +}; + +static struct virtio_driver vcpu_stall_detect_driver =3D { + .feature_table =3D features, + .feature_table_size =3D ARRAY_SIZE(features), + .feature_table_legacy =3D features_legacy, + .feature_table_size_legacy =3D ARRAY_SIZE(features_legacy), + .driver.name =3D KBUILD_MODNAME, + .driver.owner =3D THIS_MODULE, + .id_table =3D vcpu_stall_id_table, + .probe =3D vcpu_stall_detect_probe, + .remove =3D vcpu_stall_detect_remove, +}; + +module_virtio_driver(vcpu_stall_detect_driver); + +MODULE_LICENSE("GPL"); +MODULE_DEVICE_TABLE(virtio, vcpu_stall_id_table); +MODULE_AUTHOR("zhanghao1 "); +MODULE_DESCRIPTION("VCPU stall detector"); --=20 2.25.1 No virus found Checked by Hillstone Network AntiVirus