From nobody Sat Jul 25 11:03:39 2026 Received: from m16.mail.163.com (m16.mail.163.com [117.135.210.2]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id D2F4C233920; Mon, 20 Jul 2026 14:11:27 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=117.135.210.2 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1784556691; cv=none; b=JMk5FBrjynhdPbuVO3oL/YqlLxZfCrtHILjjpSH41bJgoNeFVHL93EWuJokTZ167FqlT9Oxyb/gxgGUrNEZNSZXBA7QHQENaYtEYU/Vs2pg0vSGdZYrKy8LHijDujuaDqSqfWQeeDH9Bmz8q/vb7b1KbefKZn1T29B/3peHhaFA= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1784556691; c=relaxed/simple; bh=aKgk7WJQ2y6hkMjZ2T2pN9NhrkaaUZcdCyA14QVEjIQ=; h=From:To:Cc:Subject:Date:Message-Id:In-Reply-To:References: MIME-Version:Content-Type; b=tfROlnLFSibprFCcZudVDQ1MrOfvl/zKKyRtDuXIL5mzjen2ZZG3e1VfP4Wt69mnlg/T/pszmECJMGJduWpAAuOUqvqLH+NDbuZs2WfFAysXUz6RzG1HaqyByzyDtxgHynyiAeSzN4af+a+1GNuQPJYR8F05pJe4Lg3moJd5b0g= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=163.com; spf=pass smtp.mailfrom=163.com; dkim=pass (1024-bit key) header.d=163.com header.i=@163.com header.b=O1HV7DSG; arc=none smtp.client-ip=117.135.210.2 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=163.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=163.com Authentication-Results: smtp.subspace.kernel.org; dkim=pass (1024-bit key) header.d=163.com header.i=@163.com header.b="O1HV7DSG" DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=163.com; s=s110527; h=From:To:Subject:Date:Message-Id:MIME-Version: Content-Type; bh=CO9Al5GPtC4ZgyEFmL+3ZuFzpk3dAPxBpTXH8Cw7El0=; b=O1HV7DSG1m9bI5mUv2LKR3+Hn8M7/nSd+owcWqYg/oWQ2HCFeaXyEBSKpD/o0R JrzyrsLQM6sBLXdCyVkkw7xwpJCx3qGgHz7UaTIzhk7yJeo5B0bzSz8anfEF54hm wzUigODQItMi3+91HByBVWFBI1haarrp0IoOjUQ4P9onE= Received: from localhost.localdomain (unknown []) by gzsmtp3 (Coremail) with SMTP id PigvCgD3Xw9gLF5qWUvpIg--.12525S3; Mon, 20 Jul 2026 22:10:43 +0800 (CST) From: "rom.wang" To: mst@redhat.com, jasowangio@gmail.com, eperezma@redhat.com, stefanha@redhat.com Cc: virtualization@lists.linux.dev, linux-kernel@vger.kernel.org, linux-scsi@vger.kernel.org, kvm@vger.kernel.org, qemu-devel@nongnu.org, Yufeng Wang Subject: [RFC PATCH 1/4] common: add UAPI for SQ/CQ doorbell polling Date: Mon, 20 Jul 2026 22:10:37 +0800 Message-Id: <20260720141040.181946-2-r4o5m6e8o@163.com> X-Mailer: git-send-email 2.34.1 In-Reply-To: <20260720141040.181946-1-r4o5m6e8o@163.com> References: <20260720141040.181946-1-r4o5m6e8o@163.com> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Type: text/plain; charset="utf-8" Content-Transfer-Encoding: quoted-printable X-CM-TRANSID: PigvCgD3Xw9gLF5qWUvpIg--.12525S3 X-Coremail-Antispam: 1Uf129KBjvJXoWxCw43Gr4fCr1UCw48XrW7CFg_yoWrXFyUpF WkCrW3Xr4UXa4xCrZ5G3WUtry3Xas3CF43G3yagw1j9F9IqFy8Zw4Dur4avay2vrWvyryx Xrn2qryUC34DArJanT9S1TB71UUUUU7qnTZGkaVYY2UrUUUUjbIjqfuFe4nvWSU5nxnvy2 9KBjDUYxBIdaVFxhVjvjDU0xZFpf9x07UNTmDUUUUU= X-CM-SenderInfo: 3uurkzkwhy0qqrwthudrp/xtbCzQNVympeLGONpgAA3B From: Yufeng Wang Define VIRTIO_F_SQCQ_POLL (bit 42) and the associated SQ/CQ doorbell structures for split virtqueue polling mode. - include/uapi/linux/virtio_config.h: feature bit 42 - include/uapi/linux/virtio_ring.h: struct vring_sq, struct vring_cq (128-byte aligned for cache-line isolation on ARM Neoverse N2/V2) - include/uapi/linux/virtio_pci.h: sqe/cqe_ring PCI config registers at offsets 64-76 Signed-off-by: Yufeng Wang --- include/uapi/linux/virtio_config.h | 8 +++++++- include/uapi/linux/virtio_pci.h | 9 +++++++++ include/uapi/linux/virtio_ring.h | 25 +++++++++++++++++++++++++ 3 files changed, 41 insertions(+), 1 deletion(-) diff --git a/include/uapi/linux/virtio_config.h b/include/uapi/linux/virtio= _config.h index 2445f365bce7..dbda23e44b62 100644 --- a/include/uapi/linux/virtio_config.h +++ b/include/uapi/linux/virtio_config.h @@ -52,7 +52,7 @@ * rest are per-device feature bits. */ #define VIRTIO_TRANSPORT_F_START 28 -#define VIRTIO_TRANSPORT_F_END 42 +#define VIRTIO_TRANSPORT_F_END 43 =20 #ifndef VIRTIO_CONFIG_NO_LEGACY /* Do we get callbacks when the ring is completely used, even if we've @@ -120,4 +120,10 @@ */ #define VIRTIO_F_ADMIN_VQ 41 =20 +/* + * This feature indicates that both driver and device support + * SQ/CQ polling mode to replace interrupt-based notifications. + */ +#define VIRTIO_F_SQCQ_POLL 42 + #endif /* _UAPI_LINUX_VIRTIO_CONFIG_H */ diff --git a/include/uapi/linux/virtio_pci.h b/include/uapi/linux/virtio_pc= i.h index e732e3456e27..f5058d1b4b16 100644 --- a/include/uapi/linux/virtio_pci.h +++ b/include/uapi/linux/virtio_pci.h @@ -193,6 +193,11 @@ struct virtio_pci_modern_common_cfg { =20 __le16 admin_queue_index; /* read-only */ __le16 admin_queue_num; /* read-only */ + + __le32 sqe_ring_lo; /* read-write: SQ DMA addr low */ + __le32 sqe_ring_hi; /* read-write: SQ DMA addr high */ + __le32 cqe_ring_lo; /* read-write: CQ DMA addr low */ + __le32 cqe_ring_hi; /* read-write: CQ DMA addr high */ }; =20 /* Fields in VIRTIO_PCI_CAP_PCI_CFG: */ @@ -235,6 +240,10 @@ struct virtio_pci_cfg_cap { #define VIRTIO_PCI_COMMON_Q_RESET 58 #define VIRTIO_PCI_COMMON_ADM_Q_IDX 60 #define VIRTIO_PCI_COMMON_ADM_Q_NUM 62 +#define VIRTIO_PCI_COMMON_SQE_LO 64 +#define VIRTIO_PCI_COMMON_SQE_HI 68 +#define VIRTIO_PCI_COMMON_CQE_LO 72 +#define VIRTIO_PCI_COMMON_CQE_HI 76 =20 #endif /* VIRTIO_PCI_NO_MODERN */ =20 diff --git a/include/uapi/linux/virtio_ring.h b/include/uapi/linux/virtio_r= ing.h index 3c478582a3c2..d1f2b9584d7c 100644 --- a/include/uapi/linux/virtio_ring.h +++ b/include/uapi/linux/virtio_ring.h @@ -131,6 +131,31 @@ struct vring_used { vring_used_elem_t ring[]; }; =20 +/* + * SQ/CQ doorbell structures for polling mode (VIRTIO_F_SQCQ_POLL). + * These are allocated separately from the vring and their DMA addresses + * are communicated via PCI config space registers. + * + * Ownership: + * SQ: Guest writes idx, Device reads it. + * CQ: Device writes idx, Guest reads it. + * + * Flags (8-bit): + * Set to 1 before sleeping, cleared to 0 after waking up. + * Direct assignment =E2=80=94 no read-modify-write. + */ +struct vring_sq { + __virtio64 idx; /* Producer position (updated by guest) */ + __u8 sq_need_wakeup; /* 1 =3D device should be woken on next kick */ + __u8 reserved[55]; /* Reserved for future use */ +} __attribute__((aligned(128))); + +struct vring_cq { + __virtio64 idx; /* Producer position (updated by device) */ + __u8 cq_need_wakeup; /* 1 =3D guest should be woken on completion */ + __u8 reserved[55]; /* Reserved for future use */ +} __attribute__((aligned(128))); + /* * The ring element addresses are passed between components with different * alignments assumptions. Thus, we might need to decrease the compiler-se= lected --=20 2.34.1 From nobody Sat Jul 25 11:03:39 2026 Received: from m16.mail.163.com (m16.mail.163.com [117.135.210.3]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 9CC1B428821; Mon, 20 Jul 2026 14:12:10 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=117.135.210.3 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1784556739; cv=none; b=Ur9LI+mozZfq3Rt1kGRpV0V9DfmhiAlhUdcDJfDNyNuCE7Lz/pUvQBVqgANmBBMbVC/HOIPvj1Y91C3suf+Yx+QR8/fBud8nW9nil+bxAghjE/wxDE/uC0gyHVAImlWbUgk78/xUdyWzE6GYNEvwa3ZndCphFRQ373pUSc9KfNU= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1784556739; c=relaxed/simple; bh=wFyv4zCJCgfqciWKcDdi3wOMYZ3kDnDfVWiFqUF1LbY=; h=From:To:Cc:Subject:Date:Message-Id:In-Reply-To:References: MIME-Version:Content-Type; b=bVgPFpn9VYwO/Lf3zOiGWzC6HFkLSdSFmKlZ9N4wRtwJjoJ9bz7/cgJcc+C6M1guAF6sC7v2y/kReHfqagkbF8JE8P9XZWjR5SjMsCxedW60rfi9vcu02ZJq+jm/x6+FPjVnSAnqaV5diJ410T8nm3KI5YfkEuWkoE5qMT4ujj8= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=163.com; spf=pass smtp.mailfrom=163.com; dkim=pass (1024-bit key) header.d=163.com header.i=@163.com header.b=XFUWoVho; arc=none smtp.client-ip=117.135.210.3 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=163.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=163.com Authentication-Results: smtp.subspace.kernel.org; dkim=pass (1024-bit key) header.d=163.com header.i=@163.com header.b="XFUWoVho" DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=163.com; s=s110527; h=From:To:Subject:Date:Message-Id:MIME-Version: Content-Type; bh=NK2C9evq85wkl4PEOcMxTR7CF33U9RN9RnpkiN3Uk2o=; b=XFUWoVhoKAR37L5jre0U4wdtYzIu5MtzWnWb/jYBuzAKfFLYfLx5LtDoHMGfEN 9mKQntVcBVdHYRfmCQc/malxFv+wFi0c3NYwYfniXd3987fGosIOOfslApcyiN0t 7rr27D+ERhzhnGczSJqJ0cQeZQkPJUy64hBfqP3BMeLp4= Received: from localhost.localdomain (unknown []) by gzsmtp3 (Coremail) with SMTP id PigvCgD3Xw9gLF5qWUvpIg--.12525S4; Mon, 20 Jul 2026 22:10:43 +0800 (CST) From: "rom.wang" To: mst@redhat.com, jasowangio@gmail.com, eperezma@redhat.com, stefanha@redhat.com Cc: virtualization@lists.linux.dev, linux-kernel@vger.kernel.org, linux-scsi@vger.kernel.org, kvm@vger.kernel.org, qemu-devel@nongnu.org, Yufeng Wang Subject: [RFC PATCH 2/4] vhost: host kernel support for SQ/CQ polling Date: Mon, 20 Jul 2026 22:10:38 +0800 Message-Id: <20260720141040.181946-3-r4o5m6e8o@163.com> X-Mailer: git-send-email 2.34.1 In-Reply-To: <20260720141040.181946-1-r4o5m6e8o@163.com> References: <20260720141040.181946-1-r4o5m6e8o@163.com> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Type: text/plain; charset="utf-8" Content-Transfer-Encoding: quoted-printable X-CM-TRANSID: PigvCgD3Xw9gLF5qWUvpIg--.12525S4 X-Coremail-Antispam: 1Uf129KBjvAXoWkGFW8tFy5uFy8JrW7JFWrZrb_yoW3tF1kKo ZavrW5uF4rJry3Kwn5tr1xJFyDJ3ykKF93JF4Y9ay5Wa17tay5XF97KwsxWa1fJF4rta17 A347J34fXr4kJFnxn29KB7ZKAUJUUUU8529EdanIXcx71UUUUU7v73VFW2AGmfu7bjvjm3 AaLaJ3UbIYCTnIWIevJa73UjIFyTuYvj4Rl2NKUUUUU X-CM-SenderInfo: 3uurkzkwhy0qqrwthudrp/xtbC7wRWy2peLGQvcAAA3V From: Yufeng Wang Implement SQ/CQ doorbell polling in the vhost kernel backend: - vhost.c: per-VQ poll thread (kthread) that polls sq->idx for submissions and processes completions via poll_complete callback; need_resched() cooperative spin; round-robin CPU binding with modulo to prevent overflow; sqcq_poll enabled after poll_task is valid (NULL deref fix); get_user fault retry (3 attempts, 10ms backoff); EMA overflow-safe formula - vhost.h: vhost_virtqueue extensions (sq/cq pointers, poll thread state, EMA fields) - scsi.c: vhost_scsi_poll_complete() writes cq->idx and signals Guest; SET_ENDPOINT error path stops started poll threads (fallback); 10s window diagnostic output - vhost.h UAPI: VHOST_SET_VRING_SQCQ_ADDR ioctl - vhost_types.h UAPI: struct vhost_vring_sqcq_addr Signed-off-by: Yufeng Wang --- Documentation/compare-sqcq-results.sh | 239 +++++++ Documentation/run-sqcq-compare.sh | 182 ++++++ Documentation/virtio-sqcq-poll-spec.rst | 617 ++++++++++++++++++ .../virtio-sqcq-poll-virtio-spec.rst | 355 ++++++++++ build-riscv-kernel.sh | 26 + drivers/vhost/scsi.c | 187 +++++- drivers/vhost/vhost.c | 595 ++++++++++++++++- drivers/vhost/vhost.h | 29 +- include/uapi/linux/vhost.h | 4 + include/uapi/linux/vhost_types.h | 6 + 10 files changed, 2221 insertions(+), 19 deletions(-) create mode 100644 Documentation/compare-sqcq-results.sh create mode 100644 Documentation/run-sqcq-compare.sh create mode 100644 Documentation/virtio-sqcq-poll-spec.rst create mode 100644 Documentation/virtio-sqcq-poll-virtio-spec.rst create mode 100644 build-riscv-kernel.sh diff --git a/Documentation/compare-sqcq-results.sh b/Documentation/compare-= sqcq-results.sh new file mode 100644 index 0000000000000..85bb927fe91e5 --- /dev/null +++ b/Documentation/compare-sqcq-results.sh @@ -0,0 +1,239 @@ +#!/usr/bin/env python3 +"""compare-sqcq-results.py - Generate A/B comparison report for SQ/CQ poll= ing vs baseline. + +Usage: + python3 compare-sqcq-results.py + python3 compare-sqcq-results.py # auto-detect latest results +""" +import json, sys, os, glob, re +from collections import OrderedDict + +def find_latest_dirs(): + """Find the most recent baseline and sqcq-poll result directories.""" + base =3D "perf-results" + if not os.path.isdir(base): + print(f"Error: '{base}/' not found. Run run-sqcq-compare.sh first.= ") + sys.exit(1) + + bl_dirs =3D sorted(glob.glob(os.path.join(base, "baseline_*"))) + sq_dirs =3D sorted(glob.glob(os.path.join(base, "sqcq-poll_*"))) + + if not bl_dirs: + print("Error: No baseline_* directory found in perf-results/") + sys.exit(1) + if not sq_dirs: + print("Error: No sqcq-poll_* directory found in perf-results/") + sys.exit(1) + + return bl_dirs[-1], sq_dirs[-1] + + +def parse_fio_json(filepath): + """Extract key metrics from a fio JSON output file.""" + with open(filepath) as f: + d =3D json.load(f) + + jobs =3D d.get("jobs", []) + if not jobs: + return None + + j =3D jobs[0] + rd =3D j.get("read", {}) + wr =3D j.get("write", {}) + + # Determine which direction has data + if rd.get("iops", 0) > 0: + data =3D rd + else: + data =3D wr + + iops =3D data.get("iops", 0) + bw_mib =3D data.get("bw", 0) / 1024 # KiB/s -> MiB/s + + # Latency + lat_ns =3D data.get("lat_ns", {}) + avg_us =3D lat_ns.get("mean", 0) / 1000 + min_us =3D lat_ns.get("min", 0) / 1000 + max_us =3D lat_ns.get("max", 0) / 1000 + stddev_us =3D lat_ns.get("stddev", 0) / 1000 + + # Completion latency percentiles + clat =3D data.get("clat_ns", {}) + pct =3D clat.get("percentile", {}) if isinstance(clat, dict) else {} + p50 =3D pct.get("50.000000", 0) / 1000 + p95 =3D pct.get("95.000000", 0) / 1000 + p99 =3D pct.get("99.000000", 0) / 1000 + p999 =3D pct.get("99.900000", 0) / 1000 + + return { + "iops": iops, + "bw_mib": bw_mib, + "lat_avg_us": avg_us, + "lat_min_us": min_us, + "lat_max_us": max_us, + "lat_stddev_us": stddev_us, + "p50_us": p50, + "p95_us": p95, + "p99_us": p99, + "p999_us": p999, + } + + +def pct_change(old, new): + """Calculate percentage change. Returns None if division by zero.""" + if old =3D=3D 0: + return None + return (new - old) / old * 100 + + +def fmt_pct(val): + """Format a percentage change value.""" + if val is None: + return "N/A" + sign =3D "+" if val > 0 else "" + return f"{sign}{val:.1f}%" + + +def fmt_int(val): + """Format integer with comma separators.""" + return f"{val:,.0f}" + + +def fmt_f1(val): + """Format float with 1 decimal.""" + return f"{val:.1f}" + + +def main(): + if len(sys.argv) >=3D 3: + bl_dir =3D sys.argv[1] + sq_dir =3D sys.argv[2] + else: + bl_dir, sq_dir =3D find_latest_dirs() + + print(f"Baseline: {bl_dir}") + print(f"SQ/CQ Poll: {sq_dir}") + print() + + # Collect all test labels + bl_files =3D {os.path.basename(f).replace("_best.json", ""): f + for f in glob.glob(os.path.join(bl_dir, "*_best.json"))} + sq_files =3D {os.path.basename(f).replace("_best.json", ""): f + for f in glob.glob(os.path.join(sq_dir, "*_best.json"))} + + all_labels =3D sorted(set(bl_files.keys()) | set(sq_files.keys())) + + if not all_labels: + print("Error: No *_best.json files found.") + sys.exit(1) + + # Define metric display columns + metrics =3D [ + ("IOPS", "iops", fmt_int, True), + ("BW(MiB/s)", "bw_mib", fmt_f1, True), + ("Avg(us)", "lat_avg_us", fmt_f1, False), + ("P50(us)", "p50_us", fmt_f1, False), + ("P95(us)", "p95_us", fmt_f1, False), + ("P99(us)", "p99_us", fmt_f1, False), + ("P99.9(us)", "p999_us", fmt_f1, False), + ] + + # Per-label comparison + results =3D [] + for label in all_labels: + bl_data =3D parse_fio_json(bl_files[label]) if label in bl_files e= lse None + sq_data =3D parse_fio_json(sq_files[label]) if label in sq_files e= lse None + + entry =3D {"label": label, "baseline": bl_data, "sqcq": sq_data} + results.append(entry) + + # Print grouped by workload type + groups =3D OrderedDict() + for label in all_labels: + # Extract workload type from label: e.g. "4k-randread-od1-nj1" -> = "4k-randread" + parts =3D label.rsplit("-od", 1) + wl =3D parts[0] if len(parts) > 1 else label + if wl not in groups: + groups[wl] =3D [] + groups[wl].append(label) + + print("=3D" * 100) + print(" SQ/CQ POLL vs BASELINE COMPARISON REPORT") + print("=3D" * 100) + + for wl, labels in groups.items(): + print(f"\n{'=E2=94=80' * 100}") + print(f" Workload: {wl}") + print(f"{'=E2=94=80' * 100}") + + for label in labels: + r =3D next(x for x in results if x["label"] =3D=3D label) + bl =3D r["baseline"] + sq =3D r["sqcq"] + + # Extract iodepth/numjobs from label + m =3D re.search(r'od(\d+)-nj(\d+)', label) + od =3D m.group(1) if m else "?" + nj =3D m.group(2) if m else "?" + + print(f"\n iodepth=3D{od}, numjobs=3D{nj}") + print(f" {'Metric':<14} {'Baseline':>12} {'SQ/CQ Poll':>12} {= 'Change':>10}") + print(f" {'=E2=94=80'*14} {'=E2=94=80'*12} {'=E2=94=80'*12} {= '=E2=94=80'*10}") + + for metric_name, metric_key, fmt_fn, higher_is_better in metri= cs: + bl_val =3D bl[metric_key] if bl else 0 + sq_val =3D sq[metric_key] if sq else 0 + change =3D pct_change(bl_val, sq_val) + + # For latency, negative change is improvement + line =3D f" {metric_name:<14} {fmt_fn(bl_val):>12} {fmt_f= n(sq_val):>12}" + + if change is not None: + marker =3D "" + if higher_is_better and change > 2: + marker =3D " *" + elif not higher_is_better and change < -2: + marker =3D " *" + line +=3D f" {fmt_pct(change):>10}{marker}" + else: + line +=3D f" {'N/A':>10}" + + print(line) + + # Summary table - just IOPS + print(f"\n{'=3D' * 100}") + print(" IOPS SUMMARY (all scenarios)") + print(f"{'=3D' * 100}") + print(f"\n {'Test':<30} {'Baseline':>10} {'SQ/CQ':>10} {'Change':>10}= ") + print(f" {'=E2=94=80'*30} {'=E2=94=80'*10} {'=E2=94=80'*10} {'=E2=94= =80'*10}") + + for r in results: + bl_iops =3D r["baseline"]["iops"] if r["baseline"] else 0 + sq_iops =3D r["sqcq"]["iops"] if r["sqcq"] else 0 + change =3D pct_change(bl_iops, sq_iops) + + marker =3D " **" if change and change > 5 else "" + print(f" {r['label']:<30} {bl_iops:>10,.0f} {sq_iops:>10,.0f} {fm= t_pct(change):>10}{marker}") + + # Latency summary + print(f"\n{'=3D' * 100}") + print(" P99 LATENCY SUMMARY (all scenarios)") + print(f"{'=3D' * 100}") + print(f"\n {'Test':<30} {'Base(p99)':>12} {'SQ/CQ(p99)':>12} {'Change= ':>10}") + print(f" {'=E2=94=80'*30} {'=E2=94=80'*12} {'=E2=94=80'*12} {'=E2=94= =80'*10}") + + for r in results: + bl_p99 =3D r["baseline"]["p99_us"] if r["baseline"] else 0 + sq_p99 =3D r["sqcq"]["p99_us"] if r["sqcq"] else 0 + change =3D pct_change(bl_p99, sq_p99) + + marker =3D " **" if change and change < -5 else "" + print(f" {r['label']:<30} {bl_p99:>12.1f} {sq_p99:>12.1f} {fmt_pc= t(change):>10}{marker}") + + print(f"\n * =3D notable improvement") + print(f" ** =3D significant improvement (>5% IOPS gain or >5% latency= reduction)") + print() + + +if __name__ =3D=3D "__main__": + main() diff --git a/Documentation/run-sqcq-compare.sh b/Documentation/run-sqcq-com= pare.sh new file mode 100644 index 0000000000000..3a5928bfaef2a --- /dev/null +++ b/Documentation/run-sqcq-compare.sh @@ -0,0 +1,182 @@ +#!/bin/bash +# run-sqcq-compare.sh - A/B comparison for SQ/CQ polling vs baseline +# +# Usage: +# 1. Boot Guest WITHOUT VIRTIO_F_SQCQ_POLL: ./run-sqcq-compare.sh base= line +# 2. Boot Guest WITH VIRTIO_F_SQCQ_POLL: ./run-sqcq-compare.sh sqcq= -poll +# +# Results saved to: perf-results/_/ +# +# IMPORTANT: CPU pinning uses --cpus_allowed (NOT taskset). +# Verify with: ps -eo pid,comm,psr | grep fio +# +set -euo pipefail + +MODE=3D${1:?"Usage: $0 "} +DEV=3D${2:-/dev/sda} +PIN_CPU=3D7 +RUNTIME=3D60 +RUNS=3D3 + +TIMESTAMP=3D$(date +%Y%m%d-%H%M%S) +OUTDIR=3D"perf-results/${MODE}_${TIMESTAMP}" +mkdir -p "$OUTDIR" + +# Verify CPU pinning mapping: PIN_CPU -> hwq -> single Host vq +echo "=3D=3D=3D CPU/mq mapping verification =3D=3D=3D" +echo "PIN_CPU=3D$PIN_CPU" +for d in /sys/block/$(basename $DEV)/mq/*/; do + dir=3D$(basename "$d") + cpus=3D$(cat "$d/cpu_list" 2>/dev/null || echo "?") + echo " hwq $dir: CPU $cpus" +done +echo "" + +echo "=3D=3D=3D Mode: $MODE | Device: $DEV | CPU pin: $PIN_CPU | Runs: $RU= NS =3D=3D=3D" +echo "=3D=3D=3D Output: $OUTDIR =3D=3D=3D" +echo "" + +# Collect Host dmesg marker (for correlating with Host-side vhost stats) +dmesg > "$OUTDIR/guest-dmesg-before.log" 2>/dev/null || true + +# Test matrix: label rw bs iodepth numjobs +TESTS=3D( + "4k-randread-od1-nj1 randread 4k 1 1" + "4k-randread-od32-nj1 randread 4k 32 1" + "4k-randread-od32-nj4 randread 4k 32 4" + "4k-randread-od32-nj8 randread 4k 32 8" + "4k-randwrite-od1-nj1 randwrite 4k 1 1" + "4k-randwrite-od32-nj1 randwrite 4k 32 1" + "4k-randwrite-od32-nj4 randwrite 4k 32 4" + "4k-randwrite-od32-nj8 randwrite 4k 32 8" + "128k-seqread-od1-nj1 read 128k 1 1" + "128k-seqread-od32-nj1 read 128k 32 1" + "128k-seqread-od32-nj4 read 128k 32 4" + "128k-seqread-od32-nj8 read 128k 32 8" + "128k-seqwrite-od1-nj1 write 128k 1 1" + "128k-seqwrite-od32-nj1 write 128k 32 1" + "128k-seqwrite-od32-nj4 write 128k 32 4" + "128k-seqwrite-od32-nj8 write 128k 32 8" +) + +for entry in "${TESTS[@]}"; do + read -r LABEL RW BS OD NJ <<< "$entry" + + echo "=3D=3D=3D $LABEL (iodepth=3D$OD, numjobs=3D$NJ) =3D=3D=3D" + + BEST_IOPS=3D0 + BEST_FILE=3D"" + + for run in $(seq 1 $RUNS); do + OUTFILE=3D"${OUTDIR}/${LABEL}_run${run}.json" + + # For numjobs=3D1, pin to single CPU; for numjobs>1, let fio distr= ibute + PIN_ARGS=3D"" + if [ "$NJ" -eq 1 ]; then + PIN_ARGS=3D"--cpus_allowed=3D$PIN_CPU --cpus_allowed_policy=3D= shared" + fi + + echo " run $run/$RUNS ..." + fio --ioengine=3Dlibaio \ + --direct=3D1 \ + --rw=3D"$RW" \ + --bs=3D"$BS" \ + --iodepth=3D$OD \ + --numjobs=3D$NJ \ + $PIN_ARGS \ + --time_based \ + --runtime=3D$RUNTIME \ + --group_reporting \ + --percentile_list=3D50,95,99,99.9 \ + --filename=3D"$DEV" \ + --name=3D"$LABEL" \ + --output-format=3Djson \ + --output=3D"$OUTFILE" + + # Extract IOPS from JSON + IOPS=3D$(python3 -c " +import json, sys +with open('$OUTFILE') as f: + d =3D json.load(f) +jobs =3D d.get('jobs', []) +if jobs: + rd =3D jobs[0].get('read', {}) + wr =3D jobs[0].get('write', {}) + iops =3D rd.get('iops', 0) + wr.get('iops', 0) + print(f'{iops:.0f}') +else: + print('0') +" 2>/dev/null || echo "0") + + echo " IOPS: $IOPS" + + # Track best run + if [ "$IOPS" -gt "$BEST_IOPS" ]; then + BEST_IOPS=3D$IOPS + BEST_FILE=3D$OUTFILE + fi + done + + # Copy best run as the canonical result + if [ -n "$BEST_FILE" ]; then + cp "$BEST_FILE" "${OUTDIR}/${LABEL}_best.json" + fi + + echo "" +done + +# Collect dmesg after +dmesg > "$OUTDIR/guest-dmesg-after.log" 2>/dev/null || true + +# Generate summary +echo "=3D=3D=3D Generating summary =3D=3D=3D" +python3 - <<'PYEOF' "$OUTDIR" +import json, sys, os, glob + +outdir =3D sys.argv[1] +print(f"\n{'=3D'*80}") +print(f" RESULTS: {os.path.basename(outdir)}") +print(f"{'=3D'*80}\n") + +header =3D f"{'Test':<30} {'IOPS':>10} {'p50(us)':>10} {'p99(us)':>10} {'p= 99.9(us)':>10} {'BW(MiB/s)':>10} {'lat_avg(us)':>12}" +print(header) +print("-" * len(header)) + +for best in sorted(glob.glob(os.path.join(outdir, "*_best.json"))): + label =3D os.path.basename(best).replace("_best.json", "") + try: + with open(best) as f: + d =3D json.load(f) + jobs =3D d.get("jobs", []) + if not jobs: + continue + + j =3D jobs[0] + rd =3D j.get("read", {}) + wr =3D j.get("write", {}) + iops =3D rd.get("iops", 0) + wr.get("iops", 0) + bw =3D (rd.get("bw", 0) + wr.get("bw", 0)) / 1024 # KiB/s -> MiB/s + + lat_ns =3D rd.get("lat_ns", {}) or wr.get("lat_ns", {}) + avg_us =3D lat_ns.get("mean", 0) / 1000 + + # Percentiles from clat_ns (completion latency) + clat =3D rd.get("clat_ns", {}) or wr.get("clat_ns", {}) + pct =3D clat.get("percentile", {}) if isinstance(clat, dict) else = {} + p50 =3D pct.get("50.000000", 0) / 1000 + p99 =3D pct.get("99.000000", 0) / 1000 + p999 =3D pct.get("99.900000", 0) / 1000 + + print(f"{label:<30} {iops:>10,.0f} {p50:>10.1f} {p99:>10.1f} {p999= :>10.1f} {bw:>10.1f} {avg_us:>12.1f}") + except Exception as e: + print(f"{label:<30} ERROR: {e}") + +print() +PYEOF + +echo "" +echo "=3D=3D=3D Done. Results in $OUTDIR =3D=3D=3D" +echo "=3D=3D=3D After running both modes, compare with: =3D=3D=3D" +echo " diff <(python3 summarize.py perf-results/baseline_*) <(python3 s= ummarize.py perf-results/sqcq-poll_*)" +echo "" +echo "=3D=3D=3D Or paste both summary tables side-by-side for comparison = =3D=3D=3D" diff --git a/Documentation/virtio-sqcq-poll-spec.rst b/Documentation/virtio= -sqcq-poll-spec.rst new file mode 100644 index 0000000000000..523bb3e3a1963 --- /dev/null +++ b/Documentation/virtio-sqcq-poll-spec.rst @@ -0,0 +1,617 @@ +Virtio SQ/CQ Polling Mode =E2=80=94 =E6=8A=80=E6=9C=AF=E8=A7=84=E6=A0=BC= =E4=B9=A6 +=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D + +Host Kernel vhost =E7=AB=AF=E5=B7=B2=E5=AE=9E=E7=8E=B0=E3=80=82Guest Kerne= l =E7=AB=AF=E5=B7=B2=E5=AE=9E=E7=8E=B0=E3=80=82QEMU =E7=AB=AF=E5=B7=B2=E5= =AE=9E=E7=8E=B0=E3=80=82 + +1. =E6=A6=82=E8=BF=B0 +------- + +=E4=BC=A0=E7=BB=9F virtio =E4=BD=BF=E7=94=A8 MMIO kick (Guest=E2=86=92Devi= ce) =E5=92=8C MSI-X =E4=B8=AD=E6=96=AD (Device=E2=86=92Guest) =E9=80=9A=E7= =9F=A5=EF=BC=8C +=E6=AF=8F=E6=AC=A1=E6=B6=89=E5=8F=8A VM exit=E3=80=82SQ/CQ polling =E5=80= =9F=E9=89=B4 io_uring SQPOLL =E6=A8=A1=E5=9E=8B=EF=BC=8C=E5=9C=A8 split rin= g =E6=97=81=E5=BC=95=E5=85=A5 +SQ/CQ =E9=97=A8=E9=93=83=E7=BB=93=E6=9E=84=EF=BC=8C=E7=94=B1=E8=BD=AE=E8= =AF=A2=E7=BA=BF=E7=A8=8B=E6=A3=80=E6=B5=8B=E5=8F=98=E6=9B=B4=E6=9B=BF=E4=BB= =A3=E4=B8=AD=E6=96=AD=EF=BC=8C=E6=B6=88=E9=99=A4 VM exit=E3=80=82 + +- Guest=E2=86=92Device=EF=BC=9AGuest =E6=9B=B4=E6=96=B0 SQ.idx=EF=BC=8CDev= ice =E8=BD=AE=E8=AF=A2=E7=BA=BF=E7=A8=8B=E6=A3=80=E6=B5=8B +- Device=E2=86=92Guest=EF=BC=9ADevice =E6=9B=B4=E6=96=B0 CQ.idx=EF=BC=8CGu= est =E8=BD=AE=E8=AF=A2=E7=BA=BF=E7=A8=8B=E6=A3=80=E6=B5=8B + +SQ/CQ =E9=97=A8=E9=93=83=E4=BB=85=E5=81=9A=E9=80=9A=E7=9F=A5=E4=BF=A1=E5= =8F=B7=EF=BC=8C=E5=AE=9E=E9=99=85=E6=95=B0=E6=8D=AE=E4=BB=8D=E9=80=9A=E8=BF= =87 desc/avail/used ring =E4=BC=A0=E9=80=92=E3=80=82 +=E9=80=9A=E8=BF=87 Feature Bit =E5=8D=8F=E5=95=86=EF=BC=8C=E6=9C=AA=E5=8D= =8F=E5=95=86=E6=97=B6=E9=9B=B6=E5=BC=80=E9=94=80=E3=80=82=E4=BB=85=E6=94=AF= =E6=8C=81 split ring=E3=80=82 + +1.1 =E5=9C=B0=E5=9D=80=E4=BC=A0=E9=80=92=E6=9E=B6=E6=9E=84 +~~~~~~~~~~~~~~~ + +:: + + Guest Kernel QEMU (userspace) Host Kernel (vhost) + PCI config space GPA =E2=86=92 HVA =E8=BD=AC=E6=8D=A2 VHO= ST_SET_VRING_SQCQ_ADDR + (=E5=86=99=E5=85=A5 GPA) ioctl (=E4=BC=A0=E5=85=A5 HVA) = (=E6=8E=A5=E6=94=B6 HVA) + +1.2 =E9=80=9A=E7=9F=A5=E8=B7=AF=E5=BE=84 +~~~~~~~~~~~~ + +:: + + =E6=B4=BB=E8=B7=83=E8=B7=AF=E5=BE=84=EF=BC=88=E9=9B=B6=E5=BC=80=E9=94=80= =EF=BC=89=EF=BC=9A + Guest =E6=8F=90=E4=BA=A4 =E2=86=92 SQ.idx =E2=86=92 Device =E8=BD=AE= =E8=AF=A2=E6=A3=80=E6=B5=8B=EF=BC=88=E6=B6=88=E9=99=A4 VM exit=EF=BC=89 + Device =E5=AE=8C=E6=88=90 =E2=86=92 CQ.idx =E2=86=92 Guest =E8=BD=AE= =E8=AF=A2=E6=A3=80=E6=B5=8B=EF=BC=88=E6=B6=88=E9=99=A4 VM exit=EF=BC=89 + + =E4=BC=91=E7=9C=A0=E5=94=A4=E9=86=92=E8=B7=AF=E5=BE=84=EF=BC=9A + Guest =E6=8F=90=E4=BA=A4 =E2=86=92 SQ.idx =E2=86=92 MMIO kick =E2=86= =92 QEMU =E2=86=92 eventfd =E2=86=92 vhost =E5=94=A4=E9=86=92 + Device =E5=AE=8C=E6=88=90 =E2=86=92 CQ.idx =E2=86=92 eventfd_signal = =E2=86=92 QEMU =E2=86=92 MSI-X =E2=86=92 Guest =E5=94=A4=E9=86=92 + + +2. Feature =E5=8D=8F=E5=95=86 +--------------- + +``VIRTIO_F_SQCQ_POLL =3D 42``=EF=BC=88``include/uapi/linux/virtio_config.h= ``=EF=BC=89=E3=80=82 + +``VIRTIO_TRANSPORT_F_END`` =E4=BB=8E 42 =E6=9B=B4=E6=96=B0=E4=B8=BA 43=E3= =80=82 +=E8=AF=A5 bit =E5=B1=9E=E4=BA=8E transport feature range (28..43)=EF=BC=8C= =E9=9C=80=E5=90=8C=E6=97=B6=E8=A2=AB virtio_ring =E5=92=8C virtio_pci =E6= =8E=A5=E5=8F=97=E3=80=82 +vhost =E7=AB=AF=E5=B0=86=E8=AF=A5 feature =E5=8A=A0=E5=85=A5 ``VHOST_FEATU= RES`` =E5=AE=8F=EF=BC=8C=E5=9C=A8 ``VHOST_GET_FEATURES`` ioctl =E4=B8=AD=E5= =AF=B9 Guest =E5=8F=AF=E8=A7=81=E3=80=82 + +**=E5=8D=8F=E5=95=86=E6=B5=81=E7=A8=8B=EF=BC=9A** + +1. Device =E8=AE=BE=E7=BD=AE bit 42 =E2=86=92 Guest =E6=8E=A5=E5=8F=97 =E2= =86=92 finalize_features +2. Guest =E5=9C=A8 find_vqs =E9=98=B6=E6=AE=B5=E5=88=86=E9=85=8D SQ/CQ =E7= =BB=93=E6=9E=84 +3. Guest =E5=9C=A8 vp_active_vq =E9=98=B6=E6=AE=B5=E5=B0=86 SQ/CQ DMA =E5= =9C=B0=E5=9D=80=E5=86=99=E5=85=A5 PCI config space +4. QEMU =E8=AF=BB GPA =E2=86=92 =E8=BD=AC HVA =E2=86=92 ``VHOST_SET_VRING_= SQCQ_ADDR`` ioctl =E2=86=92 vhost +5. vhost =E5=9C=A8 ``VHOST_SCSI_SET_ENDPOINT`` =E6=97=B6=E5=90=AF=E5=8A=A8= =E8=BD=AE=E8=AF=A2=E7=BA=BF=E7=A8=8B + +=E4=BB=BB=E4=B8=80=E6=96=B9=E4=B8=8D=E6=94=AF=E6=8C=81=E5=88=99=E5=9B=9E= =E9=80=80=E4=BC=A0=E7=BB=9F=E4=B8=AD=E6=96=AD=E6=A8=A1=E5=BC=8F=E3=80=82 + + +3. =E6=95=B0=E6=8D=AE=E7=BB=93=E6=9E=84 +----------- + +3.1 SQ/CQ =E9=97=A8=E9=93=83 +~~~~~~~~~~~~~~~ + +=E5=AE=9A=E4=B9=89=E4=BA=8E ``include/uapi/linux/virtio_ring.h``=EF=BC=9A + +.. code-block:: c + + struct vring_sq { + __virtio64 idx; /* Producer position: Guest =E6=9B=B4= =E6=96=B0 */ + __u8 sq_need_wakeup; /* 1 =3D Device =E5=BA=94=E8=A2=AB kic= k =E5=94=A4=E9=86=92 */ + __u8 reserved[55]; + } __attribute__((aligned(128))); /* 128 bytes, cache-line =E5=AF=B9= =E9=BD=90 */ + + struct vring_cq { + __virtio64 idx; /* Producer position: Device =E6=9B=B4= =E6=96=B0 */ + __u8 cq_need_wakeup; /* 1 =3D Guest =E5=BA=94=E8=A2=AB=E4= =B8=AD=E6=96=AD=E5=94=A4=E9=86=92 */ + __u8 reserved[55]; + } __attribute__((aligned(128))); /* 128 bytes, cache-line =E5=AF=B9= =E9=BD=90 */ + +- ``sq->idx`` =E5=AF=B9=E5=BA=94 ``avail->idx``=EF=BC=8C``cq->idx`` =E5=AF= =B9=E5=BA=94 ``used->idx`` +- 128 =E5=AD=97=E8=8A=82=E5=AF=B9=E9=BD=90=E8=A6=86=E7=9B=96 ARM Neoverse = N2/V2=E3=80=81Apple M =E7=B3=BB=E5=88=97=E7=AD=89 128 =E5=AD=97=E8=8A=82 ca= che line =E7=9A=84 CPU=EF=BC=8C + =E6=B6=88=E9=99=A4 SQ =E4=B8=8E CQ =E4=B9=8B=E9=97=B4=E7=9A=84 false sha= ring +- ``sq/cq_need_wakeup`` =E4=B8=BA 8-bit =E7=9B=B4=E6=8E=A5=E8=B5=8B=E5=80= =BC=E5=AD=97=E6=AE=B5=EF=BC=8C=E7=94=A8=E4=BA=8E NEED_WAKEUP =E5=8D=8F=E8= =AE=AE=EF=BC=88=E8=A7=81=E7=AC=AC 6 =E8=8A=82=EF=BC=89 + +3.2 PCI Config Space =E5=AF=84=E5=AD=98=E5=99=A8 +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +=E8=BF=BD=E5=8A=A0=E5=88=B0 ``struct virtio_pci_modern_common_cfg`` =E6=9C= =AB=E5=B0=BE=EF=BC=88offset 62 =E4=B9=8B=E5=90=8E=EF=BC=89=EF=BC=8C +=E5=AE=9A=E4=B9=89=E4=BA=8E ``include/uapi/linux/virtio_pci.h``=EF=BC=9A + +.. code-block:: c + + #define VIRTIO_PCI_COMMON_SQE_LO 64 + #define VIRTIO_PCI_COMMON_SQE_HI 68 + #define VIRTIO_PCI_COMMON_CQE_LO 72 + #define VIRTIO_PCI_COMMON_CQE_HI 76 + + __le32 sqe_ring_lo; /* offset 64: SQ DMA =E5=9C=B0=E5=9D=80=E4=BD= =8E 32 =E4=BD=8D */ + __le32 sqe_ring_hi; /* offset 68: SQ DMA =E5=9C=B0=E5=9D=80=E9=AB= =98 32 =E4=BD=8D */ + __le32 cqe_ring_lo; /* offset 72: CQ DMA =E5=9C=B0=E5=9D=80=E4=BD= =8E 32 =E4=BD=8D */ + __le32 cqe_ring_hi; /* offset 76: CQ DMA =E5=9C=B0=E5=9D=80=E9=AB= =98 32 =E4=BD=8D */ + +=E5=8F=97 queue_select =E7=BA=A6=E6=9D=9F=E3=80=82UAPI =E5=B7=B2=E5=AE=9A= =E4=B9=89=EF=BC=8CGuest =E5=B7=B2=E5=AE=9E=E7=8E=B0=E5=86=99=E5=85=A5=EF=BC= =8CQEMU =E5=B7=B2=E5=AE=9E=E7=8E=B0=E8=AF=BB=E5=8F=96=E3=80=82 + +3.3 vhost UAPI +~~~~~~~~~~~~~~~ + +.. code-block:: c + + // include/uapi/linux/vhost.h + #define VHOST_SET_VRING_SQCQ_ADDR _IOW(VHOST_VIRTIO, 0x27, \ + struct vhost_vring_sqcq_addr) + + // include/uapi/linux/vhost_types.h + struct vhost_vring_sqcq_addr { + unsigned int index; + __u64 sq_user_addr; /* SQ doorbell HVA */ + __u64 cq_user_addr; /* CQ doorbell HVA */ + }; + +QEMU =E8=B0=83=E7=94=A8=E6=97=B6=E6=9C=BA=EF=BC=9A=E5=9C=A8 ``VHOST_SET_VR= ING_ADDR`` =E4=B9=8B=E5=90=8E=E3=80=81``VHOST_SCSI_SET_ENDPOINT`` =E4=B9=8B= =E5=89=8D=E3=80=82 + +3.4 =E5=86=99=E5=85=A5=E6=97=B6=E5=BA=8F +~~~~~~~~~~~~ + +Guest=EF=BC=88vp_active_vq =E4=B8=AD=EF=BC=89=EF=BC=9Aqueue_select =E2=86= =92 desc/avail/used =E5=9C=B0=E5=9D=80 =E2=86=92 sqe/cqe_ring_lo/hi =E5=9C= =B0=E5=9D=80 +QEMU=EF=BC=9A=E7=BB=84=E5=90=88 GPA =E2=86=92 address_space_map =E8=BD=AC = HVA =E2=86=92 VHOST_SET_VRING_SQCQ_ADDR ioctl + + +4. =E5=AD=97=E6=AE=B5=E8=AF=BB=E5=86=99=E6=89=80=E6=9C=89=E6=9D=83 +----------------- + +.. list-table:: + :header-rows: 1 + :widths: 20 20 20 25 15 + + * - =E5=AD=97=E6=AE=B5 + - =E5=86=99=E5=85=A5=E6=96=B9 + - =E8=AF=BB=E5=8F=96=E6=96=B9 + - =E5=86=85=E5=AD=98=E5=BA=8F + - =E5=AF=B9=E5=BA=94 + + * - sq->idx + - Guest + - Device + - Guest: smp_store_release; Device: get_user + smp_rmb + - avail->idx + + * - sq->sq_need_wakeup + - Device + - Guest + - Device: smp_wmb + put_user + smp_mb (double-check) + - NEED_WAKEUP + + * - cq->idx + - Device + - Guest + - Device: smp_wmb before put_user; Guest: smp_load_acquire + - used->idx + + * - cq->cq_need_wakeup + - Guest + - Device + - Guest: smp_store_release; Device: get_user + smp_rmb + - NEED_WAKEUP + + +5. =E6=A0=B8=E5=BF=83=E5=8D=8F=E8=AE=AE +----------- + +5.1 Guest =E6=8F=90=E4=BA=A4=E8=AF=B7=E6=B1=82 (SQ =E6=96=B9=E5=90=91) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. code-block:: c + + // virtqueue_kick() =E4=B8=AD sqcq_poll =E6=A8=A1=E5=BC=8F + smp_store_release(&vring->sq->idx, avail_idx_shadow); + need_wakeup =3D smp_load_acquire(&vring->sq->sq_need_wakeup); + if (need_wakeup) + virtqueue_notify(vq); // MMIO write=EF=BC=8C=E4=BB=85=E5=9C=A8 De= vice =E4=BC=91=E7=9C=A0=E6=97=B6 + +**=E5=85=B3=E9=94=AE=EF=BC=9A** Guest =E4=BB=8D=E5=86=99 ``avail->idx``=EF= =BC=88``virtqueue_add_split()`` =E5=AE=8C=E6=88=90=EF=BC=89=E3=80=82 +SQ idx =E6=98=AF=E9=A2=9D=E5=A4=96=E9=80=9A=E7=9F=A5=E4=BF=A1=E5=8F=B7=EF= =BC=8C=E8=AE=A9 Device =E5=BF=AB=E9=80=9F=E6=A3=80=E6=B5=8B=EF=BC=88=E8=AF= =BB=E4=B8=80=E4=B8=AA cache line vs =E8=AF=BB avail ring =E4=B8=A4=E6=AC=A1= =EF=BC=89=E3=80=82 +Device =E7=9A=84 ``handle_kick()`` =E9=80=9A=E8=BF=87 ``avail->idx`` =E5= =AE=9A=E4=BD=8D descriptor=E3=80=82 + +**virtqueue_kick_prepare() =E5=AE=88=E5=8D=AB=EF=BC=9A** sqcq_poll =E6=A8= =A1=E5=BC=8F=E4=B8=8B=E7=9B=B4=E6=8E=A5=E8=BF=94=E5=9B=9E ``true``=EF=BC=8C +=E4=B8=8D=E6=A3=80=E6=9F=A5 ``used->flags``=EF=BC=88vhost =E4=B8=8D=E7=BB= =B4=E6=8A=A4=E8=AF=A5 flag=EF=BC=89=E3=80=82 + +5.2 Device =E5=A4=84=E7=90=86=E8=AF=B7=E6=B1=82 (vhost =E8=BD=AE=E8=AF=A2= =E7=BA=BF=E7=A8=8B) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +=E7=8B=AC=E7=AB=8B kthread ``vhost_sqcq_poll_thread``=E3=80=82=E7=9B=B4=E6= =8E=A5=E8=B0=83=E7=94=A8 ``handle_kick`` =E5=A4=84=E7=90=86=E6=8F=90=E4=BA= =A4=EF=BC=8C +=E9=80=9A=E8=BF=87 ``poll_complete`` =E5=9B=9E=E8=B0=83=E5=A4=84=E7=90=86= =E5=AE=8C=E6=88=90=EF=BC=8C=E5=8D=95=E7=BA=BF=E7=A8=8B=E9=97=AD=E7=8E=AF=E3= =80=82 + +=E4=B8=BB=E5=BE=AA=E7=8E=AF=E4=BC=AA=E4=BB=A3=E7=A0=81=EF=BC=88drivers/vho= st/vhost.c=EF=BC=89=EF=BC=9A + +.. code-block:: c + + while (!poll_stop): + // 0. =E5=A4=84=E7=90=86 pending completion + if (atomic_read(&vq->completion_pending) && vq->poll_complete): + atomic_set(&vq->completion_pending, 0) + vq->poll_complete(vq) // copy resp, add used, =E5=86=99 cq->i= dx, =E6=A3=80=E6=9F=A5 cq_need_wakeup + + // 1. =E8=AF=BB SQ idx + ret =3D get_user(sq_idx_val, &vq->sq->idx) + smp_rmb() // acquire + new_sq_idx =3D (u16)sq_idx_val + + // 2. =E6=9C=89=E6=96=B0=E6=8F=90=E4=BA=A4=EF=BC=9A=E5=90=8C=E6=AD= =A5=E8=B0=83=E7=94=A8 handle_kick + if (new_sq_idx !=3D vq->last_avail_idx): + vq->handle_kick(&vq->poll.work) + + // 3. =E8=87=AA=E9=80=82=E5=BA=94=E4=BC=91=E7=9C=A0=EF=BC=88EMA = =E5=8A=A8=E6=80=81=E8=B6=85=E6=97=B6=EF=BC=89 + should_sleep =3D (last_avail_idx =3D=3D last_used_idx) + if (!should_sleep && ema_ns > 0) + should_sleep =3D (active_io_vqs > 1) && (ema_us > SQCQ_SLEEP_T= HRESHOLD_US) + + if (should_sleep): + timeout =3D ema_based_or_fallback(SQCQ_IDLE_TIMEOUT_US_MAX) + // =E8=AE=BE=E7=BD=AE NEED_WAKEUP + double-check idx + smp_wmb(); put_user(1, &sq->sq_need_wakeup); smp_mb() + if (idx changed): put_user(0, &sq->sq_need_wakeup); continue + // =E6=B3=A8=E5=86=8C kick eventfd=EF=BC=8C=E5=B8=A6=E8=B6=85= =E6=97=B6=E7=AD=89=E5=BE=85 + kick_register(vq) + wait_event_interruptible_timeout(poll_wait, + poll_stop || kick_received || completion_pending, timeout) + was_kicked =3D READ_ONCE(kick_received) + kick_unregister(vq) + put_user(0, &sq->sq_need_wakeup) + else: + cpu_relax() + +5.3 I/O =E5=AE=8C=E6=88=90=E8=B7=AF=E5=BE=84 +~~~~~~~~~~~~~~~~~ + +Poll =E6=A8=A1=E5=BC=8F=E4=B8=8B completion =E4=B8=8D=E7=BB=8F=E8=BF=87 vh= ost_worker=EF=BC=9A + +1. **TCM =E5=AE=8C=E6=88=90**=EF=BC=88=E4=B8=AD=E6=96=AD/workqueue =E4=B8= =8A=E4=B8=8B=E6=96=87=EF=BC=89=EF=BC=9A``llist_add`` + ``atomic_set(complet= ion_pending, 1)`` + =E6=9D=A1=E4=BB=B6 ``wake_up`` +2. **Poll thread =E6=AD=A5=E9=AA=A4 0**=EF=BC=88poll thread =E4=B8=8A=E4= =B8=8B=E6=96=87=EF=BC=89=EF=BC=9A``poll_complete`` =E5=9B=9E=E8=B0=83 +3. **poll_complete =E5=86=85=E9=83=A8**=EF=BC=9A``llist_del_all`` =E2=86= =92 ``mutex_lock`` =E2=86=92 copy response + ``vhost_add_used`` =E2=86=92 `= `mutex_unlock`` =E2=86=92 ``put_user(cq->idx)`` =E2=86=92 =E6=A3=80=E6=9F= =A5 ``cq_need_wakeup`` =E2=86=92 ``eventfd_signal`` + +=E8=AE=BE=E8=AE=A1=E8=A6=81=E7=82=B9=EF=BC=9A=E5=8D=95=E7=BA=BF=E7=A8=8B= =E9=97=AD=E7=8E=AF=EF=BC=88handle_kick =E5=92=8C poll_complete =E9=A1=BA=E5= =BA=8F=E6=89=A7=E8=A1=8C=EF=BC=8C=E5=85=B1=E4=BA=AB mutex =E4=B8=8D=E4=BC= =9A=E6=AD=BB=E9=94=81=EF=BC=89=EF=BC=9B +CQ idx =E5=8D=95=E5=86=99=E8=80=85=EF=BC=88=E4=BB=85 poll_complete=EF=BC= =89=EF=BC=9B=E6=9D=A1=E4=BB=B6 wake_up=EF=BC=88=E4=BB=85=E5=9C=A8 poll thre= ad =E9=9D=9E RUNNING =E6=97=B6=EF=BC=89=E3=80=82 + +5.4 vhost_enable/disable_notify =E5=9C=A8 poll =E6=A8=A1=E5=BC=8F=E4=B8=8B +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +- ``vhost_enable_notify()`` =E2=86=92 ``return false`` +- ``vhost_disable_notify()`` =E2=86=92 ``return`` + +=E9=80=9A=E7=9F=A5=E6=8E=A7=E5=88=B6=E7=94=B1 sq_need_wakeup / cq_need_wak= eup =E6=89=BF=E6=8B=85=EF=BC=8Cused ring flags =E4=B8=8D=E5=86=8D=E6=9C=89= =E6=95=88=E3=80=82 +Guest =E7=AB=AF=E5=AF=B9=E7=A7=B0=EF=BC=9A``virtqueue_kick_prepare()`` =E7= =9B=B4=E6=8E=A5=E8=BF=94=E5=9B=9E true=EF=BC=8C``disable_cb/enable_cb`` =E6= =93=8D=E4=BD=9C avail->flags =E4=BD=86 vhost =E5=BF=BD=E7=95=A5=E3=80=82 + +5.5 Guest =E6=8E=A5=E6=94=B6=E5=AE=8C=E6=88=90 (CQ =E6=96=B9=E5=90=91) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Guest =E8=BD=AE=E8=AF=A2=E7=BA=BF=E7=A8=8B=EF=BC=88``virtio_sqcq_poll_thre= ad``=EF=BC=89=EF=BC=8C=E6=AF=8F device =E4=B8=80=E4=B8=AA=E7=BA=BF=E7=A8=8B= =E3=80=82 + +=E4=B8=BB=E5=BE=AA=E7=8E=AF=E8=A6=81=E7=82=B9=EF=BC=9A + +1. ``while (cq_poll_has_work())`` drain =E8=AF=A5 vq =E6=89=80=E6=9C=89=E5= =AE=8C=E6=88=90=EF=BC=8Ccallback =E5=90=8E ``do_softirq()`` +2. =E6=9C=89=E5=B7=A5=E4=BD=9C=E6=97=B6=E9=87=8D=E7=BD=AE ``last_io_ts =3D= ktime_get()``=EF=BC=8C=E4=B8=8D yield=EF=BC=8C=E5=BF=AB=E9=80=9F=E8=BD=AE= =E8=AF=A2 +3. =E6=97=A0=E5=B7=A5=E4=BD=9C=E6=97=B6=E5=9F=BA=E4=BA=8E ``last_io_ts`` = =E6=97=B6=E9=97=B4=E7=AA=97=E5=8F=A3=E8=87=AA=E6=97=8B=EF=BC=88``busy_spin_= us`` =3D 1ms=EF=BC=89 +4. =E8=87=AA=E6=97=8B=E4=B8=AD=E4=BD=BF=E7=94=A8 ``need_resched()`` + ``cp= u_relax()`` =E5=8D=8F=E4=BD=9C=E8=AE=A9=E5=87=BA=EF=BC=88io_uring SQPOLL = =E6=A8=A1=E5=9E=8B=EF=BC=89 +5. spin =E7=AA=97=E5=8F=A3=E8=BF=87=E6=9C=9F=E5=90=8E=E8=AE=BE=E7=BD=AE=E6= =89=80=E6=9C=89 vq =E7=9A=84 ``cq_need_wakeup`` + double-check ``more_used(= )`` +6. =E6=97=A0=E6=96=B0=E5=B7=A5=E4=BD=9C=E5=88=99 ``schedule_timeout(idle_t= imeout_us)`` =E4=BC=91=E7=9C=A0=EF=BC=881ms=EF=BC=89 +7. =E9=86=92=E6=9D=A5=E5=90=8E=E6=B8=85=E9=99=A4=E6=89=80=E6=9C=89 vq =E7= =9A=84 NEED_WAKEUP=EF=BC=8C=E4=B8=8D=E9=87=8D=E7=BD=AE spin =E7=AA=97=E5=8F= =A3=EF=BC=88=E8=B6=85=E6=97=B6=E5=94=A4=E9=86=92=E4=B8=8D re-arm spin=EF=BC= =89 + +**CQ =E5=AE=8C=E6=88=90=E6=A3=80=E6=B5=8B=EF=BC=9A** =E7=9B=B4=E6=8E=A5=E4= =BD=BF=E7=94=A8 ``more_used(vq)`` =E6=A3=80=E6=9F=A5 used ring=EF=BC=8C=E4= =B8=8D=E5=86=8D=E8=BD=AE=E8=AF=A2 ``cq->idx``=E3=80=82 +``cq->idx`` =E4=BB=85=E7=94=B1 Device =E5=86=99=E5=85=A5=EF=BC=8C=E4=BF=9D= =E7=95=99=E7=94=A8=E4=BA=8E NEED_WAKEUP =E5=8D=8F=E8=AE=AE=E7=9A=84 ``cq_ne= ed_wakeup`` =E5=AD=97=E6=AE=B5=EF=BC=8C +=E4=B8=8D=E5=86=8D=E4=BD=9C=E4=B8=BA=E5=AE=8C=E6=88=90=E6=A3=80=E6=B5=8B= =E7=9A=84 fast path=EF=BC=88=E6=B6=88=E9=99=A4=E4=BA=86 cq->idx =E4=B8=8E u= sed ring =E4=B9=8B=E9=97=B4=E7=9A=84=E7=AB=9E=E6=80=81=EF=BC=89=E3=80=82 + +**CQ =E8=AE=BF=E9=97=AE=E5=99=A8=E5=87=BD=E6=95=B0** (virtio_ring.c)=EF=BC= =9A + +- ``cq_poll_has_work(vq)``=EF=BC=9A``more_used(vq)`` =E2=80=94 =E7=9B=B4= =E6=8E=A5=E6=A3=80=E6=9F=A5 used ring +- ``cq_set_need_wakeup(vq)``=EF=BC=9A``smp_store_release(&cq->cq_need_wake= up, 1)`` +- ``cq_clear_need_wakeup(vq)``=EF=BC=9A``WRITE_ONCE(cq->cq_need_wakeup, 0)= `` +- ``cq_recheck(vq)``=EF=BC=9A``smp_mb()`` + ``more_used(vq)`` + +5.6 Device =E5=94=A4=E9=86=92=E8=B7=AF=E5=BE=84 (kick eventfd =E6=A1=A5=E6= =8E=A5) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +:: + + Guest MMIO kick =E2=86=92 QEMU eventfd write =E2=86=92 eventfd wq =E2=86= =92 + vhost_sqcq_kick_wakeup() =E2=86=92 kick_received=3Dtrue + wake_up =E2=86= =92 + poll thread =E2=86=92 clear sq_need_wakeup =E2=86=92 process sq->idx + +=E4=BC=91=E7=9C=A0=E5=89=8D=E9=80=9A=E8=BF=87 ``vhost_sqcq_kick_register()= `` =E6=B3=A8=E5=86=8C=E5=88=B0 kick eventfd wait queue=E3=80=82 +=E5=85=B3=E9=94=AE=EF=BC=9A``vfs_poll`` =E5=8F=AF=E8=83=BD=E5=9B=A0 stale = counter =E8=BF=94=E5=9B=9E EPOLLIN=EF=BC=8C=E5=BF=85=E9=A1=BB=E9=AA=8C=E8= =AF=81 ``sq->idx`` +=E7=A1=AE=E5=AE=9E=E6=9C=89=E6=96=B0=E5=B7=A5=E4=BD=9C=E6=89=8D=E8=A7=A6= =E5=8F=91=E5=9B=9E=E8=B0=83=EF=BC=8C=E5=90=A6=E5=88=99=E6=89=80=E6=9C=89 po= ll thread busy-loop=E3=80=82 + +QEMU =E7=AB=AF=E6=97=A0=E9=9C=80=E9=A2=9D=E5=A4=96=E6=93=8D=E4=BD=9C=EF=BC= =8C=E7=8E=B0=E6=9C=89 kick eventfd =E8=B7=AF=E5=BE=84=E5=8D=B3=E5=8F=AF=E3= =80=82 + + +6. NEED_WAKEUP =E5=8D=8F=E8=AE=AE=EF=BC=88=E6=A0=B8=E5=BF=83=EF=BC=8C=E5= =BF=85=E9=A1=BB=E4=B8=A5=E6=A0=BC=E9=81=B5=E5=AE=88=EF=BC=89 +----------------------------------------- + +=E5=8F=8C=E5=90=91=E5=AF=B9=E7=A7=B0=EF=BC=8CSQ =E5=92=8C CQ =E5=90=84=E4= =B8=80=E5=A5=97=E3=80=82=E9=81=BF=E5=85=8D=E6=AD=BB=E9=94=81=E5=92=8C=E4=B8= =A2=E5=A4=B1=E5=94=A4=E9=86=92=E7=9A=84=E6=A0=B8=E5=BF=83=E6=9C=BA=E5=88=B6= =E3=80=82 + +6.1 SQ =E6=96=B9=E5=90=91 +~~~~~~~~~~~ + +**Device =E4=BC=91=E7=9C=A0=E5=89=8D=EF=BC=9A** + +.. code-block:: pseudo + + smp_wmb() + sq->sq_need_wakeup =3D 1 + smp_mb() + new_sq_idx =3D sq->idx // double-check + if (new_sq_idx !=3D last_avail): + sq->sq_need_wakeup =3D 0 // =E5=8F=96=E6=B6=88=E4=BC=91=E7=9C= =A0 + goto process + wait_event(kick_received) + sq->sq_need_wakeup =3D 0 // =E9=86=92=E6=9D=A5=E5=90=8E=E6=B8= =85=E9=99=A4 + +**Guest =E6=8F=90=E4=BA=A4=E5=90=8E=EF=BC=9A** + +.. code-block:: pseudo + + smp_store_release(&sq->idx, new_idx) + need_wakeup =3D smp_load_acquire(&sq->sq_need_wakeup) + if (need_wakeup) + virtqueue_notify(vq) // MMIO kick + +6.2 CQ =E6=96=B9=E5=90=91 +~~~~~~~~~~~ + + **Guest =E4=BC=91=E7=9C=A0=E5=89=8D=EF=BC=9A** + +.. code-block:: pseudo + + smp_store_release(&cq->cq_need_wakeup, 1) + smp_mb() + if (more_used(vq)): // double-check used ring + cq->cq_need_wakeup =3D 0 // =E5=8F=96=E6=B6=88=E4=BC=91=E7=9C= =A0 + goto process + schedule_timeout(idle_timeout_us) + cq->cq_need_wakeup =3D 0 // =E9=86=92=E6=9D=A5=E5=90=8E=E6=B8= =85=E9=99=A4 + +**Device =E5=AE=8C=E6=88=90=E5=90=8E=EF=BC=88=E4=B8=A4=E9=98=B6=E6=AE=B5= =EF=BC=89=EF=BC=9A** + +1. release_cmd=EF=BC=9A``llist_add`` + ``atomic_set(completion_pending, 1)= `` + =E6=9D=A1=E4=BB=B6 ``wake_up`` +2. poll_complete=EF=BC=9Acopy resp =E2=86=92 add used =E2=86=92 ``put_user= (cq->idx)`` =E2=86=92 =E6=A3=80=E6=9F=A5 ``cq_need_wakeup`` =E2=86=92 ``eve= ntfd_signal`` + +6.3 =E4=B8=BA=E4=BB=80=E4=B9=88=E9=9C=80=E8=A6=81 Double-check +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +=E8=AE=BE=E7=BD=AE NEED_WAKEUP =E2=86=92 =E9=87=8D=E8=AF=BB idx =E4=B9=8B= =E9=97=B4=E5=AD=98=E5=9C=A8=E7=AB=9E=E6=80=81=EF=BC=9AGuest =E5=8F=AF=E8=83= =BD=E5=9C=A8 Device =E8=AF=BB idx =E5=90=8E=E3=80=81 +sleep =E5=89=8D=E6=8F=90=E4=BA=A4=E3=80=82Device =E5=9C=A8 sleep =E5=89=8D= =E9=87=8D=E8=AF=BB idx=EF=BC=8C=E5=A6=82=E6=9E=9C=E5=8F=98=E4=BA=86=E5=88= =99=E5=8F=96=E6=B6=88=E4=BC=91=E7=9C=A0=E3=80=82 +=E6=AD=A4=E6=97=B6 Guest =E7=9A=84 kick =E5=8F=AF=E8=83=BD=E5=B7=B2=E5=88= =B0=E4=B9=9F=E5=8F=AF=E8=83=BD=E6=9C=AA=E5=88=B0=EF=BC=8C=E4=BD=86 Device = =E5=B7=B2=E5=9C=A8=E5=A4=84=E7=90=86=E6=96=B0=E5=B7=A5=E4=BD=9C=EF=BC=8C=E4= =B8=8D=E5=BD=B1=E5=93=8D=E6=AD=A3=E7=A1=AE=E6=80=A7=E3=80=82 + +6.4 =E5=86=85=E5=AD=98=E5=BA=8F=E6=80=BB=E7=BB=93 +~~~~~~~~~~~~~~ + +- idx =E6=9B=B4=E6=96=B0=EF=BC=9A=E5=86=99=E5=85=A5=E6=96=B9 store-release= =EF=BC=88Guest: ``smp_store_release``=EF=BC=9BDevice: ``smp_wmb`` + ``put_u= ser``=EF=BC=89 +- idx =E8=AF=BB=E5=8F=96=EF=BC=9AGuest =E4=BD=BF=E7=94=A8 ``more_used()`` = =E7=9B=B4=E6=8E=A5=E6=A3=80=E6=9F=A5 used ring=EF=BC=88=E4=B8=8D=E5=86=8D= =E8=BD=AE=E8=AF=A2 ``cq->idx``=EF=BC=89 +- sq_need_wakeup =E8=AF=BB=E5=8F=96=EF=BC=9AGuest =E4=BD=BF=E7=94=A8 ``smp= _load_acquire``=EF=BC=88=E6=9B=BF=E4=BB=A3 ``smp_mb`` + ``READ_ONCE``=EF=BC= =89 +- NEED_WAKEUP =E8=AE=BE=E7=BD=AE=E5=90=8E ``smp_mb()``=EF=BC=8C=E7=84=B6= =E5=90=8E=E9=87=8D=E8=AF=BB used ring / sq->idx +- NEED_WAKEUP =E6=B8=85=E9=99=A4=E5=9C=A8 sleep =E5=94=A4=E9=86=92=E5=90= =8E=E7=AB=8B=E5=8D=B3=E6=89=A7=E8=A1=8C +- x86 =E4=B8=8A=E5=B1=8F=E9=9A=9C=E4=B8=BA no-op=EF=BC=88TSO=EF=BC=89=EF= =BC=8CARM/RISC-V =E4=B8=8A=E7=94=9F=E6=88=90=E5=AE=9E=E9=99=85=E5=B1=8F=E9= =9A=9C=E6=8C=87=E4=BB=A4 + + +7. QEMU =E5=AE=9E=E7=8E=B0=E6=8C=87=E5=8D=97 +---------------- + +7.1 Feature =E5=8D=8F=E5=95=86 +~~~~~~~~~~~~~~~~ + +.. code-block:: c + + virtio_add_feature(&vdev->host_features, VIRTIO_F_SQCQ_POLL); + +7.2 GPA =E2=86=92 HVA =E8=BD=AC=E6=8D=A2 +~~~~~~~~~~~~~~~~~~~ + +.. code-block:: c + + hwaddr sq_gpa =3D (sqe_ring_hi << 32) | sqe_ring_lo; + void *sq_hva =3D address_space_map(&address_space_memory, sq_gpa, size= of(struct vring_sq), false); + // ... cq =E7=B1=BB=E4=BC=BC + struct vhost_vring_sqcq_addr addr =3D { .index =3D idx, .sq_user_addr = =3D sq_hva, .cq_user_addr =3D cq_hva }; + ioctl(vhost_fd, VHOST_SET_VRING_SQCQ_ADDR, &addr); + +7.3 ioctl =E8=B0=83=E7=94=A8=E9=A1=BA=E5=BA=8F +~~~~~~~~~~~~~~~~~~ + +=E5=90=AF=E5=8A=A8=EF=BC=9A + +:: + + 1. VHOST_SET_VRING_ADDR (desc/avail/used) + 2. VHOST_SET_VRING_SQCQ_ADDR (sq/cq) =E2=80=94 =E6=96=B0=E5=A2=9E + 3. VHOST_SET_VRING_KICK + 4. VHOST_SET_VRING_CALL + 5. VHOST_SCSI_SET_ENDPOINT =E2=80=94 =E8=A7=A6=E5=8F=91=E8=BD=AE=E8=AF= =A2=E7=BA=BF=E7=A8=8B=E5=90=AF=E5=8A=A8 + +=E5=81=9C=E6=AD=A2=EF=BC=9A + +:: + + 1. VHOST_SCSI_CLEAR_ENDPOINT =E2=80=94 =E8=87=AA=E5=8A=A8=E5=81=9C=E6=AD= =A2=E8=BD=AE=E8=AF=A2=E7=BA=BF=E7=A8=8B + 2. VHOST_SET_VRING_KICK (null_fd) + 3. VHOST_SET_VRING_CALL (null_fd) + +7.4 call eventfd =E5=A4=84=E7=90=86 +~~~~~~~~~~~~~~~~~~~~~ + +=E6=94=B6=E5=88=B0 eventfd signal =E8=AF=B4=E6=98=8E Guest =E8=AE=BE=E7=BD= =AE=E4=BA=86 cq_need_wakeup=EF=BC=8C=E9=9C=80=E6=B3=A8=E5=85=A5 MSI-X =E4= =B8=AD=E6=96=AD=E3=80=82 +=E4=BD=BF=E7=94=A8=E7=8E=B0=E6=9C=89 ``virtio_irq()`` / ``virtio_notify_ir= qfd()`` =E5=8D=B3=E5=8F=AF=E3=80=82 + +=E5=8D=B3=E4=BD=BF=E5=90=AF=E7=94=A8 polling=EF=BC=8CDevice =E4=BB=8D=E9= =9C=80=E4=BF=9D=E7=95=99=E4=B8=AD=E6=96=AD=E8=83=BD=E5=8A=9B=EF=BC=88CQ NEE= D_WAKEUP =E5=94=A4=E9=86=92=E3=80=81config change=E3=80=81fallback=EF=BC=89= =E3=80=82 + +7.5 =E5=86=85=E5=AD=98=E5=BA=8F +~~~~~~~~~~ + +QEMU =E7=94=A8=E6=88=B7=E6=80=81=E9=9C=80=E4=BD=BF=E7=94=A8 ``atomic_store= _release`` / ``atomic_load_acquire`` =E6=88=96 C11 ``stdatomic.h``=E3=80=82 + + +8. Host Kernel vhost =E7=AB=AF=E5=AE=9E=E7=8E=B0 +---------------------------- + +8.1 vhost_virtqueue =E6=89=A9=E5=B1=95 +~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. code-block:: c + + struct vhost_virtqueue { + /* ... =E7=8E=B0=E6=9C=89=E5=AD=97=E6=AE=B5 ... */ + /* SQ/CQ polling mode */ + bool sqcq_poll; + struct vring_sq __user *sq; + struct vring_cq __user *cq; + struct task_struct *poll_task; + bool poll_stop; + wait_queue_head_t poll_wait; + struct completion poll_started, poll_stopped; + bool kick_received; + atomic_t completion_pending; + void (*poll_complete)(struct vhost_virtqueue *vq); + wait_queue_head_t *kick_wqh; + wait_queue_entry_t kick_wait; + poll_table kick_pt; + /* 10s =E7=BB=9F=E8=AE=A1=E7=AA=97=E5=8F=A3=E5=92=8C EMA */ + u64 ema_latency_ns; /* EMA of cmd processing latency */ + u64 ema_interval_ns; /* EMA of cmd-to-cmd interval */ + /* ... lat_*/stat_* =E7=BB=9F=E8=AE=A1=E5=AD=97=E6=AE=B5 ... */ + }; + +8.2 ioctl =E5=A4=84=E7=90=86 +~~~~~~~~~~~~~~ + +``vhost_vring_set_sqcq_addr()``=EF=BC=9Acopy_from_user =E2=86=92 =E9=AA=8C= =E8=AF=81=E5=9C=B0=E5=9D=80/=E5=AF=B9=E9=BD=90 =E2=86=92 access_ok=EF=BC=88= backend =E5=B7=B2=E9=85=8D=E7=BD=AE=E6=97=B6=EF=BC=89=E2=86=92 =E5=AD=98=E5= =82=A8 sq/cq=E3=80=82 +=E5=8F=A6=E5=90=AB=E8=87=AA=E5=8A=A8=E5=90=AF=E5=8A=A8=E9=80=BB=E8=BE=91= =EF=BC=9A=E5=BD=93 private_data && sq && cq && !sqcq_poll && poll_complete = && feature =E6=97=B6=E8=B0=83 ``vhost_sqcq_poll_start()``=E3=80=82 + +8.3 =E7=8B=AC=E7=AB=8B=E8=BD=AE=E8=AF=A2=E7=BA=BF=E7=A8=8B +~~~~~~~~~~~~~~~~~ + +kthread ``vhost-sqcq--``=EF=BC=8C=E9=80=9A=E8=BF=87 ``kthr= ead_use_mm(dev->mm)`` =E7=BB=91=E5=AE=9A QEMU =E5=9C=B0=E5=9D=80=E7=A9=BA= =E9=97=B4=E3=80=82 +=E7=9B=B4=E6=8E=A5=E8=B0=83=E7=94=A8 ``handle_kick`` =E5=A4=84=E7=90=86=E6= =8F=90=E4=BA=A4=EF=BC=8C``poll_complete`` =E5=A4=84=E7=90=86=E5=AE=8C=E6=88= =90=EF=BC=8C=E5=8D=95=E7=BA=BF=E7=A8=8B=E9=97=AD=E7=8E=AF=E3=80=82 + +API=EF=BC=9A``vhost_sqcq_poll_start(vq)`` / ``vhost_sqcq_poll_stop(vq)``= =E3=80=82 +start =E6=97=B6=E5=85=88=E5=88=9B=E5=BB=BA kthread =E5=B9=B6=E7=AD=89=E5= =BE=85 ``poll_started`` completion=EF=BC=8C=E7=84=B6=E5=90=8E=E6=89=8D=E8= =AE=BE=E7=BD=AE ``sqcq_poll =3D true``=EF=BC=8C +=E7=A1=AE=E4=BF=9D ``vhost_signal()`` =E7=9C=8B=E5=88=B0 sqcq_poll =E6=97= =B6 ``poll_task`` =E5=B7=B2=E6=9C=89=E6=95=88=EF=BC=88=E6=B6=88=E9=99=A4 NU= LL deref =E7=AA=97=E5=8F=A3=EF=BC=89=E3=80=82 +stop =E5=90=8E=E6=B8=85=E9=99=A4 poll_task=E3=80=81sqcq_poll=E3=80=81sq=E3= =80=81cq=EF=BC=8Cflush =E6=AE=8B=E7=95=99 completion=E3=80=82 + +8.4 =E7=AD=96=E7=95=A5=E5=B8=B8=E9=87=8F=E4=B8=8E CPU =E7=BB=91=E6=A0=B8 +~~~~~~~~~~~~~~~~~~~~~~~ + +=E7=AD=96=E7=95=A5=E5=B8=B8=E9=87=8F=EF=BC=88=E7=BC=96=E8=AF=91=E6=9C=9F= =E5=9B=BA=E5=AE=9A=EF=BC=8C``#define``=EF=BC=89=EF=BC=9A + +.. code-block:: c + + #define SQCQ_IDLE_TIMEOUT_US_MAX 1000U /* sleep =E6=97=B6=E9=95= =BF=E4=B8=8A=E7=95=8C (us) */ + #define SQCQ_SLEEP_EMA_MULT 2 /* sleep =E6=97=B6=E9=95= =BF =3D N * ema_latency */ + #define SQCQ_SLEEP_THRESHOLD_US 20U /* =E5=BF=AB=E4=BA=8E=E6= =AD=A4=E6=97=B6=E9=95=BF=E7=9A=84=E5=90=8E=E7=AB=AF=E9=80=89=E6=8B=A9 spin = */ + #define SQCQ_SPIN_EMA_MULT 2 /* inflight spin =E7=AA=97= =E5=8F=A3 =3D N * ema_latency */ + #define SQCQ_ACTIVE_TIMEOUT_NS 1000000U /* =E6=97=A0 inflight = =E5=9B=BA=E5=AE=9A=E8=87=AA=E6=97=8B=E7=AA=97=E5=8F=A3 1ms */ + #define SQCQ_STALE_TIMEOUT_NS 1000000000ULL /* sq_was_active =E5= =85=9C=E5=BA=95=E6=B8=85=E9=99=A4 1s */ + +**CPU =E7=BB=91=E6=A0=B8**=EF=BC=9Around-robin=EF=BC=88``sqcq_poll_cpu_rr`= ` =E5=85=A8=E5=B1=80=E5=8E=9F=E5=AD=90=E8=AE=A1=E6=95=B0=E5=99=A8 ``% num_o= nline_cpus()``=EF=BC=89=EF=BC=8C +=E5=88=86=E6=95=A3=E5=88=B0=E4=B8=8D=E5=90=8C=E5=9C=A8=E7=BA=BF CPU=E3=80= =82=E8=AE=A1=E6=95=B0=E5=99=A8=E5=8F=96=E6=A8=A1=E7=A1=AE=E4=BF=9D=E9=87=8D= =E5=90=AF=E5=90=8E=E4=B8=8D=E4=BC=9A=E6=BA=A2=E5=87=BA=E5=88=B0 fallback=EF= =BC=88=E5=85=A8=E7=BB=91 cpu0=EF=BC=89=E3=80=82 + +**=E5=8D=8F=E4=BD=9C=E5=BC=8F=E8=87=AA=E6=97=8B**=EF=BC=88io_uring SQPOLL = =E6=A8=A1=E5=9E=8B=EF=BC=89=EF=BC=9A=E4=B8=A4=E6=9D=A1=E8=87=AA=E6=97=8B=E8= =B7=AF=E5=BE=84=E5=9D=87=E4=BD=BF=E7=94=A8 +``need_resched()`` / ``cond_resched()`` =E5=8D=8F=E4=BD=9C=E8=AE=A9=E5=87= =BA=E2=80=94=E2=80=94=E8=87=AA=E6=97=8B=E6=97=B6=E8=8B=A5=E8=B0=83=E5=BA=A6= =E5=99=A8=E6=A0=87=E8=AE=B0=E9=9C=80=E8=A6=81=E8=AE=A9=E5=87=BA=EF=BC=8C +=E4=B8=BB=E5=8A=A8 ``cond_resched()``=EF=BC=8C=E9=81=BF=E5=85=8D=E9=A5=BF= =E6=AD=BB=E5=90=8C=E6=A0=B8=E5=85=B6=E4=BB=96=E4=BB=BB=E5=8A=A1=E3=80=82=E7= =BA=BF=E7=A8=8B=E4=BF=9D=E6=8C=81 TASK_RUNNING=EF=BC=88=E8=BD=AE=E8=AF=A2 +``sq->idx``=EF=BC=89=EF=BC=8C=E4=B8=8D=E8=BF=9B=E5=85=A5 sleep =E8=B7=AF= =E5=BE=84=E3=80=81=E4=B8=8D=E8=AE=BE ``sq_need_wakeup``=EF=BC=8C=E4=BB=8E= =E8=80=8C=E6=B6=88=E9=99=A4 Guest kick +=EF=BC=88VM exit=EF=BC=89=E3=80=82=E4=BB=85=E5=9C=A8=E6=97=A0 inflight =E4= =B8=94=E8=B6=85=E8=BF=87 1ms =E6=97=A0=E6=96=B0=E6=8F=90=E4=BA=A4=E6=97=B6= =E6=89=8D=E7=9C=9F=E6=AD=A3 sleep=E3=80=82 + +**TCM =E5=AE=8C=E6=88=90 CPU =E5=AF=BC=E5=90=91**=EF=BC=9A``sqcq_poll_acti= ve_cpu_mask`` =E8=AE=B0=E5=BD=95=E5=BF=99 poll =E7=BA=BF=E7=A8=8B=E5=8D=A0= =E7=94=A8=E7=9A=84 CPU +=EF=BC=88=E8=B7=9F=E7=9D=80 ``sq_was_active`` =E7=BF=BB=E8=BD=AC=EF=BC=89= =E3=80=82``vhost_sqcq_pick_completion_cpu()`` =E4=BB=8E=E7=A9=BA=E9=97=B2 C= PU +=E4=B8=AD=E9=80=89=E4=B8=80=E4=B8=AA=E7=BB=99 TCM =E5=AE=8C=E6=88=90=EF=BC= =8C=E9=81=BF=E5=BC=80=E5=BF=99 poll =E6=A0=B8=EF=BC=9B=E5=85=A8=E5=8D=A0=E6= =BB=A1=E6=97=B6=E5=9B=9E=E9=80=80=E5=88=B0 next-CPU=E3=80=82 + +8.5 vhost_signal / notify +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +- ``vhost_signal()``=EF=BC=9Apoll =E6=A8=A1=E5=BC=8F=E4=B8=8B=E8=AE=BE com= pletion_pending + =E6=9D=A1=E4=BB=B6 wake_up=E3=80=82SCSI completion =E4=B8= =8D=E7=BB=8F=E8=BF=87=E6=AD=A4=E8=B7=AF=E5=BE=84 +- ``vhost_enable_notify()``=EF=BC=9Apoll =E6=A8=A1=E5=BC=8F=E4=B8=8B retur= n false +- ``vhost_disable_notify()``=EF=BC=9Apoll =E6=A8=A1=E5=BC=8F=E4=B8=8B retu= rn +- ``vhost_dev_stop()``=EF=BC=9A=E4=BB=85=E5=AF=B9 sqcq_poll vq =E5=8A=A0= =E9=94=81=E5=81=9C=E6=AD=A2 + +8.6 vhost-scsi =E9=9B=86=E6=88=90 +~~~~~~~~~~~~~~~~~~~ + +``set_endpoint`` =E4=B8=AD=E5=AF=B9 I/O VQs=EF=BC=88index >=3D VHOST_SCSI_= VQ_IO=EF=BC=89=EF=BC=9A=E5=A6=82=E6=9E=9C feature + sq + cq =E5=88=99 start= =E3=80=82 +``clear_endpoint`` =E4=B8=AD=E5=85=88 stop =E5=86=8D clear backend=E3=80= =82 + +8.7 call_ctx.ctx NULL =E6=A3=80=E6=9F=A5 +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``poll_complete`` =E4=B8=AD ``eventfd_signal(call_ctx.ctx)`` =E5=89=8D=E5= =81=9A NULL =E6=A3=80=E6=9F=A5=E3=80=82 +QEMU =E5=BF=85=E9=A1=BB=E5=9C=A8 SET_ENDPOINT =E4=B9=8B=E5=89=8D=E5=AE=8C= =E6=88=90 SET_VRING_CALL =E8=AE=BE=E7=BD=AE=E3=80=82 + +8.8 =E7=A9=BA=E9=97=B2=E7=AD=96=E7=95=A5=E4=B8=8E EMA +~~~~~~~~~~~~~~~~~~ + +=E7=B3=BB=E7=BB=9F=E7=BB=B4=E6=8A=A4=E4=B8=A4=E4=B8=AA=E7=8B=AC=E7=AB=8B= =E7=9A=84 EMA=EF=BC=9A +- ``ema_latency_ns``=EF=BC=9Acmd =E5=A4=84=E7=90=86=E5=BB=B6=E8=BF=9F=EF= =BC=88submit =E2=86=92 process_completions=EF=BC=89=E7=9A=84 EMA=EF=BC=8C= =E6=AF=8F=E6=AC=A1 ``poll_complete`` =E5=90=8E=E6=9B=B4=E6=96=B0 ``old - ol= d/8 + avg/8``=EF=BC=88=E6=BA=A2=E5=87=BA=E5=AE=89=E5=85=A8=EF=BC=89 +- ``ema_interval_ns``=EF=BC=9Acmd-to-cmd =E9=97=B4=E9=9A=94=E7=9A=84 EMA= =EF=BC=8C=E6=AF=8F=E6=AC=A1=E6=A3=80=E6=B5=8B=E5=88=B0 sq->idx =E5=8F=98=E5= =8C=96=E6=97=B6=E6=9B=B4=E6=96=B0 ``old - old/8 + new/8`` + +**=E6=97=A0 inflight =E8=87=AA=E6=97=8B**=EF=BC=88io_uring SQPOLL =E6=A8= =A1=E5=9E=8B=EF=BC=89=EF=BC=9A=E5=9B=BA=E5=AE=9A ``SQCQ_ACTIVE_TIMEOUT_NS``= =EF=BC=881ms=EF=BC=89 +=E7=AA=97=E5=8F=A3 + ``cond_resched()`` =E5=8D=8F=E4=BD=9C=E8=AE=A9=E5=87= =BA=E3=80=82=E7=BA=BF=E7=A8=8B=E6=8C=81=E7=BB=AD=E8=BD=AE=E8=AF=A2 ``sq->id= x`` =E6=8E=A5=E4=BD=8F=E4=B8=8B=E4=B8=80=E6=9D=A1=E6=8F=90=E4=BA=A4=EF=BC= =8C +=E4=B8=8D=E8=BF=9B=E5=85=A5 sleep =E8=B7=AF=E5=BE=84=E3=80=81=E4=B8=8D=E8= =AE=BE ``sq_need_wakeup`` =E2=86=92 =E6=B6=88=E9=99=A4 Guest kick=EF=BC=88V= M exit=EF=BC=89=E3=80=82 +=E4=BB=85=E5=9C=A8=E8=B6=85=E8=BF=87 1ms =E6=97=A0=E6=96=B0=E6=8F=90=E4=BA= =A4=E6=97=B6=E6=89=8D sleep=E3=80=82 + +**=E6=9C=89 inflight =E8=87=AA=E6=97=8B**=EF=BC=9A``spin_ns =3D SQCQ_SPIN_= EMA_MULT * ema_latency_ns``=EF=BC=8C +``cond_resched()`` =E5=8D=8F=E4=BD=9C=E8=AE=A9=E5=87=BA=E3=80=82=E8=B6=85= =E5=87=BA=E7=AA=97=E5=8F=A3=E6=88=96=E5=A4=9A VQ + =E9=AB=98=E5=BB=B6=E8=BF= =9F=E6=97=B6 sleep=E3=80=82 + +Sleep =E5=8A=A8=E6=80=81=E8=B6=85=E6=97=B6=EF=BC=9A``timeout =3D ema_laten= cy / (1000 / SQCQ_SLEEP_EMA_MULT)``=EF=BC=8Cclamp =E5=88=B0 [10us, SQCQ_IDL= E_TIMEOUT_US_MAX]=E3=80=82 +=E5=A4=9A VQ =E4=BC=91=E7=9C=A0=E5=86=B3=E7=AD=96=EF=BC=9A=E6=9C=89 inflig= ht =E6=97=B6=E4=BB=85=E5=9C=A8 ``sqcq_active_io_vqs > 1`` =E4=B8=94 ``ema_l= atency_us > SQCQ_SLEEP_THRESHOLD_US`` =E6=97=B6=E4=BC=91=E7=9C=A0=E3=80=82 + +8.9 =E7=BB=9F=E8=AE=A1=E4=B8=8E=E8=AF=8A=E6=96=AD +~~~~~~~~~~~~~~ + +10s =E7=BB=9F=E8=AE=A1=E7=AA=97=E5=8F=A3=EF=BC=8C=E6=89=93=E5=8D=B0 cq/fai= l/sq_hits/enters/wkick/wcomp/avg/kick_pct/wait_pct/cqw_pct/spin_pct/wake_av= g/wake_n/interval/ema_lat/inflight =E5=88=B0 dmesg=E3=80=82 +Stuck =E6=A3=80=E6=B5=8B=EF=BC=9Await_event =E8=B6=85=E6=97=B6=E6=88=96=E7= =AA=97=E5=8F=A3=E5=86=85=E6=97=A0=E5=AE=8C=E6=88=90=E4=BD=86=E6=9C=89 infli= ght =E6=97=B6=E5=91=8A=E8=AD=A6=E3=80=82 + +9. Guest Kernel =E7=AB=AF=E5=AE=9E=E7=8E=B0 +----------------------- + +9.1 SQ/CQ DMA =E5=88=86=E9=85=8D +~~~~~~~~~~~~~~~~~~ + +``virtio_alloc_sqcq()``=EF=BC=9A``dma_alloc_coherent()`` =E4=B8=80=E6=AC= =A1=E5=88=86=E9=85=8D 256 bytes=EF=BC=88sq+cq =E5=90=84 128 =E5=AD=97=E8=8A= =82=EF=BC=89=EF=BC=8C +=E5=9C=A8 ``vring_create_virtqueue_split()`` =E4=B8=AD=E5=88=86=E9=85=8D= =EF=BC=8C``vring_free`` =E6=97=B6=E9=87=8A=E6=94=BE=E3=80=82 + +9.2 =E5=9C=B0=E5=9D=80=E4=BC=A0=E9=80=92 +~~~~~~~~~~~~ + +``vp_modern_queue_address_sqcq()``=EF=BC=9A=E5=9C=A8 ``vp_active_vq()`` = =E4=B8=AD=EF=BC=8C=E9=80=9A=E8=BF=87 PCI common config +=E7=9A=84 queue_select + sqe/cqe_ring_lo/hi =E5=86=99=E5=85=A5 64 =E4=BD= =8D DMA =E5=9C=B0=E5=9D=80=E3=80=82 + +9.3 =E6=8F=90=E4=BA=A4=E8=B7=AF=E5=BE=84 +~~~~~~~~~~~~ + +``virtqueue_kick()``=EF=BC=9A``smp_store_release(&sq->idx, shadow)`` =E2= =86=92 =E5=A6=82=E6=9E=9C NEED_WAKEUP =E5=88=99 MMIO kick=E3=80=82 +``virtqueue_kick_prepare()``=EF=BC=9Asqqcq_poll =E4=B8=8B=E7=9B=B4=E6=8E= =A5 return true=E3=80=82 + +9.4 CQ =E8=BD=AE=E8=AF=A2=E7=BA=BF=E7=A8=8B +~~~~~~~~~~~~~~~~~ + +=E7=8B=AC=E7=AB=8B kthread ``virtio-sqcq/``=EF=BC=8C=E6=AF=8F= device =E4=B8=80=E4=B8=AA=E7=BA=BF=E7=A8=8B=EF=BC=88``struct virtio_sqcq_p= oll_data``=EF=BC=89=E3=80=82 + +**CQ =E8=AE=BF=E9=97=AE=E5=99=A8=E5=87=BD=E6=95=B0** (virtio_ring.c)=EF=BC= =9A + +- ``cq_poll_has_work(vq)``=EF=BC=9A``smp_load_acquire(&cq->idx) !=3D last_= cq_head`` +- ``cq_poll_consume(vq)``=EF=BC=9Asnapshot cq->idx=EF=BC=88callback =E5=89= =8D=EF=BC=89 +- ``cq_set_need_wakeup(vq)``=EF=BC=9A``smp_store_release(&cq->cq_need_wake= up, 1)`` +- ``cq_clear_need_wakeup(vq)``=EF=BC=9A``WRITE_ONCE(cq->cq_need_wakeup, 0)= `` +- ``cq_recheck(vq)``=EF=BC=9A``smp_mb()`` + ``smp_load_acquire(&cq->idx) != =3D last_cq_head`` + +**=E7=BA=BF=E7=A8=8B=E7=94=9F=E5=91=BD=E5=91=A8=E6=9C=9F=EF=BC=9A** ``vp_m= odern_find_vqs()`` =E4=B8=AD=E5=90=AF=E5=8A=A8=EF=BC=88``virtio_start_sqcq_= poll(vdev, 1000, 1000)``=EF=BC=89=EF=BC=8C +=E6=B3=A8=E5=86=8C=E6=89=80=E6=9C=89 poll =E6=B4=BB=E8=B7=83=E7=9A=84 vq= =E3=80=82``vp_del_vqs()`` =E4=B8=AD=E5=81=9C=E6=AD=A2=EF=BC=88``virtio_stop= _sqcq_poll`` =E5=86=85=E9=83=A8=E9=87=8A=E6=94=BE=E6=89=80=E6=9C=89 vq node= =EF=BC=89=E3=80=82 + +**=E5=90=AF=E5=8A=A8=E5=8F=82=E6=95=B0=EF=BC=9A** ``idle_timeout_us=3D1000= ``=EF=BC=881ms sleep =E8=B6=85=E6=97=B6=EF=BC=89=EF=BC=8C``busy_spin_us=3D1= 000``=EF=BC=881ms spin =E9=A2=84=E7=AE=97=EF=BC=89=E3=80=82 +=E4=B8=8D=E7=BB=91=E6=A0=B8=EF=BC=88=E8=B0=83=E5=BA=A6=E5=99=A8=E8=87=AA= =E7=94=B1=E5=88=86=E9=85=8D=EF=BC=89=EF=BC=8C=E6=97=A0=E6=8C=87=E6=95=B0=E9= =80=80=E9=81=BF=EF=BC=88=E5=9B=BA=E5=AE=9A 1ms sleep=EF=BC=89=E3=80=82 + +**config =E7=A9=BA=E9=97=B4=E5=A4=A7=E5=B0=8F=E6=A3=80=E6=9F=A5=EF=BC=9A**= ``vp_check_common_size()`` =E9=AA=8C=E8=AF=81 common config =E8=87=B3=E5= =B0=91 80 =E5=AD=97=E8=8A=82=E3=80=82 + + +10. =E4=B8=8E=E7=8E=B0=E6=9C=89=E6=9C=BA=E5=88=B6=E7=9A=84=E5=85=B3=E7=B3= =BB +--------------------- + +- **EVENT_IDX=EF=BC=9A** =E5=8F=AF=E5=8D=8F=E5=95=86=E4=BD=86=E5=AF=B9 pol= ling vq =E6=97=A0=E6=95=88=EF=BC=8C=E8=A2=AB sq/cq_need_wakeup =E6=9B=BF=E4= =BB=A3 +- **virtio-net NAPI=EF=BC=9A** =E5=8F=AF=E5=85=B1=E5=AD=98=EF=BC=8CNAPI = =E5=8F=AF=E7=9B=B4=E6=8E=A5=E6=A3=80=E6=9F=A5 CQ +- **blk-mq iopoll=EF=BC=9A** =E4=BA=92=E8=A1=A5=EF=BC=88iopoll =E5=90=8C= =E6=AD=A5=E5=BF=AB=E8=B7=AF=E5=BE=84=EF=BC=8CSQ/CQ =E6=9B=BF=E4=BB=A3=E5=BC= =82=E6=AD=A5=E4=B8=AD=E6=96=AD=E8=B7=AF=E5=BE=84=EF=BC=89 +- **Live Migration=EF=BC=9A** reset =E6=97=B6=E5=9B=9E=E9=80=80=E4=B8=AD= =E6=96=AD=E6=A8=A1=E5=BC=8F=EF=BC=8C=E8=BF=81=E7=A7=BB=E5=90=8E Guest =E9= =87=8D=E6=96=B0=E5=8D=8F=E5=95=86 + + +11. =E6=96=87=E4=BB=B6=E6=B8=85=E5=8D=95 +------------ + +**Guest Kernel=EF=BC=9A** ``virtio_config.h`` (feature bit), ``virtio_ring= .h`` (vring_sq/cq), +``virtio_pci.h`` (PCI offsets), ``virtio_ring.c`` (=E5=88=86=E9=85=8D/SQ= =E6=8F=90=E4=BA=A4/CQ accessor), +``virtio_sqcq_poll.c`` (CQ =E8=BD=AE=E8=AF=A2=E7=BA=BF=E7=A8=8B), ``virtio= _pci_modern.c`` (feature/queue=E6=BF=80=E6=B4=BB/=E7=BA=BF=E7=A8=8B=E7=94= =9F=E5=91=BD=E5=91=A8=E6=9C=9F), +``virtio_pci_common.c`` (vp_del_vqs), ``virtio_pci_modern_dev.c`` (PCI con= fig =E6=98=A0=E5=B0=84) + +**Host Kernel vhost=EF=BC=9A** ``vhost.h`` (ioctl), ``vhost_types.h`` (sqc= q_addr struct), +``vhost.h/vhost.c`` (=E8=BD=AE=E8=AF=A2=E7=BA=BF=E7=A8=8B/ioctl/signal/=E7= =94=9F=E5=91=BD=E5=91=A8=E6=9C=9F), ``scsi.c`` (vhost-scsi =E9=9B=86=E6=88= =90) + +**QEMU=EF=BC=9A** ``virtio-pci.c`` (feature/config), ``virtio-pci-modern.c= `` (BAR/GPA=E2=86=92HVA) + + +12. =E9=AA=8C=E8=AF=81=E6=96=B9=E6=A1=88 +------------ + +**=E5=8A=9F=E8=83=BD=EF=BC=9A** QEMU =E5=B8=A6 VIRTIO_F_SQCQ_POLL + virtio= -scsi =E2=86=92 Guest =E8=AF=BB=E5=86=99=E9=AA=8C=E8=AF=81=E6=AD=A3=E7=A1= =AE=E6=80=A7=E3=80=82 + +**=E6=80=A7=E8=83=BD=EF=BC=9A** fio =E5=AF=B9=E6=AF=94=E4=B8=AD=E6=96=AD= =E6=A8=A1=E5=BC=8F vs polling=EF=BC=8C=E8=A7=82=E5=AF=9F IOPS/=E5=BB=B6=E8= =BF=9F/VM exit =E6=AC=A1=E6=95=B0=E3=80=82 + +**=E6=AD=A3=E7=A1=AE=E6=80=A7=EF=BC=9A** 24h+ stress test=EF=BC=9Bidle/wak= eup =E9=AA=8C=E8=AF=81=EF=BC=9BNEED_WAKEUP =E8=BE=B9=E7=95=8C=E6=9D=A1=E4= =BB=B6=EF=BC=9BARM/RISC-V =E5=86=85=E5=AD=98=E5=BA=8F=E9=AA=8C=E8=AF=81=E3= =80=82 diff --git a/Documentation/virtio-sqcq-poll-virtio-spec.rst b/Documentation= /virtio-sqcq-poll-virtio-spec.rst new file mode 100644 index 0000000000000..26ae41860a8e5 --- /dev/null +++ b/Documentation/virtio-sqcq-poll-virtio-spec.rst @@ -0,0 +1,355 @@ +SQ/CQ Doorbell Polling +=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D + +This section describes the SQ/CQ doorbell polling mechanism +(``VIRTIO_F_SQCQ_POLL``), which allows the driver and device to use +shared-memory doorbells instead of MMIO writes (notifications) and +interrupts, eliminating VM exits on both submission and completion paths. + +This feature is only supported with split virtqueues. A driver or +device that negotiates this feature MUST NOT also negotiate +``VIRTIO_F_RING_PACKED``. + + +Feature bit +----------- + +``VIRTIO_F_SQCQ_POLL (42)`` =E2=80=94 SQ/CQ Doorbell Polling. + +This feature is in the transport feature range (28..43). Both +``virtio_ring`` and ``virtio_pci`` MUST accept this feature for it +to be negotiated. + + +Data Structures +--------------- + +SQ Doorbell +~~~~~~~~~~~ + +The SQ (Submission Queue) doorbell is a cache-line aligned structure +written by the driver and read by the device: + +.. code-block:: c + + struct vring_sq { + le64 idx; + u8 sq_need_wakeup; + u8 reserved[55]; + }; + +The structure MUST be aligned to the cache line size of the platform +(minimum 64 bytes; 128 bytes recommended for platforms with 128-byte +cache lines such as ARM Neoverse N2/V2). + +``idx`` corresponds to ``avail->idx`` and is written by the driver +using ``smp_store_release()``. + +``sq_need_wakeup`` is a single-byte flag written by the device, +indicating whether the device's poll thread is sleeping and needs to +be woken by an MMIO write (kick). + +CQ Doorbell +~~~~~~~~~~~ + +The CQ (Completion Queue) doorbell is a cache-line aligned structure +written by the device and read by the driver: + +.. code-block:: c + + struct vring_cq { + le64 idx; + u8 cq_need_wakeup; + u8 reserved[55]; + }; + +The structure MUST be aligned to the same alignment as ``vring_sq``. + +``idx`` corresponds to ``used->idx`` and is written by the device. + +``cq_need_wakeup`` is a single-byte flag written by the driver, +indicating whether the driver's poll thread is sleeping and needs to +be woken by an interrupt. + + +PCI Configuration +----------------- + +The SQ and CQ doorbell DMA addresses are passed through the PCI +common configuration space. Four new registers are appended to +``struct virtio_pci_modern_common_cfg``: + +=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D =3D=3D=3D=3D=3D=3D=3D=3D =3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D +Offset Field Description +=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D =3D=3D=3D=3D=3D=3D=3D=3D =3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D +64 sqe_ring_lo SQ doorbell DMA address, low 32 bits +68 sqe_ring_hi SQ doorbell DMA address, high 32 bits +72 cqe_ring_lo CQ doorbell DMA address, low 32 bits +76 cqe_ring_hi CQ doorbell DMA address, high 32 bits +=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D =3D=3D=3D=3D=3D=3D=3D=3D =3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D + +These registers are scoped by ``queue_select``. The driver writes +the SQ and CQ doorbell DMA addresses for each virtqueue during +``vp_active_vq``. + +The device (QEMU) translates the guest physical address to a host +virtual address and passes it to the vhost backend via +``VHOST_SET_VRING_SQCQ_ADDR`` ioctl. + + +Allocation +---------- + +The driver allocates SQ and CQ doorbells as a single +``dma_alloc_coherent()`` call (contiguous physical memory): + +.. code-block:: c + + size_t total =3D sizeof(struct vring_sq) + sizeof(struct vring_cq); + void *buf =3D dma_alloc_coherent(dev, total, &dma_addr, GFP_KERNEL); + vq->sq =3D buf; + vq->cq =3D buf + sizeof(struct vring_sq); + +Control and event virtqueues MUST NOT use SQ/CQ polling. Only request +virtqueues (index >=3D VIRTIO_SCSI_VQ_BASE for virtio-scsi) negotiate +this feature. + + +Submission Path (Driver =E2=86=92 Device) +--------------------------------- + +When the driver has new requests available: + +1. The driver writes descriptors to the avail ring as usual +2. The driver updates ``sq->idx`` using ``smp_store_release()`` +3. The driver reads ``sq_need_wakeup`` using ``smp_load_acquire()`` + +If ``sq_need_wakeup`` is 0, the device's poll thread is actively +spinning and will detect the new ``sq->idx`` value without any +further action. + +If ``sq_need_wakeup`` is 1, the device's poll thread is sleeping. +The driver MUST perform a traditional MMIO write (kick) to wake it. + +.. code-block:: pseudo + + smp_store_release(&sq->idx, avail_idx_shadow) + need_wakeup =3D smp_load_acquire(&sq->sq_need_wakeup) + if (need_wakeup): + mmio_kick(vq) + +The ``virtqueue_kick_prepare()`` function MUST return ``true`` +unconditionally when SQ/CQ polling is active, bypassing the +``used->flags`` check. + + +Completion Path (Device =E2=86=92 Driver) +--------------------------------- + +Device Side +~~~~~~~~~~~ + +When the device processes completions: + +1. The device writes completion entries to the used ring as usual +2. The device's poll thread writes ``cq->idx`` to signal completion + +The device's poll thread is a dedicated kernel thread that processes +both submissions (via ``handle_kick``) and completions (via +``poll_complete`` callback) in a single-threaded loop. + +Driver Side +~~~~~~~~~~~ + +The driver's poll thread (one per device) detects completions by +checking ``more_used()`` =E2=80=94 directly comparing ``used->idx`` with +``last_used_idx``: + +.. code-block:: pseudo + + while more_used(vq): + consume_and_callback(vq) + do_softirq() + +Note: The driver polls ``used->idx`` directly rather than ``cq->idx``, +because ``cq->idx`` may lag behind ``used->idx`` (the device writes +the used ring entries before updating ``cq->idx``). Polling +``used->idx`` ensures the lowest-latency completion detection. + +``cq->idx`` is retained for the ``cq_need_wakeup`` sleep/wake +protocol (see below) but is not polled as a fast-path completion +detector. + + +NEED_WAKEUP Protocol +-------------------- + +The NEED_WAKEUP protocol prevents lost wakeups when either side +transitions from polling to sleeping. It is symmetric: SQ uses +``sq_need_wakeup`` and CQ uses ``cq_need_wakeup``. + +SQ NEED_WAKEUP (Device sleeping) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Before the device's poll thread sleeps: + +1. Set ``sq_need_wakeup =3D 1`` (via ``put_user``) +2. Execute ``smp_mb()`` +3. Re-read ``sq->idx`` =E2=80=94 if it changed, cancel sleep + +After waking (via eventfd or timeout): +- Clear ``sq_need_wakeup =3D 0`` + +The driver checks ``sq_need_wakeup`` after writing ``sq->idx`` and +performs an MMIO kick only if the flag is set. + +CQ NEED_WAKEUP (Driver sleeping) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Before the driver's poll thread sleeps: + +1. Set ``cq_need_wakeup =3D 1`` (via ``smp_store_release``) +2. Execute ``smp_mb()`` +3. Re-check ``more_used()`` =E2=80=94 if work found, cancel sleep + +After waking (via eventfd/interrupt or timeout): +- Clear ``cq_need_wakeup =3D 0`` + +The device checks ``cq_need_wakeup`` after writing ``cq->idx`` and +sends an eventfd signal only if the flag is set. + + +Memory Ordering +--------------- + +The following memory ordering rules MUST be observed: + +=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D =3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D = =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D +Field Writer Reader +=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D =3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D = =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D +``sq->idx`` ``smp_store_release`` ``get_user`` + ``smp_r= mb`` +``sq_need_wakeup`` ``put_user`` + ``smp_wmb`` ``smp_load_acquire`` +``cq->idx`` ``put_user`` (with ``smp_wmb``) Not polled (see note) +``cq_need_wakeup`` ``smp_store_release`` ``get_user`` + ``smp_r= mb`` +``used->idx`` Device (via ``vhost_add_used``) ``more_used()`` (pla= in read) +=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D =3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D = =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D + +Note: ``cq->idx`` is written by the device but is not polled by the +driver's fast path. The driver uses ``more_used()`` (checking +``used->idx``) for completion detection, which is the authoritative +source of completions. + +On x86 (TSO memory model), all barriers are no-ops. On ARM64 and +RISC-V, ``smp_store_release`` generates ``stlr`` and +``smp_load_acquire`` generates ``ldar``, which are lighter than full +memory barriers (``dmb ish``). + + +Poll Thread Behavior +-------------------- + +Device Poll Thread (vhost) +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +One kernel thread per virtqueue. Each thread: + +1. Polls ``sq->idx`` for new submissions and ``completion_pending`` + for completions +2. When active: busy-spins using ``need_resched()`` + ``cpu_relax()`` + for cooperative yielding (io_uring SQPOLL pattern) +3. Spin budget: fixed 1ms (``SQCQ_ACTIVE_TIMEOUT_NS``) for no-inflight, + or ``2 =C3=97 ema_latency`` for has-inflight +4. After spin budget expires: sets ``sq_need_wakeup``, sleeps via + ``wait_event_interruptible_timeout`` +5. Sleep timeout: adaptive, based on EMA latency, clamped to [10us, 1ms] + +CPU affinity: round-robin across online CPUs +(``atomic_counter % num_online_cpus()``). + +Driver Poll Thread +~~~~~~~~~~~~~~~~~~ + +One kernel thread per device (serves all registered virtqueues). The +thread: + +1. Iterates all registered virtqueues, polls ``more_used()`` for each +2. When active: busy-spins within a time budget (``busy_spin_us =3D 1ms``) + from ``last_io_ts``, using ``need_resched()`` + ``cpu_relax()`` +3. After spin budget expires: sets ``cq_need_wakeup`` on all VQs, + double-checks ``more_used()``, sleeps via ``schedule_timeout`` +4. Sleep timeout: fixed 1ms (``idle_timeout_us =3D 1000``) +5. ``do_softirq()`` called after each completion callback (no hardware + interrupts means softirqs must be explicitly flushed) + +Parameters (configurable at thread creation): + +- ``idle_timeout_us``: sleep timeout in microseconds (default: 1000) +- ``busy_spin_us``: spin budget in microseconds (default: 1000) + + +Feature Negotiation +------------------- + +1. The device offers ``VIRTIO_F_SQCQ_POLL`` in its feature bits +2. The driver accepts the feature during ``finalize_features`` +3. The driver allocates SQ/CQ doorbells in ``find_vqs`` +4. The driver writes SQ/CQ DMA addresses to PCI config space +5. The device (vhost) receives addresses and starts poll threads +6. ``virtio_features_ok()`` MUST reject + ``VIRTIO_F_SQCQ_POLL | VIRTIO_F_RING_PACKED`` + +If either side does not support the feature, traditional MMIO kick + +MSI-X interrupt mode is used with zero overhead. + + +Mutual Exclusion +---------------- + +``VIRTIO_F_SQCQ_POLL`` and ``VIRTIO_F_RING_PACKED`` MUST NOT be +negotiated simultaneously. The driver's ``virtio_features_ok()`` +function rejects this combination. + +This restriction exists because the SQ/CQ doorbell implementation +accesses split-ring-specific fields (``avail_idx_shadow``, +``vring->used->idx``) unconditionally in ``virtqueue_notify()``. +Packed ring support is a future enhancement. + + +Relationship to Existing Mechanisms +----------------------------------- + +- **EVENT_IDX** (``VIRTIO_RING_F_EVENT_IDX``): may be negotiated but + has no effect on polling virtqueues. Notification suppression is + handled by ``sq_need_wakeup`` / ``cq_need_wakeup`` instead. + +- **blk-mq iopoll**: complementary. iopoll handles synchronous + polling from the submitting task; SQ/CQ replaces the asynchronous + interrupt path. + +- **vhost-vDPA**: vDPA provides its own doorbell mmap mechanism. + SQ/CQ polling is specific to the vhost-scsi kernel backend and + does not require vDPA hardware support. + + +Performance Characteristics +--------------------------- + +Benchmark results (fio, 4K random I/O, arm64 with 8 vCPUs): + +=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D =3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D =3D=3D=3D=3D=3D=3D=3D=3D=3D +Test Baseline SQ/CQ Poll Change +=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D =3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D =3D=3D=3D=3D=3D=3D=3D=3D=3D +randread od1 nj1 22,427 IOPS 28,289 IOPS +26.1% +randread od32 nj1 89,910 IOPS 75,665 IOPS -15.8% +randread od32 nj4 186,763 IOPS 379,549 IOPS +103.2% +randread od32 nj8 199,967 IOPS 550,633 IOPS +175.4% +randwrite od1 nj1 21,912 IOPS 27,261 IOPS +24.4% +randwrite od32 nj1 85,349 IOPS 81,389 IOPS -4.6% +randwrite od32 nj4 190,443 IOPS 355,811 IOPS +86.8% +randwrite od32 nj8 196,552 IOPS 566,640 IOPS +188.3% +=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D =3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D =3D=3D=3D=3D=3D=3D=3D=3D=3D + +Multi-queue workloads (nj4/nj8) see 87-188% improvement from +eliminated VM exits on both submission and completion paths. +Single-queue high-queue-depth workloads (od32 nj1) show a +minor regression due to polling overhead vs. the VM exit savings. diff --git a/build-riscv-kernel.sh b/build-riscv-kernel.sh new file mode 100644 index 0000000000000..2ace46278369e --- /dev/null +++ b/build-riscv-kernel.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +# =E8=AE=BE=E7=BD=AE=E5=8F=98=E9=87=8F +export ARCH=3Driscv +export CROSS_COMPILE=3Driscv64-linux-gnu- +export KERNEL_DIR=3D$(pwd) +export OUTPUT_DIR=3D"${KERNEL_DIR}/output" + +# =E6=B8=85=E7=90=86 +make mrproper + +# =E9=85=8D=E7=BD=AE +make defconfig +# =E6=88=96=E4=BD=BF=E7=94=A8=E7=89=B9=E5=AE=9A=E9=85=8D=E7=BD=AE +# cp ${CONFIG_FILE} .config + +# =E7=BC=96=E8=AF=91=E5=86=85=E6=A0=B8 +echo "=E5=BC=80=E5=A7=8B=E7=BC=96=E8=AF=91=E5=86=85=E6=A0=B8..." +make -j$(nproc) || exit 1 + +# =E7=BC=96=E8=AF=91=E8=AE=BE=E5=A4=87=E6=A0=91 +echo "=E7=BC=96=E8=AF=91=E8=AE=BE=E5=A4=87=E6=A0=91..." +make dtbs || exit 1 + +echo "=E7=BC=96=E8=AF=91=E5=AE=8C=E6=88=90=EF=BC=81" +echo "=E5=86=85=E6=A0=B8=E9=95=9C=E5=83=8F: ${KERNEL_DIR}/arch/riscv/boot/= Image" diff --git a/drivers/vhost/scsi.c b/drivers/vhost/scsi.c index 1c22880e72267..4d65205fcb23e 100644 --- a/drivers/vhost/scsi.c +++ b/drivers/vhost/scsi.c @@ -142,6 +142,7 @@ struct vhost_scsi_cmd { struct llist_node tvc_completion_list; /* Used to track inflight cmd */ struct vhost_scsi_inflight *inflight; + ktime_t tvc_submit_ts; }; =20 struct vhost_scsi_nexus { @@ -234,6 +235,7 @@ struct vhost_scsi_virtqueue { =20 struct vhost_work completion_work; struct llist_head completion_list; + ktime_t stats_window; }; =20 struct vhost_scsi { @@ -490,9 +492,18 @@ static void vhost_scsi_release_cmd(struct se_cmd *se_c= md) struct vhost_scsi_virtqueue *svq =3D container_of(cmd->tvc_vq, struct vhost_scsi_virtqueue, vq); =20 - llist_add(&cmd->tvc_completion_list, &svq->completion_list); - if (!vhost_vq_work_queue(&svq->vq, &svq->completion_work)) - vhost_scsi_drop_cmds(svq); + llist_add(&cmd->tvc_completion_list, &svq->completion_list); + if (READ_ONCE(svq->vq.sqcq_poll)) { + smp_wmb(); + atomic_set(&svq->vq.completion_pending, 1); + smp_mb(); + if (!READ_ONCE(svq->vq.poll_stop) && + READ_ONCE(svq->vq.poll_task) && + !task_is_running(svq->vq.poll_task)) + wake_up(&svq->vq.poll_wait); + } else if (!vhost_vq_work_queue(&svq->vq, &svq->completion_work)) { + vhost_scsi_drop_cmds(svq); + } } } =20 @@ -678,15 +689,11 @@ static int vhost_scsi_copy_sgl_to_iov(struct vhost_sc= si_cmd *cmd) return 0; } =20 -/* Fill in status and signal that we are done processing this command - * - * This is scheduled in the vhost work queue so we are called with the own= er - * process mm and can access the vring. +/* Process completion_list: copy responses, add used entries, release cmds. + * Returns true if any completions were signalled. */ -static void vhost_scsi_complete_cmd_work(struct vhost_work *work) +static bool vhost_scsi_process_completions(struct vhost_scsi_virtqueue *sv= q) { - struct vhost_scsi_virtqueue *svq =3D container_of(work, - struct vhost_scsi_virtqueue, completion_work); struct virtio_scsi_cmd_resp v_rsp; struct vhost_scsi_cmd *cmd, *t; struct llist_node *llnode; @@ -726,6 +733,9 @@ static void vhost_scsi_complete_cmd_work(struct vhost_w= ork *work) signal =3D true; =20 vhost_add_used(cmd->tvc_vq, cmd->tvc_vq_desc, 0); + cmd->tvc_vq->lat_scsi_ns +=3D + ktime_to_ns(ktime_sub(ktime_get(), cmd->tvc_submit_ts)); + cmd->tvc_vq->stat_completions++; } else pr_err("Faulted on virtio_scsi_cmd_resp\n"); =20 @@ -737,10 +747,70 @@ static void vhost_scsi_complete_cmd_work(struct vhost= _work *work) =20 mutex_unlock(&svq->vq.mutex); =20 + return signal; +} + +/* Worker-path completion (non-poll mode only) */ +static void vhost_scsi_complete_cmd_work(struct vhost_work *work) +{ + struct vhost_scsi_virtqueue *svq =3D container_of(work, + struct vhost_scsi_virtqueue, completion_work); + struct vhost_virtqueue *vq =3D &svq->vq; + bool signal; + + signal =3D vhost_scsi_process_completions(svq); + if (signal) vhost_signal(&svq->vs->dev, &svq->vq); + + if (vq->stat_completions > 0) { + u64 now =3D ktime_get_ns(); + + if (!svq->stats_window) + svq->stats_window =3D ns_to_ktime(now); + + if (now - ktime_to_ns(svq->stats_window) >=3D 10 * NSEC_PER_SEC) { + int vi; + + for (vi =3D 0; vi < svq->vs->dev.nvqs; vi++) + if (svq->vs->dev.vqs[vi] =3D=3D vq) + break; + pr_info("vhost-baseline: vq[%d] cmd=3D%llu avg_lat=3D%lluns\n", + vi, vq->stat_completions, + div64_u64(vq->lat_scsi_ns, vq->stat_completions)); + svq->stats_window =3D ns_to_ktime(now); + vq->lat_scsi_ns =3D 0; + vq->stat_completions =3D 0; + } + } } =20 +/* Poll-thread-path completion: write cq->idx and signal Guest directly */ +static void vhost_scsi_poll_complete(struct vhost_virtqueue *vq) +{ + struct vhost_scsi_virtqueue *svq =3D container_of(vq, + struct vhost_scsi_virtqueue, vq); + bool signal; + + signal =3D vhost_scsi_process_completions(svq); + + if (signal && svq->vq.cq) { + __virtio64 cq_idx_val; + __u8 cq_need_wakeup; + + smp_mb(); + cq_idx_val =3D (__force __virtio64)READ_ONCE(svq->vq.last_used_idx); + put_user(cq_idx_val, &svq->vq.cq->idx); + smp_mb(); /* Ensure cq->idx is visible before reading cq_need_wakeup */ + if (!get_user(cq_need_wakeup, &svq->vq.cq->cq_need_wakeup)) { + smp_rmb(); + if (cq_need_wakeup && svq->vq.call_ctx.ctx) + eventfd_signal(svq->vq.call_ctx.ctx); + } + } +} + + static struct vhost_scsi_cmd * vhost_scsi_get_cmd(struct vhost_virtqueue *vq, u64 scsi_tag) { @@ -1060,9 +1130,25 @@ static void vhost_scsi_target_queue_cmd(struct vhost= _scsi_nexus *nexus, } =20 se_cmd->tag =3D 0; + cmd->tvc_submit_ts =3D ktime_get(); target_init_cmd(se_cmd, nexus->tvn_se_sess, &cmd->tvc_sense_buf[0], lun, exp_data_len, vhost_scsi_to_tcm_attr(task_attr), - data_dir, TARGET_SCF_ACK_KREF); + data_dir, + TARGET_SCF_ACK_KREF | + (READ_ONCE(cmd->tvc_vq->sqcq_poll) ? + TARGET_SCF_USE_CPUID : 0)); + + /* Poll mode: steer TCM completion to next CPU to avoid + * starvation behind poll thread. Skip if fabric has + * custom cmd_compl_affinity (-2=3DUNBOUND or specific CPU). + */ + if (READ_ONCE(cmd->tvc_vq->sqcq_poll) && num_online_cpus() > 1) { + struct se_wwn *wwn =3D se_cmd->se_sess->se_tpg->se_tpg_wwn; + + if (!wwn || wwn->cmd_compl_affinity =3D=3D SE_COMPL_AFFINITY_CPUID) + se_cmd->cpuid =3D + vhost_sqcq_pick_completion_cpu(smp_processor_id()); + } =20 if (target_submit_prep(se_cmd, cdb, sg_ptr, cmd->tvc_sgl_count, NULL, 0, sg_prot_ptr, @@ -1346,7 +1432,6 @@ vhost_scsi_handle_vq(struct vhost_scsi *vs, struct vh= ost_virtqueue *vq) ret =3D vhost_scsi_chk_size(vq, &vc); if (ret) goto err; - ret =3D vhost_scsi_get_req(vq, &vc, &tpg); if (ret) goto err; @@ -2070,6 +2155,15 @@ vhost_scsi_set_endpoint(struct vhost_scsi *vs, mutex_lock(&vq->mutex); vhost_vq_set_backend(vq, vs_tpg); vhost_vq_init_access(vq); + if (i >=3D VHOST_SCSI_VQ_IO && + vhost_has_feature(vq, VIRTIO_F_SQCQ_POLL) && + vq->sq && vq->cq) { + ret =3D vhost_sqcq_poll_start(vq); + if (ret) { + mutex_unlock(&vq->mutex); + goto stop_poll_threads; + } + } mutex_unlock(&vq->mutex); } ret =3D 0; @@ -2086,6 +2180,19 @@ vhost_scsi_set_endpoint(struct vhost_scsi *vs, vs->vs_tpg =3D vs_tpg; goto out; =20 +stop_poll_threads: + /* VQ at index i failed to start. Stop poll threads for all + * previously started VQs (indices i-1 down to VHOST_SCSI_VQ_IO). + */ + for (i =3D i - 1; i >=3D VHOST_SCSI_VQ_IO; i--) { + vq =3D &vs->vqs[i].vq; + mutex_lock(&vq->mutex); + vhost_sqcq_poll_stop(vq); + vhost_vq_set_backend(vq, NULL); + mutex_unlock(&vq->mutex); + } + /* Reset i for full vq cmds cleanup */ + i =3D vs->dev.nvqs; destroy_vq_cmds: for (i--; i >=3D VHOST_SCSI_VQ_IO; i--) { if (!vhost_vq_get_backend(&vs->vqs[i].vq)) @@ -2163,9 +2270,17 @@ vhost_scsi_clear_endpoint(struct vhost_scsi *vs, /* Prevent new cmds from starting and accessing the tpgs/sessions */ for (i =3D 0; i < vs->dev.nvqs; i++) { vq =3D &vs->vqs[i].vq; - mutex_lock(&vq->mutex); - vhost_vq_set_backend(vq, NULL); - mutex_unlock(&vq->mutex); + if (!READ_ONCE(vq->sqcq_poll)) { + mutex_lock(&vq->mutex); + vhost_vq_set_backend(vq, NULL); + mutex_unlock(&vq->mutex); + } else { + mutex_lock(&vq->mutex); + vhost_sqcq_poll_stop(vq); + vhost_vq_set_backend(vq, NULL); + mutex_unlock(&vq->mutex); + vhost_sqcq_poll_flush(vq); + } } /* Make sure cmds are not running before tearing them down. */ vhost_scsi_flush(vs); @@ -2221,6 +2336,7 @@ static int vhost_scsi_set_features(struct vhost_scsi = *vs, u64 features) bool is_log, was_log; int i; =20 + if (features & ~VHOST_SCSI_FEATURES) return -EOPNOTSUPP; =20 @@ -2321,6 +2437,7 @@ static int vhost_scsi_open(struct inode *inode, struc= t file *f) vhost_work_init(&svq->completion_work, vhost_scsi_complete_cmd_work); svq->vq.handle_kick =3D vhost_scsi_handle_kick; + svq->vq.poll_complete =3D vhost_scsi_poll_complete; } vhost_dev_init(&vs->dev, vqs, nvqs, UIO_MAXIOV, VHOST_SCSI_WEIGHT, 0, true, NULL); @@ -2415,6 +2532,45 @@ vhost_scsi_ioctl(struct file *f, if (copy_from_user(&features, featurep, sizeof features)) return -EFAULT; return vhost_scsi_set_features(vs, features); + case VHOST_GET_FEATURES_ARRAY: { + u64 count, copied; + u64 farr[VIRTIO_FEATURES_U64S] =3D {}; + + if (get_user(count, (u64 __user *)argp)) + return -EFAULT; + copied =3D min_t(u64, count, VIRTIO_FEATURES_U64S); + farr[0] =3D VHOST_SCSI_FEATURES; + if (copy_to_user((u64 __user *)(argp + sizeof(u64)), + farr, copied * sizeof(u64))) + return -EFAULT; + if (count > copied && + clear_user((u64 __user *)(argp + sizeof(u64) + + copied * sizeof(u64)), + (count - copied) * sizeof(u64))) + return -EFAULT; + return 0; + } + case VHOST_SET_FEATURES_ARRAY: { + u64 count, copied, all_features[VIRTIO_FEATURES_U64S] =3D {}; + int i; + + if (get_user(count, (u64 __user *)argp)) + return -EFAULT; + copied =3D min_t(u64, count, VIRTIO_FEATURES_U64S); + if (copy_from_user(all_features, + (u64 __user *)(argp + sizeof(u64)), + copied * sizeof(u64))) + return -EFAULT; + for (i =3D copied; i < count; i++) { + u64 tmp; + if (get_user(tmp, (u64 __user *)(argp + + sizeof(u64) + i * sizeof(u64)))) + return -EFAULT; + if (tmp) + return -EOPNOTSUPP; + } + return vhost_scsi_set_features(vs, all_features[0]); + } case VHOST_NEW_WORKER: case VHOST_FREE_WORKER: case VHOST_ATTACH_VRING_WORKER: @@ -2424,6 +2580,7 @@ vhost_scsi_ioctl(struct file *f, mutex_unlock(&vs->dev.mutex); return r; default: + mutex_lock(&vs->dev.mutex); r =3D vhost_dev_ioctl(&vs->dev, ioctl, argp); /* TODO: flush backend after dev ioctl. */ diff --git a/drivers/vhost/vhost.c b/drivers/vhost/vhost.c index 2f2c45d208832..68fb53c40b2e6 100644 --- a/drivers/vhost/vhost.c +++ b/drivers/vhost/vhost.c @@ -42,6 +42,20 @@ static int max_iotlb_entries =3D 2048; module_param(max_iotlb_entries, int, 0444); MODULE_PARM_DESC(max_iotlb_entries, "Maximum number of iotlb entries. (default: 2048)"); +#define SQCQ_IDLE_TIMEOUT_US_MAX 1000U +#define SQCQ_SLEEP_EMA_MULT 2 +#define SQCQ_SPIN_EMA_MULT 2 +#define SQCQ_ACTIVE_TIMEOUT_NS 1000000U /* cap active-VQ clear at 1 ms */ +#define SQCQ_STALE_TIMEOUT_NS 1000000000ULL /* backstop EMA/active clear = at 1 s */ + +static atomic_t sqcq_poll_cpu_rr =3D ATOMIC_INIT(-1); +static atomic_t sqcq_active_io_vqs =3D ATOMIC_INIT(0); +/* CPUs with a BUSY poll thread (actively processing cmds); TCM completion + * steers away. Idle (sleeping) poll threads don't occupy their CPU, so + * their bit is cleared via the sq_was_active transitions in the poll loop. + */ +static cpumask_t sqcq_poll_active_cpu_mask; + static bool fork_from_owner_default =3D VHOST_FORK_OWNER_TASK; =20 #ifdef CONFIG_VHOST_ENABLE_FORK_OWNER_CONTROL @@ -390,6 +404,20 @@ static void vhost_vq_reset(struct vhost_dev *dev, vhost_disable_cross_endian(vq); vhost_reset_is_le(vq); vq->busyloop_timeout =3D 0; + WRITE_ONCE(vq->sqcq_poll, false); + vq->sq =3D NULL; + vq->cq =3D NULL; + vq->poll_task =3D NULL; + WRITE_ONCE(vq->poll_stop, false); + init_waitqueue_head(&vq->poll_wait); + init_completion(&vq->poll_started); + init_completion(&vq->poll_stopped); + WRITE_ONCE(vq->kick_received, false); + atomic_set(&vq->completion_pending, 0); + vq->kick_wqh =3D NULL; + vq->stat_completions =3D 0; + vq->lat_scsi_ns =3D 0; + vq->poll_cpu =3D -1; vq->umem =3D NULL; vq->iotlb =3D NULL; rcu_assign_pointer(vq->worker, NULL); @@ -1168,6 +1196,15 @@ void vhost_dev_stop(struct vhost_dev *dev) { int i; =20 + for (i =3D 0; i < dev->nvqs; ++i) { + if (!READ_ONCE(dev->vqs[i]->sqcq_poll)) + continue; + mutex_lock(&dev->vqs[i]->mutex); + vhost_sqcq_poll_stop(dev->vqs[i]); + mutex_unlock(&dev->vqs[i]->mutex); + vhost_sqcq_poll_flush(dev->vqs[i]); + } + for (i =3D 0; i < dev->nvqs; ++i) { if (dev->vqs[i]->kick && dev->vqs[i]->handle_kick) vhost_poll_stop(&dev->vqs[i]->poll); @@ -2051,6 +2088,535 @@ static long vhost_vring_set_num(struct vhost_dev *d, return 0; } =20 +static int vhost_sqcq_kick_wakeup(wait_queue_entry_t *wait, unsigned int m= ode, + int sync, void *key) +{ + struct vhost_virtqueue *vq =3D container_of(wait, + struct vhost_virtqueue, kick_wait); + + if (!(key_to_poll(key) & EPOLLIN)) + return 0; + + WRITE_ONCE(vq->kick_received, true); + wake_up(&vq->poll_wait); + return 0; +} + +static void vhost_sqcq_kick_ptable_func(struct file *file, + wait_queue_head_t *wqh, + poll_table *pt) +{ + struct vhost_virtqueue *vq =3D container_of(pt, + struct vhost_virtqueue, kick_pt); + + vq->kick_wqh =3D wqh; + add_wait_queue(wqh, &vq->kick_wait); +} + +static void vhost_sqcq_kick_register(struct vhost_virtqueue *vq) +{ + init_waitqueue_func_entry(&vq->kick_wait, vhost_sqcq_kick_wakeup); + vq->kick_wqh =3D NULL; + + if (vq->kick) { + __poll_t mask; + + init_poll_funcptr(&vq->kick_pt, vhost_sqcq_kick_ptable_func); + mask =3D vfs_poll(vq->kick, &vq->kick_pt); + /* eventfd counter is never consumed; verify stale EPOLLIN + * against sq->idx before acting; real kicks wake via + * the registered wait queue. + */ + if (mask & EPOLLIN) { + __virtio64 sq_val; + + if (vq->sq && !get_user(sq_val, &vq->sq->idx)) { + smp_rmb(); + if ((__force u16)sq_val !=3D vq->last_avail_idx) + vhost_sqcq_kick_wakeup( + &vq->kick_wait, 0, 0, + poll_to_key(mask)); + } + } + } +} + +static void vhost_sqcq_kick_unregister(struct vhost_virtqueue *vq) +{ + if (vq->kick_wqh) { + remove_wait_queue(vq->kick_wqh, &vq->kick_wait); + vq->kick_wqh =3D NULL; + } + WRITE_ONCE(vq->kick_received, false); +} + +static int vhost_sqcq_poll_thread(void *data) +{ + struct vhost_virtqueue *vq =3D data; + struct vhost_dev *dev =3D vq->dev; + u16 new_sq_idx; + __virtio64 sq_idx_val; + __u8 sq_need_wakeup_val =3D 0; + ktime_t ts2; + ktime_t last_io_ts =3D 0; + ktime_t stats_window =3D ktime_get(); + u64 stats_cq =3D 0; + u64 stats_lat =3D 0; + u64 stats_inflight_sum =3D 0; + u64 stats_inflight_n =3D 0; + u64 stats_multi_cnt =3D 0; + u64 stats_sleep_cnt =3D 0; + u64 stats_wake_kick =3D 0; + u64 stats_wake_complete =3D 0; + u64 stats_wake_timeout =3D 0; + u64 ema_ns_val; + u64 spin_ns_val; + unsigned long _timeout_jiffies; + u64 _timeout_us; + + kthread_use_mm(dev->mm); + complete(&vq->poll_started); + + vq->stat_completions =3D 0; + vq->lat_scsi_ns =3D 0; + vq->ema_latency_ns =3D 0; + vq->ema_interval_ns =3D 0; + ts2 =3D ktime_get(); + + while (!READ_ONCE(vq->poll_stop)) { + + stats_inflight_sum +=3D (u16)(vq->last_avail_idx - vq->last_used_idx); + stats_inflight_n++; + stats_multi_cnt +=3D ((atomic_read(&sqcq_active_io_vqs) > 1) ? 1 : 0); + + // 10's output windows. + if (ktime_to_ns(ktime_sub(ktime_get(), stats_window)) >=3D + (s64)10 * NSEC_PER_SEC) { + if (stats_cq > 0) { + int vi; + + for (vi =3D 0; vi < dev->nvqs; vi++) + if (dev->vqs[vi] =3D=3D vq) + break; + pr_info("vhost-sqcq: vq[%d] cq=3D%llu avg=3D%lluns ema_lat=3D%lluns in= terval=3D%lluns avg_inflight=3D%llu multi=3D%llu%% sleep=3D%llu wk_kick=3D%= llu%% wk_cmp=3D%llu%% wk_to=3D%llu%%\n", + vi, stats_cq, + div64_u64(stats_lat, stats_cq), + vq->ema_latency_ns, + vq->ema_interval_ns, + stats_inflight_n > 0 ? + div64_u64(stats_inflight_sum, stats_inflight_n) : 0, + div64_u64(stats_multi_cnt*100, stats_inflight_n), + stats_sleep_cnt, + stats_sleep_cnt > 0 ? + div64_u64(stats_wake_kick*100, stats_sleep_cnt) : 0, + stats_sleep_cnt > 0 ? + div64_u64(stats_wake_complete*100, stats_sleep_cnt) : 0, + stats_sleep_cnt > 0 ? + div64_u64(stats_wake_timeout*100, stats_sleep_cnt) : 0); + } + stats_window =3D ktime_get(); + stats_cq =3D 0; + stats_lat =3D 0; + stats_inflight_sum =3D 0; + stats_inflight_n =3D 0; + stats_multi_cnt =3D 0; + stats_sleep_cnt =3D 0; + stats_wake_kick =3D 0; + stats_wake_complete =3D 0; + stats_wake_timeout =3D 0; + } + + /* Backstop cleanup: drop sq_was_active and reset EMAs once a VQ + * has been idle (no submission) for a fixed interval. + */ + if (vq->sq_was_active && last_io_ts > 0 && + ktime_to_ns(ktime_sub(ktime_get(), last_io_ts)) > + (s64)SQCQ_STALE_TIMEOUT_NS) { + vq->sq_was_active =3D false; + atomic_dec(&sqcq_active_io_vqs); + if (vq->poll_cpu >=3D 0) + cpumask_clear_cpu(vq->poll_cpu, &sqcq_poll_active_cpu_mask); + WRITE_ONCE(vq->ema_interval_ns, 0); + WRITE_ONCE(vq->ema_latency_ns, 0); + } + + /* complete*/ + if (atomic_read(&vq->completion_pending) && vq->poll_complete) { + smp_rmb(); + atomic_set(&vq->completion_pending, 0); + vq->poll_complete(vq); + if (vq->stat_completions > 0) { + u64 avg =3D div64_u64(vq->lat_scsi_ns, + vq->stat_completions); + + if (vq->ema_latency_ns =3D=3D 0) + vq->ema_latency_ns =3D avg; + else + vq->ema_latency_ns =3D + vq->ema_latency_ns - + vq->ema_latency_ns / 8 + + avg / 8; + stats_cq +=3D vq->stat_completions; + stats_lat +=3D vq->lat_scsi_ns; + vq->lat_scsi_ns =3D 0; + vq->stat_completions =3D 0; + } + } + + if (get_user(sq_idx_val, &vq->sq->idx)) { + int fault_retries =3D 3; + + while (fault_retries-- > 0) { + if (!get_user(sq_idx_val, &vq->sq->idx)) + break; + schedule_timeout_idle(1); + } + if (fault_retries < 0) + schedule_timeout_idle(HZ / 100); + continue; + } + /* Acquire: see all prior Guest writes (descriptors). */ + smp_rmb(); + new_sq_idx =3D (__force u16)sq_idx_val; + + if (new_sq_idx !=3D vq->last_avail_idx) { + vq->handle_kick(&vq->poll.work); + ts2 =3D ktime_get(); + if (last_io_ts > 0) { + u64 _interval =3D ktime_to_ns(ktime_sub(ts2, last_io_ts)); + if (vq->ema_interval_ns =3D=3D 0) + vq->ema_interval_ns =3D _interval; + else + vq->ema_interval_ns =3D + vq->ema_interval_ns - + vq->ema_interval_ns / 8 + + _interval / 8; + } + last_io_ts =3D ts2; + if (!vq->sq_was_active) { + vq->sq_was_active =3D true; + atomic_inc(&sqcq_active_io_vqs); + if (vq->poll_cpu >=3D 0) + cpumask_set_cpu(vq->poll_cpu, &sqcq_poll_active_cpu_mask); + } + sq_need_wakeup_val =3D 0; + smp_mb(); + put_user(sq_need_wakeup_val, &vq->sq->sq_need_wakeup); + } + + + if (vq->last_avail_idx =3D=3D vq->last_used_idx) { + /* Spin for a fixed window to catch next submission via + * sq->idx polling; need_resched+cond_resched prevents + * starving other tasks on this CPU. + */ + spin_ns_val =3D SQCQ_ACTIVE_TIMEOUT_NS; + if (last_io_ts > 0 && ktime_to_ns(ktime_sub( + ktime_get(), last_io_ts)) < spin_ns_val) { + if (need_resched()) + cond_resched(); + cpu_relax(); + continue; + } + /* Clear sq_was_active to keep active_io_vqs and + * cpu mask fresh. Has-inflight path keeps the + * flag for TCM completion steering. + */ + if (vq->sq_was_active) { + vq->sq_was_active =3D false; + atomic_dec(&sqcq_active_io_vqs); + if (vq->poll_cpu >=3D 0) + cpumask_clear_cpu(vq->poll_cpu, &sqcq_poll_active_cpu_mask); + } + } else { + /* Spin within SQCQ_SPIN_EMA_MULT * ema_latency (with + * cooperative yield), then sleep. cond_resched handles + * multi-VQ fairness without a separate yield heuristic. + */ + ema_ns_val =3D READ_ONCE(vq->ema_latency_ns); + spin_ns_val =3D ema_ns_val > 0 ? + (u64)SQCQ_SPIN_EMA_MULT * ema_ns_val : + 2 * NSEC_PER_USEC; /* fallback when EMA unknown */ + + if (ktime_to_ns(ktime_sub( + ktime_get(), last_io_ts)) < spin_ns_val) { + if (need_resched()) + cond_resched(); + cpu_relax(); + continue; + } + } + + ema_ns_val =3D READ_ONCE(vq->ema_latency_ns); + + if (ema_ns_val =3D=3D 0) { + _timeout_jiffies =3D usecs_to_jiffies( + SQCQ_IDLE_TIMEOUT_US_MAX); + } else { + _timeout_us =3D div64_u64( + ema_ns_val, + (u64)NSEC_PER_USEC / SQCQ_SLEEP_EMA_MULT); + + _timeout_us =3D clamp(_timeout_us, + 10ULL, + (u64)SQCQ_IDLE_TIMEOUT_US_MAX); + _timeout_jiffies =3D usecs_to_jiffies( + _timeout_us); + if (_timeout_jiffies =3D=3D 0) + _timeout_jiffies =3D 1; + } + + /* Set NEED_WAKEUP, then double-check sq->idx before sleeping. */ + smp_wmb(); + sq_need_wakeup_val =3D 1; + put_user(sq_need_wakeup_val, + &vq->sq->sq_need_wakeup); + smp_mb(); + + if (get_user(sq_idx_val, &vq->sq->idx)) { + int fault_retries =3D 3; + + while (fault_retries-- > 0) { + if (!get_user(sq_idx_val, &vq->sq->idx)) + break; + schedule_timeout_idle(1); + } + if (fault_retries < 0) + schedule_timeout_idle(HZ / 100); + continue; + } + smp_rmb(); + new_sq_idx =3D (__force u16)sq_idx_val; + + if (new_sq_idx !=3D vq->last_avail_idx) { + sq_need_wakeup_val =3D 0; + smp_mb(); + put_user(sq_need_wakeup_val, + &vq->sq->sq_need_wakeup); + continue; + } + + ++stats_sleep_cnt; + vhost_sqcq_kick_register(vq); + wait_event_interruptible_timeout( + vq->poll_wait, + READ_ONCE(vq->poll_stop) || + READ_ONCE(vq->kick_received) || + atomic_read(&vq->completion_pending), + _timeout_jiffies); + /* Wake reason stats: snapshot before unregister clears kick_received */ + { + bool _kicked =3D READ_ONCE(vq->kick_received); + bool _completed =3D atomic_read(&vq->completion_pending); + + if (_kicked) + stats_wake_kick++; + if (_completed) + stats_wake_complete++; + if (!_kicked && !_completed) + stats_wake_timeout++; + } + vhost_sqcq_kick_unregister(vq); + + sq_need_wakeup_val =3D 0; + put_user(sq_need_wakeup_val, + &vq->sq->sq_need_wakeup); + } + + complete(&vq->poll_stopped); + kthread_unuse_mm(dev->mm); + return 0; +} + +int vhost_sqcq_poll_start(struct vhost_virtqueue *vq) +{ + if (!vq->sq || !vq->cq || !vq->handle_kick) + return -EINVAL; + + vhost_poll_stop(&vq->poll); + + WRITE_ONCE(vq->poll_stop, false); + reinit_completion(&vq->poll_started); + reinit_completion(&vq->poll_stopped); + + int vi; + + for (vi =3D 0; vi < vq->dev->nvqs; vi++) + if (vq->dev->vqs[vi] =3D=3D vq) + break; + vq->poll_task =3D kthread_run(vhost_sqcq_poll_thread, vq, + "vhost-sqcq-%d-%d", + task_pid_nr(current), + vi); + if (IS_ERR(vq->poll_task)) { + int ret =3D PTR_ERR(vq->poll_task); + + vq->poll_task =3D NULL; + if (vq->kick) + vhost_poll_start(&vq->poll, vq->kick); + return ret; + } + + wait_for_completion(&vq->poll_started); + + /* Enable sqcq mode only after poll_task is valid and the + * thread has signaled readiness. Before this point, + * vhost_signal() routes completions via eventfd (old path). + */ + smp_mb(); + WRITE_ONCE(vq->sqcq_poll, true); + + if (vq->poll_task) { + int cpu; + int start =3D atomic_inc_return(&sqcq_poll_cpu_rr); + int target_cpu =3D -1; + int online =3D num_online_cpus(); + + if (online > 0) + start =3D start % online; + else + start =3D 0; + + for_each_online_cpu(cpu) { + if (start-- <=3D 0) { + target_cpu =3D cpu; + break; + } + } + if (target_cpu < 0) { + for_each_online_cpu(cpu) { + target_cpu =3D cpu; + break; + } + } + + if (target_cpu >=3D 0 && target_cpu < nr_cpu_ids && + cpu_online(target_cpu)) { + int vi; + + set_cpus_allowed_ptr(vq->poll_task, cpumask_of(target_cpu)); + for (vi =3D 0; vi < vq->dev->nvqs; vi++) + if (vq->dev->vqs[vi] =3D=3D vq) + break; + pr_info("vhost-sqcq: vq[%d] poll thread bound to cpu%d\n", + vi, target_cpu); + vq->poll_cpu =3D target_cpu; + } + } + + return 0; +} +EXPORT_SYMBOL_GPL(vhost_sqcq_poll_start); + +void vhost_sqcq_poll_stop(struct vhost_virtqueue *vq) +{ + if (!vq->poll_task) + return; + + WRITE_ONCE(vq->poll_stop, true); + smp_mb(); + wake_up_process(vq->poll_task); + wake_up(&vq->poll_wait); + wait_for_completion(&vq->poll_stopped); + vq->poll_task =3D NULL; + WRITE_ONCE(vq->sqcq_poll, false); + if (vq->poll_cpu >=3D 0) { + cpumask_clear_cpu(vq->poll_cpu, &sqcq_poll_active_cpu_mask); + vq->poll_cpu =3D -1; + } + smp_mb(); + + if (vq->kick) + vhost_poll_start(&vq->poll, vq->kick); + vq->sq =3D NULL; + vq->cq =3D NULL; +} +EXPORT_SYMBOL_GPL(vhost_sqcq_poll_stop); + +/* Flush residual completions after stop. Must be called outside vq->mute= x. */ +void vhost_sqcq_poll_flush(struct vhost_virtqueue *vq) +{ + if (WARN_ON_ONCE(!vq->poll_complete)) + return; + + atomic_set(&vq->completion_pending, 0); + vq->poll_complete(vq); +} +EXPORT_SYMBOL_GPL(vhost_sqcq_poll_flush); + +/* Pick a CPU for TCM completion: prefer a free CPU (no poll thread), + * searching from hint_cpu+1 with wrap. Fallback to next-online + * if every CPU has a poll thread. + */ +int vhost_sqcq_pick_completion_cpu(int hint_cpu) +{ + int first, cpu, target =3D -1; + + first =3D cpumask_next(hint_cpu, cpu_online_mask); + if (first >=3D nr_cpu_ids) + first =3D cpumask_first(cpu_online_mask); + + cpu =3D first; + do { + if (!cpumask_test_cpu(cpu, &sqcq_poll_active_cpu_mask)) { + target =3D cpu; + break; + } + cpu =3D cpumask_next(cpu, cpu_online_mask); + if (cpu >=3D nr_cpu_ids) + cpu =3D cpumask_first(cpu_online_mask); + } while (cpu !=3D first); + + if (target < 0) + target =3D first; + + return target; +} +EXPORT_SYMBOL_GPL(vhost_sqcq_pick_completion_cpu); + +static long vhost_vring_set_sqcq_addr(struct vhost_dev *d, + struct vhost_virtqueue *vq, + void __user *argp) +{ + struct vhost_vring_sqcq_addr a; + + if (copy_from_user(&a, argp, sizeof(a))) + return -EFAULT; + + if ((u64)(unsigned long)a.sq_user_addr !=3D a.sq_user_addr || + (u64)(unsigned long)a.cq_user_addr !=3D a.cq_user_addr) + return -EFAULT; + + if (a.sq_user_addr & 7 || a.cq_user_addr & 7) + return -EINVAL; + + if (vq->private_data) { + if (!access_ok((void __user *)(unsigned long)a.sq_user_addr, + sizeof(struct vring_sq)) || + !access_ok((void __user *)(unsigned long)a.cq_user_addr, + sizeof(struct vring_cq))) + return -EINVAL; + } + + vq->sq =3D (struct vring_sq __user *)(unsigned long)a.sq_user_addr; + vq->cq =3D (struct vring_cq __user *)(unsigned long)a.cq_user_addr; + + /* SET_ENDPOINT may run before SET_FEATURES/SQCQ_ADDR, so poll-start + * can fail there; this is the canonical trigger. Skip VQs without + * poll_complete (ctrl/event VQs) =E2=80=94 they stay interrupt-driven. + */ + if (vq->private_data && vq->sq && vq->cq && + !READ_ONCE(vq->sqcq_poll) && + vq->poll_complete && + vhost_has_feature(vq, VIRTIO_F_SQCQ_POLL)) { + long ret =3D vhost_sqcq_poll_start(vq); + if (ret) + return ret; + } + + return 0; +} + static long vhost_vring_set_addr(struct vhost_dev *d, struct vhost_virtqueue *vq, void __user *argp) @@ -2244,6 +2810,9 @@ long vhost_vring_ioctl(struct vhost_dev *d, unsigned = int ioctl, void __user *arg if (copy_to_user(argp, &s, sizeof(s))) r =3D -EFAULT; break; + case VHOST_SET_VRING_SQCQ_ADDR: + r =3D vhost_vring_set_sqcq_addr(d, vq, argp); + break; default: r =3D -ENOIOCTLCMD; } @@ -2256,7 +2825,7 @@ long vhost_vring_ioctl(struct vhost_dev *d, unsigned = int ioctl, void __user *arg if (filep) fput(filep); =20 - if (pollstart && vq->handle_kick) + if (pollstart && vq->handle_kick && !READ_ONCE(vq->sqcq_poll)) r =3D vhost_poll_start(&vq->poll, vq->kick); =20 mutex_unlock(&vq->mutex); @@ -2938,8 +3507,12 @@ int vhost_get_vq_desc_n(struct vhost_virtqueue *vq, *ndesc =3D c; =20 /* Assume notifications from guest are disabled at this point, - * if they aren't we would need to update avail_event index. */ - BUG_ON(!(vq->used_flags & VRING_USED_F_NO_NOTIFY)); + * if they aren't we would need to update avail_event index. + * In SQCQ poll mode, vhost_disable_notify is a no-op so + * VRING_USED_F_NO_NOTIFY is never set; skip the check. + */ + WARN_ON_ONCE(!vq->sqcq_poll && + !(vq->used_flags & VRING_USED_F_NO_NOTIFY)); return head; } EXPORT_SYMBOL_GPL(vhost_get_vq_desc_n); @@ -3167,6 +3740,16 @@ static bool vhost_notify(struct vhost_dev *dev, stru= ct vhost_virtqueue *vq) /* This actually signals the guest, using eventfd. */ void vhost_signal(struct vhost_dev *dev, struct vhost_virtqueue *vq) { + if (READ_ONCE(vq->sqcq_poll) && vq->cq) { + atomic_set(&vq->completion_pending, 1); + if (!READ_ONCE(vq->poll_stop)) { + smp_mb(); + if (!task_is_running(vq->poll_task)) + wake_up(&vq->poll_wait); + } + return; + } + /* Signal the Guest tell them we used something up. */ if (vq->call_ctx.ctx && vhost_notify(dev, vq)) eventfd_signal(vq->call_ctx.ctx); @@ -3213,6 +3796,9 @@ EXPORT_SYMBOL_GPL(vhost_vq_avail_empty); /* OK, now we need to know about added descriptors. */ bool vhost_enable_notify(struct vhost_dev *dev, struct vhost_virtqueue *vq) { + if (READ_ONCE(vq->sqcq_poll)) + return false; + int r; =20 if (!(vq->used_flags & VRING_USED_F_NO_NOTIFY)) @@ -3249,6 +3835,9 @@ EXPORT_SYMBOL_GPL(vhost_enable_notify); /* We don't need to be notified again. */ void vhost_disable_notify(struct vhost_dev *dev, struct vhost_virtqueue *v= q) { + if (READ_ONCE(vq->sqcq_poll)) + return; + int r; =20 if (vq->used_flags & VRING_USED_F_NO_NOTIFY) diff --git a/drivers/vhost/vhost.h b/drivers/vhost/vhost.h index 4fe99765c5c73..b9c15961f5a18 100644 --- a/drivers/vhost/vhost.h +++ b/drivers/vhost/vhost.h @@ -111,6 +111,7 @@ struct vhost_virtqueue { =20 /* The routine to call when the Guest pings us, or timeout. */ vhost_work_fn_t handle_kick; + void (*poll_complete)(struct vhost_virtqueue *vq); =20 /* Last available index we saw. * Values are limited to 0x7fff, and the high bit is used as @@ -164,6 +165,27 @@ struct vhost_virtqueue { bool user_be; #endif u32 busyloop_timeout; + + /* SQ/CQ polling mode */ + bool sqcq_poll; + struct vring_sq __user *sq; + struct vring_cq __user *cq; + struct task_struct *poll_task; + bool poll_stop; + wait_queue_head_t poll_wait; + struct completion poll_started; + struct completion poll_stopped; + bool kick_received; /* Set when kick eventfd wakes us */ + atomic_t completion_pending; /* Set by vhost_signal, poll thread atomical= ly clears */ + wait_queue_head_t *kick_wqh; /* kick eventfd wait queue head */ + wait_queue_entry_t kick_wait; /* Wait entry on kick eventfd wq */ + poll_table kick_pt; /* poll_table for kick eventfd */ + u64 stat_completions; + u64 lat_scsi_ns; + u64 ema_interval_ns; + u64 ema_latency_ns; + bool sq_was_active; + int poll_cpu; /* CPU this VQ's poll thread is pinned to, -1 =3D unpinned= */ }; =20 struct vhost_msg_node { @@ -256,6 +278,10 @@ void vhost_add_used_and_signal_n(struct vhost_dev *, s= truct vhost_virtqueue *, void vhost_signal(struct vhost_dev *, struct vhost_virtqueue *); void vhost_disable_notify(struct vhost_dev *, struct vhost_virtqueue *); bool vhost_vq_avail_empty(struct vhost_dev *, struct vhost_virtqueue *); +int vhost_sqcq_poll_start(struct vhost_virtqueue *vq); +void vhost_sqcq_poll_stop(struct vhost_virtqueue *vq); +void vhost_sqcq_poll_flush(struct vhost_virtqueue *vq); +int vhost_sqcq_pick_completion_cpu(int hint_cpu); bool vhost_enable_notify(struct vhost_dev *, struct vhost_virtqueue *); =20 int vhost_log_write(struct vhost_virtqueue *vq, struct vhost_log *log, @@ -294,7 +320,8 @@ void vhost_iotlb_map_free(struct vhost_iotlb *iotlb, VIRTIO_RING_F_EVENT_IDX, \ VHOST_F_LOG_ALL, \ VIRTIO_F_ANY_LAYOUT, \ - VIRTIO_F_VERSION_1 + VIRTIO_F_VERSION_1, \ + VIRTIO_F_SQCQ_POLL =20 static inline u64 vhost_features_u64(const int *features, int size, int id= x) { diff --git a/include/uapi/linux/vhost.h b/include/uapi/linux/vhost.h index c57674a6aa0db..dacb0e1fdc4bc 100644 --- a/include/uapi/linux/vhost.h +++ b/include/uapi/linux/vhost.h @@ -123,6 +123,10 @@ #define VHOST_SET_BACKEND_FEATURES _IOW(VHOST_VIRTIO, 0x25, __u64) #define VHOST_GET_BACKEND_FEATURES _IOR(VHOST_VIRTIO, 0x26, __u64) =20 +/* Set SQ/CQ doorbell addresses for polling mode */ +#define VHOST_SET_VRING_SQCQ_ADDR _IOW(VHOST_VIRTIO, 0x27, \ + struct vhost_vring_sqcq_addr) + /* VHOST_NET specific defines */ =20 /* Attach virtio net ring to a raw socket, or tap device. diff --git a/include/uapi/linux/vhost_types.h b/include/uapi/linux/vhost_ty= pes.h index 1c39cc5f5a31b..6514c1b5be20b 100644 --- a/include/uapi/linux/vhost_types.h +++ b/include/uapi/linux/vhost_types.h @@ -47,6 +47,12 @@ struct vhost_vring_addr { __u64 log_guest_addr; }; =20 +struct vhost_vring_sqcq_addr { + unsigned int index; + __u64 sq_user_addr; /* SQ doorbell HVA */ + __u64 cq_user_addr; /* CQ doorbell HVA */ +}; + struct vhost_worker_state { /* * For VHOST_NEW_WORKER the kernel will return the new vhost_worker id. --=20 2.34.1 From nobody Sat Jul 25 11:03:39 2026 Received: from m16.mail.163.com (m16.mail.163.com [117.135.210.2]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 9AE1039E6E4; Mon, 20 Jul 2026 14:11:41 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=117.135.210.2 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1784556706; cv=none; b=sOPSL68jgmY6fmwFzAIylSmjNGgo0VziQ0QZYgnTwaRq+OCsdi5nwnh0+zh8hlZM63a9iodrNaa2e9sch/fbsS69ooFuF+9sRQhPZ44aDuaTNA1lcR7gt6MY9VSAlMNIZ+oWQMjvRhlPUj7QyYFU5/7QbN3fy5sbMAmSE2q3+HM= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1784556706; c=relaxed/simple; bh=yb0qCsLKZrZcL6wihEOGi5lj/DktoRSj4Pg7irMqigU=; h=From:To:Cc:Subject:Date:Message-Id:In-Reply-To:References: MIME-Version:Content-Type; b=n90fg88QwBawKsx3aWJ7T8XMVVjOrQouNczaP90JNjk3Hfpi88W/l1UWoNseJyFUMTyZPBOWzL1ogkmT56SPfd3l0UgfjYxhzKUAjvRY0PlEh3GkATHIY7ZDj8P54nLfi+W3HqTr2bb2ig7IpX3U0SuhedZhey/hs1rP02VQOfc= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=163.com; spf=pass smtp.mailfrom=163.com; dkim=pass (1024-bit key) header.d=163.com header.i=@163.com header.b=TcDlmObR; arc=none smtp.client-ip=117.135.210.2 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=163.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=163.com Authentication-Results: smtp.subspace.kernel.org; dkim=pass (1024-bit key) header.d=163.com header.i=@163.com header.b="TcDlmObR" DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=163.com; s=s110527; h=From:To:Subject:Date:Message-Id:MIME-Version: Content-Type; bh=p7LQ9TPUGZKNlf/GqJcGoVkUE7Sm3aTjMQYFUDUjxzw=; b=TcDlmObRHpCJofBTn9ekM28ah3r98dnY6Z1rFlEXOEPNKJXbPJWOkg51ZCLCP5 TAfBursw7iVOLAF3P7CMXhShl8CtDrJ2qBZfiVM13i/JNQv33oqab+M+9Vvkezwg gIyY5KFAKCkCUZlVI5SHokHQ7llHz7KKV6k3tvnrd4iQo= Received: from localhost.localdomain (unknown []) by gzsmtp3 (Coremail) with SMTP id PigvCgD3Xw9gLF5qWUvpIg--.12525S5; Mon, 20 Jul 2026 22:10:44 +0800 (CST) From: "rom.wang" To: mst@redhat.com, jasowangio@gmail.com, eperezma@redhat.com, stefanha@redhat.com Cc: virtualization@lists.linux.dev, linux-kernel@vger.kernel.org, linux-scsi@vger.kernel.org, kvm@vger.kernel.org, qemu-devel@nongnu.org, Yufeng Wang Subject: [RFC PATCH 3/4] virtio: guest driver support for SQ/CQ polling Date: Mon, 20 Jul 2026 22:10:39 +0800 Message-Id: <20260720141040.181946-4-r4o5m6e8o@163.com> X-Mailer: git-send-email 2.34.1 In-Reply-To: <20260720141040.181946-1-r4o5m6e8o@163.com> References: <20260720141040.181946-1-r4o5m6e8o@163.com> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Type: text/plain; charset="utf-8" Content-Transfer-Encoding: quoted-printable X-CM-TRANSID: PigvCgD3Xw9gLF5qWUvpIg--.12525S5 X-Coremail-Antispam: 1Uf129KBjvAXoWfWr17GF48Zr43Zr48GrWDCFg_yoWrJF1rto WfJFsxJrn5Grn7ArWSgr1ftFWDZayqyFZxArs8urZ8Wa1jq3y5tF9rG3Z8JF1fAFyrKFn5 AF12g34ftFyUtFWfn29KB7ZKAUJUUUU8529EdanIXcx71UUUUU7v73VFW2AGmfu7bjvjm3 AaLaJ3UbIYCTnIWIevJa73UjIFyTuYvjxUfxhLUUUUU X-CM-SenderInfo: 3uurkzkwhy0qqrwthudrp/xtbCzQRWy2peLGSNuQAA3c From: Yufeng Wang Implement SQ/CQ doorbell polling in the guest kernel: - virtio_ring.c: SQ/CQ DMA allocation, virtqueue_notify() writes sq->idx via smp_store_release instead of MMIO kick; completion detection via more_used() (direct used-ring check) - virtio_sqcq_poll.c: per-device CQ poll thread with need_resched() cooperative spin, time-based spin budget, 1ms idle sleep - virtio_pci_modern.c: feature negotiation, PCI config space write for SQ/CQ DMA addresses, poll thread lifecycle - virtio.c: reject VIRTIO_F_SQCQ_POLL + VIRTIO_F_RING_PACKED - virtio_scsi.c: 10s window io_stats diagnostics output - do_softirq() after each completion callback (no hardware interrupt) Signed-off-by: Yufeng Wang --- drivers/scsi/virtio_scsi.c | 113 ++++++++- drivers/virtio/Makefile | 2 +- drivers/virtio/virtio.c | 10 + drivers/virtio/virtio_pci_common.c | 3 + drivers/virtio/virtio_pci_modern.c | 39 +++ drivers/virtio/virtio_pci_modern_dev.c | 34 ++- drivers/virtio/virtio_ring.c | 262 +++++++++++++++++++ drivers/virtio/virtio_sqcq_poll.c | 338 +++++++++++++++++++++++++ include/linux/virtio.h | 25 ++ include/linux/virtio_pci_modern.h | 2 + 10 files changed, 825 insertions(+), 3 deletions(-) create mode 100644 drivers/virtio/virtio_sqcq_poll.c diff --git a/drivers/scsi/virtio_scsi.c b/drivers/scsi/virtio_scsi.c index 0ed8558dad72..f803bb62d22a 100644 --- a/drivers/scsi/virtio_scsi.c +++ b/drivers/scsi/virtio_scsi.c @@ -30,6 +30,7 @@ #include #include #include +#include =20 #include "sd.h" =20 @@ -46,6 +47,8 @@ MODULE_PARM_DESC(virtscsi_poll_queues, struct virtio_scsi_cmd { struct scsi_cmnd *sc; struct completion *comp; + u64 submit_ns; /* ktime_get_ns() at submission */ + u64 notify_ns; /* ktime_get_ns() after SQ doorbell */ union { struct virtio_scsi_cmd_req cmd; struct virtio_scsi_cmd_req_pi cmd_pi; @@ -95,6 +98,22 @@ struct virtio_scsi { struct virtio_scsi_event events[VIRTIO_SCSI_EVENT_LEN]; __dma_from_device_group_end(); =20 + /* I/O latency statistics (debugging) */ + spinlock_t lat_lock; + u64 lat_sum_ns; + u64 lat_min_ns; + u64 lat_max_ns; + u64 lat_count; + u64 host_sum_ns; /* notify =E2=86=92 cq_detect (host transit) */ + u64 cb_wait_sum_ns; /* cq_detect =E2=86=92 process (poll sched latency) */ + u64 det_count; + u64 last_cq_detect_ns; /* timestamp of last CQ detection */ + unsigned long lat_last_print; + + /* Completion interval tracking (matches host interval metric) */ + u64 last_complete_ns; /* ktime of last completion */ + u64 ema_cmp_int_ns; /* EMA of completion-to-completion interval */ + struct virtio_scsi_vq req_vqs[]; }; =20 @@ -122,6 +141,7 @@ static void virtscsi_complete_cmd(struct virtio_scsi *v= scsi, void *buf) struct virtio_scsi_cmd *cmd =3D buf; struct scsi_cmnd *sc =3D cmd->sc; struct virtio_scsi_cmd_resp *resp =3D &cmd->resp.cmd; + u64 cb_entry_ns =3D ktime_get_ns(); =20 dev_dbg(&sc->device->sdev_gendev, "cmd %p response %u status %#02x sense_len %u\n", @@ -175,6 +195,88 @@ static void virtscsi_complete_cmd(struct virtio_scsi *= vscsi, void *buf) VIRTIO_SCSI_SENSE_SIZE)); } =20 + /* I/O latency tracking */ + if (cmd->submit_ns) { + u64 now =3D ktime_get_ns(); + u64 lat =3D now - cmd->submit_ns; + unsigned long flags; + + spin_lock_irqsave(&vscsi->lat_lock, flags); + vscsi->lat_sum_ns +=3D lat; + vscsi->lat_count++; + if (lat < vscsi->lat_min_ns || vscsi->lat_min_ns =3D=3D 0) + vscsi->lat_min_ns =3D lat; + if (lat > vscsi->lat_max_ns) + vscsi->lat_max_ns =3D lat; + + /* Completion interval EMA (matches host interval metric) */ + if (vscsi->last_complete_ns) { + u64 cmp_int =3D now - vscsi->last_complete_ns; + + if (vscsi->ema_cmp_int_ns =3D=3D 0) + vscsi->ema_cmp_int_ns =3D cmp_int; + else + vscsi->ema_cmp_int_ns =3D + vscsi->ema_cmp_int_ns - + vscsi->ema_cmp_int_ns / 8 + + cmp_int / 8; + } + vscsi->last_complete_ns =3D now; + + if (cmd->notify_ns && vscsi->last_cq_detect_ns) { + u64 host_lat =3D vscsi->last_cq_detect_ns - + cmd->notify_ns; + u64 cb_wait =3D cb_entry_ns - + vscsi->last_cq_detect_ns; + + if (host_lat < 10ULL * NSEC_PER_SEC) { + vscsi->host_sum_ns +=3D host_lat; + vscsi->cb_wait_sum_ns +=3D cb_wait; + vscsi->det_count++; + } + } + + /* 10s window =E2=80=94 single consolidated output */ + if (time_after(jiffies, vscsi->lat_last_print + + msecs_to_jiffies(10000)) && + vscsi->lat_count > 0) { + u64 avg_lat =3D vscsi->lat_sum_ns / vscsi->lat_count; + u64 host_lat =3D vscsi->det_count > 0 ? + vscsi->host_sum_ns / vscsi->det_count : 0; + u64 poll_lat =3D vscsi->det_count > 0 ? + vscsi->cb_wait_sum_ns / vscsi->det_count : 0; + u64 interval =3D 0; + unsigned int wk_kick; + unsigned long sleep_cnt; + + wk_kick =3D virtqueue_get_sq_notify_stats( + vscsi->req_vqs[0].vq, &interval); + sleep_cnt =3D virtio_sqcq_poll_get_sleep_count(vscsi->vdev); + + dev_info(&vscsi->vdev->dev, + "io_stats: cq=3D%llu avg_lat=3D%lluns host_lat=3D%lluns poll_lat=3D%l= luns min_lat=3D%lluns max_lat=3D%lluns interval=3D%lluns cmp_interval=3D%ll= uns wk_kick=3D%u sleep=3D%lu\n", + vscsi->lat_count, + avg_lat, + host_lat, + poll_lat, + vscsi->lat_min_ns, + vscsi->lat_max_ns, + interval, + vscsi->ema_cmp_int_ns, + wk_kick, + sleep_cnt); + vscsi->lat_sum_ns =3D 0; + vscsi->lat_count =3D 0; + vscsi->lat_min_ns =3D 0; + vscsi->lat_max_ns =3D 0; + vscsi->host_sum_ns =3D 0; + vscsi->cb_wait_sum_ns =3D 0; + vscsi->det_count =3D 0; + vscsi->lat_last_print =3D jiffies; + } + spin_unlock_irqrestore(&vscsi->lat_lock, flags); + } + scsi_done(sc); } =20 @@ -204,6 +306,7 @@ static void virtscsi_req_done(struct virtqueue *vq) int index =3D vq->index - VIRTIO_SCSI_VQ_BASE; struct virtio_scsi_vq *req_vq =3D &vscsi->req_vqs[index]; =20 + vscsi->last_cq_detect_ns =3D virtqueue_get_cq_detect_ns(vq); virtscsi_vq_done(vscsi, req_vq, virtscsi_complete_cmd); }; =20 @@ -514,8 +617,12 @@ static int virtscsi_add_cmd(struct virtio_scsi_vq *vq, =20 spin_unlock_irqrestore(&vq->vq_lock, flags); =20 - if (needs_kick) + if (needs_kick) { + cmd->notify_ns =3D ktime_get_ns(); virtqueue_notify(vq->vq); + } else { + cmd->notify_ns =3D 0; + } return err; } =20 @@ -605,6 +712,7 @@ static enum scsi_qc_status virtscsi_queuecommand(struct= Scsi_Host *shost, } =20 kick =3D (sc->flags & SCMD_LAST) !=3D 0; + cmd->submit_ns =3D ktime_get_ns(); ret =3D virtscsi_add_cmd(req_vq, cmd, req_size, sizeof(cmd->resp.cmd), ki= ck); if (ret =3D=3D -EIO) { cmd->resp.cmd.response =3D VIRTIO_SCSI_S_BAD_TARGET; @@ -895,6 +1003,9 @@ static int virtscsi_init(struct virtio_device *vdev, virtscsi_init_vq(&vscsi->req_vqs[i - VIRTIO_SCSI_VQ_BASE], vqs[i]); =20 + spin_lock_init(&vscsi->lat_lock); + vscsi->lat_last_print =3D jiffies; + virtscsi_config_set(vdev, cdb_size, VIRTIO_SCSI_CDB_SIZE); virtscsi_config_set(vdev, sense_size, VIRTIO_SCSI_SENSE_SIZE); =20 diff --git a/drivers/virtio/Makefile b/drivers/virtio/Makefile index eefcfe90d6b8..769e95ee5be4 100644 --- a/drivers/virtio/Makefile +++ b/drivers/virtio/Makefile @@ -1,5 +1,5 @@ # SPDX-License-Identifier: GPL-2.0 -obj-$(CONFIG_VIRTIO) +=3D virtio.o virtio_ring.o +obj-$(CONFIG_VIRTIO) +=3D virtio.o virtio_ring.o virtio_sqcq_poll.o obj-$(CONFIG_VIRTIO_ANCHOR) +=3D virtio_anchor.o obj-$(CONFIG_VIRTIO_PCI_LIB) +=3D virtio_pci_modern_dev.o obj-$(CONFIG_VIRTIO_PCI_LIB_LEGACY) +=3D virtio_pci_legacy_dev.o diff --git a/drivers/virtio/virtio.c b/drivers/virtio/virtio.c index 5bdc6b82b30b..d440e920f8e6 100644 --- a/drivers/virtio/virtio.c +++ b/drivers/virtio/virtio.c @@ -221,6 +221,16 @@ static int virtio_features_ok(struct virtio_device *de= v) } } =20 + /* SQ/CQ poll only supports split ring; reject packed ring + * to avoid split-specific field access in virtqueue_notify(). + */ + if (virtio_has_feature(dev, VIRTIO_F_SQCQ_POLL) && + virtio_has_feature(dev, VIRTIO_F_RING_PACKED)) { + dev_warn(&dev->dev, + "VIRTIO_F_SQCQ_POLL is incompatible with VIRTIO_F_RING_PACKED\n"); + return -ENODEV; + } + if (!virtio_has_feature(dev, VIRTIO_F_VERSION_1)) return 0; =20 diff --git a/drivers/virtio/virtio_pci_common.c b/drivers/virtio/virtio_pci= _common.c index da97b6a988de..cd22651c73e2 100644 --- a/drivers/virtio/virtio_pci_common.c +++ b/drivers/virtio/virtio_pci_common.c @@ -268,6 +268,9 @@ void vp_del_vqs(struct virtio_device *vdev) struct virtqueue *vq, *n; int i; =20 + /* Stop SQ/CQ poll thread before deleting virtqueues */ + virtio_stop_sqcq_poll(vdev); + list_for_each_entry_safe(vq, n, &vdev->vqs, list) { info =3D vp_is_avq(vdev, vq->index) ? vp_dev->admin_vq.info : vp_dev->vqs[vq->index]; diff --git a/drivers/virtio/virtio_pci_modern.c b/drivers/virtio/virtio_pci= _modern.c index 6d8ae2a6a8ca..7a36ec4584a6 100644 --- a/drivers/virtio/virtio_pci_modern.c +++ b/drivers/virtio/virtio_pci_modern.c @@ -18,6 +18,7 @@ #include #define VIRTIO_PCI_NO_LEGACY #define VIRTIO_RING_NO_LEGACY + #include "virtio_pci_common.h" =20 #define VIRTIO_AVQ_SGS_MAX 4 @@ -378,6 +379,9 @@ static void vp_transport_features(struct virtio_device = *vdev, u64 features) =20 if (features & BIT_ULL(VIRTIO_F_ADMIN_VQ)) __virtio_set_bit(vdev, VIRTIO_F_ADMIN_VQ); + + if (features & BIT_ULL(VIRTIO_F_SQCQ_POLL)) + __virtio_set_bit(vdev, VIRTIO_F_SQCQ_POLL); } =20 static int __vp_check_common_size_one_feature(struct virtio_device *vdev, = u32 fbit, @@ -413,6 +417,9 @@ static int vp_check_common_size(struct virtio_device *v= dev) if (vp_check_common_size_one_feature(vdev, VIRTIO_F_ADMIN_VQ, admin_queue= _num)) return -EINVAL; =20 + if (vp_check_common_size_one_feature(vdev, VIRTIO_F_SQCQ_POLL, cqe_ring_h= i)) + return -EINVAL; + return 0; } =20 @@ -578,6 +585,11 @@ static int vp_active_vq(struct virtqueue *vq, u16 msix= _vec) virtqueue_get_avail_addr(vq), virtqueue_get_used_addr(vq)); =20 + if (virtqueue_sqcq_poll_active(vq)) + vp_modern_queue_address_sqcq(mdev, index, + virtqueue_get_sq_dma_addr(vq), + virtqueue_get_cq_dma_addr(vq)); + if (msix_vec !=3D VIRTIO_MSI_NO_VECTOR) { msix_vec =3D vp_modern_queue_vector(mdev, index, msix_vec); if (msix_vec =3D=3D VIRTIO_MSI_NO_VECTOR) @@ -753,6 +765,27 @@ static int vp_modern_find_vqs(struct virtio_device *vd= ev, unsigned int nvqs, if (rc) return rc; =20 + /* Start SQ/CQ poll thread if feature negotiated */ + if (virtio_has_feature(vdev, VIRTIO_F_SQCQ_POLL)) { + dev_info(&vdev->dev, "VIRTIO_F_SQCQ_POLL negotiated, starting poll threa= d\n"); + rc =3D virtio_start_sqcq_poll(vdev, 1000, 1000); + if (rc) + goto err_del_vqs; + + list_for_each_entry(vq, &vdev->vqs, list) { + if (!virtqueue_sqcq_poll_active(vq)) + continue; + rc =3D virtio_sqcq_poll_register_vq(vdev, vq); + if (rc) { + dev_err(&vdev->dev, + "failed to register vq %d (%s) with poll thread: %d\n", + vq->index, + vq->name ? vq->name : "?", rc); + goto err_stop_poll; + } + } + } + /* Select and activate all queues. Has to be done last: once we do * this, there's no way to go back except reset. */ @@ -760,6 +793,12 @@ static int vp_modern_find_vqs(struct virtio_device *vd= ev, unsigned int nvqs, vp_modern_set_queue_enable(&vp_dev->mdev, vq->index, true); =20 return 0; + +err_stop_poll: + virtio_stop_sqcq_poll(vdev); +err_del_vqs: + vp_del_vqs(vdev); + return rc; } =20 static void del_vq(struct virtio_pci_vq_info *info) diff --git a/drivers/virtio/virtio_pci_modern_dev.c b/drivers/virtio/virtio= _pci_modern_dev.c index 413a8c353463..642b9a58751c 100644 --- a/drivers/virtio/virtio_pci_modern_dev.c +++ b/drivers/virtio/virtio_pci_modern_dev.c @@ -211,6 +211,14 @@ static inline void check_offsets(void) offsetof(struct virtio_pci_modern_common_cfg, admin_queue_index)); BUILD_BUG_ON(VIRTIO_PCI_COMMON_ADM_Q_NUM !=3D offsetof(struct virtio_pci_modern_common_cfg, admin_queue_num)); + BUILD_BUG_ON(VIRTIO_PCI_COMMON_SQE_LO !=3D + offsetof(struct virtio_pci_modern_common_cfg, sqe_ring_lo)); + BUILD_BUG_ON(VIRTIO_PCI_COMMON_SQE_HI !=3D + offsetof(struct virtio_pci_modern_common_cfg, sqe_ring_hi)); + BUILD_BUG_ON(VIRTIO_PCI_COMMON_CQE_LO !=3D + offsetof(struct virtio_pci_modern_common_cfg, cqe_ring_lo)); + BUILD_BUG_ON(VIRTIO_PCI_COMMON_CQE_HI !=3D + offsetof(struct virtio_pci_modern_common_cfg, cqe_ring_hi)); } =20 /* @@ -300,7 +308,7 @@ int vp_modern_probe(struct virtio_pci_modern_device *md= ev) mdev->common =3D vp_modern_map_capability(mdev, common, sizeof(struct virtio_pci_common_cfg), 4, 0, offsetofend(struct virtio_pci_modern_common_cfg, - admin_queue_num), + cqe_ring_hi), &mdev->common_len, NULL); if (!mdev->common) goto err_map_common; @@ -607,6 +615,30 @@ void vp_modern_queue_address(struct virtio_pci_modern_= device *mdev, } EXPORT_SYMBOL_GPL(vp_modern_queue_address); =20 +/* + * vp_modern_queue_address_sqcq - set the SQ/CQ doorbell DMA addresses + * @mdev: the modern virtio-pci device + * @index: the queue index + * @sq_addr: DMA address of the SQ doorbell + * @cq_addr: DMA address of the CQ doorbell + */ +void vp_modern_queue_address_sqcq(struct virtio_pci_modern_device *mdev, + u16 index, u64 sq_addr, u64 cq_addr) +{ + struct virtio_pci_modern_common_cfg __iomem *cfg; + + cfg =3D (struct virtio_pci_modern_common_cfg __iomem *)mdev->common; + + vp_iowrite16(index, &cfg->cfg.queue_select); + + vp_iowrite64_twopart(sq_addr, &cfg->sqe_ring_lo, + &cfg->sqe_ring_hi); + + vp_iowrite64_twopart(cq_addr, &cfg->cqe_ring_lo, + &cfg->cqe_ring_hi); +} +EXPORT_SYMBOL_GPL(vp_modern_queue_address_sqcq); + /* * vp_modern_set_queue_enable - enable a virtqueue * @mdev: the modern virtio-pci device diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c index 335692d41617..a3aaf123ee74 100644 --- a/drivers/virtio/virtio_ring.c +++ b/drivers/virtio/virtio_ring.c @@ -270,10 +270,25 @@ struct vring_virtqueue { bool last_add_time_valid; ktime_t last_add_time; #endif + + /* SQ/CQ polling state (VIRTIO_F_SQCQ_POLL) */ + bool sqcq_poll; /* Is SQ/CQ polling active? */ + struct vring_sq *sq; /* DMA-coherent SQ doorbell */ + struct vring_cq *cq; /* DMA-coherent CQ doorbell */ + dma_addr_t sq_dma_addr; /* DMA address of SQ */ + dma_addr_t cq_dma_addr; /* DMA address of CQ */ + atomic_t sq_mmio_count; /* MMIO kicks (sq_need_wakeup=3D1) */ + u64 last_cq_detect_ns; /* ktime_get_ns() when CQ work first detected */ + u64 sq_last_notify_ns; /* ktime of last sq->idx store */ + u64 sq_ema_interval_ns; /* EMA of sq->idx update interval (ns) */ }; =20 static struct vring_desc_extra *vring_alloc_desc_extra(unsigned int num); static void vring_free(struct virtqueue *_vq); +static int virtio_alloc_sqcq(struct virtio_device *vdev, + struct vring_virtqueue *vq); +static void virtio_free_sqcq(struct virtio_device *vdev, + struct vring_virtqueue *vq); =20 /* * Helpers. @@ -1334,6 +1349,10 @@ static struct virtqueue *__vring_new_virtqueue_split= (unsigned int index, vq->layout =3D virtio_has_feature(vdev, VIRTIO_F_IN_ORDER) ? VQ_LAYOUT_SPLIT_IN_ORDER : VQ_LAYOUT_SPLIT; =20 + vq->sqcq_poll =3D false; + vq->sq =3D NULL; + vq->cq =3D NULL; + if (virtio_has_feature(vdev, VIRTIO_F_ORDER_PLATFORM)) vq->weak_barriers =3D false; =20 @@ -1385,6 +1404,17 @@ static struct virtqueue *vring_create_virtqueue_spli= t( =20 to_vvq(vq)->we_own_ring =3D true; =20 + if (virtio_has_feature(vdev, VIRTIO_F_SQCQ_POLL)) { + if (virtio_alloc_sqcq(vdev, to_vvq(vq))) { + /* vq is on vdev->vqs list; use vring_del_virtqueue() + * to remove and free. vring_free_split() would + * double-free via later vp_del_vqs(). + */ + vring_del_virtqueue(vq); + return NULL; + } + } + return vq; } =20 @@ -2554,6 +2584,10 @@ static struct virtqueue *__vring_new_virtqueue_packe= d(unsigned int index, vq->layout =3D virtio_has_feature(vdev, VIRTIO_F_IN_ORDER) ? VQ_LAYOUT_PACKED_IN_ORDER : VQ_LAYOUT_PACKED; =20 + vq->sqcq_poll =3D false; + vq->sq =3D NULL; + vq->cq =3D NULL; + if (virtio_has_feature(vdev, VIRTIO_F_ORDER_PLATFORM)) vq->weak_barriers =3D false; =20 @@ -3013,6 +3047,9 @@ bool virtqueue_kick_prepare(struct virtqueue *_vq) { struct vring_virtqueue *vq =3D to_vvq(_vq); =20 + if (vq->sqcq_poll) + return true; + return VIRTQUEUE_CALL(vq, kick_prepare); } EXPORT_SYMBOL_GPL(virtqueue_kick_prepare); @@ -3032,6 +3069,45 @@ bool virtqueue_notify(struct virtqueue *_vq) if (unlikely(vq->broken)) return false; =20 + /* SQ/CQ polling mode: update SQ idx instead of MMIO kick. */ + if (vq->sqcq_poll) { + u16 new_sq_idx =3D vq->split.avail_idx_shadow; + u8 need_wakeup; + + /* Release: make descriptor writes visible before sq->idx */ + smp_store_release(&vq->sq->idx, new_sq_idx); + + /* Track submission interval (matches host's interval metric) */ + { + u64 _now =3D ktime_get_ns(); + + if (vq->sq_last_notify_ns) { + u64 _int =3D _now - vq->sq_last_notify_ns; + + if (vq->sq_ema_interval_ns =3D=3D 0) + vq->sq_ema_interval_ns =3D _int; + else + vq->sq_ema_interval_ns =3D + vq->sq_ema_interval_ns - + vq->sq_ema_interval_ns / 8 + + _int / 8; + } + vq->sq_last_notify_ns =3D _now; + } + + /* Acquire: order sq->idx store before reading sq_need_wakeup. + * Prevents stale read on ARM/RISC-V causing missed kick + * when host has set sq_need_wakeup=3D1 and gone to sleep. + */ + need_wakeup =3D smp_load_acquire(&vq->sq->sq_need_wakeup); + + /* Device poll thread is sleeping -- need MMIO kick to wake it */ + if (!need_wakeup) + return true; + atomic_inc(&vq->sq_mmio_count); + /* Fall through to MMIO notify */ + } + /* Prod other side to tell it about changes. */ if (!vq->notify(_vq)) { vq->broken =3D true; @@ -3230,6 +3306,12 @@ irqreturn_t vring_interrupt(int irq, void *_vq) { struct vring_virtqueue *vq =3D to_vvq(_vq); =20 + /* SQ/CQ poll mode: wake the poll thread, let it handle completions */ + if (vq->sqcq_poll) { + virtio_sqcq_poll_wake_thread(vq->vq.vdev); + return IRQ_HANDLED; + } + if (!more_used(vq)) { pr_debug("virtqueue interrupt with no work for %p\n", vq); return IRQ_NONE; @@ -3250,6 +3332,7 @@ irqreturn_t vring_interrupt(int irq, void *_vq) data_race(vq->event_triggered =3D true); =20 pr_debug("virtqueue callback for %p (%p)\n", vq, vq->vq.callback); + if (vq->vq.callback) vq->vq.callback(&vq->vq); =20 @@ -3468,6 +3551,8 @@ static void vring_free(struct virtqueue *_vq) kfree(vq->split.desc_state); kfree(vq->split.desc_extra); } + if (vq->sqcq_poll) + virtio_free_sqcq(vq->vq.vdev, vq); } =20 void vring_del_virtqueue(struct virtqueue *_vq) @@ -3524,6 +3609,8 @@ void vring_transport_features(struct virtio_device *v= dev) break; case VIRTIO_F_IN_ORDER: break; + case VIRTIO_F_SQCQ_POLL: + break; default: /* We don't understand this bit. */ __virtio_clear_bit(vdev, i); @@ -3663,6 +3750,181 @@ dma_addr_t virtqueue_get_used_addr(const struct vir= tqueue *_vq) } EXPORT_SYMBOL_GPL(virtqueue_get_used_addr); =20 +bool virtqueue_sqcq_poll_active(const struct virtqueue *_vq) +{ + const struct vring_virtqueue *vq =3D to_vvq(_vq); + + return vq->sqcq_poll; +} +EXPORT_SYMBOL_GPL(virtqueue_sqcq_poll_active); + +/** + * virtqueue_disable_sqcq_poll - disable SQ/CQ poll mode for a virtqueue. + * @_vq: the virtqueue + * + * Clears sqcq_poll flag; restores MMIO kick + interrupt for this vq. + * Used for control/event queues that must not use the poll thread. + */ +void virtqueue_disable_sqcq_poll(struct virtqueue *_vq) +{ + struct vring_virtqueue *vq =3D to_vvq(_vq); + + vq->sqcq_poll =3D false; +} +EXPORT_SYMBOL_GPL(virtqueue_disable_sqcq_poll); + +dma_addr_t virtqueue_get_sq_dma_addr(const struct virtqueue *_vq) +{ + const struct vring_virtqueue *vq =3D to_vvq(_vq); + + return vq->sq_dma_addr; +} +EXPORT_SYMBOL_GPL(virtqueue_get_sq_dma_addr); + +dma_addr_t virtqueue_get_cq_dma_addr(const struct virtqueue *_vq) +{ + const struct vring_virtqueue *vq =3D to_vvq(_vq); + + return vq->cq_dma_addr; +} +EXPORT_SYMBOL_GPL(virtqueue_get_cq_dma_addr); + +/* Completion detection and sleep/wake accessors for the poll thread. */ + +bool virtqueue_cq_poll_has_work(struct virtqueue *_vq) +{ + struct vring_virtqueue *vq =3D to_vvq(_vq); + + if (!vq->cq) + return false; + + if (more_used(vq)) { + WRITE_ONCE(vq->last_cq_detect_ns, ktime_get_ns()); + return true; + } + + return false; +} +EXPORT_SYMBOL_GPL(virtqueue_cq_poll_has_work); + +unsigned int virtqueue_get_sq_notify_stats(struct virtqueue *_vq, u64 *int= erval) +{ + struct vring_virtqueue *vq =3D to_vvq(_vq); + unsigned int mmio; + + mmio =3D atomic_xchg(&vq->sq_mmio_count, 0); + if (interval) + *interval =3D vq->sq_ema_interval_ns; + return mmio; +} +EXPORT_SYMBOL_GPL(virtqueue_get_sq_notify_stats); + +void virtqueue_cq_set_need_wakeup(struct virtqueue *_vq) +{ + struct vring_virtqueue *vq =3D to_vvq(_vq); + + if (vq->cq) + /* Release: order prior used-ring reads before wakeup flag */ + smp_store_release(&vq->cq->cq_need_wakeup, 1); +} +EXPORT_SYMBOL_GPL(virtqueue_cq_set_need_wakeup); + +void virtqueue_cq_clear_need_wakeup(struct virtqueue *_vq) +{ + struct vring_virtqueue *vq =3D to_vvq(_vq); + + if (vq->cq) + WRITE_ONCE(vq->cq->cq_need_wakeup, 0); +} +EXPORT_SYMBOL_GPL(virtqueue_cq_clear_need_wakeup); + +bool virtqueue_cq_recheck(struct virtqueue *_vq) +{ + struct vring_virtqueue *vq =3D to_vvq(_vq); + + if (!vq->cq) + return false; + + /* Full barrier: order cq_need_wakeup store before used ring read */ + smp_mb(); + return more_used(vq); +} +EXPORT_SYMBOL_GPL(virtqueue_cq_recheck); + +u64 virtqueue_get_cq_detect_ns(const struct virtqueue *_vq) +{ + const struct vring_virtqueue *vq =3D to_vvq(_vq); + + return READ_ONCE(vq->last_cq_detect_ns); +} +EXPORT_SYMBOL_GPL(virtqueue_get_cq_detect_ns); + +/* + * virtio_alloc_sqcq - allocate SQ/CQ doorbell structures for polling mode + * @vdev: the virtio device + * @vq: the vring_virtqueue to allocate for + * + * Called when VIRTIO_F_SQCQ_POLL is negotiated. Returns 0 on success. + */ +static int virtio_alloc_sqcq(struct virtio_device *vdev, + struct vring_virtqueue *vq) +{ + struct device *dma_dev =3D vdev->dev.parent; + + /* Control and event queues use traditional interrupt-driven I/O. */ + if (vq->vq.name && + (strcmp(vq->vq.name, "control") =3D=3D 0 || + strcmp(vq->vq.name, "event") =3D=3D 0)) + return 0; + size_t total =3D sizeof(struct vring_sq) + sizeof(struct vring_cq); + void *buf; + + buf =3D dma_alloc_coherent(dma_dev, total, &vq->sq_dma_addr, GFP_KERNEL); + if (!buf) + return -ENOMEM; + + memset(buf, 0, total); + + vq->sq =3D buf; + vq->cq =3D buf + sizeof(struct vring_sq); + vq->cq_dma_addr =3D vq->sq_dma_addr + sizeof(struct vring_sq); + vq->sqcq_poll =3D true; + atomic_set(&vq->sq_mmio_count, 0); + vq->last_cq_detect_ns =3D 0; + vq->sq_last_notify_ns =3D 0; + vq->sq_ema_interval_ns =3D 0; + + /* Diagnostic: dump SQ/CQ addresses for debug */ + dev_dbg(&vdev->dev, + "vq %s: SQ/CQ doorbells kvirt=3D%p sq_dma=3D0x%llx cq_dma=3D0x%llx\n", + vq->vq.name ? vq->vq.name : "?", + buf, + (unsigned long long)vq->sq_dma_addr, + (unsigned long long)vq->cq_dma_addr); + + return 0; +} + +/* + * virtio_free_sqcq - free SQ/CQ doorbell structures + * @vdev: the virtio device + * @vq: the vring_virtqueue to free for + */ +static void virtio_free_sqcq(struct virtio_device *vdev, + struct vring_virtqueue *vq) +{ + struct device *dma_dev =3D vdev->dev.parent; + size_t total =3D sizeof(struct vring_sq) + sizeof(struct vring_cq); + + if (vq->sq) { + dma_free_coherent(dma_dev, total, + vq->sq, vq->sq_dma_addr); + vq->sq =3D NULL; + vq->cq =3D NULL; + } + vq->sqcq_poll =3D false; +} + /* Only available for split ring */ const struct vring *virtqueue_get_vring(const struct virtqueue *vq) { diff --git a/drivers/virtio/virtio_sqcq_poll.c b/drivers/virtio/virtio_sqcq= _poll.c new file mode 100644 index 000000000000..411493218e93 --- /dev/null +++ b/drivers/virtio/virtio_sqcq_poll.c @@ -0,0 +1,338 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * virtio SQ/CQ polling thread. + * + * Replaces interrupt-based completion notification with a dedicated + * kernel thread that checks the used ring for completions. Uses the + * io_uring sqpoll pattern: busy-wait while active, then set + * NEED_WAKEUP and sleep when idle for a configurable timeout. + * + * Copyright (C) 2026 + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/** + * struct virtio_sqcq_poll_data - per-device poll thread state. + * + * Follows the Host vhost_sqcq_poll_thread pattern. + */ +struct virtio_sqcq_poll_data { + struct mutex lock; + + /* List of virtqueues served by this poll thread */ + struct list_head vq_list; + + struct task_struct *thread; + struct wait_queue_head wait; + + unsigned int idle_timeout_us; /* Sleep timeout (us) */ + unsigned int busy_spin_us; /* Busy-spin budget (us) */ + + unsigned long state; +#define VIRTIO_SQ_THREAD_SHOULD_STOP 0 + + struct completion exited; + struct device *dev; /* for dev_info() logging */ + + /* Poll thread diagnostic statistics */ + unsigned long stat_sleep_count; +}; + +/** + * struct virtio_sqcq_vq_node - links a virtqueue to its poll_data. + */ +struct virtio_sqcq_vq_node { + struct virtqueue *vq; + struct list_head node; +}; + +static void virtio_sqcq_poll_wake(struct virtio_sqcq_poll_data *sqd) +{ + wake_up(&sqd->wait); +} + +/** + * virtio_sqcq_poll_wake_thread - wake the CQ poll thread from IRQ context. + * @vdev: the virtio device + * + * Called by vring_interrupt() when SQ/CQ poll mode is active. Wakes the + * poll thread so it can check the used ring and invoke completion + * callbacks in its own context, instead of handling completions directly + * from the hard-IRQ handler. + */ +void virtio_sqcq_poll_wake_thread(struct virtio_device *vdev) +{ + struct virtio_sqcq_poll_data *sqd =3D vdev->sqcq_poll_data; + + if (sqd) + virtio_sqcq_poll_wake(sqd); +} +EXPORT_SYMBOL_GPL(virtio_sqcq_poll_wake_thread); + +unsigned long virtio_sqcq_poll_get_sleep_count(struct virtio_device *vdev) +{ + struct virtio_sqcq_poll_data *sqd =3D vdev->sqcq_poll_data; + + if (!sqd) + return 0; + + return xchg(&sqd->stat_sleep_count, 0); +} +EXPORT_SYMBOL_GPL(virtio_sqcq_poll_get_sleep_count); + +static int virtio_sqcq_poll_thread(void *data) +{ + struct virtio_sqcq_poll_data *sqd =3D data; + struct virtio_sqcq_vq_node *node; + ktime_t last_io_ts =3D ktime_get(); + u64 spin_budget_ns =3D (u64)sqd->busy_spin_us * NSEC_PER_USEC; + int vq_count =3D 0; + + /* Count registered vqs for the startup message */ + mutex_lock(&sqd->lock); + list_for_each_entry(node, &sqd->vq_list, node) + vq_count++; + mutex_unlock(&sqd->lock); + + dev_info(sqd->dev, "poll thread running, %d vqs, spin=3D%uus, idle=3D%uus= \n", + vq_count, sqd->busy_spin_us, sqd->idle_timeout_us); + + while (!kthread_should_stop()) { + bool did_work =3D false; + + if (test_bit(VIRTIO_SQ_THREAD_SHOULD_STOP, &sqd->state)) { + dev_info(sqd->dev, "poll thread stopping\n"); + break; + } + + if (signal_pending(current)) { + flush_signals(current); + continue; + } + + mutex_lock(&sqd->lock); + + /* Process completions on all registered virtqueues */ + list_for_each_entry(node, &sqd->vq_list, node) { + while (virtqueue_cq_poll_has_work(node->vq)) { + if (node->vq->callback) + node->vq->callback(node->vq); + if (local_softirq_pending()) + do_softirq(); + did_work =3D true; + } + } + + if (did_work) { + mutex_unlock(&sqd->lock); + last_io_ts =3D ktime_get(); + continue; + } + + /* No work =E2=80=94 spin within the time budget from last_io_ts. + * need_resched() + cpu_relax() for cooperative yielding, + * same pattern as Host vhost_sqcq_poll_thread. + */ + if (spin_budget_ns > 0 && + ktime_to_ns(ktime_sub(ktime_get(), last_io_ts)) < + (s64)spin_budget_ns) { + mutex_unlock(&sqd->lock); + if (need_resched()) + cond_resched(); + cpu_relax(); + continue; + } + + /* Spin budget expired =E2=80=94 set cq_need_wakeup on all VQs + * and sleep. Double-check for work before sleeping to + * close the lost-wakeup race. + */ + { + bool found_work =3D false; + DEFINE_WAIT(wait); + + list_for_each_entry(node, &sqd->vq_list, node) + virtqueue_cq_set_need_wakeup(node->vq); + + prepare_to_wait(&sqd->wait, &wait, + TASK_INTERRUPTIBLE); + + list_for_each_entry(node, &sqd->vq_list, node) { + if (virtqueue_cq_recheck(node->vq)) { + found_work =3D true; + break; + } + } + + if (!found_work) { + mutex_unlock(&sqd->lock); + schedule_timeout( + usecs_to_jiffies(sqd->idle_timeout_us)); + sqd->stat_sleep_count++; + mutex_lock(&sqd->lock); + } + + finish_wait(&sqd->wait, &wait); + + list_for_each_entry(node, &sqd->vq_list, node) + virtqueue_cq_clear_need_wakeup(node->vq); + } + + mutex_unlock(&sqd->lock); + } + + complete(&sqd->exited); + return 0; +} + +/** + * virtio_start_sqcq_poll - start the SQ/CQ poll thread for a device. + * @vdev: the virtio device + * @idle_timeout_us: idle timeout in microseconds before sleeping + * @busy_spin_us: busy-spin duration in microseconds before yielding + * + * Returns 0 on success, negative error on failure. + */ +int virtio_start_sqcq_poll(struct virtio_device *vdev, + unsigned int idle_timeout_us, + unsigned int busy_spin_us) +{ + struct virtio_sqcq_poll_data *sqd; + + sqd =3D kzalloc(sizeof(*sqd), GFP_KERNEL); + if (!sqd) + return -ENOMEM; + + mutex_init(&sqd->lock); + init_waitqueue_head(&sqd->wait); + INIT_LIST_HEAD(&sqd->vq_list); + init_completion(&sqd->exited); + sqd->idle_timeout_us =3D idle_timeout_us ? idle_timeout_us : 1000; + sqd->busy_spin_us =3D busy_spin_us; + sqd->dev =3D &vdev->dev; + + sqd->thread =3D kthread_create(virtio_sqcq_poll_thread, sqd, + "virtio-sqcq/%s", dev_name(&vdev->dev)); + if (IS_ERR(sqd->thread)) { + kfree(sqd); + return PTR_ERR(sqd->thread); + } + + vdev->sqcq_poll_data =3D sqd; + wake_up_process(sqd->thread); + + dev_info(sqd->dev, "SQ/CQ poll thread started, idle_timeout=3D%uus, spin_= us=3D%u\n", + sqd->idle_timeout_us, sqd->busy_spin_us); + + return 0; +} +EXPORT_SYMBOL_GPL(virtio_start_sqcq_poll); + +/** + * virtio_stop_sqcq_poll - stop the SQ/CQ poll thread for a device. + * @vdev: the virtio device + */ +void virtio_stop_sqcq_poll(struct virtio_device *vdev) +{ + struct virtio_sqcq_poll_data *sqd =3D vdev->sqcq_poll_data; + struct virtio_sqcq_vq_node *node, *tmp; + + if (!sqd) + return; + + set_bit(VIRTIO_SQ_THREAD_SHOULD_STOP, &sqd->state); + + dev_info(sqd->dev, "SQ/CQ poll thread stopping\n"); + virtio_sqcq_poll_wake(sqd); + wait_for_completion(&sqd->exited); + + /* Reap kthread task_struct to avoid zombie thread leak. + * Thread already exited via complete(); safe to call on + * already-exited kthread. + */ + kthread_stop(sqd->thread); + + /* Free all vq nodes */ + list_for_each_entry_safe(node, tmp, &sqd->vq_list, node) { + list_del(&node->node); + kfree(node); + } + + vdev->sqcq_poll_data =3D NULL; + kfree(sqd); +} +EXPORT_SYMBOL_GPL(virtio_stop_sqcq_poll); + +/** + * virtio_sqcq_poll_register_vq - register a virtqueue with the poll threa= d. + * @vdev: the virtio device + * @vq: the virtqueue to register + * + * The virtqueue must have VIRTIO_F_SQCQ_POLL active. + * Returns 0 on success, negative error on failure. + */ +int virtio_sqcq_poll_register_vq(struct virtio_device *vdev, + struct virtqueue *vq) +{ + struct virtio_sqcq_poll_data *sqd =3D vdev->sqcq_poll_data; + struct virtio_sqcq_vq_node *node; + + if (!sqd || !virtqueue_sqcq_poll_active(vq)) + return -EINVAL; + + node =3D kmalloc(sizeof(*node), GFP_KERNEL); + if (!node) + return -ENOMEM; + + node->vq =3D vq; + + mutex_lock(&sqd->lock); + list_add_tail(&node->node, &sqd->vq_list); + mutex_unlock(&sqd->lock); + + dev_info(sqd->dev, "vq index=3D%d (%s) registered with poll thread\n", + vq->index, vq->name ? vq->name : "?"); + + return 0; +} +EXPORT_SYMBOL_GPL(virtio_sqcq_poll_register_vq); + +/** + * virtio_sqcq_poll_unregister_vq - unregister a virtqueue from the poll t= hread. + * @vdev: the virtio device + * @vq: the virtqueue to unregister + */ +void virtio_sqcq_poll_unregister_vq(struct virtio_device *vdev, + struct virtqueue *vq) +{ + struct virtio_sqcq_poll_data *sqd =3D vdev->sqcq_poll_data; + struct virtio_sqcq_vq_node *node, *tmp; + + if (!sqd) + return; + + mutex_lock(&sqd->lock); + list_for_each_entry_safe(node, tmp, &sqd->vq_list, node) { + if (node->vq =3D=3D vq) { + list_del(&node->node); + kfree(node); + break; + } + } + mutex_unlock(&sqd->lock); +} +EXPORT_SYMBOL_GPL(virtio_sqcq_poll_unregister_vq); diff --git a/include/linux/virtio.h b/include/linux/virtio.h index 3bbc4cb6a672..8300e0946719 100644 --- a/include/linux/virtio.h +++ b/include/linux/virtio.h @@ -125,6 +125,28 @@ dma_addr_t virtqueue_get_desc_addr(const struct virtqu= eue *vq); dma_addr_t virtqueue_get_avail_addr(const struct virtqueue *vq); dma_addr_t virtqueue_get_used_addr(const struct virtqueue *vq); =20 +/* SQ/CQ polling (VIRTIO_F_SQCQ_POLL) */ +bool virtqueue_sqcq_poll_active(const struct virtqueue *vq); +void virtqueue_disable_sqcq_poll(struct virtqueue *vq); +dma_addr_t virtqueue_get_sq_dma_addr(const struct virtqueue *vq); +dma_addr_t virtqueue_get_cq_dma_addr(const struct virtqueue *vq); +u64 virtqueue_get_cq_detect_ns(const struct virtqueue *vq); +int virtio_start_sqcq_poll(struct virtio_device *vdev, + unsigned int idle_timeout_us, + unsigned int busy_spin_us); +void virtio_stop_sqcq_poll(struct virtio_device *vdev); +void virtio_sqcq_poll_wake_thread(struct virtio_device *vdev); +int virtio_sqcq_poll_register_vq(struct virtio_device *vdev, + struct virtqueue *vq); +void virtio_sqcq_poll_unregister_vq(struct virtio_device *vdev, + struct virtqueue *vq); +bool virtqueue_cq_poll_has_work(struct virtqueue *vq); +unsigned int virtqueue_get_sq_notify_stats(struct virtqueue *vq, u64 *inte= rval); +unsigned long virtio_sqcq_poll_get_sleep_count(struct virtio_device *vdev); +void virtqueue_cq_set_need_wakeup(struct virtqueue *vq); +void virtqueue_cq_clear_need_wakeup(struct virtqueue *vq); +bool virtqueue_cq_recheck(struct virtqueue *vq); + int virtqueue_resize(struct virtqueue *vq, u32 num, void (*recycle)(struct virtqueue *vq, void *buf), void (*recycle_done)(struct virtqueue *vq)); @@ -165,6 +187,8 @@ struct virtio_admin_cmd { * @debugfs_dir: debugfs directory entry. * @debugfs_filter_features: features to be filtered set by debugfs. */ +struct virtio_sqcq_poll_data; + struct virtio_device { int index; bool failed; @@ -182,6 +206,7 @@ struct virtio_device { VIRTIO_DECLARE_FEATURES(features); void *priv; union virtio_map vmap; + struct virtio_sqcq_poll_data *sqcq_poll_data; /* SQ/CQ poll thread state = */ #ifdef CONFIG_VIRTIO_DEBUG struct dentry *debugfs_dir; u64 debugfs_filter_features[VIRTIO_FEATURES_U64S]; diff --git a/include/linux/virtio_pci_modern.h b/include/linux/virtio_pci_m= odern.h index 9a3f2fc53bd6..36810f9d23f3 100644 --- a/include/linux/virtio_pci_modern.h +++ b/include/linux/virtio_pci_modern.h @@ -145,6 +145,8 @@ u16 vp_modern_config_vector(struct virtio_pci_modern_de= vice *mdev, void vp_modern_queue_address(struct virtio_pci_modern_device *mdev, u16 index, u64 desc_addr, u64 driver_addr, u64 device_addr); +void vp_modern_queue_address_sqcq(struct virtio_pci_modern_device *mdev, + u16 index, u64 sq_addr, u64 cq_addr); void vp_modern_set_queue_enable(struct virtio_pci_modern_device *mdev, u16 idx, bool enable); bool vp_modern_get_queue_enable(struct virtio_pci_modern_device *mdev, --=20 2.34.1 From nobody Sat Jul 25 11:03:39 2026 Received: from m16.mail.163.com (m16.mail.163.com [220.197.31.5]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 77857427FBD; Mon, 20 Jul 2026 14:12:10 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=220.197.31.5 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1784556734; cv=none; b=iYw6185W9efek7HIaZxSOMHuaAEZhG5ANqUOGp+lhAyT8TER//EXE5RYqPJ44pmaqVjwXNmqBqWK9TmDhJG5IqMd/9iHULVwEu5IguN62VOhSJeh5wEHot4EdJMRnaXk04XgmCuSBgtWIZ0OpyUzC5J/OX2HH/swAqG5XoPLOAs= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1784556734; c=relaxed/simple; bh=WbxGY50Ku1zh9Lld1IrD1p/ARRiklGi+UNXh7uJ52tU=; h=From:To:Cc:Subject:Date:Message-Id:In-Reply-To:References: MIME-Version; b=ZmhuA03TsKdJWcKFD/zA1KhjuWtKQVUFnscIV5xt1z3HR3AIkO8pOB5NiO4DpdosmpEhkG7u+c/x9NpJmDes6SaLutKXo5Sz9PTKQDhV2BEWvrFCqwmoouH+Z/0ov4zEHkhkDQp1Qd+cQhETzY/Hyg4ndELKacNdKZEomo2Sw4c= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=163.com; spf=pass smtp.mailfrom=163.com; dkim=pass (1024-bit key) header.d=163.com header.i=@163.com header.b=iEkuGX+g; arc=none smtp.client-ip=220.197.31.5 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=163.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=163.com Authentication-Results: smtp.subspace.kernel.org; dkim=pass (1024-bit key) header.d=163.com header.i=@163.com header.b="iEkuGX+g" DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=163.com; s=s110527; h=From:To:Subject:Date:Message-Id:MIME-Version; bh=ww 3/iniGVo4aD31R6OEa75hQHppDnik/1A2aoz8O7Yc=; b=iEkuGX+geVnx/VJ04O o66ydAr82RroxR7QFtZ5bdY7mv9Ec48daeGB/tflm4zBf0QQkv0DLalHV71LZ1pj azKwBZmSdQH+ZaKh1v2Gh8K8g+tLUFPs617gaPsqwPBBYWNSKInjfH9UslwdQDka fRZZ9P1vVWJKMJNCmE0xwXAOE= Received: from localhost.localdomain (unknown []) by gzsmtp3 (Coremail) with SMTP id PigvCgD3Xw9gLF5qWUvpIg--.12525S6; Mon, 20 Jul 2026 22:10:44 +0800 (CST) From: "rom.wang" To: mst@redhat.com, jasowangio@gmail.com, eperezma@redhat.com, stefanha@redhat.com Cc: virtualization@lists.linux.dev, linux-kernel@vger.kernel.org, linux-scsi@vger.kernel.org, kvm@vger.kernel.org, qemu-devel@nongnu.org, Yufeng Wang Subject: [RFC PATCH 4/4] virtio: add SQ/CQ polling mode support for vhost-scsi Date: Mon, 20 Jul 2026 22:10:40 +0800 Message-Id: <20260720141040.181946-5-r4o5m6e8o@163.com> X-Mailer: git-send-email 2.34.1 In-Reply-To: <20260720141040.181946-1-r4o5m6e8o@163.com> References: <20260720141040.181946-1-r4o5m6e8o@163.com> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable X-CM-TRANSID: PigvCgD3Xw9gLF5qWUvpIg--.12525S6 X-Coremail-Antispam: 1Uf129KBjvAXoWfGF45CFyrZFyfGFWrur1rZwb_yoW8AFyUGo ZxKF1Iqr4DJw1UZrnI9r1xtFW5JF1vgF4UAr15u39xKay2y343KF95KanxAa4rAFyrtF18 GFyfW3yrJay0qFZ5n29KB7ZKAUJUUUU8529EdanIXcx71UUUUU7v73VFW2AGmfu7bjvjm3 AaLaJ3UbIYCTnIWIevJa73UjIFyTuYvjxU2LIDUUUUU X-CM-SenderInfo: 3uurkzkwhy0qqrwthudrp/xtbC8ARWy2peLGQf3AAA3W Content-Type: text/plain; charset="utf-8" From: Yufeng Wang Implement the QEMU-side bridge layer for virtio SQ/CQ polling mode, which replaces interrupt-based virtio notifications with io_uring-style doorbell polling to eliminate VM exits in high-throughput scenarios. UAPI headers: sync VIRTIO_F_SQCQ_POLL (bit 42), PCI common config offsets (64-76), struct vring_sq/vring_cq, struct vhost_vring_sqcq_addr, and VHOST_SET_VRING_SQCQ_ADDR ioctl (0x27) from Linux kernel. VirtIO core: add sq/cq fields to VRing, with getter/setter functions. PCI transport: handle SQE/CQE register read/write in common config, gate SQ/CQ address forwarding on VIRTIO_F_SQCQ_POLL negotiation in Q_ENABLE handler. Vhost backend: add vhost_set_vring_sqcq_addr_op to VhostOps, implement kernel backend via VHOST_SET_VRING_SQCQ_ADDR ioctl. Vhost core: map SQ/CQ GPAs to HVAs in vhost_virtqueue_start() after VHOST_SET_VRING_ADDR, call the new ioctl before VHOST_SET_VRING_KICK; unmap in do_vhost_virtqueue_stop(). vhost-scsi: advertise VIRTIO_F_SQCQ_POLL in kernel_feature_bits[]. Signed-off-by: Yufeng Wang --- hw/scsi/vhost-scsi.c | 4 ++ hw/virtio/vhost-backend.c | 7 +++ hw/virtio/vhost.c | 54 +++++++++++++++++++ hw/virtio/virtio-pci.c | 31 +++++++++++ hw/virtio/virtio.c | 18 +++++++ include/hw/virtio/vhost-backend.h | 4 ++ include/hw/virtio/vhost.h | 6 +++ include/hw/virtio/virtio-pci.h | 2 + include/hw/virtio/virtio.h | 3 ++ include/standard-headers/linux/vhost_types.h | 7 +++ .../standard-headers/linux/virtio_config.h | 5 +- include/standard-headers/linux/virtio_pci.h | 4 ++ include/standard-headers/linux/virtio_ring.h | 30 +++++++++++ linux-headers/linux/vhost.h | 4 ++ 14 files changed, 178 insertions(+), 1 deletion(-) diff --git a/hw/scsi/vhost-scsi.c b/hw/scsi/vhost-scsi.c index 699863c..bb7f3e8 100644 --- a/hw/scsi/vhost-scsi.c +++ b/hw/scsi/vhost-scsi.c @@ -40,6 +40,7 @@ static const int kernel_feature_bits[] =3D { VIRTIO_F_RING_RESET, VIRTIO_F_IN_ORDER, VIRTIO_F_NOTIFICATION_DATA, + VIRTIO_F_SQCQ_POLL, VHOST_INVALID_FEATURE_BIT }; =20 @@ -361,6 +362,9 @@ static const Property vhost_scsi_properties[] =3D { DEFINE_PROP_BIT64("hotplug", VHostSCSICommon, host_features, VIRTIO_SCSI_F_HOTPLUG, false), + DEFINE_PROP_BIT64("sqcq_poll", VHostSCSICommon, host_features, + VIRTIO_F_SQCQ_POLL, + true), DEFINE_PROP_BOOL("migratable", VHostSCSICommon, migratable, false), DEFINE_PROP_BOOL("worker_per_virtqueue", VirtIOSCSICommon, conf.worker_per_virtqueue, false), diff --git a/hw/virtio/vhost-backend.c b/hw/virtio/vhost-backend.c index 4367db0..a71b30b 100644 --- a/hw/virtio/vhost-backend.c +++ b/hw/virtio/vhost-backend.c @@ -115,6 +115,12 @@ static int vhost_kernel_set_vring_addr(struct vhost_de= v *dev, return vhost_kernel_call(dev, VHOST_SET_VRING_ADDR, addr); } =20 +static int vhost_kernel_set_vring_sqcq_addr(struct vhost_dev *dev, + struct vhost_vring_sqcq_addr *= addr) +{ + return vhost_kernel_call(dev, VHOST_SET_VRING_SQCQ_ADDR, addr); +} + static int vhost_kernel_set_vring_endian(struct vhost_dev *dev, struct vhost_vring_state *ring) { @@ -368,6 +374,7 @@ const VhostOps kernel_ops =3D { .vhost_set_log_base =3D vhost_kernel_set_log_base, .vhost_set_mem_table =3D vhost_kernel_set_mem_table, .vhost_set_vring_addr =3D vhost_kernel_set_vring_addr, + .vhost_set_vring_sqcq_addr =3D vhost_kernel_set_vring_sqcq_addr, .vhost_set_vring_endian =3D vhost_kernel_set_vring_endian, .vhost_set_vring_num =3D vhost_kernel_set_vring_num, .vhost_set_vring_base =3D vhost_kernel_set_vring_base, diff --git a/hw/virtio/vhost.c b/hw/virtio/vhost.c index 31e9704..8c82f98 100644 --- a/hw/virtio/vhost.c +++ b/hw/virtio/vhost.c @@ -22,6 +22,7 @@ #include "qemu/memfd.h" #include "qemu/log.h" #include "standard-headers/linux/vhost_types.h" +#include "standard-headers/linux/virtio_ring.h" #include "hw/virtio/virtio-bus.h" #include "hw/mem/memory-device.h" #include "migration/blocker.h" @@ -1337,6 +1338,49 @@ int vhost_virtqueue_start(struct vhost_dev *dev, goto fail_alloc; } =20 + /* SQ/CQ polling: map and pass to vhost if feature negotiated */ + vq->sq =3D NULL; + vq->cq =3D NULL; + vq->sq_phys =3D virtio_queue_get_sq_addr(vdev, idx); + vq->cq_phys =3D virtio_queue_get_cq_addr(vdev, idx); + + if (vq->sq_phys && vq->cq_phys && + dev->vhost_ops->vhost_set_vring_sqcq_addr && + virtio_has_feature(vdev->guest_features, VIRTIO_F_SQCQ_POLL)) { + struct vhost_vring_sqcq_addr sqcq_addr; + + l =3D sizeof(struct vring_sq); + vq->sq =3D vhost_memory_map(dev, vq->sq_phys, &l, false); + if (!vq->sq) { + r =3D -ENOMEM; + goto fail_alloc; + } + vq->sq_size =3D l; + + l =3D sizeof(struct vring_cq); + vq->cq =3D vhost_memory_map(dev, vq->cq_phys, &l, false); + if (!vq->cq) { + r =3D -ENOMEM; + vhost_memory_unmap(dev, vq->sq, vq->sq_size, 0, 0); + goto fail_alloc; + } + vq->cq_size =3D l; + + memset(&sqcq_addr, 0, sizeof(sqcq_addr)); + sqcq_addr.index =3D vhost_vq_index; + sqcq_addr.sq_user_addr =3D (uint64_t)(unsigned long)vq->sq; + sqcq_addr.cq_user_addr =3D (uint64_t)(unsigned long)vq->cq; + r =3D dev->vhost_ops->vhost_set_vring_sqcq_addr(dev, &sqcq_addr); + if (r < 0) { + VHOST_OPS_DEBUG(r, "vhost_set_vring_sqcq_addr failed"); + vhost_memory_unmap(dev, vq->cq, vq->cq_size, 0, 0); + vhost_memory_unmap(dev, vq->sq, vq->sq_size, 0, 0); + vq->sq =3D NULL; + vq->cq =3D NULL; + goto fail_alloc; + } + } + file.fd =3D event_notifier_get_fd(virtio_queue_get_host_notifier(vvq)); r =3D dev->vhost_ops->vhost_set_vring_kick(dev, &file); if (r) { @@ -1425,6 +1469,16 @@ static int do_vhost_virtqueue_stop(struct vhost_dev = *dev, vhost_vq_index); } =20 + /* Unmap SQ/CQ if mapped */ + if (vq->sq) { + vhost_memory_unmap(dev, vq->sq, vq->sq_size, 0, 0); + vq->sq =3D NULL; + } + if (vq->cq) { + vhost_memory_unmap(dev, vq->cq, vq->cq_size, 0, 0); + vq->cq =3D NULL; + } + vhost_memory_unmap(dev, vq->used, virtio_queue_get_used_size(vdev, idx= ), 1, virtio_queue_get_used_size(vdev, idx)); vhost_memory_unmap(dev, vq->avail, virtio_queue_get_avail_size(vdev, i= dx), diff --git a/hw/virtio/virtio-pci.c b/hw/virtio/virtio-pci.c index b273eb2..c88cba3 100644 --- a/hw/virtio/virtio-pci.c +++ b/hw/virtio/virtio-pci.c @@ -1623,6 +1623,18 @@ static uint64_t virtio_pci_common_read(void *opaque,= hwaddr addr, case VIRTIO_PCI_COMMON_Q_RESET: val =3D proxy->vqs[vdev->queue_sel].reset; break; + case VIRTIO_PCI_COMMON_SQE_LO: + val =3D proxy->vqs[vdev->queue_sel].sq[0]; + break; + case VIRTIO_PCI_COMMON_SQE_HI: + val =3D proxy->vqs[vdev->queue_sel].sq[1]; + break; + case VIRTIO_PCI_COMMON_CQE_LO: + val =3D proxy->vqs[vdev->queue_sel].cq[0]; + break; + case VIRTIO_PCI_COMMON_CQE_HI: + val =3D proxy->vqs[vdev->queue_sel].cq[1]; + break; default: val =3D 0; } @@ -1727,6 +1739,13 @@ static void virtio_pci_common_write(void *opaque, hw= addr addr, proxy->vqs[vdev->queue_sel].avail[0], ((uint64_t)proxy->vqs[vdev->queue_sel].used[1]) << = 32 | proxy->vqs[vdev->queue_sel].used[0]); + if (virtio_has_feature(vdev->guest_features, VIRTIO_F_SQCQ_POL= L)) { + virtio_queue_set_sqcq(vdev, vdev->queue_sel, + ((uint64_t)proxy->vqs[vdev->queue_sel].sq[1] << 32) | + proxy->vqs[vdev->queue_sel].sq[0], + ((uint64_t)proxy->vqs[vdev->queue_sel].cq[1] << 32) | + proxy->vqs[vdev->queue_sel].cq[0]); + } proxy->vqs[vdev->queue_sel].enabled =3D 1; proxy->vqs[vdev->queue_sel].reset =3D 0; virtio_queue_enable(vdev, vdev->queue_sel); @@ -1752,6 +1771,18 @@ static void virtio_pci_common_write(void *opaque, hw= addr addr, case VIRTIO_PCI_COMMON_Q_USEDHI: proxy->vqs[vdev->queue_sel].used[1] =3D val; break; + case VIRTIO_PCI_COMMON_SQE_LO: + proxy->vqs[vdev->queue_sel].sq[0] =3D val; + break; + case VIRTIO_PCI_COMMON_SQE_HI: + proxy->vqs[vdev->queue_sel].sq[1] =3D val; + break; + case VIRTIO_PCI_COMMON_CQE_LO: + proxy->vqs[vdev->queue_sel].cq[0] =3D val; + break; + case VIRTIO_PCI_COMMON_CQE_HI: + proxy->vqs[vdev->queue_sel].cq[1] =3D val; + break; case VIRTIO_PCI_COMMON_Q_RESET: if (val =3D=3D 1) { proxy->vqs[vdev->queue_sel].reset =3D 1; diff --git a/hw/virtio/virtio.c b/hw/virtio/virtio.c index 3dc9423..42e39d9 100644 --- a/hw/virtio/virtio.c +++ b/hw/virtio/virtio.c @@ -111,6 +111,8 @@ typedef struct VRing hwaddr desc; hwaddr avail; hwaddr used; + hwaddr sq; + hwaddr cq; VRingMemoryRegionCaches *caches; } VRing; =20 @@ -2400,6 +2402,12 @@ void virtio_queue_set_rings(VirtIODevice *vdev, int = n, hwaddr desc, virtio_init_region_cache(vdev, n); } =20 +void virtio_queue_set_sqcq(VirtIODevice *vdev, int n, hwaddr sq, hwaddr cq) +{ + vdev->vq[n].vring.sq =3D sq; + vdev->vq[n].vring.cq =3D cq; +} + void virtio_queue_set_num(VirtIODevice *vdev, int n, int num) { /* Don't allow guest to flip queue between existent and @@ -3661,6 +3669,16 @@ hwaddr virtio_queue_get_used_addr(VirtIODevice *vdev= , int n) return vdev->vq[n].vring.used; } =20 +hwaddr virtio_queue_get_sq_addr(VirtIODevice *vdev, int n) +{ + return vdev->vq[n].vring.sq; +} + +hwaddr virtio_queue_get_cq_addr(VirtIODevice *vdev, int n) +{ + return vdev->vq[n].vring.cq; +} + hwaddr virtio_queue_get_desc_size(VirtIODevice *vdev, int n) { return sizeof(VRingDesc) * vdev->vq[n].vring.num; diff --git a/include/hw/virtio/vhost-backend.h b/include/hw/virtio/vhost-ba= ckend.h index ff94fa1..911cd15 100644 --- a/include/hw/virtio/vhost-backend.h +++ b/include/hw/virtio/vhost-backend.h @@ -45,6 +45,7 @@ struct vhost_memory; struct vhost_vring_file; struct vhost_vring_state; struct vhost_vring_addr; +struct vhost_vring_sqcq_addr; struct vhost_vring_worker; struct vhost_worker_state; struct vhost_scsi_target; @@ -71,6 +72,8 @@ typedef int (*vhost_set_mem_table_op)(struct vhost_dev *d= ev, struct vhost_memory *mem); typedef int (*vhost_set_vring_addr_op)(struct vhost_dev *dev, struct vhost_vring_addr *addr); +typedef int (*vhost_set_vring_sqcq_addr_op)(struct vhost_dev *dev, + struct vhost_vring_sqcq_addr *= addr); typedef int (*vhost_set_vring_endian_op)(struct vhost_dev *dev, struct vhost_vring_state *ring); typedef int (*vhost_set_vring_num_op)(struct vhost_dev *dev, @@ -178,6 +181,7 @@ typedef struct VhostOps { vhost_set_log_base_op vhost_set_log_base; vhost_set_mem_table_op vhost_set_mem_table; vhost_set_vring_addr_op vhost_set_vring_addr; + vhost_set_vring_sqcq_addr_op vhost_set_vring_sqcq_addr; vhost_set_vring_endian_op vhost_set_vring_endian; vhost_set_vring_num_op vhost_set_vring_num; vhost_set_vring_base_op vhost_set_vring_base; diff --git a/include/hw/virtio/vhost.h b/include/hw/virtio/vhost.h index 08bbb4d..65056a0 100644 --- a/include/hw/virtio/vhost.h +++ b/include/hw/virtio/vhost.h @@ -27,6 +27,8 @@ struct vhost_virtqueue { void *desc; void *avail; void *used; + void *sq; /* Mapped SQ doorbell */ + void *cq; /* Mapped CQ doorbell */ int num; unsigned long long desc_phys; unsigned desc_size; @@ -34,6 +36,10 @@ struct vhost_virtqueue { unsigned avail_size; unsigned long long used_phys; unsigned used_size; + unsigned long long sq_phys; /* GPA of SQ doorbell */ + unsigned long long cq_phys; /* GPA of CQ doorbell */ + unsigned sq_size; /* Size of mapped SQ region */ + unsigned cq_size; /* Size of mapped CQ region */ EventNotifier masked_notifier; EventNotifier error_notifier; EventNotifier masked_config_notifier; diff --git a/include/hw/virtio/virtio-pci.h b/include/hw/virtio/virtio-pci.h index 6397529..37103c7 100644 --- a/include/hw/virtio/virtio-pci.h +++ b/include/hw/virtio/virtio-pci.h @@ -122,6 +122,8 @@ typedef struct VirtIOPCIQueue { uint32_t desc[2]; uint32_t avail[2]; uint32_t used[2]; + uint32_t sq[2]; + uint32_t cq[2]; } VirtIOPCIQueue; =20 struct VirtIOPCIProxy { diff --git a/include/hw/virtio/virtio.h b/include/hw/virtio/virtio.h index 27cd98d..29f21f4 100644 --- a/include/hw/virtio/virtio.h +++ b/include/hw/virtio/virtio.h @@ -361,6 +361,7 @@ int virtio_queue_get_max_num(VirtIODevice *vdev, int n); int virtio_get_num_queues(VirtIODevice *vdev); void virtio_queue_set_rings(VirtIODevice *vdev, int n, hwaddr desc, hwaddr avail, hwaddr used); +void virtio_queue_set_sqcq(VirtIODevice *vdev, int n, hwaddr sq, hwaddr cq= ); void virtio_queue_update_rings(VirtIODevice *vdev, int n); void virtio_init_region_cache(VirtIODevice *vdev, int n); void virtio_queue_set_align(VirtIODevice *vdev, int n, int align); @@ -408,6 +409,8 @@ bool virtio_queue_enabled_legacy(VirtIODevice *vdev, in= t n); bool virtio_queue_enabled(VirtIODevice *vdev, int n); hwaddr virtio_queue_get_avail_addr(VirtIODevice *vdev, int n); hwaddr virtio_queue_get_used_addr(VirtIODevice *vdev, int n); +hwaddr virtio_queue_get_sq_addr(VirtIODevice *vdev, int n); +hwaddr virtio_queue_get_cq_addr(VirtIODevice *vdev, int n); hwaddr virtio_queue_get_desc_size(VirtIODevice *vdev, int n); hwaddr virtio_queue_get_avail_size(VirtIODevice *vdev, int n); hwaddr virtio_queue_get_used_size(VirtIODevice *vdev, int n); diff --git a/include/standard-headers/linux/vhost_types.h b/include/standar= d-headers/linux/vhost_types.h index 79b53a9..ea96feb 100644 --- a/include/standard-headers/linux/vhost_types.h +++ b/include/standard-headers/linux/vhost_types.h @@ -151,6 +151,13 @@ struct vhost_scsi_target { unsigned short reserved; }; =20 +/* SQ/CQ polling mode */ +struct vhost_vring_sqcq_addr { + unsigned int index; + __u64 sq_user_addr; /* SQ doorbell HVA */ + __u64 cq_user_addr; /* CQ doorbell HVA */ +}; + /* VHOST_VDPA specific definitions */ =20 struct vhost_vdpa_config { diff --git a/include/standard-headers/linux/virtio_config.h b/include/stand= ard-headers/linux/virtio_config.h index 45be0fa..d73938e 100644 --- a/include/standard-headers/linux/virtio_config.h +++ b/include/standard-headers/linux/virtio_config.h @@ -52,7 +52,7 @@ * rest are per-device feature bits. */ #define VIRTIO_TRANSPORT_F_START 28 -#define VIRTIO_TRANSPORT_F_END 42 +#define VIRTIO_TRANSPORT_F_END 43 =20 #ifndef VIRTIO_CONFIG_NO_LEGACY /* Do we get callbacks when the ring is completely used, even if we've @@ -118,4 +118,7 @@ */ #define VIRTIO_F_ADMIN_VQ 41 =20 +/* SQ/CQ polling mode for io_uring-style doorbell polling */ +#define VIRTIO_F_SQCQ_POLL 42 + #endif /* _LINUX_VIRTIO_CONFIG_H */ diff --git a/include/standard-headers/linux/virtio_pci.h b/include/standard= -headers/linux/virtio_pci.h index 4c82513..cc585b6 100644 --- a/include/standard-headers/linux/virtio_pci.h +++ b/include/standard-headers/linux/virtio_pci.h @@ -235,6 +235,10 @@ struct virtio_pci_cfg_cap { #define VIRTIO_PCI_COMMON_Q_RESET 58 #define VIRTIO_PCI_COMMON_ADM_Q_IDX 60 #define VIRTIO_PCI_COMMON_ADM_Q_NUM 62 +#define VIRTIO_PCI_COMMON_SQE_LO 64 +#define VIRTIO_PCI_COMMON_SQE_HI 68 +#define VIRTIO_PCI_COMMON_CQE_LO 72 +#define VIRTIO_PCI_COMMON_CQE_HI 76 =20 #endif /* VIRTIO_PCI_NO_MODERN */ =20 diff --git a/include/standard-headers/linux/virtio_ring.h b/include/standar= d-headers/linux/virtio_ring.h index 22f6eb8..520b758 100644 --- a/include/standard-headers/linux/virtio_ring.h +++ b/include/standard-headers/linux/virtio_ring.h @@ -245,4 +245,34 @@ struct vring_packed_desc { uint16_t flags; }; =20 +/* + * struct vring_sq - Submission Queue doorbell for polling mode. + * Placed alongside the standard virtio split ring to enable + * kick-less notification from guest to host. + * + * @idx: Producer index (guest updates, corresponds to avail->idx) + * @sq_need_wakeup: device requests guest to kick when sleeping + * @reserved: Reserved for future use, pad to cache-line + */ +struct vring_sq { + __virtio64 idx; + uint8_t sq_need_wakeup; + uint8_t reserved[55]; +} __attribute__((aligned(128))); + +/* + * struct vring_cq - Completion Queue doorbell for polling mode. + * Placed alongside the standard virtio split ring to enable + * interrupt-less notification from host to guest. + * + * @idx: Producer index (device updates, corresponds to used->idx) + * @cq_need_wakeup: guest requests host to signal when sleeping + * @reserved: Reserved for future use, pad to cache-line + */ +struct vring_cq { + __virtio64 idx; + uint8_t cq_need_wakeup; + uint8_t reserved[55]; +} __attribute__((aligned(128))); + #endif /* _LINUX_VIRTIO_RING_H */ diff --git a/linux-headers/linux/vhost.h b/linux-headers/linux/vhost.h index c57674a..746d71f 100644 --- a/linux-headers/linux/vhost.h +++ b/linux-headers/linux/vhost.h @@ -123,6 +123,10 @@ #define VHOST_SET_BACKEND_FEATURES _IOW(VHOST_VIRTIO, 0x25, __u64) #define VHOST_GET_BACKEND_FEATURES _IOR(VHOST_VIRTIO, 0x26, __u64) =20 +/* Set SQ/CQ doorbell addresses for polling mode */ +#define VHOST_SET_VRING_SQCQ_ADDR _IOW(VHOST_VIRTIO, 0x27, \ + struct vhost_vring_sqcq_addr) + /* VHOST_NET specific defines */ =20 /* Attach virtio net ring to a raw socket, or tap device. --=20 2.34.1