From nobody Fri May 17 03:00:45 2024 Delivered-To: importer@patchew.org Received-SPF: pass (zoho.com: domain of gnu.org designates 208.118.235.17 as permitted sender) client-ip=208.118.235.17; envelope-from=qemu-devel-bounces+importer=patchew.org@nongnu.org; helo=lists.gnu.org; Authentication-Results: mx.zohomail.com; spf=pass (zoho.com: domain of gnu.org designates 208.118.235.17 as permitted sender) smtp.mailfrom=qemu-devel-bounces+importer=patchew.org@nongnu.org Return-Path: Received: from lists.gnu.org (208.118.235.17 [208.118.235.17]) by mx.zohomail.com with SMTPS id 152144399728869.18821316117453; Mon, 19 Mar 2018 00:19:57 -0700 (PDT) Received: from localhost ([::1]:40608 helo=lists.gnu.org) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1exp5D-0003yK-Rl for importer@patchew.org; Mon, 19 Mar 2018 03:19:51 -0400 Received: from eggs.gnu.org ([2001:4830:134:3::10]:60849) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1exp2x-0002hS-E5 for qemu-devel@nongnu.org; Mon, 19 Mar 2018 03:17:32 -0400 Received: from Debian-exim by eggs.gnu.org with spam-scanned (Exim 4.71) (envelope-from ) id 1exp2u-0008Sh-MY for qemu-devel@nongnu.org; Mon, 19 Mar 2018 03:17:31 -0400 Received: from mga11.intel.com ([192.55.52.93]:42870) by eggs.gnu.org with esmtps (TLS1.0:DHE_RSA_AES_256_CBC_SHA1:32) (Exim 4.71) (envelope-from ) id 1exp2u-0008Rg-EN for qemu-devel@nongnu.org; Mon, 19 Mar 2018 03:17:28 -0400 Received: from fmsmga002.fm.intel.com ([10.253.24.26]) by fmsmga102.fm.intel.com with ESMTP/TLS/DHE-RSA-AES256-GCM-SHA384; 19 Mar 2018 00:17:28 -0700 Received: from debian.sh.intel.com ([10.67.104.164]) by fmsmga002.fm.intel.com with ESMTP; 19 Mar 2018 00:17:26 -0700 X-Amp-Result: SKIPPED(no attachment in message) X-Amp-File-Uploaded: False X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.48,329,1517904000"; d="scan'208";a="29181390" From: Tiwei Bie To: qemu-devel@nongnu.org, virtio-dev@lists.oasis-open.org, mst@redhat.com, alex.williamson@redhat.com, jasowang@redhat.com, pbonzini@redhat.com, stefanha@redhat.com Date: Mon, 19 Mar 2018 15:15:32 +0800 Message-Id: <20180319071537.28649-2-tiwei.bie@intel.com> X-Mailer: git-send-email 2.11.0 In-Reply-To: <20180319071537.28649-1-tiwei.bie@intel.com> References: <20180319071537.28649-1-tiwei.bie@intel.com> X-detected-operating-system: by eggs.gnu.org: Genre and OS details not recognized. X-Received-From: 192.55.52.93 Subject: [Qemu-devel] [PATCH v2 1/6] vhost-user: support receiving file descriptors in slave_read X-BeenThere: qemu-devel@nongnu.org X-Mailman-Version: 2.1.21 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Cc: jianfeng.tan@intel.com, tiwei.bie@intel.com, cunming.liang@intel.com, xiao.w.wang@intel.com, zhihong.wang@intel.com, dan.daly@intel.com Errors-To: qemu-devel-bounces+importer=patchew.org@nongnu.org Sender: "Qemu-devel" X-ZohoMail: RSF_0 Z_629925259 SPT_0 Content-Transfer-Encoding: quoted-printable MIME-Version: 1.0 Content-Type: text/plain; charset="utf-8" Signed-off-by: Tiwei Bie --- hw/virtio/vhost-user.c | 41 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/hw/virtio/vhost-user.c b/hw/virtio/vhost-user.c index 41ff5cff41..1ad6caa6a3 100644 --- a/hw/virtio/vhost-user.c +++ b/hw/virtio/vhost-user.c @@ -674,14 +674,44 @@ static void slave_read(void *opaque) VhostUserHeader hdr =3D { 0, }; VhostUserPayload payload =3D { 0, }; int size, ret =3D 0; + struct iovec iov; + struct msghdr msgh; + int fd =3D -1; + size_t fdsize =3D sizeof(fd); + char control[CMSG_SPACE(fdsize)]; + struct cmsghdr *cmsg; + + memset(&msgh, 0, sizeof(msgh)); + msgh.msg_iov =3D &iov; + msgh.msg_iovlen =3D 1; + msgh.msg_control =3D control; + msgh.msg_controllen =3D sizeof(control); =20 /* Read header */ - size =3D read(u->slave_fd, &hdr, VHOST_USER_HDR_SIZE); + iov.iov_base =3D &hdr; + iov.iov_len =3D VHOST_USER_HDR_SIZE; + + size =3D recvmsg(u->slave_fd, &msgh, 0); if (size !=3D VHOST_USER_HDR_SIZE) { error_report("Failed to read from slave."); goto err; } =20 + if (msgh.msg_flags & MSG_CTRUNC) { + error_report("Truncated message."); + goto err; + } + + for (cmsg =3D CMSG_FIRSTHDR(&msgh); cmsg !=3D NULL; + cmsg =3D CMSG_NXTHDR(&msgh, cmsg)) { + if (cmsg->cmsg_level =3D=3D SOL_SOCKET && + cmsg->cmsg_type =3D=3D SCM_RIGHTS) { + fdsize =3D cmsg->cmsg_len - CMSG_LEN(0); + memcpy(&fd, CMSG_DATA(cmsg), fdsize); + break; + } + } + if (hdr.size > VHOST_USER_PAYLOAD_SIZE) { error_report("Failed to read msg header." " Size %d exceeds the maximum %zu.", hdr.size, @@ -705,9 +735,15 @@ static void slave_read(void *opaque) break; default: error_report("Received unexpected msg type."); + if (fd !=3D -1) { + close(fd); + } ret =3D -EINVAL; } =20 + /* Message handlers need to make sure that fd will be consumed. */ + fd =3D -1; + /* * REPLY_ACK feature handling. Other reply types has to be managed * directly in their request handlers. @@ -740,6 +776,9 @@ err: qemu_set_fd_handler(u->slave_fd, NULL, NULL, NULL); close(u->slave_fd); u->slave_fd =3D -1; + if (fd !=3D -1) { + close(fd); + } return; } =20 --=20 2.11.0 From nobody Fri May 17 03:00:45 2024 Delivered-To: importer@patchew.org Received-SPF: pass (zoho.com: domain of gnu.org designates 208.118.235.17 as permitted sender) client-ip=208.118.235.17; envelope-from=qemu-devel-bounces+importer=patchew.org@nongnu.org; helo=lists.gnu.org; Authentication-Results: mx.zohomail.com; spf=pass (zoho.com: domain of gnu.org designates 208.118.235.17 as permitted sender) smtp.mailfrom=qemu-devel-bounces+importer=patchew.org@nongnu.org Return-Path: Received: from lists.gnu.org (208.118.235.17 [208.118.235.17]) by mx.zohomail.com with SMTPS id 1521443997398267.8045713926555; Mon, 19 Mar 2018 00:19:57 -0700 (PDT) Received: from localhost ([::1]:40607 helo=lists.gnu.org) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1exp5B-0003yB-6k for importer@patchew.org; Mon, 19 Mar 2018 03:19:49 -0400 Received: from eggs.gnu.org ([2001:4830:134:3::10]:60865) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1exp2y-0002hV-Dw for qemu-devel@nongnu.org; Mon, 19 Mar 2018 03:17:34 -0400 Received: from Debian-exim by eggs.gnu.org with spam-scanned (Exim 4.71) (envelope-from ) id 1exp2w-0008Th-T9 for qemu-devel@nongnu.org; Mon, 19 Mar 2018 03:17:32 -0400 Received: from mga11.intel.com ([192.55.52.93]:42870) by eggs.gnu.org with esmtps (TLS1.0:DHE_RSA_AES_256_CBC_SHA1:32) (Exim 4.71) (envelope-from ) id 1exp2w-0008Rg-I6 for qemu-devel@nongnu.org; Mon, 19 Mar 2018 03:17:30 -0400 Received: from fmsmga002.fm.intel.com ([10.253.24.26]) by fmsmga102.fm.intel.com with ESMTP/TLS/DHE-RSA-AES256-GCM-SHA384; 19 Mar 2018 00:17:30 -0700 Received: from debian.sh.intel.com ([10.67.104.164]) by fmsmga002.fm.intel.com with ESMTP; 19 Mar 2018 00:17:28 -0700 X-Amp-Result: SKIPPED(no attachment in message) X-Amp-File-Uploaded: False X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.48,329,1517904000"; d="scan'208";a="29181404" From: Tiwei Bie To: qemu-devel@nongnu.org, virtio-dev@lists.oasis-open.org, mst@redhat.com, alex.williamson@redhat.com, jasowang@redhat.com, pbonzini@redhat.com, stefanha@redhat.com Date: Mon, 19 Mar 2018 15:15:33 +0800 Message-Id: <20180319071537.28649-3-tiwei.bie@intel.com> X-Mailer: git-send-email 2.11.0 In-Reply-To: <20180319071537.28649-1-tiwei.bie@intel.com> References: <20180319071537.28649-1-tiwei.bie@intel.com> X-detected-operating-system: by eggs.gnu.org: Genre and OS details not recognized. X-Received-From: 192.55.52.93 Subject: [Qemu-devel] [PATCH v2 2/6] vhost-user: introduce shared vhost-user state X-BeenThere: qemu-devel@nongnu.org X-Mailman-Version: 2.1.21 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Cc: jianfeng.tan@intel.com, tiwei.bie@intel.com, cunming.liang@intel.com, xiao.w.wang@intel.com, zhihong.wang@intel.com, dan.daly@intel.com Errors-To: qemu-devel-bounces+importer=patchew.org@nongnu.org Sender: "Qemu-devel" X-ZohoMail: RSF_0 Z_629925259 SPT_0 Content-Transfer-Encoding: quoted-printable MIME-Version: 1.0 Content-Type: text/plain; charset="utf-8" When multi-queue is enabled for virtio-net, each virtio queue pair will have a vhost_dev, and the only thing they share currently is the chardev. This patch introduces a vhost-user state structure which will be shared by all virtio queue pairs of the same virtio device. Signed-off-by: Tiwei Bie --- hw/scsi/vhost-user-scsi.c | 6 +++--- hw/virtio/vhost-user.c | 9 +++++---- include/hw/virtio/vhost-user.h | 17 +++++++++++++++++ include/hw/virtio/virtio-scsi.h | 6 +++++- net/vhost-user.c | 30 ++++++++++++++++-------------- 5 files changed, 46 insertions(+), 22 deletions(-) create mode 100644 include/hw/virtio/vhost-user.h diff --git a/hw/scsi/vhost-user-scsi.c b/hw/scsi/vhost-user-scsi.c index 9389ed48e0..64972bdd7d 100644 --- a/hw/scsi/vhost-user-scsi.c +++ b/hw/scsi/vhost-user-scsi.c @@ -72,7 +72,7 @@ static void vhost_user_scsi_realize(DeviceState *dev, Err= or **errp) Error *err =3D NULL; int ret; =20 - if (!vs->conf.chardev.chr) { + if (!vs->conf.vhost_user.chr.chr) { error_setg(errp, "vhost-user-scsi: missing chardev"); return; } @@ -90,7 +90,7 @@ static void vhost_user_scsi_realize(DeviceState *dev, Err= or **errp) vsc->dev.vq_index =3D 0; vsc->dev.backend_features =3D 0; =20 - ret =3D vhost_dev_init(&vsc->dev, (void *)&vs->conf.chardev, + ret =3D vhost_dev_init(&vsc->dev, (void *)&vs->conf.vhost_user, VHOST_BACKEND_TYPE_USER, 0); if (ret < 0) { error_setg(errp, "vhost-user-scsi: vhost initialization failed: %s= ", @@ -131,7 +131,7 @@ static uint64_t vhost_user_scsi_get_features(VirtIODevi= ce *vdev, } =20 static Property vhost_user_scsi_properties[] =3D { - DEFINE_PROP_CHR("chardev", VirtIOSCSICommon, conf.chardev), + DEFINE_PROP_CHR("chardev", VirtIOSCSICommon, conf.vhost_user.chr), DEFINE_PROP_UINT32("boot_tpgt", VirtIOSCSICommon, conf.boot_tpgt, 0), DEFINE_PROP_UINT32("num_queues", VirtIOSCSICommon, conf.num_queues, 1), DEFINE_PROP_UINT32("virtqueue_size", VirtIOSCSICommon, conf.virtqueue_= size, diff --git a/hw/virtio/vhost-user.c b/hw/virtio/vhost-user.c index 1ad6caa6a3..b228994ffd 100644 --- a/hw/virtio/vhost-user.c +++ b/hw/virtio/vhost-user.c @@ -12,6 +12,7 @@ #include "qapi/error.h" #include "hw/virtio/vhost.h" #include "hw/virtio/vhost-backend.h" +#include "hw/virtio/vhost-user.h" #include "hw/virtio/virtio-net.h" #include "chardev/char-fe.h" #include "sysemu/kvm.h" @@ -164,7 +165,7 @@ static VhostUserMsg m __attribute__ ((unused)); #define VHOST_USER_VERSION (0x1) =20 struct vhost_user { - CharBackend *chr; + VhostUser *shared; int slave_fd; }; =20 @@ -176,7 +177,7 @@ static bool ioeventfd_enabled(void) static int vhost_user_read(struct vhost_dev *dev, VhostUserMsg *msg) { struct vhost_user *u =3D dev->opaque; - CharBackend *chr =3D u->chr; + CharBackend *chr =3D &u->shared->chr; uint8_t *p =3D (uint8_t *) msg; int r, size =3D VHOST_USER_HDR_SIZE; =20 @@ -262,7 +263,7 @@ static int vhost_user_write(struct vhost_dev *dev, Vhos= tUserMsg *msg, int *fds, int fd_num) { struct vhost_user *u =3D dev->opaque; - CharBackend *chr =3D u->chr; + CharBackend *chr =3D &u->shared->chr; int ret, size =3D VHOST_USER_HDR_SIZE + msg->hdr.size; =20 /* @@ -839,7 +840,7 @@ static int vhost_user_init(struct vhost_dev *dev, void = *opaque) assert(dev->vhost_ops->backend_type =3D=3D VHOST_BACKEND_TYPE_USER); =20 u =3D g_new0(struct vhost_user, 1); - u->chr =3D opaque; + u->shared =3D opaque; u->slave_fd =3D -1; dev->opaque =3D u; =20 diff --git a/include/hw/virtio/vhost-user.h b/include/hw/virtio/vhost-user.h new file mode 100644 index 0000000000..4f5a1477d1 --- /dev/null +++ b/include/hw/virtio/vhost-user.h @@ -0,0 +1,17 @@ +/* + * Copyright (c) 2017-2018 Intel Corporation + * + * This work is licensed under the terms of the GNU GPL, version 2. + * See the COPYING file in the top-level directory. + */ + +#ifndef HW_VIRTIO_VHOST_USER_H +#define HW_VIRTIO_VHOST_USER_H + +#include "chardev/char-fe.h" + +typedef struct VhostUser { + CharBackend chr; +} VhostUser; + +#endif diff --git a/include/hw/virtio/virtio-scsi.h b/include/hw/virtio/virtio-scs= i.h index 4c0bcdb788..885c3e84b5 100644 --- a/include/hw/virtio/virtio-scsi.h +++ b/include/hw/virtio/virtio-scsi.h @@ -19,6 +19,7 @@ #define VIRTIO_SCSI_SENSE_SIZE 0 #include "standard-headers/linux/virtio_scsi.h" #include "hw/virtio/virtio.h" +#include "hw/virtio/vhost-user.h" #include "hw/pci/pci.h" #include "hw/scsi/scsi.h" #include "chardev/char-fe.h" @@ -54,7 +55,10 @@ struct VirtIOSCSIConf { char *vhostfd; char *wwpn; #endif - CharBackend chardev; + union { + VhostUser vhost_user; + CharBackend chardev; + }; uint32_t boot_tpgt; IOThread *iothread; }; diff --git a/net/vhost-user.c b/net/vhost-user.c index e0f16c895b..49ee72bd42 100644 --- a/net/vhost-user.c +++ b/net/vhost-user.c @@ -12,6 +12,7 @@ #include "clients.h" #include "net/vhost_net.h" #include "net/vhost-user.h" +#include "hw/virtio/vhost-user.h" #include "chardev/char-fe.h" #include "qapi/error.h" #include "qapi/qapi-commands-net.h" @@ -22,7 +23,7 @@ =20 typedef struct VhostUserState { NetClientState nc; - CharBackend chr; /* only queue index 0 */ + VhostUser vhost_user; /* only queue index 0 */ VHostNetState *vhost_net; guint watch; uint64_t acked_features; @@ -64,7 +65,7 @@ static void vhost_user_stop(int queues, NetClientState *n= cs[]) } } =20 -static int vhost_user_start(int queues, NetClientState *ncs[], CharBackend= *be) +static int vhost_user_start(int queues, NetClientState *ncs[], void *be) { VhostNetOptions options; struct vhost_net *net =3D NULL; @@ -158,7 +159,7 @@ static void vhost_user_cleanup(NetClientState *nc) g_source_remove(s->watch); s->watch =3D 0; } - qemu_chr_fe_deinit(&s->chr, true); + qemu_chr_fe_deinit(&s->vhost_user.chr, true); } =20 qemu_purge_queued_packets(nc); @@ -192,7 +193,7 @@ static gboolean net_vhost_user_watch(GIOChannel *chan, = GIOCondition cond, { VhostUserState *s =3D opaque; =20 - qemu_chr_fe_disconnect(&s->chr); + qemu_chr_fe_disconnect(&s->vhost_user.chr); =20 return TRUE; } @@ -217,7 +218,8 @@ static void chr_closed_bh(void *opaque) qmp_set_link(name, false, &err); vhost_user_stop(queues, ncs); =20 - qemu_chr_fe_set_handlers(&s->chr, NULL, NULL, net_vhost_user_event, + qemu_chr_fe_set_handlers(&s->vhost_user.chr, NULL, NULL, + net_vhost_user_event, NULL, opaque, NULL, true); =20 if (err) { @@ -240,15 +242,15 @@ static void net_vhost_user_event(void *opaque, int ev= ent) assert(queues < MAX_QUEUE_NUM); =20 s =3D DO_UPCAST(VhostUserState, nc, ncs[0]); - chr =3D qemu_chr_fe_get_driver(&s->chr); + chr =3D qemu_chr_fe_get_driver(&s->vhost_user.chr); trace_vhost_user_event(chr->label, event); switch (event) { case CHR_EVENT_OPENED: - if (vhost_user_start(queues, ncs, &s->chr) < 0) { - qemu_chr_fe_disconnect(&s->chr); + if (vhost_user_start(queues, ncs, &s->vhost_user) < 0) { + qemu_chr_fe_disconnect(&s->vhost_user.chr); return; } - s->watch =3D qemu_chr_fe_add_watch(&s->chr, G_IO_HUP, + s->watch =3D qemu_chr_fe_add_watch(&s->vhost_user.chr, G_IO_HUP, net_vhost_user_watch, s); qmp_set_link(name, true, &err); s->started =3D true; @@ -264,8 +266,8 @@ static void net_vhost_user_event(void *opaque, int even= t) =20 g_source_remove(s->watch); s->watch =3D 0; - qemu_chr_fe_set_handlers(&s->chr, NULL, NULL, NULL, NULL, - NULL, NULL, false); + qemu_chr_fe_set_handlers(&s->vhost_user.chr, NULL, NULL, NULL, + NULL, NULL, NULL, false); =20 aio_bh_schedule_oneshot(ctx, chr_closed_bh, opaque); } @@ -297,7 +299,7 @@ static int net_vhost_user_init(NetClientState *peer, co= nst char *device, if (!nc0) { nc0 =3D nc; s =3D DO_UPCAST(VhostUserState, nc, nc); - if (!qemu_chr_fe_init(&s->chr, chr, &err)) { + if (!qemu_chr_fe_init(&s->vhost_user.chr, chr, &err)) { error_report_err(err); return -1; } @@ -307,11 +309,11 @@ static int net_vhost_user_init(NetClientState *peer, = const char *device, =20 s =3D DO_UPCAST(VhostUserState, nc, nc0); do { - if (qemu_chr_fe_wait_connected(&s->chr, &err) < 0) { + if (qemu_chr_fe_wait_connected(&s->vhost_user.chr, &err) < 0) { error_report_err(err); return -1; } - qemu_chr_fe_set_handlers(&s->chr, NULL, NULL, + qemu_chr_fe_set_handlers(&s->vhost_user.chr, NULL, NULL, net_vhost_user_event, NULL, nc0->name, NU= LL, true); } while (!s->started); --=20 2.11.0 From nobody Fri May 17 03:00:45 2024 Delivered-To: importer@patchew.org Received-SPF: pass (zoho.com: domain of gnu.org designates 208.118.235.17 as permitted sender) client-ip=208.118.235.17; envelope-from=qemu-devel-bounces+importer=patchew.org@nongnu.org; helo=lists.gnu.org; Authentication-Results: mx.zohomail.com; spf=pass (zoho.com: domain of gnu.org designates 208.118.235.17 as permitted sender) smtp.mailfrom=qemu-devel-bounces+importer=patchew.org@nongnu.org Return-Path: Received: from lists.gnu.org (lists.gnu.org [208.118.235.17]) by mx.zohomail.com with SMTPS id 1521443997395421.7131853957046; Mon, 19 Mar 2018 00:19:57 -0700 (PDT) Received: from localhost ([::1]:40609 helo=lists.gnu.org) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1exp5G-000410-1R for importer@patchew.org; Mon, 19 Mar 2018 03:19:54 -0400 Received: from eggs.gnu.org ([2001:4830:134:3::10]:60879) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1exp30-0002i0-FU for qemu-devel@nongnu.org; Mon, 19 Mar 2018 03:17:35 -0400 Received: from Debian-exim by eggs.gnu.org with spam-scanned (Exim 4.71) (envelope-from ) id 1exp2z-0008Up-3k for qemu-devel@nongnu.org; Mon, 19 Mar 2018 03:17:34 -0400 Received: from mga11.intel.com ([192.55.52.93]:42870) by eggs.gnu.org with esmtps (TLS1.0:DHE_RSA_AES_256_CBC_SHA1:32) (Exim 4.71) (envelope-from ) id 1exp2y-0008Rg-O9 for qemu-devel@nongnu.org; Mon, 19 Mar 2018 03:17:33 -0400 Received: from fmsmga002.fm.intel.com ([10.253.24.26]) by fmsmga102.fm.intel.com with ESMTP/TLS/DHE-RSA-AES256-GCM-SHA384; 19 Mar 2018 00:17:32 -0700 Received: from debian.sh.intel.com ([10.67.104.164]) by fmsmga002.fm.intel.com with ESMTP; 19 Mar 2018 00:17:30 -0700 X-Amp-Result: SKIPPED(no attachment in message) X-Amp-File-Uploaded: False X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.48,329,1517904000"; d="scan'208";a="29181415" From: Tiwei Bie To: qemu-devel@nongnu.org, virtio-dev@lists.oasis-open.org, mst@redhat.com, alex.williamson@redhat.com, jasowang@redhat.com, pbonzini@redhat.com, stefanha@redhat.com Date: Mon, 19 Mar 2018 15:15:34 +0800 Message-Id: <20180319071537.28649-4-tiwei.bie@intel.com> X-Mailer: git-send-email 2.11.0 In-Reply-To: <20180319071537.28649-1-tiwei.bie@intel.com> References: <20180319071537.28649-1-tiwei.bie@intel.com> X-detected-operating-system: by eggs.gnu.org: Genre and OS details not recognized. X-Received-From: 192.55.52.93 Subject: [Qemu-devel] [PATCH v2 3/6] virtio: support adding sub-regions for notify region X-BeenThere: qemu-devel@nongnu.org X-Mailman-Version: 2.1.21 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Cc: jianfeng.tan@intel.com, tiwei.bie@intel.com, cunming.liang@intel.com, xiao.w.wang@intel.com, zhihong.wang@intel.com, dan.daly@intel.com Errors-To: qemu-devel-bounces+importer=patchew.org@nongnu.org Sender: "Qemu-devel" X-ZohoMail: RSF_0 Z_629925259 SPT_0 Content-Transfer-Encoding: quoted-printable MIME-Version: 1.0 Content-Type: text/plain; charset="utf-8" Provide APIs to support querying whether the page-per-vq is enabled and adding sub-regions for notify region. Signed-off-by: Tiwei Bie --- Makefile.target | 4 ++++ hw/virtio/virtio-pci.c | 48 ++++++++++++++++++++++++++++++++++++++++++= ++++ hw/virtio/virtio-pci.h | 5 +++++ hw/virtio/virtio.c | 39 +++++++++++++++++++++++++++++++++++++ include/hw/virtio/virtio.h | 5 +++++ include/qemu/osdep.h | 1 + scripts/create_config | 3 +++ 7 files changed, 105 insertions(+) diff --git a/Makefile.target b/Makefile.target index 6549481096..b2cf618dc9 100644 --- a/Makefile.target +++ b/Makefile.target @@ -39,6 +39,9 @@ STPFILES=3D config-target.h: config-target.h-timestamp config-target.h-timestamp: config-target.mak =20 +config-devices.h: config-devices.h-timestamp +config-devices.h-timestamp: config-devices.mak + ifdef CONFIG_TRACE_SYSTEMTAP stap: $(QEMU_PROG).stp-installed $(QEMU_PROG).stp $(QEMU_PROG)-simpletrace= .stp =20 @@ -224,4 +227,5 @@ ifdef CONFIG_TRACE_SYSTEMTAP endif =20 GENERATED_FILES +=3D config-target.h +GENERATED_FILES +=3D config-devices.h Makefile: $(GENERATED_FILES) diff --git a/hw/virtio/virtio-pci.c b/hw/virtio/virtio-pci.c index 1e8ab7bbc5..b17471092a 100644 --- a/hw/virtio/virtio-pci.c +++ b/hw/virtio/virtio-pci.c @@ -1534,6 +1534,54 @@ static void virtio_pci_modern_io_region_unmap(VirtIO= PCIProxy *proxy, ®ion->mr); } =20 +static VirtIOPCIProxy *virtio_device_to_virtio_pci_proxy(VirtIODevice *vde= v) +{ + VirtIOPCIProxy *proxy =3D NULL; + + if (vdev->device_id =3D=3D VIRTIO_ID_NET) { + VirtIONetPCI *d =3D container_of(vdev, VirtIONetPCI, vdev.parent_o= bj); + proxy =3D &d->parent_obj; + } + + return proxy; +} + +bool virtio_pci_page_per_vq_enabled(VirtIODevice *vdev) +{ + VirtIOPCIProxy *proxy =3D virtio_device_to_virtio_pci_proxy(vdev); + + if (proxy =3D=3D NULL) { + return false; + } + + return !!(proxy->flags & VIRTIO_PCI_FLAG_PAGE_PER_VQ); +} + +int virtio_pci_notify_region_map(VirtIODevice *vdev, int queue_idx, + MemoryRegion *mr) +{ + VirtIOPCIProxy *proxy =3D virtio_device_to_virtio_pci_proxy(vdev); + int offset; + + if (proxy =3D=3D NULL || !virtio_pci_modern(proxy)) { + return -1; + } + + offset =3D virtio_pci_queue_mem_mult(proxy) * queue_idx; + memory_region_add_subregion(&proxy->notify.mr, offset, mr); + + return 0; +} + +void virtio_pci_notify_region_unmap(VirtIODevice *vdev, MemoryRegion *mr) +{ + VirtIOPCIProxy *proxy =3D virtio_device_to_virtio_pci_proxy(vdev); + + if (proxy !=3D NULL) { + memory_region_del_subregion(&proxy->notify.mr, mr); + } +} + static void virtio_pci_pre_plugged(DeviceState *d, Error **errp) { VirtIOPCIProxy *proxy =3D VIRTIO_PCI(d); diff --git a/hw/virtio/virtio-pci.h b/hw/virtio/virtio-pci.h index 813082b0d7..8061133741 100644 --- a/hw/virtio/virtio-pci.h +++ b/hw/virtio/virtio-pci.h @@ -213,6 +213,11 @@ static inline void virtio_pci_disable_modern(VirtIOPCI= Proxy *proxy) proxy->disable_modern =3D true; } =20 +bool virtio_pci_page_per_vq_enabled(VirtIODevice *vdev); +int virtio_pci_notify_region_map(VirtIODevice *vdev, int queue_idx, + MemoryRegion *mr); +void virtio_pci_notify_region_unmap(VirtIODevice *vdev, MemoryRegion *mr); + /* * virtio-scsi-pci: This extends VirtioPCIProxy. */ diff --git a/hw/virtio/virtio.c b/hw/virtio/virtio.c index 006d3d1148..90ee72984c 100644 --- a/hw/virtio/virtio.c +++ b/hw/virtio/virtio.c @@ -22,6 +22,7 @@ #include "qemu/atomic.h" #include "hw/virtio/virtio-bus.h" #include "hw/virtio/virtio-access.h" +#include "hw/virtio/virtio-pci.h" #include "sysemu/dma.h" =20 /* @@ -2681,6 +2682,44 @@ void virtio_device_release_ioeventfd(VirtIODevice *v= dev) virtio_bus_release_ioeventfd(vbus); } =20 +bool virtio_device_parent_is_pci_device(VirtIODevice *vdev) +{ + BusState *qbus =3D qdev_get_parent_bus(DEVICE(vdev)); + const char *typename =3D object_get_typename(OBJECT(qbus->parent)); + + return strstr(typename, "pci") !=3D NULL; +} + +bool virtio_device_page_per_vq_enabled(VirtIODevice *vdev) +{ +#ifdef CONFIG_VIRTIO_PCI + if (virtio_device_parent_is_pci_device(vdev)) { + return virtio_pci_page_per_vq_enabled(vdev); + } +#endif + return false; +} + +int virtio_device_notify_region_map(VirtIODevice *vdev, int queue_idx, + MemoryRegion *mr) +{ +#ifdef CONFIG_VIRTIO_PCI + if (virtio_device_parent_is_pci_device(vdev)) { + return virtio_pci_notify_region_map(vdev, queue_idx, mr); + } +#endif + return -1; +} + +void virtio_device_notify_region_unmap(VirtIODevice *vdev, MemoryRegion *m= r) +{ +#ifdef CONFIG_VIRTIO_PCI + if (virtio_device_parent_is_pci_device(vdev)) { + virtio_pci_notify_region_unmap(vdev, mr); + } +#endif +} + static void virtio_device_class_init(ObjectClass *klass, void *data) { /* Set the default value here. */ diff --git a/include/hw/virtio/virtio.h b/include/hw/virtio/virtio.h index 098bdaaea3..b14accdb08 100644 --- a/include/hw/virtio/virtio.h +++ b/include/hw/virtio/virtio.h @@ -285,6 +285,11 @@ void virtio_device_stop_ioeventfd(VirtIODevice *vdev); int virtio_device_grab_ioeventfd(VirtIODevice *vdev); void virtio_device_release_ioeventfd(VirtIODevice *vdev); bool virtio_device_ioeventfd_enabled(VirtIODevice *vdev); +bool virtio_device_parent_is_pci_device(VirtIODevice *vdev); +bool virtio_device_page_per_vq_enabled(VirtIODevice *vdev); +int virtio_device_notify_region_map(VirtIODevice *vdev, int queue_idx, + MemoryRegion *mr); +void virtio_device_notify_region_unmap(VirtIODevice *vdev, MemoryRegion *m= r); EventNotifier *virtio_queue_get_host_notifier(VirtQueue *vq); void virtio_queue_host_notifier_read(EventNotifier *n); void virtio_queue_aio_set_host_notifier_handler(VirtQueue *vq, AioContext = *ctx, diff --git a/include/qemu/osdep.h b/include/qemu/osdep.h index 41658060a7..2532c278ef 100644 --- a/include/qemu/osdep.h +++ b/include/qemu/osdep.h @@ -30,6 +30,7 @@ #include "config-host.h" #ifdef NEED_CPU_H #include "config-target.h" +#include "config-devices.h" #else #include "exec/poison.h" #endif diff --git a/scripts/create_config b/scripts/create_config index d727e5e36e..e4541a51ed 100755 --- a/scripts/create_config +++ b/scripts/create_config @@ -58,6 +58,9 @@ case $line in name=3D${line%=3D*} echo "#define $name 1" ;; + CONFIG_*=3D'$(CONFIG_'*')') # configuration + continue + ;; CONFIG_*=3D*) # configuration name=3D${line%=3D*} value=3D${line#*=3D} --=20 2.11.0 From nobody Fri May 17 03:00:45 2024 Delivered-To: importer@patchew.org Received-SPF: pass (zoho.com: domain of gnu.org designates 208.118.235.17 as permitted sender) client-ip=208.118.235.17; envelope-from=qemu-devel-bounces+importer=patchew.org@nongnu.org; helo=lists.gnu.org; Authentication-Results: mx.zohomail.com; spf=pass (zoho.com: domain of gnu.org designates 208.118.235.17 as permitted sender) smtp.mailfrom=qemu-devel-bounces+importer=patchew.org@nongnu.org Return-Path: Received: from lists.gnu.org (lists.gnu.org [208.118.235.17]) by mx.zohomail.com with SMTPS id 152144414749571.97594365950863; Mon, 19 Mar 2018 00:22:27 -0700 (PDT) Received: from localhost ([::1]:40628 helo=lists.gnu.org) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1exp7i-0006Aw-IK for importer@patchew.org; Mon, 19 Mar 2018 03:22:26 -0400 Received: from eggs.gnu.org ([2001:4830:134:3::10]:60900) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1exp32-0002jd-ND for qemu-devel@nongnu.org; Mon, 19 Mar 2018 03:17:38 -0400 Received: from Debian-exim by eggs.gnu.org with spam-scanned (Exim 4.71) (envelope-from ) id 1exp31-0008WM-Bv for qemu-devel@nongnu.org; Mon, 19 Mar 2018 03:17:36 -0400 Received: from mga11.intel.com ([192.55.52.93]:42870) by eggs.gnu.org with esmtps (TLS1.0:DHE_RSA_AES_256_CBC_SHA1:32) (Exim 4.71) (envelope-from ) id 1exp31-0008Rg-1Q for qemu-devel@nongnu.org; Mon, 19 Mar 2018 03:17:35 -0400 Received: from fmsmga002.fm.intel.com ([10.253.24.26]) by fmsmga102.fm.intel.com with ESMTP/TLS/DHE-RSA-AES256-GCM-SHA384; 19 Mar 2018 00:17:34 -0700 Received: from debian.sh.intel.com ([10.67.104.164]) by fmsmga002.fm.intel.com with ESMTP; 19 Mar 2018 00:17:32 -0700 X-Amp-Result: SKIPPED(no attachment in message) X-Amp-File-Uploaded: False X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.48,329,1517904000"; d="scan'208";a="29181426" From: Tiwei Bie To: qemu-devel@nongnu.org, virtio-dev@lists.oasis-open.org, mst@redhat.com, alex.williamson@redhat.com, jasowang@redhat.com, pbonzini@redhat.com, stefanha@redhat.com Date: Mon, 19 Mar 2018 15:15:35 +0800 Message-Id: <20180319071537.28649-5-tiwei.bie@intel.com> X-Mailer: git-send-email 2.11.0 In-Reply-To: <20180319071537.28649-1-tiwei.bie@intel.com> References: <20180319071537.28649-1-tiwei.bie@intel.com> X-detected-operating-system: by eggs.gnu.org: Genre and OS details not recognized. X-Received-From: 192.55.52.93 Subject: [Qemu-devel] [PATCH v2 4/6] vfio: support getting VFIOGroup from groupfd X-BeenThere: qemu-devel@nongnu.org X-Mailman-Version: 2.1.21 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Cc: jianfeng.tan@intel.com, tiwei.bie@intel.com, cunming.liang@intel.com, xiao.w.wang@intel.com, zhihong.wang@intel.com, dan.daly@intel.com Errors-To: qemu-devel-bounces+importer=patchew.org@nongnu.org Sender: "Qemu-devel" X-ZohoMail: RSF_0 Z_629925259 SPT_0 Content-Transfer-Encoding: quoted-printable MIME-Version: 1.0 Content-Type: text/plain; charset="utf-8" Add an API to support getting VFIOGroup from groupfd. When groupfd is shared by another process, the VFIOGroup may not have its container and address space in QEMU. Besides, add a reference counter to better support getting VFIOGroup multiple times. Signed-off-by: Tiwei Bie --- hw/vfio/common.c | 97 +++++++++++++++++++++++++++++++++++++++= +++- include/hw/vfio/vfio-common.h | 2 + 2 files changed, 98 insertions(+), 1 deletion(-) diff --git a/hw/vfio/common.c b/hw/vfio/common.c index 5e84716218..24ec0f2c8d 100644 --- a/hw/vfio/common.c +++ b/hw/vfio/common.c @@ -1038,6 +1038,11 @@ static int vfio_connect_container(VFIOGroup *group, = AddressSpace *as, int ret, fd; VFIOAddressSpace *space; =20 + if (as =3D=3D NULL) { + vfio_kvm_device_add_group(group); + return 0; + } + space =3D vfio_get_address_space(as); =20 QLIST_FOREACH(container, &space->containers, next) { @@ -1237,6 +1242,10 @@ static void vfio_disconnect_container(VFIOGroup *gro= up) { VFIOContainer *container =3D group->container; =20 + if (container =3D=3D NULL) { + return; + } + QLIST_REMOVE(group, container_next); group->container =3D NULL; =20 @@ -1275,6 +1284,86 @@ static void vfio_disconnect_container(VFIOGroup *gro= up) } } =20 +static int vfio_groupfd_to_groupid(int groupfd) +{ + char linkname[PATH_MAX]; + char pathname[PATH_MAX]; + char *filename; + int groupid, len; + + snprintf(linkname, sizeof(linkname), "/proc/self/fd/%d", groupfd); + + len =3D readlink(linkname, pathname, sizeof(pathname)); + if (len <=3D 0 || len >=3D sizeof(pathname)) { + return -1; + } + pathname[len] =3D '\0'; + + filename =3D g_path_get_basename(pathname); + groupid =3D atoi(filename); + g_free(filename); + + return groupid; +} + +/* + * The @as param could be NULL. In this case, groupfd is shared by + * another process which will setup the DMA mapping for this group, + * and this group won't have container and address space in QEMU. + */ +VFIOGroup *vfio_get_group_from_fd(int groupfd, AddressSpace *as, Error **e= rrp) +{ + VFIOGroup *group; + int groupid; + + groupid =3D vfio_groupfd_to_groupid(groupfd); + if (groupid < 0) { + return NULL; + } + + QLIST_FOREACH(group, &vfio_group_list, next) { + if (group->groupid =3D=3D groupid) { + /* Found it. Now is it already in the right context? */ + if ((group->container =3D=3D NULL && as =3D=3D NULL) || + (group->container && group->container->space->as =3D=3D as= )) { + group->refcnt++; + return group; + } + error_setg(errp, "group %d used in multiple address spaces", + group->groupid); + return NULL; + } + } + + group =3D g_malloc0(sizeof(*group)); + + group->fd =3D groupfd; + group->groupid =3D groupid; + group->refcnt =3D 1; + + QLIST_INIT(&group->device_list); + + if (vfio_connect_container(group, as, errp)) { + error_prepend(errp, "failed to setup container for group %d: ", + groupid); + goto free_group_exit; + } + + if (QLIST_EMPTY(&vfio_group_list)) { + qemu_register_reset(vfio_reset_handler, NULL); + } + + QLIST_INSERT_HEAD(&vfio_group_list, group, next); + + return group; + +free_group_exit: + g_free(group); + + return NULL; +} + +/* The @as param cannot be NULL. */ VFIOGroup *vfio_get_group(int groupid, AddressSpace *as, Error **errp) { VFIOGroup *group; @@ -1284,7 +1373,8 @@ VFIOGroup *vfio_get_group(int groupid, AddressSpace *= as, Error **errp) QLIST_FOREACH(group, &vfio_group_list, next) { if (group->groupid =3D=3D groupid) { /* Found it. Now is it already in the right context? */ - if (group->container->space->as =3D=3D as) { + if (group->container && group->container->space->as =3D=3D as)= { + group->refcnt++; return group; } else { error_setg(errp, "group %d used in multiple address spaces= ", @@ -1317,6 +1407,7 @@ VFIOGroup *vfio_get_group(int groupid, AddressSpace *= as, Error **errp) } =20 group->groupid =3D groupid; + group->refcnt =3D 1; QLIST_INIT(&group->device_list); =20 if (vfio_connect_container(group, as, errp)) { @@ -1348,6 +1439,10 @@ void vfio_put_group(VFIOGroup *group) return; } =20 + if (--group->refcnt > 0) { + return; + } + vfio_kvm_device_del_group(group); vfio_disconnect_container(group); QLIST_REMOVE(group, next); diff --git a/include/hw/vfio/vfio-common.h b/include/hw/vfio/vfio-common.h index d9360148e6..b820f7984c 100644 --- a/include/hw/vfio/vfio-common.h +++ b/include/hw/vfio/vfio-common.h @@ -137,6 +137,7 @@ struct VFIODeviceOps { typedef struct VFIOGroup { int fd; int groupid; + int refcnt; VFIOContainer *container; QLIST_HEAD(, VFIODevice) device_list; QLIST_ENTRY(VFIOGroup) next; @@ -180,6 +181,7 @@ void vfio_region_exit(VFIORegion *region); void vfio_region_finalize(VFIORegion *region); void vfio_reset_handler(void *opaque); VFIOGroup *vfio_get_group(int groupid, AddressSpace *as, Error **errp); +VFIOGroup *vfio_get_group_from_fd(int groupfd, AddressSpace *as, Error **e= rrp); void vfio_put_group(VFIOGroup *group); int vfio_get_device(VFIOGroup *group, const char *name, VFIODevice *vbasedev, Error **errp); --=20 2.11.0 From nobody Fri May 17 03:00:45 2024 Delivered-To: importer@patchew.org Received-SPF: pass (zoho.com: domain of gnu.org designates 208.118.235.17 as permitted sender) client-ip=208.118.235.17; envelope-from=qemu-devel-bounces+importer=patchew.org@nongnu.org; helo=lists.gnu.org; Authentication-Results: mx.zohomail.com; spf=pass (zoho.com: domain of gnu.org designates 208.118.235.17 as permitted sender) smtp.mailfrom=qemu-devel-bounces+importer=patchew.org@nongnu.org Return-Path: Received: from lists.gnu.org (lists.gnu.org [208.118.235.17]) by mx.zohomail.com with SMTPS id 1521444146639826.103953046718; Mon, 19 Mar 2018 00:22:26 -0700 (PDT) Received: from localhost ([::1]:40626 helo=lists.gnu.org) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1exp7h-00068U-Jo for importer@patchew.org; Mon, 19 Mar 2018 03:22:25 -0400 Received: from eggs.gnu.org ([2001:4830:134:3::10]:60919) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1exp37-0002mi-BZ for qemu-devel@nongnu.org; Mon, 19 Mar 2018 03:17:42 -0400 Received: from Debian-exim by eggs.gnu.org with spam-scanned (Exim 4.71) (envelope-from ) id 1exp34-00006Y-7a for qemu-devel@nongnu.org; Mon, 19 Mar 2018 03:17:41 -0400 Received: from mga11.intel.com ([192.55.52.93]:42870) by eggs.gnu.org with esmtps (TLS1.0:DHE_RSA_AES_256_CBC_SHA1:32) (Exim 4.71) (envelope-from ) id 1exp33-0008Rg-V6 for qemu-devel@nongnu.org; Mon, 19 Mar 2018 03:17:38 -0400 Received: from fmsmga002.fm.intel.com ([10.253.24.26]) by fmsmga102.fm.intel.com with ESMTP/TLS/DHE-RSA-AES256-GCM-SHA384; 19 Mar 2018 00:17:37 -0700 Received: from debian.sh.intel.com ([10.67.104.164]) by fmsmga002.fm.intel.com with ESMTP; 19 Mar 2018 00:17:34 -0700 X-Amp-Result: SKIPPED(no attachment in message) X-Amp-File-Uploaded: False X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.48,329,1517904000"; d="scan'208";a="29181443" From: Tiwei Bie To: qemu-devel@nongnu.org, virtio-dev@lists.oasis-open.org, mst@redhat.com, alex.williamson@redhat.com, jasowang@redhat.com, pbonzini@redhat.com, stefanha@redhat.com Date: Mon, 19 Mar 2018 15:15:36 +0800 Message-Id: <20180319071537.28649-6-tiwei.bie@intel.com> X-Mailer: git-send-email 2.11.0 In-Reply-To: <20180319071537.28649-1-tiwei.bie@intel.com> References: <20180319071537.28649-1-tiwei.bie@intel.com> X-detected-operating-system: by eggs.gnu.org: Genre and OS details not recognized. X-Received-From: 192.55.52.93 Subject: [Qemu-devel] [PATCH v2 5/6] vfio: remove DPRINTF() definition from vfio-common.h X-BeenThere: qemu-devel@nongnu.org X-Mailman-Version: 2.1.21 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Cc: jianfeng.tan@intel.com, tiwei.bie@intel.com, cunming.liang@intel.com, xiao.w.wang@intel.com, zhihong.wang@intel.com, dan.daly@intel.com Errors-To: qemu-devel-bounces+importer=patchew.org@nongnu.org Sender: "Qemu-devel" X-ZohoMail: RSF_0 Z_629925259 SPT_0 Content-Transfer-Encoding: quoted-printable MIME-Version: 1.0 Content-Type: text/plain; charset="utf-8" This macro isn't used by any VFIO code. And its name is too generic. The vfio-common.h (in include/hw/vfio) can be included by other modules in QEMU. It can introduce conflicts. Signed-off-by: Tiwei Bie --- include/hw/vfio/vfio-common.h | 9 --------- 1 file changed, 9 deletions(-) diff --git a/include/hw/vfio/vfio-common.h b/include/hw/vfio/vfio-common.h index b820f7984c..f6aa4ae959 100644 --- a/include/hw/vfio/vfio-common.h +++ b/include/hw/vfio/vfio-common.h @@ -34,15 +34,6 @@ #define ERR_PREFIX "vfio error: %s: " #define WARN_PREFIX "vfio warning: %s: " =20 -/*#define DEBUG_VFIO*/ -#ifdef DEBUG_VFIO -#define DPRINTF(fmt, ...) \ - do { fprintf(stderr, "vfio: " fmt, ## __VA_ARGS__); } while (0) -#else -#define DPRINTF(fmt, ...) \ - do { } while (0) -#endif - enum { VFIO_DEVICE_TYPE_PCI =3D 0, VFIO_DEVICE_TYPE_PLATFORM =3D 1, --=20 2.11.0 From nobody Fri May 17 03:00:45 2024 Delivered-To: importer@patchew.org Received-SPF: pass (zoho.com: domain of gnu.org designates 208.118.235.17 as permitted sender) client-ip=208.118.235.17; envelope-from=qemu-devel-bounces+importer=patchew.org@nongnu.org; helo=lists.gnu.org; Authentication-Results: mx.zohomail.com; spf=pass (zoho.com: domain of gnu.org designates 208.118.235.17 as permitted sender) smtp.mailfrom=qemu-devel-bounces+importer=patchew.org@nongnu.org Return-Path: Received: from lists.gnu.org (lists.gnu.org [208.118.235.17]) by mx.zohomail.com with SMTPS id 1521444148436643.6871075736912; Mon, 19 Mar 2018 00:22:28 -0700 (PDT) Received: from localhost ([::1]:40627 helo=lists.gnu.org) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1exp7j-00069t-Ij for importer@patchew.org; Mon, 19 Mar 2018 03:22:27 -0400 Received: from eggs.gnu.org ([2001:4830:134:3::10]:60941) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1exp3A-0002oy-Rf for qemu-devel@nongnu.org; Mon, 19 Mar 2018 03:17:46 -0400 Received: from Debian-exim by eggs.gnu.org with spam-scanned (Exim 4.71) (envelope-from ) id 1exp38-0000An-Tn for qemu-devel@nongnu.org; Mon, 19 Mar 2018 03:17:44 -0400 Received: from mga11.intel.com ([192.55.52.93]:42870) by eggs.gnu.org with esmtps (TLS1.0:DHE_RSA_AES_256_CBC_SHA1:32) (Exim 4.71) (envelope-from ) id 1exp38-0008Rg-Fu for qemu-devel@nongnu.org; Mon, 19 Mar 2018 03:17:42 -0400 Received: from fmsmga002.fm.intel.com ([10.253.24.26]) by fmsmga102.fm.intel.com with ESMTP/TLS/DHE-RSA-AES256-GCM-SHA384; 19 Mar 2018 00:17:40 -0700 Received: from debian.sh.intel.com ([10.67.104.164]) by fmsmga002.fm.intel.com with ESMTP; 19 Mar 2018 00:17:37 -0700 X-Amp-Result: SKIPPED(no attachment in message) X-Amp-File-Uploaded: False X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.48,329,1517904000"; d="scan'208";a="29181459" From: Tiwei Bie To: qemu-devel@nongnu.org, virtio-dev@lists.oasis-open.org, mst@redhat.com, alex.williamson@redhat.com, jasowang@redhat.com, pbonzini@redhat.com, stefanha@redhat.com Date: Mon, 19 Mar 2018 15:15:37 +0800 Message-Id: <20180319071537.28649-7-tiwei.bie@intel.com> X-Mailer: git-send-email 2.11.0 In-Reply-To: <20180319071537.28649-1-tiwei.bie@intel.com> References: <20180319071537.28649-1-tiwei.bie@intel.com> X-detected-operating-system: by eggs.gnu.org: Genre and OS details not recognized. X-Received-From: 192.55.52.93 Subject: [Qemu-devel] [PATCH v2 6/6] vhost-user: add VFIO based accelerators support X-BeenThere: qemu-devel@nongnu.org X-Mailman-Version: 2.1.21 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Cc: jianfeng.tan@intel.com, tiwei.bie@intel.com, cunming.liang@intel.com, xiao.w.wang@intel.com, zhihong.wang@intel.com, dan.daly@intel.com Errors-To: qemu-devel-bounces+importer=patchew.org@nongnu.org Sender: "Qemu-devel" X-ZohoMail: RSF_0 Z_629925259 SPT_0 Content-Transfer-Encoding: quoted-printable MIME-Version: 1.0 Content-Type: text/plain; charset="utf-8" This patch does some small extensions to vhost-user protocol to support VFIO based accelerators, and makes it possible to get the similar performance of VFIO based PCI passthru while keeping the virtio device emulation in QEMU. Any virtio ring compatible devices potentially can be used as the vhost data path accelerators. We can setup the accelerator based on the informations (e.g. memory table, features, ring info, etc) available on the vhost backend. And accelerator will be able to use the virtio ring provided by the virtio driver in the VM directly. So the virtio driver in the VM can exchange e.g. network packets with the accelerator directly via the virtio ring. But for vhost-user, the critical issue in this case is that the data path performance is relatively low and some host threads are needed for the data path, because some necessary mechanisms are missing to support: 1) guest driver notifies the device directly; 2) device interrupts the guest directly; So this patch does some small extensions to vhost-user protocol to make both of them possible. It leverages the same mechanisms as the VFIO based PCI passthru. A new protocol feature bit is added to negotiate the accelerator feature support. Two new slave message types are added to control the notify region and queue interrupt passthru for each queue. From the view of vhost-user protocol design, it's very flexible. The passthru can be enabled/disabled for each queue individually, and it's possible to accelerate each queue by different devices. The key difference with PCI passthru is that, in this case only the data path of the device (e.g. DMA ring, notify region and queue interrupt) is pass-throughed to the VM, the device control path (e.g. PCI configuration space and MMIO regions) is still defined and emulated by QEMU. The benefits of keeping virtio device emulation in QEMU compared with virtio device PCI passthru include (but not limit to): - consistent device interface for guest OS in the VM; - max flexibility on the hardware (i.e. the accelerators) design; - leveraging the existing virtio live-migration framework; Normally, vhost-user is meant for connecting to e.g. user-space switch which is shared between multiple VMs. Typically, a vhost accelerator isn't a simple NIC which is just for packet I/O, but e.g. an switch accelerator which is also shared between multiple VMs. This commit extends vhost-user to better support connecting to e.g. a user-space switch that has an accelerator. Signed-off-by: Tiwei Bie --- docs/interop/vhost-user.txt | 57 ++++++++++++ hw/virtio/vhost-user.c | 198 +++++++++++++++++++++++++++++++++++++= ++++ include/hw/virtio/vhost-user.h | 17 ++++ 3 files changed, 272 insertions(+) diff --git a/docs/interop/vhost-user.txt b/docs/interop/vhost-user.txt index cb3a7595aa..264a58a800 100644 --- a/docs/interop/vhost-user.txt +++ b/docs/interop/vhost-user.txt @@ -132,6 +132,15 @@ Depending on the request type, payload can be: Payload: Size bytes array holding the contents of the virtio device's configuration space =20 + * Vring area description + ----------------------- + | u64 | size | offset | + ----------------------- + + u64: a 64-bit unsigned integer + Size: a 64-bit size + Offset: a 64-bit offset + In QEMU the vhost-user message is implemented with the following struct: =20 typedef struct VhostUserMsg { @@ -146,6 +155,7 @@ typedef struct VhostUserMsg { VhostUserLog log; struct vhost_iotlb_msg iotlb; VhostUserConfig config; + VhostUserVringArea area; }; } QEMU_PACKED VhostUserMsg; =20 @@ -358,6 +368,17 @@ The fd is provided via VHOST_USER_SET_SLAVE_REQ_FD anc= illary data. A slave may then send VHOST_USER_SLAVE_* messages to the master using this fd communication channel. =20 +VFIO based accelerators +----------------------- + +The VFIO based accelerators feature is a protocol extension. It is support= ed +when the protocol feature VHOST_USER_PROTOCOL_F_VFIO (bit 7) is set. + +The vhost-user backend will set the accelerator context via slave channel, +and QEMU just needs to handle those messages passively. The accelerator +context will be set for each queue independently. So the page-per-vq prope= rty +should also be enabled. + Protocol features ----------------- =20 @@ -369,6 +390,7 @@ Protocol features #define VHOST_USER_PROTOCOL_F_SLAVE_REQ 5 #define VHOST_USER_PROTOCOL_F_CROSS_ENDIAN 6 #define VHOST_USER_PROTOCOL_F_CRYPTO_SESSION 7 +#define VHOST_USER_PROTOCOL_F_VFIO 8 =20 Master message types -------------------- @@ -722,6 +744,41 @@ Slave message types respond with zero when operation is successfully completed, or non-ze= ro otherwise. =20 + * VHOST_USER_SLAVE_VRING_VFIO_GROUP_MSG + + Id: 3 + Equivalent ioctl: N/A + Slave payload: u64 + Master payload: N/A + + Sets the VFIO group file descriptor which is passed as ancillary data + for a specified queue (queue index is carried in the u64 payload). + Slave sends this request to tell QEMU to add or delete a VFIO group. + QEMU will delete the current group if any for the specified queue wh= en + the message is sent without a file descriptor. A VFIO group will be + actually deleted when its reference count reaches zero. + This request should be sent only when VHOST_USER_PROTOCOL_F_VFIO pro= tocol + feature has been successfully negotiated. + + * VHOST_USER_SLAVE_VRING_NOTIFY_AREA_MSG + + Id: 4 + Equivalent ioctl: N/A + Slave payload: vring area description + Master payload: N/A + + Sets the notify area for a specified queue (queue index is carried + in the u64 field of the vring area description). A file descriptor is + passed as ancillary data (typically it's a VFIO device fd). QEMU can + mmap the file descriptor based on the information carried in the vri= ng + area description. + Slave sends this request to tell QEMU to add or delete a MemoryRegion + for a specified queue's notify MMIO region. QEMU will delete the cur= rent + MemoryRegion if any for the specified queue when the message is sent + without a file descriptor. + This request should be sent only when VHOST_USER_PROTOCOL_F_VFIO pro= tocol + feature and VIRTIO_F_VERSION_1 feature have been successfully negoti= ated. + VHOST_USER_PROTOCOL_F_REPLY_ACK: ------------------------------- The original vhost-user specification only demands replies for certain diff --git a/hw/virtio/vhost-user.c b/hw/virtio/vhost-user.c index b228994ffd..07fc63c6e8 100644 --- a/hw/virtio/vhost-user.c +++ b/hw/virtio/vhost-user.c @@ -42,6 +42,7 @@ enum VhostUserProtocolFeature { VHOST_USER_PROTOCOL_F_SLAVE_REQ =3D 5, VHOST_USER_PROTOCOL_F_CROSS_ENDIAN =3D 6, VHOST_USER_PROTOCOL_F_CRYPTO_SESSION =3D 7, + VHOST_USER_PROTOCOL_F_VFIO =3D 8, =20 VHOST_USER_PROTOCOL_F_MAX }; @@ -84,6 +85,8 @@ typedef enum VhostUserSlaveRequest { VHOST_USER_SLAVE_NONE =3D 0, VHOST_USER_SLAVE_IOTLB_MSG =3D 1, VHOST_USER_SLAVE_CONFIG_CHANGE_MSG =3D 2, + VHOST_USER_SLAVE_VRING_VFIO_GROUP_MSG =3D 3, + VHOST_USER_SLAVE_VRING_NOTIFY_AREA_MSG =3D 4, VHOST_USER_SLAVE_MAX } VhostUserSlaveRequest; =20 @@ -128,6 +131,12 @@ static VhostUserConfig c __attribute__ ((unused)); + sizeof(c.size) \ + sizeof(c.flags)) =20 +typedef struct VhostUserVringArea { + uint64_t u64; + uint64_t size; + uint64_t offset; +} VhostUserVringArea; + typedef struct { VhostUserRequest request; =20 @@ -149,6 +158,7 @@ typedef union { struct vhost_iotlb_msg iotlb; VhostUserConfig config; VhostUserCryptoSession session; + VhostUserVringArea area; } VhostUserPayload; =20 typedef struct VhostUserMsg { @@ -459,9 +469,37 @@ static int vhost_user_set_vring_num(struct vhost_dev *= dev, return vhost_set_vring(dev, VHOST_USER_SET_VRING_NUM, ring); } =20 +static void vhost_user_notify_region_remap(struct vhost_dev *dev, int queu= e_idx) +{ + struct vhost_user *u =3D dev->opaque; + VhostUserVFIOState *vfio =3D &u->shared->vfio; + VhostUserNotifyCtx *notify =3D &vfio->notify[queue_idx]; + VirtIODevice *vdev =3D dev->vdev; + + if (notify->addr && !notify->mapped) { + virtio_device_notify_region_map(vdev, queue_idx, ¬ify->mr); + notify->mapped =3D true; + } +} + +static void vhost_user_notify_region_unmap(struct vhost_dev *dev, int queu= e_idx) +{ + struct vhost_user *u =3D dev->opaque; + VhostUserVFIOState *vfio =3D &u->shared->vfio; + VhostUserNotifyCtx *notify =3D &vfio->notify[queue_idx]; + VirtIODevice *vdev =3D dev->vdev; + + if (notify->addr && notify->mapped) { + virtio_device_notify_region_unmap(vdev, ¬ify->mr); + notify->mapped =3D false; + } +} + static int vhost_user_set_vring_base(struct vhost_dev *dev, struct vhost_vring_state *ring) { + vhost_user_notify_region_remap(dev, ring->index); + return vhost_set_vring(dev, VHOST_USER_SET_VRING_BASE, ring); } =20 @@ -495,6 +533,8 @@ static int vhost_user_get_vring_base(struct vhost_dev *= dev, .hdr.size =3D sizeof(msg.payload.state), }; =20 + vhost_user_notify_region_unmap(dev, ring->index); + if (vhost_user_write(dev, &msg, NULL, 0) < 0) { return -1; } @@ -668,6 +708,133 @@ static int vhost_user_slave_handle_config_change(stru= ct vhost_dev *dev) return ret; } =20 +static int vhost_user_handle_vring_vfio_group(struct vhost_dev *dev, + uint64_t u64, + int groupfd) +{ + struct vhost_user *u =3D dev->opaque; + VhostUserVFIOState *vfio =3D &u->shared->vfio; + int queue_idx =3D u64 & VHOST_USER_VRING_IDX_MASK; + VirtIODevice *vdev =3D dev->vdev; + VFIOGroup *group; + int ret =3D 0; + + qemu_mutex_lock(&vfio->lock); + + if (!virtio_has_feature(dev->protocol_features, + VHOST_USER_PROTOCOL_F_VFIO) || + vdev =3D=3D NULL || queue_idx >=3D virtio_get_num_queues(vdev)) { + ret =3D -1; + goto out; + } + + if (vfio->group[queue_idx]) { + vfio_put_group(vfio->group[queue_idx]); + vfio->group[queue_idx] =3D NULL; + } + + if (u64 & VHOST_USER_VRING_NOFD_MASK) { + goto out; + } + + group =3D vfio_get_group_from_fd(groupfd, NULL, NULL); + if (group =3D=3D NULL) { + ret =3D -1; + goto out; + } + + if (group->fd !=3D groupfd) { + close(groupfd); + } + + vfio->group[queue_idx] =3D group; + +out: + kvm_irqchip_commit_routes(kvm_state); + qemu_mutex_unlock(&vfio->lock); + + if (ret !=3D 0 && groupfd !=3D -1) { + close(groupfd); + } + + return ret; +} + +#define NOTIFY_PAGE_SIZE 0x1000 + +static int vhost_user_handle_vring_notify_area(struct vhost_dev *dev, + VhostUserVringArea *area, + int fd) +{ + struct vhost_user *u =3D dev->opaque; + VhostUserVFIOState *vfio =3D &u->shared->vfio; + int queue_idx =3D area->u64 & VHOST_USER_VRING_IDX_MASK; + VirtIODevice *vdev =3D dev->vdev; + VhostUserNotifyCtx *notify; + void *addr =3D NULL; + int ret =3D 0; + char *name; + + qemu_mutex_lock(&vfio->lock); + + if (!virtio_has_feature(dev->protocol_features, + VHOST_USER_PROTOCOL_F_VFIO) || + vdev =3D=3D NULL || queue_idx >=3D virtio_get_num_queues(vdev) || + !virtio_device_page_per_vq_enabled(vdev)) { + ret =3D -1; + goto out; + } + + notify =3D &vfio->notify[queue_idx]; + + if (notify->addr) { + virtio_device_notify_region_unmap(vdev, ¬ify->mr); + munmap(notify->addr, NOTIFY_PAGE_SIZE); + object_unparent(OBJECT(¬ify->mr)); + notify->addr =3D NULL; + } + + if (area->u64 & VHOST_USER_VRING_NOFD_MASK) { + goto out; + } + + if (area->size < NOTIFY_PAGE_SIZE) { + ret =3D -1; + goto out; + } + + addr =3D mmap(NULL, NOTIFY_PAGE_SIZE, PROT_READ | PROT_WRITE, + MAP_SHARED, fd, area->offset); + if (addr =3D=3D MAP_FAILED) { + error_report("Can't map notify region."); + ret =3D -1; + goto out; + } + + name =3D g_strdup_printf("vhost-user/vfio@%p mmaps[%d]", vfio, queue_i= dx); + memory_region_init_ram_device_ptr(¬ify->mr, OBJECT(vdev), name, + NOTIFY_PAGE_SIZE, addr); + g_free(name); + + if (virtio_device_notify_region_map(vdev, queue_idx, ¬ify->mr)) { + ret =3D -1; + goto out; + } + + notify->addr =3D addr; + notify->mapped =3D true; + +out: + if (ret < 0 && addr !=3D NULL) { + munmap(addr, NOTIFY_PAGE_SIZE); + } + if (fd !=3D -1) { + close(fd); + } + qemu_mutex_unlock(&vfio->lock); + return ret; +} + static void slave_read(void *opaque) { struct vhost_dev *dev =3D opaque; @@ -734,6 +901,12 @@ static void slave_read(void *opaque) case VHOST_USER_SLAVE_CONFIG_CHANGE_MSG : ret =3D vhost_user_slave_handle_config_change(dev); break; + case VHOST_USER_SLAVE_VRING_VFIO_GROUP_MSG: + ret =3D vhost_user_handle_vring_vfio_group(dev, payload.u64, fd); + break; + case VHOST_USER_SLAVE_VRING_NOTIFY_AREA_MSG: + ret =3D vhost_user_handle_vring_notify_area(dev, &payload.area, fd= ); + break; default: error_report("Received unexpected msg type."); if (fd !=3D -1) { @@ -844,6 +1017,10 @@ static int vhost_user_init(struct vhost_dev *dev, voi= d *opaque) u->slave_fd =3D -1; dev->opaque =3D u; =20 + if (dev->vq_index =3D=3D 0) { + qemu_mutex_init(&u->shared->vfio.lock); + } + err =3D vhost_user_get_features(dev, &features); if (err < 0) { return err; @@ -904,6 +1081,7 @@ static int vhost_user_init(struct vhost_dev *dev, void= *opaque) static int vhost_user_cleanup(struct vhost_dev *dev) { struct vhost_user *u; + int i; =20 assert(dev->vhost_ops->backend_type =3D=3D VHOST_BACKEND_TYPE_USER); =20 @@ -913,6 +1091,26 @@ static int vhost_user_cleanup(struct vhost_dev *dev) close(u->slave_fd); u->slave_fd =3D -1; } + + if (dev->vq_index =3D=3D 0) { + VhostUserVFIOState *vfio =3D &u->shared->vfio; + + for (i =3D 0; i < VIRTIO_QUEUE_MAX; i++) { + if (vfio->notify[i].addr) { + munmap(vfio->notify[i].addr, NOTIFY_PAGE_SIZE); + object_unparent(OBJECT(&vfio->notify[i].mr)); + vfio->notify[i].addr =3D NULL; + } + + if (vfio->group[i]) { + vfio_put_group(vfio->group[i]); + vfio->group[i] =3D NULL; + } + } + + qemu_mutex_destroy(&u->shared->vfio.lock); + } + g_free(u); dev->opaque =3D 0; =20 diff --git a/include/hw/virtio/vhost-user.h b/include/hw/virtio/vhost-user.h index 4f5a1477d1..de8c647962 100644 --- a/include/hw/virtio/vhost-user.h +++ b/include/hw/virtio/vhost-user.h @@ -9,9 +9,26 @@ #define HW_VIRTIO_VHOST_USER_H =20 #include "chardev/char-fe.h" +#include "hw/virtio/virtio.h" +#include "hw/vfio/vfio-common.h" + +typedef struct VhostUserNotifyCtx { + void *addr; + MemoryRegion mr; + bool mapped; +} VhostUserNotifyCtx; + +typedef struct VhostUserVFIOState { + /* The VFIO group associated with each queue */ + VFIOGroup *group[VIRTIO_QUEUE_MAX]; + /* The notify context of each queue */ + VhostUserNotifyCtx notify[VIRTIO_QUEUE_MAX]; + QemuMutex lock; +} VhostUserVFIOState; =20 typedef struct VhostUser { CharBackend chr; + VhostUserVFIOState vfio; } VhostUser; =20 #endif --=20 2.11.0