From nobody Sat Oct 25 13:17:42 2025 Delivered-To: importer@patchew.org Received-SPF: pass (zoho.com: domain of gnu.org designates 208.118.235.17 as permitted sender) client-ip=208.118.235.17; envelope-from=qemu-devel-bounces+importer=patchew.org@nongnu.org; helo=lists.gnu.org; Authentication-Results: mx.zohomail.com; spf=pass (zoho.com: domain of gnu.org designates 208.118.235.17 as permitted sender) smtp.mailfrom=qemu-devel-bounces+importer=patchew.org@nongnu.org; dmarc=fail(p=none dis=none) header.from=redhat.com Return-Path: Received: from lists.gnu.org (208.118.235.17 [208.118.235.17]) by mx.zohomail.com with SMTPS id 1521202077812184.5678460518683; Fri, 16 Mar 2018 05:07:57 -0700 (PDT) Received: from localhost ([::1]:56338 helo=lists.gnu.org) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1ewo9F-0000Rz-JR for importer@patchew.org; Fri, 16 Mar 2018 08:07:49 -0400 Received: from eggs.gnu.org ([2001:4830:134:3::10]:59252) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1ewnwS-0006pt-9X for qemu-devel@nongnu.org; Fri, 16 Mar 2018 07:54:37 -0400 Received: from Debian-exim by eggs.gnu.org with spam-scanned (Exim 4.71) (envelope-from ) id 1ewnwQ-0002lG-Jc for qemu-devel@nongnu.org; Fri, 16 Mar 2018 07:54:36 -0400 Received: from mx3-rdu2.redhat.com ([66.187.233.73]:37992 helo=mx1.redhat.com) by eggs.gnu.org with esmtps (TLS1.0:DHE_RSA_AES_256_CBC_SHA1:32) (Exim 4.71) (envelope-from ) id 1ewnwQ-0002kx-E4 for qemu-devel@nongnu.org; Fri, 16 Mar 2018 07:54:34 -0400 Received: from smtp.corp.redhat.com (int-mx05.intmail.prod.int.rdu2.redhat.com [10.11.54.5]) (using TLSv1.2 with cipher AECDH-AES256-SHA (256/256 bits)) (No client certificate requested) by mx1.redhat.com (Postfix) with ESMTPS id F27CE8151D47 for ; Fri, 16 Mar 2018 11:54:33 +0000 (UTC) Received: from secure.mitica (ovpn-116-95.ams2.redhat.com [10.36.116.95]) by smtp.corp.redhat.com (Postfix) with ESMTP id E86CFC225F; Fri, 16 Mar 2018 11:54:32 +0000 (UTC) From: Juan Quintela To: qemu-devel@nongnu.org Date: Fri, 16 Mar 2018 12:54:03 +0100 Message-Id: <20180316115403.4148-16-quintela@redhat.com> In-Reply-To: <20180316115403.4148-1-quintela@redhat.com> References: <20180316115403.4148-1-quintela@redhat.com> X-Scanned-By: MIMEDefang 2.79 on 10.11.54.5 X-Greylist: Sender IP whitelisted, not delayed by milter-greylist-4.5.16 (mx1.redhat.com [10.11.55.8]); Fri, 16 Mar 2018 11:54:34 +0000 (UTC) X-Greylist: inspected by milter-greylist-4.5.16 (mx1.redhat.com [10.11.55.8]); Fri, 16 Mar 2018 11:54:34 +0000 (UTC) for IP:'10.11.54.5' DOMAIN:'int-mx05.intmail.prod.int.rdu2.redhat.com' HELO:'smtp.corp.redhat.com' FROM:'quintela@redhat.com' RCPT:'' X-detected-operating-system: by eggs.gnu.org: GNU/Linux 2.2.x-3.x [generic] [fuzzy] X-Received-From: 66.187.233.73 Subject: [Qemu-devel] [PATCH v11 15/15] [RFC] migration: Send pages through the multifd channels X-BeenThere: qemu-devel@nongnu.org X-Mailman-Version: 2.1.21 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Cc: lvivier@redhat.com, dgilbert@redhat.com, peterx@redhat.com Errors-To: qemu-devel-bounces+importer=patchew.org@nongnu.org Sender: "Qemu-devel" X-ZohoMail: RSF_0 Z_629925259 SPT_0 Content-Transfer-Encoding: quoted-printable MIME-Version: 1.0 Content-Type: text/plain; charset="utf-8" Migration ends correctly, but there is still a race between clean up and last synchronization. Signed-off-by: Juan Quintela --- migration/ram.c | 240 ++++++++++++++++++++++++++++++++++++++++++---= ---- migration/trace-events | 3 +- 2 files changed, 211 insertions(+), 32 deletions(-) diff --git a/migration/ram.c b/migration/ram.c index 0132de6e02..d8ad456eca 100644 --- a/migration/ram.c +++ b/migration/ram.c @@ -408,6 +408,16 @@ typedef struct { uint8_t id; } __attribute__((packed)) MultiFDInit_t; =20 +typedef struct { + uint32_t magic; + uint32_t version; + uint32_t size; + uint32_t used; + uint32_t seq; + char ramblock[256]; + uint64_t offset[]; +} __attribute__((packed)) MultiFDPacket_t; + typedef struct { /* number of used pages */ uint32_t used; @@ -422,7 +432,7 @@ typedef struct { RAMBlock *block; } MultiFDPages_t; =20 -struct MultiFDSendParams { +typedef struct { /* not changed */ uint8_t id; char *name; @@ -440,8 +450,29 @@ struct MultiFDSendParams { /* protected by multifd mutex */ /* has the thread finish the last submitted job */ bool done; -}; -typedef struct MultiFDSendParams MultiFDSendParams; + uint32_t packet_len; + MultiFDPacket_t *packet; +} MultiFDSendParams; + +typedef struct { + /* not changed */ + uint8_t id; + char *name; + QemuThread thread; + QIOChannel *c; + QemuSemaphore sem; + QemuMutex mutex; + bool running; + /* protected by param mutex */ + bool quit; + bool sync; + MultiFDPages_t *pages; + /* how many patckets has recv this channel */ + uint32_t packets_recv; + bool done; + uint32_t packet_len; + MultiFDPacket_t *packet; +} MultiFDRecvParams; =20 static int multifd_send_initial_packet(MultiFDSendParams *p, Error **errp) { @@ -502,6 +533,80 @@ static int multifd_recv_initial_packet(QIOChannel *c, = Error **errp) return msg.id; } =20 +static void multifd_send_fill_packet(MultiFDSendParams *p) +{ + MultiFDPacket_t *packet =3D p->packet; + int i; + + packet->magic =3D cpu_to_be32(MULTIFD_MAGIC); + packet->version =3D cpu_to_be32(MULTIFD_VERSION); + packet->size =3D cpu_to_be32(migrate_multifd_page_count()); + packet->used =3D cpu_to_be32(p->pages->used); + packet->seq =3D cpu_to_be32(p->pages->seq); + + for (i =3D 0; i < p->pages->used; i++) { + packet->offset[i] =3D cpu_to_be64(p->pages->offset[i]); + } + + strncpy(packet->ramblock, p->pages->block->idstr, 256); +} + +static int multifd_recv_unfill_packet(MultiFDRecvParams *p, Error **errp) +{ + MultiFDPacket_t *packet =3D p->packet; + RAMBlock *block; + int i; + + be32_to_cpus(&packet->magic); + if (packet->magic !=3D MULTIFD_MAGIC) { + error_setg(errp, "multifd: received packet " + "version %d and expected version %d", + packet->magic, MULTIFD_VERSION); + return -1; + } + + be32_to_cpus(&packet->version); + if (packet->version !=3D MULTIFD_VERSION) { + error_setg(errp, "multifd: received packet " + "version %d and expected version %d", + packet->version, MULTIFD_VERSION); + return -1; + } + + be32_to_cpus(&packet->size); + if (packet->size > migrate_multifd_page_count()) { + error_setg(errp, "multifd: received packet " + "with size %d and expected maximum size %d", + packet->size, migrate_multifd_page_count()) ; + return -1; + } + + p->pages->used =3D be32_to_cpu(packet->used); + if (p->pages->used > packet->size) { + error_setg(errp, "multifd: received packet " + "with size %d and expected maximum size %d", + p->pages->used, packet->size) ; + return -1; + } + + be32_to_cpus(&packet->seq); + + block =3D qemu_ram_block_by_name(packet->ramblock); + if (!block) { + error_setg(errp, "multifd: unknown ram block %s", + packet->ramblock); + return -1; + } + + for (i =3D 0; i < p->pages->used; i++) { + ram_addr_t offset =3D be64_to_cpu(packet->offset[i]); + + p->pages->iov[i].iov_base =3D block->host + offset; + p->pages->iov[i].iov_len =3D TARGET_PAGE_SIZE; + } + return 0; +} + struct { MultiFDSendParams *params; /* number of created threads */ @@ -583,6 +688,9 @@ int multifd_save_cleanup(Error **errp) p->name =3D NULL; multifd_pages_clear(p->pages); p->pages =3D NULL; + p->packet_len =3D 0; + g_free(p->packet); + p->packet =3D NULL; } qemu_sem_destroy(&multifd_send_state->sem_main); g_free(multifd_send_state->params); @@ -632,12 +740,13 @@ static void *multifd_send_thread(void *opaque) { MultiFDSendParams *p =3D opaque; Error *local_err =3D NULL; + int ret; =20 trace_multifd_send_thread_start(p->id); =20 - if (multifd_send_initial_packet(p, &local_err) < 0) { - multifd_send_terminate_threads(local_err); - return NULL; + ret =3D multifd_send_initial_packet(p, &local_err); + if (ret < 0) { + goto out; } qemu_sem_post(&multifd_send_state->sem); =20 @@ -651,17 +760,28 @@ static void *multifd_send_thread(void *opaque) continue; } if (p->quit) { - p->running =3D false; qemu_mutex_unlock(&p->mutex); break; } if (p->pages->used) { + Error *local_err =3D NULL; + uint32_t used; + + multifd_send_fill_packet(p); + used =3D p->pages->used; p->pages->used =3D 0; qemu_mutex_unlock(&p->mutex); =20 - trace_multifd_send(p->id, p->pages->seq, p->pages->used); - /* ToDo: send page here */ - + ret =3D qio_channel_write_all(p->c, (void *)p->packet, + p->packet_len, &local_err); + if (ret !=3D 0) { + break; + } + trace_multifd_send(p->id, p->pages->seq, used); + ret =3D qio_channel_writev_all(p->c, p->pages->iov, used, &loc= al_err); + if (ret !=3D 0) { + break; + } qemu_mutex_lock(&multifd_send_state->mutex); p->done =3D true; p->packets_sent++; @@ -671,6 +791,15 @@ static void *multifd_send_thread(void *opaque) } qemu_mutex_unlock(&p->mutex); } +out: + if (ret) { + multifd_send_terminate_threads(local_err); + } + + qemu_mutex_lock(&p->mutex); + p->running =3D false; + qemu_mutex_unlock(&p->mutex); + trace_multifd_send_thread_end(p->id, p->packets_sent); =20 return NULL; @@ -722,6 +851,9 @@ int multifd_save_setup(void) p->id =3D i; p->done =3D true; multifd_pages_init(&p->pages, page_count); + p->packet_len =3D sizeof(MultiFDPacket_t) + + sizeof(ram_addr_t) * page_count; + p->packet =3D g_malloc0(p->packet_len); p->name =3D g_strdup_printf("multifdsend_%d", i); socket_send_channel_create(multifd_new_send_channel_async, p); } @@ -774,25 +906,6 @@ static void multifd_send_page(RAMBlock *block, ram_add= r_t offset, qemu_sem_post(&p->sem); } =20 -struct MultiFDRecvParams { - /* not changed */ - uint8_t id; - char *name; - QemuThread thread; - QIOChannel *c; - QemuSemaphore sem; - QemuMutex mutex; - bool running; - /* protected by param mutex */ - bool quit; - bool sync; - /* how many patckets has recv this channel */ - uint32_t packets_recv; - MultiFDPages_t *pages; - bool done; -}; -typedef struct MultiFDRecvParams MultiFDRecvParams; - struct { MultiFDRecvParams *params; /* number of created threads */ @@ -848,6 +961,9 @@ int multifd_load_cleanup(Error **errp) p->name =3D NULL; multifd_pages_clear(p->pages); p->pages =3D NULL; + p->packet_len =3D 0; + g_free(p->packet); + p->packet =3D NULL; } qemu_sem_destroy(&multifd_recv_state->sem_main); g_free(multifd_recv_state->params); @@ -892,12 +1008,34 @@ static void multifd_recv_sync_main(void) trace_multifd_recv_sync_main(); } =20 +static gboolean recv_channel_ready(QIOChannel *ioc, + GIOCondition condition, + gpointer opaque) +{ + MultiFDRecvParams *p =3D opaque; + + if (condition !=3D G_IO_IN) { + return G_SOURCE_REMOVE; + } + + qemu_mutex_lock(&p->mutex); + p->done =3D false; + qemu_mutex_unlock(&p->mutex); + qemu_sem_post(&p->sem); + + return G_SOURCE_CONTINUE; + +} + static void *multifd_recv_thread(void *opaque) { MultiFDRecvParams *p =3D opaque; =20 trace_multifd_recv_thread_start(p->id); =20 + qio_channel_add_watch(p->c, G_IO_IN | G_IO_HUP | G_IO_ERR, + recv_channel_ready, p, NULL); + while (true) { qemu_sem_wait(&p->sem); qemu_mutex_lock(&p->mutex); @@ -907,15 +1045,51 @@ static void *multifd_recv_thread(void *opaque) qemu_sem_post(&multifd_recv_state->sem_main); continue; } + if (!p->done) { + Error *local_err =3D NULL; + int ret; + + qemu_mutex_unlock(&p->mutex); + + ret =3D qio_channel_read_all(p->c, (void *)p->packet, + p->packet_len, &local_err); + if (ret !=3D 0) { + multifd_recv_terminate_threads(local_err); + break; + } + + ret =3D multifd_recv_unfill_packet(p, &local_err); + if (ret < 0) { + multifd_recv_terminate_threads(local_err); + break; + } + + trace_multifd_recv(p->id, p->pages->seq, p->pages->used); + ret =3D qio_channel_readv_all(p->c, p->pages->iov, + p->pages->used, &local_err); + if (ret !=3D 0) { + multifd_recv_terminate_threads(local_err); + break; + } + qemu_mutex_lock(&p->mutex); + p->done =3D true; + p->packets_recv++; + qemu_mutex_unlock(&p->mutex); + + continue; + } if (p->quit) { - p->running =3D false; qemu_mutex_unlock(&p->mutex); break; } qemu_mutex_unlock(&p->mutex); } =20 - trace_multifd_recv_thread_end(p->id); + qemu_mutex_lock(&p->mutex); + p->running =3D false; + qemu_mutex_unlock(&p->mutex); + + trace_multifd_recv_thread_end(p->id, p->packets_recv); return NULL; } =20 @@ -940,9 +1114,13 @@ int multifd_load_setup(void) qemu_mutex_init(&p->mutex); qemu_sem_init(&p->sem, 0); p->quit =3D false; + p->done =3D true; p->id =3D i; p->name =3D g_strdup_printf("multifdrecv_%d", i); multifd_pages_init(&p->pages, page_count); + p->packet_len =3D sizeof(MultiFDPacket_t) + + sizeof(ram_addr_t) * page_count; + p->packet =3D g_malloc0(p->packet_len); } return 0; } diff --git a/migration/trace-events b/migration/trace-events index 06a9ead811..a6c1c4b20c 100644 --- a/migration/trace-events +++ b/migration/trace-events @@ -86,8 +86,9 @@ multifd_recv_sync_wait(uint8_t id, bool quit, bool runnin= g) "channel %d quit %d multifd_send_thread_start(uint8_t id) "%d" multifd_send_thread_end(uint8_t id, uint32_t packets) "channel %d packets = %d" multifd_recv_thread_start(uint8_t id) "%d" -multifd_recv_thread_end(uint8_t id) "%d" +multifd_recv_thread_end(uint8_t id, uint32_t packets) "channel %d packets = %d" multifd_send(uint8_t id, int seq, int num) "channel %d sequence %d num pag= es %d" +multifd_recv(uint8_t id, int seq, int num) "channel %d sequence %d num pag= es %d" =20 # migration/migration.c await_return_path_close_on_source_close(void) "" --=20 2.14.3