Series comparison

-[Qemu-devel] [PULL 00/26] Net patches
+[PULL V3 00/13] Net patches
-The following changes since commit a73549f99612f758dec0fdea6ae1c30b6c709a0b:
+The following changes since commit f8d75e10d3e0033a0a29a7a7e4777a4fbc17a016:
-  Merge remote-tracking branch 'remotes/kraxel/tags/ui-20181012-pull-request' into staging (2018-10-12 16:45:51 +0100)
+  Merge remote-tracking branch 'remotes/legoater/tags/pull-ppc-20220112' into staging (2022-01-13 11:18:24 +0000)
 are available in the git repository at:
   https://github.com/jasowang/qemu.git tags/net-pull-request
-for you to fetch changes up to a7ec0077c2db445d6bae421963188367d2695bd6:
+for you to fetch changes up to 818692f0a01587d02220916b31d5bb8e7dced611:
-  qemu-options: Fix bad "macaddr" property in the documentation (2018-10-15 16:14:15 +0800)
+  net/vmnet: update MAINTAINERS list (2022-01-14 12:58:19 +0800)
 ----------------------------------------------------------------
+Changes since V2:
+- Try to make vmnet work on some old mac version
 ----------------------------------------------------------------
-Jason Wang (4):
+Peter Foley (2):
-      ne2000: fix possible out of bound access in ne2000_receive
+      net/tap: Set return code on failure
-      rtl8139: fix possible out of bound access
+      net: Fix uninitialized data usage
       pcnet: fix possible buffer overflow
       net: ignore packet size greater than INT_MAX
-Martin Wilck (1):
+Philippe Mathieu-Daudé (1):
-      e1000: indicate dropped packets in HW counters
+      hw/net/vmxnet3: Log guest-triggerable errors using LOG_GUEST_ERROR
-Thomas Huth (1):
+Rao Lei (1):
-      qemu-options: Fix bad "macaddr" property in the documentation
+      net/filter: Optimize filter_send to coroutine
-Zhang Chen (15):
+Vladislav Yaroshchuk (7):
-      filter-rewriter: Add TCP state machine and fix memory leak in connection_track_table
+      net/vmnet: add vmnet dependency and customizable option
-      colo-compare: implement the process of checkpoint
+      net/vmnet: add vmnet backends to qapi/net
-      colo-compare: use notifier to notify packets comparing result
+      net/vmnet: implement shared mode (vmnet-shared)
-;5202;0c1;5202;0c      COLO: integrate colo compare with colo frame
+      net/vmnet: implement host mode (vmnet-host)
-      COLO: Add block replication into colo process
+      net/vmnet: implement bridged mode (vmnet-bridged)
-      COLO: Remove colo_state migration struct
+      net/vmnet: update qemu-options.hx
-      COLO: Load dirty pages into SVM's RAM cache firstly
+      net/vmnet: update MAINTAINERS list
       ram/COLO: Record the dirty pages that SVM received
       COLO: Flush memory data from ram cache
       qapi/migration.json: Rename COLO unknown mode to none mode.
       qapi: Add new command to query colo status
       savevm: split the process of different stages for loadvm/savevm
       filter: Add handle_event method for NetFilterClass
       filter-rewriter: handle checkpoint and failover event
       docs: Add COLO status diagram to COLO-FT.txt
-liujunjie (1):
+Zhang Chen (2):
-      clean up callback when del virtqueue
+      net/colo-compare.c: Optimize compare order for performance
       net/colo-compare.c: Update the default value comments
-zhanghailiang (4):
+ MAINTAINERS                   |   5 +
-      qmp event: Add COLO_EXIT event to notify users while exited COLO
+ hw/net/vmxnet3.c              |   4 +-
-      COLO: flush host dirty ram from cache
+ meson.build                   |  16 +-
-      COLO: notify net filters about checkpoint/failover event
+ meson_options.txt             |   2 +
-      COLO: quick failover process by kick COLO thread
+ net/clients.h                 |  11 ++
+ net/colo-compare.c            |  28 ++--
- docs/COLO-FT.txt          |  34 ++++++++
+ net/filter-mirror.c           |  66 +++++++--
- hw/net/e1000.c            |  16 +++-
+ net/meson.build               |   7 +
- hw/net/ne2000.c           |   4 +-
+ net/net.c                     |  10 ++
- hw/net/pcnet.c            |   4 +-
+ net/tap-linux.c               |   1 +
- hw/net/rtl8139.c          |   8 +-
+ net/tap.c                     |   1 +
- hw/net/trace-events       |   3 +
+ net/vmnet-bridged.m           | 120 +++++++++++++++
- hw/virtio/virtio.c        |   2 +
+ net/vmnet-common.m            | 333 ++++++++++++++++++++++++++++++++++++++++++
- include/exec/ram_addr.h   |   1 +
+ net/vmnet-host.c              | 122 ++++++++++++++++
- include/migration/colo.h  |  11 ++-
+ net/vmnet-shared.c            | 100 +++++++++++++
- include/net/filter.h      |   5 ++
+ net/vmnet_int.h               |  48 ++++++
- migration/Makefile.objs   |   2 +-
+ qapi/net.json                 | 133 ++++++++++++++++-
- migration/colo-comm.c     |  76 -----------------
+ qemu-options.hx               |  25 ++++
- migration/colo-failover.c |   2 +-
+ scripts/meson-buildoptions.sh |   3 +
- migration/colo.c          | 212 +++++++++++++++++++++++++++++++++++++++++++---
+files changed, 1004 insertions(+), 31 deletions(-)
- migration/migration.c     |  46 ++++++++--
+ create mode 100644 net/vmnet-bridged.m
- migration/ram.c           | 166 +++++++++++++++++++++++++++++++++++-
+ create mode 100644 net/vmnet-common.m
- migration/ram.h           |   4 +
+ create mode 100644 net/vmnet-host.c
- migration/savevm.c        |  53 ++++++++++--
+ create mode 100644 net/vmnet-shared.c
- migration/savevm.h        |   5 ++
+ create mode 100644 net/vmnet_int.h
  migration/trace-events    |   3 +
  net/colo-compare.c        | 115 ++++++++++++++++++++++---
  net/colo-compare.h        |  24 ++++++
  net/colo.c                |  10 ++-
  net/colo.h                |  11 +--
  net/filter-rewriter.c     | 166 +++++++++++++++++++++++++++++++++---
  net/filter.c              |  17 ++++
  net/net.c                 |  26 +++++-
  qapi/migration.json       |  80 +++++++++++++++--
  qemu-options.hx           |   2 +-
  vl.c                      |   2 -
 files changed, 958 insertions(+), 152 deletions(-)
  delete mode 100644 migration/colo-comm.c
  create mode 100644 net/colo-compare.h

-[Qemu-devel] [PULL 01/26] filter-rewriter: Add TCP state machine and fix memory leak in connection_track_table
+Deleted patch
-From: Zhang Chen <zhangckid@gmail.com>
-We add almost full TCP state machine in filter-rewriter, except
-TCPS_LISTEN and some simplify in VM active close FIN states.
-The reason for this simplify job is because guest kernel will track
-the TCP status and wait 2MSL time too, if client resend the FIN packet,
-guest will resend the last ACK, so we needn't wait 2MSL time in filter-rewriter.
-After a net connection is closed, we didn't clear its related resources
-in connection_track_table, which will lead to memory leak.
-Let's track the state of net connection, if it is closed, its related
-resources will be cleared up.
-Signed-off-by: zhanghailiang <zhang.zhanghailiang@huawei.com>
-Signed-off-by: Zhang Chen <zhangckid@gmail.com>
-Signed-off-by: Zhang Chen <chen.zhang@intel.com>
-Signed-off-by: Jason Wang <jasowang@redhat.com>
----
- net/colo.c            |   2 +-
- net/colo.h            |   9 ++---
- net/filter-rewriter.c | 109 +++++++++++++++++++++++++++++++++++++++++++++-----
-files changed, 104 insertions(+), 16 deletions(-)
-diff --git a/net/colo.c b/net/colo.c
-index XXXXXXX..XXXXXXX 100644
---- a/net/colo.c
-+++ b/net/colo.c
-@@ -XXX,XX +XXX,XX @@ Connection *connection_new(ConnectionKey *key)
-     conn->ip_proto = key->ip_proto;
-     conn->processing = false;
-     conn->offset = 0;
--    conn->syn_flag = 0;
-+    conn->tcp_state = TCPS_CLOSED;
-     conn->pack = 0;
-     conn->sack = 0;
-     g_queue_init(&conn->primary_list);
-diff --git a/net/colo.h b/net/colo.h
-index XXXXXXX..XXXXXXX 100644
---- a/net/colo.h
-+++ b/net/colo.h
-@@ -XXX,XX +XXX,XX @@
- #include "slirp/slirp.h"
- #include "qemu/jhash.h"
- #include "qemu/timer.h"
-+#include "slirp/tcp.h"
- #define HASHTABLE_MAX_SIZE 16384
-@@ -XXX,XX +XXX,XX @@ typedef struct Connection {
-     uint32_t sack;
-     /* offset = secondary_seq - primary_seq */
-     tcp_seq  offset;
--    /*
--     * we use this flag update offset func
--     * run once in independent tcp connection
--     */
--    int syn_flag;
-+
-+    int tcp_state; /* TCP FSM state */
-+    tcp_seq fin_ack_seq; /* the seq of 'fin=1,ack=1' */
- } Connection;
- uint32_t connection_key_hash(const void *opaque);
-diff --git a/net/filter-rewriter.c b/net/filter-rewriter.c
-index XXXXXXX..XXXXXXX 100644
---- a/net/filter-rewriter.c
-+++ b/net/filter-rewriter.c
-@@ -XXX,XX +XXX,XX @@ static int is_tcp_packet(Packet *pkt)
- }
- /* handle tcp packet from primary guest */
--static int handle_primary_tcp_pkt(NetFilterState *nf,
-+static int handle_primary_tcp_pkt(RewriterState *rf,
-                                   Connection *conn,
--                                  Packet *pkt)
-+                                  Packet *pkt, ConnectionKey *key)
- {
-     struct tcphdr *tcp_pkt;
-@@ -XXX,XX +XXX,XX @@ static int handle_primary_tcp_pkt(NetFilterState *nf,
-         trace_colo_filter_rewriter_conn_offset(conn->offset);
-     }
-+    if (((tcp_pkt->th_flags & (TH_ACK | TH_SYN)) == (TH_ACK | TH_SYN)) &&
-+        conn->tcp_state == TCPS_SYN_SENT) {
-+        conn->tcp_state = TCPS_ESTABLISHED;
-+    }
-+
-     if (((tcp_pkt->th_flags & (TH_ACK | TH_SYN)) == TH_SYN)) {
-         /*
-          * we use this flag update offset func
-          * run once in independent tcp connection
-          */
--        conn->syn_flag = 1;
-+        conn->tcp_state = TCPS_SYN_RECEIVED;
-     }
-     if (((tcp_pkt->th_flags & (TH_ACK | TH_SYN)) == TH_ACK)) {
--        if (conn->syn_flag) {
-+        if (conn->tcp_state == TCPS_SYN_RECEIVED) {
-             /*
-              * offset = secondary_seq - primary seq
-              * ack packet sent by guest from primary node,
-              * so we use th_ack - 1 get primary_seq
-              */
-             conn->offset -= (ntohl(tcp_pkt->th_ack) - 1);
--            conn->syn_flag = 0;
-+            conn->tcp_state = TCPS_ESTABLISHED;
-         }
-         if (conn->offset) {
-             /* handle packets to the secondary from the primary */
-@@ -XXX,XX +XXX,XX @@ static int handle_primary_tcp_pkt(NetFilterState *nf,
-             net_checksum_calculate((uint8_t *)pkt->data + pkt->vnet_hdr_len,
-                                    pkt->size - pkt->vnet_hdr_len);
-         }
-+
-+        /*
-+         * Passive close step 3
-+         */
-+        if ((conn->tcp_state == TCPS_LAST_ACK) &&
-+            (ntohl(tcp_pkt->th_ack) == (conn->fin_ack_seq + 1))) {
-+            conn->tcp_state = TCPS_CLOSED;
-+            g_hash_table_remove(rf->connection_track_table, key);
-+        }
-+    }
-+
-+    if ((tcp_pkt->th_flags & TH_FIN) == TH_FIN) {
-+        /*
-+         * Passive close.
-+         * Step 1:
-+         * The *server* side of this connect is VM, *client* tries to close
-+         * the connection. We will into CLOSE_WAIT status.
-+         *
-+         * Step 2:
-+         * In this step we will into LAST_ACK status.
-+         *
-+         * We got 'fin=1, ack=1' packet from server side, we need to
-+         * record the seq of 'fin=1, ack=1' packet.
-+         *
-+         * Step 3:
-+         * We got 'ack=1' packets from client side, it acks 'fin=1, ack=1'
-+         * packet from server side. From this point, we can ensure that there
-+         * will be no packets in the connection, except that, some errors
-+         * happen between the path of 'filter object' and vNIC, if this rare
-+         * case really happen, we can still create a new connection,
-+         * So it is safe to remove the connection from connection_track_table.
-+         *
-+         */
-+        if (conn->tcp_state == TCPS_ESTABLISHED) {
-+            conn->tcp_state = TCPS_CLOSE_WAIT;
-+        }
-+
-+        /*
-+         * Active close step 2.
-+         */
-+        if (conn->tcp_state == TCPS_FIN_WAIT_1) {
-+            conn->tcp_state = TCPS_TIME_WAIT;
-+            /*
-+             * For simplify implementation, we needn't wait 2MSL time
-+             * in filter rewriter. Because guest kernel will track the
-+             * TCP status and wait 2MSL time, if client resend the FIN
-+             * packet, guest will apply the last ACK too.
-+             */
-+            conn->tcp_state = TCPS_CLOSED;
-+            g_hash_table_remove(rf->connection_track_table, key);
-+        }
-     }
-     return 0;
- }
- /* handle tcp packet from secondary guest */
--static int handle_secondary_tcp_pkt(NetFilterState *nf,
-+static int handle_secondary_tcp_pkt(RewriterState *rf,
-                                     Connection *conn,
--                                    Packet *pkt)
-+                                    Packet *pkt, ConnectionKey *key)
- {
-     struct tcphdr *tcp_pkt;
-@@ -XXX,XX +XXX,XX @@ static int handle_secondary_tcp_pkt(NetFilterState *nf,
-         trace_colo_filter_rewriter_conn_offset(conn->offset);
-     }
--    if (((tcp_pkt->th_flags & (TH_ACK | TH_SYN)) == (TH_ACK | TH_SYN))) {
-+    if (conn->tcp_state == TCPS_SYN_RECEIVED &&
-+        ((tcp_pkt->th_flags & (TH_ACK | TH_SYN)) == (TH_ACK | TH_SYN))) {
-         /*
-          * save offset = secondary_seq and then
-          * in handle_primary_tcp_pkt make offset
-@@ -XXX,XX +XXX,XX @@ static int handle_secondary_tcp_pkt(NetFilterState *nf,
-         conn->offset = ntohl(tcp_pkt->th_seq);
-     }
-+    /* VM active connect */
-+    if (conn->tcp_state == TCPS_CLOSED &&
-+        ((tcp_pkt->th_flags & (TH_ACK | TH_SYN)) == TH_SYN)) {
-+        conn->tcp_state = TCPS_SYN_SENT;
-+    }
-+
-     if ((tcp_pkt->th_flags & (TH_ACK | TH_SYN)) == TH_ACK) {
-         /* Only need to adjust seq while offset is Non-zero */
-         if (conn->offset) {
-@@ -XXX,XX +XXX,XX @@ static int handle_secondary_tcp_pkt(NetFilterState *nf,
-         }
-     }
-+    /*
-+     * Passive close step 2:
-+     */
-+    if (conn->tcp_state == TCPS_CLOSE_WAIT &&
-+        (tcp_pkt->th_flags & (TH_ACK | TH_FIN)) == (TH_ACK | TH_FIN)) {
-+        conn->fin_ack_seq = ntohl(tcp_pkt->th_seq);
-+        conn->tcp_state = TCPS_LAST_ACK;
-+    }
-+
-+    /*
-+     * Active close
-+     *
-+     * Step 1:
-+     * The *server* side of this connect is VM, *server* tries to close
-+     * the connection.
-+     *
-+     * Step 2:
-+     * We will into CLOSE_WAIT status.
-+     * We simplify the TCPS_FIN_WAIT_2, TCPS_TIME_WAIT and
-+     * CLOSING status.
-+     */
-+    if (conn->tcp_state == TCPS_ESTABLISHED &&
-+        (tcp_pkt->th_flags & (TH_ACK | TH_FIN)) == TH_FIN) {
-+        conn->tcp_state = TCPS_FIN_WAIT_1;
-+    }
-+
-     return 0;
- }
-@@ -XXX,XX +XXX,XX @@ static ssize_t colo_rewriter_receive_iov(NetFilterState *nf,
-         if (sender == nf->netdev) {
-             /* NET_FILTER_DIRECTION_TX */
--            if (!handle_primary_tcp_pkt(nf, conn, pkt)) {
-+            if (!handle_primary_tcp_pkt(s, conn, pkt, &key)) {
-                 qemu_net_queue_send(s->incoming_queue, sender, 0,
-                 (const uint8_t *)pkt->data, pkt->size, NULL);
-                 packet_destroy(pkt, NULL);
-@@ -XXX,XX +XXX,XX @@ static ssize_t colo_rewriter_receive_iov(NetFilterState *nf,
-             }
-         } else {
-             /* NET_FILTER_DIRECTION_RX */
--            if (!handle_secondary_tcp_pkt(nf, conn, pkt)) {
-+            if (!handle_secondary_tcp_pkt(s, conn, pkt, &key)) {
-                 qemu_net_queue_send(s->incoming_queue, sender, 0,
-                 (const uint8_t *)pkt->data, pkt->size, NULL);
-                 packet_destroy(pkt, NULL);
---
-.5.0

-[Qemu-devel] [PULL 25/26] e1000: indicate dropped packets in HW counters
+[PULL V3 01/13] hw/net/vmxnet3: Log guest-triggerable errors using LOG_GUEST_ERROR
-From: Martin Wilck <mwilck@suse.com>
+From: Philippe Mathieu-Daudé <philmd@redhat.com>
-The e1000 emulation silently discards RX packets if there's
+The "Interrupt Cause" register (VMXNET3_REG_ICR) is read-only.
-insufficient space in the ring buffer. This leads to errors
+Write accesses are ignored. Log them with as LOG_GUEST_ERROR
-on higher-level protocols in the guest, with no indication
+instead of aborting:
 about the error cause.
-This patch increments the "Missed Packets Count" (MPC) and
+  [R +0.239743] writeq 0xe0002031 0x46291a5a55460800
-"Receive No Buffers Count" (RNBC) HW counters in this case.
+  ERROR:hw/net/vmxnet3.c:1819:vmxnet3_io_bar1_write: code should not be reached
-As the emulation has no FIFO for buffering packets that can't
+  Thread 1 "qemu-system-i38" received signal SIGABRT, Aborted.
-immediately be pushed to the guest, these two registers are
+  (gdb) bt
-practically equivalent (see 10.2.7.4, 10.2.7.33 in
+  #3  0x74c397d3 in __GI_abort () at abort.c:79
-https://www.intel.com/content/www/us/en/embedded/products/networking/82574l-gbe-controller-datasheet.html).
+  #4  0x76d3cd4c in g_assertion_message (domain=<optimized out>, file=<optimized out>, line=<optimized out>, func=<optimized out>, message=<optimized out>) at ../glib/gtestutils.c:3223
   #5  0x76d9d45f in g_assertion_message_expr
       (domain=0x0, file=0x59fc2e53 "hw/net/vmxnet3.c", line=1819, func=0x59fc11e0 <__func__.vmxnet3_io_bar1_write> "vmxnet3_io_bar1_write", expr=<optimized out>)
       at ../glib/gtestutils.c:3249
   #6  0x57e80a3a in vmxnet3_io_bar1_write (opaque=0x62814100, addr=56, val=70, size=4) at hw/net/vmxnet3.c:1819
   #7  0x58c2d894 in memory_region_write_accessor (mr=0x62816b90, addr=56, value=0x7fff9450, size=4, shift=0, mask=4294967295, attrs=...) at softmmu/memory.c:492
   #8  0x58c2d1d2 in access_with_adjusted_size (addr=56, value=0x7fff9450, size=1, access_size_min=4, access_size_max=4, access_fn=
 x58c2d290 <memory_region_write_accessor>, mr=0x62816b90, attrs=...) at softmmu/memory.c:554
   #9  0x58c2bae7 in memory_region_dispatch_write (mr=0x62816b90, addr=56, data=70, op=MO_8, attrs=...) at softmmu/memory.c:1504
   #10 0x58bfd034 in flatview_write_continue (fv=0x606000181700, addr=0xe0002038, attrs=..., ptr=0x7fffb9e0, len=1, addr1=56, l=1, mr=0x62816b90)
       at softmmu/physmem.c:2782
   #11 0x58beba00 in flatview_write (fv=0x606000181700, addr=0xe0002031, attrs=..., buf=0x7fffb9e0, len=8) at softmmu/physmem.c:2822
   #12 0x58beb589 in address_space_write (as=0x608000015f20, addr=0xe0002031, attrs=..., buf=0x7fffb9e0, len=8) at softmmu/physmem.c:2914
-On a Linux guest, the register content  will be reflected in
+Reported-by: Dike <dike199774@qq.com>
-the "rx_missed_errors" and "rx_no_buffer_count" stats from
+Reported-by: Duhao <504224090@qq.com>
-"ethtool -S", and in the "missed" stat from "ip -s -s link show",
+BugLink: https://bugzilla.redhat.com/show_bug.cgi?id=2032932
-giving at least some hint about the error cause inside the guest.
+Signed-off-by: Philippe Mathieu-Daudé <philmd@redhat.com>
 If the cause is known, problems like this can often be avoided
 easily, by increasing the number of RX descriptors in the guest
 e1000 driver (e.g under Linux, "e1000.RxDescriptors=1024").
 The patch also adds a qemu trace message for this condition.
 Signed-off-by: Martin Wilck <mwilck@suse.com>
 Signed-off-by: Jason Wang <jasowang@redhat.com>
 ---
- hw/net/e1000.c      | 16 +++++++++++++---
+ hw/net/vmxnet3.c | 4 +++-
- hw/net/trace-events |  3 +++
+file changed, 3 insertions(+), 1 deletion(-)
 files changed, 16 insertions(+), 3 deletions(-)
-diff --git a/hw/net/e1000.c b/hw/net/e1000.c
+diff --git a/hw/net/vmxnet3.c b/hw/net/vmxnet3.c
 index XXXXXXX..XXXXXXX 100644
---- a/hw/net/e1000.c
+--- a/hw/net/vmxnet3.c
-+++ b/hw/net/e1000.c
++++ b/hw/net/vmxnet3.c
-@@ -XXX,XX +XXX,XX @@
+@@ -XXX,XX +XXX,XX @@ vmxnet3_io_bar1_write(void *opaque,
- #include "qemu/range.h"
+     case VMXNET3_REG_ICR:
+         VMW_CBPRN("Write BAR1 [VMXNET3_REG_ICR] = %" PRIx64 ", size %d",
- #include "e1000x_common.h"
+                   val, size);
-+#include "trace.h"
+-        g_assert_not_reached();
++        qemu_log_mask(LOG_GUEST_ERROR,
- static const uint8_t bcast[] = {0xff, 0xff, 0xff, 0xff, 0xff, 0xff};
++                      "%s: write to read-only register VMXNET3_REG_ICR\n",
++                      TYPE_VMXNET3);
-@@ -XXX,XX +XXX,XX @@ static uint64_t rx_desc_base(E1000State *s)
+         break;
-     return (bah << 32) + bal;
- }
+     /* Event Cause Register */
 +static void
 +e1000_receiver_overrun(E1000State *s, size_t size)
 +{
 +    trace_e1000_receiver_overrun(size, s->mac_reg[RDH], s->mac_reg[RDT]);
 +    e1000x_inc_reg_if_not_full(s->mac_reg, RNBC);
 +    e1000x_inc_reg_if_not_full(s->mac_reg, MPC);
 +    set_ics(s, 0, E1000_ICS_RXO);
 +}
 +
  static ssize_t
  e1000_receive_iov(NetClientState *nc, const struct iovec *iov, int iovcnt)
  {
@@ -XXX,XX +XXX,XX @@ e1000_receive_iov(NetClientState *nc, const struct iovec *iov, int iovcnt)
      desc_offset = 0;
      total_size = size + e1000x_fcs_len(s->mac_reg);
      if (!e1000_has_rxbufs(s, total_size)) {
 -            set_ics(s, 0, E1000_ICS_RXO);
 -            return -1;
 +        e1000_receiver_overrun(s, total_size);
 +        return -1;
      }
      do {
          desc_size = total_size - desc_offset;
@@ -XXX,XX +XXX,XX @@ e1000_receive_iov(NetClientState *nc, const struct iovec *iov, int iovcnt)
              rdh_start >= s->mac_reg[RDLEN] / sizeof(desc)) {
              DBGOUT(RXERR, "RDH wraparound @%x, RDT %x, RDLEN %x\n",
                     rdh_start, s->mac_reg[RDT], s->mac_reg[RDLEN]);
 -            set_ics(s, 0, E1000_ICS_RXO);
 +            e1000_receiver_overrun(s, total_size);
              return -1;
          }
      } while (desc_offset < total_size);
 diff --git a/hw/net/trace-events b/hw/net/trace-events
 index XXXXXXX..XXXXXXX 100644
 --- a/hw/net/trace-events
 +++ b/hw/net/trace-events
@@ -XXX,XX +XXX,XX @@ net_rx_pkt_rss_ip6_ex(void) "Calculating IPv6/EX RSS  hash"
  net_rx_pkt_rss_hash(size_t rss_length, uint32_t rss_hash) "RSS hash for %zu bytes: 0x%X"
  net_rx_pkt_rss_add_chunk(void* ptr, size_t size, size_t input_offset) "Add RSS chunk %p, %zu bytes, RSS input offset %zu bytes"
 +# hw/net/e1000.c
 +e1000_receiver_overrun(size_t s, uint32_t rdh, uint32_t rdt) "Receiver overrun: dropped packet of %lu bytes, RDH=%u, RDT=%u"
 +
  # hw/net/e1000x_common.c
  e1000x_rx_can_recv_disabled(bool link_up, bool rx_enabled, bool pci_master) "link_up: %d, rx_enabled %d, pci_master %d"
  e1000x_vlan_is_vlan_pkt(bool is_vlan_pkt, uint16_t eth_proto, uint16_t vet) "Is VLAN packet: %d, ETH proto: 0x%X, VET: 0x%X"
 --
-.5.0
+.7.4

-[Qemu-devel] [PULL 24/26] net: ignore packet size greater than INT_MAX
+[PULL V3 02/13] net/tap: Set return code on failure
-There should not be a reason for passing a packet size greater than
+From: Peter Foley <pefoley@google.com>
 INT_MAX. It's usually a hint of bug somewhere, so ignore packet size
 greater than INT_MAX in qemu_deliver_packet_iov()
-CC: qemu-stable@nongnu.org
+Match the other error handling in this function.
-Reported-by: Daniel Shapira <daniel@twistlock.com>
-Reviewed-by: Michael S. Tsirkin <mst@redhat.com>
+Fixes: e7b347d0bf6 ("net: detect errors from probing vnet hdr flag for TAP devices")
 Reviewed-by: Patrick Venture <venture@google.com>
 Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org>
 Signed-off-by: Peter Foley <pefoley@google.com>
 Signed-off-by: Jason Wang <jasowang@redhat.com>
 ---
- net/net.c | 7 ++++++-
+ net/tap.c | 1 +
-file changed, 6 insertions(+), 1 deletion(-)
+file changed, 1 insertion(+)
-diff --git a/net/net.c b/net/net.c
+diff --git a/net/tap.c b/net/tap.c
 index XXXXXXX..XXXXXXX 100644
---- a/net/net.c
+--- a/net/tap.c
-+++ b/net/net.c
++++ b/net/tap.c
-@@ -XXX,XX +XXX,XX @@ ssize_t qemu_deliver_packet_iov(NetClientState *sender,
+@@ -XXX,XX +XXX,XX @@ int net_init_tap(const Netdev *netdev, const char *name,
-                                 void *opaque)
+             if (i == 0) {
- {
+                 vnet_hdr = tap_probe_vnet_hdr(fd, errp);
-     NetClientState *nc = opaque;
+                 if (vnet_hdr < 0) {
-+    size_t size = iov_size(iov, iovcnt);
++                    ret = -1;
-     int ret;
+                     goto free_fail;
+                 }
-+    if (size > INT_MAX) {
+             } else if (vnet_hdr != tap_probe_vnet_hdr(fd, NULL)) {
 +        return size;
 +    }
 +
      if (nc->link_down) {
 -        return iov_size(iov, iovcnt);
 +        return size;
      }
      if (nc->receive_disabled) {
 --
-.5.0
+.7.4

-[Qemu-devel] [PULL 23/26] pcnet: fix possible buffer overflow
+[PULL V3 03/13] net: Fix uninitialized data usage
-In pcnet_receive(), we try to assign size_ to size which converts from
+From: Peter Foley <pefoley@google.com>
 size_t to integer. This will cause troubles when size_ is greater
 INT_MAX, this will lead a negative value in size and it can then pass
 the check of size < MIN_BUF_SIZE which may lead out of bound access
 for both buf and buf1.
-Fixing by converting the type of size to size_t.
+e.g.
 15:16:20.151506 Uninitialized bytes in ioctl_common_pre at offset 0 inside [0x7ffc516af9b8, 4)
 15:16:20.151659 ==588974==WARNING: MemorySanitizer: use-of-uninitialized-value
 15:16:20.312923     #0 0x5639b88acb21 in tap_probe_vnet_hdr_len third_party/qemu/net/tap-linux.c:183:9
 15:16:20.312952     #1 0x5639b88afd66 in net_tap_fd_init third_party/qemu/net/tap.c:409:9
 15:16:20.312954     #2 0x5639b88b2d1b in net_init_tap_one third_party/qemu/net/tap.c:681:19
 15:16:20.312956     #3 0x5639b88b16a8 in net_init_tap third_party/qemu/net/tap.c:912:13
 15:16:20.312957     #4 0x5639b8890175 in net_client_init1 third_party/qemu/net/net.c:1110:9
 15:16:20.312958     #5 0x5639b888f912 in net_client_init third_party/qemu/net/net.c:1208:15
 15:16:20.312960     #6 0x5639b8894aa5 in net_param_nic third_party/qemu/net/net.c:1588:11
 15:16:20.312961     #7 0x5639b900cd18 in qemu_opts_foreach third_party/qemu/util/qemu-option.c:1135:14
 15:16:20.312962     #8 0x5639b889393c in net_init_clients third_party/qemu/net/net.c:1612:9
 15:16:20.312964     #9 0x5639b717aaf3 in qemu_create_late_backends third_party/qemu/softmmu/vl.c:1962:5
 15:16:20.312965     #10 0x5639b717aaf3 in qemu_init third_party/qemu/softmmu/vl.c:3694:5
 15:16:20.312967     #11 0x5639b71083b8 in main third_party/qemu/softmmu/main.c:49:5
 15:16:20.312968     #12 0x7f464de1d8d2 in __libc_start_main (/usr/grte/v5/lib64/libc.so.6+0x628d2)
 15:16:20.312969     #13 0x5639b6bbd389 in _start /usr/grte/v5/debug-src/src/csu/../sysdeps/x86_64/start.S:120
 15:16:20.312970
 15:16:20.312975   Uninitialized value was stored to memory at
 15:16:20.313393     #0 0x5639b88acbee in tap_probe_vnet_hdr_len third_party/qemu/net/tap-linux.c
 15:16:20.313396     #1 0x5639b88afd66 in net_tap_fd_init third_party/qemu/net/tap.c:409:9
 15:16:20.313398     #2 0x5639b88b2d1b in net_init_tap_one third_party/qemu/net/tap.c:681:19
 15:16:20.313399     #3 0x5639b88b16a8 in net_init_tap third_party/qemu/net/tap.c:912:13
 15:16:20.313400     #4 0x5639b8890175 in net_client_init1 third_party/qemu/net/net.c:1110:9
 15:16:20.313401     #5 0x5639b888f912 in net_client_init third_party/qemu/net/net.c:1208:15
 15:16:20.313403     #6 0x5639b8894aa5 in net_param_nic third_party/qemu/net/net.c:1588:11
 15:16:20.313404     #7 0x5639b900cd18 in qemu_opts_foreach third_party/qemu/util/qemu-option.c:1135:14
 15:16:20.313405     #8 0x5639b889393c in net_init_clients third_party/qemu/net/net.c:1612:9
 15:16:20.313407     #9 0x5639b717aaf3 in qemu_create_late_backends third_party/qemu/softmmu/vl.c:1962:5
 15:16:20.313408     #10 0x5639b717aaf3 in qemu_init third_party/qemu/softmmu/vl.c:3694:5
 15:16:20.313409     #11 0x5639b71083b8 in main third_party/qemu/softmmu/main.c:49:5
 15:16:20.313410     #12 0x7f464de1d8d2 in __libc_start_main (/usr/grte/v5/lib64/libc.so.6+0x628d2)
 15:16:20.313412     #13 0x5639b6bbd389 in _start /usr/grte/v5/debug-src/src/csu/../sysdeps/x86_64/start.S:120
 15:16:20.313413
 15:16:20.313417   Uninitialized value was stored to memory at
 15:16:20.313791     #0 0x5639b88affbd in net_tap_fd_init third_party/qemu/net/tap.c:400:26
 15:16:20.313826     #1 0x5639b88b2d1b in net_init_tap_one third_party/qemu/net/tap.c:681:19
 15:16:20.313829     #2 0x5639b88b16a8 in net_init_tap third_party/qemu/net/tap.c:912:13
 15:16:20.313831     #3 0x5639b8890175 in net_client_init1 third_party/qemu/net/net.c:1110:9
 15:16:20.313836     #4 0x5639b888f912 in net_client_init third_party/qemu/net/net.c:1208:15
 15:16:20.313838     #5 0x5639b8894aa5 in net_param_nic third_party/qemu/net/net.c:1588:11
 15:16:20.313839     #6 0x5639b900cd18 in qemu_opts_foreach third_party/qemu/util/qemu-option.c:1135:14
 15:16:20.313841     #7 0x5639b889393c in net_init_clients third_party/qemu/net/net.c:1612:9
 15:16:20.313843     #8 0x5639b717aaf3 in qemu_create_late_backends third_party/qemu/softmmu/vl.c:1962:5
 15:16:20.313844     #9 0x5639b717aaf3 in qemu_init third_party/qemu/softmmu/vl.c:3694:5
 15:16:20.313845     #10 0x5639b71083b8 in main third_party/qemu/softmmu/main.c:49:5
 15:16:20.313846     #11 0x7f464de1d8d2 in __libc_start_main (/usr/grte/v5/lib64/libc.so.6+0x628d2)
 15:16:20.313847     #12 0x5639b6bbd389 in _start /usr/grte/v5/debug-src/src/csu/../sysdeps/x86_64/start.S:120
 15:16:20.313849
 15:16:20.313851   Uninitialized value was created by an allocation of 'ifr' in the stack frame of function 'tap_probe_vnet_hdr'
 15:16:20.313855     #0 0x5639b88ac680 in tap_probe_vnet_hdr third_party/qemu/net/tap-linux.c:151
 15:16:20.313856
 15:16:20.313878 SUMMARY: MemorySanitizer: use-of-uninitialized-value third_party/qemu/net/tap-linux.c:183:9 in tap_probe_vnet_hdr_len
-CC: qemu-stable@nongnu.org
+Fixes: dc69004c7d8 ("net: move tap_probe_vnet_hdr() to tap-linux.c")
-Reported-by: Daniel Shapira <daniel@twistlock.com>
+Reviewed-by: Hao Wu <wuhaotsh@google.com>
-Reviewed-by: Michael S. Tsirkin <mst@redhat.com>
+Reviewed-by: Patrick Venture <venture@google.com>
 Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org>
 Signed-off-by: Peter Foley <pefoley@google.com>
 Signed-off-by: Jason Wang <jasowang@redhat.com>
 ---
- hw/net/pcnet.c | 4 ++--
+ net/tap-linux.c | 1 +
-file changed, 2 insertions(+), 2 deletions(-)
+file changed, 1 insertion(+)
-diff --git a/hw/net/pcnet.c b/hw/net/pcnet.c
+diff --git a/net/tap-linux.c b/net/tap-linux.c
 index XXXXXXX..XXXXXXX 100644
---- a/hw/net/pcnet.c
+--- a/net/tap-linux.c
-+++ b/hw/net/pcnet.c
++++ b/net/tap-linux.c
-@@ -XXX,XX +XXX,XX @@ ssize_t pcnet_receive(NetClientState *nc, const uint8_t *buf, size_t size_)
+@@ -XXX,XX +XXX,XX @@ void tap_set_sndbuf(int fd, const NetdevTapOptions *tap, Error **errp)
-     uint8_t buf1[60];
+ int tap_probe_vnet_hdr(int fd, Error **errp)
-     int remaining;
+ {
-     int crc_err = 0;
+     struct ifreq ifr;
--    int size = size_;
++    memset(&ifr, 0, sizeof(ifr));
-+    size_t size = size_;
+     if (ioctl(fd, TUNGETIFF, &ifr) != 0) {
-     if (CSR_DRX(s) || CSR_STOP(s) || CSR_SPND(s) || !size ||
+         /* TUNGETIFF is available since kernel v2.6.27 */
          (CSR_LOOP(s) && !s->looptest)) {
          return -1;
      }
  #ifdef PCNET_DEBUG
 -    printf("pcnet_receive size=%d\n", size);
 +    printf("pcnet_receive size=%zu\n", size);
  #endif
      /* if too small buffer, then expand it */
 --
-.5.0
+.7.4

-[Qemu-devel] [PULL 03/26] colo-compare: use notifier to notify packets comparing result
+[PULL V3 04/13] net/colo-compare.c: Optimize compare order for performance
-From: Zhang Chen <zhangckid@gmail.com>
+From: Zhang Chen <chen.zhang@intel.com>
-It's a good idea to use notifier to notify COLO frame of
+COLO-compare use the glib function g_queue_find_custom to dump
-inconsistent packets comparing.
+another VM's networking packet to compare. But this function always
 start find from the queue->head(here is the newest packet), It will
 reduce the success rate of comparison. So this patch reversed
 the order of the queues for performance.
-Signed-off-by: Zhang Chen <zhangckid@gmail.com>
 Signed-off-by: Zhang Chen <chen.zhang@intel.com>
-Signed-off-by: zhanghailiang <zhang.zhanghailiang@huawei.com>
+Reported-by: leirao <lei.rao@intel.com>
 Signed-off-by: Jason Wang <jasowang@redhat.com>
 ---
- net/colo-compare.c | 37 ++++++++++++++++++++++++++-----------
+ net/colo-compare.c | 26 +++++++++++++-------------
- net/colo-compare.h |  2 ++
+file changed, 13 insertions(+), 13 deletions(-)
 files changed, 28 insertions(+), 11 deletions(-)
 diff --git a/net/colo-compare.c b/net/colo-compare.c
 index XXXXXXX..XXXXXXX 100644
 --- a/net/colo-compare.c
 +++ b/net/colo-compare.c
-@@ -XXX,XX +XXX,XX @@
+@@ -XXX,XX +XXX,XX @@ static void colo_compare_inconsistency_notify(CompareState *s)
- #include "sysemu/iothread.h"
+ /* Use restricted to colo_insert_packet() */
- #include "net/colo-compare.h"
+ static gint seq_sorter(Packet *a, Packet *b, gpointer data)
- #include "migration/colo.h"
+ {
-+#include "migration/migration.h"
+-    return a->tcp_seq - b->tcp_seq;
++    return b->tcp_seq - a->tcp_seq;
  #define TYPE_COLO_COMPARE "colo-compare"
  #define COLO_COMPARE(obj) \
@@ -XXX,XX +XXX,XX @@
  static QTAILQ_HEAD(, CompareState) net_compares =
         QTAILQ_HEAD_INITIALIZER(net_compares);
 +static NotifierList colo_compare_notifiers =
 +    NOTIFIER_LIST_INITIALIZER(colo_compare_notifiers);
 +
  #define COMPARE_READ_LEN_MAX NET_BUFSIZE
  #define MAX_QUEUE_SIZE 1024
@@ -XXX,XX +XXX,XX @@ static bool colo_mark_tcp_pkt(Packet *ppkt, Packet *spkt,
      return false;
  }
-+static void colo_compare_inconsistency_notify(void)
+ static void fill_pkt_tcp_info(void *data, uint32_t *max_ack)
-+{
+@@ -XXX,XX +XXX,XX @@ pri:
-+    notifier_list_notify(&colo_compare_notifiers,
+     if (g_queue_is_empty(&conn->primary_list)) {
-+                migrate_get_current());
+         return;
-+}
+     }
-+
+-    ppkt = g_queue_pop_head(&conn->primary_list);
- static void colo_compare_tcp(CompareState *s, Connection *conn)
++    ppkt = g_queue_pop_tail(&conn->primary_list);
- {
+ sec:
-     Packet *ppkt = NULL, *spkt = NULL;
+     if (g_queue_is_empty(&conn->secondary_list)) {
 -        g_queue_push_head(&conn->primary_list, ppkt);
 +        g_queue_push_tail(&conn->primary_list, ppkt);
          return;
      }
 -    spkt = g_queue_pop_head(&conn->secondary_list);
 +    spkt = g_queue_pop_tail(&conn->secondary_list);
      if (ppkt->tcp_seq == ppkt->seq_end) {
          colo_release_primary_pkt(s, ppkt);
 @@ -XXX,XX +XXX,XX @@ sec:
-         qemu_hexdump((char *)spkt->data, stderr,
+             }
-                      "colo-compare spkt", spkt->size);
+         }
+         if (!ppkt) {
--        /*
+-            g_queue_push_head(&conn->secondary_list, spkt);
--         * colo_compare_inconsistent_notify();
++            g_queue_push_tail(&conn->secondary_list, spkt);
--         * TODO: notice to checkpoint();
+             goto pri;
--         */
+         }
-+        colo_compare_inconsistency_notify();
+     }
@@ -XXX,XX +XXX,XX @@ sec:
          if (mark == COLO_COMPARE_FREE_PRIMARY) {
              conn->compare_seq = ppkt->seq_end;
              colo_release_primary_pkt(s, ppkt);
 -            g_queue_push_head(&conn->secondary_list, spkt);
 +            g_queue_push_tail(&conn->secondary_list, spkt);
              goto pri;
          } else if (mark == COLO_COMPARE_FREE_SECONDARY) {
              conn->compare_seq = spkt->seq_end;
@@ -XXX,XX +XXX,XX @@ sec:
              goto pri;
          }
      } else {
 -        g_queue_push_head(&conn->primary_list, ppkt);
 -        g_queue_push_head(&conn->secondary_list, spkt);
 +        g_queue_push_tail(&conn->primary_list, ppkt);
 +        g_queue_push_tail(&conn->secondary_list, spkt);
  #ifdef DEBUG_COLO_PACKETS
          qemu_hexdump(stderr, "colo-compare ppkt", ppkt->data, ppkt->size);
@@ -XXX,XX +XXX,XX @@ static void colo_compare_packet(CompareState *s, Connection *conn,
      while (!g_queue_is_empty(&conn->primary_list) &&
             !g_queue_is_empty(&conn->secondary_list)) {
 -        pkt = g_queue_pop_head(&conn->primary_list);
 +        pkt = g_queue_pop_tail(&conn->primary_list);
          result = g_queue_find_custom(&conn->secondary_list,
                   pkt, (GCompareFunc)HandlePacket);
@@ -XXX,XX +XXX,XX @@ static void colo_compare_packet(CompareState *s, Connection *conn,
               * timeout, it will trigger a checkpoint request.
               */
              trace_colo_compare_main("packet different");
 -            g_queue_push_head(&conn->primary_list, pkt);
 +            g_queue_push_tail(&conn->primary_list, pkt);
              colo_compare_inconsistency_notify(s);
              break;
@@ -XXX,XX +XXX,XX @@ static int compare_chr_send(CompareState *s,
          entry->buf = g_malloc(size);
          memcpy(entry->buf, buf, size);
      }
 -    g_queue_push_head(&sendco->send_list, entry);
 +    g_queue_push_tail(&sendco->send_list, entry);
      if (sendco->done) {
          sendco->co = qemu_coroutine_create(_compare_chr_send, sendco);
@@ -XXX,XX +XXX,XX @@ static void colo_flush_packets(void *opaque, void *user_data)
      Packet *pkt = NULL;
      while (!g_queue_is_empty(&conn->primary_list)) {
 -        pkt = g_queue_pop_head(&conn->primary_list);
 +        pkt = g_queue_pop_tail(&conn->primary_list);
          compare_chr_send(s,
                           pkt->data,
                           pkt->size,
@@ -XXX,XX +XXX,XX @@ static void colo_flush_packets(void *opaque, void *user_data)
          packet_destroy_partial(pkt, NULL);
      }
      while (!g_queue_is_empty(&conn->secondary_list)) {
 -        pkt = g_queue_pop_head(&conn->secondary_list);
 +        pkt = g_queue_pop_tail(&conn->secondary_list);
          packet_destroy(pkt, NULL);
      }
  }
-@@ -XXX,XX +XXX,XX @@ static int colo_old_packet_check_one(Packet *pkt, int64_t *check_time)
-     }
- }
-+void colo_compare_register_notifier(Notifier *notify)
-+{
-+    notifier_list_add(&colo_compare_notifiers, notify);
-+}
-+
-+void colo_compare_unregister_notifier(Notifier *notify)
-+{
-+    notifier_remove(notify);
-+}
-+
- static int colo_old_packet_check_one_conn(Connection *conn,
--                                          void *user_data)
-+                                           void *user_data)
- {
-     GList *result = NULL;
-     int64_t check_time = REGULAR_PACKET_CHECK_MS;
-@@ -XXX,XX +XXX,XX @@ static int colo_old_packet_check_one_conn(Connection *conn,
-     if (result) {
-         /* Do checkpoint will flush old packet */
--        /*
--         * TODO: Notify colo frame to do checkpoint.
--         * colo_compare_inconsistent_notify();
--         */
-+        colo_compare_inconsistency_notify();
-         return 0;
-     }
-@@ -XXX,XX +XXX,XX @@ static void colo_compare_packet(CompareState *s, Connection *conn,
-             /*
-              * If one packet arrive late, the secondary_list or
-              * primary_list will be empty, so we can't compare it
--             * until next comparison.
-+             * until next comparison. If the packets in the list are
-+             * timeout, it will trigger a checkpoint request.
-              */
-             trace_colo_compare_main("packet different");
-             g_queue_push_head(&conn->primary_list, pkt);
--            /* TODO: colo_notify_checkpoint();*/
-+            colo_compare_inconsistency_notify();
-             break;
-         }
-     }
-diff --git a/net/colo-compare.h b/net/colo-compare.h
-index XXXXXXX..XXXXXXX 100644
---- a/net/colo-compare.h
-+++ b/net/colo-compare.h
-@@ -XXX,XX +XXX,XX @@
- #define QEMU_COLO_COMPARE_H
- void colo_notify_compares_event(void *opaque, int event, Error **errp);
-+void colo_compare_register_notifier(Notifier *notify);
-+void colo_compare_unregister_notifier(Notifier *notify);
- #endif /* QEMU_COLO_COMPARE_H */
 --
-.5.0
+.7.4

-[Qemu-devel] [PULL 16/26] filter-rewriter: handle checkpoint and failover event
+[PULL V3 05/13] net/colo-compare.c: Update the default value comments
-From: Zhang Chen <zhangckid@gmail.com>
+From: Zhang Chen <chen.zhang@intel.com>
-After one round of checkpoint, the states between PVM and SVM
+Make the comments consistent with the REGULAR_PACKET_CHECK_MS.
 become consistent, so it is unnecessary to adjust the sequence
 of net packets for old connections, besides, while failover
 happens, filter-rewriter will into failover mode that needn't
 handle the new TCP connection.
-Signed-off-by: zhanghailiang <zhang.zhanghailiang@huawei.com>
-Signed-off-by: Zhang Chen <zhangckid@gmail.com>
 Signed-off-by: Zhang Chen <chen.zhang@intel.com>
 Signed-off-by: Jason Wang <jasowang@redhat.com>
 ---
- net/colo-compare.c    | 12 +++++------
+ net/colo-compare.c | 2 +-
- net/colo.c            |  8 ++++++++
+file changed, 1 insertion(+), 1 deletion(-)
  net/colo.h            |  2 ++
  net/filter-rewriter.c | 57 +++++++++++++++++++++++++++++++++++++++++++++++++++
 files changed, 73 insertions(+), 6 deletions(-)
 diff --git a/net/colo-compare.c b/net/colo-compare.c
 index XXXXXXX..XXXXXXX 100644
 --- a/net/colo-compare.c
 +++ b/net/colo-compare.c
-@@ -XXX,XX +XXX,XX @@ enum {
+@@ -XXX,XX +XXX,XX @@ static void colo_compare_complete(UserCreatable *uc, Error **errp)
      SECONDARY_IN,
  };
 +static void colo_compare_inconsistency_notify(void)
 +{
 +    notifier_list_notify(&colo_compare_notifiers,
 +                migrate_get_current());
 +}
 +
  static int compare_chr_send(CompareState *s,
                              const uint8_t *buf,
                              uint32_t size,
@@ -XXX,XX +XXX,XX @@ static bool colo_mark_tcp_pkt(Packet *ppkt, Packet *spkt,
      return false;
  }
 -static void colo_compare_inconsistency_notify(void)
 -{
 -    notifier_list_notify(&colo_compare_notifiers,
 -                migrate_get_current());
 -}
 -
  static void colo_compare_tcp(CompareState *s, Connection *conn)
  {
      Packet *ppkt = NULL, *spkt = NULL;
 diff --git a/net/colo.c b/net/colo.c
 index XXXXXXX..XXXXXXX 100644
 --- a/net/colo.c
 +++ b/net/colo.c
@@ -XXX,XX +XXX,XX @@ Connection *connection_get(GHashTable *connection_track_table,
      return conn;
  }
 +
 +bool connection_has_tracked(GHashTable *connection_track_table,
 +                            ConnectionKey *key)
 +{
 +    Connection *conn = g_hash_table_lookup(connection_track_table, key);
 +
 +    return conn ? true : false;
 +}
 diff --git a/net/colo.h b/net/colo.h
 index XXXXXXX..XXXXXXX 100644
 --- a/net/colo.h
 +++ b/net/colo.h
@@ -XXX,XX +XXX,XX @@ void connection_destroy(void *opaque);
  Connection *connection_get(GHashTable *connection_track_table,
                             ConnectionKey *key,
                             GQueue *conn_list);
 +bool connection_has_tracked(GHashTable *connection_track_table,
 +                            ConnectionKey *key);
  void connection_hashtable_reset(GHashTable *connection_track_table);
  Packet *packet_new(const void *data, int size, int vnet_hdr_len);
  void packet_destroy(void *opaque, void *user_data);
 diff --git a/net/filter-rewriter.c b/net/filter-rewriter.c
 index XXXXXXX..XXXXXXX 100644
 --- a/net/filter-rewriter.c
 +++ b/net/filter-rewriter.c
@@ -XXX,XX +XXX,XX @@
  #include "qemu/main-loop.h"
  #include "qemu/iov.h"
  #include "net/checksum.h"
 +#include "net/colo.h"
 +#include "migration/colo.h"
  #define FILTER_COLO_REWRITER(obj) \
      OBJECT_CHECK(RewriterState, (obj), TYPE_FILTER_REWRITER)
  #define TYPE_FILTER_REWRITER "filter-rewriter"
 +#define FAILOVER_MODE_ON  true
 +#define FAILOVER_MODE_OFF false
  typedef struct RewriterState {
      NetFilterState parent_obj;
@@ -XXX,XX +XXX,XX @@ typedef struct RewriterState {
      /* hashtable to save connection */
      GHashTable *connection_track_table;
      bool vnet_hdr;
 +    bool failover_mode;
  } RewriterState;
 +static void filter_rewriter_failover_mode(RewriterState *s)
 +{
 +    s->failover_mode = FAILOVER_MODE_ON;
 +}
 +
  static void filter_rewriter_flush(NetFilterState *nf)
  {
      RewriterState *s = FILTER_COLO_REWRITER(nf);
@@ -XXX,XX +XXX,XX @@ static ssize_t colo_rewriter_receive_iov(NetFilterState *nf,
               */
              reverse_connection_key(&key);
          }
 +
 +        /* After failover we needn't change new TCP packet */
 +        if (s->failover_mode &&
 +            !connection_has_tracked(s->connection_track_table, &key)) {
 +            goto out;
 +        }
 +
          conn = connection_get(s->connection_track_table,
                                &key,
                                NULL);
@@ -XXX,XX +XXX,XX @@ static ssize_t colo_rewriter_receive_iov(NetFilterState *nf,
          }
      }
-+out:
+     if (!s->expired_scan_cycle) {
-     packet_destroy(pkt, NULL);
+-        /* Set default value to 3000 MS */
-     pkt = NULL;
++        /* Set default value to 1000 MS */
-     return 0;
+         s->expired_scan_cycle = REGULAR_PACKET_CHECK_MS;
- }
+     }
 +static void reset_seq_offset(gpointer key, gpointer value, gpointer user_data)
 +{
 +    Connection *conn = (Connection *)value;
 +
 +    conn->offset = 0;
 +}
 +
 +static gboolean offset_is_nonzero(gpointer key,
 +                                  gpointer value,
 +                                  gpointer user_data)
 +{
 +    Connection *conn = (Connection *)value;
 +
 +    return conn->offset ? true : false;
 +}
 +
 +static void colo_rewriter_handle_event(NetFilterState *nf, int event,
 +                                       Error **errp)
 +{
 +    RewriterState *rs = FILTER_COLO_REWRITER(nf);
 +
 +    switch (event) {
 +    case COLO_EVENT_CHECKPOINT:
 +        g_hash_table_foreach(rs->connection_track_table,
 +                            reset_seq_offset, NULL);
 +        break;
 +    case COLO_EVENT_FAILOVER:
 +        if (!g_hash_table_find(rs->connection_track_table,
 +                              offset_is_nonzero, NULL)) {
 +            filter_rewriter_failover_mode(rs);
 +        }
 +        break;
 +    default:
 +        break;
 +    }
 +}
 +
  static void colo_rewriter_cleanup(NetFilterState *nf)
  {
      RewriterState *s = FILTER_COLO_REWRITER(nf);
@@ -XXX,XX +XXX,XX @@ static void filter_rewriter_init(Object *obj)
      RewriterState *s = FILTER_COLO_REWRITER(obj);
      s->vnet_hdr = false;
 +    s->failover_mode = FAILOVER_MODE_OFF;
      object_property_add_bool(obj, "vnet_hdr_support",
                               filter_rewriter_get_vnet_hdr,
                               filter_rewriter_set_vnet_hdr, NULL);
@@ -XXX,XX +XXX,XX @@ static void colo_rewriter_class_init(ObjectClass *oc, void *data)
      nfc->setup = colo_rewriter_setup;
      nfc->cleanup = colo_rewriter_cleanup;
      nfc->receive_iov = colo_rewriter_receive_iov;
 +    nfc->handle_event = colo_rewriter_handle_event;
  }
  static const TypeInfo colo_rewriter_info = {
 --
-.5.0
+.7.4

-[Qemu-devel] [PULL 04/26] COLO: integrate colo compare with colo frame
+[PULL V3 06/13] net/filter: Optimize filter_send to coroutine
-From: Zhang Chen <zhangckid@gmail.com>
+From: Rao Lei <lei.rao@intel.com>
-For COLO FT, both the PVM and SVM run at the same time,
+This patch is to improve the logic of QEMU main thread sleep code in
-only sync the state while it needs.
+qemu_chr_write_buffer() where it can be blocked and can't run other
 coroutines during COLO IO stress test.
-So here, let SVM runs while not doing checkpoint, change
+Our approach is to put filter_send() in a coroutine. In this way,
-DEFAULT_MIGRATE_X_CHECKPOINT_DELAY to 200*100.
+filter_send() will call qemu_coroutine_yield() in qemu_co_sleep_ns(),
 so that it can be scheduled out and QEMU main thread has opportunity to
 run other tasks.
-Besides, we forgot to release colo_checkpoint_semd and
+Signed-off-by: Lei Rao <lei.rao@intel.com>
 colo_delay_timer, fix them here.
 Signed-off-by: zhanghailiang <zhang.zhanghailiang@huawei.com>
 Signed-off-by: Zhang Chen <zhangckid@gmail.com>
 Signed-off-by: Zhang Chen <chen.zhang@intel.com>
-Reviewed-by: Dr. David Alan Gilbert <dgilbert@redhat.com>
+Reviewed-by: Li Zhijian <lizhijian@fujitsu.com>
 Reviewed-by: Zhang Chen <chen.zhang@intel.com>
 Signed-off-by: Jason Wang <jasowang@redhat.com>
 ---
- migration/colo.c      | 42 ++++++++++++++++++++++++++++++++++++++++--
+ net/filter-mirror.c | 66 ++++++++++++++++++++++++++++++++++++++++++-----------
- migration/migration.c |  6 ++----
+file changed, 53 insertions(+), 13 deletions(-)
 files changed, 42 insertions(+), 6 deletions(-)
-diff --git a/migration/colo.c b/migration/colo.c
+diff --git a/net/filter-mirror.c b/net/filter-mirror.c
 index XXXXXXX..XXXXXXX 100644
---- a/migration/colo.c
+--- a/net/filter-mirror.c
-+++ b/migration/colo.c
++++ b/net/filter-mirror.c
 @@ -XXX,XX +XXX,XX @@
- #include "qemu/error-report.h"
+ #include "chardev/char-fe.h"
- #include "migration/failover.h"
+ #include "qemu/iov.h"
- #include "replication.h"
+ #include "qemu/sockets.h"
-+#include "net/colo-compare.h"
++#include "block/aio-wait.h"
-+#include "net/colo.h"
+ #define TYPE_FILTER_MIRROR "filter-mirror"
- static bool vmstate_loading;
+ typedef struct MirrorState MirrorState;
-+static Notifier packets_compare_notifier;
+@@ -XXX,XX +XXX,XX @@ struct MirrorState {
+     bool vnet_hdr;
- #define COLO_BUFFER_BASE_SIZE (4 * 1024 * 1024)
+ };
-@@ -XXX,XX +XXX,XX @@ static int colo_do_checkpoint_transaction(MigrationState *s,
+-static int filter_send(MirrorState *s,
-         goto out;
+-                       const struct iovec *iov,
 -                       int iovcnt)
 +typedef struct FilterSendCo {
 +    MirrorState *s;
 +    char *buf;
 +    ssize_t size;
 +    bool done;
 +    int ret;
 +} FilterSendCo;
 +
 +static int _filter_send(MirrorState *s,
 +                       char *buf,
 +                       ssize_t size)
  {
      NetFilterState *nf = NETFILTER(s);
      int ret = 0;
 -    ssize_t size = 0;
      uint32_t len = 0;
 -    char *buf;
 -
 -    size = iov_size(iov, iovcnt);
 -    if (!size) {
 -        return 0;
 -    }
      len = htonl(size);
      ret = qemu_chr_fe_write_all(&s->chr_out, (uint8_t *)&len, sizeof(len));
@@ -XXX,XX +XXX,XX @@ static int filter_send(MirrorState *s,
          }
      }
-+    colo_notify_compares_event(NULL, COLO_EVENT_CHECKPOINT, &local_err);
+-    buf = g_malloc(size);
-+    if (local_err) {
+-    iov_to_buf(iov, iovcnt, 0, buf, size);
-+        goto out;
+     ret = qemu_chr_fe_write_all(&s->chr_out, (uint8_t *)buf, size);
 -    g_free(buf);
      if (ret != size) {
          goto err;
      }
@@ -XXX,XX +XXX,XX @@ err:
      return ret < 0 ? ret : -EIO;
  }
 +static void coroutine_fn filter_send_co(void *opaque)
 +{
 +    FilterSendCo *data = opaque;
 +
 +    data->ret = _filter_send(data->s, data->buf, data->size);
 +    data->done = true;
 +    g_free(data->buf);
 +    aio_wait_kick();
 +}
 +
 +static int filter_send(MirrorState *s,
 +                       const struct iovec *iov,
 +                       int iovcnt)
 +{
 +    ssize_t size = iov_size(iov, iovcnt);
 +    char *buf = NULL;
 +
 +    if (!size) {
 +        return 0;
 +    }
 +
-     /* Disable block migration */
++    buf = g_malloc(size);
-     migrate_set_block_enabled(false, &local_err);
++    iov_to_buf(iov, iovcnt, 0, buf, size);
-     qemu_savevm_state_header(fb);
++
-@@ -XXX,XX +XXX,XX @@ out:
++    FilterSendCo data = {
-     return ret;
++        .s = s,
- }
++        .size = size,
++        .buf = buf,
-+static void colo_compare_notify_checkpoint(Notifier *notifier, void *data)
++        .ret = 0,
-+{
++    };
-+    colo_checkpoint_notify(data);
++
 +    Coroutine *co = qemu_coroutine_create(filter_send_co, &data);
 +    qemu_coroutine_enter(co);
 +
 +    while (!data.done) {
 +        aio_poll(qemu_get_aio_context(), true);
 +    }
 +
 +    return data.ret;
 +}
 +
- static void colo_process_checkpoint(MigrationState *s)
+ static void redirector_to_filter(NetFilterState *nf,
- {
+                                  const uint8_t *buf,
-     QIOChannelBuffer *bioc;
+                                  int len)
@@ -XXX,XX +XXX,XX @@ static void colo_process_checkpoint(MigrationState *s)
          goto out;
      }
 +    packets_compare_notifier.notify = colo_compare_notify_checkpoint;
 +    colo_compare_register_notifier(&packets_compare_notifier);
 +
      /*
       * Wait for Secondary finish loading VM states and enter COLO
       * restore.
@@ -XXX,XX +XXX,XX @@ out:
          qemu_fclose(fb);
      }
 -    timer_del(s->colo_delay_timer);
 -
      /* Hope this not to be too long to wait here */
      qemu_sem_wait(&s->colo_exit_sem);
      qemu_sem_destroy(&s->colo_exit_sem);
 +
 +    /*
 +     * It is safe to unregister notifier after failover finished.
 +     * Besides, colo_delay_timer and colo_checkpoint_sem can't be
 +     * released befor unregister notifier, or there will be use-after-free
 +     * error.
 +     */
 +    colo_compare_unregister_notifier(&packets_compare_notifier);
 +    timer_del(s->colo_delay_timer);
 +    timer_free(s->colo_delay_timer);
 +    qemu_sem_destroy(&s->colo_checkpoint_sem);
 +
      /*
       * Must be called after failover BH is completed,
       * Or the failover BH may shutdown the wrong fd that
@@ -XXX,XX +XXX,XX @@ void *colo_process_incoming_thread(void *opaque)
      fb = qemu_fopen_channel_input(QIO_CHANNEL(bioc));
      object_unref(OBJECT(bioc));
 +    qemu_mutex_lock_iothread();
 +    vm_start();
 +    trace_colo_vm_state_change("stop", "run");
 +    qemu_mutex_unlock_iothread();
 +
      colo_send_message(mis->to_src_file, COLO_MESSAGE_CHECKPOINT_READY,
                        &local_err);
      if (local_err) {
@@ -XXX,XX +XXX,XX @@ void *colo_process_incoming_thread(void *opaque)
              goto out;
          }
 +        qemu_mutex_lock_iothread();
 +        vm_stop_force_state(RUN_STATE_COLO);
 +        trace_colo_vm_state_change("run", "stop");
 +        qemu_mutex_unlock_iothread();
 +
          /* FIXME: This is unnecessary for periodic checkpoint mode */
          colo_send_message(mis->to_src_file, COLO_MESSAGE_CHECKPOINT_REPLY,
                       &local_err);
@@ -XXX,XX +XXX,XX @@ void *colo_process_incoming_thread(void *opaque)
          }
          vmstate_loading = false;
 +        vm_start();
 +        trace_colo_vm_state_change("stop", "run");
          qemu_mutex_unlock_iothread();
          if (failover_get_state() == FAILOVER_STATUS_RELAUNCH) {
 diff --git a/migration/migration.c b/migration/migration.c
 index XXXXXXX..XXXXXXX 100644
 --- a/migration/migration.c
 +++ b/migration/migration.c
@@ -XXX,XX +XXX,XX @@
  /* Migration XBZRLE default cache size */
  #define DEFAULT_MIGRATE_XBZRLE_CACHE_SIZE (64 * 1024 * 1024)
 -/* The delay time (in ms) between two COLO checkpoints
 - * Note: Please change this default value to 10000 when we support hybrid mode.
 - */
 -#define DEFAULT_MIGRATE_X_CHECKPOINT_DELAY 200
 +/* The delay time (in ms) between two COLO checkpoints */
 +#define DEFAULT_MIGRATE_X_CHECKPOINT_DELAY (200 * 100)
  #define DEFAULT_MIGRATE_MULTIFD_CHANNELS 2
  #define DEFAULT_MIGRATE_MULTIFD_PAGE_COUNT 16
 --
-.5.0
+.7.4

-[Qemu-devel] [PULL 22/26] rtl8139: fix possible out of bound access
+[PULL V3 07/13] net/vmnet: add vmnet dependency and customizable option
-In rtl8139_do_receive(), we try to assign size_ to size which converts
+From: Vladislav Yaroshchuk <yaroshchuk2000@gmail.com>
 from size_t to integer. This will cause troubles when size_ is greater
 INT_MAX, this will lead a negative value in size and it can then pass
 the check of size < MIN_BUF_SIZE which may lead out of bound access of
 for both buf and buf1.
-Fixing by converting the type of size to size_t.
+vmnet.framework dependency is added with 'vmnet' option
 to enable or disable it. Default value is 'auto'.
-CC: qemu-stable@nongnu.org
+vmnet features to be used are available since macOS 11.0,
-Reported-by: Daniel Shapira <daniel@twistlock.com>
+corresponding probe is created into meson.build.
-Reviewed-by: Michael S. Tsirkin <mst@redhat.com>
 Signed-off-by: Vladislav Yaroshchuk <yaroshchuk2000@gmail.com>
 Signed-off-by: Jason Wang <jasowang@redhat.com>
 ---
- hw/net/rtl8139.c | 8 ++++----
+ meson.build                   | 16 +++++++++++++++-
-file changed, 4 insertions(+), 4 deletions(-)
+ meson_options.txt             |  2 ++
  scripts/meson-buildoptions.sh |  3 +++
 files changed, 20 insertions(+), 1 deletion(-)
-diff --git a/hw/net/rtl8139.c b/hw/net/rtl8139.c
+diff --git a/meson.build b/meson.build
 index XXXXXXX..XXXXXXX 100644
---- a/hw/net/rtl8139.c
+--- a/meson.build
-+++ b/hw/net/rtl8139.c
++++ b/meson.build
-@@ -XXX,XX +XXX,XX @@ static ssize_t rtl8139_do_receive(NetClientState *nc, const uint8_t *buf, size_t
+@@ -XXX,XX +XXX,XX @@ if cocoa.found() and get_option('gtk').enabled()
-     RTL8139State *s = qemu_get_nic_opaque(nc);
+   error('Cocoa and GTK+ cannot be enabled at the same time')
-     PCIDevice *d = PCI_DEVICE(s);
+ endif
-     /* size is the length of the buffer passed to the driver */
--    int size = size_;
++vmnet = dependency('appleframeworks', modules: 'vmnet', required: get_option('vmnet'))
-+    size_t size = size_;
++if vmnet.found() and not cc.has_header_symbol('vmnet/vmnet.h',
-     const uint8_t *dot1q_buf = NULL;
++                                              'VMNET_BRIDGED_MODE',
++                                              dependencies: vmnet)
-     uint32_t packet_header = 0;
++  vmnet = not_found
-@@ -XXX,XX +XXX,XX @@ static ssize_t rtl8139_do_receive(NetClientState *nc, const uint8_t *buf, size_t
++  if get_option('vmnet').enabled()
-     static const uint8_t broadcast_macaddr[6] =
++    error('vmnet.framework API is outdated')
-         { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff };
++  else
++    warning('vmnet.framework API is outdated, disabling')
--    DPRINTF(">>> received len=%d\n", size);
++  endif
-+    DPRINTF(">>> received len=%zu\n", size);
++endif
++
-     /* test if board clock is stopped */
+ seccomp = not_found
-     if (!s->clock_enabled)
+ if not get_option('seccomp').auto() or have_system or have_tools
-@@ -XXX,XX +XXX,XX @@ static ssize_t rtl8139_do_receive(NetClientState *nc, const uint8_t *buf, size_t
+   seccomp = dependency('libseccomp', version: '>=2.3.0',
+@@ -XXX,XX +XXX,XX @@ config_host_data.set('CONFIG_SECCOMP', seccomp.found())
-         if (size+4 > rx_space)
+ config_host_data.set('CONFIG_SNAPPY', snappy.found())
-         {
+ config_host_data.set('CONFIG_USB_LIBUSB', libusb.found())
--            DPRINTF("C+ Rx mode : descriptor %d size %d received %d + 4\n",
+ config_host_data.set('CONFIG_VDE', vde.found())
-+            DPRINTF("C+ Rx mode : descriptor %d size %d received %zu + 4\n",
++config_host_data.set('CONFIG_VMNET', vmnet.found())
-                 descriptor, rx_space, size);
+ config_host_data.set('CONFIG_VHOST_USER_BLK_SERVER', have_vhost_user_blk_server)
+ config_host_data.set('CONFIG_VNC', vnc.found())
-             s->IntrStatus |= RxOverflow;
+ config_host_data.set('CONFIG_VNC_JPEG', jpeg.found())
-@@ -XXX,XX +XXX,XX @@ static ssize_t rtl8139_do_receive(NetClientState *nc, const uint8_t *buf, size_t
+@@ -XXX,XX +XXX,XX @@ summary(summary_info, bool_yn: true, section: 'Crypto')
-         if (avail != 0 && RX_ALIGN(size + 8) >= avail)
+ # Libraries
-         {
+ summary_info = {}
-             DPRINTF("rx overflow: rx buffer length %d head 0x%04x "
+ if targetos == 'darwin'
--                "read 0x%04x === available 0x%04x need 0x%04x\n",
+-  summary_info += {'Cocoa support':   cocoa}
-+                "read 0x%04x === available 0x%04x need 0x%04zx\n",
++  summary_info += {'Cocoa support':           cocoa}
-                 s->RxBufferSize, s->RxBufAddr, s->RxBufPtr, avail, size + 8);
++  summary_info += {'vmnet.framework support': vmnet}
+ endif
-             s->IntrStatus |= RxOverflow;
+ summary_info += {'SDL support':       sdl}
  summary_info += {'SDL image support': sdl_image}
 diff --git a/meson_options.txt b/meson_options.txt
 index XXXXXXX..XXXXXXX 100644
 --- a/meson_options.txt
 +++ b/meson_options.txt
@@ -XXX,XX +XXX,XX @@ option('netmap', type : 'feature', value : 'auto',
         description: 'netmap network backend support')
  option('vde', type : 'feature', value : 'auto',
         description: 'vde network backend support')
 +option('vmnet', type : 'feature', value : 'auto',
 +       description: 'vmnet.framework network backend support')
  option('virglrenderer', type : 'feature', value : 'auto',
         description: 'virgl rendering support')
  option('vnc', type : 'feature', value : 'auto',
 diff --git a/scripts/meson-buildoptions.sh b/scripts/meson-buildoptions.sh
 index XXXXXXX..XXXXXXX 100644
 --- a/scripts/meson-buildoptions.sh
 +++ b/scripts/meson-buildoptions.sh
@@ -XXX,XX +XXX,XX @@ meson_options_help() {
    printf "%s\n" '  u2f             U2F emulation support'
    printf "%s\n" '  usb-redir       libusbredir support'
    printf "%s\n" '  vde             vde network backend support'
 +  printf "%s\n" '  vmnet           vmnet.framework network backend support'
    printf "%s\n" '  vhost-user-blk-server'
    printf "%s\n" '                  build vhost-user-blk server'
    printf "%s\n" '  virglrenderer   virgl rendering support'
@@ -XXX,XX +XXX,XX @@ _meson_option_parse() {
      --disable-usb-redir) printf "%s" -Dusb_redir=disabled ;;
      --enable-vde) printf "%s" -Dvde=enabled ;;
      --disable-vde) printf "%s" -Dvde=disabled ;;
 +    --enable-vmnet) printf "%s" -Dvmnet=enabled ;;
 +    --disable-vmnet) printf "%s" -Dvmnet=disabled ;;
      --enable-vhost-user-blk-server) printf "%s" -Dvhost_user_blk_server=enabled ;;
      --disable-vhost-user-blk-server) printf "%s" -Dvhost_user_blk_server=disabled ;;
      --enable-virglrenderer) printf "%s" -Dvirglrenderer=enabled ;;
 --
-.5.0
+.7.4

-[Qemu-devel] [PULL 15/26] filter: Add handle_event method for NetFilterClass
+[PULL V3 08/13] net/vmnet: add vmnet backends to qapi/net
-From: Zhang Chen <zhangckid@gmail.com>
+From: Vladislav Yaroshchuk <yaroshchuk2000@gmail.com>
-Filter needs to process the event of checkpoint/failover or
+Create separate netdevs for each vmnet operating mode:
-other event passed by COLO frame.
+- vmnet-host
 - vmnet-shared
 - vmnet-bridged
-Signed-off-by: zhanghailiang <zhang.zhanghailiang@huawei.com>
+Signed-off-by: Vladislav Yaroshchuk <yaroshchuk2000@gmail.com>
 Signed-off-by: Zhang Chen <zhangckid@gmail.com>
 Signed-off-by: Zhang Chen <chen.zhang@intel.com>
 Signed-off-by: Jason Wang <jasowang@redhat.com>
 ---
- include/net/filter.h |  5 +++++
+ net/clients.h       |  11 +++++
- net/filter.c         | 17 +++++++++++++++++
+ net/meson.build     |   7 +++
- net/net.c            | 19 +++++++++++++++++++
+ net/net.c           |  10 ++++
-files changed, 41 insertions(+)
+ net/vmnet-bridged.m |  25 ++++++++++
  net/vmnet-common.m  |  19 ++++++++
  net/vmnet-host.c    |  24 ++++++++++
  net/vmnet-shared.c  |  25 ++++++++++
  net/vmnet_int.h     |  25 ++++++++++
  qapi/net.json       | 133 +++++++++++++++++++++++++++++++++++++++++++++++++++-
 files changed, 277 insertions(+), 2 deletions(-)
  create mode 100644 net/vmnet-bridged.m
  create mode 100644 net/vmnet-common.m
  create mode 100644 net/vmnet-host.c
  create mode 100644 net/vmnet-shared.c
  create mode 100644 net/vmnet_int.h
-diff --git a/include/net/filter.h b/include/net/filter.h
+diff --git a/net/clients.h b/net/clients.h
 index XXXXXXX..XXXXXXX 100644
---- a/include/net/filter.h
+--- a/net/clients.h
-+++ b/include/net/filter.h
++++ b/net/clients.h
-@@ -XXX,XX +XXX,XX @@ typedef ssize_t (FilterReceiveIOV)(NetFilterState *nc,
+@@ -XXX,XX +XXX,XX @@ int net_init_vhost_user(const Netdev *netdev, const char *name,
- typedef void (FilterStatusChanged) (NetFilterState *nf, Error **errp);
+ int net_init_vhost_vdpa(const Netdev *netdev, const char *name,
+                         NetClientState *peer, Error **errp);
-+typedef void (FilterHandleEvent) (NetFilterState *nf, int event, Error **errp);
++#ifdef CONFIG_VMNET
-+
++int net_init_vmnet_host(const Netdev *netdev, const char *name,
- typedef struct NetFilterClass {
++                          NetClientState *peer, Error **errp);
-     ObjectClass parent_class;
++
++int net_init_vmnet_shared(const Netdev *netdev, const char *name,
-@@ -XXX,XX +XXX,XX @@ typedef struct NetFilterClass {
++                          NetClientState *peer, Error **errp);
-     FilterSetup *setup;
++
-     FilterCleanup *cleanup;
++int net_init_vmnet_bridged(const Netdev *netdev, const char *name,
-     FilterStatusChanged *status_changed;
++                          NetClientState *peer, Error **errp);
-+    FilterHandleEvent *handle_event;
++#endif /* CONFIG_VMNET */
-     /* mandatory */
++
-     FilterReceiveIOV *receive_iov;
+ #endif /* QEMU_NET_CLIENTS_H */
- } NetFilterClass;
+diff --git a/net/meson.build b/net/meson.build
@@ -XXX,XX +XXX,XX @@ ssize_t qemu_netfilter_pass_to_next(NetClientState *sender,
                                      int iovcnt,
                                      void *opaque);
 +void colo_notify_filters_event(int event, Error **errp);
 +
  #endif /* QEMU_NET_FILTER_H */
 diff --git a/net/filter.c b/net/filter.c
 index XXXXXXX..XXXXXXX 100644
---- a/net/filter.c
+--- a/net/meson.build
-+++ b/net/filter.c
++++ b/net/meson.build
-@@ -XXX,XX +XXX,XX @@
+@@ -XXX,XX +XXX,XX @@ softmmu_ss.add(when: 'CONFIG_POSIX', if_true: files(tap_posix))
- #include "net/vhost_net.h"
+ softmmu_ss.add(when: 'CONFIG_WIN32', if_true: files('tap-win32.c'))
- #include "qom/object_interfaces.h"
+ softmmu_ss.add(when: 'CONFIG_VHOST_NET_VDPA', if_true: files('vhost-vdpa.c'))
- #include "qemu/iov.h"
-+#include "net/colo.h"
++vmnet_files = files(
-+#include "migration/colo.h"
++  'vmnet-common.m',
++  'vmnet-bridged.m',
- static inline bool qemu_can_skip_netfilter(NetFilterState *nf)
++  'vmnet-host.c',
- {
++  'vmnet-shared.c'
-@@ -XXX,XX +XXX,XX @@ static void netfilter_finalize(Object *obj)
++)
-     g_free(nf->netdev_id);
++softmmu_ss.add(when: vmnet, if_true: vmnet_files)
- }
+ subdir('can')
 +static void default_handle_event(NetFilterState *nf, int event, Error **errp)
 +{
 +    switch (event) {
 +    case COLO_EVENT_CHECKPOINT:
 +        break;
 +    case COLO_EVENT_FAILOVER:
 +        object_property_set_str(OBJECT(nf), "off", "status", errp);
 +        break;
 +    default:
 +        break;
 +    }
 +}
 +
  static void netfilter_class_init(ObjectClass *oc, void *data)
  {
      UserCreatableClass *ucc = USER_CREATABLE_CLASS(oc);
 +    NetFilterClass *nfc = NETFILTER_CLASS(oc);
      ucc->complete = netfilter_complete;
 +    nfc->handle_event = default_handle_event;
  }
  static const TypeInfo netfilter_info = {
 diff --git a/net/net.c b/net/net.c
 index XXXXXXX..XXXXXXX 100644
 --- a/net/net.c
 +++ b/net/net.c
-@@ -XXX,XX +XXX,XX @@ void hmp_info_network(Monitor *mon, const QDict *qdict)
+@@ -XXX,XX +XXX,XX @@ static int (* const net_client_init_fun[NET_CLIENT_DRIVER__MAX])(
-     }
+ #ifdef CONFIG_L2TPV3
- }
+         [NET_CLIENT_DRIVER_L2TPV3]    = net_init_l2tpv3,
+ #endif
-+void colo_notify_filters_event(int event, Error **errp)
++#ifdef CONFIG_VMNET
 +        [NET_CLIENT_DRIVER_VMNET_HOST] = net_init_vmnet_host,
 +        [NET_CLIENT_DRIVER_VMNET_SHARED] = net_init_vmnet_shared,
 +        [NET_CLIENT_DRIVER_VMNET_BRIDGED] = net_init_vmnet_bridged,
 +#endif /* CONFIG_VMNET */
  };
@@ -XXX,XX +XXX,XX @@ void show_netdevs(void)
  #ifdef CONFIG_VHOST_VDPA
          "vhost-vdpa",
  #endif
 +#ifdef CONFIG_VMNET
 +        "vmnet-host",
 +        "vmnet-shared",
 +        "vmnet-bridged",
 +#endif
      };
      qemu_printf("Available netdev backend types:\n");
 diff --git a/net/vmnet-bridged.m b/net/vmnet-bridged.m
 new file mode 100644
 index XXXXXXX..XXXXXXX
 --- /dev/null
 +++ b/net/vmnet-bridged.m
@@ -XXX,XX +XXX,XX @@
 +/*
 + * vmnet-bridged.m
 + *
 + * Copyright(c) 2021 Vladislav Yaroshchuk <yaroshchuk2000@gmail.com>
 + *
 + * This work is licensed under the terms of the GNU GPL, version 2 or later.
 + * See the COPYING file in the top-level directory.
 + *
 + */
 +
 +#include "qemu/osdep.h"
 +#include "qapi/qapi-types-net.h"
 +#include "vmnet_int.h"
 +#include "clients.h"
 +#include "qemu/error-report.h"
 +#include "qapi/error.h"
 +
 +#include <vmnet/vmnet.h>
 +
 +int net_init_vmnet_bridged(const Netdev *netdev, const char *name,
 +                           NetClientState *peer, Error **errp)
 +{
-+    NetClientState *nc;
++  error_setg(errp, "vmnet-bridged is not implemented yet");
-+    NetFilterState *nf;
++  return -1;
 +    NetFilterClass *nfc = NULL;
 +    Error *local_err = NULL;
 +
 +    QTAILQ_FOREACH(nc, &net_clients, next) {
 +        QTAILQ_FOREACH(nf, &nc->filters, next) {
 +            nfc = NETFILTER_GET_CLASS(OBJECT(nf));
 +            nfc->handle_event(nf, event, &local_err);
 +            if (local_err) {
 +                error_propagate(errp, local_err);
 +                return;
 +            }
 +        }
 +    }
 +}
-+
+diff --git a/net/vmnet-common.m b/net/vmnet-common.m
- void qmp_set_link(const char *name, bool up, Error **errp)
+new file mode 100644
- {
+index XXXXXXX..XXXXXXX
-     NetClientState *ncs[MAX_QUEUE_NUM];
+--- /dev/null
 +++ b/net/vmnet-common.m
@@ -XXX,XX +XXX,XX @@
 +/*
 + * vmnet-common.m - network client wrapper for Apple vmnet.framework
 + *
 + * Copyright(c) 2021 Vladislav Yaroshchuk <yaroshchuk2000@gmail.com>
 + * Copyright(c) 2021 Phillip Tennen <phillip@axleos.com>
 + *
 + * This work is licensed under the terms of the GNU GPL, version 2 or later.
 + * See the COPYING file in the top-level directory.
 + *
 + */
 +
 +#include "qemu/osdep.h"
 +#include "qapi/qapi-types-net.h"
 +#include "vmnet_int.h"
 +#include "clients.h"
 +#include "qemu/error-report.h"
 +#include "qapi/error.h"
 +
 +#include <vmnet/vmnet.h>
 diff --git a/net/vmnet-host.c b/net/vmnet-host.c
 new file mode 100644
 index XXXXXXX..XXXXXXX
 --- /dev/null
 +++ b/net/vmnet-host.c
@@ -XXX,XX +XXX,XX @@
 +/*
 + * vmnet-host.c
 + *
 + * Copyright(c) 2021 Vladislav Yaroshchuk <yaroshchuk2000@gmail.com>
 + *
 + * This work is licensed under the terms of the GNU GPL, version 2 or later.
 + * See the COPYING file in the top-level directory.
 + *
 + */
 +
 +#include "qemu/osdep.h"
 +#include "qapi/qapi-types-net.h"
 +#include "vmnet_int.h"
 +#include "clients.h"
 +#include "qemu/error-report.h"
 +#include "qapi/error.h"
 +
 +#include <vmnet/vmnet.h>
 +
 +int net_init_vmnet_host(const Netdev *netdev, const char *name,
 +                        NetClientState *peer, Error **errp) {
 +  error_setg(errp, "vmnet-host is not implemented yet");
 +  return -1;
 +}
 diff --git a/net/vmnet-shared.c b/net/vmnet-shared.c
 new file mode 100644
 index XXXXXXX..XXXXXXX
 --- /dev/null
 +++ b/net/vmnet-shared.c
@@ -XXX,XX +XXX,XX @@
 +/*
 + * vmnet-shared.c
 + *
 + * Copyright(c) 2021 Vladislav Yaroshchuk <yaroshchuk2000@gmail.com>
 + *
 + * This work is licensed under the terms of the GNU GPL, version 2 or later.
 + * See the COPYING file in the top-level directory.
 + *
 + */
 +
 +#include "qemu/osdep.h"
 +#include "qapi/qapi-types-net.h"
 +#include "vmnet_int.h"
 +#include "clients.h"
 +#include "qemu/error-report.h"
 +#include "qapi/error.h"
 +
 +#include <vmnet/vmnet.h>
 +
 +int net_init_vmnet_shared(const Netdev *netdev, const char *name,
 +                          NetClientState *peer, Error **errp)
 +{
 +  error_setg(errp, "vmnet-shared is not implemented yet");
 +  return -1;
 +}
 diff --git a/net/vmnet_int.h b/net/vmnet_int.h
 new file mode 100644
 index XXXXXXX..XXXXXXX
 --- /dev/null
 +++ b/net/vmnet_int.h
@@ -XXX,XX +XXX,XX @@
 +/*
 + * vmnet_int.h
 + *
 + * Copyright(c) 2021 Vladislav Yaroshchuk <yaroshchuk2000@gmail.com>
 + *
 + * This work is licensed under the terms of the GNU GPL, version 2 or later.
 + * See the COPYING file in the top-level directory.
 + *
 + */
 +#ifndef VMNET_INT_H
 +#define VMNET_INT_H
 +
 +#include "qemu/osdep.h"
 +#include "vmnet_int.h"
 +#include "clients.h"
 +
 +#include <vmnet/vmnet.h>
 +
 +typedef struct VmnetCommonState {
 +  NetClientState nc;
 +
 +} VmnetCommonState;
 +
 +
 +#endif /* VMNET_INT_H */
 diff --git a/qapi/net.json b/qapi/net.json
 index XXXXXXX..XXXXXXX 100644
 --- a/qapi/net.json
 +++ b/qapi/net.json
@@ -XXX,XX +XXX,XX @@
      '*queues':       'int' } }
  ##
 +# @NetdevVmnetHostOptions:
 +#
 +# vmnet (host mode) network backend.
 +#
 +# Allows the vmnet interface to communicate with other vmnet
 +# interfaces that are in host mode and also with the host.
 +#
 +# @start-address: The starting IPv4 address to use for the interface.
 +#                 Must be in the private IP range (RFC 1918). Must be
 +#                 specified along with @end-address and @subnet-mask.
 +#                 This address is used as the gateway address. The
 +#                 subsequent address up to and including end-address are
 +#                 placed in the DHCP pool.
 +#
 +# @end-address: The DHCP IPv4 range end address to use for the
 +#               interface. Must be in the private IP range (RFC 1918).
 +#               Must be specified along with @start-address and
 +#               @subnet-mask.
 +#
 +# @subnet-mask: The IPv4 subnet mask to use on the interface. Must
 +#               be specified along with @start-address and @subnet-mask.
 +#
 +# @isolated: Enable isolation for this interface. Interface isolation
 +#            ensures that vmnet interface is not able to communicate
 +#            with any other vmnet interfaces. Only communication with
 +#            host is allowed. Available since macOS Big Sur 11.0.
 +#
 +# @net-uuid: The identifier (UUID) to uniquely identify the isolated
 +#            network vmnet interface should be added to. If
 +#            set, no DHCP service is provided for this interface and
 +#            network communication is allowed only with other interfaces
 +#            added to this network identified by the UUID. Available
 +#            since macOS Big Sur 11.0.
 +#
 +# Since: 7.0
 +##
 +{ 'struct': 'NetdevVmnetHostOptions',
 +  'data': {
 +    '*start-address': 'str',
 +    '*end-address':   'str',
 +    '*subnet-mask':   'str',
 +    '*isolated':      'bool',
 +    '*net-uuid':      'str' },
 +  'if': 'CONFIG_VMNET' }
 +
 +##
 +# @NetdevVmnetSharedOptions:
 +#
 +# vmnet (shared mode) network backend.
 +#
 +# Allows traffic originating from the vmnet interface to reach the
 +# Internet through a network address translator (NAT).
 +# The vmnet interface can communicate with the host and with
 +# other shared mode interfaces on the same subnet. If no DHCP
 +# settings, subnet mask and IPv6 prefix specified, the interface can
 +# communicate with any of other interfaces in shared mode.
 +#
 +# @start-address: The starting IPv4 address to use for the interface.
 +#                 Must be in the private IP range (RFC 1918). Must be
 +#                 specified along with @end-address and @subnet-mask.
 +#                 This address is used as the gateway address. The
 +#                 subsequent address up to and including end-address are
 +#                 placed in the DHCP pool.
 +#
 +# @end-address: The DHCP IPv4 range end address to use for the
 +#               interface. Must be in the private IP range (RFC 1918).
 +#               Must be specified along with @start-address and @subnet-mask.
 +#
 +# @subnet-mask: The IPv4 subnet mask to use on the interface. Must
 +#                be specified along with @start-address and @subnet-mask.
 +#
 +# @isolated: Enable isolation for this interface. Interface isolation
 +#            ensures that vmnet interface is not able to communicate
 +#            with any other vmnet interfaces. Only communication with
 +#            host is allowed. Available since macOS Big Sur 11.0.
 +#
 +# @nat66-prefix: The IPv6 prefix to use into guest network. Must be a
 +#                unique local address i.e. start with fd00::/8 and have
 +#                length of 64.
 +#
 +# Since: 7.0
 +##
 +{ 'struct': 'NetdevVmnetSharedOptions',
 +  'data': {
 +    '*start-address': 'str',
 +    '*end-address':   'str',
 +    '*subnet-mask':   'str',
 +    '*isolated':      'bool',
 +    '*nat66-prefix':  'str' },
 +  'if': 'CONFIG_VMNET' }
 +
 +##
 +# @NetdevVmnetBridgedOptions:
 +#
 +# vmnet (bridged mode) network backend.
 +#
 +# Bridges the vmnet interface with a physical network interface.
 +#
 +# @ifname: The name of the physical interface to be bridged.
 +#
 +# @isolated: Enable isolation for this interface. Interface isolation
 +#            ensures that vmnet interface is not able to communicate
 +#            with any other vmnet interfaces. Only communication with
 +#            host is allowed. Available since macOS Big Sur 11.0.
 +#
 +# Since: 7.0
 +##
 +{ 'struct': 'NetdevVmnetBridgedOptions',
 +  'data': {
 +    'ifname':     'str',
 +    '*isolated':  'bool' },
 +  'if': 'CONFIG_VMNET' }
 +
 +##
  # @NetClientDriver:
  #
  # Available netdev drivers.
@@ -XXX,XX +XXX,XX @@
  # Since: 2.7
  #
  #        @vhost-vdpa since 5.1
 +#        @vmnet-host since 7.0
 +#        @vmnet-shared since 7.0
 +#        @vmnet-bridged since 7.0
  ##
  { 'enum': 'NetClientDriver',
    'data': [ 'none', 'nic', 'user', 'tap', 'l2tpv3', 'socket', 'vde',
 -            'bridge', 'hubport', 'netmap', 'vhost-user', 'vhost-vdpa' ] }
 +            'bridge', 'hubport', 'netmap', 'vhost-user', 'vhost-vdpa',
 +            { 'name': 'vmnet-host', 'if': 'CONFIG_VMNET' },
 +            { 'name': 'vmnet-shared', 'if': 'CONFIG_VMNET' },
 +            { 'name': 'vmnet-bridged', 'if': 'CONFIG_VMNET' }] }
  ##
  # @Netdev:
@@ -XXX,XX +XXX,XX @@
  # Since: 1.2
  #
  #        'l2tpv3' - since 2.1
 +#        'vmnet-host' - since 7.0
 +#        'vmnet-shared' - since 7.0
 +#        'vmnet-bridged' - since 7.0
  ##
  { 'union': 'Netdev',
    'base': { 'id': 'str', 'type': 'NetClientDriver' },
@@ -XXX,XX +XXX,XX @@
      'hubport':  'NetdevHubPortOptions',
      'netmap':   'NetdevNetmapOptions',
      'vhost-user': 'NetdevVhostUserOptions',
 -    'vhost-vdpa': 'NetdevVhostVDPAOptions' } }
 +    'vhost-vdpa': 'NetdevVhostVDPAOptions',
 +    'vmnet-host': { 'type': 'NetdevVmnetHostOptions',
 +                    'if': 'CONFIG_VMNET' },
 +    'vmnet-shared': { 'type': 'NetdevVmnetSharedOptions',
 +                      'if': 'CONFIG_VMNET' },
 +    'vmnet-bridged': { 'type': 'NetdevVmnetBridgedOptions',
 +                       'if': 'CONFIG_VMNET' } } }
  ##
  # @RxState:
 --
-.5.0
+.7.4

-[Qemu-devel] [PULL 02/26] colo-compare: implement the process of checkpoint
+[PULL V3 09/13] net/vmnet: implement shared mode (vmnet-shared)
-From: Zhang Chen <zhangckid@gmail.com>
+From: Vladislav Yaroshchuk <yaroshchuk2000@gmail.com>
-While do checkpoint, we need to flush all the unhandled packets,
+Interaction with vmnet.framework in different modes
-By using the filter notifier mechanism, we can easily to notify
+differs only on configuration stage, so we can create
-every compare object to do this process, which runs inside
+common `send`, `receive`, etc. procedures and reuse them.
-of compare threads as a coroutine.
+vmnet.framework supports iov, but writing more than
-Signed-off-by: zhanghailiang <zhang.zhanghailiang@huawei.com>
+one iov into vmnet interface fails with
-Signed-off-by: Zhang Chen <zhangckid@gmail.com>
+'VMNET_INVALID_ARGUMENT'. Collecting provided iovs into
-Signed-off-by: Zhang Chen <chen.zhang@intel.com>
+one and passing it to vmnet works fine. That's the
 reason why receive_iov() left unimplemented. But it still
 works with good enough performance having .receive()
 implemented only.
 Also, there is no way to unsubscribe from vmnet packages
 receiving except registering and unregistering event
 callback or simply drop packages just ignoring and
 not processing them when related flag is set. Here we do
 using the second way.
 Signed-off-by: Phillip Tennen <phillip@axleos.com>
 Signed-off-by: Vladislav Yaroshchuk <yaroshchuk2000@gmail.com>
 Signed-off-by: Jason Wang <jasowang@redhat.com>
 ---
- include/migration/colo.h |  6 ++++
+ net/vmnet-common.m | 314 +++++++++++++++++++++++++++++++++++++++++++++++++++++
- net/colo-compare.c       | 78 ++++++++++++++++++++++++++++++++++++++++++++++++
+ net/vmnet-shared.c |  83 +++++++++++++-
- net/colo-compare.h       | 22 ++++++++++++++
+ net/vmnet_int.h    |  23 ++++
-files changed, 106 insertions(+)
+files changed, 416 insertions(+), 4 deletions(-)
- create mode 100644 net/colo-compare.h
+diff --git a/net/vmnet-common.m b/net/vmnet-common.m
 diff --git a/include/migration/colo.h b/include/migration/colo.h
 index XXXXXXX..XXXXXXX 100644
---- a/include/migration/colo.h
+--- a/net/vmnet-common.m
-+++ b/include/migration/colo.h
++++ b/net/vmnet-common.m
 @@ -XXX,XX +XXX,XX @@
- #include "qemu-common.h"
+  */
- #include "qapi/qapi-types-migration.h"
+ #include "qemu/osdep.h"
-+enum colo_event {
++#include "qemu/main-loop.h"
-+    COLO_EVENT_NONE,
++#include "qemu/log.h"
-+    COLO_EVENT_CHECKPOINT,
+ #include "qapi/qapi-types-net.h"
-+    COLO_EVENT_FAILOVER,
+ #include "vmnet_int.h"
-+};
+ #include "clients.h"
 +
  void colo_info_init(void);
  void migrate_start_colo_process(MigrationState *s);
 diff --git a/net/colo-compare.c b/net/colo-compare.c
 index XXXXXXX..XXXXXXX 100644
 --- a/net/colo-compare.c
 +++ b/net/colo-compare.c
 @@ -XXX,XX +XXX,XX @@
- #include "qemu/sockets.h"
+ #include "qapi/error.h"
- #include "colo.h"
- #include "sysemu/iothread.h"
+ #include <vmnet/vmnet.h>
-+#include "net/colo-compare.h"
++#include <dispatch/dispatch.h>
-+#include "migration/colo.h"
++
++#ifdef DEBUG
- #define TYPE_COLO_COMPARE "colo-compare"
++#define D(x) x
- #define COLO_COMPARE(obj) \
++#define D_LOG(...) qemu_log(__VA_ARGS__)
-     OBJECT_CHECK(CompareState, (obj), TYPE_COLO_COMPARE)
++#else
++#define D(x) do { } while (0)
-+static QTAILQ_HEAD(, CompareState) net_compares =
++#define D_LOG(...) do { } while (0)
-+       QTAILQ_HEAD_INITIALIZER(net_compares);
++#endif
 +
- #define COMPARE_READ_LEN_MAX NET_BUFSIZE
++typedef struct vmpktdesc vmpktdesc_t;
- #define MAX_QUEUE_SIZE 1024
++typedef struct iovec iovec_t;
++
-@@ -XXX,XX +XXX,XX @@
++static void vmnet_set_send_enabled(VmnetCommonState *s, bool enable)
- /* TODO: Should be configurable */
++{
- #define REGULAR_PACKET_CHECK_MS 3000
++    s->send_enabled = enable;
++}
-+static QemuMutex event_mtx;
++
-+static QemuCond event_complete_cond;
++
-+static int event_unhandled_count;
++static void vmnet_send_completed(NetClientState *nc, ssize_t len)
-+
++{
- /*
++    VmnetCommonState *s = DO_UPCAST(VmnetCommonState, nc, nc);
-  *  + CompareState ++
++    vmnet_set_send_enabled(s, true);
-  *  |               |
++}
-@@ -XXX,XX +XXX,XX @@ typedef struct CompareState {
++
-     IOThread *iothread;
++
-     GMainContext *worker_context;
++static void vmnet_send(NetClientState *nc,
-     QEMUTimer *packet_check_timer;
++                       interface_event_t event_id,
-+
++                       xpc_object_t event)
-+    QEMUBH *event_bh;
++{
-+    enum colo_event event;
++    assert(event_id == VMNET_INTERFACE_PACKETS_AVAILABLE);
 +
-+    QTAILQ_ENTRY(CompareState) next;
++    VmnetCommonState *s;
- } CompareState;
++    uint64_t packets_available;
++
- typedef struct CompareClass {
++    struct iovec *iov;
-@@ -XXX,XX +XXX,XX @@ static void check_old_packet_regular(void *opaque)
++    struct vmpktdesc *packets;
-                 REGULAR_PACKET_CHECK_MS);
++    int pkt_cnt;
- }
++    int i;
++
-+/* Public API, Used for COLO frame to notify compare event */
++    vmnet_return_t if_status;
-+void colo_notify_compares_event(void *opaque, int event, Error **errp)
++    ssize_t size;
-+{
++
-+    CompareState *s;
++    s = DO_UPCAST(VmnetCommonState, nc, nc);
 +
-+    qemu_mutex_lock(&event_mtx);
++    packets_available = xpc_dictionary_get_uint64(
-+    QTAILQ_FOREACH(s, &net_compares, next) {
++        event,
-+        s->event = event;
++        vmnet_estimated_packets_available_key
-+        qemu_bh_schedule(s->event_bh);
++    );
-+        event_unhandled_count++;
++
-+    }
++    pkt_cnt = (packets_available < VMNET_PACKETS_LIMIT) ?
-+    /* Wait all compare threads to finish handling this event */
++              packets_available :
-+    while (event_unhandled_count > 0) {
++              VMNET_PACKETS_LIMIT;
-+        qemu_cond_wait(&event_complete_cond, &event_mtx);
++
-+    }
++
-+
++    iov = s->iov_buf;
-+    qemu_mutex_unlock(&event_mtx);
++    packets = s->packets_buf;
-+}
++
-+
++    for (i = 0; i < pkt_cnt; ++i) {
- static void colo_compare_timer_init(CompareState *s)
++        packets[i].vm_pkt_size = s->max_packet_size;
- {
++        packets[i].vm_pkt_iovcnt = 1;
-     AioContext *ctx = iothread_get_aio_context(s->iothread);
++        packets[i].vm_flags = 0;
-@@ -XXX,XX +XXX,XX @@ static void colo_compare_timer_del(CompareState *s)
++    }
-     }
++
-  }
++    if_status = vmnet_read(s->vmnet_if, packets, &pkt_cnt);
++    if (if_status != VMNET_SUCCESS) {
-+static void colo_flush_packets(void *opaque, void *user_data);
++        error_printf("vmnet: read failed: %s\n",
-+
++                     vmnet_status_map_str(if_status));
-+static void colo_compare_handle_event(void *opaque)
++    }
-+{
++    qemu_mutex_lock_iothread();
-+    CompareState *s = opaque;
++    for (i = 0; i < pkt_cnt; ++i) {
-+
++        size = qemu_send_packet_async(nc,
-+    switch (s->event) {
++                                      iov[i].iov_base,
-+    case COLO_EVENT_CHECKPOINT:
++                                      packets[i].vm_pkt_size,
-+        g_queue_foreach(&s->conn_list, colo_flush_packets, s);
++                                      vmnet_send_completed);
-+        break;
++        if (size == 0) {
-+    case COLO_EVENT_FAILOVER:
++            vmnet_set_send_enabled(s, false);
-+        break;
++        } else if (size < 0) {
 +    default:
 +        break;
 +    }
 +
 +    assert(event_unhandled_count > 0);
 +
 +    qemu_mutex_lock(&event_mtx);
 +    event_unhandled_count--;
 +    qemu_cond_broadcast(&event_complete_cond);
 +    qemu_mutex_unlock(&event_mtx);
 +}
 +
  static void colo_compare_iothread(CompareState *s)
  {
      object_ref(OBJECT(s->iothread));
@@ -XXX,XX +XXX,XX @@ static void colo_compare_iothread(CompareState *s)
                               s, s->worker_context, true);
      colo_compare_timer_init(s);
 +    s->event_bh = qemu_bh_new(colo_compare_handle_event, s);
  }
  static char *compare_get_pri_indev(Object *obj, Error **errp)
@@ -XXX,XX +XXX,XX @@ static void colo_compare_complete(UserCreatable *uc, Error **errp)
      net_socket_rs_init(&s->pri_rs, compare_pri_rs_finalize, s->vnet_hdr);
      net_socket_rs_init(&s->sec_rs, compare_sec_rs_finalize, s->vnet_hdr);
 +    QTAILQ_INSERT_TAIL(&net_compares, s, next);
 +
      g_queue_init(&s->conn_list);
 +    qemu_mutex_init(&event_mtx);
 +    qemu_cond_init(&event_complete_cond);
 +
      s->connection_track_table = g_hash_table_new_full(connection_key_hash,
                                                        connection_key_equal,
                                                        g_free,
@@ -XXX,XX +XXX,XX @@ static void colo_compare_init(Object *obj)
  static void colo_compare_finalize(Object *obj)
  {
      CompareState *s = COLO_COMPARE(obj);
 +    CompareState *tmp = NULL;
      qemu_chr_fe_deinit(&s->chr_pri_in, false);
      qemu_chr_fe_deinit(&s->chr_sec_in, false);
@@ -XXX,XX +XXX,XX @@ static void colo_compare_finalize(Object *obj)
      if (s->iothread) {
          colo_compare_timer_del(s);
      }
 +
 +    qemu_bh_delete(s->event_bh);
 +
 +    QTAILQ_FOREACH(tmp, &net_compares, next) {
 +        if (tmp == s) {
 +            QTAILQ_REMOVE(&net_compares, s, next);
 +            break;
 +        }
 +    }
-+
++    qemu_mutex_unlock_iothread();
-     /* Release all unhandled packets after compare thead exited */
++
-     g_queue_foreach(&s->conn_list, colo_flush_packets, s);
++}
++
-@@ -XXX,XX +XXX,XX @@ static void colo_compare_finalize(Object *obj)
++
-     if (s->iothread) {
++static void vmnet_register_event_callback(VmnetCommonState *s)
-         object_unref(OBJECT(s->iothread));
++{
-     }
++    dispatch_queue_t avail_pkt_q = dispatch_queue_create(
-+
++        "org.qemu.vmnet.if_queue",
-+    qemu_mutex_destroy(&event_mtx);
++        DISPATCH_QUEUE_SERIAL
-+    qemu_cond_destroy(&event_complete_cond);
++    );
 +
-     g_free(s->pri_indev);
++    vmnet_interface_set_event_callback(
-     g_free(s->sec_indev);
++        s->vmnet_if,
-     g_free(s->outdev);
++        VMNET_INTERFACE_PACKETS_AVAILABLE,
-diff --git a/net/colo-compare.h b/net/colo-compare.h
++        avail_pkt_q,
-new file mode 100644
++        ^(interface_event_t event_id, xpc_object_t event) {
-index XXXXXXX..XXXXXXX
++          if (s->send_enabled) {
---- /dev/null
++              vmnet_send(&s->nc, event_id, event);
-+++ b/net/colo-compare.h
++          }
 +        });
 +}
 +
 +
 +static void vmnet_bufs_init(VmnetCommonState *s)
 +{
 +    int i;
 +    struct vmpktdesc *packets;
 +    struct iovec *iov;
 +
 +    packets = s->packets_buf;
 +    iov = s->iov_buf;
 +
 +    for (i = 0; i < VMNET_PACKETS_LIMIT; ++i) {
 +        iov[i].iov_len = s->max_packet_size;
 +        iov[i].iov_base = g_malloc0(iov[i].iov_len);
 +        packets[i].vm_pkt_iov = iov + i;
 +    }
 +}
 +
 +
 +const char *vmnet_status_map_str(vmnet_return_t status)
 +{
 +    switch (status) {
 +    case VMNET_SUCCESS:
 +        return "success";
 +    case VMNET_FAILURE:
 +        return "general failure";
 +    case VMNET_MEM_FAILURE:
 +        return "memory allocation failure";
 +    case VMNET_INVALID_ARGUMENT:
 +        return "invalid argument specified";
 +    case VMNET_SETUP_INCOMPLETE:
 +        return "interface setup is not complete";
 +    case VMNET_INVALID_ACCESS:
 +        return "invalid access, permission denied";
 +    case VMNET_PACKET_TOO_BIG:
 +        return "packet size is larger than MTU";
 +    case VMNET_BUFFER_EXHAUSTED:
 +        return "buffers exhausted in kernel";
 +    case VMNET_TOO_MANY_PACKETS:
 +        return "packet count exceeds limit";
 +#if defined(MAC_OS_VERSION_11_0) && \
 +    MAC_OS_X_VERSION_MIN_REQUIRED >= MAC_OS_VERSION_11_0
 +    case VMNET_SHARING_SERVICE_BUSY:
 +        return "conflict, sharing service is in use";
 +#endif
 +    default:
 +        return "unknown vmnet error";
 +    }
 +}
 +
 +
 +int vmnet_if_create(NetClientState *nc,
 +                    xpc_object_t if_desc,
 +                    Error **errp,
 +                    void (*completion_callback)(xpc_object_t interface_param))
 +{
 +    VmnetCommonState *s;
 +
 +    dispatch_queue_t if_create_q;
 +    dispatch_semaphore_t if_created_sem;
 +
 +    __block vmnet_return_t if_status;
 +
 +    if_create_q = dispatch_queue_create("org.qemu.vmnet.create",
 +                                        DISPATCH_QUEUE_SERIAL);
 +    if_created_sem = dispatch_semaphore_create(0);
 +
 +    xpc_dictionary_set_bool(
 +        if_desc,
 +        vmnet_allocate_mac_address_key,
 +        false
 +    );
 +
 +    D(D_LOG("vmnet.start.interface_desc:\n");
 +    xpc_dictionary_apply(if_desc,
 +                         ^bool(const char *k, xpc_object_t v) {
 +                           char *desc = xpc_copy_description(v);
 +                           D_LOG("  %s=%s\n", k, desc);
 +                           free(desc);
 +                           return true;
 +                         }));
 +
 +    s = DO_UPCAST(VmnetCommonState, nc, nc);
 +    s->vmnet_if = vmnet_start_interface(
 +        if_desc,
 +        if_create_q,
 +        ^(vmnet_return_t status, xpc_object_t interface_param) {
 +          if_status = status;
 +          if (status != VMNET_SUCCESS || !interface_param) {
 +              dispatch_semaphore_signal(if_created_sem);
 +              return;
 +          }
 +
 +          D(D_LOG("vmnet.start.interface_param:\n");
 +                xpc_dictionary_apply(interface_param,
 +                                     ^bool(const char *k, xpc_object_t v) {
 +                                       char *desc = xpc_copy_description(v);
 +                                       D_LOG("  %s=%s\n", k, desc);
 +                                       free(desc);
 +                                       return true;
 +                                     }));
 +
 +          s->mtu = xpc_dictionary_get_uint64(
 +              interface_param,
 +              vmnet_mtu_key);
 +          s->max_packet_size = xpc_dictionary_get_uint64(
 +              interface_param,
 +              vmnet_max_packet_size_key);
 +
 +          if (completion_callback) {
 +              completion_callback(interface_param);
 +          }
 +          dispatch_semaphore_signal(if_created_sem);
 +        });
 +
 +    if (s->vmnet_if == NULL) {
 +        error_setg(errp, "unable to create interface with requested params");
 +        return -1;
 +    }
 +
 +    dispatch_semaphore_wait(if_created_sem, DISPATCH_TIME_FOREVER);
 +    dispatch_release(if_create_q);
 +
 +    if (if_status != VMNET_SUCCESS) {
 +        error_setg(errp,
 +                   "cannot create vmnet interface: %s",
 +                   vmnet_status_map_str(if_status));
 +        return -1;
 +    }
 +
 +    vmnet_register_event_callback(s);
 +    vmnet_bufs_init(s);
 +    vmnet_set_send_enabled(s, true);
 +
 +    return 0;
 +}
 +
 +
 +ssize_t vmnet_receive_common(NetClientState *nc,
 +                             const uint8_t *buf,
 +                             size_t size)
 +{
 +    VmnetCommonState *s;
 +    vmpktdesc_t packet;
 +    iovec_t iov;
 +    int pkt_cnt;
 +    vmnet_return_t if_status;
 +
 +    s = DO_UPCAST(VmnetCommonState, nc, nc);
 +
 +    if (size > s->max_packet_size) {
 +        warn_report("vmnet: packet is too big, %zu > %llu\n",
 +                    packet.vm_pkt_size,
 +                    s->max_packet_size);
 +        return -1;
 +    }
 +
 +    iov.iov_base = (char *) buf;
 +    iov.iov_len = size;
 +
 +    packet.vm_pkt_iovcnt = 1;
 +    packet.vm_flags = 0;
 +    packet.vm_pkt_size = size;
 +    packet.vm_pkt_iov = &iov;
 +
 +    pkt_cnt = 1;
 +    if_status = vmnet_write(s->vmnet_if, &packet, &pkt_cnt);
 +
 +    if (if_status != VMNET_SUCCESS) {
 +        error_report("vmnet: write error: %s\n",
 +                     vmnet_status_map_str(if_status));
 +    }
 +
 +    if (if_status == VMNET_SUCCESS && pkt_cnt) {
 +        return size;
 +    }
 +    return 0;
 +}
 +
 +
 +void vmnet_cleanup_common(NetClientState *nc)
 +{
 +    VmnetCommonState *s;
 +    dispatch_queue_t if_destroy_q;
 +
 +    s = DO_UPCAST(VmnetCommonState, nc, nc);
 +
 +    qemu_purge_queued_packets(nc);
 +    vmnet_set_send_enabled(s, false);
 +
 +    if (s->vmnet_if == NULL) {
 +        return;
 +    }
 +
 +    if_destroy_q = dispatch_queue_create(
 +        "org.qemu.vmnet.destroy",
 +        DISPATCH_QUEUE_SERIAL
 +    );
 +
 +    vmnet_stop_interface(
 +        s->vmnet_if,
 +        if_destroy_q,
 +        ^(vmnet_return_t status) {
 +        });
 +
 +    for (int i = 0; i < VMNET_PACKETS_LIMIT; ++i) {
 +        g_free(s->iov_buf[i].iov_base);
 +    }
 +}
 diff --git a/net/vmnet-shared.c b/net/vmnet-shared.c
 index XXXXXXX..XXXXXXX 100644
 --- a/net/vmnet-shared.c
 +++ b/net/vmnet-shared.c
 @@ -XXX,XX +XXX,XX @@
-+/*
-+ * COarse-grain LOck-stepping Virtual Machines for Non-stop Service (COLO)
+ #include "qemu/osdep.h"
-+ * (a.k.a. Fault Tolerance or Continuous Replication)
+ #include "qapi/qapi-types-net.h"
-+ *
++#include "qapi/error.h"
-+ * Copyright (c) 2017 HUAWEI TECHNOLOGIES CO., LTD.
+ #include "vmnet_int.h"
-+ * Copyright (c) 2017 FUJITSU LIMITED
+ #include "clients.h"
-+ * Copyright (c) 2017 Intel Corporation
+-#include "qemu/error-report.h"
-+ *
+-#include "qapi/error.h"
-+ * Authors:
-+ *    zhanghailiang <zhang.zhanghailiang@huawei.com>
+ #include <vmnet/vmnet.h>
-+ *    Zhang Chen <zhangckid@gmail.com>
-+ *
++typedef struct VmnetSharedState {
-+ * This work is licensed under the terms of the GNU GPL, version 2 or
++  VmnetCommonState cs;
-+ * later.  See the COPYING file in the top-level directory.
++} VmnetSharedState;
-+ */
++
 +
-+#ifndef QEMU_COLO_COMPARE_H
++static xpc_object_t create_if_desc(const Netdev *netdev, Error **errp)
-+#define QEMU_COLO_COMPARE_H
++{
-+
++    const NetdevVmnetSharedOptions *options = &(netdev->u.vmnet_shared);
-+void colo_notify_compares_event(void *opaque, int event, Error **errp);
++    xpc_object_t if_desc = xpc_dictionary_create(NULL, NULL, 0);
 +
-+#endif /* QEMU_COLO_COMPARE_H */
++    xpc_dictionary_set_uint64(
 +        if_desc,
 +        vmnet_operation_mode_key,
 +        VMNET_SHARED_MODE
 +    );
 +
 +#if defined(MAC_OS_VERSION_11_0) && \
 +    MAC_OS_X_VERSION_MIN_REQUIRED >= MAC_OS_VERSION_11_0
 +    xpc_dictionary_set_bool(
 +        if_desc,
 +        vmnet_enable_isolation_key,
 +        options->isolated
 +    );
 +#else
 +    if (options->has_isolated) {
 +        error_setg(errp,
 +                   "vmnet-shared.isolated feature is "
 +                   "unavailable: outdated vmnet.framework API");
 +    }
 +#endif
 +
 +    if (options->has_nat66_prefix) {
 +        xpc_dictionary_set_string(if_desc,
 +                                  vmnet_nat66_prefix_key,
 +                                  options->nat66_prefix);
 +    }
 +
 +    if (options->has_start_address ||
 +        options->has_end_address ||
 +        options->has_subnet_mask) {
 +
 +        if (options->has_start_address &&
 +            options->has_end_address &&
 +            options->has_subnet_mask) {
 +
 +            xpc_dictionary_set_string(if_desc,
 +                                      vmnet_start_address_key,
 +                                      options->start_address);
 +            xpc_dictionary_set_string(if_desc,
 +                                      vmnet_end_address_key,
 +                                      options->end_address);
 +            xpc_dictionary_set_string(if_desc,
 +                                      vmnet_subnet_mask_key,
 +                                      options->subnet_mask);
 +        } else {
 +            error_setg(
 +                errp,
 +                "'start-address', 'end-address', 'subnet-mask' "
 +                "should be provided together"
 +            );
 +        }
 +    }
 +
 +    return if_desc;
 +}
 +
 +static NetClientInfo net_vmnet_shared_info = {
 +    .type = NET_CLIENT_DRIVER_VMNET_SHARED,
 +    .size = sizeof(VmnetSharedState),
 +    .receive = vmnet_receive_common,
 +    .cleanup = vmnet_cleanup_common,
 +};
 +
  int net_init_vmnet_shared(const Netdev *netdev, const char *name,
                            NetClientState *peer, Error **errp)
  {
 -  error_setg(errp, "vmnet-shared is not implemented yet");
 -  return -1;
 +    NetClientState *nc = qemu_new_net_client(&net_vmnet_shared_info,
 +                                             peer, "vmnet-shared", name);
 +    xpc_object_t if_desc = create_if_desc(netdev, errp);
 +
 +    return vmnet_if_create(nc, if_desc, errp, NULL);
  }
 diff --git a/net/vmnet_int.h b/net/vmnet_int.h
 index XXXXXXX..XXXXXXX 100644
 --- a/net/vmnet_int.h
 +++ b/net/vmnet_int.h
@@ -XXX,XX +XXX,XX @@
  #include <vmnet/vmnet.h>
 +#define VMNET_PACKETS_LIMIT 50
 +
  typedef struct VmnetCommonState {
    NetClientState nc;
 +  interface_ref vmnet_if;
 +
 +  bool send_enabled;
 +
 +  uint64_t mtu;
 +  uint64_t max_packet_size;
 +
 +  struct vmpktdesc packets_buf[VMNET_PACKETS_LIMIT];
 +  struct iovec iov_buf[VMNET_PACKETS_LIMIT];
  } VmnetCommonState;
 +const char *vmnet_status_map_str(vmnet_return_t status);
 +
 +int vmnet_if_create(NetClientState *nc,
 +                    xpc_object_t if_desc,
 +                    Error **errp,
 +                    void (*completion_callback)(xpc_object_t interface_param));
 +
 +ssize_t vmnet_receive_common(NetClientState *nc,
 +                             const uint8_t *buf,
 +                             size_t size);
 +
 +void vmnet_cleanup_common(NetClientState *nc);
  #endif /* VMNET_INT_H */
 --
-.5.0
+.7.4

-[Qemu-devel] [PULL 05/26] COLO: Add block replication into colo process
+Deleted patch
-From: Zhang Chen <zhangckid@gmail.com>
-Make sure master start block replication after slave's block
-replication started.
-Besides, we need to activate VM's blocks before goes into
-COLO state.
-Signed-off-by: zhanghailiang <zhang.zhanghailiang@huawei.com>
-Signed-off-by: Li Zhijian <lizhijian@cn.fujitsu.com>
-Signed-off-by: Zhang Chen <zhangckid@gmail.com>
-Signed-off-by: Zhang Chen <chen.zhang@intel.com>
-Signed-off-by: Jason Wang <jasowang@redhat.com>
----
- migration/colo.c      | 43 +++++++++++++++++++++++++++++++++++++++++++
- migration/migration.c | 10 ++++++++++
-files changed, 53 insertions(+)
-diff --git a/migration/colo.c b/migration/colo.c
-index XXXXXXX..XXXXXXX 100644
---- a/migration/colo.c
-+++ b/migration/colo.c
-@@ -XXX,XX +XXX,XX @@
- #include "replication.h"
- #include "net/colo-compare.h"
- #include "net/colo.h"
-+#include "block/block.h"
- static bool vmstate_loading;
- static Notifier packets_compare_notifier;
-@@ -XXX,XX +XXX,XX @@ static void secondary_vm_do_failover(void)
- {
-     int old_state;
-     MigrationIncomingState *mis = migration_incoming_get_current();
-+    Error *local_err = NULL;
-     /* Can not do failover during the process of VM's loading VMstate, Or
-      * it will break the secondary VM.
-@@ -XXX,XX +XXX,XX @@ static void secondary_vm_do_failover(void)
-     migrate_set_state(&mis->state, MIGRATION_STATUS_COLO,
-                       MIGRATION_STATUS_COMPLETED);
-+    replication_stop_all(true, &local_err);
-+    if (local_err) {
-+        error_report_err(local_err);
-+    }
-+
-     if (!autostart) {
-         error_report("\"-S\" qemu option will be ignored in secondary side");
-         /* recover runstate to normal migration finish state */
-@@ -XXX,XX +XXX,XX @@ static void primary_vm_do_failover(void)
- {
-     MigrationState *s = migrate_get_current();
-     int old_state;
-+    Error *local_err = NULL;
-     migrate_set_state(&s->state, MIGRATION_STATUS_COLO,
-                       MIGRATION_STATUS_COMPLETED);
-@@ -XXX,XX +XXX,XX @@ static void primary_vm_do_failover(void)
-                      FailoverStatus_str(old_state));
-         return;
-     }
-+
-+    replication_stop_all(true, &local_err);
-+    if (local_err) {
-+        error_report_err(local_err);
-+        local_err = NULL;
-+    }
-+
-     /* Notify COLO thread that failover work is finished */
-     qemu_sem_post(&s->colo_exit_sem);
- }
-@@ -XXX,XX +XXX,XX @@ static int colo_do_checkpoint_transaction(MigrationState *s,
-     qemu_savevm_state_header(fb);
-     qemu_savevm_state_setup(fb);
-     qemu_mutex_lock_iothread();
-+    replication_do_checkpoint_all(&local_err);
-+    if (local_err) {
-+        qemu_mutex_unlock_iothread();
-+        goto out;
-+    }
-     qemu_savevm_state_complete_precopy(fb, false, false);
-     qemu_mutex_unlock_iothread();
-@@ -XXX,XX +XXX,XX @@ static void colo_process_checkpoint(MigrationState *s)
-     object_unref(OBJECT(bioc));
-     qemu_mutex_lock_iothread();
-+    replication_start_all(REPLICATION_MODE_PRIMARY, &local_err);
-+    if (local_err) {
-+        qemu_mutex_unlock_iothread();
-+        goto out;
-+    }
-+
-     vm_start();
-     qemu_mutex_unlock_iothread();
-     trace_colo_vm_state_change("stop", "run");
-@@ -XXX,XX +XXX,XX @@ void *colo_process_incoming_thread(void *opaque)
-     object_unref(OBJECT(bioc));
-     qemu_mutex_lock_iothread();
-+    replication_start_all(REPLICATION_MODE_SECONDARY, &local_err);
-+    if (local_err) {
-+        qemu_mutex_unlock_iothread();
-+        goto out;
-+    }
-     vm_start();
-     trace_colo_vm_state_change("stop", "run");
-     qemu_mutex_unlock_iothread();
-@@ -XXX,XX +XXX,XX @@ void *colo_process_incoming_thread(void *opaque)
-             goto out;
-         }
-+        replication_get_error_all(&local_err);
-+        if (local_err) {
-+            qemu_mutex_unlock_iothread();
-+            goto out;
-+        }
-+        /* discard colo disk buffer */
-+        replication_do_checkpoint_all(&local_err);
-+        if (local_err) {
-+            qemu_mutex_unlock_iothread();
-+            goto out;
-+        }
-+
-         vmstate_loading = false;
-         vm_start();
-         trace_colo_vm_state_change("stop", "run");
-diff --git a/migration/migration.c b/migration/migration.c
-index XXXXXXX..XXXXXXX 100644
---- a/migration/migration.c
-+++ b/migration/migration.c
-@@ -XXX,XX +XXX,XX @@ static void process_incoming_migration_co(void *opaque)
-     MigrationIncomingState *mis = migration_incoming_get_current();
-     PostcopyState ps;
-     int ret;
-+    Error *local_err = NULL;
-     assert(mis->from_src_file);
-     mis->migration_incoming_co = qemu_coroutine_self();
-@@ -XXX,XX +XXX,XX @@ static void process_incoming_migration_co(void *opaque)
-     /* we get COLO info, and know if we are in COLO mode */
-     if (!ret && migration_incoming_enable_colo()) {
-+        /* Make sure all file formats flush their mutable metadata */
-+        bdrv_invalidate_cache_all(&local_err);
-+        if (local_err) {
-+            migrate_set_state(&mis->state, MIGRATION_STATUS_ACTIVE,
-+                    MIGRATION_STATUS_FAILED);
-+            error_report_err(local_err);
-+            exit(EXIT_FAILURE);
-+        }
-+
-         qemu_thread_create(&mis->colo_incoming_thread, "COLO incoming",
-              colo_process_incoming_thread, mis, QEMU_THREAD_JOINABLE);
-         mis->have_colo_incoming_thread = true;
---
-.5.0

-[Qemu-devel] [PULL 07/26] COLO: Load dirty pages into SVM's RAM cache firstly
+[PULL V3 10/13] net/vmnet: implement host mode (vmnet-host)
-From: Zhang Chen <zhangckid@gmail.com>
+From: Vladislav Yaroshchuk <yaroshchuk2000@gmail.com>
-We should not load PVM's state directly into SVM, because there maybe some
+Signed-off-by: Vladislav Yaroshchuk <yaroshchuk2000@gmail.com>
 errors happen when SVM is receving data, which will break SVM.
 We need to ensure receving all data before load the state into SVM. We use
 an extra memory to cache these data (PVM's ram). The ram cache in secondary side
 is initially the same as SVM/PVM's memory. And in the process of checkpoint,
 we cache the dirty pages of PVM into this ram cache firstly, so this ram cache
 always the same as PVM's memory at every checkpoint, then we flush this cached ram
 to SVM after we receive all PVM's state.
 Signed-off-by: zhanghailiang <zhang.zhanghailiang@huawei.com>
 Signed-off-by: Li Zhijian <lizhijian@cn.fujitsu.com>
 Signed-off-by: Zhang Chen <zhangckid@gmail.com>
 Signed-off-by: Zhang Chen <chen.zhang@intel.com>
 Reviewed-by: Dr. David Alan Gilbert <dgilbert@redhat.com>
 Signed-off-by: Jason Wang <jasowang@redhat.com>
 ---
- include/exec/ram_addr.h |  1 +
+ net/vmnet-host.c | 110 ++++++++++++++++++++++++++++++++++++++++++++++++++++---
- migration/migration.c   |  7 +++++
+file changed, 104 insertions(+), 6 deletions(-)
  migration/ram.c         | 83 +++++++++++++++++++++++++++++++++++++++++++++++--
  migration/ram.h         |  4 +++
  migration/savevm.c      |  2 +-
 files changed, 94 insertions(+), 3 deletions(-)
-diff --git a/include/exec/ram_addr.h b/include/exec/ram_addr.h
+diff --git a/net/vmnet-host.c b/net/vmnet-host.c
 index XXXXXXX..XXXXXXX 100644
---- a/include/exec/ram_addr.h
+--- a/net/vmnet-host.c
-+++ b/include/exec/ram_addr.h
++++ b/net/vmnet-host.c
-@@ -XXX,XX +XXX,XX @@ struct RAMBlock {
+@@ -XXX,XX +XXX,XX @@
-     struct rcu_head rcu;
+  */
-     struct MemoryRegion *mr;
-     uint8_t *host;
+ #include "qemu/osdep.h"
-+    uint8_t *colo_cache; /* For colo, VM's ram cache */
++#include "qemu/uuid.h"
-     ram_addr_t offset;
+ #include "qapi/qapi-types-net.h"
-     ram_addr_t used_length;
+-#include "vmnet_int.h"
-     ram_addr_t max_length;
+-#include "clients.h"
-diff --git a/migration/migration.c b/migration/migration.c
+-#include "qemu/error-report.h"
-index XXXXXXX..XXXXXXX 100644
+ #include "qapi/error.h"
---- a/migration/migration.c
++#include "clients.h"
-+++ b/migration/migration.c
++#include "vmnet_int.h"
-@@ -XXX,XX +XXX,XX @@ static void process_incoming_migration_co(void *opaque)
-             exit(EXIT_FAILURE);
+ #include <vmnet/vmnet.h>
-         }
++typedef struct VmnetHostState {
-+        if (colo_init_ram_cache() < 0) {
++  VmnetCommonState cs;
-+            error_report("Init ram cache failed");
++  QemuUUID network_uuid;
-+            exit(EXIT_FAILURE);
++} VmnetHostState;
 +
 +static xpc_object_t create_if_desc(const Netdev *netdev,
 +                                   NetClientState *nc,
 +                                   Error **errp)
 +{
 +    const NetdevVmnetHostOptions *options = &(netdev->u.vmnet_host);
 +
 +    xpc_object_t if_desc = xpc_dictionary_create(NULL, NULL, 0);
 +
 +    xpc_dictionary_set_uint64(
 +        if_desc,
 +        vmnet_operation_mode_key,
 +        VMNET_HOST_MODE
 +    );
 +
 +#if defined(MAC_OS_VERSION_11_0) && \
 +    MAC_OS_X_VERSION_MIN_REQUIRED >= MAC_OS_VERSION_11_0
 +
 +    VmnetCommonState *cs = DO_UPCAST(VmnetCommonState, nc, nc);
 +    VmnetHostState *hs = DO_UPCAST(VmnetHostState, cs, cs);
 +
 +    xpc_dictionary_set_bool(
 +        if_desc,
 +        vmnet_enable_isolation_key,
 +        options->isolated
 +    );
 +
 +    if (options->has_net_uuid) {
 +        if (qemu_uuid_parse(options->net_uuid, &hs->network_uuid) < 0) {
 +            error_setg(errp, "Invalid UUID provided in 'net-uuid'");
 +        }
 +
-         qemu_thread_create(&mis->colo_incoming_thread, "COLO incoming",
++        xpc_dictionary_set_uuid(
-              colo_process_incoming_thread, mis, QEMU_THREAD_JOINABLE);
++            if_desc,
-         mis->have_colo_incoming_thread = true;
++            vmnet_network_identifier_key,
-@@ -XXX,XX +XXX,XX @@ static void process_incoming_migration_co(void *opaque)
++            hs->network_uuid.data
++        );
          /* Wait checkpoint incoming thread exit before free resource */
          qemu_thread_join(&mis->colo_incoming_thread);
 +        /* We hold the global iothread lock, so it is safe here */
 +        colo_release_ram_cache();
      }
      if (ret < 0) {
 diff --git a/migration/ram.c b/migration/ram.c
 index XXXXXXX..XXXXXXX 100644
 --- a/migration/ram.c
 +++ b/migration/ram.c
@@ -XXX,XX +XXX,XX @@ static inline void *host_from_ram_block_offset(RAMBlock *block,
      return block->host + offset;
  }
 +static inline void *colo_cache_from_block_offset(RAMBlock *block,
 +                                                 ram_addr_t offset)
 +{
 +    if (!offset_in_ramblock(block, offset)) {
 +        return NULL;
 +    }
-+    if (!block->colo_cache) {
++#else
-+        error_report("%s: colo_cache is NULL in block :%s",
++    if (options->has_isolated) {
-+                     __func__, block->idstr);
++        error_setg(errp,
-+        return NULL;
++                   "vmnet-host.isolated feature is "
 +                   "unavailable: outdated vmnet.framework API");
 +    }
-+    return block->colo_cache + offset;
-+}
 +
- /**
++    if (options->has_net_uuid) {
-  * ram_handle_compressed: handle the zero page case
++        error_setg(errp,
-  *
++                   "vmnet-host.net-uuid feature is "
-@@ -XXX,XX +XXX,XX @@ static void decompress_data_with_multi_threads(QEMUFile *f,
++                   "unavailable: outdated vmnet.framework API");
-     qemu_mutex_unlock(&decomp_done_lock);
++    }
- }
++#endif
 +/*
 + * colo cache: this is for secondary VM, we cache the whole
 + * memory of the secondary VM, it is need to hold the global lock
 + * to call this helper.
 + */
 +int colo_init_ram_cache(void)
 +{
 +    RAMBlock *block;
 +
-+    rcu_read_lock();
++    if (options->has_start_address ||
-+    QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
++        options->has_end_address ||
-+        block->colo_cache = qemu_anon_ram_alloc(block->used_length,
++        options->has_subnet_mask) {
 +                                                NULL,
 +                                                false);
 +        if (!block->colo_cache) {
 +            error_report("%s: Can't alloc memory for COLO cache of block %s,"
 +                         "size 0x" RAM_ADDR_FMT, __func__, block->idstr,
 +                         block->used_length);
 +            goto out_locked;
 +        }
 +        memcpy(block->colo_cache, block->host, block->used_length);
 +    }
 +    rcu_read_unlock();
 +    return 0;
 +
-+out_locked:
++        if (options->has_start_address &&
-+    QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
++            options->has_end_address &&
-+        if (block->colo_cache) {
++            options->has_subnet_mask) {
-+            qemu_anon_ram_free(block->colo_cache, block->used_length);
++
-+            block->colo_cache = NULL;
++            xpc_dictionary_set_string(if_desc,
 +                                      vmnet_start_address_key,
 +                                      options->start_address);
 +            xpc_dictionary_set_string(if_desc,
 +                                      vmnet_end_address_key,
 +                                      options->end_address);
 +            xpc_dictionary_set_string(if_desc,
 +                                      vmnet_subnet_mask_key,
 +                                      options->subnet_mask);
 +        } else {
 +            error_setg(
 +                errp,
 +                "'start-address', 'end-address', 'subnet-mask' "
 +                "should be provided together"
 +            );
 +        }
 +    }
 +
-+    rcu_read_unlock();
++    return if_desc;
 +    return -errno;
 +}
 +
-+/* It is need to hold the global lock to call this helper */
++static NetClientInfo net_vmnet_host_info = {
-+void colo_release_ram_cache(void)
++    .type = NET_CLIENT_DRIVER_VMNET_HOST,
 +    .size = sizeof(VmnetHostState),
 +    .receive = vmnet_receive_common,
 +    .cleanup = vmnet_cleanup_common,
 +};
 +
  int net_init_vmnet_host(const Netdev *netdev, const char *name,
 -                        NetClientState *peer, Error **errp) {
 -  error_setg(errp, "vmnet-host is not implemented yet");
 -  return -1;
 +                        NetClientState *peer, Error **errp)
 +{
-+    RAMBlock *block;
++    NetClientState *nc;
 +    xpc_object_t if_desc;
 +
-+    rcu_read_lock();
++    nc = qemu_new_net_client(&net_vmnet_host_info,
-+    QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
++                             peer, "vmnet-host", name);
-+        if (block->colo_cache) {
++    if_desc = create_if_desc(netdev, nc, errp);
-+            qemu_anon_ram_free(block->colo_cache, block->used_length);
++    return vmnet_if_create(nc, if_desc, errp, NULL);
 +            block->colo_cache = NULL;
 +        }
 +    }
 +    rcu_read_unlock();
 +}
 +
  /**
   * ram_load_setup: Setup RAM for migration incoming side
   *
@@ -XXX,XX +XXX,XX @@ static int ram_load_setup(QEMUFile *f, void *opaque)
      xbzrle_load_setup();
      ramblock_recv_map_init();
 +
      return 0;
  }
-@@ -XXX,XX +XXX,XX @@ static int ram_load_cleanup(void *opaque)
-         g_free(rb->receivedmap);
-         rb->receivedmap = NULL;
-     }
-+
-     return 0;
- }
-@@ -XXX,XX +XXX,XX @@ static int ram_load(QEMUFile *f, void *opaque, int version_id)
-                      RAM_SAVE_FLAG_COMPRESS_PAGE | RAM_SAVE_FLAG_XBZRLE)) {
-             RAMBlock *block = ram_block_from_stream(f, flags);
--            host = host_from_ram_block_offset(block, addr);
-+            /*
-+             * After going into COLO, we should load the Page into colo_cache.
-+             */
-+            if (migration_incoming_in_colo_state()) {
-+                host = colo_cache_from_block_offset(block, addr);
-+            } else {
-+                host = host_from_ram_block_offset(block, addr);
-+            }
-             if (!host) {
-                 error_report("Illegal RAM offset " RAM_ADDR_FMT, addr);
-                 ret = -EINVAL;
-                 break;
-             }
--            ramblock_recv_bitmap_set(block, host);
-+
-+            if (!migration_incoming_in_colo_state()) {
-+                ramblock_recv_bitmap_set(block, host);
-+            }
-+
-             trace_ram_load_loop(block->idstr, (uint64_t)addr, flags, host);
-         }
-diff --git a/migration/ram.h b/migration/ram.h
-index XXXXXXX..XXXXXXX 100644
---- a/migration/ram.h
-+++ b/migration/ram.h
-@@ -XXX,XX +XXX,XX @@ int64_t ramblock_recv_bitmap_send(QEMUFile *file,
-                                   const char *block_name);
- int ram_dirty_bitmap_reload(MigrationState *s, RAMBlock *rb);
-+/* ram cache */
-+int colo_init_ram_cache(void);
-+void colo_release_ram_cache(void);
-+
- #endif
-diff --git a/migration/savevm.c b/migration/savevm.c
-index XXXXXXX..XXXXXXX 100644
---- a/migration/savevm.c
-+++ b/migration/savevm.c
-@@ -XXX,XX +XXX,XX @@ static int loadvm_handle_recv_bitmap(MigrationIncomingState *mis,
- static int loadvm_process_enable_colo(MigrationIncomingState *mis)
- {
-     migration_incoming_enable_colo();
--    return 0;
-+    return colo_init_ram_cache();
- }
- /*
 --
-.5.0
+.7.4

-[Qemu-devel] [PULL 06/26] COLO: Remove colo_state migration struct
+[PULL V3 11/13] net/vmnet: implement bridged mode (vmnet-bridged)
-From: Zhang Chen <zhangckid@gmail.com>
+From: Vladislav Yaroshchuk <yaroshchuk2000@gmail.com>
-We need to know if migration is going into COLO state for
+Signed-off-by: Vladislav Yaroshchuk <yaroshchuk2000@gmail.com>
 incoming side before start normal migration.
 Instead by using the VMStateDescription to send colo_state
 from source side to destination side, we use MIG_CMD_ENABLE_COLO
 to indicate whether COLO is enabled or not.
 Signed-off-by: zhanghailiang <zhang.zhanghailiang@huawei.com>
 Signed-off-by: Zhang Chen <zhangckid@gmail.com>
 Signed-off-by: Zhang Chen <chen.zhang@intel.com>
 Reviewed-by: Dr. David Alan Gilbert <dgilbert@redhat.com>
 Signed-off-by: Jason Wang <jasowang@redhat.com>
 ---
- include/migration/colo.h |  5 ++--
+ net/vmnet-bridged.m | 105 +++++++++++++++++++++++++++++++++++++++++++++++++---
- migration/Makefile.objs  |  2 +-
+file changed, 100 insertions(+), 5 deletions(-)
  migration/colo-comm.c    | 76 ------------------------------------------------
  migration/colo.c         | 13 ++++++++-
  migration/migration.c    | 23 ++++++++++++++-
  migration/savevm.c       | 17 +++++++++++
  migration/savevm.h       |  1 +
  migration/trace-events   |  1 +
  vl.c                     |  2 --
 files changed, 57 insertions(+), 83 deletions(-)
  delete mode 100644 migration/colo-comm.c
-diff --git a/include/migration/colo.h b/include/migration/colo.h
+diff --git a/net/vmnet-bridged.m b/net/vmnet-bridged.m
 index XXXXXXX..XXXXXXX 100644
---- a/include/migration/colo.h
+--- a/net/vmnet-bridged.m
-+++ b/include/migration/colo.h
++++ b/net/vmnet-bridged.m
@@ -XXX,XX +XXX,XX @@ void migrate_start_colo_process(MigrationState *s);
  bool migration_in_colo_state(void);
  /* loadvm */
 -bool migration_incoming_enable_colo(void);
 -void migration_incoming_exit_colo(void);
 +void migration_incoming_enable_colo(void);
 +void migration_incoming_disable_colo(void);
 +bool migration_incoming_colo_enabled(void);
  void *colo_process_incoming_thread(void *opaque);
  bool migration_incoming_in_colo_state(void);
 diff --git a/migration/Makefile.objs b/migration/Makefile.objs
 index XXXXXXX..XXXXXXX 100644
 --- a/migration/Makefile.objs
 +++ b/migration/Makefile.objs
 @@ -XXX,XX +XXX,XX @@
- common-obj-y += migration.o socket.o fd.o exec.o
- common-obj-y += tls.o channel.o savevm.o
+ #include "qemu/osdep.h"
--common-obj-y += colo-comm.o colo.o colo-failover.o
+ #include "qapi/qapi-types-net.h"
-+common-obj-y += colo.o colo-failover.o
+-#include "vmnet_int.h"
- common-obj-y += vmstate.o vmstate-types.o page_cache.o
+-#include "clients.h"
- common-obj-y += qemu-file.o global_state.o
+-#include "qemu/error-report.h"
- common-obj-y += qemu-file-channel.o
+ #include "qapi/error.h"
-diff --git a/migration/colo-comm.c b/migration/colo-comm.c
++#include "clients.h"
-deleted file mode 100644
++#include "vmnet_int.h"
-index XXXXXXX..XXXXXXX
---- a/migration/colo-comm.c
+ #include <vmnet/vmnet.h>
-+++ /dev/null
-@@ -XXX,XX +XXX,XX @@
++typedef struct VmnetBridgedState {
--/*
++  VmnetCommonState cs;
-- * COarse-grain LOck-stepping Virtual Machines for Non-stop Service (COLO)
++} VmnetBridgedState;
-- * (a.k.a. Fault Tolerance or Continuous Replication)
++
-- *
++static bool validate_ifname(const char *ifname)
 - * Copyright (c) 2016 HUAWEI TECHNOLOGIES CO., LTD.
 - * Copyright (c) 2016 FUJITSU LIMITED
 - * Copyright (c) 2016 Intel Corporation
 - *
 - * This work is licensed under the terms of the GNU GPL, version 2 or
 - * later. See the COPYING file in the top-level directory.
 - *
 - */
 -
 -#include "qemu/osdep.h"
 -#include "migration.h"
 -#include "migration/colo.h"
 -#include "migration/vmstate.h"
 -#include "trace.h"
 -
 -typedef struct {
 -     bool colo_requested;
 -} COLOInfo;
 -
 -static COLOInfo colo_info;
 -
 -COLOMode get_colo_mode(void)
 -{
 -    if (migration_in_colo_state()) {
 -        return COLO_MODE_PRIMARY;
 -    } else if (migration_incoming_in_colo_state()) {
 -        return COLO_MODE_SECONDARY;
 -    } else {
 -        return COLO_MODE_UNKNOWN;
 -    }
 -}
 -
 -static int colo_info_pre_save(void *opaque)
 -{
 -    COLOInfo *s = opaque;
 -
 -    s->colo_requested = migrate_colo_enabled();
 -
 -    return 0;
 -}
 -
 -static bool colo_info_need(void *opaque)
 -{
 -   return migrate_colo_enabled();
 -}
 -
 -static const VMStateDescription colo_state = {
 -    .name = "COLOState",
 -    .version_id = 1,
 -    .minimum_version_id = 1,
 -    .pre_save = colo_info_pre_save,
 -    .needed = colo_info_need,
 -    .fields = (VMStateField[]) {
 -        VMSTATE_BOOL(colo_requested, COLOInfo),
 -        VMSTATE_END_OF_LIST()
 -    },
 -};
 -
 -void colo_info_init(void)
 -{
 -    vmstate_register(NULL, 0, &colo_state, &colo_info);
 -}
 -
 -bool migration_incoming_enable_colo(void)
 -{
 -    return colo_info.colo_requested;
 -}
 -
 -void migration_incoming_exit_colo(void)
 -{
 -    colo_info.colo_requested = false;
 -}
 diff --git a/migration/colo.c b/migration/colo.c
 index XXXXXXX..XXXXXXX 100644
 --- a/migration/colo.c
 +++ b/migration/colo.c
@@ -XXX,XX +XXX,XX @@ static void primary_vm_do_failover(void)
      qemu_sem_post(&s->colo_exit_sem);
  }
 +COLOMode get_colo_mode(void)
 +{
-+    if (migration_in_colo_state()) {
++    xpc_object_t shared_if_list = vmnet_copy_shared_interface_list();
-+        return COLO_MODE_PRIMARY;
++    __block bool match = false;
-+    } else if (migration_incoming_in_colo_state()) {
++
-+        return COLO_MODE_SECONDARY;
++    xpc_array_apply(
-+    } else {
++        shared_if_list,
-+        return COLO_MODE_UNKNOWN;
++        ^bool(size_t index, xpc_object_t value) {
-+    }
++          if (strcmp(xpc_string_get_string_ptr(value), ifname) == 0) {
 +              match = true;
 +              return false;
 +          }
 +          return true;
 +        });
 +
 +    return match;
 +}
 +
- void colo_do_failover(MigrationState *s)
++static const char *get_valid_ifnames(void)
  {
      /* Make sure VM stopped while failover happened. */
@@ -XXX,XX +XXX,XX @@ out:
      if (mis->to_src_file) {
          qemu_fclose(mis->to_src_file);
      }
 -    migration_incoming_exit_colo();
 +    migration_incoming_disable_colo();
      rcu_unregister_thread();
      return NULL;
 diff --git a/migration/migration.c b/migration/migration.c
 index XXXXXXX..XXXXXXX 100644
 --- a/migration/migration.c
 +++ b/migration/migration.c
@@ -XXX,XX +XXX,XX @@ int migrate_send_rp_req_pages(MigrationIncomingState *mis, const char *rbname,
      return migrate_send_rp_message(mis, msg_type, msglen, bufc);
  }
 +static bool migration_colo_enabled;
 +bool migration_incoming_colo_enabled(void)
 +{
-+    return migration_colo_enabled;
++    xpc_object_t shared_if_list = vmnet_copy_shared_interface_list();
 +    __block char *if_list = NULL;
 +
 +    xpc_array_apply(
 +        shared_if_list,
 +        ^bool(size_t index, xpc_object_t value) {
 +          if_list = g_strconcat(xpc_string_get_string_ptr(value),
 +                                " ",
 +                                if_list,
 +                                NULL);
 +          return true;
 +        });
 +
 +    if (if_list) {
 +        return if_list;
 +    }
 +    return "[no interfaces]";
 +}
 +
-+void migration_incoming_disable_colo(void)
++static xpc_object_t create_if_desc(const Netdev *netdev, Error **errp)
 +{
-+    migration_colo_enabled = false;
++    const NetdevVmnetBridgedOptions *options = &(netdev->u.vmnet_bridged);
 +    xpc_object_t if_desc = xpc_dictionary_create(NULL, NULL, 0);
 +
 +    xpc_dictionary_set_uint64(
 +        if_desc,
 +        vmnet_operation_mode_key,
 +        VMNET_BRIDGED_MODE
 +    );
 +
 +#if defined(MAC_OS_VERSION_11_0) && \
 +    MAC_OS_X_VERSION_MIN_REQUIRED >= MAC_OS_VERSION_11_0
 +    xpc_dictionary_set_bool(
 +        if_desc,
 +        vmnet_enable_isolation_key,
 +        options->isolated
 +    );
 +#else
 +    if (options->has_isolated) {
 +        error_setg(errp,
 +                   "vmnet-bridged.isolated feature is "
 +                   "unavailable: outdated vmnet.framework API");
 +    }
 +#endif
 +
 +    if (validate_ifname(options->ifname)) {
 +        xpc_dictionary_set_string(if_desc,
 +                                  vmnet_shared_interface_name_key,
 +                                  options->ifname);
 +    } else {
 +        return NULL;
 +    }
 +    return if_desc;
 +}
 +
-+void migration_incoming_enable_colo(void)
++static NetClientInfo net_vmnet_bridged_info = {
-+{
++    .type = NET_CLIENT_DRIVER_VMNET_BRIDGED,
-+    migration_colo_enabled = true;
++    .size = sizeof(VmnetBridgedState),
-+}
++    .receive = vmnet_receive_common,
 +    .cleanup = vmnet_cleanup_common,
 +};
 +
- void qemu_start_incoming_migration(const char *uri, Error **errp)
+ int net_init_vmnet_bridged(const Netdev *netdev, const char *name,
                             NetClientState *peer, Error **errp)
  {
-     const char *p;
+-  error_setg(errp, "vmnet-bridged is not implemented yet");
-@@ -XXX,XX +XXX,XX @@ static void process_incoming_migration_co(void *opaque)
+-  return -1;
-     }
++    NetClientState *nc = qemu_new_net_client(&net_vmnet_bridged_info,
++                                             peer, "vmnet-bridged", name);
-     /* we get COLO info, and know if we are in COLO mode */
++    xpc_object_t if_desc = create_if_desc(netdev, errp);;
--    if (!ret && migration_incoming_enable_colo()) {
++
-+    if (!ret && migration_incoming_colo_enabled()) {
++    if (!if_desc) {
-         /* Make sure all file formats flush their mutable metadata */
++        error_setg(errp,
-         bdrv_invalidate_cache_all(&local_err);
++                   "unsupported ifname, should be one of: %s",
-         if (local_err) {
++                   get_valid_ifnames());
-@@ -XXX,XX +XXX,XX @@ static void *migration_thread(void *opaque)
++        return -1;
          qemu_savevm_send_postcopy_advise(s->to_dst_file);
      }
 +    if (migrate_colo_enabled()) {
 +        /* Notify migration destination that we enable COLO */
 +        qemu_savevm_send_colo_enable(s->to_dst_file);
 +    }
 +
-     qemu_savevm_state_setup(s->to_dst_file);
++    return vmnet_if_create(nc, if_desc, errp, NULL);
      s->setup_time = qemu_clock_get_ms(QEMU_CLOCK_HOST) - setup_start;
 diff --git a/migration/savevm.c b/migration/savevm.c
 index XXXXXXX..XXXXXXX 100644
 --- a/migration/savevm.c
 +++ b/migration/savevm.c
@@ -XXX,XX +XXX,XX @@
  #include "io/channel-file.h"
  #include "sysemu/replay.h"
  #include "qjson.h"
 +#include "migration/colo.h"
  #ifndef ETH_P_RARP
  #define ETH_P_RARP 0x8035
@@ -XXX,XX +XXX,XX @@ enum qemu_vm_cmd {
                                        were previously sent during
                                        precopy but are dirty. */
      MIG_CMD_PACKAGED,          /* Send a wrapped stream within this stream */
 +    MIG_CMD_ENABLE_COLO,       /* Enable COLO */
      MIG_CMD_POSTCOPY_RESUME,   /* resume postcopy on dest */
      MIG_CMD_RECV_BITMAP,       /* Request for recved bitmap on dst */
      MIG_CMD_MAX
@@ -XXX,XX +XXX,XX @@ static void qemu_savevm_command_send(QEMUFile *f,
      qemu_fflush(f);
  }
-+void qemu_savevm_send_colo_enable(QEMUFile *f)
-+{
-+    trace_savevm_send_colo_enable();
-+    qemu_savevm_command_send(f, MIG_CMD_ENABLE_COLO, 0, NULL);
-+}
-+
- void qemu_savevm_send_ping(QEMUFile *f, uint32_t value)
- {
-     uint32_t buf;
-@@ -XXX,XX +XXX,XX @@ static int loadvm_handle_recv_bitmap(MigrationIncomingState *mis,
-     return 0;
- }
-+static int loadvm_process_enable_colo(MigrationIncomingState *mis)
-+{
-+    migration_incoming_enable_colo();
-+    return 0;
-+}
-+
- /*
-  * Process an incoming 'QEMU_VM_COMMAND'
-  * 0           just a normal return
-@@ -XXX,XX +XXX,XX @@ static int loadvm_process_command(QEMUFile *f)
-     case MIG_CMD_RECV_BITMAP:
-         return loadvm_handle_recv_bitmap(mis, len);
-+
-+    case MIG_CMD_ENABLE_COLO:
-+        return loadvm_process_enable_colo(mis);
-     }
-     return 0;
-diff --git a/migration/savevm.h b/migration/savevm.h
-index XXXXXXX..XXXXXXX 100644
---- a/migration/savevm.h
-+++ b/migration/savevm.h
-@@ -XXX,XX +XXX,XX @@ void qemu_savevm_send_postcopy_ram_discard(QEMUFile *f, const char *name,
-                                            uint16_t len,
-                                            uint64_t *start_list,
-                                            uint64_t *length_list);
-+void qemu_savevm_send_colo_enable(QEMUFile *f);
- int qemu_loadvm_state(QEMUFile *f);
- void qemu_loadvm_state_cleanup(void);
-diff --git a/migration/trace-events b/migration/trace-events
-index XXXXXXX..XXXXXXX 100644
---- a/migration/trace-events
-+++ b/migration/trace-events
-@@ -XXX,XX +XXX,XX @@ savevm_send_ping(uint32_t val) "0x%x"
- savevm_send_postcopy_listen(void) ""
- savevm_send_postcopy_run(void) ""
- savevm_send_postcopy_resume(void) ""
-+savevm_send_colo_enable(void) ""
- savevm_send_recv_bitmap(char *name) "%s"
- savevm_state_setup(void) ""
- savevm_state_resume_prepare(void) ""
-diff --git a/vl.c b/vl.c
-index XXXXXXX..XXXXXXX 100644
---- a/vl.c
-+++ b/vl.c
-@@ -XXX,XX +XXX,XX @@ int main(int argc, char **argv, char **envp)
- #endif
-     }
--    colo_info_init();
--
-     if (net_init_clients(&err) < 0) {
-         error_report_err(err);
-         exit(1);
 --
-.5.0
+.7.4

-[Qemu-devel] [PULL 08/26] ram/COLO: Record the dirty pages that SVM received
+Deleted patch
-From: Zhang Chen <zhangckid@gmail.com>
-We record the address of the dirty pages that received,
-it will help flushing pages that cached into SVM.
-Here, it is a trick, we record dirty pages by re-using migration
-dirty bitmap. In the later patch, we will start the dirty log
-for SVM, just like migration, in this way, we can record both
-the dirty pages caused by PVM and SVM, we only flush those dirty
-pages from RAM cache while do checkpoint.
-Signed-off-by: zhanghailiang <zhang.zhanghailiang@huawei.com>
-Signed-off-by: Zhang Chen <zhangckid@gmail.com>
-Signed-off-by: Zhang Chen <chen.zhang@intel.com>
-Reviewed-by: Dr. David Alan Gilbert <dgilbert@redhat.com>
-Signed-off-by: Jason Wang <jasowang@redhat.com>
----
- migration/ram.c | 43 ++++++++++++++++++++++++++++++++++++++++---
-file changed, 40 insertions(+), 3 deletions(-)
-diff --git a/migration/ram.c b/migration/ram.c
-index XXXXXXX..XXXXXXX 100644
---- a/migration/ram.c
-+++ b/migration/ram.c
-@@ -XXX,XX +XXX,XX @@ static inline void *colo_cache_from_block_offset(RAMBlock *block,
-                      __func__, block->idstr);
-         return NULL;
-     }
-+
-+    /*
-+    * During colo checkpoint, we need bitmap of these migrated pages.
-+    * It help us to decide which pages in ram cache should be flushed
-+    * into VM's RAM later.
-+    */
-+    if (!test_and_set_bit(offset >> TARGET_PAGE_BITS, block->bmap)) {
-+        ram_state->migration_dirty_pages++;
-+    }
-     return block->colo_cache + offset;
- }
-@@ -XXX,XX +XXX,XX @@ int colo_init_ram_cache(void)
-     RAMBlock *block;
-     rcu_read_lock();
--    QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
-+    RAMBLOCK_FOREACH_MIGRATABLE(block) {
-         block->colo_cache = qemu_anon_ram_alloc(block->used_length,
-                                                 NULL,
-                                                 false);
-@@ -XXX,XX +XXX,XX @@ int colo_init_ram_cache(void)
-         memcpy(block->colo_cache, block->host, block->used_length);
-     }
-     rcu_read_unlock();
-+    /*
-+    * Record the dirty pages that sent by PVM, we use this dirty bitmap together
-+    * with to decide which page in cache should be flushed into SVM's RAM. Here
-+    * we use the same name 'ram_bitmap' as for migration.
-+    */
-+    if (ram_bytes_total()) {
-+        RAMBlock *block;
-+
-+        RAMBLOCK_FOREACH_MIGRATABLE(block) {
-+            unsigned long pages = block->max_length >> TARGET_PAGE_BITS;
-+
-+            block->bmap = bitmap_new(pages);
-+            bitmap_set(block->bmap, 0, pages);
-+        }
-+    }
-+    ram_state = g_new0(RAMState, 1);
-+    ram_state->migration_dirty_pages = 0;
-+
-     return 0;
- out_locked:
--    QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
-+
-+    RAMBLOCK_FOREACH_MIGRATABLE(block) {
-         if (block->colo_cache) {
-             qemu_anon_ram_free(block->colo_cache, block->used_length);
-             block->colo_cache = NULL;
-@@ -XXX,XX +XXX,XX @@ void colo_release_ram_cache(void)
- {
-     RAMBlock *block;
-+    RAMBLOCK_FOREACH_MIGRATABLE(block) {
-+        g_free(block->bmap);
-+        block->bmap = NULL;
-+    }
-+
-     rcu_read_lock();
--    QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
-+
-+    RAMBLOCK_FOREACH_MIGRATABLE(block) {
-         if (block->colo_cache) {
-             qemu_anon_ram_free(block->colo_cache, block->used_length);
-             block->colo_cache = NULL;
-         }
-     }
-+
-     rcu_read_unlock();
-+    g_free(ram_state);
-+    ram_state = NULL;
- }
- /**
---
-.5.0

-[Qemu-devel] [PULL 09/26] COLO: Flush memory data from ram cache
+Deleted patch
-From: Zhang Chen <zhangckid@gmail.com>
-During the time of VM's running, PVM may dirty some pages, we will transfer
-PVM's dirty pages to SVM and store them into SVM's RAM cache at next checkpoint
-time. So, the content of SVM's RAM cache will always be same with PVM's memory
-after checkpoint.
-Instead of flushing all content of PVM's RAM cache into SVM's MEMORY,
-we do this in a more efficient way:
-Only flush any page that dirtied by PVM since last checkpoint.
-In this way, we can ensure SVM's memory same with PVM's.
-Besides, we must ensure flush RAM cache before load device state.
-Signed-off-by: zhanghailiang <zhang.zhanghailiang@huawei.com>
-Signed-off-by: Li Zhijian <lizhijian@cn.fujitsu.com>
-Reviewed-by: Dr. David Alan Gilbert <dgilbert@redhat.com>
-Signed-off-by: Jason Wang <jasowang@redhat.com>
----
- migration/ram.c        | 37 +++++++++++++++++++++++++++++++++++++
- migration/trace-events |  2 ++
-files changed, 39 insertions(+)
-diff --git a/migration/ram.c b/migration/ram.c
-index XXXXXXX..XXXXXXX 100644
---- a/migration/ram.c
-+++ b/migration/ram.c
-@@ -XXX,XX +XXX,XX @@ static bool postcopy_is_running(void)
-     return ps >= POSTCOPY_INCOMING_LISTENING && ps < POSTCOPY_INCOMING_END;
- }
-+/*
-+ * Flush content of RAM cache into SVM's memory.
-+ * Only flush the pages that be dirtied by PVM or SVM or both.
-+ */
-+static void colo_flush_ram_cache(void)
-+{
-+    RAMBlock *block = NULL;
-+    void *dst_host;
-+    void *src_host;
-+    unsigned long offset = 0;
-+
-+    trace_colo_flush_ram_cache_begin(ram_state->migration_dirty_pages);
-+    rcu_read_lock();
-+    block = QLIST_FIRST_RCU(&ram_list.blocks);
-+
-+    while (block) {
-+        offset = migration_bitmap_find_dirty(ram_state, block, offset);
-+
-+        if (offset << TARGET_PAGE_BITS >= block->used_length) {
-+            offset = 0;
-+            block = QLIST_NEXT_RCU(block, next);
-+        } else {
-+            migration_bitmap_clear_dirty(ram_state, block, offset);
-+            dst_host = block->host + (offset << TARGET_PAGE_BITS);
-+            src_host = block->colo_cache + (offset << TARGET_PAGE_BITS);
-+            memcpy(dst_host, src_host, TARGET_PAGE_SIZE);
-+        }
-+    }
-+
-+    rcu_read_unlock();
-+    trace_colo_flush_ram_cache_end();
-+}
-+
- static int ram_load(QEMUFile *f, void *opaque, int version_id)
- {
-     int flags = 0, ret = 0, invalid_flags = 0;
-@@ -XXX,XX +XXX,XX @@ static int ram_load(QEMUFile *f, void *opaque, int version_id)
-     ret |= wait_for_decompress_done();
-     rcu_read_unlock();
-     trace_ram_load_complete(ret, seq_iter);
-+
-+    if (!ret  && migration_incoming_in_colo_state()) {
-+        colo_flush_ram_cache();
-+    }
-     return ret;
- }
-diff --git a/migration/trace-events b/migration/trace-events
-index XXXXXXX..XXXXXXX 100644
---- a/migration/trace-events
-+++ b/migration/trace-events
-@@ -XXX,XX +XXX,XX @@ ram_dirty_bitmap_sync_start(void) ""
- ram_dirty_bitmap_sync_wait(void) ""
- ram_dirty_bitmap_sync_complete(void) ""
- ram_state_resume_prepare(uint64_t v) "%" PRId64
-+colo_flush_ram_cache_begin(uint64_t dirty_pages) "dirty_pages %" PRIu64
-+colo_flush_ram_cache_end(void) ""
- # migration/migration.c
- await_return_path_close_on_source_close(void) ""
---
-.5.0

-[Qemu-devel] [PULL 10/26] qmp event: Add COLO_EXIT event to notify users while exited COLO
+Deleted patch
-From: zhanghailiang <zhang.zhanghailiang@huawei.com>
-If some errors happen during VM's COLO FT stage, it's important to
-notify the users of this event. Together with 'x-colo-lost-heartbeat',
-Users can intervene in COLO's failover work immediately.
-If users don't want to get involved in COLO's failover verdict,
-it is still necessary to notify users that we exited COLO mode.
-Signed-off-by: zhanghailiang <zhang.zhanghailiang@huawei.com>
-Signed-off-by: Li Zhijian <lizhijian@cn.fujitsu.com>
-Signed-off-by: Zhang Chen <zhangckid@gmail.com>
-Signed-off-by: Zhang Chen <chen.zhang@intel.com>
-Signed-off-by: Jason Wang <jasowang@redhat.com>
----
- migration/colo.c    | 31 +++++++++++++++++++++++++++++++
- qapi/migration.json | 38 ++++++++++++++++++++++++++++++++++++++
-files changed, 69 insertions(+)
-diff --git a/migration/colo.c b/migration/colo.c
-index XXXXXXX..XXXXXXX 100644
---- a/migration/colo.c
-+++ b/migration/colo.c
-@@ -XXX,XX +XXX,XX @@
- #include "net/colo-compare.h"
- #include "net/colo.h"
- #include "block/block.h"
-+#include "qapi/qapi-events-migration.h"
- static bool vmstate_loading;
- static Notifier packets_compare_notifier;
-@@ -XXX,XX +XXX,XX @@ out:
-         qemu_fclose(fb);
-     }
-+    /*
-+     * There are only two reasons we can get here, some error happened
-+     * or the user triggered failover.
-+     */
-+    switch (failover_get_state()) {
-+    case FAILOVER_STATUS_NONE:
-+        qapi_event_send_colo_exit(COLO_MODE_PRIMARY,
-+                                  COLO_EXIT_REASON_ERROR);
-+        break;
-+    case FAILOVER_STATUS_REQUIRE:
-+        qapi_event_send_colo_exit(COLO_MODE_PRIMARY,
-+                                  COLO_EXIT_REASON_REQUEST);
-+        break;
-+    default:
-+        abort();
-+    }
-+
-     /* Hope this not to be too long to wait here */
-     qemu_sem_wait(&s->colo_exit_sem);
-     qemu_sem_destroy(&s->colo_exit_sem);
-@@ -XXX,XX +XXX,XX @@ out:
-         error_report_err(local_err);
-     }
-+    switch (failover_get_state()) {
-+    case FAILOVER_STATUS_NONE:
-+        qapi_event_send_colo_exit(COLO_MODE_SECONDARY,
-+                                  COLO_EXIT_REASON_ERROR);
-+        break;
-+    case FAILOVER_STATUS_REQUIRE:
-+        qapi_event_send_colo_exit(COLO_MODE_SECONDARY,
-+                                  COLO_EXIT_REASON_REQUEST);
-+        break;
-+    default:
-+        abort();
-+    }
-+
-     if (fb) {
-         qemu_fclose(fb);
-     }
-diff --git a/qapi/migration.json b/qapi/migration.json
-index XXXXXXX..XXXXXXX 100644
---- a/qapi/migration.json
-+++ b/qapi/migration.json
-@@ -XXX,XX +XXX,XX @@
-   'data': [ 'none', 'require', 'active', 'completed', 'relaunch' ] }
- ##
-+# @COLO_EXIT:
-+#
-+# Emitted when VM finishes COLO mode due to some errors happening or
-+# at the request of users.
-+#
-+# @mode: report COLO mode when COLO exited.
-+#
-+# @reason: describes the reason for the COLO exit.
-+#
-+# Since: 3.1
-+#
-+# Example:
-+#
-+# <- { "timestamp": {"seconds": 2032141960, "microseconds": 417172},
-+#      "event": "COLO_EXIT", "data": {"mode": "primary", "reason": "request" } }
-+#
-+##
-+{ 'event': 'COLO_EXIT',
-+  'data': {'mode': 'COLOMode', 'reason': 'COLOExitReason' } }
-+
-+##
-+# @COLOExitReason:
-+#
-+# The reason for a COLO exit
-+#
-+# @none: no failover has ever happened. This can't occur in the
-+# COLO_EXIT event, only in the result of query-colo-status.
-+#
-+# @request: COLO exit is due to an external request
-+#
-+# @error: COLO exit is due to an internal error
-+#
-+# Since: 3.1
-+##
-+{ 'enum': 'COLOExitReason',
-+  'data': [ 'none', 'request', 'error' ] }
-+
-+##
- # @x-colo-lost-heartbeat:
- #
- # Tell qemu that heartbeat is lost, request it to do takeover procedures.
---
-.5.0

-[Qemu-devel] [PULL 11/26] qapi/migration.json: Rename COLO unknown mode to none mode.
+Deleted patch
-From: Zhang Chen <chen.zhang@intel.com>
-Suggested by Markus Armbruster rename COLO unknown mode to none mode.
-Signed-off-by: Zhang Chen <zhangckid@gmail.com>
-Signed-off-by: Zhang Chen <chen.zhang@intel.com>
-Reviewed-by: Eric Blake <eblake@redhat.com>
-Reviewed-by: Markus Armbruster <armbru@redhat.com>
-Signed-off-by: Jason Wang <jasowang@redhat.com>
----
- migration/colo-failover.c |  2 +-
- migration/colo.c          |  2 +-
- qapi/migration.json       | 10 +++++-----
-files changed, 7 insertions(+), 7 deletions(-)
-diff --git a/migration/colo-failover.c b/migration/colo-failover.c
-index XXXXXXX..XXXXXXX 100644
---- a/migration/colo-failover.c
-+++ b/migration/colo-failover.c
-@@ -XXX,XX +XXX,XX @@ FailoverStatus failover_get_state(void)
- void qmp_x_colo_lost_heartbeat(Error **errp)
- {
--    if (get_colo_mode() == COLO_MODE_UNKNOWN) {
-+    if (get_colo_mode() == COLO_MODE_NONE) {
-         error_setg(errp, QERR_FEATURE_DISABLED, "colo");
-         return;
-     }
-diff --git a/migration/colo.c b/migration/colo.c
-index XXXXXXX..XXXXXXX 100644
---- a/migration/colo.c
-+++ b/migration/colo.c
-@@ -XXX,XX +XXX,XX @@ COLOMode get_colo_mode(void)
-     } else if (migration_incoming_in_colo_state()) {
-         return COLO_MODE_SECONDARY;
-     } else {
--        return COLO_MODE_UNKNOWN;
-+        return COLO_MODE_NONE;
-     }
- }
-diff --git a/qapi/migration.json b/qapi/migration.json
-index XXXXXXX..XXXXXXX 100644
---- a/qapi/migration.json
-+++ b/qapi/migration.json
-@@ -XXX,XX +XXX,XX @@
- ##
- # @COLOMode:
- #
--# The colo mode
-+# The COLO current mode.
- #
--# @unknown: unknown mode
-+# @none: COLO is disabled.
- #
--# @primary: master side
-+# @primary: COLO node in primary side.
- #
--# @secondary: slave side
-+# @secondary: COLO node in slave side.
- #
- # Since: 2.8
- ##
- { 'enum': 'COLOMode',
--  'data': [ 'unknown', 'primary', 'secondary'] }
-+  'data': [ 'none', 'primary', 'secondary'] }
- ##
- # @FailoverStatus:
---
-.5.0

-[Qemu-devel] [PULL 12/26] qapi: Add new command to query colo status
+Deleted patch
-From: Zhang Chen <zhangckid@gmail.com>
-Libvirt or other high level software can use this command query colo status.
-You can test this command like that:
-{'execute':'query-colo-status'}
-Signed-off-by: Zhang Chen <zhangckid@gmail.com>
-Signed-off-by: Zhang Chen <chen.zhang@intel.com>
-Signed-off-by: Jason Wang <jasowang@redhat.com>
----
- migration/colo.c    | 21 +++++++++++++++++++++
- qapi/migration.json | 32 ++++++++++++++++++++++++++++++++
-files changed, 53 insertions(+)
-diff --git a/migration/colo.c b/migration/colo.c
-index XXXXXXX..XXXXXXX 100644
---- a/migration/colo.c
-+++ b/migration/colo.c
-@@ -XXX,XX +XXX,XX @@
- #include "net/colo.h"
- #include "block/block.h"
- #include "qapi/qapi-events-migration.h"
-+#include "qapi/qmp/qerror.h"
- static bool vmstate_loading;
- static Notifier packets_compare_notifier;
-@@ -XXX,XX +XXX,XX @@ void qmp_xen_colo_do_checkpoint(Error **errp)
- #endif
- }
-+COLOStatus *qmp_query_colo_status(Error **errp)
-+{
-+    COLOStatus *s = g_new0(COLOStatus, 1);
-+
-+    s->mode = get_colo_mode();
-+
-+    switch (failover_get_state()) {
-+    case FAILOVER_STATUS_NONE:
-+        s->reason = COLO_EXIT_REASON_NONE;
-+        break;
-+    case FAILOVER_STATUS_REQUIRE:
-+        s->reason = COLO_EXIT_REASON_REQUEST;
-+        break;
-+    default:
-+        s->reason = COLO_EXIT_REASON_ERROR;
-+    }
-+
-+    return s;
-+}
-+
- static void colo_send_message(QEMUFile *f, COLOMessage msg,
-                               Error **errp)
- {
-diff --git a/qapi/migration.json b/qapi/migration.json
-index XXXXXXX..XXXXXXX 100644
---- a/qapi/migration.json
-+++ b/qapi/migration.json
-@@ -XXX,XX +XXX,XX @@
- { 'command': 'xen-colo-do-checkpoint' }
- ##
-+# @COLOStatus:
-+#
-+# The result format for 'query-colo-status'.
-+#
-+# @mode: COLO running mode. If COLO is running, this field will return
-+#        'primary' or 'secondary'.
-+#
-+# @reason: describes the reason for the COLO exit.
-+#
-+# Since: 3.0
-+##
-+{ 'struct': 'COLOStatus',
-+  'data': { 'mode': 'COLOMode', 'reason': 'COLOExitReason' } }
-+
-+##
-+# @query-colo-status:
-+#
-+# Query COLO status while the vm is running.
-+#
-+# Returns: A @COLOStatus object showing the status.
-+#
-+# Example:
-+#
-+# -> { "execute": "query-colo-status" }
-+# <- { "return": { "mode": "primary", "active": true, "reason": "request" } }
-+#
-+# Since: 3.0
-+##
-+{ 'command': 'query-colo-status',
-+  'returns': 'COLOStatus' }
-+
-+##
- # @migrate-recover:
- #
- # Provide a recovery migration stream URI.
---
-.5.0

-[Qemu-devel] [PULL 13/26] savevm: split the process of different stages for loadvm/savevm
+Deleted patch
-From: Zhang Chen <zhangckid@gmail.com>
-There are several stages during loadvm/savevm process. In different stage,
-migration incoming processes different types of sections.
-We want to control these stages more accuracy, it will benefit COLO
-performance, we don't have to save type of QEMU_VM_SECTION_START
-sections everytime while do checkpoint, besides, we want to separate
-the process of saving/loading memory and devices state.
-So we add three new helper functions: qemu_load_device_state() and
-qemu_savevm_live_state() to achieve different process during migration.
-Besides, we make qemu_loadvm_state_main() and qemu_save_device_state()
-public, and simplify the codes of qemu_save_device_state() by calling the
-wrapper qemu_savevm_state_header().
-Signed-off-by: zhanghailiang <zhang.zhanghailiang@huawei.com>
-Signed-off-by: Li Zhijian <lizhijian@cn.fujitsu.com>
-Signed-off-by: Zhang Chen <zhangckid@gmail.com>
-Signed-off-by: Zhang Chen <chen.zhang@intel.com>
-Reviewed-by: Dr. David Alan Gilbert <dgilbert@redhat.com>
-Signed-off-by: Jason Wang <jasowang@redhat.com>
----
- migration/colo.c   | 41 ++++++++++++++++++++++++++++++++---------
- migration/savevm.c | 36 +++++++++++++++++++++++++++++-------
- migration/savevm.h |  4 ++++
-files changed, 65 insertions(+), 16 deletions(-)
-diff --git a/migration/colo.c b/migration/colo.c
-index XXXXXXX..XXXXXXX 100644
---- a/migration/colo.c
-+++ b/migration/colo.c
-@@ -XXX,XX +XXX,XX @@
- #include "block/block.h"
- #include "qapi/qapi-events-migration.h"
- #include "qapi/qmp/qerror.h"
-+#include "sysemu/cpus.h"
- static bool vmstate_loading;
- static Notifier packets_compare_notifier;
-@@ -XXX,XX +XXX,XX @@ static int colo_do_checkpoint_transaction(MigrationState *s,
-     /* Disable block migration */
-     migrate_set_block_enabled(false, &local_err);
--    qemu_savevm_state_header(fb);
--    qemu_savevm_state_setup(fb);
-     qemu_mutex_lock_iothread();
-     replication_do_checkpoint_all(&local_err);
-     if (local_err) {
-         qemu_mutex_unlock_iothread();
-         goto out;
-     }
--    qemu_savevm_state_complete_precopy(fb, false, false);
--    qemu_mutex_unlock_iothread();
--
--    qemu_fflush(fb);
-     colo_send_message(s->to_dst_file, COLO_MESSAGE_VMSTATE_SEND, &local_err);
-     if (local_err) {
-+        qemu_mutex_unlock_iothread();
-+        goto out;
-+    }
-+    /* Note: device state is saved into buffer */
-+    ret = qemu_save_device_state(fb);
-+
-+    qemu_mutex_unlock_iothread();
-+    if (ret < 0) {
-         goto out;
-     }
-     /*
-+     * Only save VM's live state, which not including device state.
-+     * TODO: We may need a timeout mechanism to prevent COLO process
-+     * to be blocked here.
-+     */
-+    qemu_savevm_live_state(s->to_dst_file);
-+
-+    qemu_fflush(fb);
-+
-+    /*
-      * We need the size of the VMstate data in Secondary side,
-      * With which we can decide how much data should be read.
-      */
-@@ -XXX,XX +XXX,XX @@ void *colo_process_incoming_thread(void *opaque)
-     uint64_t total_size;
-     uint64_t value;
-     Error *local_err = NULL;
-+    int ret;
-     rcu_register_thread();
-     qemu_sem_init(&mis->colo_incoming_sem, 0);
-@@ -XXX,XX +XXX,XX @@ void *colo_process_incoming_thread(void *opaque)
-             goto out;
-         }
-+        qemu_mutex_lock_iothread();
-+        cpu_synchronize_all_pre_loadvm();
-+        ret = qemu_loadvm_state_main(mis->from_src_file, mis);
-+        qemu_mutex_unlock_iothread();
-+
-+        if (ret < 0) {
-+            error_report("Load VM's live state (ram) error");
-+            goto out;
-+        }
-+
-         value = colo_receive_message_value(mis->from_src_file,
-                                  COLO_MESSAGE_VMSTATE_SIZE, &local_err);
-         if (local_err) {
-@@ -XXX,XX +XXX,XX @@ void *colo_process_incoming_thread(void *opaque)
-         }
-         qemu_mutex_lock_iothread();
--        qemu_system_reset(SHUTDOWN_CAUSE_NONE);
-         vmstate_loading = true;
--        if (qemu_loadvm_state(fb) < 0) {
--            error_report("COLO: loadvm failed");
-+        ret = qemu_load_device_state(fb);
-+        if (ret < 0) {
-+            error_report("COLO: load device state failed");
-             qemu_mutex_unlock_iothread();
-             goto out;
-         }
-diff --git a/migration/savevm.c b/migration/savevm.c
-index XXXXXXX..XXXXXXX 100644
---- a/migration/savevm.c
-+++ b/migration/savevm.c
-@@ -XXX,XX +XXX,XX @@ done:
-     return ret;
- }
--static int qemu_save_device_state(QEMUFile *f)
-+void qemu_savevm_live_state(QEMUFile *f)
- {
--    SaveStateEntry *se;
-+    /* save QEMU_VM_SECTION_END section */
-+    qemu_savevm_state_complete_precopy(f, true, false);
-+    qemu_put_byte(f, QEMU_VM_EOF);
-+}
--    qemu_put_be32(f, QEMU_VM_FILE_MAGIC);
--    qemu_put_be32(f, QEMU_VM_FILE_VERSION);
-+int qemu_save_device_state(QEMUFile *f)
-+{
-+    SaveStateEntry *se;
-+    if (!migration_in_colo_state()) {
-+        qemu_put_be32(f, QEMU_VM_FILE_MAGIC);
-+        qemu_put_be32(f, QEMU_VM_FILE_VERSION);
-+    }
-     cpu_synchronize_all_states();
-     QTAILQ_FOREACH(se, &savevm_state.handlers, entry) {
-@@ -XXX,XX +XXX,XX @@ enum LoadVMExitCodes {
-     LOADVM_QUIT     =  1,
- };
--static int qemu_loadvm_state_main(QEMUFile *f, MigrationIncomingState *mis);
--
- /* ------ incoming postcopy messages ------ */
- /* 'advise' arrives before any transfers just to tell us that a postcopy
-  * *might* happen - it might be skipped if precopy transferred everything
-@@ -XXX,XX +XXX,XX @@ static bool postcopy_pause_incoming(MigrationIncomingState *mis)
-     return true;
- }
--static int qemu_loadvm_state_main(QEMUFile *f, MigrationIncomingState *mis)
-+int qemu_loadvm_state_main(QEMUFile *f, MigrationIncomingState *mis)
- {
-     uint8_t section_type;
-     int ret = 0;
-@@ -XXX,XX +XXX,XX @@ int qemu_loadvm_state(QEMUFile *f)
-     return ret;
- }
-+int qemu_load_device_state(QEMUFile *f)
-+{
-+    MigrationIncomingState *mis = migration_incoming_get_current();
-+    int ret;
-+
-+    /* Load QEMU_VM_SECTION_FULL section */
-+    ret = qemu_loadvm_state_main(f, mis);
-+    if (ret < 0) {
-+        error_report("Failed to load device state: %d", ret);
-+        return ret;
-+    }
-+
-+    cpu_synchronize_all_post_init();
-+    return 0;
-+}
-+
- int save_snapshot(const char *name, Error **errp)
- {
-     BlockDriverState *bs, *bs1;
-diff --git a/migration/savevm.h b/migration/savevm.h
-index XXXXXXX..XXXXXXX 100644
---- a/migration/savevm.h
-+++ b/migration/savevm.h
-@@ -XXX,XX +XXX,XX @@ void qemu_savevm_send_postcopy_ram_discard(QEMUFile *f, const char *name,
-                                            uint64_t *start_list,
-                                            uint64_t *length_list);
- void qemu_savevm_send_colo_enable(QEMUFile *f);
-+void qemu_savevm_live_state(QEMUFile *f);
-+int qemu_save_device_state(QEMUFile *f);
- int qemu_loadvm_state(QEMUFile *f);
- void qemu_loadvm_state_cleanup(void);
-+int qemu_loadvm_state_main(QEMUFile *f, MigrationIncomingState *mis);
-+int qemu_load_device_state(QEMUFile *f);
- #endif
---
-.5.0

-[Qemu-devel] [PULL 14/26] COLO: flush host dirty ram from cache
+Deleted patch
-From: zhanghailiang <zhang.zhanghailiang@huawei.com>
-Don't need to flush all VM's ram from cache, only
-flush the dirty pages since last checkpoint
-Signed-off-by: Li Zhijian <lizhijian@cn.fujitsu.com>
-Signed-off-by: Zhang Chen <zhangckid@gmail.com>
-Signed-off-by: Zhang Chen <chen.zhang@intel.com>
-Signed-off-by: zhanghailiang <zhang.zhanghailiang@huawei.com>
-Reviewed-by: Dr. David Alan Gilbert <dgilbert@redhat.com>
-Signed-off-by: Jason Wang <jasowang@redhat.com>
----
- migration/ram.c | 9 +++++++++
-file changed, 9 insertions(+)
-diff --git a/migration/ram.c b/migration/ram.c
-index XXXXXXX..XXXXXXX 100644
---- a/migration/ram.c
-+++ b/migration/ram.c
-@@ -XXX,XX +XXX,XX @@ int colo_init_ram_cache(void)
-     }
-     ram_state = g_new0(RAMState, 1);
-     ram_state->migration_dirty_pages = 0;
-+    memory_global_dirty_log_start();
-     return 0;
-@@ -XXX,XX +XXX,XX @@ void colo_release_ram_cache(void)
- {
-     RAMBlock *block;
-+    memory_global_dirty_log_stop();
-     RAMBLOCK_FOREACH_MIGRATABLE(block) {
-         g_free(block->bmap);
-         block->bmap = NULL;
-@@ -XXX,XX +XXX,XX @@ static void colo_flush_ram_cache(void)
-     void *src_host;
-     unsigned long offset = 0;
-+    memory_global_dirty_log_sync();
-+    rcu_read_lock();
-+    RAMBLOCK_FOREACH_MIGRATABLE(block) {
-+        migration_bitmap_sync_range(ram_state, block, 0, block->used_length);
-+    }
-+    rcu_read_unlock();
-+
-     trace_colo_flush_ram_cache_begin(ram_state->migration_dirty_pages);
-     rcu_read_lock();
-     block = QLIST_FIRST_RCU(&ram_list.blocks);
---
-.5.0

-[Qemu-devel] [PULL 17/26] COLO: notify net filters about checkpoint/failover event
+Deleted patch
-From: zhanghailiang <zhang.zhanghailiang@huawei.com>
-Notify all net filters about the checkpoint and failover event.
-Signed-off-by: zhanghailiang <zhang.zhanghailiang@huawei.com>
-Reviewed-by: Dr. David Alan Gilbert <dgilbert@redhat.com>
-Signed-off-by: Jason Wang <jasowang@redhat.com>
----
- migration/colo.c | 15 +++++++++++++++
-file changed, 15 insertions(+)
-diff --git a/migration/colo.c b/migration/colo.c
-index XXXXXXX..XXXXXXX 100644
---- a/migration/colo.c
-+++ b/migration/colo.c
-@@ -XXX,XX +XXX,XX @@
- #include "qapi/qapi-events-migration.h"
- #include "qapi/qmp/qerror.h"
- #include "sysemu/cpus.h"
-+#include "net/filter.h"
- static bool vmstate_loading;
- static Notifier packets_compare_notifier;
-@@ -XXX,XX +XXX,XX @@ static void secondary_vm_do_failover(void)
-         error_report_err(local_err);
-     }
-+    /* Notify all filters of all NIC to do checkpoint */
-+    colo_notify_filters_event(COLO_EVENT_FAILOVER, &local_err);
-+    if (local_err) {
-+        error_report_err(local_err);
-+    }
-+
-     if (!autostart) {
-         error_report("\"-S\" qemu option will be ignored in secondary side");
-         /* recover runstate to normal migration finish state */
-@@ -XXX,XX +XXX,XX @@ void *colo_process_incoming_thread(void *opaque)
-             goto out;
-         }
-+        /* Notify all filters of all NIC to do checkpoint */
-+        colo_notify_filters_event(COLO_EVENT_CHECKPOINT, &local_err);
-+
-+        if (local_err) {
-+            qemu_mutex_unlock_iothread();
-+            goto out;
-+        }
-+
-         vmstate_loading = false;
-         vm_start();
-         trace_colo_vm_state_change("stop", "run");
---
-.5.0

-[Qemu-devel] [PULL 18/26] COLO: quick failover process by kick COLO thread
+Deleted patch
-From: zhanghailiang <zhang.zhanghailiang@huawei.com>
-COLO thread may sleep at qemu_sem_wait(&s->colo_checkpoint_sem),
-while failover works begin, It's better to wakeup it to quick
-the process.
-Signed-off-by: zhanghailiang <zhang.zhanghailiang@huawei.com>
-Reviewed-by: Dr. David Alan Gilbert <dgilbert@redhat.com>
-Signed-off-by: Jason Wang <jasowang@redhat.com>
----
- migration/colo.c | 8 ++++++++
-file changed, 8 insertions(+)
-diff --git a/migration/colo.c b/migration/colo.c
-index XXXXXXX..XXXXXXX 100644
---- a/migration/colo.c
-+++ b/migration/colo.c
-@@ -XXX,XX +XXX,XX @@ static void primary_vm_do_failover(void)
-     migrate_set_state(&s->state, MIGRATION_STATUS_COLO,
-                       MIGRATION_STATUS_COMPLETED);
-+    /*
-+     * kick COLO thread which might wait at
-+     * qemu_sem_wait(&s->colo_checkpoint_sem).
-+     */
-+    colo_checkpoint_notify(migrate_get_current());
-     /*
-      * Wake up COLO thread which may blocked in recv() or send(),
-@@ -XXX,XX +XXX,XX @@ static void colo_process_checkpoint(MigrationState *s)
-         qemu_sem_wait(&s->colo_checkpoint_sem);
-+        if (s->state != MIGRATION_STATUS_COLO) {
-+            goto out;
-+        }
-         ret = colo_do_checkpoint_transaction(s, bioc, fb);
-         if (ret < 0) {
-             goto out;
---
-.5.0

-[Qemu-devel] [PULL 19/26] docs: Add COLO status diagram to COLO-FT.txt
+Deleted patch
-From: Zhang Chen <chen.zhang@intel.com>
-This diagram make user better understand COLO.
-Suggested by Markus Armbruster.
-Signed-off-by: Zhang Chen <zhangckid@gmail.com>
-Signed-off-by: Zhang Chen <chen.zhang@intel.com>
-Signed-off-by: Jason Wang <jasowang@redhat.com>
----
- docs/COLO-FT.txt | 34 ++++++++++++++++++++++++++++++++++
-file changed, 34 insertions(+)
-diff --git a/docs/COLO-FT.txt b/docs/COLO-FT.txt
-index XXXXXXX..XXXXXXX 100644
---- a/docs/COLO-FT.txt
-+++ b/docs/COLO-FT.txt
-@@ -XXX,XX +XXX,XX @@ Note:
- HeartBeat has not been implemented yet, so you need to trigger failover process
- by using 'x-colo-lost-heartbeat' command.
-+== COLO operation status ==
-+
-++-----------------+
-+|                 |
-+|    Start COLO   |
-+|                 |
-++--------+--------+
-+         |
-+         |  Main qmp command:
-+         |  migrate-set-capabilities with x-colo
-+         |  migrate
-+         |
-+         v
-++--------+--------+
-+|                 |
-+|  COLO running   |
-+|                 |
-++--------+--------+
-+         |
-+         |  Main qmp command:
-+         |  x-colo-lost-heartbeat
-+         |  or
-+         |  some error happened
-+         v
-++--------+--------+
-+|                 |  send qmp event:
-+|  COLO failover  |  COLO_EXIT
-+|                 |
-++-----------------+
-+
-+COLO use the qmp command to switch and report operation status.
-+The diagram just shows the main qmp command, you can get the detail
-+in test procedure.
-+
- == Test procedure ==
-. Startup qemu
- Primary:
---
-.5.0

-[Qemu-devel] [PULL 20/26] clean up callback when del virtqueue
+Deleted patch
-From: liujunjie <liujunjie23@huawei.com>
-Before, we did not clear callback like handle_output when delete
-the virtqueue which may result be segmentfault.
-The scene is as follows:
-. Start a vm with multiqueue vhost-net,
-. then we write VIRTIO_PCI_GUEST_FEATURES in PCI configuration to
-triger multiqueue disable in this vm which will delete the virtqueue.
-In this step, the tx_bh is deleted but the callback virtio_net_handle_tx_bh
-still exist.
-. Finally, we write VIRTIO_PCI_QUEUE_NOTIFY in PCI configuration to
-notify the deleted virtqueue. In this way, virtio_net_handle_tx_bh
-will be called and qemu will be crashed.
-Although the way described above is uncommon, we had better reinforce it.
-CC: qemu-stable@nongnu.org
-Signed-off-by: liujunjie <liujunjie23@huawei.com>
-Signed-off-by: Jason Wang <jasowang@redhat.com>
----
- hw/virtio/virtio.c | 2 ++
-file changed, 2 insertions(+)
-diff --git a/hw/virtio/virtio.c b/hw/virtio/virtio.c
-index XXXXXXX..XXXXXXX 100644
---- a/hw/virtio/virtio.c
-+++ b/hw/virtio/virtio.c
-@@ -XXX,XX +XXX,XX @@ void virtio_del_queue(VirtIODevice *vdev, int n)
-     vdev->vq[n].vring.num = 0;
-     vdev->vq[n].vring.num_default = 0;
-+    vdev->vq[n].handle_output = NULL;
-+    vdev->vq[n].handle_aio_output = NULL;
- }
- static void virtio_set_isr(VirtIODevice *vdev, int value)
---
-.5.0

-[Qemu-devel] [PULL 26/26] qemu-options: Fix bad "macaddr" property in the documentation
+[PULL V3 12/13] net/vmnet: update qemu-options.hx
-From: Thomas Huth <thuth@redhat.com>
+From: Vladislav Yaroshchuk <yaroshchuk2000@gmail.com>
-When using the "-device" option, the property is called "mac".
+Signed-off-by: Vladislav Yaroshchuk <yaroshchuk2000@gmail.com>
 "macaddr" is only used for the legacy "-net nic" option.
 Reported-by: Harald Hoyer <harald@redhat.com>
 Reviewed-by: Markus Armbruster <armbru@redhat.com>
 Signed-off-by: Thomas Huth <thuth@redhat.com>
 Signed-off-by: Jason Wang <jasowang@redhat.com>
 ---
- qemu-options.hx | 2 +-
+ qemu-options.hx | 25 +++++++++++++++++++++++++
-file changed, 1 insertion(+), 1 deletion(-)
+file changed, 25 insertions(+)
 diff --git a/qemu-options.hx b/qemu-options.hx
 index XXXXXXX..XXXXXXX 100644
 --- a/qemu-options.hx
 +++ b/qemu-options.hx
-@@ -XXX,XX +XXX,XX @@ qemu-system-i386 linux.img \
+@@ -XXX,XX +XXX,XX @@ DEF("netdev", HAS_ARG, QEMU_OPTION_netdev,
-                  -netdev socket,id=n2,mcast=230.0.0.1:1234
+     "-netdev vhost-vdpa,id=str,vhostdev=/path/to/dev\n"
- # launch yet another QEMU instance on same "bus"
+     "                configure a vhost-vdpa network,Establish a vhost-vdpa netdev\n"
- qemu-system-i386 linux.img \
+ #endif
--                 -device e1000,netdev=n3,macaddr=52:54:00:12:34:58 \
++#ifdef CONFIG_VMNET
-+                 -device e1000,netdev=n3,mac=52:54:00:12:34:58 \
++    "-netdev vmnet-host,id=str[,isolated=on|off][,net-uuid=uuid]\n"
-                  -netdev socket,id=n3,mcast=230.0.0.1:1234
++    "         [,start-address=addr,end-address=addr,subnet-mask=mask]\n"
- @end example
++    "                configure a vmnet network backend in host mode with ID 'str',\n"
++    "                isolate this interface from others with 'isolated',\n"
 +    "                configure the address range and choose a subnet mask,\n"
 +    "                specify network UUID 'uuid' to disable DHCP and interact with\n"
 +    "                vmnet-host interfaces within this isolated network\n"
 +    "-netdev vmnet-shared,id=str[,isolated=on|off][,nat66-prefix=addr]\n"
 +    "         [,start-address=addr,end-address=addr,subnet-mask=mask]\n"
 +    "                configure a vmnet network backend in shared mode with ID 'str',\n"
 +    "                configure the address range and choose a subnet mask,\n"
 +    "                set IPv6 ULA prefix (of length 64) to use for internal network,\n"
 +    "                isolate this interface from others with 'isolated'\n"
 +    "-netdev vmnet-bridged,id=str,ifname=name[,isolated=on|off]\n"
 +    "                configure a vmnet network backend in bridged mode with ID 'str',\n"
 +    "                use 'ifname=name' to select a physical network interface to be bridged,\n"
 +    "                isolate this interface from others with 'isolated'\n"
 +#endif
      "-netdev hubport,id=str,hubid=n[,netdev=nd]\n"
      "                configure a hub port on the hub with ID 'n'\n", QEMU_ARCH_ALL)
  DEF("nic", HAS_ARG, QEMU_OPTION_nic,
@@ -XXX,XX +XXX,XX @@ DEF("nic", HAS_ARG, QEMU_OPTION_nic,
  #ifdef CONFIG_POSIX
      "vhost-user|"
  #endif
 +#ifdef CONFIG_VMNET
 +    "vmnet-host|vmnet-shared|vmnet-bridged|"
 +#endif
      "socket][,option][,...][mac=macaddr]\n"
      "                initialize an on-board / default host NIC (using MAC address\n"
      "                macaddr) and connect it to the given host network backend\n"
@@ -XXX,XX +XXX,XX @@ DEF("net", HAS_ARG, QEMU_OPTION_net,
  #ifdef CONFIG_NETMAP
      "netmap|"
  #endif
 +#ifdef CONFIG_VMNET
 +    "vmnet-host|vmnet-shared|vmnet-bridged|"
 +#endif
      "socket][,option][,option][,...]\n"
      "                old way to initialize a host network interface\n"
      "                (use the -netdev option if possible instead)\n", QEMU_ARCH_ALL)
 --
-.5.0
+.7.4

-[Qemu-devel] [PULL 21/26] ne2000: fix possible out of bound access in ne2000_receive
+[PULL V3 13/13] net/vmnet: update MAINTAINERS list
-In ne2000_receive(), we try to assign size_ to size which converts
+From: Vladislav Yaroshchuk <yaroshchuk2000@gmail.com>
 from size_t to integer. This will cause troubles when size_ is greater
 INT_MAX, this will lead a negative value in size and it can then pass
 the check of size < MIN_BUF_SIZE which may lead out of bound access of
 for both buf and buf1.
-Fixing by converting the type of size to size_t.
+Signed-off-by: Vladislav Yaroshchuk <yaroshchuk2000@gmail.com>
 CC: qemu-stable@nongnu.org
 Reported-by: Daniel Shapira <daniel@twistlock.com>
 Reviewed-by: Michael S. Tsirkin <mst@redhat.com>
 Signed-off-by: Jason Wang <jasowang@redhat.com>
 ---
- hw/net/ne2000.c | 4 ++--
+ MAINTAINERS | 5 +++++
-file changed, 2 insertions(+), 2 deletions(-)
+file changed, 5 insertions(+)
-diff --git a/hw/net/ne2000.c b/hw/net/ne2000.c
+diff --git a/MAINTAINERS b/MAINTAINERS
 index XXXXXXX..XXXXXXX 100644
---- a/hw/net/ne2000.c
+--- a/MAINTAINERS
-+++ b/hw/net/ne2000.c
++++ b/MAINTAINERS
-@@ -XXX,XX +XXX,XX @@ static int ne2000_buffer_full(NE2000State *s)
+@@ -XXX,XX +XXX,XX @@ W: http://info.iet.unipi.it/~luigi/netmap/
- ssize_t ne2000_receive(NetClientState *nc, const uint8_t *buf, size_t size_)
+ S: Maintained
- {
+ F: net/netmap.c
-     NE2000State *s = qemu_get_nic_opaque(nc);
--    int size = size_;
++Apple vmnet network backends
-+    size_t size = size_;
++M: Vladislav Yaroshchuk <yaroshchuk2000@gmail.com>
-     uint8_t *p;
++S: Maintained
-     unsigned int total_len, next, avail, len, index, mcast_idx;
++F: net/vmnet*
-     uint8_t buf1[60];
++
-@@ -XXX,XX +XXX,XX @@ ssize_t ne2000_receive(NetClientState *nc, const uint8_t *buf, size_t size_)
+ Host Memory Backends
-         { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff };
+ M: David Hildenbrand <david@redhat.com>
+ M: Igor Mammedov <imammedo@redhat.com>
  #if defined(DEBUG_NE2000)
 -    printf("NE2000: received len=%d\n", size);
 +    printf("NE2000: received len=%zu\n", size);
  #endif
      if (s->cmd & E8390_STOP || ne2000_buffer_full(s))
 --
-.5.0
+.7.4

The following changes since commit a73549f99612f758dec0fdea6ae1c30b6c709a0b:

Merge remote-tracking branch 'remotes/kraxel/tags/ui-20181012-pull-request' into staging (2018-10-12 16:45:51 +0100)

are available in the git repository at:

https://github.com/jasowang/qemu.git tags/net-pull-request

for you to fetch changes up to a7ec0077c2db445d6bae421963188367d2695bd6:

qemu-options: Fix bad "macaddr" property in the documentation (2018-10-15 16:14:15 +0800)

----------------------------------------------------------------

----------------------------------------------------------------
Jason Wang (4):
      ne2000: fix possible out of bound access in ne2000_receive
      rtl8139: fix possible out of bound access
      pcnet: fix possible buffer overflow
      net: ignore packet size greater than INT_MAX

Martin Wilck (1):
      e1000: indicate dropped packets in HW counters

Thomas Huth (1):
      qemu-options: Fix bad "macaddr" property in the documentation

Zhang Chen (15):
      filter-rewriter: Add TCP state machine and fix memory leak in connection_track_table
      colo-compare: implement the process of checkpoint
      colo-compare: use notifier to notify packets comparing result
1;5202;0c1;5202;0c      COLO: integrate colo compare with colo frame
      COLO: Add block replication into colo process
      COLO: Remove colo_state migration struct
      COLO: Load dirty pages into SVM's RAM cache firstly
      ram/COLO: Record the dirty pages that SVM received
      COLO: Flush memory data from ram cache
      qapi/migration.json: Rename COLO unknown mode to none mode.
      qapi: Add new command to query colo status
      savevm: split the process of different stages for loadvm/savevm
      filter: Add handle_event method for NetFilterClass
      filter-rewriter: handle checkpoint and failover event
      docs: Add COLO status diagram to COLO-FT.txt

liujunjie (1):
      clean up callback when del virtqueue

zhanghailiang (4):
      qmp event: Add COLO_EXIT event to notify users while exited COLO
      COLO: flush host dirty ram from cache
      COLO: notify net filters about checkpoint/failover event
      COLO: quick failover process by kick COLO thread

From: Zhang Chen <zhangckid@gmail.com>

We add almost full TCP state machine in filter-rewriter, except
TCPS_LISTEN and some simplify in VM active close FIN states.
The reason for this simplify job is because guest kernel will track
the TCP status and wait 2MSL time too, if client resend the FIN packet,
guest will resend the last ACK, so we needn't wait 2MSL time in filter-rewriter.

After a net connection is closed, we didn't clear its related resources
in connection_track_table, which will lead to memory leak.

Let's track the state of net connection, if it is closed, its related
resources will be cleared up.

Signed-off-by: zhanghailiang <zhang.zhanghailiang@huawei.com>
Signed-off-by: Zhang Chen <zhangckid@gmail.com>
Signed-off-by: Zhang Chen <chen.zhang@intel.com>
Signed-off-by: Jason Wang <jasowang@redhat.com>
---
 net/colo.c            |   2 +-
 net/colo.h            |   9 ++---
 net/filter-rewriter.c | 109 +++++++++++++++++++++++++++++++++++++++++++++-----
 3 files changed, 104 insertions(+), 16 deletions(-)

diff --git a/net/colo.c b/net/colo.c
index XXXXXXX..XXXXXXX 100644
--- a/net/colo.c
+++ b/net/colo.c
@@ -XXX,XX +XXX,XX @@ Connection *connection_new(ConnectionKey *key)
     conn->ip_proto = key->ip_proto;
     conn->processing = false;
     conn->offset = 0;
-    conn->syn_flag = 0;
+    conn->tcp_state = TCPS_CLOSED;
     conn->pack = 0;
     conn->sack = 0;
     g_queue_init(&conn->primary_list);
diff --git a/net/colo.h b/net/colo.h
index XXXXXXX..XXXXXXX 100644
--- a/net/colo.h
+++ b/net/colo.h
@@ -XXX,XX +XXX,XX @@
 #include "slirp/slirp.h"
 #include "qemu/jhash.h"
 #include "qemu/timer.h"
+#include "slirp/tcp.h"
 
 #define HASHTABLE_MAX_SIZE 16384
 
@@ -XXX,XX +XXX,XX @@ typedef struct Connection {
     uint32_t sack;
     /* offset = secondary_seq - primary_seq */
     tcp_seq  offset;
-    /*
-     * we use this flag update offset func
-     * run once in independent tcp connection
-     */
-    int syn_flag;
+
+    int tcp_state; /* TCP FSM state */
+    tcp_seq fin_ack_seq; /* the seq of 'fin=1,ack=1' */
 } Connection;
 
 uint32_t connection_key_hash(const void *opaque);
diff --git a/net/filter-rewriter.c b/net/filter-rewriter.c
index XXXXXXX..XXXXXXX 100644
--- a/net/filter-rewriter.c
+++ b/net/filter-rewriter.c
@@ -XXX,XX +XXX,XX @@ static int is_tcp_packet(Packet *pkt)
 }
 
 /* handle tcp packet from primary guest */
-static int handle_primary_tcp_pkt(NetFilterState *nf,
+static int handle_primary_tcp_pkt(RewriterState *rf,
                                   Connection *conn,
-                                  Packet *pkt)
+                                  Packet *pkt, ConnectionKey *key)
 {
     struct tcphdr *tcp_pkt;
 
@@ -XXX,XX +XXX,XX @@ static int handle_primary_tcp_pkt(NetFilterState *nf,
         trace_colo_filter_rewriter_conn_offset(conn->offset);
     }
 
+    if (((tcp_pkt->th_flags & (TH_ACK | TH_SYN)) == (TH_ACK | TH_SYN)) &&
+        conn->tcp_state == TCPS_SYN_SENT) {
+        conn->tcp_state = TCPS_ESTABLISHED;
+    }
+
     if (((tcp_pkt->th_flags & (TH_ACK | TH_SYN)) == TH_SYN)) {
         /*
          * we use this flag update offset func
          * run once in independent tcp connection
          */
-        conn->syn_flag = 1;
+        conn->tcp_state = TCPS_SYN_RECEIVED;
     }
 
     if (((tcp_pkt->th_flags & (TH_ACK | TH_SYN)) == TH_ACK)) {
-        if (conn->syn_flag) {
+        if (conn->tcp_state == TCPS_SYN_RECEIVED) {
             /*
              * offset = secondary_seq - primary seq
              * ack packet sent by guest from primary node,
              * so we use th_ack - 1 get primary_seq
              */
             conn->offset -= (ntohl(tcp_pkt->th_ack) - 1);
-            conn->syn_flag = 0;
+            conn->tcp_state = TCPS_ESTABLISHED;
         }
         if (conn->offset) {
             /* handle packets to the secondary from the primary */
@@ -XXX,XX +XXX,XX @@ static int handle_primary_tcp_pkt(NetFilterState *nf,
             net_checksum_calculate((uint8_t *)pkt->data + pkt->vnet_hdr_len,
                                    pkt->size - pkt->vnet_hdr_len);
         }
+
+        /*
+         * Passive close step 3
+         */
+        if ((conn->tcp_state == TCPS_LAST_ACK) &&
+            (ntohl(tcp_pkt->th_ack) == (conn->fin_ack_seq + 1))) {
+            conn->tcp_state = TCPS_CLOSED;
+            g_hash_table_remove(rf->connection_track_table, key);
+        }
+    }
+
+    if ((tcp_pkt->th_flags & TH_FIN) == TH_FIN) {
+        /*
+         * Passive close.
+         * Step 1:
+         * The *server* side of this connect is VM, *client* tries to close
+         * the connection. We will into CLOSE_WAIT status.
+         *
+         * Step 2:
+         * In this step we will into LAST_ACK status.
+         *
+         * We got 'fin=1, ack=1' packet from server side, we need to
+         * record the seq of 'fin=1, ack=1' packet.
+         *
+         * Step 3:
+         * We got 'ack=1' packets from client side, it acks 'fin=1, ack=1'
+         * packet from server side. From this point, we can ensure that there
+         * will be no packets in the connection, except that, some errors
+         * happen between the path of 'filter object' and vNIC, if this rare
+         * case really happen, we can still create a new connection,
+         * So it is safe to remove the connection from connection_track_table.
+         *
+         */
+        if (conn->tcp_state == TCPS_ESTABLISHED) {
+            conn->tcp_state = TCPS_CLOSE_WAIT;
+        }
+
+        /*
+         * Active close step 2.
+         */
+        if (conn->tcp_state == TCPS_FIN_WAIT_1) {
+            conn->tcp_state = TCPS_TIME_WAIT;
+            /*
+             * For simplify implementation, we needn't wait 2MSL time
+             * in filter rewriter. Because guest kernel will track the
+             * TCP status and wait 2MSL time, if client resend the FIN
+             * packet, guest will apply the last ACK too.
+             */
+            conn->tcp_state = TCPS_CLOSED;
+            g_hash_table_remove(rf->connection_track_table, key);
+        }
     }
 
     return 0;
 }
 
 /* handle tcp packet from secondary guest */
-static int handle_secondary_tcp_pkt(NetFilterState *nf,
+static int handle_secondary_tcp_pkt(RewriterState *rf,
                                     Connection *conn,
-                                    Packet *pkt)
+                                    Packet *pkt, ConnectionKey *key)
 {
     struct tcphdr *tcp_pkt;
 
@@ -XXX,XX +XXX,XX @@ static int handle_secondary_tcp_pkt(NetFilterState *nf,
         trace_colo_filter_rewriter_conn_offset(conn->offset);
     }
 
-    if (((tcp_pkt->th_flags & (TH_ACK | TH_SYN)) == (TH_ACK | TH_SYN))) {
+    if (conn->tcp_state == TCPS_SYN_RECEIVED &&
+        ((tcp_pkt->th_flags & (TH_ACK | TH_SYN)) == (TH_ACK | TH_SYN))) {
         /*
          * save offset = secondary_seq and then
          * in handle_primary_tcp_pkt make offset
@@ -XXX,XX +XXX,XX @@ static int handle_secondary_tcp_pkt(NetFilterState *nf,
         conn->offset = ntohl(tcp_pkt->th_seq);
     }
 
+    /* VM active connect */
+    if (conn->tcp_state == TCPS_CLOSED &&
+        ((tcp_pkt->th_flags & (TH_ACK | TH_SYN)) == TH_SYN)) {
+        conn->tcp_state = TCPS_SYN_SENT;
+    }
+
     if ((tcp_pkt->th_flags & (TH_ACK | TH_SYN)) == TH_ACK) {
         /* Only need to adjust seq while offset is Non-zero */
         if (conn->offset) {
@@ -XXX,XX +XXX,XX @@ static int handle_secondary_tcp_pkt(NetFilterState *nf,
         }
     }
 
+    /*
+     * Passive close step 2:
+     */
+    if (conn->tcp_state == TCPS_CLOSE_WAIT &&
+        (tcp_pkt->th_flags & (TH_ACK | TH_FIN)) == (TH_ACK | TH_FIN)) {
+        conn->fin_ack_seq = ntohl(tcp_pkt->th_seq);
+        conn->tcp_state = TCPS_LAST_ACK;
+    }
+
+    /*
+     * Active close
+     *
+     * Step 1:
+     * The *server* side of this connect is VM, *server* tries to close
+     * the connection.
+     *
+     * Step 2:
+     * We will into CLOSE_WAIT status.
+     * We simplify the TCPS_FIN_WAIT_2, TCPS_TIME_WAIT and
+     * CLOSING status.
+     */
+    if (conn->tcp_state == TCPS_ESTABLISHED &&
+        (tcp_pkt->th_flags & (TH_ACK | TH_FIN)) == TH_FIN) {
+        conn->tcp_state = TCPS_FIN_WAIT_1;
+    }
+
     return 0;
 }
 
@@ -XXX,XX +XXX,XX @@ static ssize_t colo_rewriter_receive_iov(NetFilterState *nf,
 
         if (sender == nf->netdev) {
             /* NET_FILTER_DIRECTION_TX */
-            if (!handle_primary_tcp_pkt(nf, conn, pkt)) {
+            if (!handle_primary_tcp_pkt(s, conn, pkt, &key)) {
                 qemu_net_queue_send(s->incoming_queue, sender, 0,
                 (const uint8_t *)pkt->data, pkt->size, NULL);
                 packet_destroy(pkt, NULL);
@@ -XXX,XX +XXX,XX @@ static ssize_t colo_rewriter_receive_iov(NetFilterState *nf,
             }
         } else {
             /* NET_FILTER_DIRECTION_RX */
-            if (!handle_secondary_tcp_pkt(nf, conn, pkt)) {
+            if (!handle_secondary_tcp_pkt(s, conn, pkt, &key)) {
                 qemu_net_queue_send(s->incoming_queue, sender, 0,
                 (const uint8_t *)pkt->data, pkt->size, NULL);
                 packet_destroy(pkt, NULL);
-- 
2.5.0

From: Zhang Chen <zhangckid@gmail.com>

While do checkpoint, we need to flush all the unhandled packets,
By using the filter notifier mechanism, we can easily to notify
every compare object to do this process, which runs inside
of compare threads as a coroutine.

Signed-off-by: zhanghailiang <zhang.zhanghailiang@huawei.com>
Signed-off-by: Zhang Chen <zhangckid@gmail.com>
Signed-off-by: Zhang Chen <chen.zhang@intel.com>
Signed-off-by: Jason Wang <jasowang@redhat.com>
---
 include/migration/colo.h |  6 ++++
 net/colo-compare.c       | 78 ++++++++++++++++++++++++++++++++++++++++++++++++
 net/colo-compare.h       | 22 ++++++++++++++
 3 files changed, 106 insertions(+)
 create mode 100644 net/colo-compare.h

diff --git a/include/migration/colo.h b/include/migration/colo.h
index XXXXXXX..XXXXXXX 100644
--- a/include/migration/colo.h
+++ b/include/migration/colo.h
@@ -XXX,XX +XXX,XX @@
 #include "qemu-common.h"
 #include "qapi/qapi-types-migration.h"
 
+enum colo_event {
+    COLO_EVENT_NONE,
+    COLO_EVENT_CHECKPOINT,
+    COLO_EVENT_FAILOVER,
+};
+
 void colo_info_init(void);
 
 void migrate_start_colo_process(MigrationState *s);
diff --git a/net/colo-compare.c b/net/colo-compare.c
index XXXXXXX..XXXXXXX 100644
--- a/net/colo-compare.c
+++ b/net/colo-compare.c
@@ -XXX,XX +XXX,XX @@
 #include "qemu/sockets.h"
 #include "colo.h"
 #include "sysemu/iothread.h"
+#include "net/colo-compare.h"
+#include "migration/colo.h"
 
 #define TYPE_COLO_COMPARE "colo-compare"
 #define COLO_COMPARE(obj) \
     OBJECT_CHECK(CompareState, (obj), TYPE_COLO_COMPARE)
 
+static QTAILQ_HEAD(, CompareState) net_compares =
+       QTAILQ_HEAD_INITIALIZER(net_compares);
+
 #define COMPARE_READ_LEN_MAX NET_BUFSIZE
 #define MAX_QUEUE_SIZE 1024
 
@@ -XXX,XX +XXX,XX @@
 /* TODO: Should be configurable */
 #define REGULAR_PACKET_CHECK_MS 3000
 
+static QemuMutex event_mtx;
+static QemuCond event_complete_cond;
+static int event_unhandled_count;
+
 /*
  *  + CompareState ++
  *  |               |
@@ -XXX,XX +XXX,XX @@ typedef struct CompareState {
     IOThread *iothread;
     GMainContext *worker_context;
     QEMUTimer *packet_check_timer;
+
+    QEMUBH *event_bh;
+    enum colo_event event;
+
+    QTAILQ_ENTRY(CompareState) next;
 } CompareState;
 
 typedef struct CompareClass {
@@ -XXX,XX +XXX,XX @@ static void check_old_packet_regular(void *opaque)
                 REGULAR_PACKET_CHECK_MS);
 }
 
+/* Public API, Used for COLO frame to notify compare event */
+void colo_notify_compares_event(void *opaque, int event, Error **errp)
+{
+    CompareState *s;
+
+    qemu_mutex_lock(&event_mtx);
+    QTAILQ_FOREACH(s, &net_compares, next) {
+        s->event = event;
+        qemu_bh_schedule(s->event_bh);
+        event_unhandled_count++;
+    }
+    /* Wait all compare threads to finish handling this event */
+    while (event_unhandled_count > 0) {
+        qemu_cond_wait(&event_complete_cond, &event_mtx);
+    }
+
+    qemu_mutex_unlock(&event_mtx);
+}
+
 static void colo_compare_timer_init(CompareState *s)
 {
     AioContext *ctx = iothread_get_aio_context(s->iothread);
@@ -XXX,XX +XXX,XX @@ static void colo_compare_timer_del(CompareState *s)
     }
  }
 
+static void colo_flush_packets(void *opaque, void *user_data);
+
+static void colo_compare_handle_event(void *opaque)
+{
+    CompareState *s = opaque;
+
+    switch (s->event) {
+    case COLO_EVENT_CHECKPOINT:
+        g_queue_foreach(&s->conn_list, colo_flush_packets, s);
+        break;
+    case COLO_EVENT_FAILOVER:
+        break;
+    default:
+        break;
+    }
+
+    assert(event_unhandled_count > 0);
+
+    qemu_mutex_lock(&event_mtx);
+    event_unhandled_count--;
+    qemu_cond_broadcast(&event_complete_cond);
+    qemu_mutex_unlock(&event_mtx);
+}
+
 static void colo_compare_iothread(CompareState *s)
 {
     object_ref(OBJECT(s->iothread));
@@ -XXX,XX +XXX,XX @@ static void colo_compare_iothread(CompareState *s)
                              s, s->worker_context, true);
 
     colo_compare_timer_init(s);
+    s->event_bh = qemu_bh_new(colo_compare_handle_event, s);
 }
 
 static char *compare_get_pri_indev(Object *obj, Error **errp)
@@ -XXX,XX +XXX,XX @@ static void colo_compare_complete(UserCreatable *uc, Error **errp)
     net_socket_rs_init(&s->pri_rs, compare_pri_rs_finalize, s->vnet_hdr);
     net_socket_rs_init(&s->sec_rs, compare_sec_rs_finalize, s->vnet_hdr);
 
+    QTAILQ_INSERT_TAIL(&net_compares, s, next);
+
     g_queue_init(&s->conn_list);
 
+    qemu_mutex_init(&event_mtx);
+    qemu_cond_init(&event_complete_cond);
+
     s->connection_track_table = g_hash_table_new_full(connection_key_hash,
                                                       connection_key_equal,
                                                       g_free,
@@ -XXX,XX +XXX,XX @@ static void colo_compare_init(Object *obj)
 static void colo_compare_finalize(Object *obj)
 {
     CompareState *s = COLO_COMPARE(obj);
+    CompareState *tmp = NULL;
 
     qemu_chr_fe_deinit(&s->chr_pri_in, false);
     qemu_chr_fe_deinit(&s->chr_sec_in, false);
@@ -XXX,XX +XXX,XX @@ static void colo_compare_finalize(Object *obj)
     if (s->iothread) {
         colo_compare_timer_del(s);
     }
+
+    qemu_bh_delete(s->event_bh);
+
+    QTAILQ_FOREACH(tmp, &net_compares, next) {
+        if (tmp == s) {
+            QTAILQ_REMOVE(&net_compares, s, next);
+            break;
+        }
+    }
+
     /* Release all unhandled packets after compare thead exited */
     g_queue_foreach(&s->conn_list, colo_flush_packets, s);
 
@@ -XXX,XX +XXX,XX @@ static void colo_compare_finalize(Object *obj)
     if (s->iothread) {
         object_unref(OBJECT(s->iothread));
     }
+
+    qemu_mutex_destroy(&event_mtx);
+    qemu_cond_destroy(&event_complete_cond);
+
     g_free(s->pri_indev);
     g_free(s->sec_indev);
     g_free(s->outdev);
diff --git a/net/colo-compare.h b/net/colo-compare.h
new file mode 100644
index XXXXXXX..XXXXXXX
--- /dev/null
+++ b/net/colo-compare.h
@@ -XXX,XX +XXX,XX @@
+/*
+ * COarse-grain LOck-stepping Virtual Machines for Non-stop Service (COLO)
+ * (a.k.a. Fault Tolerance or Continuous Replication)
+ *
+ * Copyright (c) 2017 HUAWEI TECHNOLOGIES CO., LTD.
+ * Copyright (c) 2017 FUJITSU LIMITED
+ * Copyright (c) 2017 Intel Corporation
+ *
+ * Authors:
+ *    zhanghailiang <zhang.zhanghailiang@huawei.com>
+ *    Zhang Chen <zhangckid@gmail.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or
+ * later.  See the COPYING file in the top-level directory.
+ */
+
+#ifndef QEMU_COLO_COMPARE_H
+#define QEMU_COLO_COMPARE_H
+
+void colo_notify_compares_event(void *opaque, int event, Error **errp);
+
+#endif /* QEMU_COLO_COMPARE_H */
-- 
2.5.0

From: Zhang Chen <zhangckid@gmail.com>

It's a good idea to use notifier to notify COLO frame of
inconsistent packets comparing.

Signed-off-by: Zhang Chen <zhangckid@gmail.com>
Signed-off-by: Zhang Chen <chen.zhang@intel.com>
Signed-off-by: zhanghailiang <zhang.zhanghailiang@huawei.com>
Signed-off-by: Jason Wang <jasowang@redhat.com>
---
 net/colo-compare.c | 37 ++++++++++++++++++++++++++-----------
 net/colo-compare.h |  2 ++
 2 files changed, 28 insertions(+), 11 deletions(-)

diff --git a/net/colo-compare.c b/net/colo-compare.c
index XXXXXXX..XXXXXXX 100644
--- a/net/colo-compare.c
+++ b/net/colo-compare.c
@@ -XXX,XX +XXX,XX @@
 #include "sysemu/iothread.h"
 #include "net/colo-compare.h"
 #include "migration/colo.h"
+#include "migration/migration.h"
 
 #define TYPE_COLO_COMPARE "colo-compare"
 #define COLO_COMPARE(obj) \
@@ -XXX,XX +XXX,XX @@
 static QTAILQ_HEAD(, CompareState) net_compares =
        QTAILQ_HEAD_INITIALIZER(net_compares);
 
+static NotifierList colo_compare_notifiers =
+    NOTIFIER_LIST_INITIALIZER(colo_compare_notifiers);
+
 #define COMPARE_READ_LEN_MAX NET_BUFSIZE
 #define MAX_QUEUE_SIZE 1024
 
@@ -XXX,XX +XXX,XX @@ static bool colo_mark_tcp_pkt(Packet *ppkt, Packet *spkt,
     return false;
 }
 
+static void colo_compare_inconsistency_notify(void)
+{
+    notifier_list_notify(&colo_compare_notifiers,
+                migrate_get_current());
+}
+
 static void colo_compare_tcp(CompareState *s, Connection *conn)
 {
     Packet *ppkt = NULL, *spkt = NULL;
@@ -XXX,XX +XXX,XX @@ sec:
         qemu_hexdump((char *)spkt->data, stderr,
                      "colo-compare spkt", spkt->size);
 
-        /*
-         * colo_compare_inconsistent_notify();
-         * TODO: notice to checkpoint();
-         */
+        colo_compare_inconsistency_notify();
     }
 }
 
@@ -XXX,XX +XXX,XX @@ static int colo_old_packet_check_one(Packet *pkt, int64_t *check_time)
     }
 }
 
+void colo_compare_register_notifier(Notifier *notify)
+{
+    notifier_list_add(&colo_compare_notifiers, notify);
+}
+
+void colo_compare_unregister_notifier(Notifier *notify)
+{
+    notifier_remove(notify);
+}
+
 static int colo_old_packet_check_one_conn(Connection *conn,
-                                          void *user_data)
+                                           void *user_data)
 {
     GList *result = NULL;
     int64_t check_time = REGULAR_PACKET_CHECK_MS;
@@ -XXX,XX +XXX,XX @@ static int colo_old_packet_check_one_conn(Connection *conn,
 
     if (result) {
         /* Do checkpoint will flush old packet */
-        /*
-         * TODO: Notify colo frame to do checkpoint.
-         * colo_compare_inconsistent_notify();
-         */
+        colo_compare_inconsistency_notify();
         return 0;
     }
 
@@ -XXX,XX +XXX,XX @@ static void colo_compare_packet(CompareState *s, Connection *conn,
             /*
              * If one packet arrive late, the secondary_list or
              * primary_list will be empty, so we can't compare it
-             * until next comparison.
+             * until next comparison. If the packets in the list are
+             * timeout, it will trigger a checkpoint request.
              */
             trace_colo_compare_main("packet different");
             g_queue_push_head(&conn->primary_list, pkt);
-            /* TODO: colo_notify_checkpoint();*/
+            colo_compare_inconsistency_notify();
             break;
         }
     }
diff --git a/net/colo-compare.h b/net/colo-compare.h
index XXXXXXX..XXXXXXX 100644
--- a/net/colo-compare.h
+++ b/net/colo-compare.h
@@ -XXX,XX +XXX,XX @@
 #define QEMU_COLO_COMPARE_H
 
 void colo_notify_compares_event(void *opaque, int event, Error **errp);
+void colo_compare_register_notifier(Notifier *notify);
+void colo_compare_unregister_notifier(Notifier *notify);
 
 #endif /* QEMU_COLO_COMPARE_H */
-- 
2.5.0

From: Zhang Chen <zhangckid@gmail.com>

For COLO FT, both the PVM and SVM run at the same time,
only sync the state while it needs.

So here, let SVM runs while not doing checkpoint, change
DEFAULT_MIGRATE_X_CHECKPOINT_DELAY to 200*100.

Besides, we forgot to release colo_checkpoint_semd and
colo_delay_timer, fix them here.

diff --git a/migration/colo.c b/migration/colo.c
index XXXXXXX..XXXXXXX 100644
--- a/migration/colo.c
+++ b/migration/colo.c
@@ -XXX,XX +XXX,XX @@
 #include "qemu/error-report.h"
 #include "migration/failover.h"
 #include "replication.h"
+#include "net/colo-compare.h"
+#include "net/colo.h"
 
 static bool vmstate_loading;
+static Notifier packets_compare_notifier;
 
 #define COLO_BUFFER_BASE_SIZE (4 * 1024 * 1024)
 
@@ -XXX,XX +XXX,XX @@ static int colo_do_checkpoint_transaction(MigrationState *s,
         goto out;
     }
 
+    colo_notify_compares_event(NULL, COLO_EVENT_CHECKPOINT, &local_err);
+    if (local_err) {
+        goto out;
+    }
+
     /* Disable block migration */
     migrate_set_block_enabled(false, &local_err);
     qemu_savevm_state_header(fb);
@@ -XXX,XX +XXX,XX @@ out:
     return ret;
 }
 
+static void colo_compare_notify_checkpoint(Notifier *notifier, void *data)
+{
+    colo_checkpoint_notify(data);
+}
+
 static void colo_process_checkpoint(MigrationState *s)
 {
     QIOChannelBuffer *bioc;
@@ -XXX,XX +XXX,XX @@ static void colo_process_checkpoint(MigrationState *s)
         goto out;
     }
 
+    packets_compare_notifier.notify = colo_compare_notify_checkpoint;
+    colo_compare_register_notifier(&packets_compare_notifier);
+
     /*
      * Wait for Secondary finish loading VM states and enter COLO
      * restore.
@@ -XXX,XX +XXX,XX @@ out:
         qemu_fclose(fb);
     }
 
-    timer_del(s->colo_delay_timer);
-
     /* Hope this not to be too long to wait here */
     qemu_sem_wait(&s->colo_exit_sem);
     qemu_sem_destroy(&s->colo_exit_sem);
+
+    /*
+     * It is safe to unregister notifier after failover finished.
+     * Besides, colo_delay_timer and colo_checkpoint_sem can't be
+     * released befor unregister notifier, or there will be use-after-free
+     * error.
+     */
+    colo_compare_unregister_notifier(&packets_compare_notifier);
+    timer_del(s->colo_delay_timer);
+    timer_free(s->colo_delay_timer);
+    qemu_sem_destroy(&s->colo_checkpoint_sem);
+
     /*
      * Must be called after failover BH is completed,
      * Or the failover BH may shutdown the wrong fd that
@@ -XXX,XX +XXX,XX @@ void *colo_process_incoming_thread(void *opaque)
     fb = qemu_fopen_channel_input(QIO_CHANNEL(bioc));
     object_unref(OBJECT(bioc));
 
+    qemu_mutex_lock_iothread();
+    vm_start();
+    trace_colo_vm_state_change("stop", "run");
+    qemu_mutex_unlock_iothread();
+
     colo_send_message(mis->to_src_file, COLO_MESSAGE_CHECKPOINT_READY,
                       &local_err);
     if (local_err) {
@@ -XXX,XX +XXX,XX @@ void *colo_process_incoming_thread(void *opaque)
             goto out;
         }
 
+        qemu_mutex_lock_iothread();
+        vm_stop_force_state(RUN_STATE_COLO);
+        trace_colo_vm_state_change("run", "stop");
+        qemu_mutex_unlock_iothread();
+
         /* FIXME: This is unnecessary for periodic checkpoint mode */
         colo_send_message(mis->to_src_file, COLO_MESSAGE_CHECKPOINT_REPLY,
                      &local_err);
@@ -XXX,XX +XXX,XX @@ void *colo_process_incoming_thread(void *opaque)
         }
 
         vmstate_loading = false;
+        vm_start();
+        trace_colo_vm_state_change("stop", "run");
         qemu_mutex_unlock_iothread();
 
         if (failover_get_state() == FAILOVER_STATUS_RELAUNCH) {
diff --git a/migration/migration.c b/migration/migration.c
index XXXXXXX..XXXXXXX 100644
--- a/migration/migration.c
+++ b/migration/migration.c
@@ -XXX,XX +XXX,XX @@
 /* Migration XBZRLE default cache size */
 #define DEFAULT_MIGRATE_XBZRLE_CACHE_SIZE (64 * 1024 * 1024)
 
-/* The delay time (in ms) between two COLO checkpoints
- * Note: Please change this default value to 10000 when we support hybrid mode.
- */
-#define DEFAULT_MIGRATE_X_CHECKPOINT_DELAY 200
+/* The delay time (in ms) between two COLO checkpoints */
+#define DEFAULT_MIGRATE_X_CHECKPOINT_DELAY (200 * 100)
 #define DEFAULT_MIGRATE_MULTIFD_CHANNELS 2
 #define DEFAULT_MIGRATE_MULTIFD_PAGE_COUNT 16
 
-- 
2.5.0

From: Zhang Chen <zhangckid@gmail.com>

Make sure master start block replication after slave's block
replication started.

Besides, we need to activate VM's blocks before goes into
COLO state.

Signed-off-by: zhanghailiang <zhang.zhanghailiang@huawei.com>
Signed-off-by: Li Zhijian <lizhijian@cn.fujitsu.com>
Signed-off-by: Zhang Chen <zhangckid@gmail.com>
Signed-off-by: Zhang Chen <chen.zhang@intel.com>
Signed-off-by: Jason Wang <jasowang@redhat.com>
---
 migration/colo.c      | 43 +++++++++++++++++++++++++++++++++++++++++++
 migration/migration.c | 10 ++++++++++
 2 files changed, 53 insertions(+)

diff --git a/migration/colo.c b/migration/colo.c
index XXXXXXX..XXXXXXX 100644
--- a/migration/colo.c
+++ b/migration/colo.c
@@ -XXX,XX +XXX,XX @@
 #include "replication.h"
 #include "net/colo-compare.h"
 #include "net/colo.h"
+#include "block/block.h"
 
 static bool vmstate_loading;
 static Notifier packets_compare_notifier;
@@ -XXX,XX +XXX,XX @@ static void secondary_vm_do_failover(void)
 {
     int old_state;
     MigrationIncomingState *mis = migration_incoming_get_current();
+    Error *local_err = NULL;
 
     /* Can not do failover during the process of VM's loading VMstate, Or
      * it will break the secondary VM.
@@ -XXX,XX +XXX,XX @@ static void secondary_vm_do_failover(void)
     migrate_set_state(&mis->state, MIGRATION_STATUS_COLO,
                       MIGRATION_STATUS_COMPLETED);
 
+    replication_stop_all(true, &local_err);
+    if (local_err) {
+        error_report_err(local_err);
+    }
+
     if (!autostart) {
         error_report("\"-S\" qemu option will be ignored in secondary side");
         /* recover runstate to normal migration finish state */
@@ -XXX,XX +XXX,XX @@ static void primary_vm_do_failover(void)
 {
     MigrationState *s = migrate_get_current();
     int old_state;
+    Error *local_err = NULL;
 
     migrate_set_state(&s->state, MIGRATION_STATUS_COLO,
                       MIGRATION_STATUS_COMPLETED);
@@ -XXX,XX +XXX,XX @@ static void primary_vm_do_failover(void)
                      FailoverStatus_str(old_state));
         return;
     }
+
+    replication_stop_all(true, &local_err);
+    if (local_err) {
+        error_report_err(local_err);
+        local_err = NULL;
+    }
+
     /* Notify COLO thread that failover work is finished */
     qemu_sem_post(&s->colo_exit_sem);
 }
@@ -XXX,XX +XXX,XX @@ static int colo_do_checkpoint_transaction(MigrationState *s,
     qemu_savevm_state_header(fb);
     qemu_savevm_state_setup(fb);
     qemu_mutex_lock_iothread();
+    replication_do_checkpoint_all(&local_err);
+    if (local_err) {
+        qemu_mutex_unlock_iothread();
+        goto out;
+    }
     qemu_savevm_state_complete_precopy(fb, false, false);
     qemu_mutex_unlock_iothread();
 
@@ -XXX,XX +XXX,XX @@ static void colo_process_checkpoint(MigrationState *s)
     object_unref(OBJECT(bioc));
 
     qemu_mutex_lock_iothread();
+    replication_start_all(REPLICATION_MODE_PRIMARY, &local_err);
+    if (local_err) {
+        qemu_mutex_unlock_iothread();
+        goto out;
+    }
+
     vm_start();
     qemu_mutex_unlock_iothread();
     trace_colo_vm_state_change("stop", "run");
@@ -XXX,XX +XXX,XX @@ void *colo_process_incoming_thread(void *opaque)
     object_unref(OBJECT(bioc));
 
     qemu_mutex_lock_iothread();
+    replication_start_all(REPLICATION_MODE_SECONDARY, &local_err);
+    if (local_err) {
+        qemu_mutex_unlock_iothread();
+        goto out;
+    }
     vm_start();
     trace_colo_vm_state_change("stop", "run");
     qemu_mutex_unlock_iothread();
@@ -XXX,XX +XXX,XX @@ void *colo_process_incoming_thread(void *opaque)
             goto out;
         }
 
+        replication_get_error_all(&local_err);
+        if (local_err) {
+            qemu_mutex_unlock_iothread();
+            goto out;
+        }
+        /* discard colo disk buffer */
+        replication_do_checkpoint_all(&local_err);
+        if (local_err) {
+            qemu_mutex_unlock_iothread();
+            goto out;
+        }
+
         vmstate_loading = false;
         vm_start();
         trace_colo_vm_state_change("stop", "run");
diff --git a/migration/migration.c b/migration/migration.c
index XXXXXXX..XXXXXXX 100644
--- a/migration/migration.c
+++ b/migration/migration.c
@@ -XXX,XX +XXX,XX @@ static void process_incoming_migration_co(void *opaque)
     MigrationIncomingState *mis = migration_incoming_get_current();
     PostcopyState ps;
     int ret;
+    Error *local_err = NULL;
 
     assert(mis->from_src_file);
     mis->migration_incoming_co = qemu_coroutine_self();
@@ -XXX,XX +XXX,XX @@ static void process_incoming_migration_co(void *opaque)
 
     /* we get COLO info, and know if we are in COLO mode */
     if (!ret && migration_incoming_enable_colo()) {
+        /* Make sure all file formats flush their mutable metadata */
+        bdrv_invalidate_cache_all(&local_err);
+        if (local_err) {
+            migrate_set_state(&mis->state, MIGRATION_STATUS_ACTIVE,
+                    MIGRATION_STATUS_FAILED);
+            error_report_err(local_err);
+            exit(EXIT_FAILURE);
+        }
+
         qemu_thread_create(&mis->colo_incoming_thread, "COLO incoming",
              colo_process_incoming_thread, mis, QEMU_THREAD_JOINABLE);
         mis->have_colo_incoming_thread = true;
-- 
2.5.0

From: Zhang Chen <zhangckid@gmail.com>

We need to know if migration is going into COLO state for
incoming side before start normal migration.

Instead by using the VMStateDescription to send colo_state
from source side to destination side, we use MIG_CMD_ENABLE_COLO
to indicate whether COLO is enabled or not.

Signed-off-by: zhanghailiang <zhang.zhanghailiang@huawei.com>
Signed-off-by: Zhang Chen <zhangckid@gmail.com>
Signed-off-by: Zhang Chen <chen.zhang@intel.com>
Reviewed-by: Dr. David Alan Gilbert <dgilbert@redhat.com>
Signed-off-by: Jason Wang <jasowang@redhat.com>
---
 include/migration/colo.h |  5 ++--
 migration/Makefile.objs  |  2 +-
 migration/colo-comm.c    | 76 ------------------------------------------------
 migration/colo.c         | 13 ++++++++-
 migration/migration.c    | 23 ++++++++++++++-
 migration/savevm.c       | 17 +++++++++++
 migration/savevm.h       |  1 +
 migration/trace-events   |  1 +
 vl.c                     |  2 --
 9 files changed, 57 insertions(+), 83 deletions(-)
 delete mode 100644 migration/colo-comm.c

diff --git a/include/migration/colo.h b/include/migration/colo.h
index XXXXXXX..XXXXXXX 100644
--- a/include/migration/colo.h
+++ b/include/migration/colo.h
@@ -XXX,XX +XXX,XX @@ void migrate_start_colo_process(MigrationState *s);
 bool migration_in_colo_state(void);
 
 /* loadvm */
-bool migration_incoming_enable_colo(void);
-void migration_incoming_exit_colo(void);
+void migration_incoming_enable_colo(void);
+void migration_incoming_disable_colo(void);
+bool migration_incoming_colo_enabled(void);
 void *colo_process_incoming_thread(void *opaque);
 bool migration_incoming_in_colo_state(void);
 
diff --git a/migration/Makefile.objs b/migration/Makefile.objs
index XXXXXXX..XXXXXXX 100644
--- a/migration/Makefile.objs
+++ b/migration/Makefile.objs
@@ -XXX,XX +XXX,XX @@
 common-obj-y += migration.o socket.o fd.o exec.o
 common-obj-y += tls.o channel.o savevm.o
-common-obj-y += colo-comm.o colo.o colo-failover.o
+common-obj-y += colo.o colo-failover.o
 common-obj-y += vmstate.o vmstate-types.o page_cache.o
 common-obj-y += qemu-file.o global_state.o
 common-obj-y += qemu-file-channel.o
diff --git a/migration/colo-comm.c b/migration/colo-comm.c
deleted file mode 100644
index XXXXXXX..XXXXXXX
--- a/migration/colo-comm.c
+++ /dev/null
@@ -XXX,XX +XXX,XX @@
-/*
- * COarse-grain LOck-stepping Virtual Machines for Non-stop Service (COLO)
- * (a.k.a. Fault Tolerance or Continuous Replication)
- *
- * Copyright (c) 2016 HUAWEI TECHNOLOGIES CO., LTD.
- * Copyright (c) 2016 FUJITSU LIMITED
- * Copyright (c) 2016 Intel Corporation
- *
- * This work is licensed under the terms of the GNU GPL, version 2 or
- * later. See the COPYING file in the top-level directory.
- *
- */
-
-#include "qemu/osdep.h"
-#include "migration.h"
-#include "migration/colo.h"
-#include "migration/vmstate.h"
-#include "trace.h"
-
-typedef struct {
-     bool colo_requested;
-} COLOInfo;
-
-static COLOInfo colo_info;
-
-COLOMode get_colo_mode(void)
-{
-    if (migration_in_colo_state()) {
-        return COLO_MODE_PRIMARY;
-    } else if (migration_incoming_in_colo_state()) {
-        return COLO_MODE_SECONDARY;
-    } else {
-        return COLO_MODE_UNKNOWN;
-    }
-}
-
-static int colo_info_pre_save(void *opaque)
-{
-    COLOInfo *s = opaque;
-
-    s->colo_requested = migrate_colo_enabled();
-
-    return 0;
-}
-
-static bool colo_info_need(void *opaque)
-{
-   return migrate_colo_enabled();
-}
-
-static const VMStateDescription colo_state = {
-    .name = "COLOState",
-    .version_id = 1,
-    .minimum_version_id = 1,
-    .pre_save = colo_info_pre_save,
-    .needed = colo_info_need,
-    .fields = (VMStateField[]) {
-        VMSTATE_BOOL(colo_requested, COLOInfo),
-        VMSTATE_END_OF_LIST()
-    },
-};
-
-void colo_info_init(void)
-{
-    vmstate_register(NULL, 0, &colo_state, &colo_info);
-}
-
-bool migration_incoming_enable_colo(void)
-{
-    return colo_info.colo_requested;
-}
-
-void migration_incoming_exit_colo(void)
-{
-    colo_info.colo_requested = false;
-}
diff --git a/migration/colo.c b/migration/colo.c
index XXXXXXX..XXXXXXX 100644
--- a/migration/colo.c
+++ b/migration/colo.c
@@ -XXX,XX +XXX,XX @@ static void primary_vm_do_failover(void)
     qemu_sem_post(&s->colo_exit_sem);
 }
 
+COLOMode get_colo_mode(void)
+{
+    if (migration_in_colo_state()) {
+        return COLO_MODE_PRIMARY;
+    } else if (migration_incoming_in_colo_state()) {
+        return COLO_MODE_SECONDARY;
+    } else {
+        return COLO_MODE_UNKNOWN;
+    }
+}
+
 void colo_do_failover(MigrationState *s)
 {
     /* Make sure VM stopped while failover happened. */
@@ -XXX,XX +XXX,XX @@ out:
     if (mis->to_src_file) {
         qemu_fclose(mis->to_src_file);
     }
-    migration_incoming_exit_colo();
+    migration_incoming_disable_colo();
 
     rcu_unregister_thread();
     return NULL;
diff --git a/migration/migration.c b/migration/migration.c
index XXXXXXX..XXXXXXX 100644
--- a/migration/migration.c
+++ b/migration/migration.c
@@ -XXX,XX +XXX,XX @@ int migrate_send_rp_req_pages(MigrationIncomingState *mis, const char *rbname,
     return migrate_send_rp_message(mis, msg_type, msglen, bufc);
 }
 
+static bool migration_colo_enabled;
+bool migration_incoming_colo_enabled(void)
+{
+    return migration_colo_enabled;
+}
+
+void migration_incoming_disable_colo(void)
+{
+    migration_colo_enabled = false;
+}
+
+void migration_incoming_enable_colo(void)
+{
+    migration_colo_enabled = true;
+}
+
 void qemu_start_incoming_migration(const char *uri, Error **errp)
 {
     const char *p;
@@ -XXX,XX +XXX,XX @@ static void process_incoming_migration_co(void *opaque)
     }
 
     /* we get COLO info, and know if we are in COLO mode */
-    if (!ret && migration_incoming_enable_colo()) {
+    if (!ret && migration_incoming_colo_enabled()) {
         /* Make sure all file formats flush their mutable metadata */
         bdrv_invalidate_cache_all(&local_err);
         if (local_err) {
@@ -XXX,XX +XXX,XX @@ static void *migration_thread(void *opaque)
         qemu_savevm_send_postcopy_advise(s->to_dst_file);
     }
 
+    if (migrate_colo_enabled()) {
+        /* Notify migration destination that we enable COLO */
+        qemu_savevm_send_colo_enable(s->to_dst_file);
+    }
+
     qemu_savevm_state_setup(s->to_dst_file);
 
     s->setup_time = qemu_clock_get_ms(QEMU_CLOCK_HOST) - setup_start;
diff --git a/migration/savevm.c b/migration/savevm.c
index XXXXXXX..XXXXXXX 100644
--- a/migration/savevm.c
+++ b/migration/savevm.c
@@ -XXX,XX +XXX,XX @@
 #include "io/channel-file.h"
 #include "sysemu/replay.h"
 #include "qjson.h"
+#include "migration/colo.h"
 
 #ifndef ETH_P_RARP
 #define ETH_P_RARP 0x8035
@@ -XXX,XX +XXX,XX @@ enum qemu_vm_cmd {
                                       were previously sent during
                                       precopy but are dirty. */
     MIG_CMD_PACKAGED,          /* Send a wrapped stream within this stream */
+    MIG_CMD_ENABLE_COLO,       /* Enable COLO */
     MIG_CMD_POSTCOPY_RESUME,   /* resume postcopy on dest */
     MIG_CMD_RECV_BITMAP,       /* Request for recved bitmap on dst */
     MIG_CMD_MAX
@@ -XXX,XX +XXX,XX @@ static void qemu_savevm_command_send(QEMUFile *f,
     qemu_fflush(f);
 }
 
+void qemu_savevm_send_colo_enable(QEMUFile *f)
+{
+    trace_savevm_send_colo_enable();
+    qemu_savevm_command_send(f, MIG_CMD_ENABLE_COLO, 0, NULL);
+}
+
 void qemu_savevm_send_ping(QEMUFile *f, uint32_t value)
 {
     uint32_t buf;
@@ -XXX,XX +XXX,XX @@ static int loadvm_handle_recv_bitmap(MigrationIncomingState *mis,
     return 0;
 }
 
+static int loadvm_process_enable_colo(MigrationIncomingState *mis)
+{
+    migration_incoming_enable_colo();
+    return 0;
+}
+
 /*
  * Process an incoming 'QEMU_VM_COMMAND'
  * 0           just a normal return
@@ -XXX,XX +XXX,XX @@ static int loadvm_process_command(QEMUFile *f)
 
     case MIG_CMD_RECV_BITMAP:
         return loadvm_handle_recv_bitmap(mis, len);
+
+    case MIG_CMD_ENABLE_COLO:
+        return loadvm_process_enable_colo(mis);
     }
 
     return 0;
diff --git a/migration/savevm.h b/migration/savevm.h
index XXXXXXX..XXXXXXX 100644
--- a/migration/savevm.h
+++ b/migration/savevm.h
@@ -XXX,XX +XXX,XX @@ void qemu_savevm_send_postcopy_ram_discard(QEMUFile *f, const char *name,
                                            uint16_t len,
                                            uint64_t *start_list,
                                            uint64_t *length_list);
+void qemu_savevm_send_colo_enable(QEMUFile *f);
 
 int qemu_loadvm_state(QEMUFile *f);
 void qemu_loadvm_state_cleanup(void);
diff --git a/migration/trace-events b/migration/trace-events
index XXXXXXX..XXXXXXX 100644
--- a/migration/trace-events
+++ b/migration/trace-events
@@ -XXX,XX +XXX,XX @@ savevm_send_ping(uint32_t val) "0x%x"
 savevm_send_postcopy_listen(void) ""
 savevm_send_postcopy_run(void) ""
 savevm_send_postcopy_resume(void) ""
+savevm_send_colo_enable(void) ""
 savevm_send_recv_bitmap(char *name) "%s"
 savevm_state_setup(void) ""
 savevm_state_resume_prepare(void) ""
diff --git a/vl.c b/vl.c
index XXXXXXX..XXXXXXX 100644
--- a/vl.c
+++ b/vl.c
@@ -XXX,XX +XXX,XX @@ int main(int argc, char **argv, char **envp)
 #endif
     }
 
-    colo_info_init();
-
     if (net_init_clients(&err) < 0) {
         error_report_err(err);
         exit(1);
-- 
2.5.0

From: Zhang Chen <zhangckid@gmail.com>

We should not load PVM's state directly into SVM, because there maybe some
errors happen when SVM is receving data, which will break SVM.

We need to ensure receving all data before load the state into SVM. We use
an extra memory to cache these data (PVM's ram). The ram cache in secondary side
is initially the same as SVM/PVM's memory. And in the process of checkpoint,
we cache the dirty pages of PVM into this ram cache firstly, so this ram cache
always the same as PVM's memory at every checkpoint, then we flush this cached ram
to SVM after we receive all PVM's state.

Signed-off-by: zhanghailiang <zhang.zhanghailiang@huawei.com>
Signed-off-by: Li Zhijian <lizhijian@cn.fujitsu.com>
Signed-off-by: Zhang Chen <zhangckid@gmail.com>
Signed-off-by: Zhang Chen <chen.zhang@intel.com>
Reviewed-by: Dr. David Alan Gilbert <dgilbert@redhat.com>
Signed-off-by: Jason Wang <jasowang@redhat.com>
---
 include/exec/ram_addr.h |  1 +
 migration/migration.c   |  7 +++++
 migration/ram.c         | 83 +++++++++++++++++++++++++++++++++++++++++++++++--
 migration/ram.h         |  4 +++
 migration/savevm.c      |  2 +-
 5 files changed, 94 insertions(+), 3 deletions(-)

diff --git a/include/exec/ram_addr.h b/include/exec/ram_addr.h
index XXXXXXX..XXXXXXX 100644
--- a/include/exec/ram_addr.h
+++ b/include/exec/ram_addr.h
@@ -XXX,XX +XXX,XX @@ struct RAMBlock {
     struct rcu_head rcu;
     struct MemoryRegion *mr;
     uint8_t *host;
+    uint8_t *colo_cache; /* For colo, VM's ram cache */
     ram_addr_t offset;
     ram_addr_t used_length;
     ram_addr_t max_length;
diff --git a/migration/migration.c b/migration/migration.c
index XXXXXXX..XXXXXXX 100644
--- a/migration/migration.c
+++ b/migration/migration.c
@@ -XXX,XX +XXX,XX @@ static void process_incoming_migration_co(void *opaque)
             exit(EXIT_FAILURE);
         }
 
+        if (colo_init_ram_cache() < 0) {
+            error_report("Init ram cache failed");
+            exit(EXIT_FAILURE);
+        }
+
         qemu_thread_create(&mis->colo_incoming_thread, "COLO incoming",
              colo_process_incoming_thread, mis, QEMU_THREAD_JOINABLE);
         mis->have_colo_incoming_thread = true;
@@ -XXX,XX +XXX,XX @@ static void process_incoming_migration_co(void *opaque)
 
         /* Wait checkpoint incoming thread exit before free resource */
         qemu_thread_join(&mis->colo_incoming_thread);
+        /* We hold the global iothread lock, so it is safe here */
+        colo_release_ram_cache();
     }
 
     if (ret < 0) {
diff --git a/migration/ram.c b/migration/ram.c
index XXXXXXX..XXXXXXX 100644
--- a/migration/ram.c
+++ b/migration/ram.c
@@ -XXX,XX +XXX,XX @@ static inline void *host_from_ram_block_offset(RAMBlock *block,
     return block->host + offset;
 }
 
+static inline void *colo_cache_from_block_offset(RAMBlock *block,
+                                                 ram_addr_t offset)
+{
+    if (!offset_in_ramblock(block, offset)) {
+        return NULL;
+    }
+    if (!block->colo_cache) {
+        error_report("%s: colo_cache is NULL in block :%s",
+                     __func__, block->idstr);
+        return NULL;
+    }
+    return block->colo_cache + offset;
+}
+
 /**
  * ram_handle_compressed: handle the zero page case
  *
@@ -XXX,XX +XXX,XX @@ static void decompress_data_with_multi_threads(QEMUFile *f,
     qemu_mutex_unlock(&decomp_done_lock);
 }
 
+/*
+ * colo cache: this is for secondary VM, we cache the whole
+ * memory of the secondary VM, it is need to hold the global lock
+ * to call this helper.
+ */
+int colo_init_ram_cache(void)
+{
+    RAMBlock *block;
+
+    rcu_read_lock();
+    QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
+        block->colo_cache = qemu_anon_ram_alloc(block->used_length,
+                                                NULL,
+                                                false);
+        if (!block->colo_cache) {
+            error_report("%s: Can't alloc memory for COLO cache of block %s,"
+                         "size 0x" RAM_ADDR_FMT, __func__, block->idstr,
+                         block->used_length);
+            goto out_locked;
+        }
+        memcpy(block->colo_cache, block->host, block->used_length);
+    }
+    rcu_read_unlock();
+    return 0;
+
+out_locked:
+    QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
+        if (block->colo_cache) {
+            qemu_anon_ram_free(block->colo_cache, block->used_length);
+            block->colo_cache = NULL;
+        }
+    }
+
+    rcu_read_unlock();
+    return -errno;
+}
+
+/* It is need to hold the global lock to call this helper */
+void colo_release_ram_cache(void)
+{
+    RAMBlock *block;
+
+    rcu_read_lock();
+    QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
+        if (block->colo_cache) {
+            qemu_anon_ram_free(block->colo_cache, block->used_length);
+            block->colo_cache = NULL;
+        }
+    }
+    rcu_read_unlock();
+}
+
 /**
  * ram_load_setup: Setup RAM for migration incoming side
  *
@@ -XXX,XX +XXX,XX @@ static int ram_load_setup(QEMUFile *f, void *opaque)
 
     xbzrle_load_setup();
     ramblock_recv_map_init();
+
     return 0;
 }
 
@@ -XXX,XX +XXX,XX @@ static int ram_load_cleanup(void *opaque)
         g_free(rb->receivedmap);
         rb->receivedmap = NULL;
     }
+
     return 0;
 }
 
@@ -XXX,XX +XXX,XX @@ static int ram_load(QEMUFile *f, void *opaque, int version_id)
                      RAM_SAVE_FLAG_COMPRESS_PAGE | RAM_SAVE_FLAG_XBZRLE)) {
             RAMBlock *block = ram_block_from_stream(f, flags);
 
-            host = host_from_ram_block_offset(block, addr);
+            /*
+             * After going into COLO, we should load the Page into colo_cache.
+             */
+            if (migration_incoming_in_colo_state()) {
+                host = colo_cache_from_block_offset(block, addr);
+            } else {
+                host = host_from_ram_block_offset(block, addr);
+            }
             if (!host) {
                 error_report("Illegal RAM offset " RAM_ADDR_FMT, addr);
                 ret = -EINVAL;
                 break;
             }
-            ramblock_recv_bitmap_set(block, host);
+
+            if (!migration_incoming_in_colo_state()) {
+                ramblock_recv_bitmap_set(block, host);
+            }
+
             trace_ram_load_loop(block->idstr, (uint64_t)addr, flags, host);
         }
 
diff --git a/migration/ram.h b/migration/ram.h
index XXXXXXX..XXXXXXX 100644
--- a/migration/ram.h
+++ b/migration/ram.h
@@ -XXX,XX +XXX,XX @@ int64_t ramblock_recv_bitmap_send(QEMUFile *file,
                                   const char *block_name);
 int ram_dirty_bitmap_reload(MigrationState *s, RAMBlock *rb);
 
+/* ram cache */
+int colo_init_ram_cache(void);
+void colo_release_ram_cache(void);
+
 #endif
diff --git a/migration/savevm.c b/migration/savevm.c
index XXXXXXX..XXXXXXX 100644
--- a/migration/savevm.c
+++ b/migration/savevm.c
@@ -XXX,XX +XXX,XX @@ static int loadvm_handle_recv_bitmap(MigrationIncomingState *mis,
 static int loadvm_process_enable_colo(MigrationIncomingState *mis)
 {
     migration_incoming_enable_colo();
-    return 0;
+    return colo_init_ram_cache();
 }
 
 /*
-- 
2.5.0

From: Zhang Chen <zhangckid@gmail.com>

We record the address of the dirty pages that received,
it will help flushing pages that cached into SVM.

Here, it is a trick, we record dirty pages by re-using migration
dirty bitmap. In the later patch, we will start the dirty log
for SVM, just like migration, in this way, we can record both
the dirty pages caused by PVM and SVM, we only flush those dirty
pages from RAM cache while do checkpoint.

diff --git a/migration/ram.c b/migration/ram.c
index XXXXXXX..XXXXXXX 100644
--- a/migration/ram.c
+++ b/migration/ram.c
@@ -XXX,XX +XXX,XX @@ static inline void *colo_cache_from_block_offset(RAMBlock *block,
                      __func__, block->idstr);
         return NULL;
     }
+
+    /*
+    * During colo checkpoint, we need bitmap of these migrated pages.
+    * It help us to decide which pages in ram cache should be flushed
+    * into VM's RAM later.
+    */
+    if (!test_and_set_bit(offset >> TARGET_PAGE_BITS, block->bmap)) {
+        ram_state->migration_dirty_pages++;
+    }
     return block->colo_cache + offset;
 }
 
@@ -XXX,XX +XXX,XX @@ int colo_init_ram_cache(void)
     RAMBlock *block;
 
     rcu_read_lock();
-    QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
+    RAMBLOCK_FOREACH_MIGRATABLE(block) {
         block->colo_cache = qemu_anon_ram_alloc(block->used_length,
                                                 NULL,
                                                 false);
@@ -XXX,XX +XXX,XX @@ int colo_init_ram_cache(void)
         memcpy(block->colo_cache, block->host, block->used_length);
     }
     rcu_read_unlock();
+    /*
+    * Record the dirty pages that sent by PVM, we use this dirty bitmap together
+    * with to decide which page in cache should be flushed into SVM's RAM. Here
+    * we use the same name 'ram_bitmap' as for migration.
+    */
+    if (ram_bytes_total()) {
+        RAMBlock *block;
+
+        RAMBLOCK_FOREACH_MIGRATABLE(block) {
+            unsigned long pages = block->max_length >> TARGET_PAGE_BITS;
+
+            block->bmap = bitmap_new(pages);
+            bitmap_set(block->bmap, 0, pages);
+        }
+    }
+    ram_state = g_new0(RAMState, 1);
+    ram_state->migration_dirty_pages = 0;
+
     return 0;
 
 out_locked:
-    QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
+
+    RAMBLOCK_FOREACH_MIGRATABLE(block) {
         if (block->colo_cache) {
             qemu_anon_ram_free(block->colo_cache, block->used_length);
             block->colo_cache = NULL;
@@ -XXX,XX +XXX,XX @@ void colo_release_ram_cache(void)
 {
     RAMBlock *block;
 
+    RAMBLOCK_FOREACH_MIGRATABLE(block) {
+        g_free(block->bmap);
+        block->bmap = NULL;
+    }
+
     rcu_read_lock();
-    QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
+
+    RAMBLOCK_FOREACH_MIGRATABLE(block) {
         if (block->colo_cache) {
             qemu_anon_ram_free(block->colo_cache, block->used_length);
             block->colo_cache = NULL;
         }
     }
+
     rcu_read_unlock();
+    g_free(ram_state);
+    ram_state = NULL;
 }
 
 /**
-- 
2.5.0

From: Zhang Chen <zhangckid@gmail.com>

During the time of VM's running, PVM may dirty some pages, we will transfer
PVM's dirty pages to SVM and store them into SVM's RAM cache at next checkpoint
time. So, the content of SVM's RAM cache will always be same with PVM's memory
after checkpoint.

Instead of flushing all content of PVM's RAM cache into SVM's MEMORY,
we do this in a more efficient way:
Only flush any page that dirtied by PVM since last checkpoint.
In this way, we can ensure SVM's memory same with PVM's.

Besides, we must ensure flush RAM cache before load device state.

Signed-off-by: zhanghailiang <zhang.zhanghailiang@huawei.com>
Signed-off-by: Li Zhijian <lizhijian@cn.fujitsu.com>
Reviewed-by: Dr. David Alan Gilbert <dgilbert@redhat.com>
Signed-off-by: Jason Wang <jasowang@redhat.com>
---
 migration/ram.c        | 37 +++++++++++++++++++++++++++++++++++++
 migration/trace-events |  2 ++
 2 files changed, 39 insertions(+)

diff --git a/migration/ram.c b/migration/ram.c
index XXXXXXX..XXXXXXX 100644
--- a/migration/ram.c
+++ b/migration/ram.c
@@ -XXX,XX +XXX,XX @@ static bool postcopy_is_running(void)
     return ps >= POSTCOPY_INCOMING_LISTENING && ps < POSTCOPY_INCOMING_END;
 }
 
+/*
+ * Flush content of RAM cache into SVM's memory.
+ * Only flush the pages that be dirtied by PVM or SVM or both.
+ */
+static void colo_flush_ram_cache(void)
+{
+    RAMBlock *block = NULL;
+    void *dst_host;
+    void *src_host;
+    unsigned long offset = 0;
+
+    trace_colo_flush_ram_cache_begin(ram_state->migration_dirty_pages);
+    rcu_read_lock();
+    block = QLIST_FIRST_RCU(&ram_list.blocks);
+
+    while (block) {
+        offset = migration_bitmap_find_dirty(ram_state, block, offset);
+
+        if (offset << TARGET_PAGE_BITS >= block->used_length) {
+            offset = 0;
+            block = QLIST_NEXT_RCU(block, next);
+        } else {
+            migration_bitmap_clear_dirty(ram_state, block, offset);
+            dst_host = block->host + (offset << TARGET_PAGE_BITS);
+            src_host = block->colo_cache + (offset << TARGET_PAGE_BITS);
+            memcpy(dst_host, src_host, TARGET_PAGE_SIZE);
+        }
+    }
+
+    rcu_read_unlock();
+    trace_colo_flush_ram_cache_end();
+}
+
 static int ram_load(QEMUFile *f, void *opaque, int version_id)
 {
     int flags = 0, ret = 0, invalid_flags = 0;
@@ -XXX,XX +XXX,XX @@ static int ram_load(QEMUFile *f, void *opaque, int version_id)
     ret |= wait_for_decompress_done();
     rcu_read_unlock();
     trace_ram_load_complete(ret, seq_iter);
+
+    if (!ret  && migration_incoming_in_colo_state()) {
+        colo_flush_ram_cache();
+    }
     return ret;
 }
 
diff --git a/migration/trace-events b/migration/trace-events
index XXXXXXX..XXXXXXX 100644
--- a/migration/trace-events
+++ b/migration/trace-events
@@ -XXX,XX +XXX,XX @@ ram_dirty_bitmap_sync_start(void) ""
 ram_dirty_bitmap_sync_wait(void) ""
 ram_dirty_bitmap_sync_complete(void) ""
 ram_state_resume_prepare(uint64_t v) "%" PRId64
+colo_flush_ram_cache_begin(uint64_t dirty_pages) "dirty_pages %" PRIu64
+colo_flush_ram_cache_end(void) ""
 
 # migration/migration.c
 await_return_path_close_on_source_close(void) ""
-- 
2.5.0

From: zhanghailiang <zhang.zhanghailiang@huawei.com>

If some errors happen during VM's COLO FT stage, it's important to
notify the users of this event. Together with 'x-colo-lost-heartbeat',
Users can intervene in COLO's failover work immediately.
If users don't want to get involved in COLO's failover verdict,
it is still necessary to notify users that we exited COLO mode.

Signed-off-by: zhanghailiang <zhang.zhanghailiang@huawei.com>
Signed-off-by: Li Zhijian <lizhijian@cn.fujitsu.com>
Signed-off-by: Zhang Chen <zhangckid@gmail.com>
Signed-off-by: Zhang Chen <chen.zhang@intel.com>
Signed-off-by: Jason Wang <jasowang@redhat.com>
---
 migration/colo.c    | 31 +++++++++++++++++++++++++++++++
 qapi/migration.json | 38 ++++++++++++++++++++++++++++++++++++++
 2 files changed, 69 insertions(+)

diff --git a/migration/colo.c b/migration/colo.c
index XXXXXXX..XXXXXXX 100644
--- a/migration/colo.c
+++ b/migration/colo.c
@@ -XXX,XX +XXX,XX @@
 #include "net/colo-compare.h"
 #include "net/colo.h"
 #include "block/block.h"
+#include "qapi/qapi-events-migration.h"
 
 static bool vmstate_loading;
 static Notifier packets_compare_notifier;
@@ -XXX,XX +XXX,XX @@ out:
         qemu_fclose(fb);
     }
 
+    /*
+     * There are only two reasons we can get here, some error happened
+     * or the user triggered failover.
+     */
+    switch (failover_get_state()) {
+    case FAILOVER_STATUS_NONE:
+        qapi_event_send_colo_exit(COLO_MODE_PRIMARY,
+                                  COLO_EXIT_REASON_ERROR);
+        break;
+    case FAILOVER_STATUS_REQUIRE:
+        qapi_event_send_colo_exit(COLO_MODE_PRIMARY,
+                                  COLO_EXIT_REASON_REQUEST);
+        break;
+    default:
+        abort();
+    }
+
     /* Hope this not to be too long to wait here */
     qemu_sem_wait(&s->colo_exit_sem);
     qemu_sem_destroy(&s->colo_exit_sem);
@@ -XXX,XX +XXX,XX @@ out:
         error_report_err(local_err);
     }
 
+    switch (failover_get_state()) {
+    case FAILOVER_STATUS_NONE:
+        qapi_event_send_colo_exit(COLO_MODE_SECONDARY,
+                                  COLO_EXIT_REASON_ERROR);
+        break;
+    case FAILOVER_STATUS_REQUIRE:
+        qapi_event_send_colo_exit(COLO_MODE_SECONDARY,
+                                  COLO_EXIT_REASON_REQUEST);
+        break;
+    default:
+        abort();
+    }
+
     if (fb) {
         qemu_fclose(fb);
     }
diff --git a/qapi/migration.json b/qapi/migration.json
index XXXXXXX..XXXXXXX 100644
--- a/qapi/migration.json
+++ b/qapi/migration.json
@@ -XXX,XX +XXX,XX @@
   'data': [ 'none', 'require', 'active', 'completed', 'relaunch' ] }
 
 ##
+# @COLO_EXIT:
+#
+# Emitted when VM finishes COLO mode due to some errors happening or
+# at the request of users.
+#
+# @mode: report COLO mode when COLO exited.
+#
+# @reason: describes the reason for the COLO exit.
+#
+# Since: 3.1
+#
+# Example:
+#
+# <- { "timestamp": {"seconds": 2032141960, "microseconds": 417172},
+#      "event": "COLO_EXIT", "data": {"mode": "primary", "reason": "request" } }
+#
+##
+{ 'event': 'COLO_EXIT',
+  'data': {'mode': 'COLOMode', 'reason': 'COLOExitReason' } }
+
+##
+# @COLOExitReason:
+#
+# The reason for a COLO exit
+#
+# @none: no failover has ever happened. This can't occur in the
+# COLO_EXIT event, only in the result of query-colo-status.
+#
+# @request: COLO exit is due to an external request
+#
+# @error: COLO exit is due to an internal error
+#
+# Since: 3.1
+##
+{ 'enum': 'COLOExitReason',
+  'data': [ 'none', 'request', 'error' ] }
+
+##
 # @x-colo-lost-heartbeat:
 #
 # Tell qemu that heartbeat is lost, request it to do takeover procedures.
-- 
2.5.0

From: Zhang Chen <chen.zhang@intel.com>

Suggested by Markus Armbruster rename COLO unknown mode to none mode.

Signed-off-by: Zhang Chen <zhangckid@gmail.com>
Signed-off-by: Zhang Chen <chen.zhang@intel.com>
Reviewed-by: Eric Blake <eblake@redhat.com>
Reviewed-by: Markus Armbruster <armbru@redhat.com>
Signed-off-by: Jason Wang <jasowang@redhat.com>
---
 migration/colo-failover.c |  2 +-
 migration/colo.c          |  2 +-
 qapi/migration.json       | 10 +++++-----
 3 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/migration/colo-failover.c b/migration/colo-failover.c
index XXXXXXX..XXXXXXX 100644
--- a/migration/colo-failover.c
+++ b/migration/colo-failover.c
@@ -XXX,XX +XXX,XX @@ FailoverStatus failover_get_state(void)
 
 void qmp_x_colo_lost_heartbeat(Error **errp)
 {
-    if (get_colo_mode() == COLO_MODE_UNKNOWN) {
+    if (get_colo_mode() == COLO_MODE_NONE) {
         error_setg(errp, QERR_FEATURE_DISABLED, "colo");
         return;
     }
diff --git a/migration/colo.c b/migration/colo.c
index XXXXXXX..XXXXXXX 100644
--- a/migration/colo.c
+++ b/migration/colo.c
@@ -XXX,XX +XXX,XX @@ COLOMode get_colo_mode(void)
     } else if (migration_incoming_in_colo_state()) {
         return COLO_MODE_SECONDARY;
     } else {
-        return COLO_MODE_UNKNOWN;
+        return COLO_MODE_NONE;
     }
 }
 
diff --git a/qapi/migration.json b/qapi/migration.json
index XXXXXXX..XXXXXXX 100644
--- a/qapi/migration.json
+++ b/qapi/migration.json
@@ -XXX,XX +XXX,XX @@
 ##
 # @COLOMode:
 #
-# The colo mode
+# The COLO current mode.
 #
-# @unknown: unknown mode
+# @none: COLO is disabled.
 #
-# @primary: master side
+# @primary: COLO node in primary side.
 #
-# @secondary: slave side
+# @secondary: COLO node in slave side.
 #
 # Since: 2.8
 ##
 { 'enum': 'COLOMode',
-  'data': [ 'unknown', 'primary', 'secondary'] }
+  'data': [ 'none', 'primary', 'secondary'] }
 
 ##
 # @FailoverStatus:
-- 
2.5.0

From: Zhang Chen <zhangckid@gmail.com>

Libvirt or other high level software can use this command query colo status.
You can test this command like that:
{'execute':'query-colo-status'}

Signed-off-by: Zhang Chen <zhangckid@gmail.com>
Signed-off-by: Zhang Chen <chen.zhang@intel.com>
Signed-off-by: Jason Wang <jasowang@redhat.com>
---
 migration/colo.c    | 21 +++++++++++++++++++++
 qapi/migration.json | 32 ++++++++++++++++++++++++++++++++
 2 files changed, 53 insertions(+)

diff --git a/migration/colo.c b/migration/colo.c
index XXXXXXX..XXXXXXX 100644
--- a/migration/colo.c
+++ b/migration/colo.c
@@ -XXX,XX +XXX,XX @@
 #include "net/colo.h"
 #include "block/block.h"
 #include "qapi/qapi-events-migration.h"
+#include "qapi/qmp/qerror.h"
 
 static bool vmstate_loading;
 static Notifier packets_compare_notifier;
@@ -XXX,XX +XXX,XX @@ void qmp_xen_colo_do_checkpoint(Error **errp)
 #endif
 }
 
+COLOStatus *qmp_query_colo_status(Error **errp)
+{
+    COLOStatus *s = g_new0(COLOStatus, 1);
+
+    s->mode = get_colo_mode();
+
+    switch (failover_get_state()) {
+    case FAILOVER_STATUS_NONE:
+        s->reason = COLO_EXIT_REASON_NONE;
+        break;
+    case FAILOVER_STATUS_REQUIRE:
+        s->reason = COLO_EXIT_REASON_REQUEST;
+        break;
+    default:
+        s->reason = COLO_EXIT_REASON_ERROR;
+    }
+
+    return s;
+}
+
 static void colo_send_message(QEMUFile *f, COLOMessage msg,
                               Error **errp)
 {
diff --git a/qapi/migration.json b/qapi/migration.json
index XXXXXXX..XXXXXXX 100644
--- a/qapi/migration.json
+++ b/qapi/migration.json
@@ -XXX,XX +XXX,XX @@
 { 'command': 'xen-colo-do-checkpoint' }
 
 ##
+# @COLOStatus:
+#
+# The result format for 'query-colo-status'.
+#
+# @mode: COLO running mode. If COLO is running, this field will return
+#        'primary' or 'secondary'.
+#
+# @reason: describes the reason for the COLO exit.
+#
+# Since: 3.0
+##
+{ 'struct': 'COLOStatus',
+  'data': { 'mode': 'COLOMode', 'reason': 'COLOExitReason' } }
+
+##
+# @query-colo-status:
+#
+# Query COLO status while the vm is running.
+#
+# Returns: A @COLOStatus object showing the status.
+#
+# Example:
+#
+# -> { "execute": "query-colo-status" }
+# <- { "return": { "mode": "primary", "active": true, "reason": "request" } }
+#
+# Since: 3.0
+##
+{ 'command': 'query-colo-status',
+  'returns': 'COLOStatus' }
+
+##
 # @migrate-recover:
 #
 # Provide a recovery migration stream URI.
-- 
2.5.0

From: Zhang Chen <zhangckid@gmail.com>

There are several stages during loadvm/savevm process. In different stage,
migration incoming processes different types of sections.
We want to control these stages more accuracy, it will benefit COLO
performance, we don't have to save type of QEMU_VM_SECTION_START
sections everytime while do checkpoint, besides, we want to separate
the process of saving/loading memory and devices state.

So we add three new helper functions: qemu_load_device_state() and
qemu_savevm_live_state() to achieve different process during migration.

Besides, we make qemu_loadvm_state_main() and qemu_save_device_state()
public, and simplify the codes of qemu_save_device_state() by calling the
wrapper qemu_savevm_state_header().

Signed-off-by: zhanghailiang <zhang.zhanghailiang@huawei.com>
Signed-off-by: Li Zhijian <lizhijian@cn.fujitsu.com>
Signed-off-by: Zhang Chen <zhangckid@gmail.com>
Signed-off-by: Zhang Chen <chen.zhang@intel.com>
Reviewed-by: Dr. David Alan Gilbert <dgilbert@redhat.com>
Signed-off-by: Jason Wang <jasowang@redhat.com>
---
 migration/colo.c   | 41 ++++++++++++++++++++++++++++++++---------
 migration/savevm.c | 36 +++++++++++++++++++++++++++++-------
 migration/savevm.h |  4 ++++
 3 files changed, 65 insertions(+), 16 deletions(-)

diff --git a/migration/colo.c b/migration/colo.c
index XXXXXXX..XXXXXXX 100644
--- a/migration/colo.c
+++ b/migration/colo.c
@@ -XXX,XX +XXX,XX @@
 #include "block/block.h"
 #include "qapi/qapi-events-migration.h"
 #include "qapi/qmp/qerror.h"
+#include "sysemu/cpus.h"
 
 static bool vmstate_loading;
 static Notifier packets_compare_notifier;
@@ -XXX,XX +XXX,XX @@ static int colo_do_checkpoint_transaction(MigrationState *s,
 
     /* Disable block migration */
     migrate_set_block_enabled(false, &local_err);
-    qemu_savevm_state_header(fb);
-    qemu_savevm_state_setup(fb);
     qemu_mutex_lock_iothread();
     replication_do_checkpoint_all(&local_err);
     if (local_err) {
         qemu_mutex_unlock_iothread();
         goto out;
     }
-    qemu_savevm_state_complete_precopy(fb, false, false);
-    qemu_mutex_unlock_iothread();
-
-    qemu_fflush(fb);
 
     colo_send_message(s->to_dst_file, COLO_MESSAGE_VMSTATE_SEND, &local_err);
     if (local_err) {
+        qemu_mutex_unlock_iothread();
+        goto out;
+    }
+    /* Note: device state is saved into buffer */
+    ret = qemu_save_device_state(fb);
+
+    qemu_mutex_unlock_iothread();
+    if (ret < 0) {
         goto out;
     }
     /*
+     * Only save VM's live state, which not including device state.
+     * TODO: We may need a timeout mechanism to prevent COLO process
+     * to be blocked here.
+     */
+    qemu_savevm_live_state(s->to_dst_file);
+
+    qemu_fflush(fb);
+
+    /*
      * We need the size of the VMstate data in Secondary side,
      * With which we can decide how much data should be read.
      */
@@ -XXX,XX +XXX,XX @@ void *colo_process_incoming_thread(void *opaque)
     uint64_t total_size;
     uint64_t value;
     Error *local_err = NULL;
+    int ret;
 
     rcu_register_thread();
     qemu_sem_init(&mis->colo_incoming_sem, 0);
@@ -XXX,XX +XXX,XX @@ void *colo_process_incoming_thread(void *opaque)
             goto out;
         }
 
+        qemu_mutex_lock_iothread();
+        cpu_synchronize_all_pre_loadvm();
+        ret = qemu_loadvm_state_main(mis->from_src_file, mis);
+        qemu_mutex_unlock_iothread();
+
+        if (ret < 0) {
+            error_report("Load VM's live state (ram) error");
+            goto out;
+        }
+
         value = colo_receive_message_value(mis->from_src_file,
                                  COLO_MESSAGE_VMSTATE_SIZE, &local_err);
         if (local_err) {
@@ -XXX,XX +XXX,XX @@ void *colo_process_incoming_thread(void *opaque)
         }
 
         qemu_mutex_lock_iothread();
-        qemu_system_reset(SHUTDOWN_CAUSE_NONE);
         vmstate_loading = true;
-        if (qemu_loadvm_state(fb) < 0) {
-            error_report("COLO: loadvm failed");
+        ret = qemu_load_device_state(fb);
+        if (ret < 0) {
+            error_report("COLO: load device state failed");
             qemu_mutex_unlock_iothread();
             goto out;
         }
diff --git a/migration/savevm.c b/migration/savevm.c
index XXXXXXX..XXXXXXX 100644
--- a/migration/savevm.c
+++ b/migration/savevm.c
@@ -XXX,XX +XXX,XX @@ done:
     return ret;
 }
 
-static int qemu_save_device_state(QEMUFile *f)
+void qemu_savevm_live_state(QEMUFile *f)
 {
-    SaveStateEntry *se;
+    /* save QEMU_VM_SECTION_END section */
+    qemu_savevm_state_complete_precopy(f, true, false);
+    qemu_put_byte(f, QEMU_VM_EOF);
+}
 
-    qemu_put_be32(f, QEMU_VM_FILE_MAGIC);
-    qemu_put_be32(f, QEMU_VM_FILE_VERSION);
+int qemu_save_device_state(QEMUFile *f)
+{
+    SaveStateEntry *se;
 
+    if (!migration_in_colo_state()) {
+        qemu_put_be32(f, QEMU_VM_FILE_MAGIC);
+        qemu_put_be32(f, QEMU_VM_FILE_VERSION);
+    }
     cpu_synchronize_all_states();
 
     QTAILQ_FOREACH(se, &savevm_state.handlers, entry) {
@@ -XXX,XX +XXX,XX @@ enum LoadVMExitCodes {
     LOADVM_QUIT     =  1,
 };
 
-static int qemu_loadvm_state_main(QEMUFile *f, MigrationIncomingState *mis);
-
 /* ------ incoming postcopy messages ------ */
 /* 'advise' arrives before any transfers just to tell us that a postcopy
  * *might* happen - it might be skipped if precopy transferred everything
@@ -XXX,XX +XXX,XX @@ static bool postcopy_pause_incoming(MigrationIncomingState *mis)
     return true;
 }
 
-static int qemu_loadvm_state_main(QEMUFile *f, MigrationIncomingState *mis)
+int qemu_loadvm_state_main(QEMUFile *f, MigrationIncomingState *mis)
 {
     uint8_t section_type;
     int ret = 0;
@@ -XXX,XX +XXX,XX @@ int qemu_loadvm_state(QEMUFile *f)
     return ret;
 }
 
+int qemu_load_device_state(QEMUFile *f)
+{
+    MigrationIncomingState *mis = migration_incoming_get_current();
+    int ret;
+
+    /* Load QEMU_VM_SECTION_FULL section */
+    ret = qemu_loadvm_state_main(f, mis);
+    if (ret < 0) {
+        error_report("Failed to load device state: %d", ret);
+        return ret;
+    }
+
+    cpu_synchronize_all_post_init();
+    return 0;
+}
+
 int save_snapshot(const char *name, Error **errp)
 {
     BlockDriverState *bs, *bs1;
diff --git a/migration/savevm.h b/migration/savevm.h
index XXXXXXX..XXXXXXX 100644
--- a/migration/savevm.h
+++ b/migration/savevm.h
@@ -XXX,XX +XXX,XX @@ void qemu_savevm_send_postcopy_ram_discard(QEMUFile *f, const char *name,
                                            uint64_t *start_list,
                                            uint64_t *length_list);
 void qemu_savevm_send_colo_enable(QEMUFile *f);
+void qemu_savevm_live_state(QEMUFile *f);
+int qemu_save_device_state(QEMUFile *f);
 
 int qemu_loadvm_state(QEMUFile *f);
 void qemu_loadvm_state_cleanup(void);
+int qemu_loadvm_state_main(QEMUFile *f, MigrationIncomingState *mis);
+int qemu_load_device_state(QEMUFile *f);
 
 #endif
-- 
2.5.0

From: zhanghailiang <zhang.zhanghailiang@huawei.com>

Don't need to flush all VM's ram from cache, only
flush the dirty pages since last checkpoint

Signed-off-by: Li Zhijian <lizhijian@cn.fujitsu.com>
Signed-off-by: Zhang Chen <zhangckid@gmail.com>
Signed-off-by: Zhang Chen <chen.zhang@intel.com>
Signed-off-by: zhanghailiang <zhang.zhanghailiang@huawei.com>
Reviewed-by: Dr. David Alan Gilbert <dgilbert@redhat.com>
Signed-off-by: Jason Wang <jasowang@redhat.com>
---
 migration/ram.c | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/migration/ram.c b/migration/ram.c
index XXXXXXX..XXXXXXX 100644
--- a/migration/ram.c
+++ b/migration/ram.c
@@ -XXX,XX +XXX,XX @@ int colo_init_ram_cache(void)
     }
     ram_state = g_new0(RAMState, 1);
     ram_state->migration_dirty_pages = 0;
+    memory_global_dirty_log_start();
 
     return 0;
 
@@ -XXX,XX +XXX,XX @@ void colo_release_ram_cache(void)
 {
     RAMBlock *block;
 
+    memory_global_dirty_log_stop();
     RAMBLOCK_FOREACH_MIGRATABLE(block) {
         g_free(block->bmap);
         block->bmap = NULL;
@@ -XXX,XX +XXX,XX @@ static void colo_flush_ram_cache(void)
     void *src_host;
     unsigned long offset = 0;
 
+    memory_global_dirty_log_sync();
+    rcu_read_lock();
+    RAMBLOCK_FOREACH_MIGRATABLE(block) {
+        migration_bitmap_sync_range(ram_state, block, 0, block->used_length);
+    }
+    rcu_read_unlock();
+
     trace_colo_flush_ram_cache_begin(ram_state->migration_dirty_pages);
     rcu_read_lock();
     block = QLIST_FIRST_RCU(&ram_list.blocks);
-- 
2.5.0

From: Zhang Chen <zhangckid@gmail.com>

Filter needs to process the event of checkpoint/failover or
other event passed by COLO frame.

Signed-off-by: zhanghailiang <zhang.zhanghailiang@huawei.com>
Signed-off-by: Zhang Chen <zhangckid@gmail.com>
Signed-off-by: Zhang Chen <chen.zhang@intel.com>
Signed-off-by: Jason Wang <jasowang@redhat.com>
---
 include/net/filter.h |  5 +++++
 net/filter.c         | 17 +++++++++++++++++
 net/net.c            | 19 +++++++++++++++++++
 3 files changed, 41 insertions(+)

diff --git a/include/net/filter.h b/include/net/filter.h
index XXXXXXX..XXXXXXX 100644
--- a/include/net/filter.h
+++ b/include/net/filter.h
@@ -XXX,XX +XXX,XX @@ typedef ssize_t (FilterReceiveIOV)(NetFilterState *nc,
 
 typedef void (FilterStatusChanged) (NetFilterState *nf, Error **errp);
 
+typedef void (FilterHandleEvent) (NetFilterState *nf, int event, Error **errp);
+
 typedef struct NetFilterClass {
     ObjectClass parent_class;
 
@@ -XXX,XX +XXX,XX @@ typedef struct NetFilterClass {
     FilterSetup *setup;
     FilterCleanup *cleanup;
     FilterStatusChanged *status_changed;
+    FilterHandleEvent *handle_event;
     /* mandatory */
     FilterReceiveIOV *receive_iov;
 } NetFilterClass;
@@ -XXX,XX +XXX,XX @@ ssize_t qemu_netfilter_pass_to_next(NetClientState *sender,
                                     int iovcnt,
                                     void *opaque);
 
+void colo_notify_filters_event(int event, Error **errp);
+
 #endif /* QEMU_NET_FILTER_H */
diff --git a/net/filter.c b/net/filter.c
index XXXXXXX..XXXXXXX 100644
--- a/net/filter.c
+++ b/net/filter.c
@@ -XXX,XX +XXX,XX @@
 #include "net/vhost_net.h"
 #include "qom/object_interfaces.h"
 #include "qemu/iov.h"
+#include "net/colo.h"
+#include "migration/colo.h"
 
 static inline bool qemu_can_skip_netfilter(NetFilterState *nf)
 {
@@ -XXX,XX +XXX,XX @@ static void netfilter_finalize(Object *obj)
     g_free(nf->netdev_id);
 }
 
+static void default_handle_event(NetFilterState *nf, int event, Error **errp)
+{
+    switch (event) {
+    case COLO_EVENT_CHECKPOINT:
+        break;
+    case COLO_EVENT_FAILOVER:
+        object_property_set_str(OBJECT(nf), "off", "status", errp);
+        break;
+    default:
+        break;
+    }
+}
+
 static void netfilter_class_init(ObjectClass *oc, void *data)
 {
     UserCreatableClass *ucc = USER_CREATABLE_CLASS(oc);
+    NetFilterClass *nfc = NETFILTER_CLASS(oc);
 
     ucc->complete = netfilter_complete;
+    nfc->handle_event = default_handle_event;
 }
 
 static const TypeInfo netfilter_info = {
diff --git a/net/net.c b/net/net.c
index XXXXXXX..XXXXXXX 100644
--- a/net/net.c
+++ b/net/net.c
@@ -XXX,XX +XXX,XX @@ void hmp_info_network(Monitor *mon, const QDict *qdict)
     }
 }
 
+void colo_notify_filters_event(int event, Error **errp)
+{
+    NetClientState *nc;
+    NetFilterState *nf;
+    NetFilterClass *nfc = NULL;
+    Error *local_err = NULL;
+
+    QTAILQ_FOREACH(nc, &net_clients, next) {
+        QTAILQ_FOREACH(nf, &nc->filters, next) {
+            nfc = NETFILTER_GET_CLASS(OBJECT(nf));
+            nfc->handle_event(nf, event, &local_err);
+            if (local_err) {
+                error_propagate(errp, local_err);
+                return;
+            }
+        }
+    }
+}
+
 void qmp_set_link(const char *name, bool up, Error **errp)
 {
     NetClientState *ncs[MAX_QUEUE_NUM];
-- 
2.5.0

From: Zhang Chen <zhangckid@gmail.com>

After one round of checkpoint, the states between PVM and SVM
become consistent, so it is unnecessary to adjust the sequence
of net packets for old connections, besides, while failover
happens, filter-rewriter will into failover mode that needn't
handle the new TCP connection.

Signed-off-by: zhanghailiang <zhang.zhanghailiang@huawei.com>
Signed-off-by: Zhang Chen <zhangckid@gmail.com>
Signed-off-by: Zhang Chen <chen.zhang@intel.com>
Signed-off-by: Jason Wang <jasowang@redhat.com>
---
 net/colo-compare.c    | 12 +++++------
 net/colo.c            |  8 ++++++++
 net/colo.h            |  2 ++
 net/filter-rewriter.c | 57 +++++++++++++++++++++++++++++++++++++++++++++++++++
 4 files changed, 73 insertions(+), 6 deletions(-)

diff --git a/net/colo-compare.c b/net/colo-compare.c
index XXXXXXX..XXXXXXX 100644
--- a/net/colo-compare.c
+++ b/net/colo-compare.c
@@ -XXX,XX +XXX,XX @@ enum {
     SECONDARY_IN,
 };
 
+static void colo_compare_inconsistency_notify(void)
+{
+    notifier_list_notify(&colo_compare_notifiers,
+                migrate_get_current());
+}
+
 static int compare_chr_send(CompareState *s,
                             const uint8_t *buf,
                             uint32_t size,
@@ -XXX,XX +XXX,XX @@ static bool colo_mark_tcp_pkt(Packet *ppkt, Packet *spkt,
     return false;
 }
 
-static void colo_compare_inconsistency_notify(void)
-{
-    notifier_list_notify(&colo_compare_notifiers,
-                migrate_get_current());
-}
-
 static void colo_compare_tcp(CompareState *s, Connection *conn)
 {
     Packet *ppkt = NULL, *spkt = NULL;
diff --git a/net/colo.c b/net/colo.c
index XXXXXXX..XXXXXXX 100644
--- a/net/colo.c
+++ b/net/colo.c
@@ -XXX,XX +XXX,XX @@ Connection *connection_get(GHashTable *connection_track_table,
 
     return conn;
 }
+
+bool connection_has_tracked(GHashTable *connection_track_table,
+                            ConnectionKey *key)
+{
+    Connection *conn = g_hash_table_lookup(connection_track_table, key);
+
+    return conn ? true : false;
+}
diff --git a/net/colo.h b/net/colo.h
index XXXXXXX..XXXXXXX 100644
--- a/net/colo.h
+++ b/net/colo.h
@@ -XXX,XX +XXX,XX @@ void connection_destroy(void *opaque);
 Connection *connection_get(GHashTable *connection_track_table,
                            ConnectionKey *key,
                            GQueue *conn_list);
+bool connection_has_tracked(GHashTable *connection_track_table,
+                            ConnectionKey *key);
 void connection_hashtable_reset(GHashTable *connection_track_table);
 Packet *packet_new(const void *data, int size, int vnet_hdr_len);
 void packet_destroy(void *opaque, void *user_data);
diff --git a/net/filter-rewriter.c b/net/filter-rewriter.c
index XXXXXXX..XXXXXXX 100644
--- a/net/filter-rewriter.c
+++ b/net/filter-rewriter.c
@@ -XXX,XX +XXX,XX @@
 #include "qemu/main-loop.h"
 #include "qemu/iov.h"
 #include "net/checksum.h"
+#include "net/colo.h"
+#include "migration/colo.h"
 
 #define FILTER_COLO_REWRITER(obj) \
     OBJECT_CHECK(RewriterState, (obj), TYPE_FILTER_REWRITER)
 
 #define TYPE_FILTER_REWRITER "filter-rewriter"
+#define FAILOVER_MODE_ON  true
+#define FAILOVER_MODE_OFF false
 
 typedef struct RewriterState {
     NetFilterState parent_obj;
@@ -XXX,XX +XXX,XX @@ typedef struct RewriterState {
     /* hashtable to save connection */
     GHashTable *connection_track_table;
     bool vnet_hdr;
+    bool failover_mode;
 } RewriterState;
 
+static void filter_rewriter_failover_mode(RewriterState *s)
+{
+    s->failover_mode = FAILOVER_MODE_ON;
+}
+
 static void filter_rewriter_flush(NetFilterState *nf)
 {
     RewriterState *s = FILTER_COLO_REWRITER(nf);
@@ -XXX,XX +XXX,XX @@ static ssize_t colo_rewriter_receive_iov(NetFilterState *nf,
              */
             reverse_connection_key(&key);
         }
+
+        /* After failover we needn't change new TCP packet */
+        if (s->failover_mode &&
+            !connection_has_tracked(s->connection_track_table, &key)) {
+            goto out;
+        }
+
         conn = connection_get(s->connection_track_table,
                               &key,
                               NULL);
@@ -XXX,XX +XXX,XX @@ static ssize_t colo_rewriter_receive_iov(NetFilterState *nf,
         }
     }
 
+out:
     packet_destroy(pkt, NULL);
     pkt = NULL;
     return 0;
 }
 
+static void reset_seq_offset(gpointer key, gpointer value, gpointer user_data)
+{
+    Connection *conn = (Connection *)value;
+
+    conn->offset = 0;
+}
+
+static gboolean offset_is_nonzero(gpointer key,
+                                  gpointer value,
+                                  gpointer user_data)
+{
+    Connection *conn = (Connection *)value;
+
+    return conn->offset ? true : false;
+}
+
+static void colo_rewriter_handle_event(NetFilterState *nf, int event,
+                                       Error **errp)
+{
+    RewriterState *rs = FILTER_COLO_REWRITER(nf);
+
+    switch (event) {
+    case COLO_EVENT_CHECKPOINT:
+        g_hash_table_foreach(rs->connection_track_table,
+                            reset_seq_offset, NULL);
+        break;
+    case COLO_EVENT_FAILOVER:
+        if (!g_hash_table_find(rs->connection_track_table,
+                              offset_is_nonzero, NULL)) {
+            filter_rewriter_failover_mode(rs);
+        }
+        break;
+    default:
+        break;
+    }
+}
+
 static void colo_rewriter_cleanup(NetFilterState *nf)
 {
     RewriterState *s = FILTER_COLO_REWRITER(nf);
@@ -XXX,XX +XXX,XX @@ static void filter_rewriter_init(Object *obj)
     RewriterState *s = FILTER_COLO_REWRITER(obj);
 
     s->vnet_hdr = false;
+    s->failover_mode = FAILOVER_MODE_OFF;
     object_property_add_bool(obj, "vnet_hdr_support",
                              filter_rewriter_get_vnet_hdr,
                              filter_rewriter_set_vnet_hdr, NULL);
@@ -XXX,XX +XXX,XX @@ static void colo_rewriter_class_init(ObjectClass *oc, void *data)
     nfc->setup = colo_rewriter_setup;
     nfc->cleanup = colo_rewriter_cleanup;
     nfc->receive_iov = colo_rewriter_receive_iov;
+    nfc->handle_event = colo_rewriter_handle_event;
 }
 
 static const TypeInfo colo_rewriter_info = {
-- 
2.5.0

From: zhanghailiang <zhang.zhanghailiang@huawei.com>

Notify all net filters about the checkpoint and failover event.

Signed-off-by: zhanghailiang <zhang.zhanghailiang@huawei.com>
Reviewed-by: Dr. David Alan Gilbert <dgilbert@redhat.com>
Signed-off-by: Jason Wang <jasowang@redhat.com>
---
 migration/colo.c | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/migration/colo.c b/migration/colo.c
index XXXXXXX..XXXXXXX 100644
--- a/migration/colo.c
+++ b/migration/colo.c
@@ -XXX,XX +XXX,XX @@
 #include "qapi/qapi-events-migration.h"
 #include "qapi/qmp/qerror.h"
 #include "sysemu/cpus.h"
+#include "net/filter.h"
 
 static bool vmstate_loading;
 static Notifier packets_compare_notifier;
@@ -XXX,XX +XXX,XX @@ static void secondary_vm_do_failover(void)
         error_report_err(local_err);
     }
 
+    /* Notify all filters of all NIC to do checkpoint */
+    colo_notify_filters_event(COLO_EVENT_FAILOVER, &local_err);
+    if (local_err) {
+        error_report_err(local_err);
+    }
+
     if (!autostart) {
         error_report("\"-S\" qemu option will be ignored in secondary side");
         /* recover runstate to normal migration finish state */
@@ -XXX,XX +XXX,XX @@ void *colo_process_incoming_thread(void *opaque)
             goto out;
         }
 
+        /* Notify all filters of all NIC to do checkpoint */
+        colo_notify_filters_event(COLO_EVENT_CHECKPOINT, &local_err);
+
+        if (local_err) {
+            qemu_mutex_unlock_iothread();
+            goto out;
+        }
+
         vmstate_loading = false;
         vm_start();
         trace_colo_vm_state_change("stop", "run");
-- 
2.5.0

From: zhanghailiang <zhang.zhanghailiang@huawei.com>

COLO thread may sleep at qemu_sem_wait(&s->colo_checkpoint_sem),
while failover works begin, It's better to wakeup it to quick
the process.

diff --git a/migration/colo.c b/migration/colo.c
index XXXXXXX..XXXXXXX 100644
--- a/migration/colo.c
+++ b/migration/colo.c
@@ -XXX,XX +XXX,XX @@ static void primary_vm_do_failover(void)
 
     migrate_set_state(&s->state, MIGRATION_STATUS_COLO,
                       MIGRATION_STATUS_COMPLETED);
+    /*
+     * kick COLO thread which might wait at
+     * qemu_sem_wait(&s->colo_checkpoint_sem).
+     */
+    colo_checkpoint_notify(migrate_get_current());
 
     /*
      * Wake up COLO thread which may blocked in recv() or send(),
@@ -XXX,XX +XXX,XX @@ static void colo_process_checkpoint(MigrationState *s)
 
         qemu_sem_wait(&s->colo_checkpoint_sem);
 
+        if (s->state != MIGRATION_STATUS_COLO) {
+            goto out;
+        }
         ret = colo_do_checkpoint_transaction(s, bioc, fb);
         if (ret < 0) {
             goto out;
-- 
2.5.0

From: Zhang Chen <chen.zhang@intel.com>

This diagram make user better understand COLO.
Suggested by Markus Armbruster.

Signed-off-by: Zhang Chen <zhangckid@gmail.com>
Signed-off-by: Zhang Chen <chen.zhang@intel.com>
Signed-off-by: Jason Wang <jasowang@redhat.com>
---
 docs/COLO-FT.txt | 34 ++++++++++++++++++++++++++++++++++
 1 file changed, 34 insertions(+)

diff --git a/docs/COLO-FT.txt b/docs/COLO-FT.txt
index XXXXXXX..XXXXXXX 100644
--- a/docs/COLO-FT.txt
+++ b/docs/COLO-FT.txt
@@ -XXX,XX +XXX,XX @@ Note:
 HeartBeat has not been implemented yet, so you need to trigger failover process
 by using 'x-colo-lost-heartbeat' command.
 
+== COLO operation status ==
+
++-----------------+
+|                 |
+|    Start COLO   |
+|                 |
++--------+--------+
+         |
+         |  Main qmp command:
+         |  migrate-set-capabilities with x-colo
+         |  migrate
+         |
+         v
++--------+--------+
+|                 |
+|  COLO running   |
+|                 |
++--------+--------+
+         |
+         |  Main qmp command:
+         |  x-colo-lost-heartbeat
+         |  or
+         |  some error happened
+         v
++--------+--------+
+|                 |  send qmp event:
+|  COLO failover  |  COLO_EXIT
+|                 |
++-----------------+
+
+COLO use the qmp command to switch and report operation status.
+The diagram just shows the main qmp command, you can get the detail
+in test procedure.
+
 == Test procedure ==
 1. Startup qemu
 Primary:
-- 
2.5.0

From: liujunjie <liujunjie23@huawei.com>

Before, we did not clear callback like handle_output when delete
the virtqueue which may result be segmentfault.
The scene is as follows:
1. Start a vm with multiqueue vhost-net,
2. then we write VIRTIO_PCI_GUEST_FEATURES in PCI configuration to
triger multiqueue disable in this vm which will delete the virtqueue.
In this step, the tx_bh is deleted but the callback virtio_net_handle_tx_bh
still exist.
3. Finally, we write VIRTIO_PCI_QUEUE_NOTIFY in PCI configuration to
notify the deleted virtqueue. In this way, virtio_net_handle_tx_bh
will be called and qemu will be crashed.

Although the way described above is uncommon, we had better reinforce it.

CC: qemu-stable@nongnu.org
Signed-off-by: liujunjie <liujunjie23@huawei.com>
Signed-off-by: Jason Wang <jasowang@redhat.com>
---
 hw/virtio/virtio.c | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/hw/virtio/virtio.c b/hw/virtio/virtio.c
index XXXXXXX..XXXXXXX 100644
--- a/hw/virtio/virtio.c
+++ b/hw/virtio/virtio.c
@@ -XXX,XX +XXX,XX @@ void virtio_del_queue(VirtIODevice *vdev, int n)
 
     vdev->vq[n].vring.num = 0;
     vdev->vq[n].vring.num_default = 0;
+    vdev->vq[n].handle_output = NULL;
+    vdev->vq[n].handle_aio_output = NULL;
 }
 
 static void virtio_set_isr(VirtIODevice *vdev, int value)
-- 
2.5.0

In ne2000_receive(), we try to assign size_ to size which converts
from size_t to integer. This will cause troubles when size_ is greater
INT_MAX, this will lead a negative value in size and it can then pass
the check of size < MIN_BUF_SIZE which may lead out of bound access of
for both buf and buf1.

Fixing by converting the type of size to size_t.

CC: qemu-stable@nongnu.org
Reported-by: Daniel Shapira <daniel@twistlock.com>
Reviewed-by: Michael S. Tsirkin <mst@redhat.com>
Signed-off-by: Jason Wang <jasowang@redhat.com>
---
 hw/net/ne2000.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/hw/net/ne2000.c b/hw/net/ne2000.c
index XXXXXXX..XXXXXXX 100644
--- a/hw/net/ne2000.c
+++ b/hw/net/ne2000.c
@@ -XXX,XX +XXX,XX @@ static int ne2000_buffer_full(NE2000State *s)
 ssize_t ne2000_receive(NetClientState *nc, const uint8_t *buf, size_t size_)
 {
     NE2000State *s = qemu_get_nic_opaque(nc);
-    int size = size_;
+    size_t size = size_;
     uint8_t *p;
     unsigned int total_len, next, avail, len, index, mcast_idx;
     uint8_t buf1[60];
@@ -XXX,XX +XXX,XX @@ ssize_t ne2000_receive(NetClientState *nc, const uint8_t *buf, size_t size_)
         { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff };
 
 #if defined(DEBUG_NE2000)
-    printf("NE2000: received len=%d\n", size);
+    printf("NE2000: received len=%zu\n", size);
 #endif
 
     if (s->cmd & E8390_STOP || ne2000_buffer_full(s))
-- 
2.5.0

In rtl8139_do_receive(), we try to assign size_ to size which converts
from size_t to integer. This will cause troubles when size_ is greater
INT_MAX, this will lead a negative value in size and it can then pass
the check of size < MIN_BUF_SIZE which may lead out of bound access of
for both buf and buf1.

Fixing by converting the type of size to size_t.

CC: qemu-stable@nongnu.org
Reported-by: Daniel Shapira <daniel@twistlock.com>
Reviewed-by: Michael S. Tsirkin <mst@redhat.com>
Signed-off-by: Jason Wang <jasowang@redhat.com>
---
 hw/net/rtl8139.c | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/hw/net/rtl8139.c b/hw/net/rtl8139.c
index XXXXXXX..XXXXXXX 100644
--- a/hw/net/rtl8139.c
+++ b/hw/net/rtl8139.c
@@ -XXX,XX +XXX,XX @@ static ssize_t rtl8139_do_receive(NetClientState *nc, const uint8_t *buf, size_t
     RTL8139State *s = qemu_get_nic_opaque(nc);
     PCIDevice *d = PCI_DEVICE(s);
     /* size is the length of the buffer passed to the driver */
-    int size = size_;
+    size_t size = size_;
     const uint8_t *dot1q_buf = NULL;
 
     uint32_t packet_header = 0;
@@ -XXX,XX +XXX,XX @@ static ssize_t rtl8139_do_receive(NetClientState *nc, const uint8_t *buf, size_t
     static const uint8_t broadcast_macaddr[6] =
         { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff };
 
-    DPRINTF(">>> received len=%d\n", size);
+    DPRINTF(">>> received len=%zu\n", size);
 
     /* test if board clock is stopped */
     if (!s->clock_enabled)
@@ -XXX,XX +XXX,XX @@ static ssize_t rtl8139_do_receive(NetClientState *nc, const uint8_t *buf, size_t
 
         if (size+4 > rx_space)
         {
-            DPRINTF("C+ Rx mode : descriptor %d size %d received %d + 4\n",
+            DPRINTF("C+ Rx mode : descriptor %d size %d received %zu + 4\n",
                 descriptor, rx_space, size);
 
             s->IntrStatus |= RxOverflow;
@@ -XXX,XX +XXX,XX @@ static ssize_t rtl8139_do_receive(NetClientState *nc, const uint8_t *buf, size_t
         if (avail != 0 && RX_ALIGN(size + 8) >= avail)
         {
             DPRINTF("rx overflow: rx buffer length %d head 0x%04x "
-                "read 0x%04x === available 0x%04x need 0x%04x\n",
+                "read 0x%04x === available 0x%04x need 0x%04zx\n",
                 s->RxBufferSize, s->RxBufAddr, s->RxBufPtr, avail, size + 8);
 
             s->IntrStatus |= RxOverflow;
-- 
2.5.0

In pcnet_receive(), we try to assign size_ to size which converts from
size_t to integer. This will cause troubles when size_ is greater
INT_MAX, this will lead a negative value in size and it can then pass
the check of size < MIN_BUF_SIZE which may lead out of bound access
for both buf and buf1.

Fixing by converting the type of size to size_t.

CC: qemu-stable@nongnu.org
Reported-by: Daniel Shapira <daniel@twistlock.com>
Reviewed-by: Michael S. Tsirkin <mst@redhat.com>
Signed-off-by: Jason Wang <jasowang@redhat.com>
---
 hw/net/pcnet.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/hw/net/pcnet.c b/hw/net/pcnet.c
index XXXXXXX..XXXXXXX 100644
--- a/hw/net/pcnet.c
+++ b/hw/net/pcnet.c
@@ -XXX,XX +XXX,XX @@ ssize_t pcnet_receive(NetClientState *nc, const uint8_t *buf, size_t size_)
     uint8_t buf1[60];
     int remaining;
     int crc_err = 0;
-    int size = size_;
+    size_t size = size_;
 
     if (CSR_DRX(s) || CSR_STOP(s) || CSR_SPND(s) || !size ||
         (CSR_LOOP(s) && !s->looptest)) {
         return -1;
     }
 #ifdef PCNET_DEBUG
-    printf("pcnet_receive size=%d\n", size);
+    printf("pcnet_receive size=%zu\n", size);
 #endif
 
     /* if too small buffer, then expand it */
-- 
2.5.0

From: Martin Wilck <mwilck@suse.com>

The e1000 emulation silently discards RX packets if there's
insufficient space in the ring buffer. This leads to errors
on higher-level protocols in the guest, with no indication
about the error cause.

This patch increments the "Missed Packets Count" (MPC) and
"Receive No Buffers Count" (RNBC) HW counters in this case.
As the emulation has no FIFO for buffering packets that can't
immediately be pushed to the guest, these two registers are
practically equivalent (see 10.2.7.4, 10.2.7.33 in
https://www.intel.com/content/www/us/en/embedded/products/networking/82574l-gbe-controller-datasheet.html).

On a Linux guest, the register content  will be reflected in
the "rx_missed_errors" and "rx_no_buffer_count" stats from
"ethtool -S", and in the "missed" stat from "ip -s -s link show",
giving at least some hint about the error cause inside the guest.

If the cause is known, problems like this can often be avoided
easily, by increasing the number of RX descriptors in the guest
e1000 driver (e.g under Linux, "e1000.RxDescriptors=1024").

The patch also adds a qemu trace message for this condition.

Signed-off-by: Martin Wilck <mwilck@suse.com>
Signed-off-by: Jason Wang <jasowang@redhat.com>
---
 hw/net/e1000.c      | 16 +++++++++++++---
 hw/net/trace-events |  3 +++
 2 files changed, 16 insertions(+), 3 deletions(-)

diff --git a/hw/net/e1000.c b/hw/net/e1000.c
index XXXXXXX..XXXXXXX 100644
--- a/hw/net/e1000.c
+++ b/hw/net/e1000.c
@@ -XXX,XX +XXX,XX @@
 #include "qemu/range.h"
 
 #include "e1000x_common.h"
+#include "trace.h"
 
 static const uint8_t bcast[] = {0xff, 0xff, 0xff, 0xff, 0xff, 0xff};
 
@@ -XXX,XX +XXX,XX @@ static uint64_t rx_desc_base(E1000State *s)
     return (bah << 32) + bal;
 }
 
+static void
+e1000_receiver_overrun(E1000State *s, size_t size)
+{
+    trace_e1000_receiver_overrun(size, s->mac_reg[RDH], s->mac_reg[RDT]);
+    e1000x_inc_reg_if_not_full(s->mac_reg, RNBC);
+    e1000x_inc_reg_if_not_full(s->mac_reg, MPC);
+    set_ics(s, 0, E1000_ICS_RXO);
+}
+
 static ssize_t
 e1000_receive_iov(NetClientState *nc, const struct iovec *iov, int iovcnt)
 {
@@ -XXX,XX +XXX,XX @@ e1000_receive_iov(NetClientState *nc, const struct iovec *iov, int iovcnt)
     desc_offset = 0;
     total_size = size + e1000x_fcs_len(s->mac_reg);
     if (!e1000_has_rxbufs(s, total_size)) {
-            set_ics(s, 0, E1000_ICS_RXO);
-            return -1;
+        e1000_receiver_overrun(s, total_size);
+        return -1;
     }
     do {
         desc_size = total_size - desc_offset;
@@ -XXX,XX +XXX,XX @@ e1000_receive_iov(NetClientState *nc, const struct iovec *iov, int iovcnt)
             rdh_start >= s->mac_reg[RDLEN] / sizeof(desc)) {
             DBGOUT(RXERR, "RDH wraparound @%x, RDT %x, RDLEN %x\n",
                    rdh_start, s->mac_reg[RDT], s->mac_reg[RDLEN]);
-            set_ics(s, 0, E1000_ICS_RXO);
+            e1000_receiver_overrun(s, total_size);
             return -1;
         }
     } while (desc_offset < total_size);
diff --git a/hw/net/trace-events b/hw/net/trace-events
index XXXXXXX..XXXXXXX 100644
--- a/hw/net/trace-events
+++ b/hw/net/trace-events
@@ -XXX,XX +XXX,XX @@ net_rx_pkt_rss_ip6_ex(void) "Calculating IPv6/EX RSS  hash"
 net_rx_pkt_rss_hash(size_t rss_length, uint32_t rss_hash) "RSS hash for %zu bytes: 0x%X"
 net_rx_pkt_rss_add_chunk(void* ptr, size_t size, size_t input_offset) "Add RSS chunk %p, %zu bytes, RSS input offset %zu bytes"
 
+# hw/net/e1000.c
+e1000_receiver_overrun(size_t s, uint32_t rdh, uint32_t rdt) "Receiver overrun: dropped packet of %lu bytes, RDH=%u, RDT=%u"
+
 # hw/net/e1000x_common.c
 e1000x_rx_can_recv_disabled(bool link_up, bool rx_enabled, bool pci_master) "link_up: %d, rx_enabled %d, pci_master %d"
 e1000x_vlan_is_vlan_pkt(bool is_vlan_pkt, uint16_t eth_proto, uint16_t vet) "Is VLAN packet: %d, ETH proto: 0x%X, VET: 0x%X"
-- 
2.5.0

The following changes since commit f8d75e10d3e0033a0a29a7a7e4777a4fbc17a016:

Merge remote-tracking branch 'remotes/legoater/tags/pull-ppc-20220112' into staging (2022-01-13 11:18:24 +0000)

are available in the git repository at:

https://github.com/jasowang/qemu.git tags/net-pull-request

for you to fetch changes up to 818692f0a01587d02220916b31d5bb8e7dced611:

net/vmnet: update MAINTAINERS list (2022-01-14 12:58:19 +0800)

----------------------------------------------------------------

Changes since V2:

- Try to make vmnet work on some old mac version

----------------------------------------------------------------
Peter Foley (2):
      net/tap: Set return code on failure
      net: Fix uninitialized data usage

Philippe Mathieu-Daudé (1):
      hw/net/vmxnet3: Log guest-triggerable errors using LOG_GUEST_ERROR

Rao Lei (1):
      net/filter: Optimize filter_send to coroutine

Vladislav Yaroshchuk (7):
      net/vmnet: add vmnet dependency and customizable option
      net/vmnet: add vmnet backends to qapi/net
      net/vmnet: implement shared mode (vmnet-shared)
      net/vmnet: implement host mode (vmnet-host)
      net/vmnet: implement bridged mode (vmnet-bridged)
      net/vmnet: update qemu-options.hx
      net/vmnet: update MAINTAINERS list

Zhang Chen (2):
      net/colo-compare.c: Optimize compare order for performance
      net/colo-compare.c: Update the default value comments

From: Philippe Mathieu-Daudé <philmd@redhat.com>

The "Interrupt Cause" register (VMXNET3_REG_ICR) is read-only.
Write accesses are ignored. Log them with as LOG_GUEST_ERROR
instead of aborting:

[R +0.239743] writeq 0xe0002031 0x46291a5a55460800
  ERROR:hw/net/vmxnet3.c:1819:vmxnet3_io_bar1_write: code should not be reached
  Thread 1 "qemu-system-i38" received signal SIGABRT, Aborted.
  (gdb) bt
  #3  0x74c397d3 in __GI_abort () at abort.c:79
  #4  0x76d3cd4c in g_assertion_message (domain=<optimized out>, file=<optimized out>, line=<optimized out>, func=<optimized out>, message=<optimized out>) at ../glib/gtestutils.c:3223
  #5  0x76d9d45f in g_assertion_message_expr
      (domain=0x0, file=0x59fc2e53 "hw/net/vmxnet3.c", line=1819, func=0x59fc11e0 <__func__.vmxnet3_io_bar1_write> "vmxnet3_io_bar1_write", expr=<optimized out>)
      at ../glib/gtestutils.c:3249
  #6  0x57e80a3a in vmxnet3_io_bar1_write (opaque=0x62814100, addr=56, val=70, size=4) at hw/net/vmxnet3.c:1819
  #7  0x58c2d894 in memory_region_write_accessor (mr=0x62816b90, addr=56, value=0x7fff9450, size=4, shift=0, mask=4294967295, attrs=...) at softmmu/memory.c:492
  #8  0x58c2d1d2 in access_with_adjusted_size (addr=56, value=0x7fff9450, size=1, access_size_min=4, access_size_max=4, access_fn=
      0x58c2d290 <memory_region_write_accessor>, mr=0x62816b90, attrs=...) at softmmu/memory.c:554
  #9  0x58c2bae7 in memory_region_dispatch_write (mr=0x62816b90, addr=56, data=70, op=MO_8, attrs=...) at softmmu/memory.c:1504
  #10 0x58bfd034 in flatview_write_continue (fv=0x606000181700, addr=0xe0002038, attrs=..., ptr=0x7fffb9e0, len=1, addr1=56, l=1, mr=0x62816b90)
      at softmmu/physmem.c:2782
  #11 0x58beba00 in flatview_write (fv=0x606000181700, addr=0xe0002031, attrs=..., buf=0x7fffb9e0, len=8) at softmmu/physmem.c:2822
  #12 0x58beb589 in address_space_write (as=0x608000015f20, addr=0xe0002031, attrs=..., buf=0x7fffb9e0, len=8) at softmmu/physmem.c:2914

Reported-by: Dike <dike199774@qq.com>
Reported-by: Duhao <504224090@qq.com>
BugLink: https://bugzilla.redhat.com/show_bug.cgi?id=2032932
Signed-off-by: Philippe Mathieu-Daudé <philmd@redhat.com>
Signed-off-by: Jason Wang <jasowang@redhat.com>
---
 hw/net/vmxnet3.c | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/hw/net/vmxnet3.c b/hw/net/vmxnet3.c
index XXXXXXX..XXXXXXX 100644
--- a/hw/net/vmxnet3.c
+++ b/hw/net/vmxnet3.c
@@ -XXX,XX +XXX,XX @@ vmxnet3_io_bar1_write(void *opaque,
     case VMXNET3_REG_ICR:
         VMW_CBPRN("Write BAR1 [VMXNET3_REG_ICR] = %" PRIx64 ", size %d",
                   val, size);
-        g_assert_not_reached();
+        qemu_log_mask(LOG_GUEST_ERROR,
+                      "%s: write to read-only register VMXNET3_REG_ICR\n",
+                      TYPE_VMXNET3);
         break;
 
     /* Event Cause Register */
-- 
2.7.4

From: Peter Foley <pefoley@google.com>

e.g.
1109 15:16:20.151506 Uninitialized bytes in ioctl_common_pre at offset 0 inside [0x7ffc516af9b8, 4)
 1109 15:16:20.151659 ==588974==WARNING: MemorySanitizer: use-of-uninitialized-value
 1109 15:16:20.312923     #0 0x5639b88acb21 in tap_probe_vnet_hdr_len third_party/qemu/net/tap-linux.c:183:9
 1109 15:16:20.312952     #1 0x5639b88afd66 in net_tap_fd_init third_party/qemu/net/tap.c:409:9
 1109 15:16:20.312954     #2 0x5639b88b2d1b in net_init_tap_one third_party/qemu/net/tap.c:681:19
 1109 15:16:20.312956     #3 0x5639b88b16a8 in net_init_tap third_party/qemu/net/tap.c:912:13
 1109 15:16:20.312957     #4 0x5639b8890175 in net_client_init1 third_party/qemu/net/net.c:1110:9
 1109 15:16:20.312958     #5 0x5639b888f912 in net_client_init third_party/qemu/net/net.c:1208:15
 1109 15:16:20.312960     #6 0x5639b8894aa5 in net_param_nic third_party/qemu/net/net.c:1588:11
 1109 15:16:20.312961     #7 0x5639b900cd18 in qemu_opts_foreach third_party/qemu/util/qemu-option.c:1135:14
 1109 15:16:20.312962     #8 0x5639b889393c in net_init_clients third_party/qemu/net/net.c:1612:9
 1109 15:16:20.312964     #9 0x5639b717aaf3 in qemu_create_late_backends third_party/qemu/softmmu/vl.c:1962:5
 1109 15:16:20.312965     #10 0x5639b717aaf3 in qemu_init third_party/qemu/softmmu/vl.c:3694:5
 1109 15:16:20.312967     #11 0x5639b71083b8 in main third_party/qemu/softmmu/main.c:49:5
 1109 15:16:20.312968     #12 0x7f464de1d8d2 in __libc_start_main (/usr/grte/v5/lib64/libc.so.6+0x628d2)
 1109 15:16:20.312969     #13 0x5639b6bbd389 in _start /usr/grte/v5/debug-src/src/csu/../sysdeps/x86_64/start.S:120
 1109 15:16:20.312970
 1109 15:16:20.312975   Uninitialized value was stored to memory at
 1109 15:16:20.313393     #0 0x5639b88acbee in tap_probe_vnet_hdr_len third_party/qemu/net/tap-linux.c
 1109 15:16:20.313396     #1 0x5639b88afd66 in net_tap_fd_init third_party/qemu/net/tap.c:409:9
 1109 15:16:20.313398     #2 0x5639b88b2d1b in net_init_tap_one third_party/qemu/net/tap.c:681:19
 1109 15:16:20.313399     #3 0x5639b88b16a8 in net_init_tap third_party/qemu/net/tap.c:912:13
 1109 15:16:20.313400     #4 0x5639b8890175 in net_client_init1 third_party/qemu/net/net.c:1110:9
 1109 15:16:20.313401     #5 0x5639b888f912 in net_client_init third_party/qemu/net/net.c:1208:15
 1109 15:16:20.313403     #6 0x5639b8894aa5 in net_param_nic third_party/qemu/net/net.c:1588:11
 1109 15:16:20.313404     #7 0x5639b900cd18 in qemu_opts_foreach third_party/qemu/util/qemu-option.c:1135:14
 1109 15:16:20.313405     #8 0x5639b889393c in net_init_clients third_party/qemu/net/net.c:1612:9
 1109 15:16:20.313407     #9 0x5639b717aaf3 in qemu_create_late_backends third_party/qemu/softmmu/vl.c:1962:5
 1109 15:16:20.313408     #10 0x5639b717aaf3 in qemu_init third_party/qemu/softmmu/vl.c:3694:5
 1109 15:16:20.313409     #11 0x5639b71083b8 in main third_party/qemu/softmmu/main.c:49:5
 1109 15:16:20.313410     #12 0x7f464de1d8d2 in __libc_start_main (/usr/grte/v5/lib64/libc.so.6+0x628d2)
 1109 15:16:20.313412     #13 0x5639b6bbd389 in _start /usr/grte/v5/debug-src/src/csu/../sysdeps/x86_64/start.S:120
 1109 15:16:20.313413
 1109 15:16:20.313417   Uninitialized value was stored to memory at
 1109 15:16:20.313791     #0 0x5639b88affbd in net_tap_fd_init third_party/qemu/net/tap.c:400:26
 1109 15:16:20.313826     #1 0x5639b88b2d1b in net_init_tap_one third_party/qemu/net/tap.c:681:19
 1109 15:16:20.313829     #2 0x5639b88b16a8 in net_init_tap third_party/qemu/net/tap.c:912:13
 1109 15:16:20.313831     #3 0x5639b8890175 in net_client_init1 third_party/qemu/net/net.c:1110:9
 1109 15:16:20.313836     #4 0x5639b888f912 in net_client_init third_party/qemu/net/net.c:1208:15
 1109 15:16:20.313838     #5 0x5639b8894aa5 in net_param_nic third_party/qemu/net/net.c:1588:11
 1109 15:16:20.313839     #6 0x5639b900cd18 in qemu_opts_foreach third_party/qemu/util/qemu-option.c:1135:14
 1109 15:16:20.313841     #7 0x5639b889393c in net_init_clients third_party/qemu/net/net.c:1612:9
 1109 15:16:20.313843     #8 0x5639b717aaf3 in qemu_create_late_backends third_party/qemu/softmmu/vl.c:1962:5
 1109 15:16:20.313844     #9 0x5639b717aaf3 in qemu_init third_party/qemu/softmmu/vl.c:3694:5
 1109 15:16:20.313845     #10 0x5639b71083b8 in main third_party/qemu/softmmu/main.c:49:5
 1109 15:16:20.313846     #11 0x7f464de1d8d2 in __libc_start_main (/usr/grte/v5/lib64/libc.so.6+0x628d2)
 1109 15:16:20.313847     #12 0x5639b6bbd389 in _start /usr/grte/v5/debug-src/src/csu/../sysdeps/x86_64/start.S:120
 1109 15:16:20.313849
 1109 15:16:20.313851   Uninitialized value was created by an allocation of 'ifr' in the stack frame of function 'tap_probe_vnet_hdr'
 1109 15:16:20.313855     #0 0x5639b88ac680 in tap_probe_vnet_hdr third_party/qemu/net/tap-linux.c:151
 1109 15:16:20.313856
 1109 15:16:20.313878 SUMMARY: MemorySanitizer: use-of-uninitialized-value third_party/qemu/net/tap-linux.c:183:9 in tap_probe_vnet_hdr_len

Fixes: dc69004c7d8 ("net: move tap_probe_vnet_hdr() to tap-linux.c")
Reviewed-by: Hao Wu <wuhaotsh@google.com>
Reviewed-by: Patrick Venture <venture@google.com>
Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org>
Signed-off-by: Peter Foley <pefoley@google.com>
Signed-off-by: Jason Wang <jasowang@redhat.com>
---
 net/tap-linux.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/net/tap-linux.c b/net/tap-linux.c
index XXXXXXX..XXXXXXX 100644
--- a/net/tap-linux.c
+++ b/net/tap-linux.c
@@ -XXX,XX +XXX,XX @@ void tap_set_sndbuf(int fd, const NetdevTapOptions *tap, Error **errp)
 int tap_probe_vnet_hdr(int fd, Error **errp)
 {
     struct ifreq ifr;
+    memset(&ifr, 0, sizeof(ifr));
 
     if (ioctl(fd, TUNGETIFF, &ifr) != 0) {
         /* TUNGETIFF is available since kernel v2.6.27 */
-- 
2.7.4

From: Zhang Chen <chen.zhang@intel.com>

COLO-compare use the glib function g_queue_find_custom to dump
another VM's networking packet to compare. But this function always
start find from the queue->head(here is the newest packet), It will
reduce the success rate of comparison. So this patch reversed
the order of the queues for performance.

Signed-off-by: Zhang Chen <chen.zhang@intel.com>
Reported-by: leirao <lei.rao@intel.com>
Signed-off-by: Jason Wang <jasowang@redhat.com>
---
 net/colo-compare.c | 26 +++++++++++++-------------
 1 file changed, 13 insertions(+), 13 deletions(-)

diff --git a/net/colo-compare.c b/net/colo-compare.c
index XXXXXXX..XXXXXXX 100644
--- a/net/colo-compare.c
+++ b/net/colo-compare.c
@@ -XXX,XX +XXX,XX @@ static void colo_compare_inconsistency_notify(CompareState *s)
 /* Use restricted to colo_insert_packet() */
 static gint seq_sorter(Packet *a, Packet *b, gpointer data)
 {
-    return a->tcp_seq - b->tcp_seq;
+    return b->tcp_seq - a->tcp_seq;
 }
 
 static void fill_pkt_tcp_info(void *data, uint32_t *max_ack)
@@ -XXX,XX +XXX,XX @@ pri:
     if (g_queue_is_empty(&conn->primary_list)) {
         return;
     }
-    ppkt = g_queue_pop_head(&conn->primary_list);
+    ppkt = g_queue_pop_tail(&conn->primary_list);
 sec:
     if (g_queue_is_empty(&conn->secondary_list)) {
-        g_queue_push_head(&conn->primary_list, ppkt);
+        g_queue_push_tail(&conn->primary_list, ppkt);
         return;
     }
-    spkt = g_queue_pop_head(&conn->secondary_list);
+    spkt = g_queue_pop_tail(&conn->secondary_list);
 
     if (ppkt->tcp_seq == ppkt->seq_end) {
         colo_release_primary_pkt(s, ppkt);
@@ -XXX,XX +XXX,XX @@ sec:
             }
         }
         if (!ppkt) {
-            g_queue_push_head(&conn->secondary_list, spkt);
+            g_queue_push_tail(&conn->secondary_list, spkt);
             goto pri;
         }
     }
@@ -XXX,XX +XXX,XX @@ sec:
         if (mark == COLO_COMPARE_FREE_PRIMARY) {
             conn->compare_seq = ppkt->seq_end;
             colo_release_primary_pkt(s, ppkt);
-            g_queue_push_head(&conn->secondary_list, spkt);
+            g_queue_push_tail(&conn->secondary_list, spkt);
             goto pri;
         } else if (mark == COLO_COMPARE_FREE_SECONDARY) {
             conn->compare_seq = spkt->seq_end;
@@ -XXX,XX +XXX,XX @@ sec:
             goto pri;
         }
     } else {
-        g_queue_push_head(&conn->primary_list, ppkt);
-        g_queue_push_head(&conn->secondary_list, spkt);
+        g_queue_push_tail(&conn->primary_list, ppkt);
+        g_queue_push_tail(&conn->secondary_list, spkt);
 
 #ifdef DEBUG_COLO_PACKETS
         qemu_hexdump(stderr, "colo-compare ppkt", ppkt->data, ppkt->size);
@@ -XXX,XX +XXX,XX @@ static void colo_compare_packet(CompareState *s, Connection *conn,
 
     while (!g_queue_is_empty(&conn->primary_list) &&
            !g_queue_is_empty(&conn->secondary_list)) {
-        pkt = g_queue_pop_head(&conn->primary_list);
+        pkt = g_queue_pop_tail(&conn->primary_list);
         result = g_queue_find_custom(&conn->secondary_list,
                  pkt, (GCompareFunc)HandlePacket);
 
@@ -XXX,XX +XXX,XX @@ static void colo_compare_packet(CompareState *s, Connection *conn,
              * timeout, it will trigger a checkpoint request.
              */
             trace_colo_compare_main("packet different");
-            g_queue_push_head(&conn->primary_list, pkt);
+            g_queue_push_tail(&conn->primary_list, pkt);
 
             colo_compare_inconsistency_notify(s);
             break;
@@ -XXX,XX +XXX,XX @@ static int compare_chr_send(CompareState *s,
         entry->buf = g_malloc(size);
         memcpy(entry->buf, buf, size);
     }
-    g_queue_push_head(&sendco->send_list, entry);
+    g_queue_push_tail(&sendco->send_list, entry);
 
     if (sendco->done) {
         sendco->co = qemu_coroutine_create(_compare_chr_send, sendco);
@@ -XXX,XX +XXX,XX @@ static void colo_flush_packets(void *opaque, void *user_data)
     Packet *pkt = NULL;
 
     while (!g_queue_is_empty(&conn->primary_list)) {
-        pkt = g_queue_pop_head(&conn->primary_list);
+        pkt = g_queue_pop_tail(&conn->primary_list);
         compare_chr_send(s,
                          pkt->data,
                          pkt->size,
@@ -XXX,XX +XXX,XX @@ static void colo_flush_packets(void *opaque, void *user_data)
         packet_destroy_partial(pkt, NULL);
     }
     while (!g_queue_is_empty(&conn->secondary_list)) {
-        pkt = g_queue_pop_head(&conn->secondary_list);
+        pkt = g_queue_pop_tail(&conn->secondary_list);
         packet_destroy(pkt, NULL);
     }
 }
-- 
2.7.4

From: Rao Lei <lei.rao@intel.com>

This patch is to improve the logic of QEMU main thread sleep code in
qemu_chr_write_buffer() where it can be blocked and can't run other
coroutines during COLO IO stress test.

Our approach is to put filter_send() in a coroutine. In this way,
filter_send() will call qemu_coroutine_yield() in qemu_co_sleep_ns(),
so that it can be scheduled out and QEMU main thread has opportunity to
run other tasks.

Signed-off-by: Lei Rao <lei.rao@intel.com>
Signed-off-by: Zhang Chen <chen.zhang@intel.com>
Reviewed-by: Li Zhijian <lizhijian@fujitsu.com>
Reviewed-by: Zhang Chen <chen.zhang@intel.com>
Signed-off-by: Jason Wang <jasowang@redhat.com>
---
 net/filter-mirror.c | 66 ++++++++++++++++++++++++++++++++++++++++++-----------
 1 file changed, 53 insertions(+), 13 deletions(-)

diff --git a/net/filter-mirror.c b/net/filter-mirror.c
index XXXXXXX..XXXXXXX 100644
--- a/net/filter-mirror.c
+++ b/net/filter-mirror.c
@@ -XXX,XX +XXX,XX @@
 #include "chardev/char-fe.h"
 #include "qemu/iov.h"
 #include "qemu/sockets.h"
+#include "block/aio-wait.h"
 
 #define TYPE_FILTER_MIRROR "filter-mirror"
 typedef struct MirrorState MirrorState;
@@ -XXX,XX +XXX,XX @@ struct MirrorState {
     bool vnet_hdr;
 };
 
-static int filter_send(MirrorState *s,
-                       const struct iovec *iov,
-                       int iovcnt)
+typedef struct FilterSendCo {
+    MirrorState *s;
+    char *buf;
+    ssize_t size;
+    bool done;
+    int ret;
+} FilterSendCo;
+
+static int _filter_send(MirrorState *s,
+                       char *buf,
+                       ssize_t size)
 {
     NetFilterState *nf = NETFILTER(s);
     int ret = 0;
-    ssize_t size = 0;
     uint32_t len = 0;
-    char *buf;
-
-    size = iov_size(iov, iovcnt);
-    if (!size) {
-        return 0;
-    }
 
     len = htonl(size);
     ret = qemu_chr_fe_write_all(&s->chr_out, (uint8_t *)&len, sizeof(len));
@@ -XXX,XX +XXX,XX @@ static int filter_send(MirrorState *s,
         }
     }
 
-    buf = g_malloc(size);
-    iov_to_buf(iov, iovcnt, 0, buf, size);
     ret = qemu_chr_fe_write_all(&s->chr_out, (uint8_t *)buf, size);
-    g_free(buf);
     if (ret != size) {
         goto err;
     }
@@ -XXX,XX +XXX,XX @@ err:
     return ret < 0 ? ret : -EIO;
 }
 
+static void coroutine_fn filter_send_co(void *opaque)
+{
+    FilterSendCo *data = opaque;
+
+    data->ret = _filter_send(data->s, data->buf, data->size);
+    data->done = true;
+    g_free(data->buf);
+    aio_wait_kick();
+}
+
+static int filter_send(MirrorState *s,
+                       const struct iovec *iov,
+                       int iovcnt)
+{
+    ssize_t size = iov_size(iov, iovcnt);
+    char *buf = NULL;
+
+    if (!size) {
+        return 0;
+    }
+
+    buf = g_malloc(size);
+    iov_to_buf(iov, iovcnt, 0, buf, size);
+
+    FilterSendCo data = {
+        .s = s,
+        .size = size,
+        .buf = buf,
+        .ret = 0,
+    };
+
+    Coroutine *co = qemu_coroutine_create(filter_send_co, &data);
+    qemu_coroutine_enter(co);
+
+    while (!data.done) {
+        aio_poll(qemu_get_aio_context(), true);
+    }
+
+    return data.ret;
+}
+
 static void redirector_to_filter(NetFilterState *nf,
                                  const uint8_t *buf,
                                  int len)
-- 
2.7.4

From: Vladislav Yaroshchuk <yaroshchuk2000@gmail.com>

vmnet.framework dependency is added with 'vmnet' option
to enable or disable it. Default value is 'auto'.

vmnet features to be used are available since macOS 11.0,
corresponding probe is created into meson.build.

Signed-off-by: Vladislav Yaroshchuk <yaroshchuk2000@gmail.com>
Signed-off-by: Jason Wang <jasowang@redhat.com>
---
 meson.build                   | 16 +++++++++++++++-
 meson_options.txt             |  2 ++
 scripts/meson-buildoptions.sh |  3 +++
 3 files changed, 20 insertions(+), 1 deletion(-)

diff --git a/meson.build b/meson.build
index XXXXXXX..XXXXXXX 100644
--- a/meson.build
+++ b/meson.build
@@ -XXX,XX +XXX,XX @@ if cocoa.found() and get_option('gtk').enabled()
   error('Cocoa and GTK+ cannot be enabled at the same time')
 endif
 
+vmnet = dependency('appleframeworks', modules: 'vmnet', required: get_option('vmnet'))
+if vmnet.found() and not cc.has_header_symbol('vmnet/vmnet.h',
+                                              'VMNET_BRIDGED_MODE',
+                                              dependencies: vmnet)
+  vmnet = not_found
+  if get_option('vmnet').enabled()
+    error('vmnet.framework API is outdated')
+  else
+    warning('vmnet.framework API is outdated, disabling')
+  endif
+endif
+
 seccomp = not_found
 if not get_option('seccomp').auto() or have_system or have_tools
   seccomp = dependency('libseccomp', version: '>=2.3.0',
@@ -XXX,XX +XXX,XX @@ config_host_data.set('CONFIG_SECCOMP', seccomp.found())
 config_host_data.set('CONFIG_SNAPPY', snappy.found())
 config_host_data.set('CONFIG_USB_LIBUSB', libusb.found())
 config_host_data.set('CONFIG_VDE', vde.found())
+config_host_data.set('CONFIG_VMNET', vmnet.found())
 config_host_data.set('CONFIG_VHOST_USER_BLK_SERVER', have_vhost_user_blk_server)
 config_host_data.set('CONFIG_VNC', vnc.found())
 config_host_data.set('CONFIG_VNC_JPEG', jpeg.found())
@@ -XXX,XX +XXX,XX @@ summary(summary_info, bool_yn: true, section: 'Crypto')
 # Libraries
 summary_info = {}
 if targetos == 'darwin'
-  summary_info += {'Cocoa support':   cocoa}
+  summary_info += {'Cocoa support':           cocoa}
+  summary_info += {'vmnet.framework support': vmnet}
 endif
 summary_info += {'SDL support':       sdl}
 summary_info += {'SDL image support': sdl_image}
diff --git a/meson_options.txt b/meson_options.txt
index XXXXXXX..XXXXXXX 100644
--- a/meson_options.txt
+++ b/meson_options.txt
@@ -XXX,XX +XXX,XX @@ option('netmap', type : 'feature', value : 'auto',
        description: 'netmap network backend support')
 option('vde', type : 'feature', value : 'auto',
        description: 'vde network backend support')
+option('vmnet', type : 'feature', value : 'auto',
+       description: 'vmnet.framework network backend support')
 option('virglrenderer', type : 'feature', value : 'auto',
        description: 'virgl rendering support')
 option('vnc', type : 'feature', value : 'auto',
diff --git a/scripts/meson-buildoptions.sh b/scripts/meson-buildoptions.sh
index XXXXXXX..XXXXXXX 100644
--- a/scripts/meson-buildoptions.sh
+++ b/scripts/meson-buildoptions.sh
@@ -XXX,XX +XXX,XX @@ meson_options_help() {
   printf "%s\n" '  u2f             U2F emulation support'
   printf "%s\n" '  usb-redir       libusbredir support'
   printf "%s\n" '  vde             vde network backend support'
+  printf "%s\n" '  vmnet           vmnet.framework network backend support'
   printf "%s\n" '  vhost-user-blk-server'
   printf "%s\n" '                  build vhost-user-blk server'
   printf "%s\n" '  virglrenderer   virgl rendering support'
@@ -XXX,XX +XXX,XX @@ _meson_option_parse() {
     --disable-usb-redir) printf "%s" -Dusb_redir=disabled ;;
     --enable-vde) printf "%s" -Dvde=enabled ;;
     --disable-vde) printf "%s" -Dvde=disabled ;;
+    --enable-vmnet) printf "%s" -Dvmnet=enabled ;;
+    --disable-vmnet) printf "%s" -Dvmnet=disabled ;;
     --enable-vhost-user-blk-server) printf "%s" -Dvhost_user_blk_server=enabled ;;
     --disable-vhost-user-blk-server) printf "%s" -Dvhost_user_blk_server=disabled ;;
     --enable-virglrenderer) printf "%s" -Dvirglrenderer=enabled ;;
-- 
2.7.4

From: Vladislav Yaroshchuk <yaroshchuk2000@gmail.com>

Create separate netdevs for each vmnet operating mode:
- vmnet-host
- vmnet-shared
- vmnet-bridged

Signed-off-by: Vladislav Yaroshchuk <yaroshchuk2000@gmail.com>
Signed-off-by: Jason Wang <jasowang@redhat.com>
---
 net/clients.h       |  11 +++++
 net/meson.build     |   7 +++
 net/net.c           |  10 ++++
 net/vmnet-bridged.m |  25 ++++++++++
 net/vmnet-common.m  |  19 ++++++++
 net/vmnet-host.c    |  24 ++++++++++
 net/vmnet-shared.c  |  25 ++++++++++
 net/vmnet_int.h     |  25 ++++++++++
 qapi/net.json       | 133 +++++++++++++++++++++++++++++++++++++++++++++++++++-
 9 files changed, 277 insertions(+), 2 deletions(-)
 create mode 100644 net/vmnet-bridged.m
 create mode 100644 net/vmnet-common.m
 create mode 100644 net/vmnet-host.c
 create mode 100644 net/vmnet-shared.c
 create mode 100644 net/vmnet_int.h

diff --git a/net/clients.h b/net/clients.h
index XXXXXXX..XXXXXXX 100644
--- a/net/clients.h
+++ b/net/clients.h
@@ -XXX,XX +XXX,XX @@ int net_init_vhost_user(const Netdev *netdev, const char *name,
 
 int net_init_vhost_vdpa(const Netdev *netdev, const char *name,
                         NetClientState *peer, Error **errp);
+#ifdef CONFIG_VMNET
+int net_init_vmnet_host(const Netdev *netdev, const char *name,
+                          NetClientState *peer, Error **errp);
+
+int net_init_vmnet_shared(const Netdev *netdev, const char *name,
+                          NetClientState *peer, Error **errp);
+
+int net_init_vmnet_bridged(const Netdev *netdev, const char *name,
+                          NetClientState *peer, Error **errp);
+#endif /* CONFIG_VMNET */
+
 #endif /* QEMU_NET_CLIENTS_H */
diff --git a/net/meson.build b/net/meson.build
index XXXXXXX..XXXXXXX 100644
--- a/net/meson.build
+++ b/net/meson.build
@@ -XXX,XX +XXX,XX @@ softmmu_ss.add(when: 'CONFIG_POSIX', if_true: files(tap_posix))
 softmmu_ss.add(when: 'CONFIG_WIN32', if_true: files('tap-win32.c'))
 softmmu_ss.add(when: 'CONFIG_VHOST_NET_VDPA', if_true: files('vhost-vdpa.c'))
 
+vmnet_files = files(
+  'vmnet-common.m',
+  'vmnet-bridged.m',
+  'vmnet-host.c',
+  'vmnet-shared.c'
+)
+softmmu_ss.add(when: vmnet, if_true: vmnet_files)
 subdir('can')
diff --git a/net/net.c b/net/net.c
index XXXXXXX..XXXXXXX 100644
--- a/net/net.c
+++ b/net/net.c
@@ -XXX,XX +XXX,XX @@ static int (* const net_client_init_fun[NET_CLIENT_DRIVER__MAX])(
 #ifdef CONFIG_L2TPV3
         [NET_CLIENT_DRIVER_L2TPV3]    = net_init_l2tpv3,
 #endif
+#ifdef CONFIG_VMNET
+        [NET_CLIENT_DRIVER_VMNET_HOST] = net_init_vmnet_host,
+        [NET_CLIENT_DRIVER_VMNET_SHARED] = net_init_vmnet_shared,
+        [NET_CLIENT_DRIVER_VMNET_BRIDGED] = net_init_vmnet_bridged,
+#endif /* CONFIG_VMNET */
 };
 
 
@@ -XXX,XX +XXX,XX @@ void show_netdevs(void)
 #ifdef CONFIG_VHOST_VDPA
         "vhost-vdpa",
 #endif
+#ifdef CONFIG_VMNET
+        "vmnet-host",
+        "vmnet-shared",
+        "vmnet-bridged",
+#endif
     };
 
     qemu_printf("Available netdev backend types:\n");
diff --git a/net/vmnet-bridged.m b/net/vmnet-bridged.m
new file mode 100644
index XXXXXXX..XXXXXXX
--- /dev/null
+++ b/net/vmnet-bridged.m
@@ -XXX,XX +XXX,XX @@
+/*
+ * vmnet-bridged.m
+ *
+ * Copyright(c) 2021 Vladislav Yaroshchuk <yaroshchuk2000@gmail.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "qapi/qapi-types-net.h"
+#include "vmnet_int.h"
+#include "clients.h"
+#include "qemu/error-report.h"
+#include "qapi/error.h"
+
+#include <vmnet/vmnet.h>
+
+int net_init_vmnet_bridged(const Netdev *netdev, const char *name,
+                           NetClientState *peer, Error **errp)
+{
+  error_setg(errp, "vmnet-bridged is not implemented yet");
+  return -1;
+}
diff --git a/net/vmnet-common.m b/net/vmnet-common.m
new file mode 100644
index XXXXXXX..XXXXXXX
--- /dev/null
+++ b/net/vmnet-common.m
@@ -XXX,XX +XXX,XX @@
+/*
+ * vmnet-common.m - network client wrapper for Apple vmnet.framework
+ *
+ * Copyright(c) 2021 Vladislav Yaroshchuk <yaroshchuk2000@gmail.com>
+ * Copyright(c) 2021 Phillip Tennen <phillip@axleos.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "qapi/qapi-types-net.h"
+#include "vmnet_int.h"
+#include "clients.h"
+#include "qemu/error-report.h"
+#include "qapi/error.h"
+
+#include <vmnet/vmnet.h>
diff --git a/net/vmnet-host.c b/net/vmnet-host.c
new file mode 100644
index XXXXXXX..XXXXXXX
--- /dev/null
+++ b/net/vmnet-host.c
@@ -XXX,XX +XXX,XX @@
+/*
+ * vmnet-host.c
+ *
+ * Copyright(c) 2021 Vladislav Yaroshchuk <yaroshchuk2000@gmail.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "qapi/qapi-types-net.h"
+#include "vmnet_int.h"
+#include "clients.h"
+#include "qemu/error-report.h"
+#include "qapi/error.h"
+
+#include <vmnet/vmnet.h>
+
+int net_init_vmnet_host(const Netdev *netdev, const char *name,
+                        NetClientState *peer, Error **errp) {
+  error_setg(errp, "vmnet-host is not implemented yet");
+  return -1;
+}
diff --git a/net/vmnet-shared.c b/net/vmnet-shared.c
new file mode 100644
index XXXXXXX..XXXXXXX
--- /dev/null
+++ b/net/vmnet-shared.c
@@ -XXX,XX +XXX,XX @@
+/*
+ * vmnet-shared.c
+ *
+ * Copyright(c) 2021 Vladislav Yaroshchuk <yaroshchuk2000@gmail.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "qapi/qapi-types-net.h"
+#include "vmnet_int.h"
+#include "clients.h"
+#include "qemu/error-report.h"
+#include "qapi/error.h"
+
+#include <vmnet/vmnet.h>
+
+int net_init_vmnet_shared(const Netdev *netdev, const char *name,
+                          NetClientState *peer, Error **errp)
+{
+  error_setg(errp, "vmnet-shared is not implemented yet");
+  return -1;
+}
diff --git a/net/vmnet_int.h b/net/vmnet_int.h
new file mode 100644
index XXXXXXX..XXXXXXX
--- /dev/null
+++ b/net/vmnet_int.h
@@ -XXX,XX +XXX,XX @@
+/*
+ * vmnet_int.h
+ *
+ * Copyright(c) 2021 Vladislav Yaroshchuk <yaroshchuk2000@gmail.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */
+#ifndef VMNET_INT_H
+#define VMNET_INT_H
+
+#include "qemu/osdep.h"
+#include "vmnet_int.h"
+#include "clients.h"
+
+#include <vmnet/vmnet.h>
+
+typedef struct VmnetCommonState {
+  NetClientState nc;
+
+} VmnetCommonState;
+
+
+#endif /* VMNET_INT_H */
diff --git a/qapi/net.json b/qapi/net.json
index XXXXXXX..XXXXXXX 100644
--- a/qapi/net.json
+++ b/qapi/net.json
@@ -XXX,XX +XXX,XX @@
     '*queues':       'int' } }
 
 ##
+# @NetdevVmnetHostOptions:
+#
+# vmnet (host mode) network backend.
+#
+# Allows the vmnet interface to communicate with other vmnet
+# interfaces that are in host mode and also with the host.
+#
+# @start-address: The starting IPv4 address to use for the interface.
+#                 Must be in the private IP range (RFC 1918). Must be
+#                 specified along with @end-address and @subnet-mask.
+#                 This address is used as the gateway address. The
+#                 subsequent address up to and including end-address are
+#                 placed in the DHCP pool.
+#
+# @end-address: The DHCP IPv4 range end address to use for the
+#               interface. Must be in the private IP range (RFC 1918).
+#               Must be specified along with @start-address and
+#               @subnet-mask.
+#
+# @subnet-mask: The IPv4 subnet mask to use on the interface. Must
+#               be specified along with @start-address and @subnet-mask.
+#
+# @isolated: Enable isolation for this interface. Interface isolation
+#            ensures that vmnet interface is not able to communicate
+#            with any other vmnet interfaces. Only communication with
+#            host is allowed. Available since macOS Big Sur 11.0.
+#
+# @net-uuid: The identifier (UUID) to uniquely identify the isolated
+#            network vmnet interface should be added to. If
+#            set, no DHCP service is provided for this interface and
+#            network communication is allowed only with other interfaces
+#            added to this network identified by the UUID. Available
+#            since macOS Big Sur 11.0.
+#
+# Since: 7.0
+##
+{ 'struct': 'NetdevVmnetHostOptions',
+  'data': {
+    '*start-address': 'str',
+    '*end-address':   'str',
+    '*subnet-mask':   'str',
+    '*isolated':      'bool',
+    '*net-uuid':      'str' },
+  'if': 'CONFIG_VMNET' }
+
+##
+# @NetdevVmnetSharedOptions:
+#
+# vmnet (shared mode) network backend.
+#
+# Allows traffic originating from the vmnet interface to reach the
+# Internet through a network address translator (NAT).
+# The vmnet interface can communicate with the host and with
+# other shared mode interfaces on the same subnet. If no DHCP
+# settings, subnet mask and IPv6 prefix specified, the interface can
+# communicate with any of other interfaces in shared mode.
+#
+# @start-address: The starting IPv4 address to use for the interface.
+#                 Must be in the private IP range (RFC 1918). Must be
+#                 specified along with @end-address and @subnet-mask.
+#                 This address is used as the gateway address. The
+#                 subsequent address up to and including end-address are
+#                 placed in the DHCP pool.
+#
+# @end-address: The DHCP IPv4 range end address to use for the
+#               interface. Must be in the private IP range (RFC 1918).
+#               Must be specified along with @start-address and @subnet-mask.
+#
+# @subnet-mask: The IPv4 subnet mask to use on the interface. Must
+#                be specified along with @start-address and @subnet-mask.
+#
+# @isolated: Enable isolation for this interface. Interface isolation
+#            ensures that vmnet interface is not able to communicate
+#            with any other vmnet interfaces. Only communication with
+#            host is allowed. Available since macOS Big Sur 11.0.
+#
+# @nat66-prefix: The IPv6 prefix to use into guest network. Must be a
+#                unique local address i.e. start with fd00::/8 and have
+#                length of 64.
+#
+# Since: 7.0
+##
+{ 'struct': 'NetdevVmnetSharedOptions',
+  'data': {
+    '*start-address': 'str',
+    '*end-address':   'str',
+    '*subnet-mask':   'str',
+    '*isolated':      'bool',
+    '*nat66-prefix':  'str' },
+  'if': 'CONFIG_VMNET' }
+
+##
+# @NetdevVmnetBridgedOptions:
+#
+# vmnet (bridged mode) network backend.
+#
+# Bridges the vmnet interface with a physical network interface.
+#
+# @ifname: The name of the physical interface to be bridged.
+#
+# @isolated: Enable isolation for this interface. Interface isolation
+#            ensures that vmnet interface is not able to communicate
+#            with any other vmnet interfaces. Only communication with
+#            host is allowed. Available since macOS Big Sur 11.0.
+#
+# Since: 7.0
+##
+{ 'struct': 'NetdevVmnetBridgedOptions',
+  'data': {
+    'ifname':     'str',
+    '*isolated':  'bool' },
+  'if': 'CONFIG_VMNET' }
+
+##
 # @NetClientDriver:
 #
 # Available netdev drivers.
@@ -XXX,XX +XXX,XX @@
 # Since: 2.7
 #
 #        @vhost-vdpa since 5.1
+#        @vmnet-host since 7.0
+#        @vmnet-shared since 7.0
+#        @vmnet-bridged since 7.0
 ##
 { 'enum': 'NetClientDriver',
   'data': [ 'none', 'nic', 'user', 'tap', 'l2tpv3', 'socket', 'vde',
-            'bridge', 'hubport', 'netmap', 'vhost-user', 'vhost-vdpa' ] }
+            'bridge', 'hubport', 'netmap', 'vhost-user', 'vhost-vdpa',
+            { 'name': 'vmnet-host', 'if': 'CONFIG_VMNET' },
+            { 'name': 'vmnet-shared', 'if': 'CONFIG_VMNET' },
+            { 'name': 'vmnet-bridged', 'if': 'CONFIG_VMNET' }] }
 
 ##
 # @Netdev:
@@ -XXX,XX +XXX,XX @@
 # Since: 1.2
 #
 #        'l2tpv3' - since 2.1
+#        'vmnet-host' - since 7.0
+#        'vmnet-shared' - since 7.0
+#        'vmnet-bridged' - since 7.0
 ##
 { 'union': 'Netdev',
   'base': { 'id': 'str', 'type': 'NetClientDriver' },
@@ -XXX,XX +XXX,XX @@
     'hubport':  'NetdevHubPortOptions',
     'netmap':   'NetdevNetmapOptions',
     'vhost-user': 'NetdevVhostUserOptions',
-    'vhost-vdpa': 'NetdevVhostVDPAOptions' } }
+    'vhost-vdpa': 'NetdevVhostVDPAOptions',
+    'vmnet-host': { 'type': 'NetdevVmnetHostOptions',
+                    'if': 'CONFIG_VMNET' },
+    'vmnet-shared': { 'type': 'NetdevVmnetSharedOptions',
+                      'if': 'CONFIG_VMNET' },
+    'vmnet-bridged': { 'type': 'NetdevVmnetBridgedOptions',
+                       'if': 'CONFIG_VMNET' } } }
 
 ##
 # @RxState:
-- 
2.7.4

From: Vladislav Yaroshchuk <yaroshchuk2000@gmail.com>

Interaction with vmnet.framework in different modes
differs only on configuration stage, so we can create
common `send`, `receive`, etc. procedures and reuse them.

vmnet.framework supports iov, but writing more than
one iov into vmnet interface fails with
'VMNET_INVALID_ARGUMENT'. Collecting provided iovs into
one and passing it to vmnet works fine. That's the
reason why receive_iov() left unimplemented. But it still
works with good enough performance having .receive()
implemented only.

Also, there is no way to unsubscribe from vmnet packages
receiving except registering and unregistering event
callback or simply drop packages just ignoring and
not processing them when related flag is set. Here we do
using the second way.

Signed-off-by: Phillip Tennen <phillip@axleos.com>
Signed-off-by: Vladislav Yaroshchuk <yaroshchuk2000@gmail.com>
Signed-off-by: Jason Wang <jasowang@redhat.com>
---
 net/vmnet-common.m | 314 +++++++++++++++++++++++++++++++++++++++++++++++++++++
 net/vmnet-shared.c |  83 +++++++++++++-
 net/vmnet_int.h    |  23 ++++
 3 files changed, 416 insertions(+), 4 deletions(-)

diff --git a/net/vmnet-common.m b/net/vmnet-common.m
index XXXXXXX..XXXXXXX 100644
--- a/net/vmnet-common.m
+++ b/net/vmnet-common.m
@@ -XXX,XX +XXX,XX @@
  */
 
 #include "qemu/osdep.h"
+#include "qemu/main-loop.h"
+#include "qemu/log.h"
 #include "qapi/qapi-types-net.h"
 #include "vmnet_int.h"
 #include "clients.h"
@@ -XXX,XX +XXX,XX @@
 #include "qapi/error.h"
 
 #include <vmnet/vmnet.h>
+#include <dispatch/dispatch.h>
+
+#ifdef DEBUG
+#define D(x) x
+#define D_LOG(...) qemu_log(__VA_ARGS__)
+#else
+#define D(x) do { } while (0)
+#define D_LOG(...) do { } while (0)
+#endif
+
+typedef struct vmpktdesc vmpktdesc_t;
+typedef struct iovec iovec_t;
+
+static void vmnet_set_send_enabled(VmnetCommonState *s, bool enable)
+{
+    s->send_enabled = enable;
+}
+
+
+static void vmnet_send_completed(NetClientState *nc, ssize_t len)
+{
+    VmnetCommonState *s = DO_UPCAST(VmnetCommonState, nc, nc);
+    vmnet_set_send_enabled(s, true);
+}
+
+
+static void vmnet_send(NetClientState *nc,
+                       interface_event_t event_id,
+                       xpc_object_t event)
+{
+    assert(event_id == VMNET_INTERFACE_PACKETS_AVAILABLE);
+
+    VmnetCommonState *s;
+    uint64_t packets_available;
+
+    struct iovec *iov;
+    struct vmpktdesc *packets;
+    int pkt_cnt;
+    int i;
+
+    vmnet_return_t if_status;
+    ssize_t size;
+
+    s = DO_UPCAST(VmnetCommonState, nc, nc);
+
+    packets_available = xpc_dictionary_get_uint64(
+        event,
+        vmnet_estimated_packets_available_key
+    );
+
+    pkt_cnt = (packets_available < VMNET_PACKETS_LIMIT) ?
+              packets_available :
+              VMNET_PACKETS_LIMIT;
+
+
+    iov = s->iov_buf;
+    packets = s->packets_buf;
+
+    for (i = 0; i < pkt_cnt; ++i) {
+        packets[i].vm_pkt_size = s->max_packet_size;
+        packets[i].vm_pkt_iovcnt = 1;
+        packets[i].vm_flags = 0;
+    }
+
+    if_status = vmnet_read(s->vmnet_if, packets, &pkt_cnt);
+    if (if_status != VMNET_SUCCESS) {
+        error_printf("vmnet: read failed: %s\n",
+                     vmnet_status_map_str(if_status));
+    }
+    qemu_mutex_lock_iothread();
+    for (i = 0; i < pkt_cnt; ++i) {
+        size = qemu_send_packet_async(nc,
+                                      iov[i].iov_base,
+                                      packets[i].vm_pkt_size,
+                                      vmnet_send_completed);
+        if (size == 0) {
+            vmnet_set_send_enabled(s, false);
+        } else if (size < 0) {
+            break;
+        }
+    }
+    qemu_mutex_unlock_iothread();
+
+}
+
+
+static void vmnet_register_event_callback(VmnetCommonState *s)
+{
+    dispatch_queue_t avail_pkt_q = dispatch_queue_create(
+        "org.qemu.vmnet.if_queue",
+        DISPATCH_QUEUE_SERIAL
+    );
+
+    vmnet_interface_set_event_callback(
+        s->vmnet_if,
+        VMNET_INTERFACE_PACKETS_AVAILABLE,
+        avail_pkt_q,
+        ^(interface_event_t event_id, xpc_object_t event) {
+          if (s->send_enabled) {
+              vmnet_send(&s->nc, event_id, event);
+          }
+        });
+}
+
+
+static void vmnet_bufs_init(VmnetCommonState *s)
+{
+    int i;
+    struct vmpktdesc *packets;
+    struct iovec *iov;
+
+    packets = s->packets_buf;
+    iov = s->iov_buf;
+
+    for (i = 0; i < VMNET_PACKETS_LIMIT; ++i) {
+        iov[i].iov_len = s->max_packet_size;
+        iov[i].iov_base = g_malloc0(iov[i].iov_len);
+        packets[i].vm_pkt_iov = iov + i;
+    }
+}
+
+
+const char *vmnet_status_map_str(vmnet_return_t status)
+{
+    switch (status) {
+    case VMNET_SUCCESS:
+        return "success";
+    case VMNET_FAILURE:
+        return "general failure";
+    case VMNET_MEM_FAILURE:
+        return "memory allocation failure";
+    case VMNET_INVALID_ARGUMENT:
+        return "invalid argument specified";
+    case VMNET_SETUP_INCOMPLETE:
+        return "interface setup is not complete";
+    case VMNET_INVALID_ACCESS:
+        return "invalid access, permission denied";
+    case VMNET_PACKET_TOO_BIG:
+        return "packet size is larger than MTU";
+    case VMNET_BUFFER_EXHAUSTED:
+        return "buffers exhausted in kernel";
+    case VMNET_TOO_MANY_PACKETS:
+        return "packet count exceeds limit";
+#if defined(MAC_OS_VERSION_11_0) && \
+    MAC_OS_X_VERSION_MIN_REQUIRED >= MAC_OS_VERSION_11_0
+    case VMNET_SHARING_SERVICE_BUSY:
+        return "conflict, sharing service is in use";
+#endif
+    default:
+        return "unknown vmnet error";
+    }
+}
+
+
+int vmnet_if_create(NetClientState *nc,
+                    xpc_object_t if_desc,
+                    Error **errp,
+                    void (*completion_callback)(xpc_object_t interface_param))
+{
+    VmnetCommonState *s;
+
+    dispatch_queue_t if_create_q;
+    dispatch_semaphore_t if_created_sem;
+
+    __block vmnet_return_t if_status;
+
+    if_create_q = dispatch_queue_create("org.qemu.vmnet.create",
+                                        DISPATCH_QUEUE_SERIAL);
+    if_created_sem = dispatch_semaphore_create(0);
+
+    xpc_dictionary_set_bool(
+        if_desc,
+        vmnet_allocate_mac_address_key,
+        false
+    );
+
+    D(D_LOG("vmnet.start.interface_desc:\n");
+    xpc_dictionary_apply(if_desc,
+                         ^bool(const char *k, xpc_object_t v) {
+                           char *desc = xpc_copy_description(v);
+                           D_LOG("  %s=%s\n", k, desc);
+                           free(desc);
+                           return true;
+                         }));
+
+    s = DO_UPCAST(VmnetCommonState, nc, nc);
+    s->vmnet_if = vmnet_start_interface(
+        if_desc,
+        if_create_q,
+        ^(vmnet_return_t status, xpc_object_t interface_param) {
+          if_status = status;
+          if (status != VMNET_SUCCESS || !interface_param) {
+              dispatch_semaphore_signal(if_created_sem);
+              return;
+          }
+
+          D(D_LOG("vmnet.start.interface_param:\n");
+                xpc_dictionary_apply(interface_param,
+                                     ^bool(const char *k, xpc_object_t v) {
+                                       char *desc = xpc_copy_description(v);
+                                       D_LOG("  %s=%s\n", k, desc);
+                                       free(desc);
+                                       return true;
+                                     }));
+
+          s->mtu = xpc_dictionary_get_uint64(
+              interface_param,
+              vmnet_mtu_key);
+          s->max_packet_size = xpc_dictionary_get_uint64(
+              interface_param,
+              vmnet_max_packet_size_key);
+
+          if (completion_callback) {
+              completion_callback(interface_param);
+          }
+          dispatch_semaphore_signal(if_created_sem);
+        });
+
+    if (s->vmnet_if == NULL) {
+        error_setg(errp, "unable to create interface with requested params");
+        return -1;
+    }
+
+    dispatch_semaphore_wait(if_created_sem, DISPATCH_TIME_FOREVER);
+    dispatch_release(if_create_q);
+
+    if (if_status != VMNET_SUCCESS) {
+        error_setg(errp,
+                   "cannot create vmnet interface: %s",
+                   vmnet_status_map_str(if_status));
+        return -1;
+    }
+
+    vmnet_register_event_callback(s);
+    vmnet_bufs_init(s);
+    vmnet_set_send_enabled(s, true);
+
+    return 0;
+}
+
+
+ssize_t vmnet_receive_common(NetClientState *nc,
+                             const uint8_t *buf,
+                             size_t size)
+{
+    VmnetCommonState *s;
+    vmpktdesc_t packet;
+    iovec_t iov;
+    int pkt_cnt;
+    vmnet_return_t if_status;
+
+    s = DO_UPCAST(VmnetCommonState, nc, nc);
+
+    if (size > s->max_packet_size) {
+        warn_report("vmnet: packet is too big, %zu > %llu\n",
+                    packet.vm_pkt_size,
+                    s->max_packet_size);
+        return -1;
+    }
+
+    iov.iov_base = (char *) buf;
+    iov.iov_len = size;
+
+    packet.vm_pkt_iovcnt = 1;
+    packet.vm_flags = 0;
+    packet.vm_pkt_size = size;
+    packet.vm_pkt_iov = &iov;
+
+    pkt_cnt = 1;
+    if_status = vmnet_write(s->vmnet_if, &packet, &pkt_cnt);
+
+    if (if_status != VMNET_SUCCESS) {
+        error_report("vmnet: write error: %s\n",
+                     vmnet_status_map_str(if_status));
+    }
+
+    if (if_status == VMNET_SUCCESS && pkt_cnt) {
+        return size;
+    }
+    return 0;
+}
+
+
+void vmnet_cleanup_common(NetClientState *nc)
+{
+    VmnetCommonState *s;
+    dispatch_queue_t if_destroy_q;
+
+    s = DO_UPCAST(VmnetCommonState, nc, nc);
+
+    qemu_purge_queued_packets(nc);
+    vmnet_set_send_enabled(s, false);
+
+    if (s->vmnet_if == NULL) {
+        return;
+    }
+
+    if_destroy_q = dispatch_queue_create(
+        "org.qemu.vmnet.destroy",
+        DISPATCH_QUEUE_SERIAL
+    );
+
+    vmnet_stop_interface(
+        s->vmnet_if,
+        if_destroy_q,
+        ^(vmnet_return_t status) {
+        });
+
+    for (int i = 0; i < VMNET_PACKETS_LIMIT; ++i) {
+        g_free(s->iov_buf[i].iov_base);
+    }
+}
diff --git a/net/vmnet-shared.c b/net/vmnet-shared.c
index XXXXXXX..XXXXXXX 100644
--- a/net/vmnet-shared.c
+++ b/net/vmnet-shared.c
@@ -XXX,XX +XXX,XX @@
 
 #include "qemu/osdep.h"
 #include "qapi/qapi-types-net.h"
+#include "qapi/error.h"
 #include "vmnet_int.h"
 #include "clients.h"
-#include "qemu/error-report.h"
-#include "qapi/error.h"
 
 #include <vmnet/vmnet.h>
 
+typedef struct VmnetSharedState {
+  VmnetCommonState cs;
+} VmnetSharedState;
+
+
+static xpc_object_t create_if_desc(const Netdev *netdev, Error **errp)
+{
+    const NetdevVmnetSharedOptions *options = &(netdev->u.vmnet_shared);
+    xpc_object_t if_desc = xpc_dictionary_create(NULL, NULL, 0);
+
+    xpc_dictionary_set_uint64(
+        if_desc,
+        vmnet_operation_mode_key,
+        VMNET_SHARED_MODE
+    );
+
+#if defined(MAC_OS_VERSION_11_0) && \
+    MAC_OS_X_VERSION_MIN_REQUIRED >= MAC_OS_VERSION_11_0
+    xpc_dictionary_set_bool(
+        if_desc,
+        vmnet_enable_isolation_key,
+        options->isolated
+    );
+#else
+    if (options->has_isolated) {
+        error_setg(errp,
+                   "vmnet-shared.isolated feature is "
+                   "unavailable: outdated vmnet.framework API");
+    }
+#endif
+
+    if (options->has_nat66_prefix) {
+        xpc_dictionary_set_string(if_desc,
+                                  vmnet_nat66_prefix_key,
+                                  options->nat66_prefix);
+    }
+
+    if (options->has_start_address ||
+        options->has_end_address ||
+        options->has_subnet_mask) {
+
+        if (options->has_start_address &&
+            options->has_end_address &&
+            options->has_subnet_mask) {
+
+            xpc_dictionary_set_string(if_desc,
+                                      vmnet_start_address_key,
+                                      options->start_address);
+            xpc_dictionary_set_string(if_desc,
+                                      vmnet_end_address_key,
+                                      options->end_address);
+            xpc_dictionary_set_string(if_desc,
+                                      vmnet_subnet_mask_key,
+                                      options->subnet_mask);
+        } else {
+            error_setg(
+                errp,
+                "'start-address', 'end-address', 'subnet-mask' "
+                "should be provided together"
+            );
+        }
+    }
+
+    return if_desc;
+}
+
+static NetClientInfo net_vmnet_shared_info = {
+    .type = NET_CLIENT_DRIVER_VMNET_SHARED,
+    .size = sizeof(VmnetSharedState),
+    .receive = vmnet_receive_common,
+    .cleanup = vmnet_cleanup_common,
+};
+
 int net_init_vmnet_shared(const Netdev *netdev, const char *name,
                           NetClientState *peer, Error **errp)
 {
-  error_setg(errp, "vmnet-shared is not implemented yet");
-  return -1;
+    NetClientState *nc = qemu_new_net_client(&net_vmnet_shared_info,
+                                             peer, "vmnet-shared", name);
+    xpc_object_t if_desc = create_if_desc(netdev, errp);
+
+    return vmnet_if_create(nc, if_desc, errp, NULL);
 }
diff --git a/net/vmnet_int.h b/net/vmnet_int.h
index XXXXXXX..XXXXXXX 100644
--- a/net/vmnet_int.h
+++ b/net/vmnet_int.h
@@ -XXX,XX +XXX,XX @@
 
 #include <vmnet/vmnet.h>
 
+#define VMNET_PACKETS_LIMIT 50
+
 typedef struct VmnetCommonState {
   NetClientState nc;
+  interface_ref vmnet_if;
+
+  bool send_enabled;
+
+  uint64_t mtu;
+  uint64_t max_packet_size;
+
+  struct vmpktdesc packets_buf[VMNET_PACKETS_LIMIT];
+  struct iovec iov_buf[VMNET_PACKETS_LIMIT];
 
 } VmnetCommonState;
 
+const char *vmnet_status_map_str(vmnet_return_t status);
+
+int vmnet_if_create(NetClientState *nc,
+                    xpc_object_t if_desc,
+                    Error **errp,
+                    void (*completion_callback)(xpc_object_t interface_param));
+
+ssize_t vmnet_receive_common(NetClientState *nc,
+                             const uint8_t *buf,
+                             size_t size);
+
+void vmnet_cleanup_common(NetClientState *nc);
 
 #endif /* VMNET_INT_H */
-- 
2.7.4

From: Vladislav Yaroshchuk <yaroshchuk2000@gmail.com>

Signed-off-by: Vladislav Yaroshchuk <yaroshchuk2000@gmail.com>
Signed-off-by: Jason Wang <jasowang@redhat.com>
---
 net/vmnet-host.c | 110 ++++++++++++++++++++++++++++++++++++++++++++++++++++---
 1 file changed, 104 insertions(+), 6 deletions(-)

diff --git a/net/vmnet-host.c b/net/vmnet-host.c
index XXXXXXX..XXXXXXX 100644
--- a/net/vmnet-host.c
+++ b/net/vmnet-host.c
@@ -XXX,XX +XXX,XX @@
  */
 
 #include "qemu/osdep.h"
+#include "qemu/uuid.h"
 #include "qapi/qapi-types-net.h"
-#include "vmnet_int.h"
-#include "clients.h"
-#include "qemu/error-report.h"
 #include "qapi/error.h"
+#include "clients.h"
+#include "vmnet_int.h"
 
 #include <vmnet/vmnet.h>
 
+typedef struct VmnetHostState {
+  VmnetCommonState cs;
+  QemuUUID network_uuid;
+} VmnetHostState;
+
+static xpc_object_t create_if_desc(const Netdev *netdev,
+                                   NetClientState *nc,
+                                   Error **errp)
+{
+    const NetdevVmnetHostOptions *options = &(netdev->u.vmnet_host);
+
+    xpc_object_t if_desc = xpc_dictionary_create(NULL, NULL, 0);
+
+    xpc_dictionary_set_uint64(
+        if_desc,
+        vmnet_operation_mode_key,
+        VMNET_HOST_MODE
+    );
+
+#if defined(MAC_OS_VERSION_11_0) && \
+    MAC_OS_X_VERSION_MIN_REQUIRED >= MAC_OS_VERSION_11_0
+
+    VmnetCommonState *cs = DO_UPCAST(VmnetCommonState, nc, nc);
+    VmnetHostState *hs = DO_UPCAST(VmnetHostState, cs, cs);
+
+    xpc_dictionary_set_bool(
+        if_desc,
+        vmnet_enable_isolation_key,
+        options->isolated
+    );
+
+    if (options->has_net_uuid) {
+        if (qemu_uuid_parse(options->net_uuid, &hs->network_uuid) < 0) {
+            error_setg(errp, "Invalid UUID provided in 'net-uuid'");
+        }
+
+        xpc_dictionary_set_uuid(
+            if_desc,
+            vmnet_network_identifier_key,
+            hs->network_uuid.data
+        );
+    }
+#else
+    if (options->has_isolated) {
+        error_setg(errp,
+                   "vmnet-host.isolated feature is "
+                   "unavailable: outdated vmnet.framework API");
+    }
+
+    if (options->has_net_uuid) {
+        error_setg(errp,
+                   "vmnet-host.net-uuid feature is "
+                   "unavailable: outdated vmnet.framework API");
+    }
+#endif
+
+    if (options->has_start_address ||
+        options->has_end_address ||
+        options->has_subnet_mask) {
+
+        if (options->has_start_address &&
+            options->has_end_address &&
+            options->has_subnet_mask) {
+
+            xpc_dictionary_set_string(if_desc,
+                                      vmnet_start_address_key,
+                                      options->start_address);
+            xpc_dictionary_set_string(if_desc,
+                                      vmnet_end_address_key,
+                                      options->end_address);
+            xpc_dictionary_set_string(if_desc,
+                                      vmnet_subnet_mask_key,
+                                      options->subnet_mask);
+        } else {
+            error_setg(
+                errp,
+                "'start-address', 'end-address', 'subnet-mask' "
+                "should be provided together"
+            );
+        }
+    }
+
+    return if_desc;
+}
+
+static NetClientInfo net_vmnet_host_info = {
+    .type = NET_CLIENT_DRIVER_VMNET_HOST,
+    .size = sizeof(VmnetHostState),
+    .receive = vmnet_receive_common,
+    .cleanup = vmnet_cleanup_common,
+};
+
 int net_init_vmnet_host(const Netdev *netdev, const char *name,
-                        NetClientState *peer, Error **errp) {
-  error_setg(errp, "vmnet-host is not implemented yet");
-  return -1;
+                        NetClientState *peer, Error **errp)
+{
+    NetClientState *nc;
+    xpc_object_t if_desc;
+
+    nc = qemu_new_net_client(&net_vmnet_host_info,
+                             peer, "vmnet-host", name);
+    if_desc = create_if_desc(netdev, nc, errp);
+    return vmnet_if_create(nc, if_desc, errp, NULL);
 }
-- 
2.7.4

From: Vladislav Yaroshchuk <yaroshchuk2000@gmail.com>

Signed-off-by: Vladislav Yaroshchuk <yaroshchuk2000@gmail.com>
Signed-off-by: Jason Wang <jasowang@redhat.com>
---
 net/vmnet-bridged.m | 105 +++++++++++++++++++++++++++++++++++++++++++++++++---
 1 file changed, 100 insertions(+), 5 deletions(-)

diff --git a/net/vmnet-bridged.m b/net/vmnet-bridged.m
index XXXXXXX..XXXXXXX 100644
--- a/net/vmnet-bridged.m
+++ b/net/vmnet-bridged.m
@@ -XXX,XX +XXX,XX @@
 
 #include "qemu/osdep.h"
 #include "qapi/qapi-types-net.h"
-#include "vmnet_int.h"
-#include "clients.h"
-#include "qemu/error-report.h"
 #include "qapi/error.h"
+#include "clients.h"
+#include "vmnet_int.h"
 
 #include <vmnet/vmnet.h>
 
+typedef struct VmnetBridgedState {
+  VmnetCommonState cs;
+} VmnetBridgedState;
+
+static bool validate_ifname(const char *ifname)
+{
+    xpc_object_t shared_if_list = vmnet_copy_shared_interface_list();
+    __block bool match = false;
+
+    xpc_array_apply(
+        shared_if_list,
+        ^bool(size_t index, xpc_object_t value) {
+          if (strcmp(xpc_string_get_string_ptr(value), ifname) == 0) {
+              match = true;
+              return false;
+          }
+          return true;
+        });
+
+    return match;
+}
+
+static const char *get_valid_ifnames(void)
+{
+    xpc_object_t shared_if_list = vmnet_copy_shared_interface_list();
+    __block char *if_list = NULL;
+
+    xpc_array_apply(
+        shared_if_list,
+        ^bool(size_t index, xpc_object_t value) {
+          if_list = g_strconcat(xpc_string_get_string_ptr(value),
+                                " ",
+                                if_list,
+                                NULL);
+          return true;
+        });
+
+    if (if_list) {
+        return if_list;
+    }
+    return "[no interfaces]";
+}
+
+static xpc_object_t create_if_desc(const Netdev *netdev, Error **errp)
+{
+    const NetdevVmnetBridgedOptions *options = &(netdev->u.vmnet_bridged);
+    xpc_object_t if_desc = xpc_dictionary_create(NULL, NULL, 0);
+
+    xpc_dictionary_set_uint64(
+        if_desc,
+        vmnet_operation_mode_key,
+        VMNET_BRIDGED_MODE
+    );
+
+#if defined(MAC_OS_VERSION_11_0) && \
+    MAC_OS_X_VERSION_MIN_REQUIRED >= MAC_OS_VERSION_11_0
+    xpc_dictionary_set_bool(
+        if_desc,
+        vmnet_enable_isolation_key,
+        options->isolated
+    );
+#else
+    if (options->has_isolated) {
+        error_setg(errp,
+                   "vmnet-bridged.isolated feature is "
+                   "unavailable: outdated vmnet.framework API");
+    }
+#endif
+
+    if (validate_ifname(options->ifname)) {
+        xpc_dictionary_set_string(if_desc,
+                                  vmnet_shared_interface_name_key,
+                                  options->ifname);
+    } else {
+        return NULL;
+    }
+    return if_desc;
+}
+
+static NetClientInfo net_vmnet_bridged_info = {
+    .type = NET_CLIENT_DRIVER_VMNET_BRIDGED,
+    .size = sizeof(VmnetBridgedState),
+    .receive = vmnet_receive_common,
+    .cleanup = vmnet_cleanup_common,
+};
+
 int net_init_vmnet_bridged(const Netdev *netdev, const char *name,
                            NetClientState *peer, Error **errp)
 {
-  error_setg(errp, "vmnet-bridged is not implemented yet");
-  return -1;
+    NetClientState *nc = qemu_new_net_client(&net_vmnet_bridged_info,
+                                             peer, "vmnet-bridged", name);
+    xpc_object_t if_desc = create_if_desc(netdev, errp);;
+
+    if (!if_desc) {
+        error_setg(errp,
+                   "unsupported ifname, should be one of: %s",
+                   get_valid_ifnames());
+        return -1;
+    }
+
+    return vmnet_if_create(nc, if_desc, errp, NULL);
 }
-- 
2.7.4

From: Vladislav Yaroshchuk <yaroshchuk2000@gmail.com>

Signed-off-by: Vladislav Yaroshchuk <yaroshchuk2000@gmail.com>
Signed-off-by: Jason Wang <jasowang@redhat.com>
---
 qemu-options.hx | 25 +++++++++++++++++++++++++
 1 file changed, 25 insertions(+)

diff --git a/qemu-options.hx b/qemu-options.hx
index XXXXXXX..XXXXXXX 100644
--- a/qemu-options.hx
+++ b/qemu-options.hx
@@ -XXX,XX +XXX,XX @@ DEF("netdev", HAS_ARG, QEMU_OPTION_netdev,
     "-netdev vhost-vdpa,id=str,vhostdev=/path/to/dev\n"
     "                configure a vhost-vdpa network,Establish a vhost-vdpa netdev\n"
 #endif
+#ifdef CONFIG_VMNET
+    "-netdev vmnet-host,id=str[,isolated=on|off][,net-uuid=uuid]\n"
+    "         [,start-address=addr,end-address=addr,subnet-mask=mask]\n"
+    "                configure a vmnet network backend in host mode with ID 'str',\n"
+    "                isolate this interface from others with 'isolated',\n"
+    "                configure the address range and choose a subnet mask,\n"
+    "                specify network UUID 'uuid' to disable DHCP and interact with\n"
+    "                vmnet-host interfaces within this isolated network\n"
+    "-netdev vmnet-shared,id=str[,isolated=on|off][,nat66-prefix=addr]\n"
+    "         [,start-address=addr,end-address=addr,subnet-mask=mask]\n"
+    "                configure a vmnet network backend in shared mode with ID 'str',\n"
+    "                configure the address range and choose a subnet mask,\n"
+    "                set IPv6 ULA prefix (of length 64) to use for internal network,\n"
+    "                isolate this interface from others with 'isolated'\n"
+    "-netdev vmnet-bridged,id=str,ifname=name[,isolated=on|off]\n"
+    "                configure a vmnet network backend in bridged mode with ID 'str',\n"
+    "                use 'ifname=name' to select a physical network interface to be bridged,\n"
+    "                isolate this interface from others with 'isolated'\n"
+#endif
     "-netdev hubport,id=str,hubid=n[,netdev=nd]\n"
     "                configure a hub port on the hub with ID 'n'\n", QEMU_ARCH_ALL)
 DEF("nic", HAS_ARG, QEMU_OPTION_nic,
@@ -XXX,XX +XXX,XX @@ DEF("nic", HAS_ARG, QEMU_OPTION_nic,
 #ifdef CONFIG_POSIX
     "vhost-user|"
 #endif
+#ifdef CONFIG_VMNET
+    "vmnet-host|vmnet-shared|vmnet-bridged|"
+#endif
     "socket][,option][,...][mac=macaddr]\n"
     "                initialize an on-board / default host NIC (using MAC address\n"
     "                macaddr) and connect it to the given host network backend\n"
@@ -XXX,XX +XXX,XX @@ DEF("net", HAS_ARG, QEMU_OPTION_net,
 #ifdef CONFIG_NETMAP
     "netmap|"
 #endif
+#ifdef CONFIG_VMNET
+    "vmnet-host|vmnet-shared|vmnet-bridged|"
+#endif
     "socket][,option][,option][,...]\n"
     "                old way to initialize a host network interface\n"
     "                (use the -netdev option if possible instead)\n", QEMU_ARCH_ALL)
-- 
2.7.4