The following changes since commit 17783ac828adc694d986698d2d7014aedfeb48c6:

Merge remote-tracking branch 'remotes/dgibson/tags/ppc-for-2.9-20170303' into staging (2017-03-04 16:31:14 +0000)

for you to fetch changes up to f0aabd5c4ae11fde704d138e8f06c69e5c902a16:

net/filter-mirror: Follow CODING_STYLE (2017-03-06 11:46:02 +0800)

- fix vlan issues for e1000e

- convert to use vmstate for vmxnet3

- several fixes for colo

Dmitry Fleytman (5):

eth: Extend vlan stripping functions

NetRxPkt: Fix memory corruption on VLAN header stripping

NetRxPkt: Do not try to pull more data than present

NetRxPkt: Account buffer with ETH header in IOV length

NetRxPkt: Remove code duplication in net_rx_pkt_pull_data()

Dr. David Alan Gilbert (2):

vmxnet3: Convert ring values to uint32_t's

vmxnet3: VMStatify rx/tx q_descr and int_state

Fam Zheng (1):

net: Remove useless local var pkt

Zhang Chen (5):

net/colo-compare: Fix memory free error

COLO-compare: Rename compare function and remove duplicate codes

COLO-compare: Optimize compare_common and compare_tcp

COLO-compare: Fix icmp and udp compare different packet always dump bug

net/filter-mirror: Follow CODING_STYLE

zhanghailiang (6):

colo-compare: use g_timeout_source_new() to process the stale packets

colo-compare: kick compare thread to exit after some cleanup in finalization

char: remove the right fd been watched in qemu_chr_fe_set_handlers()

colo-compare: Fix removing fds been watched incorrectly in finalization

net/colo: fix memory double free error

filter-rewriter: skip net_checksum_calculate() while offset = 0

chardev/char-fd.c | 6 +-

chardev/char-io.c | 8 +-

chardev/char-io.h | 2 +-

chardev/char-pty.c | 2 +-

chardev/char-socket.c | 4 +-

chardev/char-udp.c | 6 +-

chardev/char.c | 2 +-

hw/net/net_rx_pkt.c | 41 ++++----

hw/net/vmxnet3.c | 284 +++++++++++++++++---------------------------------

include/net/eth.h | 4 +-

net/colo-compare.c | 189 +++++++++++++++++++--------------

net/colo.c | 4 +-

net/eth.c | 25 +++--

net/filter-mirror.c | 7 +-

net/filter-rewriter.c | 17 +--

15 files changed, 273 insertions(+), 328 deletions(-)

From: Zhang Chen <zhangchen.fnst@cn.fujitsu.com>

Signed-off-by: Zhang Chen <zhangchen.fnst@cn.fujitsu.com>

net/colo-compare.c | 18 ++++++++++++------

1 file changed, 12 insertions(+), 6 deletions(-)

diff --git a/net/colo-compare.c b/net/colo-compare.c

--- a/net/colo-compare.c

+++ b/net/colo-compare.c

@@ -XXX,XX +XXX,XX @@ static int colo_packet_compare_udp(Packet *spkt, Packet *ppkt)

if (ret) {

trace_colo_compare_udp_miscompare("primary pkt size", ppkt->size);

- qemu_hexdump((char *)ppkt->data, stderr, "colo-compare", ppkt->size);

trace_colo_compare_udp_miscompare("Secondary pkt size", spkt->size);

- qemu_hexdump((char *)spkt->data, stderr, "colo-compare", spkt->size);

+ if (trace_event_get_state(TRACE_COLO_COMPARE_MISCOMPARE)) {

+ qemu_hexdump((char *)ppkt->data, stderr, "colo-compare pri pkt",

+ ppkt->size);

+ qemu_hexdump((char *)spkt->data, stderr, "colo-compare sec pkt",

+ spkt->size);

+ }

From: Zhang Chen <zhangchen.fnst@cn.fujitsu.com>

Signed-off-by: Zhang Chen <zhangchen.fnst@cn.fujitsu.com>

net/filter-mirror.c | 7 ++++---

1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/net/filter-mirror.c b/net/filter-mirror.c

--- a/net/filter-mirror.c

+++ b/net/filter-mirror.c

@@ -XXX,XX +XXX,XX @@ static int filter_mirror_send(CharBackend *chr_out,

{

int ret = 0;

ssize_t size = 0;

- uint32_t len = 0;

+ uint32_t len = 0;

char *buf;

size = iov_size(iov, iovcnt);

@@ -XXX,XX +XXX,XX @@ err:

return ret < 0 ? ret : -EIO;

}

-static void

-redirector_to_filter(NetFilterState *nf, const uint8_t *buf, int len)

+static void redirector_to_filter(NetFilterState *nf,

+ const uint8_t *buf,

+ int len)

{

struct iovec iov = {

.iov_base = (void *)buf,

From: zhanghailiang <zhang.zhanghailiang@huawei.com>

We should call g_main_loop_quit() to notify colo compare thread to

exit, Or it will run in g_main_loop_run() forever.

Besides, the finalizing process can't happen in context of colo thread,

it is reasonable to remove the 'if (qemu_thread_is_self(&s->thread))'

branch.

Before compare thead exits, some cleanup works need to be

done, All unhandled packets need to be released and connection_track_table

needs to be freed, or there will be memory leak.

Signed-off-by: zhanghailiang <zhang.zhanghailiang@huawei.com>

Reviewed-by: Zhang Chen <zhangchen.fnst@cn.fujitsu.com>

net/colo-compare.c | 39 +++++++++++++++++++++++++++++----------

1 file changed, 29 insertions(+), 10 deletions(-)

diff --git a/net/colo-compare.c b/net/colo-compare.c

--- a/net/colo-compare.c

+++ b/net/colo-compare.c

@@ -XXX,XX +XXX,XX @@ typedef struct CompareState {

GHashTable *connection_track_table;

/* compare thread, a thread for each NIC */

QemuThread thread;

+

+ GMainLoop *compare_loop;

} CompareState;

typedef struct CompareClass {

@@ -XXX,XX +XXX,XX @@ static gboolean check_old_packet_regular(void *opaque)

static void *colo_compare_thread(void *opaque)

{

GMainContext *worker_context;

- GMainLoop *compare_loop;

CompareState *s = opaque;

GSource *timeout_source;

@@ -XXX,XX +XXX,XX @@ static void *colo_compare_thread(void *opaque)

qemu_chr_fe_set_handlers(&s->chr_sec_in, compare_chr_can_read,

compare_sec_chr_in, NULL, s, worker_context, true);

- compare_loop = g_main_loop_new(worker_context, FALSE);

+ s->compare_loop = g_main_loop_new(worker_context, FALSE);

/* To kick any packets that the secondary doesn't match */

timeout_source = g_timeout_source_new(REGULAR_PACKET_CHECK_MS);

@@ -XXX,XX +XXX,XX @@ static void *colo_compare_thread(void *opaque)

(GSourceFunc)check_old_packet_regular, s, NULL);

g_source_attach(timeout_source, worker_context);

- g_main_loop_run(compare_loop);

+ g_main_loop_run(s->compare_loop);

g_source_unref(timeout_source);

- g_main_loop_unref(compare_loop);

+ g_main_loop_unref(s->compare_loop);

g_main_context_unref(worker_context);

return NULL;

}

@@ -XXX,XX +XXX,XX @@ static void colo_compare_complete(UserCreatable *uc, Error **errp)

+static void colo_flush_packets(void *opaque, void *user_data)

+{

+ CompareState *s = user_data;

+ Connection *conn = opaque;

+ Packet *pkt = NULL;

+

+ while (!g_queue_is_empty(&conn->primary_list)) {

+ pkt = g_queue_pop_head(&conn->primary_list);

+ compare_chr_send(&s->chr_out, pkt->data, pkt->size);

+ packet_destroy(pkt, NULL);

+ }

+ while (!g_queue_is_empty(&conn->secondary_list)) {

+ pkt = g_queue_pop_head(&conn->secondary_list);

+ packet_destroy(pkt, NULL);

+ }

+}

+

static void colo_compare_class_init(ObjectClass *oc, void *data)

{

UserCreatableClass *ucc = USER_CREATABLE_CLASS(oc);

@@ -XXX,XX +XXX,XX @@ static void colo_compare_finalize(Object *obj)

qemu_chr_fe_deinit(&s->chr_sec_in);

qemu_chr_fe_deinit(&s->chr_out);

- g_queue_free(&s->conn_list);

+ g_main_loop_quit(s->compare_loop);

+ qemu_thread_join(&s->thread);

- if (qemu_thread_is_self(&s->thread)) {

- /* compare connection */

- g_queue_foreach(&s->conn_list, colo_compare_connection, s);

- qemu_thread_join(&s->thread);

- }

+ /* Release all unhandled packets after compare thead exited */

+ g_queue_foreach(&s->conn_list, colo_flush_packets, s);

+

+ g_queue_free(&s->conn_list);

+ g_hash_table_destroy(s->connection_track_table);

g_free(s->pri_indev);

g_free(s->sec_indev);

g_free(s->outdev);

From: zhanghailiang <zhang.zhanghailiang@huawei.com>

While the offset of packets's sequence for primary side and

secondary side is zero, it is unnecessary to call net_checksum_calculate()

to recalculate the checksume value of packets.

Signed-off-by: zhanghailiang <zhang.zhanghailiang@huawei.com>

net/filter-rewriter.c | 17 +++++++++++------

1 file changed, 11 insertions(+), 6 deletions(-)

diff --git a/net/filter-rewriter.c b/net/filter-rewriter.c

--- a/net/filter-rewriter.c

+++ b/net/filter-rewriter.c

@@ -XXX,XX +XXX,XX @@ static int handle_primary_tcp_pkt(NetFilterState *nf,

conn->offset -= (ntohl(tcp_pkt->th_ack) - 1);

conn->syn_flag = 0;

}

- /* handle packets to the secondary from the primary */

- tcp_pkt->th_ack = htonl(ntohl(tcp_pkt->th_ack) + conn->offset);

+ if (conn->offset) {

+ /* handle packets to the secondary from the primary */

+ tcp_pkt->th_ack = htonl(ntohl(tcp_pkt->th_ack) + conn->offset);

- net_checksum_calculate((uint8_t *)pkt->data, pkt->size);

+ net_checksum_calculate((uint8_t *)pkt->data, pkt->size);

+ }

return 0;

@@ -XXX,XX +XXX,XX @@ static int handle_secondary_tcp_pkt(NetFilterState *nf,

if ((tcp_pkt->th_flags & (TH_ACK | TH_SYN)) == TH_ACK) {

- /* handle packets to the primary from the secondary*/

- tcp_pkt->th_seq = htonl(ntohl(tcp_pkt->th_seq) - conn->offset);

+ /* Only need to adjust seq while offset is Non-zero */

+ if (conn->offset) {

+ /* handle packets to the primary from the secondary*/

+ tcp_pkt->th_seq = htonl(ntohl(tcp_pkt->th_seq) - conn->offset);

- net_checksum_calculate((uint8_t *)pkt->data, pkt->size);

+ net_checksum_calculate((uint8_t *)pkt->data, pkt->size);

+ }

}

return 0;

From: Zhang Chen <zhangchen.fnst@cn.fujitsu.com>

Add offset args for colo_packet_compare_common, optimize

colo_packet_compare_icmp() and colo_packet_compare_udp()

just compare the IP payload. Before compare all tcp packet,

we compare tcp checksum firstly, this function can get

better performance.

Signed-off-by: Zhang Chen <zhangchen.fnst@cn.fujitsu.com>

net/colo-compare.c | 50 ++++++++++++++++++++++++++++++++++++--------------

1 file changed, 36 insertions(+), 14 deletions(-)

diff --git a/net/colo-compare.c b/net/colo-compare.c

--- a/net/colo-compare.c

+++ b/net/colo-compare.c

@@ -XXX,XX +XXX,XX @@ static int packet_enqueue(CompareState *s, int mode)

* return: 0 means packet same

* > 0 || < 0 means packet different

*/

-static int colo_packet_compare_common(Packet *ppkt, Packet *spkt)

+static int colo_packet_compare_common(Packet *ppkt, Packet *spkt, int offset)

trace_colo_compare_ip_info(ppkt->size, inet_ntoa(ppkt->ip->ip_src),

inet_ntoa(ppkt->ip->ip_dst), spkt->size,

@@ -XXX,XX +XXX,XX @@ static int colo_packet_compare_common(Packet *ppkt, Packet *spkt)

inet_ntoa(spkt->ip->ip_dst));

if (ppkt->size == spkt->size) {

- return memcmp(ppkt->data, spkt->data, spkt->size);

+ return memcmp(ppkt->data + offset, spkt->data + offset,

+ spkt->size - offset);

} else {

trace_colo_compare_main("Net packet size are not the same");

return -1;

@@ -XXX,XX +XXX,XX @@ static int colo_packet_compare_tcp(Packet *spkt, Packet *ppkt)

trace_colo_compare_main("compare tcp");

- if (ppkt->size != spkt->size) {

- if (trace_event_get_state(TRACE_COLO_COMPARE_MISCOMPARE)) {

- trace_colo_compare_main("pkt size not same");

- }

- return -1;

- }

-

ptcp = (struct tcphdr *)ppkt->transport_header;

stcp = (struct tcphdr *)spkt->transport_header;

@@ -XXX,XX +XXX,XX @@ static int colo_packet_compare_tcp(Packet *spkt, Packet *ppkt)

spkt->ip->ip_sum = ppkt->ip->ip_sum;

- res = memcmp(ppkt->data + ETH_HLEN, spkt->data + ETH_HLEN,

- (spkt->size - ETH_HLEN));

+ if (ptcp->th_sum == stcp->th_sum) {

+ res = colo_packet_compare_common(ppkt, spkt, ETH_HLEN);

+ } else {

+ res = -1;

trace_colo_compare_main("compare icmp");

- if (colo_packet_compare_common(ppkt, spkt)) {

+ /*

+ * Because of ppkt and spkt are both in the same connection,

+ * The ppkt's src ip, dst ip, src port, dst port, ip_proto all are

+ * same with spkt. In addition, IP header's Identification is a random

+ * field, we can handle it in IP fragmentation function later.

+ * COLO just concern the response net packet payload from primary guest

+ * and secondary guest are same or not, So we ignored all IP header include

+ * other field like TOS,TTL,IP Checksum. we only need to compare

+ * the ip payload here.

+ */

+ if (colo_packet_compare_common(ppkt, spkt,

+ network_header_length + ETH_HLEN)) {

trace_colo_compare_icmp_miscompare("primary pkt size",

ppkt->size);

qemu_hexdump((char *)ppkt->data, stderr, "colo-compare",

@@ -XXX,XX +XXX,XX @@ static int colo_packet_compare_other(Packet *spkt, Packet *ppkt)

inet_ntoa(ppkt->ip->ip_dst), spkt->size,

inet_ntoa(spkt->ip->ip_src),

inet_ntoa(spkt->ip->ip_dst));

- return colo_packet_compare_common(ppkt, spkt);

+ return colo_packet_compare_common(ppkt, spkt, 0);

}

static int colo_old_packet_check_one(Packet *pkt, int64_t *check_time)

From: "Dr. David Alan Gilbert" <dgilbert@redhat.com>

The index's in the Vmxnet3Ring were migrated as 32bit ints

yet are declared as size_t's. They appear to be derived

from 32bit values loaded from guest memory, so actually

store them as that.

Signed-off-by: Dr. David Alan Gilbert <dgilbert@redhat.com>

Acked-by: Dmitry Fleytman <dmitry@daynix.com>

hw/net/vmxnet3.c | 12 ++++++------

1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/hw/net/vmxnet3.c b/hw/net/vmxnet3.c

--- a/hw/net/vmxnet3.c

+++ b/hw/net/vmxnet3.c

@@ -XXX,XX +XXX,XX @@ typedef struct VMXNET3Class {

/* Cyclic ring abstraction */

typedef struct {

hwaddr pa;

- size_t size;

- size_t cell_size;

- size_t next;

+ uint32_t size;

+ uint32_t cell_size;

+ uint32_t next;

uint8_t gen;

} Vmxnet3Ring;

static inline void vmxnet3_ring_init(PCIDevice *d,

                 Vmxnet3Ring *ring,

hwaddr pa,

- size_t size,

- size_t cell_size,

+ uint32_t size,

+ uint32_t cell_size,

bool zero_region)

{

ring->pa = pa;

@@ -XXX,XX +XXX,XX @@ static inline void vmxnet3_ring_init(PCIDevice *d,

#define VMXNET3_RING_DUMP(macro, ring_name, ridx, r) \

- macro("%s#%d: base %" PRIx64 " size %zu cell_size %zu gen %d next %zu", \

+ macro("%s#%d: base %" PRIx64 " size %u cell_size %u gen %d next %u", \

(ring_name), (ridx), \

(r)->pa, (r)->size, (r)->cell_size, (r)->gen, (r)->next)

From: zhanghailiang <zhang.zhanghailiang@huawei.com>

Instead of using qemu timer to process the stale packets,

We re-use the colo compare thread to process these packets

by creating a new timeout coroutine.

Besides, since we process all the same vNIC's net connection/packets

in one thread, it is safe to remove the timer_check_lock.

Signed-off-by: zhanghailiang <zhang.zhanghailiang@huawei.com>

net/colo-compare.c | 62 +++++++++++++++++++-----------------------------------

1 file changed, 22 insertions(+), 40 deletions(-)

diff --git a/net/colo-compare.c b/net/colo-compare.c

--- a/net/colo-compare.c

+++ b/net/colo-compare.c

@@ -XXX,XX +XXX,XX @@ typedef struct CompareState {

GHashTable *connection_track_table;

/* compare thread, a thread for each NIC */

QemuThread thread;

- /* Timer used on the primary to find packets that are never matched */

- QEMUTimer *timer;

- QemuMutex timer_check_lock;

} CompareState;

typedef struct CompareClass {

@@ -XXX,XX +XXX,XX @@ static void colo_compare_connection(void *opaque, void *user_data)

while (!g_queue_is_empty(&conn->primary_list) &&

!g_queue_is_empty(&conn->secondary_list)) {

- qemu_mutex_lock(&s->timer_check_lock);

pkt = g_queue_pop_tail(&conn->primary_list);

- qemu_mutex_unlock(&s->timer_check_lock);

switch (conn->ip_proto) {

case IPPROTO_TCP:

result = g_queue_find_custom(&conn->secondary_list,

@@ -XXX,XX +XXX,XX @@ static void colo_compare_connection(void *opaque, void *user_data)

* until next comparison.

*/

trace_colo_compare_main("packet different");

- qemu_mutex_lock(&s->timer_check_lock);

g_queue_push_tail(&conn->primary_list, pkt);

- qemu_mutex_unlock(&s->timer_check_lock);

/* TODO: colo_notify_checkpoint();*/

break;

}

@@ -XXX,XX +XXX,XX @@ static void compare_sec_chr_in(void *opaque, const uint8_t *buf, int size)

}

}

+/*

+ * Check old packet regularly so it can watch for any packets

+ * that the secondary hasn't produced equivalents of.

+ */

+static gboolean check_old_packet_regular(void *opaque)

+ CompareState *s = opaque;

+

+ /* if have old packet we will notify checkpoint */

+ colo_old_packet_check(s);

+