The following changes since commit 6632f6ff96f0537fc34cdc00c760656fc62e23c5:

Merge remote-tracking branch 'remotes/famz/tags/block-and-testing-pull-request' into staging (2017-07-17 11:46:36 +0100)

for you to fetch changes up to 189ae6bb5ce1f5a322f8691d00fe942ba43dd601:

virtio-net: fix offload ctrl endian (2017-07-17 20:13:56 +0800)

- fix virtio-net ctrl offload endian

- vnet header support for variou COLO netfilters and compare thread

Jason Wang (1):

virtio-net: fix offload ctrl endian

Michal Privoznik (1):

virtion-net: Prefer is_power_of_2()

Zhang Chen (12):

net: Add vnet_hdr_len arguments in NetClientState

net/net.c: Add vnet_hdr support in SocketReadState

net/filter-mirror.c: Introduce parameter for filter_send()

net/filter-mirror.c: Make filter mirror support vnet support.

net/filter-mirror.c: Add new option to enable vnet support for filter-redirector

net/colo.c: Make vnet_hdr_len as packet property

net/colo-compare.c: Introduce parameter for compare_chr_send()

net/colo-compare.c: Make colo-compare support vnet_hdr_len

net/colo.c: Add vnet packet parse feature in colo-proxy

net/colo-compare.c: Add vnet packet's tcp/udp/icmp compare

net/filter-rewriter.c: Make filter-rewriter support vnet_hdr_len

docs/colo-proxy.txt: Update colo-proxy usage of net driver with vnet_header

docs/colo-proxy.txt | 26 ++++++++++++++++

hw/net/virtio-net.c | 4 ++-

include/net/net.h | 10 ++++--

net/colo-compare.c | 84 ++++++++++++++++++++++++++++++++++++++++++---------

net/colo.c | 9 +++---

net/colo.h | 4 ++-

net/filter-mirror.c | 75 +++++++++++++++++++++++++++++++++++++++++----

net/filter-rewriter.c | 37 ++++++++++++++++++++++-

net/net.c | 37 ++++++++++++++++++++---

net/socket.c | 8 ++---

qemu-options.hx | 19 ++++++------

11 files changed, 265 insertions(+), 48 deletions(-)

From: Zhang Chen <zhangchen.fnst@cn.fujitsu.com>

Signed-off-by: Zhang Chen <zhangchen.fnst@cn.fujitsu.com>

docs/colo-proxy.txt | 26 ++++++++++++++++++++++++++

1 file changed, 26 insertions(+)

diff --git a/docs/colo-proxy.txt b/docs/colo-proxy.txt

--- a/docs/colo-proxy.txt

+++ b/docs/colo-proxy.txt

@@ -XXX,XX +XXX,XX @@ Secondary(ip:3.3.3.8):

-chardev socket,id=red1,host=3.3.3.3,port=9004

-object filter-redirector,id=f1,netdev=hn0,queue=tx,indev=red0

-object filter-redirector,id=f2,netdev=hn0,queue=rx,outdev=red1

+-object filter-rewriter,id=f3,netdev=hn0,queue=all

+If you want to use virtio-net-pci or other driver with vnet_header:

+

+Primary(ip:3.3.3.3):

+-netdev tap,id=hn0,vhost=off,script=/etc/qemu-ifup,downscript=/etc/qemu-ifdown

+-device e1000,id=e0,netdev=hn0,mac=52:a4:00:12:78:66

+-chardev socket,id=mirror0,host=3.3.3.3,port=9003,server,nowait

+-chardev socket,id=compare1,host=3.3.3.3,port=9004,server,nowait

+-chardev socket,id=compare0,host=3.3.3.3,port=9001,server,nowait

+-chardev socket,id=compare0-0,host=3.3.3.3,port=9001

+-chardev socket,id=compare_out,host=3.3.3.3,port=9005,server,nowait

+-chardev socket,id=compare_out0,host=3.3.3.3,port=9005

+-object filter-mirror,id=m0,netdev=hn0,queue=tx,outdev=mirror0,vnet_hdr_support

+-object filter-redirector,netdev=hn0,id=redire0,queue=rx,indev=compare_out,vnet_hdr_support

+-object filter-redirector,netdev=hn0,id=redire1,queue=rx,outdev=compare0,vnet_hdr_support

+-object colo-compare,id=comp0,primary_in=compare0-0,secondary_in=compare1,outdev=compare_out0,vnet_hdr_support

+

+Secondary(ip:3.3.3.8):

+-netdev tap,id=hn0,vhost=off,script=/etc/qemu-ifup,down script=/etc/qemu-ifdown

+-device e1000,netdev=hn0,mac=52:a4:00:12:78:66

+-chardev socket,id=red0,host=3.3.3.3,port=9003

+-chardev socket,id=red1,host=3.3.3.3,port=9004

+-object filter-redirector,id=f1,netdev=hn0,queue=tx,indev=red0,vnet_hdr_support

+-object filter-redirector,id=f2,netdev=hn0,queue=rx,outdev=red1,vnet_hdr_support

+-object filter-rewriter,id=f3,netdev=hn0,queue=all,vnet_hdr_support

Note:

a.COLO-proxy must work with COLO-frame and Block-replication.

2.7.4

From: Zhang Chen <zhangchen.fnst@cn.fujitsu.com>

COLO-Proxy just focus on packet payload, so we skip vnet header.

Signed-off-by: Zhang Chen <zhangchen.fnst@cn.fujitsu.com>

net/colo-compare.c | 8 ++++++--

1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/net/colo-compare.c b/net/colo-compare.c

--- a/net/colo-compare.c

+++ b/net/colo-compare.c

@@ -XXX,XX +XXX,XX @@ static int colo_packet_compare_common(Packet *ppkt, Packet *spkt, int offset)

sec_ip_src, sec_ip_dst);

+ offset = ppkt->vnet_hdr_len + offset;

if (ppkt->size == spkt->size) {

- return memcmp(ppkt->data + offset, spkt->data + offset,

+ return memcmp(ppkt->data + offset,

+ spkt->data + offset,

spkt->size - offset);

} else {

trace_colo_compare_main("Net packet size are not the same");

@@ -XXX,XX +XXX,XX @@ static int colo_packet_compare_tcp(Packet *spkt, Packet *ppkt)

*/

if (ptcp->th_off > 5) {

ptrdiff_t tcp_offset;

tcp_offset = ppkt->transport_header - (uint8_t *)ppkt->data

- + (ptcp->th_off * 4);

+ + (ptcp->th_off * 4) - ppkt->vnet_hdr_len;

res = colo_packet_compare_common(ppkt, spkt, tcp_offset);

} else if (ptcp->th_sum == stcp->th_sum) {

res = colo_packet_compare_common(ppkt, spkt, ETH_HLEN);

2.7.4

From: Michal Privoznik <mprivozn@redhat.com>

We have a function that checks if given number is power of two.

We should prefer it instead of expanding the check on our own.

Signed-off-by: Michal Privoznik <mprivozn@redhat.com>

hw/net/virtio-net.c | 2 +-

diff --git a/hw/net/virtio-net.c b/hw/net/virtio-net.c

--- a/hw/net/virtio-net.c

+++ b/hw/net/virtio-net.c

@@ -XXX,XX +XXX,XX @@ static void virtio_net_device_realize(DeviceState *dev, Error **errp)

*/

if (n->net_conf.rx_queue_size < VIRTIO_NET_RX_QUEUE_MIN_SIZE ||

n->net_conf.rx_queue_size > VIRTQUEUE_MAX_SIZE ||

- (n->net_conf.rx_queue_size & (n->net_conf.rx_queue_size - 1))) {

+ !is_power_of_2(n->net_conf.rx_queue_size)) {

error_setg(errp, "Invalid rx_queue_size (= %" PRIu16 "), "

"must be a power of 2 between %d and %d.",

n->net_conf.rx_queue_size, VIRTIO_NET_RX_QUEUE_MIN_SIZE,

2.7.4

From: Zhang Chen <zhangchen.fnst@cn.fujitsu.com>

This patch change the compare_chr_send() parameter from CharBackend to CompareState,

we can get more information like vnet_hdr(We use it to support packet with vnet_header).

Signed-off-by: Zhang Chen <zhangchen.fnst@cn.fujitsu.com>

net/colo-compare.c | 14 +++++++-------

1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/net/colo-compare.c b/net/colo-compare.c

--- a/net/colo-compare.c

+++ b/net/colo-compare.c

@@ -XXX,XX +XXX,XX @@ enum {

SECONDARY_IN,

};

-static int compare_chr_send(CharBackend *out,

+static int compare_chr_send(CompareState *s,

const uint8_t *buf,

uint32_t size);

@@ -XXX,XX +XXX,XX @@ static void colo_compare_connection(void *opaque, void *user_data)

}

if (result) {

- ret = compare_chr_send(&s->chr_out, pkt->data, pkt->size);

+ ret = compare_chr_send(s, pkt->data, pkt->size);

if (ret < 0) {

error_report("colo_send_primary_packet failed");

}

@@ -XXX,XX +XXX,XX @@ static void colo_compare_connection(void *opaque, void *user_data)

-static int compare_chr_send(CharBackend *out,

+static int compare_chr_send(CompareState *s,

const uint8_t *buf,

uint32_t size)

@@ -XXX,XX +XXX,XX @@ static int compare_chr_send(CharBackend *out,

return 0;

}

- ret = qemu_chr_fe_write_all(out, (uint8_t *)&len, sizeof(len));

+ ret = qemu_chr_fe_write_all(&s->chr_out, (uint8_t *)&len, sizeof(len));

if (ret != sizeof(len)) {

goto err;

}

- ret = qemu_chr_fe_write_all(out, (uint8_t *)buf, size);

+ ret = qemu_chr_fe_write_all(&s->chr_out, (uint8_t *)buf, size);

if (ret != size) {

goto err;

}

@@ -XXX,XX +XXX,XX @@ static void compare_pri_rs_finalize(SocketReadState *pri_rs)

if (packet_enqueue(s, PRIMARY_IN)) {

trace_colo_compare_main("primary: unsupported packet in");

- compare_chr_send(&s->chr_out, pri_rs->buf, pri_rs->packet_len);

+ compare_chr_send(s, pri_rs->buf, pri_rs->packet_len);

} else {

/* compare connection */

g_queue_foreach(&s->conn_list, colo_compare_connection, s);

@@ -XXX,XX +XXX,XX @@ static void colo_flush_packets(void *opaque, void *user_data)

while (!g_queue_is_empty(&conn->primary_list)) {

pkt = g_queue_pop_head(&conn->primary_list);

- compare_chr_send(&s->chr_out, pkt->data, pkt->size);

+ compare_chr_send(s, pkt->data, pkt->size);

packet_destroy(pkt, NULL);

}

while (!g_queue_is_empty(&conn->secondary_list)) {

2.7.4

From: Zhang Chen <zhangchen.fnst@cn.fujitsu.com>

We can use this property flush and send packet with vnet_hdr_len.

Signed-off-by: Zhang Chen <zhangchen.fnst@cn.fujitsu.com>

net/colo-compare.c | 8 ++++++--

net/colo.c | 3 ++-

net/colo.h | 4 +++-

net/filter-rewriter.c | 2 +-

4 files changed, 12 insertions(+), 5 deletions(-)

diff --git a/net/colo-compare.c b/net/colo-compare.c

--- a/net/colo-compare.c

+++ b/net/colo-compare.c

@@ -XXX,XX +XXX,XX @@ static int packet_enqueue(CompareState *s, int mode)

Connection *conn;

if (mode == PRIMARY_IN) {

- pkt = packet_new(s->pri_rs.buf, s->pri_rs.packet_len);

+ pkt = packet_new(s->pri_rs.buf,

+ s->pri_rs.packet_len,

+ s->pri_rs.vnet_hdr_len);

} else {

- pkt = packet_new(s->sec_rs.buf, s->sec_rs.packet_len);

+ pkt = packet_new(s->sec_rs.buf,

+ s->sec_rs.packet_len,

+ s->sec_rs.vnet_hdr_len);

if (parse_packet_early(pkt)) {

diff --git a/net/colo.c b/net/colo.c

index XXXXXXX..XXXXXXX 100644

--- a/net/colo.c

+++ b/net/colo.c

@@ -XXX,XX +XXX,XX @@ void connection_destroy(void *opaque)

g_slice_free(Connection, conn);

-Packet *packet_new(const void *data, int size)

+Packet *packet_new(const void *data, int size, int vnet_hdr_len)

{

Packet *pkt = g_slice_new(Packet);

pkt->data = g_memdup(data, size);

pkt->size = size;

pkt->creation_ms = qemu_clock_get_ms(QEMU_CLOCK_HOST);

+ pkt->vnet_hdr_len = vnet_hdr_len;

return pkt;

}

diff --git a/net/colo.h b/net/colo.h

index XXXXXXX..XXXXXXX 100644

--- a/net/colo.h

+++ b/net/colo.h

@@ -XXX,XX +XXX,XX @@ typedef struct Packet {

int size;

/* Time of packet creation, in wall clock ms */

int64_t creation_ms;

+ /* Get vnet_hdr_len from filter */

+ uint32_t vnet_hdr_len;

} Packet;

typedef struct ConnectionKey {

@@ -XXX,XX +XXX,XX @@ Connection *connection_get(GHashTable *connection_track_table,

ConnectionKey *key,

GQueue *conn_list);

void connection_hashtable_reset(GHashTable *connection_track_table);

-Packet *packet_new(const void *data, int size);

+Packet *packet_new(const void *data, int size, int vnet_hdr_len);

void packet_destroy(void *opaque, void *user_data);

#endif /* QEMU_COLO_PROXY_H */

diff --git a/net/filter-rewriter.c b/net/filter-rewriter.c

index XXXXXXX..XXXXXXX 100644

--- a/net/filter-rewriter.c

+++ b/net/filter-rewriter.c

@@ -XXX,XX +XXX,XX @@ static ssize_t colo_rewriter_receive_iov(NetFilterState *nf,

char *buf = g_malloc0(size);

iov_to_buf(iov, iovcnt, 0, buf, size);

- pkt = packet_new(buf, size);

+ pkt = packet_new(buf, size, 0);

g_free(buf);

/*

2.7.4

From: Zhang Chen <zhangchen.fnst@cn.fujitsu.com>

Add vnet_hdr_len arguments in NetClientState

that make other module get real vnet_hdr_len easily.

Signed-off-by: Zhang Chen <zhangchen.fnst@cn.fujitsu.com>

include/net/net.h | 1 +

net/net.c | 1 +

2 files changed, 2 insertions(+)

diff --git a/include/net/net.h b/include/net/net.h

--- a/include/net/net.h

+++ b/include/net/net.h

@@ -XXX,XX +XXX,XX @@ struct NetClientState {

unsigned int queue_index;

unsigned rxfilter_notify_enabled:1;

int vring_enable;

+ int vnet_hdr_len;

QTAILQ_HEAD(NetFilterHead, NetFilterState) filters;

};

diff --git a/net/net.c b/net/net.c

index XXXXXXX..XXXXXXX 100644

--- a/net/net.c

+++ b/net/net.c

@@ -XXX,XX +XXX,XX @@ void qemu_set_vnet_hdr_len(NetClientState *nc, int len)

return;

+ nc->vnet_hdr_len = len;

nc->info->set_vnet_hdr_len(nc, len);

2.7.4

From: Zhang Chen <zhangchen.fnst@cn.fujitsu.com>

Make colo-compare and filter-rewriter can parse vnet packet.

Signed-off-by: Zhang Chen <zhangchen.fnst@cn.fujitsu.com>

net/colo.c | 6 +++---

1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/net/colo.c b/net/colo.c

--- a/net/colo.c

+++ b/net/colo.c

@@ -XXX,XX +XXX,XX @@ int parse_packet_early(Packet *pkt)

{

int network_length;

static const uint8_t vlan[] = {0x81, 0x00};

- uint8_t *data = pkt->data;

+ uint8_t *data = pkt->data + pkt->vnet_hdr_len;

uint16_t l3_proto;

ssize_t l2hdr_len = eth_get_l2_hdr_length(data);

- if (pkt->size < ETH_HLEN) {

+ if (pkt->size < ETH_HLEN + pkt->vnet_hdr_len) {

trace_colo_proxy_main("pkt->size < ETH_HLEN");

return 1;

}

@@ -XXX,XX +XXX,XX @@ int parse_packet_early(Packet *pkt)

}

network_length = pkt->ip->ip_hl * 4;

- if (pkt->size < l2hdr_len + network_length) {

+ if (pkt->size < l2hdr_len + network_length + pkt->vnet_hdr_len) {

trace_colo_proxy_main("pkt->size < network_header + network_length");

return 1;

}

2.7.4

Spec said offloads should be le64, so use virtio_ldq_p() to guarantee

valid endian.

Fixes: 644c98587d4c ("virtio-net: dynamic network offloads configuration")

Cc: qemu-stable@nongnu.org

Cc: Dmitry Fleytman <dfleytma@redhat.com>

hw/net/virtio-net.c | 2 ++

1 file changed, 2 insertions(+)

@@ -XXX,XX +XXX,XX @@ static int virtio_net_handle_offloads(VirtIONet *n, uint8_t cmd,

if (cmd == VIRTIO_NET_CTRL_GUEST_OFFLOADS_SET) {

uint64_t supported_offloads;

+ offloads = virtio_ldq_p(vdev, &offloads);

+

if (!n->has_vnet_hdr) {

return VIRTIO_NET_ERR;

2.7.4

From: Zhang Chen <zhangchen.fnst@cn.fujitsu.com>

We add the vnet_hdr_support option for filter-mirror, default is disabled.

If you use virtio-net-pci or other driver needs vnet_hdr, please enable it.

You can use it for example:

-object filter-mirror,id=m0,netdev=hn0,queue=tx,outdev=mirror0,vnet_hdr_support

If it has vnet_hdr_support flag, we will change the sending packet format from

struct {int size; const uint8_t buf[];} to {int size; int vnet_hdr_len; const uint8_t buf[];}.

make other module(like colo-compare) know how to parse net packet correctly.

Signed-off-by: Zhang Chen <zhangchen.fnst@cn.fujitsu.com>

net/filter-mirror.c | 42 +++++++++++++++++++++++++++++++++++++++++-

qemu-options.hx | 5 ++---

2 files changed, 43 insertions(+), 4 deletions(-)

diff --git a/net/filter-mirror.c b/net/filter-mirror.c

--- a/net/filter-mirror.c

+++ b/net/filter-mirror.c

@@ -XXX,XX +XXX,XX @@ typedef struct MirrorState {

CharBackend chr_in;

CharBackend chr_out;

SocketReadState rs;

+ bool vnet_hdr;

} MirrorState;

static int filter_send(MirrorState *s,

const struct iovec *iov,

int iovcnt)

+ NetFilterState *nf = NETFILTER(s);

int ret = 0;

ssize_t size = 0;

uint32_t len = 0;

@@ -XXX,XX +XXX,XX @@ static int filter_send(MirrorState *s,

goto err;

}

+ if (s->vnet_hdr) {

+ /*

+ * If vnet_hdr = on, we send vnet header len to make other

+ * module(like colo-compare) know how to parse net

+ * packet correctly.

+ */

+ ssize_t vnet_hdr_len;

+

+ vnet_hdr_len = nf->netdev->vnet_hdr_len;

+

+ len = htonl(vnet_hdr_len);

+ ret = qemu_chr_fe_write_all(&s->chr_out, (uint8_t *)&len, sizeof(len));

+ if (ret != sizeof(len)) {

+ goto err;

+ }

+ }

+

buf = g_malloc(size);

iov_to_buf(iov, iovcnt, 0, buf, size);

ret = qemu_chr_fe_write_all(&s->chr_out, (uint8_t *)buf, size);

@@ -XXX,XX +XXX,XX @@ static void filter_redirector_setup(NetFilterState *nf, Error **errp)

- net_socket_rs_init(&s->rs, redirector_rs_finalize, false);

+ net_socket_rs_init(&s->rs, redirector_rs_finalize, s->vnet_hdr);

if (s->indev) {

chr = qemu_chr_find(s->indev);

@@ -XXX,XX +XXX,XX @@ static void filter_mirror_set_outdev(Object *obj,

+static bool filter_mirror_get_vnet_hdr(Object *obj, Error **errp)

+{

+ MirrorState *s = FILTER_MIRROR(obj);

+

+ return s->vnet_hdr;

+}

+

+static void filter_mirror_set_vnet_hdr(Object *obj, bool value, Error **errp)

+{

+ MirrorState *s = FILTER_MIRROR(obj);

+

+ s->vnet_hdr = value;

+}

+

static char *filter_redirector_get_outdev(Object *obj, Error **errp)

MirrorState *s = FILTER_REDIRECTOR(obj);

@@ -XXX,XX +XXX,XX @@ static void filter_redirector_set_outdev(Object *obj,

static void filter_mirror_init(Object *obj)

{

+ MirrorState *s = FILTER_MIRROR(obj);

+

object_property_add_str(obj, "outdev", filter_mirror_get_outdev,

filter_mirror_set_outdev, NULL);

+

+ s->vnet_hdr = false;

+ object_property_add_bool(obj, "vnet_hdr_support",

+ filter_mirror_get_vnet_hdr,

+ filter_mirror_set_vnet_hdr, NULL);

static void filter_redirector_init(Object *obj)

diff --git a/qemu-options.hx b/qemu-options.hx

index XXXXXXX..XXXXXXX 100644

--- a/qemu-options.hx

+++ b/qemu-options.hx

@@ -XXX,XX +XXX,XX @@ queue @var{all|rx|tx} is an option that can be applied to any netfilter.

@option{tx}: the filter is attached to the transmit queue of the netdev,

where it will receive packets sent by the netdev.

-@item -object filter-mirror,id=@var{id},netdev=@var{netdevid},outdev=@var{chardevid}[,queue=@var{all|rx|tx}]

+@item -object filter-mirror,id=@var{id},netdev=@var{netdevid},outdev=@var{chardevid},queue=@var{all|rx|tx}[,vnet_hdr_support]

-filter-mirror on netdev @var{netdevid},mirror net packet to chardev

-@var{chardevid}

+filter-mirror on netdev @var{netdevid},mirror net packet to chardev@var{chardevid}, if it has the vnet_hdr_support flag, filter-mirror will mirror packet with vnet_hdr_len.

@item -object filter-redirector,id=@var{id},netdev=@var{netdevid},indev=@var{chardevid},

outdev=@var{chardevid}[,queue=@var{all|rx|tx}]

2.7.4

From: Zhang Chen <zhangchen.fnst@cn.fujitsu.com>