Series comparison

-[PULL 00/17] Net patches
+[PULL V3 00/15] Net patches
-The following changes since commit 03a3a62fbd0aa5227e978eef3c67d3978aec9e5f:
+The following changes since commit 352998df1c53b366413690d95b35f76d0721ebed:
-  Merge tag 'for-upstream' of https://gitlab.com/bonzini/qemu into staging (2023-09-07 10:29:06 -0400)
+  Merge tag 'i2c-20220314' of https://github.com/philmd/qemu into staging (2022-03-14 14:39:33 +0000)
 are available in the git repository at:
   https://github.com/jasowang/qemu.git tags/net-pull-request
-for you to fetch changes up to 049cfda145e96b2605cdf9739f1bcf9ebf3a83e1:
+for you to fetch changes up to 12a195fa343aae2ead1301ce04727bd0ae25eb15:
-  ebpf: Updated eBPF program and skeleton. (2023-09-08 14:33:46 +0800)
+  vdpa: Expose VHOST_F_LOG_ALL on SVQ (2022-03-15 13:57:44 +0800)
 ----------------------------------------------------------------
+Changes since V2:
+- fix 32bit build errros
 ----------------------------------------------------------------
-Andrew Melnychenko (7):
+Eugenio Pérez (14):
-      tap: Add USO support to tap device.
+      vhost: Add VhostShadowVirtqueue
-      virtio-net: Add USO flags to vhost support.
+      vhost: Add Shadow VirtQueue kick forwarding capabilities
-      ebpf: Added eBPF map update through mmap.
+      vhost: Add Shadow VirtQueue call forwarding capabilities
-      ebpf: Added eBPF initialization by fds.
+      vhost: Add vhost_svq_valid_features to shadow vq
-      virtio-net: Added property to load eBPF RSS with fds.
+      virtio: Add vhost_svq_get_vring_addr
-      qmp: Added new command to retrieve eBPF blob.
+      vdpa: adapt vhost_ops callbacks to svq
-      ebpf: Updated eBPF program and skeleton.
+      vhost: Shadow virtqueue buffers forwarding
       util: Add iova_tree_alloc_map
       util: add iova_tree_find_iova
       vhost: Add VhostIOVATree
       vdpa: Add custom IOTLB translations to SVQ
       vdpa: Adapt vhost_vdpa_get_vring_base to SVQ
       vdpa: Never set log_base addr if SVQ is enabled
       vdpa: Expose VHOST_F_LOG_ALL on SVQ
-Ilya Maximets (1):
+Jason Wang (1):
-      net: add initial support for AF_XDP network backend
+      virtio-net: fix map leaking on error during receive
-Tomasz Dzieciol (7):
+ hw/net/virtio-net.c                |   1 +
-      igb: remove TCP ACK detection
+ hw/virtio/meson.build              |   2 +-
-      igb: rename E1000E_RingInfo_st
+ hw/virtio/vhost-iova-tree.c        | 110 +++++++
-      igb: RX descriptors guest writting refactoring
+ hw/virtio/vhost-iova-tree.h        |  27 ++
-      igb: RX payload guest writting refactoring
+ hw/virtio/vhost-shadow-virtqueue.c | 636 +++++++++++++++++++++++++++++++++++++
-      igb: add IPv6 extended headers traffic detection
+ hw/virtio/vhost-shadow-virtqueue.h |  87 +++++
-      igb: packet-split descriptors support
+ hw/virtio/vhost-vdpa.c             | 522 +++++++++++++++++++++++++++++-
-      e1000e: rename e1000e_ba_state and e1000e_write_hdr_to_rx_buffers
+ include/hw/virtio/vhost-vdpa.h     |   8 +
  include/qemu/iova-tree.h           |  38 ++-
  util/iova-tree.c                   | 170 ++++++++++
 files changed, 1584 insertions(+), 17 deletions(-)
  create mode 100644 hw/virtio/vhost-iova-tree.c
  create mode 100644 hw/virtio/vhost-iova-tree.h
  create mode 100644 hw/virtio/vhost-shadow-virtqueue.c
  create mode 100644 hw/virtio/vhost-shadow-virtqueue.h
-Yuri Benditovich (2):
-      tap: Add check for USO features
-      virtio-net: Add support for USO features
- MAINTAINERS                                     |    4 +
  ebpf/ebpf.c                                     |   70 ++
  ebpf/ebpf.h                                     |   31 +
  ebpf/ebpf_rss-stub.c                            |    6 +
  ebpf/ebpf_rss.c                                 |  150 ++-
  ebpf/ebpf_rss.h                                 |   10 +
  ebpf/meson.build                                |    2 +-
  ebpf/rss.bpf.skeleton.h                         | 1460 ++++++++++++-----------
  hmp-commands.hx                                 |    3 +
  hw/core/machine.c                               |    4 +
  hw/net/e1000e_core.c                            |   80 +-
  hw/net/igb_core.c                               |  732 ++++++++----
  hw/net/igb_regs.h                               |   20 +-
  hw/net/trace-events                             |    6 +-
  hw/net/vhost_net.c                              |    3 +
  hw/net/virtio-net.c                             |   90 +-
  hw/net/vmxnet3.c                                |    2 +
  include/hw/virtio/virtio-net.h                  |    1 +
  include/net/net.h                               |    7 +-
  meson.build                                     |   19 +-
  meson_options.txt                               |    2 +
  net/af-xdp.c                                    |  526 ++++++++
  net/clients.h                                   |    5 +
  net/meson.build                                 |    3 +
  net/net.c                                       |   19 +-
  net/tap-bsd.c                                   |    7 +-
  net/tap-linux.c                                 |   27 +-
  net/tap-linux.h                                 |    2 +
  net/tap-solaris.c                               |    7 +-
  net/tap-stub.c                                  |    7 +-
  net/tap-win32.c                                 |    2 +-
  net/tap.c                                       |   18 +-
  net/tap_int.h                                   |    4 +-
  net/vhost-vdpa.c                                |    3 +
  qapi/ebpf.json                                  |   66 +
  qapi/meson.build                                |    1 +
  qapi/net.json                                   |   58 +
  qapi/qapi-schema.json                           |    1 +
  qemu-options.hx                                 |   70 +-
  scripts/ci/org.centos/stream/8/x86_64/configure |    1 +
  scripts/meson-buildoptions.sh                   |    3 +
  tests/docker/dockerfiles/debian-amd64.docker    |    1 +
  tests/qtest/libqos/igb.c                        |    5 +
  tools/ebpf/rss.bpf.c                            |    5 +-
 files changed, 2518 insertions(+), 1025 deletions(-)
  create mode 100644 ebpf/ebpf.c
  create mode 100644 ebpf/ebpf.h
  create mode 100644 net/af-xdp.c
  create mode 100644 qapi/ebpf.json

-[PULL 01/17] tap: Add USO support to tap device.
+Deleted patch
-From: Andrew Melnychenko <andrew@daynix.com>
-Passing additional parameters (USOv4 and USOv6 offloads) when
-setting TAP offloads
-Signed-off-by: Yuri Benditovich <yuri.benditovich@daynix.com>
-Signed-off-by: Andrew Melnychenko <andrew@daynix.com>
-Signed-off-by: Jason Wang <jasowang@redhat.com>
----
- hw/net/e1000e_core.c |  2 +-
- hw/net/igb_core.c    |  2 +-
- hw/net/virtio-net.c  |  4 +++-
- hw/net/vmxnet3.c     |  2 ++
- include/net/net.h    |  4 ++--
- net/net.c            |  4 ++--
- net/tap-bsd.c        |  2 +-
- net/tap-linux.c      | 15 ++++++++++++---
- net/tap-linux.h      |  2 ++
- net/tap-solaris.c    |  2 +-
- net/tap-stub.c       |  2 +-
- net/tap-win32.c      |  2 +-
- net/tap.c            |  6 +++---
- net/tap_int.h        |  3 ++-
-files changed, 34 insertions(+), 18 deletions(-)
-diff --git a/hw/net/e1000e_core.c b/hw/net/e1000e_core.c
-index XXXXXXX..XXXXXXX 100644
---- a/hw/net/e1000e_core.c
-+++ b/hw/net/e1000e_core.c
-@@ -XXX,XX +XXX,XX @@ e1000e_update_rx_offloads(E1000ECore *core)
-     if (core->has_vnet) {
-         qemu_set_offload(qemu_get_queue(core->owner_nic)->peer,
--                         cso_state, 0, 0, 0, 0);
-+                         cso_state, 0, 0, 0, 0, 0, 0);
-     }
- }
-diff --git a/hw/net/igb_core.c b/hw/net/igb_core.c
-index XXXXXXX..XXXXXXX 100644
---- a/hw/net/igb_core.c
-+++ b/hw/net/igb_core.c
-@@ -XXX,XX +XXX,XX @@ igb_update_rx_offloads(IGBCore *core)
-     if (core->has_vnet) {
-         qemu_set_offload(qemu_get_queue(core->owner_nic)->peer,
--                         cso_state, 0, 0, 0, 0);
-+                         cso_state, 0, 0, 0, 0, 0, 0);
-     }
- }
-diff --git a/hw/net/virtio-net.c b/hw/net/virtio-net.c
-index XXXXXXX..XXXXXXX 100644
---- a/hw/net/virtio-net.c
-+++ b/hw/net/virtio-net.c
-@@ -XXX,XX +XXX,XX @@ static void virtio_net_apply_guest_offloads(VirtIONet *n)
-             !!(n->curr_guest_offloads & (1ULL << VIRTIO_NET_F_GUEST_TSO4)),
-             !!(n->curr_guest_offloads & (1ULL << VIRTIO_NET_F_GUEST_TSO6)),
-             !!(n->curr_guest_offloads & (1ULL << VIRTIO_NET_F_GUEST_ECN)),
--            !!(n->curr_guest_offloads & (1ULL << VIRTIO_NET_F_GUEST_UFO)));
-+            !!(n->curr_guest_offloads & (1ULL << VIRTIO_NET_F_GUEST_UFO)),
-+            !!(n->curr_guest_offloads & (1ULL << VIRTIO_NET_F_GUEST_USO4)),
-+            !!(n->curr_guest_offloads & (1ULL << VIRTIO_NET_F_GUEST_USO6)));
- }
- static uint64_t virtio_net_guest_offloads_by_features(uint32_t features)
-diff --git a/hw/net/vmxnet3.c b/hw/net/vmxnet3.c
-index XXXXXXX..XXXXXXX 100644
---- a/hw/net/vmxnet3.c
-+++ b/hw/net/vmxnet3.c
-@@ -XXX,XX +XXX,XX @@ static void vmxnet3_update_features(VMXNET3State *s)
-                          s->lro_supported,
-                          s->lro_supported,
-,
-+                         0,
-+                         0,
-);
-     }
- }
-diff --git a/include/net/net.h b/include/net/net.h
-index XXXXXXX..XXXXXXX 100644
---- a/include/net/net.h
-+++ b/include/net/net.h
-@@ -XXX,XX +XXX,XX @@ typedef bool (HasVnetHdr)(NetClientState *);
- typedef bool (HasVnetHdrLen)(NetClientState *, int);
- typedef bool (GetUsingVnetHdr)(NetClientState *);
- typedef void (UsingVnetHdr)(NetClientState *, bool);
--typedef void (SetOffload)(NetClientState *, int, int, int, int, int);
-+typedef void (SetOffload)(NetClientState *, int, int, int, int, int, int, int);
- typedef int (GetVnetHdrLen)(NetClientState *);
- typedef void (SetVnetHdrLen)(NetClientState *, int);
- typedef int (SetVnetLE)(NetClientState *, bool);
-@@ -XXX,XX +XXX,XX @@ bool qemu_has_vnet_hdr_len(NetClientState *nc, int len);
- bool qemu_get_using_vnet_hdr(NetClientState *nc);
- void qemu_using_vnet_hdr(NetClientState *nc, bool enable);
- void qemu_set_offload(NetClientState *nc, int csum, int tso4, int tso6,
--                      int ecn, int ufo);
-+                      int ecn, int ufo, int uso4, int uso6);
- int qemu_get_vnet_hdr_len(NetClientState *nc);
- void qemu_set_vnet_hdr_len(NetClientState *nc, int len);
- int qemu_set_vnet_le(NetClientState *nc, bool is_le);
-diff --git a/net/net.c b/net/net.c
-index XXXXXXX..XXXXXXX 100644
---- a/net/net.c
-+++ b/net/net.c
-@@ -XXX,XX +XXX,XX @@ void qemu_using_vnet_hdr(NetClientState *nc, bool enable)
- }
- void qemu_set_offload(NetClientState *nc, int csum, int tso4, int tso6,
--                          int ecn, int ufo)
-+                          int ecn, int ufo, int uso4, int uso6)
- {
-     if (!nc || !nc->info->set_offload) {
-         return;
-     }
--    nc->info->set_offload(nc, csum, tso4, tso6, ecn, ufo);
-+    nc->info->set_offload(nc, csum, tso4, tso6, ecn, ufo, uso4, uso6);
- }
- int qemu_get_vnet_hdr_len(NetClientState *nc)
-diff --git a/net/tap-bsd.c b/net/tap-bsd.c
-index XXXXXXX..XXXXXXX 100644
---- a/net/tap-bsd.c
-+++ b/net/tap-bsd.c
-@@ -XXX,XX +XXX,XX @@ int tap_fd_set_vnet_be(int fd, int is_be)
- }
- void tap_fd_set_offload(int fd, int csum, int tso4,
--                        int tso6, int ecn, int ufo)
-+                        int tso6, int ecn, int ufo, int uso4, int uso6)
- {
- }
-diff --git a/net/tap-linux.c b/net/tap-linux.c
-index XXXXXXX..XXXXXXX 100644
---- a/net/tap-linux.c
-+++ b/net/tap-linux.c
-@@ -XXX,XX +XXX,XX @@ int tap_fd_set_vnet_be(int fd, int is_be)
- }
- void tap_fd_set_offload(int fd, int csum, int tso4,
--                        int tso6, int ecn, int ufo)
-+                        int tso6, int ecn, int ufo, int uso4, int uso6)
- {
-     unsigned int offload = 0;
-@@ -XXX,XX +XXX,XX @@ void tap_fd_set_offload(int fd, int csum, int tso4,
-             offload |= TUN_F_TSO_ECN;
-         if (ufo)
-             offload |= TUN_F_UFO;
-+        if (uso4) {
-+            offload |= TUN_F_USO4;
-+        }
-+        if (uso6) {
-+            offload |= TUN_F_USO6;
-+        }
-     }
-     if (ioctl(fd, TUNSETOFFLOAD, offload) != 0) {
--        offload &= ~TUN_F_UFO;
-+        offload &= ~(TUN_F_USO4 | TUN_F_USO6);
-         if (ioctl(fd, TUNSETOFFLOAD, offload) != 0) {
--            fprintf(stderr, "TUNSETOFFLOAD ioctl() failed: %s\n",
-+            offload &= ~TUN_F_UFO;
-+            if (ioctl(fd, TUNSETOFFLOAD, offload) != 0) {
-+                fprintf(stderr, "TUNSETOFFLOAD ioctl() failed: %s\n",
-                     strerror(errno));
-+            }
-         }
-     }
- }
-diff --git a/net/tap-linux.h b/net/tap-linux.h
-index XXXXXXX..XXXXXXX 100644
---- a/net/tap-linux.h
-+++ b/net/tap-linux.h
-@@ -XXX,XX +XXX,XX @@
- #define TUN_F_TSO6    0x04    /* I can handle TSO for IPv6 packets */
- #define TUN_F_TSO_ECN 0x08    /* I can handle TSO with ECN bits. */
- #define TUN_F_UFO     0x10    /* I can handle UFO packets */
-+#define TUN_F_USO4    0x20    /* I can handle USO for IPv4 packets */
-+#define TUN_F_USO6    0x40    /* I can handle USO for IPv6 packets */
- #endif /* QEMU_TAP_LINUX_H */
-diff --git a/net/tap-solaris.c b/net/tap-solaris.c
-index XXXXXXX..XXXXXXX 100644
---- a/net/tap-solaris.c
-+++ b/net/tap-solaris.c
-@@ -XXX,XX +XXX,XX @@ int tap_fd_set_vnet_be(int fd, int is_be)
- }
- void tap_fd_set_offload(int fd, int csum, int tso4,
--                        int tso6, int ecn, int ufo)
-+                        int tso6, int ecn, int ufo, int uso4, int uso6)
- {
- }
-diff --git a/net/tap-stub.c b/net/tap-stub.c
-index XXXXXXX..XXXXXXX 100644
---- a/net/tap-stub.c
-+++ b/net/tap-stub.c
-@@ -XXX,XX +XXX,XX @@ int tap_fd_set_vnet_be(int fd, int is_be)
- }
- void tap_fd_set_offload(int fd, int csum, int tso4,
--                        int tso6, int ecn, int ufo)
-+                        int tso6, int ecn, int ufo, int uso4, int uso6)
- {
- }
-diff --git a/net/tap-win32.c b/net/tap-win32.c
-index XXXXXXX..XXXXXXX 100644
---- a/net/tap-win32.c
-+++ b/net/tap-win32.c
-@@ -XXX,XX +XXX,XX @@ static void tap_using_vnet_hdr(NetClientState *nc, bool using_vnet_hdr)
- }
- static void tap_set_offload(NetClientState *nc, int csum, int tso4,
--                     int tso6, int ecn, int ufo)
-+                     int tso6, int ecn, int ufo, int uso4, int uso6)
- {
- }
-diff --git a/net/tap.c b/net/tap.c
-index XXXXXXX..XXXXXXX 100644
---- a/net/tap.c
-+++ b/net/tap.c
-@@ -XXX,XX +XXX,XX @@ static int tap_set_vnet_be(NetClientState *nc, bool is_be)
- }
- static void tap_set_offload(NetClientState *nc, int csum, int tso4,
--                     int tso6, int ecn, int ufo)
-+                     int tso6, int ecn, int ufo, int uso4, int uso6)
- {
-     TAPState *s = DO_UPCAST(TAPState, nc, nc);
-     if (s->fd < 0) {
-         return;
-     }
--    tap_fd_set_offload(s->fd, csum, tso4, tso6, ecn, ufo);
-+    tap_fd_set_offload(s->fd, csum, tso4, tso6, ecn, ufo, uso4, uso6);
- }
- static void tap_exit_notify(Notifier *notifier, void *data)
-@@ -XXX,XX +XXX,XX @@ static TAPState *net_tap_fd_init(NetClientState *peer,
-     s->using_vnet_hdr = false;
-     s->has_ufo = tap_probe_has_ufo(s->fd);
-     s->enabled = true;
--    tap_set_offload(&s->nc, 0, 0, 0, 0, 0);
-+    tap_set_offload(&s->nc, 0, 0, 0, 0, 0, 0, 0);
-     /*
-      * Make sure host header length is set correctly in tap:
-      * it might have been modified by another instance of qemu.
-diff --git a/net/tap_int.h b/net/tap_int.h
-index XXXXXXX..XXXXXXX 100644
---- a/net/tap_int.h
-+++ b/net/tap_int.h
-@@ -XXX,XX +XXX,XX @@ void tap_set_sndbuf(int fd, const NetdevTapOptions *tap, Error **errp);
- int tap_probe_vnet_hdr(int fd, Error **errp);
- int tap_probe_vnet_hdr_len(int fd, int len);
- int tap_probe_has_ufo(int fd);
--void tap_fd_set_offload(int fd, int csum, int tso4, int tso6, int ecn, int ufo);
-+void tap_fd_set_offload(int fd, int csum, int tso4, int tso6, int ecn, int ufo,
-+                        int uso4, int uso6);
- void tap_fd_set_vnet_hdr_len(int fd, int len);
- int tap_fd_set_vnet_le(int fd, int vnet_is_le);
- int tap_fd_set_vnet_be(int fd, int vnet_is_be);
---
-.7.4

-[PULL 15/17] virtio-net: Added property to load eBPF RSS with fds.
+[PULL V3 01/15] virtio-net: fix map leaking on error during receive
-From: Andrew Melnychenko <andrew@daynix.com>
+Commit bedd7e93d0196 ("virtio-net: fix use after unmap/free for sg")
 tries to fix the use after free of the sg by caching the virtqueue
 elements in an array and unmap them at once after receiving the
 packets, But it forgot to unmap the cached elements on error which
 will lead to leaking of mapping and other unexpected results.
-eBPF RSS program and maps may now be passed during initialization.
+Fixing this by detaching the cached elements on error. This addresses
-Initially was implemented for libvirt to launch qemu without permissions,
+CVE-2022-26353.
 and initialized eBPF program through the helper.
-Signed-off-by: Andrew Melnychenko <andrew@daynix.com>
+Reported-by: Victor Tom <vv474172261@gmail.com>
 Cc: qemu-stable@nongnu.org
 Fixes: CVE-2022-26353
 Fixes: bedd7e93d0196 ("virtio-net: fix use after unmap/free for sg")
 Reviewed-by: Michael S. Tsirkin <mst@redhat.com>
 Signed-off-by: Jason Wang <jasowang@redhat.com>
 ---
- hw/net/virtio-net.c            | 55 +++++++++++++++++++++++++++++++++++++-----
+ hw/net/virtio-net.c | 1 +
- include/hw/virtio/virtio-net.h |  1 +
+file changed, 1 insertion(+)
 files changed, 50 insertions(+), 6 deletions(-)
 diff --git a/hw/net/virtio-net.c b/hw/net/virtio-net.c
 index XXXXXXX..XXXXXXX 100644
 --- a/hw/net/virtio-net.c
 +++ b/hw/net/virtio-net.c
-@@ -XXX,XX +XXX,XX @@
+@@ -XXX,XX +XXX,XX @@ static ssize_t virtio_net_receive_rcu(NetClientState *nc, const uint8_t *buf,
- #include "sysemu/sysemu.h"
- #include "trace.h"
+ err:
- #include "monitor/qdev.h"
+     for (j = 0; j < i; j++) {
-+#include "monitor/monitor.h"
++        virtqueue_detach_element(q->rx_vq, elems[j], lens[j]);
- #include "hw/pci/pci_device.h"
+         g_free(elems[j]);
  #include "net_rx_pkt.h"
  #include "hw/virtio/vhost.h"
@@ -XXX,XX +XXX,XX @@ static void virtio_net_detach_epbf_rss(VirtIONet *n)
      virtio_net_attach_ebpf_to_backend(n->nic, -1);
  }
 -static bool virtio_net_load_ebpf(VirtIONet *n)
 +static bool virtio_net_load_ebpf_fds(VirtIONet *n, Error **errp)
  {
 -    if (!virtio_net_attach_ebpf_to_backend(n->nic, -1)) {
 -        /* backend does't support steering ebpf */
 -        return false;
 +    int fds[EBPF_RSS_MAX_FDS] = { [0 ... EBPF_RSS_MAX_FDS - 1] = -1};
 +    int nfds = 0;
 +    int ret = true;
 +    int i = 0;
 +    g_auto(GStrv) fds_strs = g_strsplit(n->ebpf_rss_fds, ":", 0);
 +
 +    ERRP_GUARD();
 +
 +    if (g_strv_length(fds_strs) != EBPF_RSS_MAX_FDS) {
 +        error_setg(errp,
 +                  "Expected %d file descriptors but got %d",
 +                  EBPF_RSS_MAX_FDS, g_strv_length(fds_strs));
 +       return false;
 +   }
 +
 +    for (i = 0; i < nfds; i++) {
 +        fds[i] = monitor_fd_param(monitor_cur(), fds_strs[i], errp);
 +        if (*errp) {
 +            ret = false;
 +            goto exit;
 +        }
 +    }
 +
 +    ret = ebpf_rss_load_fds(&n->ebpf_rss, fds[0], fds[1], fds[2], fds[3]);
 +
 +exit:
 +    if (!ret || *errp) {
 +        for (i = 0; i < nfds && fds[i] != -1; i++) {
 +            close(fds[i]);
 +        }
      }
--    return ebpf_rss_load(&n->ebpf_rss);
-+    return ret;
-+}
-+
-+static bool virtio_net_load_ebpf(VirtIONet *n, Error **errp)
-+{
-+    bool ret = false;
-+
-+    if (virtio_net_attach_ebpf_to_backend(n->nic, -1)) {
-+        if (!(n->ebpf_rss_fds
-+                && virtio_net_load_ebpf_fds(n, errp))) {
-+            ret = ebpf_rss_load(&n->ebpf_rss);
-+        }
-+    }
-+
-+    return ret;
- }
- static void virtio_net_unload_ebpf(VirtIONet *n)
-@@ -XXX,XX +XXX,XX @@ static void virtio_net_device_realize(DeviceState *dev, Error **errp)
-     net_rx_pkt_init(&n->rx_pkt);
-     if (virtio_has_feature(n->host_features, VIRTIO_NET_F_RSS)) {
--        virtio_net_load_ebpf(n);
-+        virtio_net_load_ebpf(n, errp);
-     }
- }
-@@ -XXX,XX +XXX,XX @@ static Property virtio_net_properties[] = {
-                     VIRTIO_NET_F_RSS, false),
-     DEFINE_PROP_BIT64("hash", VirtIONet, host_features,
-                     VIRTIO_NET_F_HASH_REPORT, false),
-+    DEFINE_PROP_STRING("ebpf_rss_fds", VirtIONet, ebpf_rss_fds),
-     DEFINE_PROP_BIT64("guest_rsc_ext", VirtIONet, host_features,
-                     VIRTIO_NET_F_RSC_EXT, false),
-     DEFINE_PROP_UINT32("rsc_interval", VirtIONet, rsc_timeout,
-diff --git a/include/hw/virtio/virtio-net.h b/include/hw/virtio/virtio-net.h
-index XXXXXXX..XXXXXXX 100644
---- a/include/hw/virtio/virtio-net.h
-+++ b/include/hw/virtio/virtio-net.h
-@@ -XXX,XX +XXX,XX @@ struct VirtIONet {
-     VirtioNetRssData rss_data;
-     struct NetRxPkt *rx_pkt;
-     struct EBPFRSSContext ebpf_rss;
-+    char *ebpf_rss_fds;
- };
- size_t virtio_net_handle_ctrl_iov(VirtIODevice *vdev,
 --
 .7.4

-[PULL 16/17] qmp: Added new command to retrieve eBPF blob.
+[PULL V3 02/15] vhost: Add VhostShadowVirtqueue
-From: Andrew Melnychenko <andrew@daynix.com>
+From: Eugenio Pérez <eperezma@redhat.com>
-Now, the binary objects may be retrieved by id.
+Vhost shadow virtqueue (SVQ) is an intermediate jump for virtqueue
-It would require for future qmp commands that may require specific
+notifications and buffers, allowing qemu to track them. While qemu is
-eBPF blob.
+forwarding the buffers and virtqueue changes, it is able to commit the
 memory it's being dirtied, the same way regular qemu's VirtIO devices
 do.
-Added command "request-ebpf". This command returns
+This commit only exposes basic SVQ allocation and free. Next patches of
-eBPF program encoded base64. The program taken from the
+the series add functionality like notifications and buffers forwarding.
 skeleton and essentially is an ELF object that can be
 loaded in the future with libbpf.
-The reason to use the command to provide the eBPF object
+Signed-off-by: Eugenio Pérez <eperezma@redhat.com>
-instead of a separate artifact was to avoid issues related
+Acked-by: Michael S. Tsirkin <mst@redhat.com>
 to finding the eBPF itself. eBPF object is an ELF binary
 that contains the eBPF program and eBPF map description(BTF).
 Overall, eBPF object should contain the program and enough
 metadata to create/load eBPF with libbpf. As the eBPF
 maps/program should correspond to QEMU, the eBPF can't
 be used from different QEMU build.
 The first solution was a helper that comes with QEMU
 and loads appropriate eBPF objects. And the issue is
 to find a proper helper if the system has several
 different QEMUs installed and/or built from the source,
 which helpers may not be compatible.
 Another issue is QEMU updating while there is a running
 QEMU instance. With an updated helper, it may not be
 possible to hotplug virtio-net device to the already
 running QEMU. Overall, requesting the eBPF object from
 QEMU itself solves possible failures with acceptable effort.
 Links:
 [PATCH 3/5] qmp: Added the helper stamp check.
 https://lore.kernel.org/all/20230219162100.174318-4-andrew@daynix.com/
 Signed-off-by: Andrew Melnychenko <andrew@daynix.com>
 Signed-off-by: Jason Wang <jasowang@redhat.com>
 ---
- ebpf/ebpf.c           | 70 +++++++++++++++++++++++++++++++++++++++++++++++++++
+ hw/virtio/meson.build              |  2 +-
- ebpf/ebpf.h           | 31 +++++++++++++++++++++++
+ hw/virtio/vhost-shadow-virtqueue.c | 62 ++++++++++++++++++++++++++++++++++++++
- ebpf/ebpf_rss.c       |  6 +++++
+ hw/virtio/vhost-shadow-virtqueue.h | 28 +++++++++++++++++
- ebpf/meson.build      |  2 +-
+files changed, 91 insertions(+), 1 deletion(-)
- qapi/ebpf.json        | 66 ++++++++++++++++++++++++++++++++++++++++++++++++
+ create mode 100644 hw/virtio/vhost-shadow-virtqueue.c
- qapi/meson.build      |  1 +
+ create mode 100644 hw/virtio/vhost-shadow-virtqueue.h
  qapi/qapi-schema.json |  1 +
 files changed, 176 insertions(+), 1 deletion(-)
  create mode 100644 ebpf/ebpf.c
  create mode 100644 ebpf/ebpf.h
  create mode 100644 qapi/ebpf.json
-diff --git a/ebpf/ebpf.c b/ebpf/ebpf.c
+diff --git a/hw/virtio/meson.build b/hw/virtio/meson.build
 index XXXXXXX..XXXXXXX 100644
 --- a/hw/virtio/meson.build
 +++ b/hw/virtio/meson.build
@@ -XXX,XX +XXX,XX @@ softmmu_ss.add(when: 'CONFIG_ALL', if_true: files('vhost-stub.c'))
  virtio_ss = ss.source_set()
  virtio_ss.add(files('virtio.c'))
 -virtio_ss.add(when: 'CONFIG_VHOST', if_true: files('vhost.c', 'vhost-backend.c'))
 +virtio_ss.add(when: 'CONFIG_VHOST', if_true: files('vhost.c', 'vhost-backend.c', 'vhost-shadow-virtqueue.c'))
  virtio_ss.add(when: 'CONFIG_VHOST_USER', if_true: files('vhost-user.c'))
  virtio_ss.add(when: 'CONFIG_VHOST_VDPA', if_true: files('vhost-vdpa.c'))
  virtio_ss.add(when: 'CONFIG_VIRTIO_BALLOON', if_true: files('virtio-balloon.c'))
 diff --git a/hw/virtio/vhost-shadow-virtqueue.c b/hw/virtio/vhost-shadow-virtqueue.c
 new file mode 100644
 index XXXXXXX..XXXXXXX
 --- /dev/null
-+++ b/ebpf/ebpf.c
++++ b/hw/virtio/vhost-shadow-virtqueue.c
 @@ -XXX,XX +XXX,XX @@
 +/*
-+ * QEMU eBPF binary declaration routine.
++ * vhost shadow virtqueue
 + *
-+ * Developed by Daynix Computing LTD (http://www.daynix.com)
++ * SPDX-FileCopyrightText: Red Hat, Inc. 2021
 + * SPDX-FileContributor: Author: Eugenio Pérez <eperezma@redhat.com>
 + *
-+ * Authors:
++ * SPDX-License-Identifier: GPL-2.0-or-later
 + *  Andrew Melnychenko <andrew@daynix.com>
 + *
 + * This work is licensed under the terms of the GNU GPL, version 2 or
 + * later.  See the COPYING file in the top-level directory.
 + */
 +
 +#include "qemu/osdep.h"
-+#include "qemu/queue.h"
++#include "hw/virtio/vhost-shadow-virtqueue.h"
 +#include "qapi/error.h"
 +#include "qapi/qapi-commands-ebpf.h"
 +#include "ebpf/ebpf.h"
 +
-+struct ElfBinaryDataEntry {
++#include "qemu/error-report.h"
 +    int id;
 +    const void *data;
 +    size_t datalen;
 +
-+    QSLIST_ENTRY(ElfBinaryDataEntry) node;
++/**
-+};
++ * Creates vhost shadow virtqueue, and instructs the vhost device to use the
 + * shadow methods and file descriptors.
 + *
 + * Returns the new virtqueue or NULL.
 + *
 + * In case of error, reason is reported through error_report.
 + */
 +VhostShadowVirtqueue *vhost_svq_new(void)
 +{
 +    g_autofree VhostShadowVirtqueue *svq = g_new0(VhostShadowVirtqueue, 1);
 +    int r;
 +
-+static QSLIST_HEAD(, ElfBinaryDataEntry) ebpf_elf_obj_list =
++    r = event_notifier_init(&svq->hdev_kick, 0);
-+                                            QSLIST_HEAD_INITIALIZER();
++    if (r != 0) {
-+
++        error_report("Couldn't create kick event notifier: %s (%d)",
-+void ebpf_register_binary_data(int id, const void *data, size_t datalen)
++                     g_strerror(errno), errno);
-+{
++        goto err_init_hdev_kick;
 +    struct ElfBinaryDataEntry *dataentry = NULL;
 +
 +    dataentry = g_new0(struct ElfBinaryDataEntry, 1);
 +    dataentry->data = data;
 +    dataentry->datalen = datalen;
 +    dataentry->id = id;
 +
 +    QSLIST_INSERT_HEAD(&ebpf_elf_obj_list, dataentry, node);
 +}
 +
 +const void *ebpf_find_binary_by_id(int id, size_t *sz, Error **errp)
 +{
 +    struct ElfBinaryDataEntry *it = NULL;
 +    QSLIST_FOREACH(it, &ebpf_elf_obj_list, node) {
 +        if (id == it->id) {
 +            *sz = it->datalen;
 +            return it->data;
 +        }
 +    }
 +
-+    error_setg(errp, "can't find eBPF object with id: %d", id);
++    r = event_notifier_init(&svq->hdev_call, 0);
 +    if (r != 0) {
 +        error_report("Couldn't create call event notifier: %s (%d)",
 +                     g_strerror(errno), errno);
 +        goto err_init_hdev_call;
 +    }
 +
++    return g_steal_pointer(&svq);
++
++err_init_hdev_call:
++    event_notifier_cleanup(&svq->hdev_kick);
++
++err_init_hdev_kick:
 +    return NULL;
 +}
 +
-+EbpfObject *qmp_request_ebpf(EbpfProgramID id, Error **errp)
++/**
 + * Free the resources of the shadow virtqueue.
 + *
 + * @pvq: gpointer to SVQ so it can be used by autofree functions.
 + */
 +void vhost_svq_free(gpointer pvq)
 +{
-+    EbpfObject *ret = NULL;
++    VhostShadowVirtqueue *vq = pvq;
-+    size_t size = 0;
++    event_notifier_cleanup(&vq->hdev_kick);
-+    const void *data = ebpf_find_binary_by_id(id, &size, errp);
++    event_notifier_cleanup(&vq->hdev_call);
-+    if (!data) {
++    g_free(vq);
 +        return NULL;
 +    }
 +
 +    ret = g_new0(EbpfObject, 1);
 +    ret->object = g_base64_encode(data, size);
 +
 +    return ret;
 +}
-diff --git a/ebpf/ebpf.h b/ebpf/ebpf.h
+diff --git a/hw/virtio/vhost-shadow-virtqueue.h b/hw/virtio/vhost-shadow-virtqueue.h
 new file mode 100644
 index XXXXXXX..XXXXXXX
 --- /dev/null
-+++ b/ebpf/ebpf.h
++++ b/hw/virtio/vhost-shadow-virtqueue.h
 @@ -XXX,XX +XXX,XX @@
 +/*
-+ * QEMU eBPF binary declaration routine.
++ * vhost shadow virtqueue
 + *
-+ * Developed by Daynix Computing LTD (http://www.daynix.com)
++ * SPDX-FileCopyrightText: Red Hat, Inc. 2021
 + * SPDX-FileContributor: Author: Eugenio Pérez <eperezma@redhat.com>
 + *
-+ * Authors:
++ * SPDX-License-Identifier: GPL-2.0-or-later
 + *  Andrew Melnychenko <andrew@daynix.com>
 + *
 + * This work is licensed under the terms of the GNU GPL, version 2 or
 + * later.  See the COPYING file in the top-level directory.
 + */
 +
-+#ifndef EBPF_H
++#ifndef VHOST_SHADOW_VIRTQUEUE_H
-+#define EBPF_H
++#define VHOST_SHADOW_VIRTQUEUE_H
 +
-+struct Error;
++#include "qemu/event_notifier.h"
 +
-+void ebpf_register_binary_data(int id, const void *data,
++/* Shadow virtqueue to relay notifications */
-+                               size_t datalen);
++typedef struct VhostShadowVirtqueue {
-+const void *ebpf_find_binary_by_id(int id, size_t *sz,
++    /* Shadow kick notifier, sent to vhost */
-+                                   struct Error **errp);
++    EventNotifier hdev_kick;
 +    /* Shadow call notifier, sent to vhost */
 +    EventNotifier hdev_call;
 +} VhostShadowVirtqueue;
 +
-+#define ebpf_binary_init(id, fn)                                           \
++VhostShadowVirtqueue *vhost_svq_new(void);
 +static void __attribute__((constructor)) ebpf_binary_init_ ## fn(void)     \
 +{                                                                          \
 +    size_t datalen = 0;                                                    \
 +    const void *data = fn(&datalen);                                       \
 +    ebpf_register_binary_data(id, data, datalen);                          \
 +}
 +
-+#endif /* EBPF_H */
++void vhost_svq_free(gpointer vq);
-diff --git a/ebpf/ebpf_rss.c b/ebpf/ebpf_rss.c
++G_DEFINE_AUTOPTR_CLEANUP_FUNC(VhostShadowVirtqueue, vhost_svq_free);
 index XXXXXXX..XXXXXXX 100644
 --- a/ebpf/ebpf_rss.c
 +++ b/ebpf/ebpf_rss.c
@@ -XXX,XX +XXX,XX @@
  #include "qemu/osdep.h"
  #include "qemu/error-report.h"
 +#include "qapi/qapi-types-misc.h"
 +#include "qapi/qapi-commands-ebpf.h"
  #include <bpf/libbpf.h>
  #include <bpf/bpf.h>
@@ -XXX,XX +XXX,XX @@
  #include "ebpf/ebpf_rss.h"
  #include "ebpf/rss.bpf.skeleton.h"
 +#include "ebpf/ebpf.h"
 +
- #include "trace.h"
++#endif
  void ebpf_rss_init(struct EBPFRSSContext *ctx)
@@ -XXX,XX +XXX,XX @@ void ebpf_rss_unload(struct EBPFRSSContext *ctx)
      ctx->map_toeplitz_key = -1;
      ctx->map_indirections_table = -1;
  }
 +
 +ebpf_binary_init(EBPF_PROGRAMID_RSS, rss_bpf__elf_bytes)
 diff --git a/ebpf/meson.build b/ebpf/meson.build
 index XXXXXXX..XXXXXXX 100644
 --- a/ebpf/meson.build
 +++ b/ebpf/meson.build
@@ -1 +1 @@
 -system_ss.add(when: libbpf, if_true: files('ebpf_rss.c'), if_false: files('ebpf_rss-stub.c'))
 +common_ss.add(when: libbpf, if_true: files('ebpf.c', 'ebpf_rss.c'), if_false: files('ebpf_rss-stub.c'))
 diff --git a/qapi/ebpf.json b/qapi/ebpf.json
 new file mode 100644
 index XXXXXXX..XXXXXXX
 --- /dev/null
 +++ b/qapi/ebpf.json
@@ -XXX,XX +XXX,XX @@
 +# -*- Mode: Python -*-
 +# vim: filetype=python
 +#
 +# This work is licensed under the terms of the GNU GPL, version 2 or later.
 +# See the COPYING file in the top-level directory.
 +
 +##
 +# = eBPF Objects
 +#
 +# eBPF object is an ELF binary that contains the eBPF
 +# program and eBPF map description(BTF). Overall, eBPF
 +# object should contain the program and enough metadata
 +# to create/load eBPF with libbpf. As the eBPF maps/program
 +# should correspond to QEMU, the eBPF can't be used from
 +# different QEMU build.
 +#
 +# Currently, there is a possible eBPF for receive-side scaling (RSS).
 +#
 +##
 +
 +##
 +# @EbpfObject:
 +#
 +# An eBPF ELF object.
 +#
 +# @object: the eBPF object encoded in base64
 +#
 +# Since: 8.2
 +##
 +{ 'struct': 'EbpfObject',
 +  'data': {'object': 'str'},
 +  'if': 'CONFIG_EBPF' }
 +
 +##
 +# @EbpfProgramID:
 +#
 +# The eBPF programs that can be gotten with request-ebpf.
 +#
 +# @rss: Receive side scaling, technology that allows steering traffic
 +#     between queues by calculation hash.  Users may set up
 +#     indirection table and hash/packet types configurations.  Used
 +#     with virtio-net.
 +#
 +# Since: 8.2
 +##
 +{ 'enum': 'EbpfProgramID',
 +  'if': 'CONFIG_EBPF',
 +  'data': [ { 'name': 'rss' } ] }
 +
 +##
 +# @request-ebpf:
 +#
 +# Retrieve an eBPF object that can be loaded with libbpf.  Management
 +# applications (g.e. libvirt) may load it and pass file descriptors to
 +# QEMU, so they can run running QEMU without BPF capabilities.
 +#
 +# @id: The ID of the program to return.
 +#
 +# Returns: eBPF object encoded in base64.
 +#
 +# Since: 8.2
 +##
 +{ 'command': 'request-ebpf',
 +  'data': { 'id': 'EbpfProgramID' },
 +  'returns': 'EbpfObject',
 +  'if': 'CONFIG_EBPF' }
 diff --git a/qapi/meson.build b/qapi/meson.build
 index XXXXXXX..XXXXXXX 100644
 --- a/qapi/meson.build
 +++ b/qapi/meson.build
@@ -XXX,XX +XXX,XX @@ qapi_all_modules = [
    'crypto',
    'cxl',
    'dump',
 +  'ebpf',
    'error',
    'introspect',
    'job',
 diff --git a/qapi/qapi-schema.json b/qapi/qapi-schema.json
 index XXXXXXX..XXXXXXX 100644
 --- a/qapi/qapi-schema.json
 +++ b/qapi/qapi-schema.json
@@ -XXX,XX +XXX,XX @@
  { 'include': 'char.json' }
  { 'include': 'dump.json' }
  { 'include': 'net.json' }
 +{ 'include': 'ebpf.json' }
  { 'include': 'rdma.json' }
  { 'include': 'rocker.json' }
  { 'include': 'tpm.json' }
 --
 .7.4

-[PULL 14/17] ebpf: Added eBPF initialization by fds.
+[PULL V3 03/15] vhost: Add Shadow VirtQueue kick forwarding capabilities
-From: Andrew Melnychenko <andrew@daynix.com>
+From: Eugenio Pérez <eperezma@redhat.com>
-It allows using file descriptors of eBPF provided
+At this mode no buffer forwarding will be performed in SVQ mode: Qemu
-outside of QEMU.
+will just forward the guest's kicks to the device.
-QEMU may be run without capabilities for eBPF and run
-RSS program provided by management tool(g.e. libvirt).
+Host memory notifiers regions are left out for simplicity, and they will
+not be addressed in this series.
-Signed-off-by: Andrew Melnychenko <andrew@daynix.com>
 Signed-off-by: Eugenio Pérez <eperezma@redhat.com>
 Acked-by: Michael S. Tsirkin <mst@redhat.com>
 Signed-off-by: Jason Wang <jasowang@redhat.com>
 ---
- ebpf/ebpf_rss-stub.c |  6 ++++++
+ hw/virtio/vhost-shadow-virtqueue.c |  55 ++++++++++++++
- ebpf/ebpf_rss.c      | 27 +++++++++++++++++++++++++++
+ hw/virtio/vhost-shadow-virtqueue.h |  14 ++++
- ebpf/ebpf_rss.h      |  5 +++++
+ hw/virtio/vhost-vdpa.c             | 144 ++++++++++++++++++++++++++++++++++++-
-files changed, 38 insertions(+)
+ include/hw/virtio/vhost-vdpa.h     |   4 ++
+files changed, 215 insertions(+), 2 deletions(-)
-diff --git a/ebpf/ebpf_rss-stub.c b/ebpf/ebpf_rss-stub.c
 diff --git a/hw/virtio/vhost-shadow-virtqueue.c b/hw/virtio/vhost-shadow-virtqueue.c
 index XXXXXXX..XXXXXXX 100644
---- a/ebpf/ebpf_rss-stub.c
+--- a/hw/virtio/vhost-shadow-virtqueue.c
-+++ b/ebpf/ebpf_rss-stub.c
++++ b/hw/virtio/vhost-shadow-virtqueue.c
-@@ -XXX,XX +XXX,XX @@ bool ebpf_rss_load(struct EBPFRSSContext *ctx)
+@@ -XXX,XX +XXX,XX @@
-     return false;
+ #include "hw/virtio/vhost-shadow-virtqueue.h"
- }
+ #include "qemu/error-report.h"
-+bool ebpf_rss_load_fds(struct EBPFRSSContext *ctx, int program_fd,
++#include "qemu/main-loop.h"
-+                       int config_fd, int toeplitz_fd, int table_fd)
++#include "linux-headers/linux/vhost.h"
-+{
++
-+    return false;
++/**
-+}
++ * Forward guest notifications.
-+
++ *
- bool ebpf_rss_set_all(struct EBPFRSSContext *ctx, struct EBPFRSSConfig *config,
++ * @n: guest kick event notifier, the one that guest set to notify svq.
-                       uint16_t *indirections_table, uint8_t *toeplitz_key)
++ */
- {
++static void vhost_handle_guest_kick(EventNotifier *n)
-diff --git a/ebpf/ebpf_rss.c b/ebpf/ebpf_rss.c
++{
 +    VhostShadowVirtqueue *svq = container_of(n, VhostShadowVirtqueue, svq_kick);
 +    event_notifier_test_and_clear(n);
 +    event_notifier_set(&svq->hdev_kick);
 +}
 +
 +/**
 + * Set a new file descriptor for the guest to kick the SVQ and notify for avail
 + *
 + * @svq: The svq
 + * @svq_kick_fd: The svq kick fd
 + *
 + * Note that the SVQ will never close the old file descriptor.
 + */
 +void vhost_svq_set_svq_kick_fd(VhostShadowVirtqueue *svq, int svq_kick_fd)
 +{
 +    EventNotifier *svq_kick = &svq->svq_kick;
 +    bool poll_stop = VHOST_FILE_UNBIND != event_notifier_get_fd(svq_kick);
 +    bool poll_start = svq_kick_fd != VHOST_FILE_UNBIND;
 +
 +    if (poll_stop) {
 +        event_notifier_set_handler(svq_kick, NULL);
 +    }
 +
 +    /*
 +     * event_notifier_set_handler already checks for guest's notifications if
 +     * they arrive at the new file descriptor in the switch, so there is no
 +     * need to explicitly check for them.
 +     */
 +    if (poll_start) {
 +        event_notifier_init_fd(svq_kick, svq_kick_fd);
 +        event_notifier_set(svq_kick);
 +        event_notifier_set_handler(svq_kick, vhost_handle_guest_kick);
 +    }
 +}
 +
 +/**
 + * Stop the shadow virtqueue operation.
 + * @svq: Shadow Virtqueue
 + */
 +void vhost_svq_stop(VhostShadowVirtqueue *svq)
 +{
 +    event_notifier_set_handler(&svq->svq_kick, NULL);
 +}
  /**
   * Creates vhost shadow virtqueue, and instructs the vhost device to use the
@@ -XXX,XX +XXX,XX @@ VhostShadowVirtqueue *vhost_svq_new(void)
          goto err_init_hdev_call;
      }
 +    event_notifier_init_fd(&svq->svq_kick, VHOST_FILE_UNBIND);
      return g_steal_pointer(&svq);
  err_init_hdev_call:
@@ -XXX,XX +XXX,XX @@ err_init_hdev_kick:
  void vhost_svq_free(gpointer pvq)
  {
      VhostShadowVirtqueue *vq = pvq;
 +    vhost_svq_stop(vq);
      event_notifier_cleanup(&vq->hdev_kick);
      event_notifier_cleanup(&vq->hdev_call);
      g_free(vq);
 diff --git a/hw/virtio/vhost-shadow-virtqueue.h b/hw/virtio/vhost-shadow-virtqueue.h
 index XXXXXXX..XXXXXXX 100644
---- a/ebpf/ebpf_rss.c
+--- a/hw/virtio/vhost-shadow-virtqueue.h
-+++ b/ebpf/ebpf_rss.c
++++ b/hw/virtio/vhost-shadow-virtqueue.h
-@@ -XXX,XX +XXX,XX @@ error:
+@@ -XXX,XX +XXX,XX @@ typedef struct VhostShadowVirtqueue {
-     return false;
+     EventNotifier hdev_kick;
- }
+     /* Shadow call notifier, sent to vhost */
+     EventNotifier hdev_call;
-+bool ebpf_rss_load_fds(struct EBPFRSSContext *ctx, int program_fd,
++
-+                       int config_fd, int toeplitz_fd, int table_fd)
++    /*
-+{
++     * Borrowed virtqueue's guest to host notifier. To borrow it in this event
-+    if (ctx == NULL || ebpf_rss_is_loaded(ctx)) {
++     * notifier allows to recover the VhostShadowVirtqueue from the event loop
-+        return false;
++     * easily. If we use the VirtQueue's one, we don't have an easy way to
-+    }
++     * retrieve VhostShadowVirtqueue.
-+
++     *
-+    if (program_fd < 0 || config_fd < 0 || toeplitz_fd < 0 || table_fd < 0) {
++     * So shadow virtqueue must not clean it, or we would lose VirtQueue one.
-+        return false;
++     */
-+    }
++    EventNotifier svq_kick;
-+
+ } VhostShadowVirtqueue;
-+    ctx->program_fd = program_fd;
-+    ctx->map_configuration = config_fd;
++void vhost_svq_set_svq_kick_fd(VhostShadowVirtqueue *svq, int svq_kick_fd);
-+    ctx->map_toeplitz_key = toeplitz_fd;
++
-+    ctx->map_indirections_table = table_fd;
++void vhost_svq_stop(VhostShadowVirtqueue *svq);
 +
-+    if (!ebpf_rss_mmap(ctx)) {
+ VhostShadowVirtqueue *vhost_svq_new(void);
-+        ctx->program_fd = -1;
-+        ctx->map_configuration = -1;
+ void vhost_svq_free(gpointer vq);
-+        ctx->map_toeplitz_key = -1;
+diff --git a/hw/virtio/vhost-vdpa.c b/hw/virtio/vhost-vdpa.c
-+        ctx->map_indirections_table = -1;
+index XXXXXXX..XXXXXXX 100644
-+        return false;
+--- a/hw/virtio/vhost-vdpa.c
 +++ b/hw/virtio/vhost-vdpa.c
@@ -XXX,XX +XXX,XX @@
  #include "hw/virtio/vhost.h"
  #include "hw/virtio/vhost-backend.h"
  #include "hw/virtio/virtio-net.h"
 +#include "hw/virtio/vhost-shadow-virtqueue.h"
  #include "hw/virtio/vhost-vdpa.h"
  #include "exec/address-spaces.h"
  #include "qemu/main-loop.h"
  #include "cpu.h"
  #include "trace.h"
  #include "qemu-common.h"
 +#include "qapi/error.h"
  /*
   * Return one past the end of the end of section. Be careful with uint64_t
@@ -XXX,XX +XXX,XX @@ static bool vhost_vdpa_one_time_request(struct vhost_dev *dev)
      return v->index != 0;
  }
 +static int vhost_vdpa_init_svq(struct vhost_dev *hdev, struct vhost_vdpa *v,
 +                               Error **errp)
 +{
 +    g_autoptr(GPtrArray) shadow_vqs = NULL;
 +
 +    if (!v->shadow_vqs_enabled) {
 +        return 0;
 +    }
 +
 +    shadow_vqs = g_ptr_array_new_full(hdev->nvqs, vhost_svq_free);
 +    for (unsigned n = 0; n < hdev->nvqs; ++n) {
 +        g_autoptr(VhostShadowVirtqueue) svq = vhost_svq_new();
 +
 +        if (unlikely(!svq)) {
 +            error_setg(errp, "Cannot create svq %u", n);
 +            return -1;
 +        }
 +        g_ptr_array_add(shadow_vqs, g_steal_pointer(&svq));
 +    }
 +
 +    v->shadow_vqs = g_steal_pointer(&shadow_vqs);
 +    return 0;
 +}
 +
  static int vhost_vdpa_init(struct vhost_dev *dev, void *opaque, Error **errp)
  {
      struct vhost_vdpa *v;
@@ -XXX,XX +XXX,XX @@ static int vhost_vdpa_init(struct vhost_dev *dev, void *opaque, Error **errp)
      dev->opaque =  opaque ;
      v->listener = vhost_vdpa_memory_listener;
      v->msg_type = VHOST_IOTLB_MSG_V2;
 +    ret = vhost_vdpa_init_svq(dev, v, errp);
 +    if (ret) {
 +        goto err;
 +    }
      vhost_vdpa_get_iova_range(v);
@@ -XXX,XX +XXX,XX @@ static int vhost_vdpa_init(struct vhost_dev *dev, void *opaque, Error **errp)
                                 VIRTIO_CONFIG_S_DRIVER);
      return 0;
 +
 +err:
 +    ram_block_discard_disable(false);
 +    return ret;
  }
  static void vhost_vdpa_host_notifier_uninit(struct vhost_dev *dev,
@@ -XXX,XX +XXX,XX @@ static void vhost_vdpa_host_notifiers_uninit(struct vhost_dev *dev, int n)
  static void vhost_vdpa_host_notifiers_init(struct vhost_dev *dev)
  {
 +    struct vhost_vdpa *v = dev->opaque;
      int i;
 +    if (v->shadow_vqs_enabled) {
 +        /* FIXME SVQ is not compatible with host notifiers mr */
 +        return;
 +    }
 +
      for (i = dev->vq_index; i < dev->vq_index + dev->nvqs; i++) {
          if (vhost_vdpa_host_notifier_init(dev, i)) {
              goto err;
@@ -XXX,XX +XXX,XX @@ err:
      return;
  }
 +static void vhost_vdpa_svq_cleanup(struct vhost_dev *dev)
 +{
 +    struct vhost_vdpa *v = dev->opaque;
 +    size_t idx;
 +
 +    if (!v->shadow_vqs) {
 +        return;
 +    }
 +
 +    for (idx = 0; idx < v->shadow_vqs->len; ++idx) {
 +        vhost_svq_stop(g_ptr_array_index(v->shadow_vqs, idx));
 +    }
 +    g_ptr_array_free(v->shadow_vqs, true);
 +}
 +
  static int vhost_vdpa_cleanup(struct vhost_dev *dev)
  {
      struct vhost_vdpa *v;
@@ -XXX,XX +XXX,XX @@ static int vhost_vdpa_cleanup(struct vhost_dev *dev)
      trace_vhost_vdpa_cleanup(dev, v);
      vhost_vdpa_host_notifiers_uninit(dev, dev->nvqs);
      memory_listener_unregister(&v->listener);
 +    vhost_vdpa_svq_cleanup(dev);
      dev->opaque = NULL;
      ram_block_discard_disable(false);
@@ -XXX,XX +XXX,XX @@ static int vhost_vdpa_get_device_id(struct vhost_dev *dev,
      return ret;
  }
 +static void vhost_vdpa_reset_svq(struct vhost_vdpa *v)
 +{
 +    if (!v->shadow_vqs_enabled) {
 +        return;
 +    }
 +
 +    for (unsigned i = 0; i < v->shadow_vqs->len; ++i) {
 +        VhostShadowVirtqueue *svq = g_ptr_array_index(v->shadow_vqs, i);
 +        vhost_svq_stop(svq);
 +    }
 +}
 +
  static int vhost_vdpa_reset_device(struct vhost_dev *dev)
  {
 +    struct vhost_vdpa *v = dev->opaque;
      int ret;
      uint8_t status = 0;
 +    vhost_vdpa_reset_svq(v);
 +
      ret = vhost_vdpa_call(dev, VHOST_VDPA_SET_STATUS, &status);
      trace_vhost_vdpa_reset_device(dev, status);
      return ret;
@@ -XXX,XX +XXX,XX @@ static int vhost_vdpa_get_config(struct vhost_dev *dev, uint8_t *config,
      return ret;
   }
 +static int vhost_vdpa_set_vring_dev_kick(struct vhost_dev *dev,
 +                                         struct vhost_vring_file *file)
 +{
 +    trace_vhost_vdpa_set_vring_kick(dev, file->index, file->fd);
 +    return vhost_vdpa_call(dev, VHOST_SET_VRING_KICK, file);
 +}
 +
 +/**
 + * Set the shadow virtqueue descriptors to the device
 + *
 + * @dev: The vhost device model
 + * @svq: The shadow virtqueue
 + * @idx: The index of the virtqueue in the vhost device
 + * @errp: Error
 + */
 +static bool vhost_vdpa_svq_setup(struct vhost_dev *dev,
 +                                 VhostShadowVirtqueue *svq, unsigned idx,
 +                                 Error **errp)
 +{
 +    struct vhost_vring_file file = {
 +        .index = dev->vq_index + idx,
 +    };
 +    const EventNotifier *event_notifier = &svq->hdev_kick;
 +    int r;
 +
 +    file.fd = event_notifier_get_fd(event_notifier);
 +    r = vhost_vdpa_set_vring_dev_kick(dev, &file);
 +    if (unlikely(r != 0)) {
 +        error_setg_errno(errp, -r, "Can't set device kick fd");
 +    }
 +
 +    return r == 0;
 +}
 +
 +static bool vhost_vdpa_svqs_start(struct vhost_dev *dev)
 +{
 +    struct vhost_vdpa *v = dev->opaque;
 +    Error *err = NULL;
 +    unsigned i;
 +
 +    if (!v->shadow_vqs) {
 +        return true;
 +    }
 +
 +    for (i = 0; i < v->shadow_vqs->len; ++i) {
 +        VhostShadowVirtqueue *svq = g_ptr_array_index(v->shadow_vqs, i);
 +        bool ok = vhost_vdpa_svq_setup(dev, svq, i, &err);
 +        if (unlikely(!ok)) {
 +            error_reportf_err(err, "Cannot setup SVQ %u: ", i);
 +            return false;
 +        }
 +    }
 +
 +    return true;
 +}
 +
- static bool ebpf_rss_set_config(struct EBPFRSSContext *ctx,
+ static int vhost_vdpa_dev_start(struct vhost_dev *dev, bool started)
-                                 struct EBPFRSSConfig *config)
+ {
- {
+     struct vhost_vdpa *v = dev->opaque;
-diff --git a/ebpf/ebpf_rss.h b/ebpf/ebpf_rss.h
++    bool ok;
      trace_vhost_vdpa_dev_start(dev, started);
      if (started) {
          vhost_vdpa_host_notifiers_init(dev);
 +        ok = vhost_vdpa_svqs_start(dev);
 +        if (unlikely(!ok)) {
 +            return -1;
 +        }
          vhost_vdpa_set_vring_ready(dev);
      } else {
          vhost_vdpa_host_notifiers_uninit(dev, dev->nvqs);
@@ -XXX,XX +XXX,XX @@ static int vhost_vdpa_get_vring_base(struct vhost_dev *dev,
  static int vhost_vdpa_set_vring_kick(struct vhost_dev *dev,
                                         struct vhost_vring_file *file)
  {
 -    trace_vhost_vdpa_set_vring_kick(dev, file->index, file->fd);
 -    return vhost_vdpa_call(dev, VHOST_SET_VRING_KICK, file);
 +    struct vhost_vdpa *v = dev->opaque;
 +    int vdpa_idx = file->index - dev->vq_index;
 +
 +    if (v->shadow_vqs_enabled) {
 +        VhostShadowVirtqueue *svq = g_ptr_array_index(v->shadow_vqs, vdpa_idx);
 +        vhost_svq_set_svq_kick_fd(svq, file->fd);
 +        return 0;
 +    } else {
 +        return vhost_vdpa_set_vring_dev_kick(dev, file);
 +    }
  }
  static int vhost_vdpa_set_vring_call(struct vhost_dev *dev,
 diff --git a/include/hw/virtio/vhost-vdpa.h b/include/hw/virtio/vhost-vdpa.h
 index XXXXXXX..XXXXXXX 100644
---- a/ebpf/ebpf_rss.h
+--- a/include/hw/virtio/vhost-vdpa.h
-+++ b/ebpf/ebpf_rss.h
++++ b/include/hw/virtio/vhost-vdpa.h
 @@ -XXX,XX +XXX,XX @@
- #ifndef QEMU_EBPF_RSS_H
+ #ifndef HW_VIRTIO_VHOST_VDPA_H
- #define QEMU_EBPF_RSS_H
+ #define HW_VIRTIO_VHOST_VDPA_H
-+#define EBPF_RSS_MAX_FDS 4
++#include <gmodule.h>
 +
- struct EBPFRSSContext {
+ #include "hw/virtio/virtio.h"
-     void *obj;
+ #include "standard-headers/linux/vhost_types.h"
-     int program_fd;
-@@ -XXX,XX +XXX,XX @@ bool ebpf_rss_is_loaded(struct EBPFRSSContext *ctx);
+@@ -XXX,XX +XXX,XX @@ typedef struct vhost_vdpa {
+     bool iotlb_batch_begin_sent;
- bool ebpf_rss_load(struct EBPFRSSContext *ctx);
+     MemoryListener listener;
+     struct vhost_vdpa_iova_range iova_range;
-+bool ebpf_rss_load_fds(struct EBPFRSSContext *ctx, int program_fd,
++    bool shadow_vqs_enabled;
-+                       int config_fd, int toeplitz_fd, int table_fd);
++    GPtrArray *shadow_vqs;
-+
+     struct vhost_dev *dev;
- bool ebpf_rss_set_all(struct EBPFRSSContext *ctx, struct EBPFRSSConfig *config,
+     VhostVDPAHostNotifier notifier[VIRTIO_QUEUE_MAX];
-                       uint16_t *indirections_table, uint8_t *toeplitz_key);
+ } VhostVDPA;
 --
 .7.4

-[PULL 10/17] igb: packet-split descriptors support
+[PULL V3 04/15] vhost: Add Shadow VirtQueue call forwarding capabilities
-From: Tomasz Dzieciol <t.dzieciol@partner.samsung.com>
+From: Eugenio Pérez <eperezma@redhat.com>
-Packet-split descriptors are used by Linux VF driver for MTU values from 2048
+This will make qemu aware of the device used buffers, allowing it to
 write the guest memory with its contents if needed.
-Signed-off-by: Tomasz Dzieciol <t.dzieciol@partner.samsung.com>
+Signed-off-by: Eugenio Pérez <eperezma@redhat.com>
-Reviewed-by: Akihiko Odaki <akihiko.odaki@daynix.com>
+Acked-by: Michael S. Tsirkin <mst@redhat.com>
 Tested-by: Akihiko Odaki <akihiko.odaki@daynix.com>
 Signed-off-by: Jason Wang <jasowang@redhat.com>
 ---
- hw/net/igb_core.c   | 348 +++++++++++++++++++++++++++++++++++++++++++++-------
+ hw/virtio/vhost-shadow-virtqueue.c | 38 ++++++++++++++++++++++++++++++++++++++
- hw/net/igb_regs.h   |   9 ++
+ hw/virtio/vhost-shadow-virtqueue.h |  4 ++++
- hw/net/trace-events |   2 +-
+ hw/virtio/vhost-vdpa.c             | 31 +++++++++++++++++++++++++++++--
-files changed, 316 insertions(+), 43 deletions(-)
+files changed, 71 insertions(+), 2 deletions(-)
-diff --git a/hw/net/igb_core.c b/hw/net/igb_core.c
+diff --git a/hw/virtio/vhost-shadow-virtqueue.c b/hw/virtio/vhost-shadow-virtqueue.c
 index XXXXXXX..XXXXXXX 100644
---- a/hw/net/igb_core.c
+--- a/hw/virtio/vhost-shadow-virtqueue.c
-+++ b/hw/net/igb_core.c
++++ b/hw/virtio/vhost-shadow-virtqueue.c
-@@ -XXX,XX +XXX,XX @@ igb_rx_use_legacy_descriptor(IGBCore *core)
+@@ -XXX,XX +XXX,XX @@ static void vhost_handle_guest_kick(EventNotifier *n)
      return false;
  }
-+typedef struct E1000ERingInfo {
+ /**
-+    int dbah;
++ * Forward vhost notifications
-+    int dbal;
++ *
-+    int dlen;
++ * @n: hdev call event notifier, the one that device set to notify svq.
-+    int dh;
++ */
-+    int dt;
++static void vhost_svq_handle_call(EventNotifier *n)
 +    int idx;
 +} E1000ERingInfo;
 +
 +static uint32_t
 +igb_rx_queue_desctyp_get(IGBCore *core, const E1000ERingInfo *r)
 +{
-+    return core->mac[E1000_SRRCTL(r->idx) >> 2] & E1000_SRRCTL_DESCTYPE_MASK;
++    VhostShadowVirtqueue *svq = container_of(n, VhostShadowVirtqueue,
 +                                             hdev_call);
 +    event_notifier_test_and_clear(n);
 +    event_notifier_set(&svq->svq_call);
 +}
 +
-+static bool
++/**
-+igb_rx_use_ps_descriptor(IGBCore *core, const E1000ERingInfo *r)
++ * Set the call notifier for the SVQ to call the guest
 + *
 + * @svq: Shadow virtqueue
 + * @call_fd: call notifier
 + *
 + * Called on BQL context.
 + */
 +void vhost_svq_set_svq_call_fd(VhostShadowVirtqueue *svq, int call_fd)
 +{
-+    uint32_t desctyp = igb_rx_queue_desctyp_get(core, r);
++    if (call_fd == VHOST_FILE_UNBIND) {
-+    return desctyp == E1000_SRRCTL_DESCTYPE_HDR_SPLIT ||
++        /*
-+           desctyp == E1000_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS;
++         * Fail event_notifier_set if called handling device call.
 +         *
 +         * SVQ still needs device notifications, since it needs to keep
 +         * forwarding used buffers even with the unbind.
 +         */
 +        memset(&svq->svq_call, 0, sizeof(svq->svq_call));
 +    } else {
 +        event_notifier_init_fd(&svq->svq_call, call_fd);
 +    }
 +}
 +
- static inline bool
++/**
- igb_rss_enabled(IGBCore *core)
+  * Set a new file descriptor for the guest to kick the SVQ and notify for avail
- {
+  *
-@@ -XXX,XX +XXX,XX @@ static uint32_t igb_rx_wb_eic(IGBCore *core, int queue_idx)
+  * @svq: The svq
-     return (ent & E1000_IVAR_VALID) ? BIT(ent & 0x1f) : 0;
+@@ -XXX,XX +XXX,XX @@ VhostShadowVirtqueue *vhost_svq_new(void)
      }
      event_notifier_init_fd(&svq->svq_kick, VHOST_FILE_UNBIND);
 +    event_notifier_set_handler(&svq->hdev_call, vhost_svq_handle_call);
      return g_steal_pointer(&svq);
  err_init_hdev_call:
@@ -XXX,XX +XXX,XX @@ void vhost_svq_free(gpointer pvq)
      VhostShadowVirtqueue *vq = pvq;
      vhost_svq_stop(vq);
      event_notifier_cleanup(&vq->hdev_kick);
 +    event_notifier_set_handler(&vq->hdev_call, NULL);
      event_notifier_cleanup(&vq->hdev_call);
      g_free(vq);
  }
+diff --git a/hw/virtio/vhost-shadow-virtqueue.h b/hw/virtio/vhost-shadow-virtqueue.h
--typedef struct E1000ERingInfo {
+index XXXXXXX..XXXXXXX 100644
--    int dbah;
+--- a/hw/virtio/vhost-shadow-virtqueue.h
--    int dbal;
++++ b/hw/virtio/vhost-shadow-virtqueue.h
--    int dlen;
+@@ -XXX,XX +XXX,XX @@ typedef struct VhostShadowVirtqueue {
--    int dh;
+      * So shadow virtqueue must not clean it, or we would lose VirtQueue one.
--    int dt;
+      */
--    int idx;
+     EventNotifier svq_kick;
--} E1000ERingInfo;
++
--
++    /* Guest's call notifier, where the SVQ calls guest. */
- static inline bool
++    EventNotifier svq_call;
- igb_ring_empty(IGBCore *core, const E1000ERingInfo *r)
+ } VhostShadowVirtqueue;
- {
-@@ -XXX,XX +XXX,XX @@ igb_read_lgcy_rx_descr(IGBCore *core, struct e1000_rx_desc *desc,
+ void vhost_svq_set_svq_kick_fd(VhostShadowVirtqueue *svq, int svq_kick_fd);
 +void vhost_svq_set_svq_call_fd(VhostShadowVirtqueue *svq, int call_fd);
  void vhost_svq_stop(VhostShadowVirtqueue *svq);
 diff --git a/hw/virtio/vhost-vdpa.c b/hw/virtio/vhost-vdpa.c
 index XXXXXXX..XXXXXXX 100644
 --- a/hw/virtio/vhost-vdpa.c
 +++ b/hw/virtio/vhost-vdpa.c
@@ -XXX,XX +XXX,XX @@ static int vhost_vdpa_set_vring_dev_kick(struct vhost_dev *dev,
      return vhost_vdpa_call(dev, VHOST_SET_VRING_KICK, file);
  }
- static inline void
++static int vhost_vdpa_set_vring_dev_call(struct vhost_dev *dev,
--igb_read_adv_rx_descr(IGBCore *core, union e1000_adv_rx_desc *desc,
++                                         struct vhost_vring_file *file)
 -                      hwaddr *buff_addr)
 +igb_read_adv_rx_single_buf_descr(IGBCore *core, union e1000_adv_rx_desc *desc,
 +                                 hwaddr *buff_addr)
  {
      *buff_addr = le64_to_cpu(desc->read.pkt_addr);
  }
 +static inline void
 +igb_read_adv_rx_split_buf_descr(IGBCore *core, union e1000_adv_rx_desc *desc,
 +                                hwaddr *buff_addr)
 +{
-+    buff_addr[0] = le64_to_cpu(desc->read.hdr_addr);
++    trace_vhost_vdpa_set_vring_call(dev, file->index, file->fd);
-+    buff_addr[1] = le64_to_cpu(desc->read.pkt_addr);
++    return vhost_vdpa_call(dev, VHOST_SET_VRING_CALL, file);
 +}
 +
-+typedef struct IGBBAState {
+ /**
-+    uint16_t written[IGB_MAX_PS_BUFFERS];
+  * Set the shadow virtqueue descriptors to the device
-+    uint8_t cur_idx;
+  *
-+} IGBBAState;
+@@ -XXX,XX +XXX,XX @@ static int vhost_vdpa_set_vring_dev_kick(struct vhost_dev *dev,
-+
+  * @svq: The shadow virtqueue
-+typedef struct IGBSplitDescriptorData {
+  * @idx: The index of the virtqueue in the vhost device
-+    bool sph;
+  * @errp: Error
-+    bool hbo;
++ *
-+    size_t hdr_len;
++ * Note that this function does not rewind kick file descriptor if cannot set
-+} IGBSplitDescriptorData;
++ * call one.
-+
+  */
- typedef struct IGBPacketRxDMAState {
+ static bool vhost_vdpa_svq_setup(struct vhost_dev *dev,
-     size_t size;
+                                  VhostShadowVirtqueue *svq, unsigned idx,
-     size_t total_size;
+@@ -XXX,XX +XXX,XX @@ static bool vhost_vdpa_svq_setup(struct vhost_dev *dev,
-@@ -XXX,XX +XXX,XX @@ typedef struct IGBPacketRxDMAState {
+     r = vhost_vdpa_set_vring_dev_kick(dev, &file);
-     uint32_t rx_desc_header_buf_size;
+     if (unlikely(r != 0)) {
-     struct iovec *iov;
+         error_setg_errno(errp, -r, "Can't set device kick fd");
      size_t iov_ofs;
 +    bool do_ps;
      bool is_first;
 -    uint16_t written;
 -    hwaddr ba;
 +    IGBBAState bastate;
 +    hwaddr ba[IGB_MAX_PS_BUFFERS];
 +    IGBSplitDescriptorData ps_desc_data;
  } IGBPacketRxDMAState;
  static inline void
 -igb_read_rx_descr(IGBCore *core, union e1000_rx_desc_union *desc,
 -                  hwaddr *buff_addr)
 +igb_read_rx_descr(IGBCore *core,
 +                  union e1000_rx_desc_union *desc,
 +                  IGBPacketRxDMAState *pdma_st,
 +                  const E1000ERingInfo *r)
  {
 +    uint32_t desc_type;
 +
      if (igb_rx_use_legacy_descriptor(core)) {
 -        igb_read_lgcy_rx_descr(core, &desc->legacy, buff_addr);
 -    } else {
 -        igb_read_adv_rx_descr(core, &desc->adv, buff_addr);
 +        igb_read_lgcy_rx_descr(core, &desc->legacy, &pdma_st->ba[1]);
 +        pdma_st->ba[0] = 0;
 +        return;
 +    }
 +
 +    /* advanced header split descriptor */
 +    if (igb_rx_use_ps_descriptor(core, r)) {
 +        igb_read_adv_rx_split_buf_descr(core, &desc->adv, &pdma_st->ba[0]);
 +        return;
 +    }
 +
 +    /* descriptor replication modes not supported */
 +    desc_type = igb_rx_queue_desctyp_get(core, r);
 +    if (desc_type != E1000_SRRCTL_DESCTYPE_ADV_ONEBUF) {
 +        trace_igb_wrn_rx_desc_modes_not_supp(desc_type);
      }
 +
 +    /* advanced single buffer descriptor */
 +    igb_read_adv_rx_single_buf_descr(core, &desc->adv, &pdma_st->ba[1]);
 +    pdma_st->ba[0] = 0;
  }
  static void
@@ -XXX,XX +XXX,XX @@ igb_write_lgcy_rx_descr(IGBCore *core, struct e1000_rx_desc *desc,
      desc->status = (uint8_t) le32_to_cpu(status_flags);
  }
 +static bool
 +igb_rx_ps_descriptor_split_always(IGBCore *core, const E1000ERingInfo *r)
 +{
 +    uint32_t desctyp = igb_rx_queue_desctyp_get(core, r);
 +    return desctyp == E1000_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS;
 +}
 +
  static uint16_t
  igb_rx_desc_get_packet_type(IGBCore *core, struct NetRxPkt *pkt, uint16_t etqf)
  {
@@ -XXX,XX +XXX,XX @@ igb_write_adv_rx_descr(IGBCore *core, union e1000_adv_rx_desc *desc,
  }
  static inline void
 -igb_write_rx_descr(IGBCore *core, union e1000_rx_desc_union *desc,
 -                   struct NetRxPkt *pkt, const E1000E_RSSInfo *rss_info,
 -                   uint16_t etqf, bool ts, uint16_t length)
 +igb_write_adv_ps_rx_descr(IGBCore *core,
 +                          union e1000_adv_rx_desc *desc,
 +                          struct NetRxPkt *pkt,
 +                          const E1000E_RSSInfo *rss_info,
 +                          const E1000ERingInfo *r,
 +                          uint16_t etqf,
 +                          bool ts,
 +                          IGBPacketRxDMAState *pdma_st)
 +{
 +    size_t pkt_len;
 +    uint16_t hdr_info = 0;
 +
 +    if (pdma_st->do_ps) {
 +        pkt_len = pdma_st->bastate.written[1];
 +    } else {
 +        pkt_len = pdma_st->bastate.written[0] + pdma_st->bastate.written[1];
 +    }
 +
 +    igb_write_adv_rx_descr(core, desc, pkt, rss_info, etqf, ts, pkt_len);
 +
 +    hdr_info = (pdma_st->ps_desc_data.hdr_len << E1000_ADVRXD_HDR_LEN_OFFSET) &
 +               E1000_ADVRXD_ADV_HDR_LEN_MASK;
 +    hdr_info |= pdma_st->ps_desc_data.sph ? E1000_ADVRXD_HDR_SPH : 0;
 +    desc->wb.lower.lo_dword.hdr_info = cpu_to_le16(hdr_info);
 +
 +    desc->wb.upper.status_error |= cpu_to_le32(
 +        pdma_st->ps_desc_data.hbo ? E1000_ADVRXD_ST_ERR_HBO_OFFSET : 0);
 +}
 +
 +static inline void
 +igb_write_rx_descr(IGBCore *core,
 +                   union e1000_rx_desc_union *desc,
 +                   struct NetRxPkt *pkt,
 +                   const E1000E_RSSInfo *rss_info,
 +                   uint16_t etqf,
 +                   bool ts,
 +                   IGBPacketRxDMAState *pdma_st,
 +                   const E1000ERingInfo *r)
  {
      if (igb_rx_use_legacy_descriptor(core)) {
 -        igb_write_lgcy_rx_descr(core, &desc->legacy, pkt, rss_info, length);
 +        igb_write_lgcy_rx_descr(core, &desc->legacy, pkt, rss_info,
 +                                pdma_st->bastate.written[1]);
 +    } else if (igb_rx_use_ps_descriptor(core, r)) {
 +        igb_write_adv_ps_rx_descr(core, &desc->adv, pkt, rss_info, r, etqf, ts,
 +                                  pdma_st);
      } else {
          igb_write_adv_rx_descr(core, &desc->adv, pkt, rss_info,
 -                               etqf, ts, length);
 +                               etqf, ts, pdma_st->bastate.written[1]);
      }
  }
@@ -XXX,XX +XXX,XX @@ igb_rx_descr_threshold_hit(IGBCore *core, const E1000ERingInfo *rxi)
             ((core->mac[E1000_SRRCTL(rxi->idx) >> 2] >> 20) & 31) * 16;
  }
 +static bool
 +igb_do_ps(IGBCore *core,
 +          const E1000ERingInfo *r,
 +          struct NetRxPkt *pkt,
 +          IGBPacketRxDMAState *pdma_st)
 +{
 +    bool hasip4, hasip6;
 +    EthL4HdrProto l4hdr_proto;
 +    bool fragment;
 +    bool split_always;
 +    size_t bheader_size;
 +    size_t total_pkt_len;
 +
 +    if (!igb_rx_use_ps_descriptor(core, r)) {
 +        return false;
 +    }
 +
-+    total_pkt_len = net_rx_pkt_get_total_len(pkt);
++    event_notifier = &svq->hdev_call;
-+    bheader_size = igb_rxhdrbufsize(core, r);
++    file.fd = event_notifier_get_fd(event_notifier);
-+    split_always = igb_rx_ps_descriptor_split_always(core, r);
++    r = vhost_vdpa_set_vring_dev_call(dev, &file);
-+    if (split_always && total_pkt_len <= bheader_size) {
++    if (unlikely(r != 0)) {
-+        pdma_st->ps_hdr_len = total_pkt_len;
++        error_setg_errno(errp, -r, "Can't set device call fd");
-+        pdma_st->ps_desc_data.hdr_len = total_pkt_len;
+     }
-+        return true;
-+    }
+     return r == 0;
@@ -XXX,XX +XXX,XX @@ static int vhost_vdpa_set_vring_kick(struct vhost_dev *dev,
  static int vhost_vdpa_set_vring_call(struct vhost_dev *dev,
                                         struct vhost_vring_file *file)
  {
 -    trace_vhost_vdpa_set_vring_call(dev, file->index, file->fd);
 -    return vhost_vdpa_call(dev, VHOST_SET_VRING_CALL, file);
 +    struct vhost_vdpa *v = dev->opaque;
 +
-+    net_rx_pkt_get_protocols(pkt, &hasip4, &hasip6, &l4hdr_proto);
++    if (v->shadow_vqs_enabled) {
 +        int vdpa_idx = file->index - dev->vq_index;
 +        VhostShadowVirtqueue *svq = g_ptr_array_index(v->shadow_vqs, vdpa_idx);
 +
-+    if (hasip4) {
++        vhost_svq_set_svq_call_fd(svq, file->fd);
-+        fragment = net_rx_pkt_get_ip4_info(pkt)->fragment;
++        return 0;
 +    } else if (hasip6) {
 +        fragment = net_rx_pkt_get_ip6_info(pkt)->fragment;
 +    } else {
-+        pdma_st->ps_desc_data.hdr_len = bheader_size;
++        return vhost_vdpa_set_vring_dev_call(dev, file);
 +        goto header_not_handled;
 +    }
 +
 +    if (fragment && (core->mac[RFCTL] & E1000_RFCTL_IPFRSP_DIS)) {
 +        pdma_st->ps_desc_data.hdr_len = bheader_size;
 +        goto header_not_handled;
 +    }
 +
 +    /* no header splitting for SCTP */
 +    if (!fragment && (l4hdr_proto == ETH_L4_HDR_PROTO_UDP ||
 +                      l4hdr_proto == ETH_L4_HDR_PROTO_TCP)) {
 +        pdma_st->ps_hdr_len = net_rx_pkt_get_l5_hdr_offset(pkt);
 +    } else {
 +        pdma_st->ps_hdr_len = net_rx_pkt_get_l4_hdr_offset(pkt);
 +    }
 +
 +    pdma_st->ps_desc_data.sph = true;
 +    pdma_st->ps_desc_data.hdr_len = pdma_st->ps_hdr_len;
 +
 +    if (pdma_st->ps_hdr_len > bheader_size) {
 +        pdma_st->ps_desc_data.hbo = true;
 +        goto header_not_handled;
 +    }
 +
 +    return true;
 +
 +header_not_handled:
 +    if (split_always) {
 +        pdma_st->ps_hdr_len = bheader_size;
 +        return true;
 +    }
 +
 +    return false;
 +}
 +
  static void
  igb_truncate_to_descriptor_size(IGBPacketRxDMAState *pdma_st, size_t *size)
  {
 -    if (*size > pdma_st->rx_desc_packet_buf_size) {
 -        *size = pdma_st->rx_desc_packet_buf_size;
 +    if (pdma_st->do_ps && pdma_st->is_first) {
 +        if (*size > pdma_st->rx_desc_packet_buf_size + pdma_st->ps_hdr_len) {
 +            *size = pdma_st->rx_desc_packet_buf_size + pdma_st->ps_hdr_len;
 +        }
 +    } else {
 +        if (*size > pdma_st->rx_desc_packet_buf_size) {
 +            *size = pdma_st->rx_desc_packet_buf_size;
 +        }
 +    }
 +}
 +
 +static inline void
 +igb_write_hdr_frag_to_rx_buffers(IGBCore *core,
 +                                 PCIDevice *d,
 +                                 IGBPacketRxDMAState *pdma_st,
 +                                 const char *data,
 +                                 dma_addr_t data_len)
 +{
 +    assert(data_len <= pdma_st->rx_desc_header_buf_size -
 +                       pdma_st->bastate.written[0]);
 +    pci_dma_write(d,
 +                  pdma_st->ba[0] + pdma_st->bastate.written[0],
 +                  data, data_len);
 +    pdma_st->bastate.written[0] += data_len;
 +    pdma_st->bastate.cur_idx = 1;
 +}
 +
 +static void
 +igb_write_header_to_rx_buffers(IGBCore *core,
 +                               struct NetRxPkt *pkt,
 +                               PCIDevice *d,
 +                               IGBPacketRxDMAState *pdma_st,
 +                               size_t *copy_size)
 +{
 +    size_t iov_copy;
 +    size_t ps_hdr_copied = 0;
 +
 +    if (!pdma_st->is_first) {
 +        /* Leave buffer 0 of each descriptor except first */
 +        /* empty                                          */
 +        pdma_st->bastate.cur_idx = 1;
 +        return;
      }
 +
 +    do {
 +        iov_copy = MIN(pdma_st->ps_hdr_len - ps_hdr_copied,
 +                       pdma_st->iov->iov_len - pdma_st->iov_ofs);
 +
 +        igb_write_hdr_frag_to_rx_buffers(core, d, pdma_st,
 +                                         pdma_st->iov->iov_base,
 +                                         iov_copy);
 +
 +        *copy_size -= iov_copy;
 +        ps_hdr_copied += iov_copy;
 +
 +        pdma_st->iov_ofs += iov_copy;
 +        if (pdma_st->iov_ofs == pdma_st->iov->iov_len) {
 +            pdma_st->iov++;
 +            pdma_st->iov_ofs = 0;
 +        }
 +    } while (ps_hdr_copied < pdma_st->ps_hdr_len);
 +
 +    pdma_st->is_first = false;
  }
  static void
  igb_write_payload_frag_to_rx_buffers(IGBCore *core,
                                       PCIDevice *d,
 -                                     hwaddr ba,
 -                                     uint16_t *written,
 -                                     uint32_t cur_buf_len,
 +                                     IGBPacketRxDMAState *pdma_st,
                                       const char *data,
                                       dma_addr_t data_len)
  {
 -    trace_igb_rx_desc_buff_write(ba, *written, data, data_len);
 -    pci_dma_write(d, ba + *written, data, data_len);
 -    *written += data_len;
 +    while (data_len > 0) {
 +        assert(pdma_st->bastate.cur_idx < IGB_MAX_PS_BUFFERS);
 +
 +        uint32_t cur_buf_bytes_left =
 +            pdma_st->rx_desc_packet_buf_size -
 +            pdma_st->bastate.written[pdma_st->bastate.cur_idx];
 +        uint32_t bytes_to_write = MIN(data_len, cur_buf_bytes_left);
 +
 +        trace_igb_rx_desc_buff_write(
 +            pdma_st->bastate.cur_idx,
 +            pdma_st->ba[pdma_st->bastate.cur_idx],
 +            pdma_st->bastate.written[pdma_st->bastate.cur_idx],
 +            data,
 +            bytes_to_write);
 +
 +        pci_dma_write(d,
 +                      pdma_st->ba[pdma_st->bastate.cur_idx] +
 +                      pdma_st->bastate.written[pdma_st->bastate.cur_idx],
 +                      data, bytes_to_write);
 +
 +        pdma_st->bastate.written[pdma_st->bastate.cur_idx] += bytes_to_write;
 +        data += bytes_to_write;
 +        data_len -= bytes_to_write;
 +
 +        if (pdma_st->bastate.written[pdma_st->bastate.cur_idx] ==
 +            pdma_st->rx_desc_packet_buf_size) {
 +            pdma_st->bastate.cur_idx++;
 +        }
 +    }
  }
- static void
+ static int vhost_vdpa_get_features(struct vhost_dev *dev,
@@ -XXX,XX +XXX,XX @@ igb_write_payload_to_rx_buffers(IGBCore *core,
      while (*copy_size) {
          iov_copy = MIN(*copy_size, pdma_st->iov->iov_len - pdma_st->iov_ofs);
          igb_write_payload_frag_to_rx_buffers(core, d,
 -                                             pdma_st->ba,
 -                                             &pdma_st->written,
 -                                             pdma_st->rx_desc_packet_buf_size,
 +                                             pdma_st,
                                               pdma_st->iov->iov_base +
                                               pdma_st->iov_ofs,
                                               iov_copy);
@@ -XXX,XX +XXX,XX @@ igb_write_payload_to_rx_buffers(IGBCore *core,
      if (pdma_st->desc_offset + pdma_st->desc_size >= pdma_st->total_size) {
          /* Simulate FCS checksum presence in the last descriptor */
          igb_write_payload_frag_to_rx_buffers(core, d,
 -                                             pdma_st->ba,
 -                                             &pdma_st->written,
 -                                             pdma_st->rx_desc_packet_buf_size,
 +                                             pdma_st,
                                               (const char *) &fcs_pad,
                                               e1000x_fcs_len(core->mac));
      }
@@ -XXX,XX +XXX,XX @@ igb_write_to_rx_buffers(IGBCore *core,
  {
      size_t copy_size;
 -    if (!pdma_st->ba) {
 +    if (!(pdma_st->ba)[1] || (pdma_st->do_ps && !(pdma_st->ba[0]))) {
          /* as per intel docs; skip descriptors with null buf addr */
          trace_e1000e_rx_null_descriptor();
          return;
@@ -XXX,XX +XXX,XX @@ igb_write_to_rx_buffers(IGBCore *core,
      igb_truncate_to_descriptor_size(pdma_st, &pdma_st->desc_size);
      copy_size = pdma_st->size - pdma_st->desc_offset;
      igb_truncate_to_descriptor_size(pdma_st, &copy_size);
 +
 +    /* For PS mode copy the packet header first */
 +    if (pdma_st->do_ps) {
 +        igb_write_header_to_rx_buffers(core, pkt, d, pdma_st, &copy_size);
 +    } else {
 +        pdma_st->bastate.cur_idx = 1;
 +    }
 +
      igb_write_payload_to_rx_buffers(core, pkt, d, pdma_st, &copy_size);
  }
@@ -XXX,XX +XXX,XX @@ igb_write_packet_to_guest(IGBCore *core, struct NetRxPkt *pkt,
          d = core->owner;
      }
 +    pdma_st.do_ps = igb_do_ps(core, rxi, pkt, &pdma_st);
 +
      do {
 -        pdma_st.written = 0;
 +        memset(&pdma_st.bastate, 0, sizeof(IGBBAState));
          bool is_last = false;
          if (igb_ring_empty(core, rxi)) {
@@ -XXX,XX +XXX,XX @@ igb_write_packet_to_guest(IGBCore *core, struct NetRxPkt *pkt,
          pci_dma_read(d, base, &desc, rx_desc_len);
          trace_e1000e_rx_descr(rxi->idx, base, rx_desc_len);
 -        igb_read_rx_descr(core, &desc, &pdma_st.ba);
 +        igb_read_rx_descr(core, &desc, &pdma_st, rxi);
          igb_write_to_rx_buffers(core, pkt, d, &pdma_st);
          pdma_st.desc_offset += pdma_st.desc_size;
@@ -XXX,XX +XXX,XX @@ igb_write_packet_to_guest(IGBCore *core, struct NetRxPkt *pkt,
              is_last = true;
          }
 -        igb_write_rx_descr(core, &desc, is_last ? core->rx_pkt : NULL,
 -                           rss_info, etqf, ts, pdma_st.written);
 +        igb_write_rx_descr(core, &desc,
 +                           is_last ? pkt : NULL,
 +                           rss_info,
 +                           etqf, ts,
 +                           &pdma_st,
 +                           rxi);
          igb_pci_dma_write_rx_desc(core, d, base, &desc, rx_desc_len);
          igb_ring_advance(core, rxi, rx_desc_len / E1000_MIN_RX_DESC_LEN);
      } while (pdma_st.desc_offset < pdma_st.total_size);
 diff --git a/hw/net/igb_regs.h b/hw/net/igb_regs.h
 index XXXXXXX..XXXXXXX 100644
 --- a/hw/net/igb_regs.h
 +++ b/hw/net/igb_regs.h
@@ -XXX,XX +XXX,XX @@ union e1000_adv_rx_desc {
  #define E1000_SRRCTL_BSIZEHDRSIZE_MASK         0x00000F00
  #define E1000_SRRCTL_BSIZEHDRSIZE_SHIFT        2  /* Shift _left_ */
  #define E1000_SRRCTL_DESCTYPE_ADV_ONEBUF       0x02000000
 +#define E1000_SRRCTL_DESCTYPE_HDR_SPLIT        0x04000000
  #define E1000_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS 0x0A000000
  #define E1000_SRRCTL_DESCTYPE_MASK             0x0E000000
  #define E1000_SRRCTL_DROP_EN                   0x80000000
@@ -XXX,XX +XXX,XX @@ union e1000_adv_rx_desc {
  #define E1000_ADVRXD_PKT_UDP  BIT(5)
  #define E1000_ADVRXD_PKT_SCTP BIT(6)
 +#define IGB_MAX_PS_BUFFERS 2
 +
 +#define E1000_ADVRXD_HDR_LEN_OFFSET    (21 - 16)
 +#define E1000_ADVRXD_ADV_HDR_LEN_MASK  ((BIT(10) - 1) << \
 +                                        E1000_ADVRXD_HDR_LEN_OFFSET)
 +#define E1000_ADVRXD_HDR_SPH           BIT(15)
 +#define E1000_ADVRXD_ST_ERR_HBO_OFFSET BIT(3 + 20)
 +
  static inline uint8_t igb_ivar_entry_rx(uint8_t i)
  {
      return i < 8 ? i * 4 : (i - 8) * 4 + 2;
 diff --git a/hw/net/trace-events b/hw/net/trace-events
 index XXXXXXX..XXXXXXX 100644
 --- a/hw/net/trace-events
 +++ b/hw/net/trace-events
@@ -XXX,XX +XXX,XX @@ igb_core_mdic_write_unhandled(uint32_t addr) "MDIC WRITE: PHY[%u] UNHANDLED"
  igb_link_set_ext_params(bool asd_check, bool speed_select_bypass, bool pfrstd) "Set extended link params: ASD check: %d, Speed select bypass: %d, PF reset done: %d"
  igb_rx_desc_buff_size(uint32_t b) "buffer size: %u"
 -igb_rx_desc_buff_write(uint64_t addr, uint16_t offset, const void* source, uint32_t len) "addr: 0x%"PRIx64", offset: %u, from: %p, length: %u"
 +igb_rx_desc_buff_write(uint8_t idx, uint64_t addr, uint16_t offset, const void* source, uint32_t len) "buffer %u, addr: 0x%"PRIx64", offset: %u, from: %p, length: %u"
  igb_rx_metadata_rss(uint32_t rss, uint16_t rss_pkt_type) "RSS data: rss: 0x%X, rss_pkt_type: 0x%X"
 --
 .7.4

-[PULL 04/17] virtio-net: Add support for USO features
+[PULL V3 05/15] vhost: Add vhost_svq_valid_features to shadow vq
-From: Yuri Benditovich <yuri.benditovich@daynix.com>
+From: Eugenio Pérez <eperezma@redhat.com>
-USO features of virtio-net device depend on kernel ability
+This allows SVQ to negotiate features with the guest and the device. For
-to support them, for backward compatibility by default the
+the device, SVQ is a driver. While this function bypasses all
-features are disabled on 8.0 and earlier.
+non-transport features, it needs to disable the features that SVQ does
 not support when forwarding buffers. This includes packed vq layout,
 indirect descriptors or event idx.
-Signed-off-by: Yuri Benditovich <yuri.benditovich@daynix.com>
+Future changes can add support to offer more features to the guest,
-Signed-off-by: Andrew Melnychecnko <andrew@daynix.com>
+since the use of VirtQueue gives this for free. This is left out at the
 moment for simplicity.
 Signed-off-by: Eugenio Pérez <eperezma@redhat.com>
 Acked-by: Michael S. Tsirkin <mst@redhat.com>
 Signed-off-by: Jason Wang <jasowang@redhat.com>
 ---
- hw/core/machine.c   |  4 ++++
+ hw/virtio/vhost-shadow-virtqueue.c | 44 ++++++++++++++++++++++++++++++++++++++
- hw/net/virtio-net.c | 31 +++++++++++++++++++++++++++++--
+ hw/virtio/vhost-shadow-virtqueue.h |  2 ++
-files changed, 33 insertions(+), 2 deletions(-)
+ hw/virtio/vhost-vdpa.c             | 15 +++++++++++++
 files changed, 61 insertions(+)
-diff --git a/hw/core/machine.c b/hw/core/machine.c
+diff --git a/hw/virtio/vhost-shadow-virtqueue.c b/hw/virtio/vhost-shadow-virtqueue.c
 index XXXXXXX..XXXXXXX 100644
---- a/hw/core/machine.c
+--- a/hw/virtio/vhost-shadow-virtqueue.c
-+++ b/hw/core/machine.c
++++ b/hw/virtio/vhost-shadow-virtqueue.c
 @@ -XXX,XX +XXX,XX @@
- #include "exec/confidential-guest-support.h"
+ #include "hw/virtio/vhost-shadow-virtqueue.h"
- #include "hw/virtio/virtio.h"
- #include "hw/virtio/virtio-pci.h"
+ #include "qemu/error-report.h"
-+#include "hw/virtio/virtio-net.h"
++#include "qapi/error.h"
+ #include "qemu/main-loop.h"
- GlobalProperty hw_compat_8_1[] = {};
+ #include "linux-headers/linux/vhost.h"
- const size_t hw_compat_8_1_len = G_N_ELEMENTS(hw_compat_8_1);
-@@ -XXX,XX +XXX,XX @@ const size_t hw_compat_8_1_len = G_N_ELEMENTS(hw_compat_8_1);
+ /**
- GlobalProperty hw_compat_8_0[] = {
++ * Validate the transport device features that both guests can use with the SVQ
-     { "migration", "multifd-flush-after-each-section", "on"},
++ * and SVQs can use with the device.
-     { TYPE_PCI_DEVICE, "x-pcie-ari-nextfn-1", "on" },
++ *
-+    { TYPE_VIRTIO_NET, "host_uso", "off"},
++ * @dev_features: The features
-+    { TYPE_VIRTIO_NET, "guest_uso4", "off"},
++ * @errp: Error pointer
-+    { TYPE_VIRTIO_NET, "guest_uso6", "off"},
++ */
- };
++bool vhost_svq_valid_features(uint64_t features, Error **errp)
  const size_t hw_compat_8_0_len = G_N_ELEMENTS(hw_compat_8_0);
 diff --git a/hw/net/virtio-net.c b/hw/net/virtio-net.c
 index XXXXXXX..XXXXXXX 100644
 --- a/hw/net/virtio-net.c
 +++ b/hw/net/virtio-net.c
@@ -XXX,XX +XXX,XX @@ static int peer_has_ufo(VirtIONet *n)
      return n->has_ufo;
  }
 +static int peer_has_uso(VirtIONet *n)
 +{
-+    if (!peer_has_vnet_hdr(n)) {
++    bool ok = true;
-+        return 0;
++    uint64_t svq_features = features;
 +
 +    for (uint64_t b = VIRTIO_TRANSPORT_F_START; b <= VIRTIO_TRANSPORT_F_END;
 +         ++b) {
 +        switch (b) {
 +        case VIRTIO_F_ANY_LAYOUT:
 +            continue;
 +
 +        case VIRTIO_F_ACCESS_PLATFORM:
 +            /* SVQ trust in the host's IOMMU to translate addresses */
 +        case VIRTIO_F_VERSION_1:
 +            /* SVQ trust that the guest vring is little endian */
 +            if (!(svq_features & BIT_ULL(b))) {
 +                svq_features |= BIT_ULL(b);
 +                ok = false;
 +            }
 +            continue;
 +
 +        default:
 +            if (svq_features & BIT_ULL(b)) {
 +                svq_features &= ~BIT_ULL(b);
 +                ok = false;
 +            }
 +        }
 +    }
 +
-+    return qemu_has_uso(qemu_get_queue(n->nic)->peer);
++    if (!ok) {
 +        error_setg(errp, "SVQ Invalid device feature flags, offer: 0x%"PRIx64
 +                         ", ok: 0x%"PRIx64, features, svq_features);
 +    }
 +    return ok;
 +}
 +
- static void virtio_net_set_mrg_rx_bufs(VirtIONet *n, int mergeable_rx_bufs,
++/**
-                                        int version_1, int hash_report)
+  * Forward guest notifications.
   *
   * @n: guest kick event notifier, the one that guest set to notify svq.
 diff --git a/hw/virtio/vhost-shadow-virtqueue.h b/hw/virtio/vhost-shadow-virtqueue.h
 index XXXXXXX..XXXXXXX 100644
 --- a/hw/virtio/vhost-shadow-virtqueue.h
 +++ b/hw/virtio/vhost-shadow-virtqueue.h
@@ -XXX,XX +XXX,XX @@ typedef struct VhostShadowVirtqueue {
      EventNotifier svq_call;
  } VhostShadowVirtqueue;
 +bool vhost_svq_valid_features(uint64_t features, Error **errp);
 +
  void vhost_svq_set_svq_kick_fd(VhostShadowVirtqueue *svq, int svq_kick_fd);
  void vhost_svq_set_svq_call_fd(VhostShadowVirtqueue *svq, int call_fd);
 diff --git a/hw/virtio/vhost-vdpa.c b/hw/virtio/vhost-vdpa.c
 index XXXXXXX..XXXXXXX 100644
 --- a/hw/virtio/vhost-vdpa.c
 +++ b/hw/virtio/vhost-vdpa.c
@@ -XXX,XX +XXX,XX @@ static int vhost_vdpa_init_svq(struct vhost_dev *hdev, struct vhost_vdpa *v,
                                 Error **errp)
  {
-@@ -XXX,XX +XXX,XX @@ static uint64_t virtio_net_get_features(VirtIODevice *vdev, uint64_t features,
+     g_autoptr(GPtrArray) shadow_vqs = NULL;
-         virtio_clear_feature(&features, VIRTIO_NET_F_GUEST_TSO6);
++    uint64_t dev_features, svq_features;
-         virtio_clear_feature(&features, VIRTIO_NET_F_GUEST_ECN);
++    int r;
++    bool ok;
-+        virtio_clear_feature(&features, VIRTIO_NET_F_HOST_USO);
-+        virtio_clear_feature(&features, VIRTIO_NET_F_GUEST_USO4);
+     if (!v->shadow_vqs_enabled) {
-+        virtio_clear_feature(&features, VIRTIO_NET_F_GUEST_USO6);
+         return 0;
 +
          virtio_clear_feature(&features, VIRTIO_NET_F_HASH_REPORT);
      }
-@@ -XXX,XX +XXX,XX @@ static uint64_t virtio_net_get_features(VirtIODevice *vdev, uint64_t features,
++    r = hdev->vhost_ops->vhost_get_features(hdev, &dev_features);
-         virtio_clear_feature(&features, VIRTIO_NET_F_HOST_UFO);
++    if (r != 0) {
-     }
++        error_setg_errno(errp, -r, "Can't get vdpa device features");
++        return r;
 +    if (!peer_has_uso(n)) {
 +        virtio_clear_feature(&features, VIRTIO_NET_F_HOST_USO);
 +        virtio_clear_feature(&features, VIRTIO_NET_F_GUEST_USO4);
 +        virtio_clear_feature(&features, VIRTIO_NET_F_GUEST_USO6);
 +    }
 +
-     if (!get_vhost_net(nc->peer)) {
++    svq_features = dev_features;
-         return features;
++    ok = vhost_svq_valid_features(svq_features, errp);
-     }
++    if (unlikely(!ok)) {
-@@ -XXX,XX +XXX,XX @@ static void virtio_net_apply_guest_offloads(VirtIONet *n)
++        return -1;
-             !!(n->curr_guest_offloads & (1ULL << VIRTIO_NET_F_GUEST_USO6)));
++    }
- }
++
+     shadow_vqs = g_ptr_array_new_full(hdev->nvqs, vhost_svq_free);
--static uint64_t virtio_net_guest_offloads_by_features(uint32_t features)
+     for (unsigned n = 0; n < hdev->nvqs; ++n) {
-+static uint64_t virtio_net_guest_offloads_by_features(uint64_t features)
+         g_autoptr(VhostShadowVirtqueue) svq = vhost_svq_new();
  {
      static const uint64_t guest_offloads_mask =
          (1ULL << VIRTIO_NET_F_GUEST_CSUM) |
          (1ULL << VIRTIO_NET_F_GUEST_TSO4) |
          (1ULL << VIRTIO_NET_F_GUEST_TSO6) |
          (1ULL << VIRTIO_NET_F_GUEST_ECN)  |
 -        (1ULL << VIRTIO_NET_F_GUEST_UFO);
 +        (1ULL << VIRTIO_NET_F_GUEST_UFO)  |
 +        (1ULL << VIRTIO_NET_F_GUEST_USO4) |
 +        (1ULL << VIRTIO_NET_F_GUEST_USO6);
      return guest_offloads_mask & features;
  }
@@ -XXX,XX +XXX,XX @@ static Property virtio_net_properties[] = {
      DEFINE_PROP_INT32("speed", VirtIONet, net_conf.speed, SPEED_UNKNOWN),
      DEFINE_PROP_STRING("duplex", VirtIONet, net_conf.duplex_str),
      DEFINE_PROP_BOOL("failover", VirtIONet, failover, false),
 +    DEFINE_PROP_BIT64("guest_uso4", VirtIONet, host_features,
 +                      VIRTIO_NET_F_GUEST_USO4, true),
 +    DEFINE_PROP_BIT64("guest_uso6", VirtIONet, host_features,
 +                      VIRTIO_NET_F_GUEST_USO6, true),
 +    DEFINE_PROP_BIT64("host_uso", VirtIONet, host_features,
 +                      VIRTIO_NET_F_HOST_USO, true),
      DEFINE_PROP_END_OF_LIST(),
  };
 --
 .7.4

-[PULL 17/17] ebpf: Updated eBPF program and skeleton.
+[PULL V3 06/15] virtio: Add vhost_svq_get_vring_addr
-From: Andrew Melnychenko <andrew@daynix.com>
+From: Eugenio Pérez <eperezma@redhat.com>
-Updated section name, so libbpf should init/gues proper
+It reports the shadow virtqueue address from qemu virtual address space.
 program type without specifications during open/load.
 Also, added map_flags with explicitly declared BPF_F_MMAPABLE.
 Added check for BPF_F_MMAPABLE flag to meson script and
 requirements to libbpf version.
-Signed-off-by: Andrew Melnychenko <andrew@daynix.com>
+Since this will be different from the guest's vaddr, but the device can
 access it, SVQ takes special care about its alignment & lack of garbage
 data. It assumes that IOMMU will work in host_page_size ranges for that.
 Signed-off-by: Eugenio Pérez <eperezma@redhat.com>
 Acked-by: Michael S. Tsirkin <mst@redhat.com>
 Signed-off-by: Jason Wang <jasowang@redhat.com>
 ---
- ebpf/rss.bpf.skeleton.h | 1460 ++++++++++++++++++++++++-----------------------
+ hw/virtio/vhost-shadow-virtqueue.c | 29 +++++++++++++++++++++++++++++
- meson.build             |   10 +-
+ hw/virtio/vhost-shadow-virtqueue.h |  9 +++++++++
- tools/ebpf/rss.bpf.c    |    5 +-
+files changed, 38 insertions(+)
 files changed, 748 insertions(+), 727 deletions(-)
-diff --git a/ebpf/rss.bpf.skeleton.h b/ebpf/rss.bpf.skeleton.h
+diff --git a/hw/virtio/vhost-shadow-virtqueue.c b/hw/virtio/vhost-shadow-virtqueue.c
 index XXXXXXX..XXXXXXX 100644
---- a/ebpf/rss.bpf.skeleton.h
+--- a/hw/virtio/vhost-shadow-virtqueue.c
-+++ b/ebpf/rss.bpf.skeleton.h
++++ b/hw/virtio/vhost-shadow-virtqueue.c
-@@ -XXX,XX +XXX,XX @@ err:
+@@ -XXX,XX +XXX,XX @@ void vhost_svq_set_svq_call_fd(VhostShadowVirtqueue *svq, int call_fd)
  static inline const void *rss_bpf__elf_bytes(size_t *sz)
  {
 -    *sz = 20440;
 +    *sz = 20824;
      return (const void *)"\
  \x7f\x45\x4c\x46\x02\x01\x01\0\0\0\0\0\0\0\0\0\x01\0\xf7\0\x01\0\0\0\0\0\0\0\0\
 -\0\0\0\0\0\0\0\0\0\0\0\x98\x4c\0\0\0\0\0\0\0\0\0\0\x40\0\0\0\0\0\x40\0\x0d\0\
 -\x01\0\xbf\x19\0\0\0\0\0\0\xb7\x01\0\0\0\0\0\0\x63\x1a\x54\xff\0\0\0\0\xbf\xa7\
 -\0\0\0\0\0\0\x07\x07\0\0\x54\xff\xff\xff\x18\x01\0\0\0\0\0\0\0\0\0\0\0\0\0\0\
 +\0\0\0\0\0\0\0\0\0\0\0\x18\x4e\0\0\0\0\0\0\0\0\0\0\x40\0\0\0\0\0\x40\0\x0d\0\
 +\x01\0\xbf\x19\0\0\0\0\0\0\xb7\x01\0\0\0\0\0\0\x63\x1a\x4c\xff\0\0\0\0\xbf\xa7\
 +\0\0\0\0\0\0\x07\x07\0\0\x4c\xff\xff\xff\x18\x01\0\0\0\0\0\0\0\0\0\0\0\0\0\0\
  \xbf\x72\0\0\0\0\0\0\x85\0\0\0\x01\0\0\0\xbf\x06\0\0\0\0\0\0\x18\x01\0\0\0\0\0\
  \0\0\0\0\0\0\0\0\0\xbf\x72\0\0\0\0\0\0\x85\0\0\0\x01\0\0\0\xbf\x08\0\0\0\0\0\0\
 -\x18\0\0\0\xff\xff\xff\xff\0\0\0\0\0\0\0\0\x15\x06\x67\x02\0\0\0\0\xbf\x87\0\0\
 -\0\0\0\0\x15\x07\x65\x02\0\0\0\0\x71\x61\0\0\0\0\0\0\x55\x01\x01\0\0\0\0\0\x05\
 -\0\x5e\x02\0\0\0\0\xb7\x01\0\0\0\0\0\0\x63\x1a\xc8\xff\0\0\0\0\x7b\x1a\xc0\xff\
 -\0\0\0\0\x7b\x1a\xb8\xff\0\0\0\0\x7b\x1a\xb0\xff\0\0\0\0\x7b\x1a\xa8\xff\0\0\0\
 -\0\x63\x1a\xa0\xff\0\0\0\0\x7b\x1a\x98\xff\0\0\0\0\x7b\x1a\x90\xff\0\0\0\0\x7b\
 -\x1a\x88\xff\0\0\0\0\x7b\x1a\x80\xff\0\0\0\0\x7b\x1a\x78\xff\0\0\0\0\x7b\x1a\
 -\x70\xff\0\0\0\0\x7b\x1a\x68\xff\0\0\0\0\x7b\x1a\x60\xff\0\0\0\0\x7b\x1a\x58\
 -\xff\0\0\0\0\x15\x09\x4d\x02\0\0\0\0\x6b\x1a\xd0\xff\0\0\0\0\xbf\xa3\0\0\0\0\0\
 -\0\x07\x03\0\0\xd0\xff\xff\xff\xbf\x91\0\0\0\0\0\0\xb7\x02\0\0\x0c\0\0\0\xb7\
 +\x18\0\0\0\xff\xff\xff\xff\0\0\0\0\0\0\0\0\x15\x06\x64\x02\0\0\0\0\xbf\x87\0\0\
 +\0\0\0\0\x15\x07\x62\x02\0\0\0\0\x71\x61\0\0\0\0\0\0\x55\x01\x01\0\0\0\0\0\x05\
 +\0\x5b\x02\0\0\0\0\xb7\x01\0\0\0\0\0\0\x63\x1a\xc0\xff\0\0\0\0\x7b\x1a\xb8\xff\
 +\0\0\0\0\x7b\x1a\xb0\xff\0\0\0\0\x7b\x1a\xa8\xff\0\0\0\0\x7b\x1a\xa0\xff\0\0\0\
 +\0\x63\x1a\x98\xff\0\0\0\0\x7b\x1a\x90\xff\0\0\0\0\x7b\x1a\x88\xff\0\0\0\0\x7b\
 +\x1a\x80\xff\0\0\0\0\x7b\x1a\x78\xff\0\0\0\0\x7b\x1a\x70\xff\0\0\0\0\x7b\x1a\
 +\x68\xff\0\0\0\0\x7b\x1a\x60\xff\0\0\0\0\x7b\x1a\x58\xff\0\0\0\0\x7b\x1a\x50\
 +\xff\0\0\0\0\x15\x09\x4a\x02\0\0\0\0\x6b\x1a\xc8\xff\0\0\0\0\xbf\xa3\0\0\0\0\0\
 +\0\x07\x03\0\0\xc8\xff\xff\xff\xbf\x91\0\0\0\0\0\0\xb7\x02\0\0\x0c\0\0\0\xb7\
  \x04\0\0\x02\0\0\0\xb7\x05\0\0\0\0\0\0\x85\0\0\0\x44\0\0\0\x67\0\0\0\x20\0\0\0\
 -\x77\0\0\0\x20\0\0\0\x55\0\x42\x02\0\0\0\0\xb7\x02\0\0\x10\0\0\0\x69\xa1\xd0\
 +\x77\0\0\0\x20\0\0\0\x55\0\x3f\x02\0\0\0\0\xb7\x02\0\0\x10\0\0\0\x69\xa1\xc8\
  \xff\0\0\0\0\xbf\x13\0\0\0\0\0\0\xdc\x03\0\0\x10\0\0\0\x15\x03\x02\0\0\x81\0\0\
  \x55\x03\x0b\0\xa8\x88\0\0\xb7\x02\0\0\x14\0\0\0\xbf\xa3\0\0\0\0\0\0\x07\x03\0\
 -\0\xd0\xff\xff\xff\xbf\x91\0\0\0\0\0\0\xb7\x04\0\0\x02\0\0\0\xb7\x05\0\0\0\0\0\
 -\0\x85\0\0\0\x44\0\0\0\x67\0\0\0\x20\0\0\0\x77\0\0\0\x20\0\0\0\x55\0\x32\x02\0\
 -\0\0\0\x69\xa1\xd0\xff\0\0\0\0\x15\x01\x30\x02\0\0\0\0\x7b\x7a\x38\xff\0\0\0\0\
 -\x7b\x9a\x40\xff\0\0\0\0\x15\x01\x55\0\x86\xdd\0\0\x55\x01\x39\0\x08\0\0\0\xb7\
 -\x07\0\0\x01\0\0\0\x73\x7a\x58\xff\0\0\0\0\xb7\x01\0\0\0\0\0\0\x63\x1a\xe0\xff\
 -\0\0\0\0\x7b\x1a\xd8\xff\0\0\0\0\x7b\x1a\xd0\xff\0\0\0\0\xbf\xa3\0\0\0\0\0\0\
 -\x07\x03\0\0\xd0\xff\xff\xff\x79\xa1\x40\xff\0\0\0\0\xb7\x02\0\0\0\0\0\0\xb7\
 +\0\xc8\xff\xff\xff\xbf\x91\0\0\0\0\0\0\xb7\x04\0\0\x02\0\0\0\xb7\x05\0\0\0\0\0\
 +\0\x85\0\0\0\x44\0\0\0\x67\0\0\0\x20\0\0\0\x77\0\0\0\x20\0\0\0\x55\0\x2f\x02\0\
 +\0\0\0\x69\xa1\xc8\xff\0\0\0\0\x15\x01\x2d\x02\0\0\0\0\x7b\x7a\x30\xff\0\0\0\0\
 +\x7b\x9a\x38\xff\0\0\0\0\x15\x01\x55\0\x86\xdd\0\0\x55\x01\x39\0\x08\0\0\0\xb7\
 +\x07\0\0\x01\0\0\0\x73\x7a\x50\xff\0\0\0\0\xb7\x01\0\0\0\0\0\0\x63\x1a\xd8\xff\
 +\0\0\0\0\x7b\x1a\xd0\xff\0\0\0\0\x7b\x1a\xc8\xff\0\0\0\0\xbf\xa3\0\0\0\0\0\0\
 +\x07\x03\0\0\xc8\xff\xff\xff\x79\xa1\x38\xff\0\0\0\0\xb7\x02\0\0\0\0\0\0\xb7\
  \x04\0\0\x14\0\0\0\xb7\x05\0\0\x01\0\0\0\x85\0\0\0\x44\0\0\0\x67\0\0\0\x20\0\0\
 -\0\x77\0\0\0\x20\0\0\0\x55\0\x1c\x02\0\0\0\0\x69\xa1\xd6\xff\0\0\0\0\x55\x01\
 -\x01\0\0\0\0\0\xb7\x07\0\0\0\0\0\0\x61\xa1\xdc\xff\0\0\0\0\x63\x1a\x64\xff\0\0\
 -\0\0\x61\xa1\xe0\xff\0\0\0\0\x63\x1a\x68\xff\0\0\0\0\x71\xa9\xd9\xff\0\0\0\0\
 -\x73\x7a\x5e\xff\0\0\0\0\x71\xa1\xd0\xff\0\0\0\0\x67\x01\0\0\x02\0\0\0\x57\x01\
 -\0\0\x3c\0\0\0\x7b\x1a\x48\xff\0\0\0\0\xbf\x91\0\0\0\0\0\0\x57\x01\0\0\xff\0\0\
 +\0\x77\0\0\0\x20\0\0\0\x55\0\x19\x02\0\0\0\0\x69\xa1\xce\xff\0\0\0\0\x55\x01\
 +\x01\0\0\0\0\0\xb7\x07\0\0\0\0\0\0\x61\xa1\xd4\xff\0\0\0\0\x63\x1a\x5c\xff\0\0\
 +\0\0\x61\xa1\xd8\xff\0\0\0\0\x63\x1a\x60\xff\0\0\0\0\x71\xa9\xd1\xff\0\0\0\0\
 +\x73\x7a\x56\xff\0\0\0\0\x71\xa1\xc8\xff\0\0\0\0\x67\x01\0\0\x02\0\0\0\x57\x01\
 +\0\0\x3c\0\0\0\x7b\x1a\x40\xff\0\0\0\0\xbf\x91\0\0\0\0\0\0\x57\x01\0\0\xff\0\0\
  \0\x15\x01\x19\0\0\0\0\0\x57\x07\0\0\xff\0\0\0\x55\x07\x17\0\0\0\0\0\x57\x09\0\
  \0\xff\0\0\0\x15\x09\x5a\x01\x11\0\0\0\x55\x09\x14\0\x06\0\0\0\xb7\x01\0\0\x01\
 -\0\0\0\x73\x1a\x5b\xff\0\0\0\0\xb7\x01\0\0\0\0\0\0\x63\x1a\xe0\xff\0\0\0\0\x7b\
 -\x1a\xd8\xff\0\0\0\0\x7b\x1a\xd0\xff\0\0\0\0\xbf\xa3\0\0\0\0\0\0\x07\x03\0\0\
 -\xd0\xff\xff\xff\x79\xa1\x40\xff\0\0\0\0\x79\xa2\x48\xff\0\0\0\0\xb7\x04\0\0\
 +\0\0\0\x73\x1a\x53\xff\0\0\0\0\xb7\x01\0\0\0\0\0\0\x63\x1a\xd8\xff\0\0\0\0\x7b\
 +\x1a\xd0\xff\0\0\0\0\x7b\x1a\xc8\xff\0\0\0\0\xbf\xa3\0\0\0\0\0\0\x07\x03\0\0\
 +\xc8\xff\xff\xff\x79\xa1\x38\xff\0\0\0\0\x79\xa2\x40\xff\0\0\0\0\xb7\x04\0\0\
  \x14\0\0\0\xb7\x05\0\0\x01\0\0\0\x85\0\0\0\x44\0\0\0\x67\0\0\0\x20\0\0\0\x77\0\
 -\0\0\x20\0\0\0\x55\0\xf7\x01\0\0\0\0\x69\xa1\xd0\xff\0\0\0\0\x6b\x1a\x60\xff\0\
 -\0\0\0\x69\xa1\xd2\xff\0\0\0\0\x6b\x1a\x62\xff\0\0\0\0\x71\xa1\x58\xff\0\0\0\0\
 +\0\0\x20\0\0\0\x55\0\xf4\x01\0\0\0\0\x69\xa1\xc8\xff\0\0\0\0\x6b\x1a\x58\xff\0\
 +\0\0\0\x69\xa1\xca\xff\0\0\0\0\x6b\x1a\x5a\xff\0\0\0\0\x71\xa1\x50\xff\0\0\0\0\
  \x15\x01\xdb\0\0\0\0\0\x71\x62\x03\0\0\0\0\0\x67\x02\0\0\x08\0\0\0\x71\x61\x02\
  \0\0\0\0\0\x4f\x12\0\0\0\0\0\0\x71\x63\x04\0\0\0\0\0\x71\x61\x05\0\0\0\0\0\x67\
  \x01\0\0\x08\0\0\0\x4f\x31\0\0\0\0\0\0\x67\x01\0\0\x10\0\0\0\x4f\x21\0\0\0\0\0\
 -\0\x71\xa2\x5b\xff\0\0\0\0\x79\xa0\x38\xff\0\0\0\0\x15\x02\x0c\x01\0\0\0\0\xbf\
 -\x12\0\0\0\0\0\0\x57\x02\0\0\x02\0\0\0\x15\x02\x09\x01\0\0\0\0\x61\xa1\x64\xff\
 -\0\0\0\0\x63\x1a\xa8\xff\0\0\0\0\x61\xa1\x68\xff\0\0\0\0\x63\x1a\xac\xff\0\0\0\
 -\0\x69\xa1\x60\xff\0\0\0\0\x6b\x1a\xb0\xff\0\0\0\0\x69\xa1\x62\xff\0\0\0\0\x6b\
 -\x1a\xb2\xff\0\0\0\0\x05\0\x6b\x01\0\0\0\0\xb7\x01\0\0\x01\0\0\0\x73\x1a\x59\
 -\xff\0\0\0\0\xb7\x01\0\0\0\0\0\0\x7b\x1a\xf0\xff\0\0\0\0\x7b\x1a\xe8\xff\0\0\0\
 -\0\x7b\x1a\xe0\xff\0\0\0\0\x7b\x1a\xd8\xff\0\0\0\0\x7b\x1a\xd0\xff\0\0\0\0\xbf\
 -\xa3\0\0\0\0\0\0\x07\x03\0\0\xd0\xff\xff\xff\xb7\x01\0\0\x28\0\0\0\x7b\x1a\x48\
 +\0\x71\xa2\x53\xff\0\0\0\0\x79\xa0\x30\xff\0\0\0\0\x15\x02\x0c\x01\0\0\0\0\xbf\
 +\x12\0\0\0\0\0\0\x57\x02\0\0\x02\0\0\0\x15\x02\x09\x01\0\0\0\0\x61\xa1\x5c\xff\
 +\0\0\0\0\x63\x1a\xa0\xff\0\0\0\0\x61\xa1\x60\xff\0\0\0\0\x63\x1a\xa4\xff\0\0\0\
 +\0\x69\xa1\x58\xff\0\0\0\0\x6b\x1a\xa8\xff\0\0\0\0\x69\xa1\x5a\xff\0\0\0\0\x6b\
 +\x1a\xaa\xff\0\0\0\0\x05\0\x6b\x01\0\0\0\0\xb7\x01\0\0\x01\0\0\0\x73\x1a\x51\
 +\xff\0\0\0\0\xb7\x01\0\0\0\0\0\0\x7b\x1a\xe8\xff\0\0\0\0\x7b\x1a\xe0\xff\0\0\0\
 +\0\x7b\x1a\xd8\xff\0\0\0\0\x7b\x1a\xd0\xff\0\0\0\0\x7b\x1a\xc8\xff\0\0\0\0\xbf\
 +\xa3\0\0\0\0\0\0\x07\x03\0\0\xc8\xff\xff\xff\xb7\x01\0\0\x28\0\0\0\x7b\x1a\x40\
  \xff\0\0\0\0\xbf\x91\0\0\0\0\0\0\xb7\x02\0\0\0\0\0\0\xb7\x04\0\0\x28\0\0\0\xb7\
  \x05\0\0\x01\0\0\0\x85\0\0\0\x44\0\0\0\x67\0\0\0\x20\0\0\0\x77\0\0\0\x20\0\0\0\
 -\x55\0\xfe\0\0\0\0\0\x79\xa1\xe0\xff\0\0\0\0\x63\x1a\x6c\xff\0\0\0\0\x77\x01\0\
 -\0\x20\0\0\0\x63\x1a\x70\xff\0\0\0\0\x79\xa1\xd8\xff\0\0\0\0\x63\x1a\x64\xff\0\
 -\0\0\0\x77\x01\0\0\x20\0\0\0\x63\x1a\x68\xff\0\0\0\0\x79\xa1\xe8\xff\0\0\0\0\
 -\x63\x1a\x74\xff\0\0\0\0\x77\x01\0\0\x20\0\0\0\x63\x1a\x78\xff\0\0\0\0\x79\xa1\
 -\xf0\xff\0\0\0\0\x63\x1a\x7c\xff\0\0\0\0\x77\x01\0\0\x20\0\0\0\x63\x1a\x80\xff\
 -\0\0\0\0\x71\xa9\xd6\xff\0\0\0\0\x25\x09\x13\x01\x3c\0\0\0\xb7\x01\0\0\x01\0\0\
 +\x55\0\xfe\0\0\0\0\0\x79\xa1\xd8\xff\0\0\0\0\x63\x1a\x64\xff\0\0\0\0\x77\x01\0\
 +\0\x20\0\0\0\x63\x1a\x68\xff\0\0\0\0\x79\xa1\xd0\xff\0\0\0\0\x63\x1a\x5c\xff\0\
 +\0\0\0\x77\x01\0\0\x20\0\0\0\x63\x1a\x60\xff\0\0\0\0\x79\xa1\xe0\xff\0\0\0\0\
 +\x63\x1a\x6c\xff\0\0\0\0\x77\x01\0\0\x20\0\0\0\x63\x1a\x70\xff\0\0\0\0\x79\xa1\
 +\xe8\xff\0\0\0\0\x63\x1a\x74\xff\0\0\0\0\x77\x01\0\0\x20\0\0\0\x63\x1a\x78\xff\
 +\0\0\0\0\x71\xa9\xce\xff\0\0\0\0\x25\x09\x13\x01\x3c\0\0\0\xb7\x01\0\0\x01\0\0\
  \0\x6f\x91\0\0\0\0\0\0\x18\x02\0\0\x01\0\0\0\0\0\0\0\0\x18\0\x1c\x5f\x21\0\0\0\
  \0\0\0\x55\x01\x01\0\0\0\0\0\x05\0\x0c\x01\0\0\0\0\xb7\x01\0\0\0\0\0\0\x6b\x1a\
 -\xfe\xff\0\0\0\0\xb7\x01\0\0\x28\0\0\0\x7b\x1a\x48\xff\0\0\0\0\xbf\xa1\0\0\0\0\
 -\0\0\x07\x01\0\0\x94\xff\xff\xff\x7b\x1a\x20\xff\0\0\0\0\xbf\xa1\0\0\0\0\0\0\
 -\x07\x01\0\0\x84\xff\xff\xff\x7b\x1a\x18\xff\0\0\0\0\x18\x07\0\0\x01\0\0\0\0\0\
 -\0\0\0\x18\0\x1c\xb7\x02\0\0\0\0\0\0\x7b\x8a\x28\xff\0\0\0\0\x7b\x2a\x30\xff\0\
 -\0\0\0\xbf\xa3\0\0\0\0\0\0\x07\x03\0\0\xfe\xff\xff\xff\x79\xa1\x40\xff\0\0\0\0\
 -\x79\xa2\x48\xff\0\0\0\0\xb7\x04\0\0\x02\0\0\0\xb7\x05\0\0\x01\0\0\0\x85\0\0\0\
 -\x44\0\0\0\x67\0\0\0\x20\0\0\0\x77\0\0\0\x20\0\0\0\x15\0\x01\0\0\0\0\0\x05\0\
 -\x91\x01\0\0\0\0\xbf\x91\0\0\0\0\0\0\x15\x01\x26\0\x3c\0\0\0\x15\x01\x5f\0\x2c\
 -\0\0\0\x55\x01\x60\0\x2b\0\0\0\xb7\x01\0\0\0\0\0\0\x63\x1a\xf8\xff\0\0\0\0\xbf\
 -\xa3\0\0\0\0\0\0\x07\x03\0\0\xf8\xff\xff\xff\x79\xa7\x40\xff\0\0\0\0\xbf\x71\0\
 -\0\0\0\0\0\x79\xa2\x48\xff\0\0\0\0\xb7\x04\0\0\x04\0\0\0\xb7\x05\0\0\x01\0\0\0\
 -\x85\0\0\0\x44\0\0\0\xbf\x01\0\0\0\0\0\0\x67\x01\0\0\x20\0\0\0\x77\x01\0\0\x20\
 -\0\0\0\x55\x01\x06\x01\0\0\0\0\x71\xa1\xfa\xff\0\0\0\0\x55\x01\x11\0\x02\0\0\0\
 -\x71\xa1\xf9\xff\0\0\0\0\x55\x01\x0f\0\x02\0\0\0\x71\xa1\xfb\xff\0\0\0\0\x55\
 -\x01\x0d\0\x01\0\0\0\x79\xa2\x48\xff\0\0\0\0\x07\x02\0\0\x08\0\0\0\xbf\x71\0\0\
 -\0\0\0\0\x79\xa3\x20\xff\0\0\0\0\xb7\x04\0\0\x10\0\0\0\xb7\x05\0\0\x01\0\0\0\
 -\x85\0\0\0\x44\0\0\0\xbf\x01\0\0\0\0\0\0\x67\x01\0\0\x20\0\0\0\x77\x01\0\0\x20\
 -\0\0\0\x55\x01\xf5\0\0\0\0\0\xb7\x01\0\0\x01\0\0\0\x73\x1a\x5d\xff\0\0\0\0\x18\
 -\x07\0\0\x01\0\0\0\0\0\0\0\0\x18\0\x1c\x05\0\x3c\0\0\0\0\0\xb7\x08\0\0\x02\0\0\
 -\0\xb7\x07\0\0\0\0\0\0\x6b\x7a\xf8\xff\0\0\0\0\x05\0\x13\0\0\0\0\0\x0f\x81\0\0\
 -\0\0\0\0\xbf\x12\0\0\0\0\0\0\x07\x02\0\0\x01\0\0\0\x71\xa3\xff\xff\0\0\0\0\x67\
 -\x03\0\0\x03\0\0\0\x3d\x32\x09\0\0\0\0\0\xbf\x72\0\0\0\0\0\0\x07\x02\0\0\x01\0\
 -\0\0\x67\x07\0\0\x20\0\0\0\xbf\x73\0\0\0\0\0\0\x77\x03\0\0\x20\0\0\0\xbf\x27\0\
 -\0\0\0\0\0\xbf\x18\0\0\0\0\0\0\xb7\x01\0\0\x1d\0\0\0\x2d\x31\x04\0\0\0\0\0\x79\
 -\xa8\x28\xff\0\0\0\0\x18\x07\0\0\x01\0\0\0\0\0\0\0\0\x18\0\x1c\x05\0\x25\0\0\0\
 -\0\0\xbf\x89\0\0\0\0\0\0\x79\xa1\x48\xff\0\0\0\0\x0f\x19\0\0\0\0\0\0\xbf\xa3\0\
 -\0\0\0\0\0\x07\x03\0\0\xf8\xff\xff\xff\x79\xa1\x40\xff\0\0\0\0\xbf\x92\0\0\0\0\
 -\0\0\xb7\x04\0\0\x02\0\0\0\xb7\x05\0\0\x01\0\0\0\x85\0\0\0\x44\0\0\0\xbf\x01\0\
 -\0\0\0\0\0\x67\x01\0\0\x20\0\0\0\x77\x01\0\0\x20\0\0\0\x55\x01\x79\0\0\0\0\0\
 -\x71\xa2\xf8\xff\0\0\0\0\x55\x02\x0e\0\xc9\0\0\0\x07\x09\0\0\x02\0\0\0\x79\xa1\
 -\x40\xff\0\0\0\0\xbf\x92\0\0\0\0\0\0\x79\xa3\x18\xff\0\0\0\0\xb7\x04\0\0\x10\0\
 -\0\0\xb7\x05\0\0\x01\0\0\0\x85\0\0\0\x44\0\0\0\xbf\x01\0\0\0\0\0\0\x67\x01\0\0\
 -\x20\0\0\0\x77\x01\0\0\x20\0\0\0\x55\x01\x6c\0\0\0\0\0\xb7\x01\0\0\x01\0\0\0\
 -\x73\x1a\x5c\xff\0\0\0\0\x05\0\xde\xff\0\0\0\0\xb7\x01\0\0\x01\0\0\0\x15\x02\
 -\xcd\xff\0\0\0\0\x71\xa1\xf9\xff\0\0\0\0\x07\x01\0\0\x02\0\0\0\x05\0\xca\xff\0\
 -\0\0\0\xb7\x01\0\0\x01\0\0\0\x73\x1a\x5e\xff\0\0\0\0\x71\xa1\xff\xff\0\0\0\0\
 -\x67\x01\0\0\x03\0\0\0\x79\xa2\x48\xff\0\0\0\0\x0f\x12\0\0\0\0\0\0\x07\x02\0\0\
 -\x08\0\0\0\x7b\x2a\x48\xff\0\0\0\0\x71\xa9\xfe\xff\0\0\0\0\x79\xa2\x30\xff\0\0\
 -\0\0\x25\x09\x0c\0\x3c\0\0\0\xb7\x01\0\0\x01\0\0\0\x6f\x91\0\0\0\0\0\0\x5f\x71\
 -\0\0\0\0\0\0\x55\x01\x01\0\0\0\0\0\x05\0\x07\0\0\0\0\0\x07\x02\0\0\x01\0\0\0\
 -\xbf\x21\0\0\0\0\0\0\x67\x01\0\0\x20\0\0\0\x77\x01\0\0\x20\0\0\0\x55\x01\x7d\
 -\xff\x0b\0\0\0\x71\xa7\x5e\xff\0\0\0\0\x05\0\x09\xff\0\0\0\0\x15\x09\xf8\xff\
 -\x87\0\0\0\x05\0\xfc\xff\0\0\0\0\x71\xa1\x59\xff\0\0\0\0\x79\xa0\x38\xff\0\0\0\
 -\0\x15\x01\x13\x01\0\0\0\0\x71\x62\x03\0\0\0\0\0\x67\x02\0\0\x08\0\0\0\x71\x61\
 +\xf8\xff\0\0\0\0\xb7\x01\0\0\x28\0\0\0\x7b\x1a\x40\xff\0\0\0\0\xbf\xa1\0\0\0\0\
 +\0\0\x07\x01\0\0\x8c\xff\xff\xff\x7b\x1a\x18\xff\0\0\0\0\xbf\xa1\0\0\0\0\0\0\
 +\x07\x01\0\0\x7c\xff\xff\xff\x7b\x1a\x10\xff\0\0\0\0\x18\x07\0\0\x01\0\0\0\0\0\
 +\0\0\0\x18\0\x1c\xb7\x02\0\0\0\0\0\0\x7b\x8a\x20\xff\0\0\0\0\x7b\x2a\x28\xff\0\
 +\0\0\0\xbf\xa3\0\0\0\0\0\0\x07\x03\0\0\xf8\xff\xff\xff\x79\xa1\x38\xff\0\0\0\0\
 +\x79\xa2\x40\xff\0\0\0\0\xb7\x04\0\0\x02\0\0\0\xb7\x05\0\0\x01\0\0\0\x85\0\0\0\
 +\x44\0\0\0\xbf\x01\0\0\0\0\0\0\x67\x01\0\0\x20\0\0\0\x77\x01\0\0\x20\0\0\0\x55\
 +\x01\xc8\0\0\0\0\0\xbf\x91\0\0\0\0\0\0\x15\x01\x26\0\x3c\0\0\0\x15\x01\x5f\0\
 +\x2c\0\0\0\x55\x01\x60\0\x2b\0\0\0\xb7\x01\0\0\0\0\0\0\x63\x1a\xf0\xff\0\0\0\0\
 +\xbf\xa3\0\0\0\0\0\0\x07\x03\0\0\xf0\xff\xff\xff\x79\xa7\x38\xff\0\0\0\0\xbf\
 +\x71\0\0\0\0\0\0\x79\xa2\x40\xff\0\0\0\0\xb7\x04\0\0\x04\0\0\0\xb7\x05\0\0\x01\
 +\0\0\0\x85\0\0\0\x44\0\0\0\xbf\x01\0\0\0\0\0\0\x67\x01\0\0\x20\0\0\0\x77\x01\0\
 +\0\x20\0\0\0\x55\x01\x06\x01\0\0\0\0\x71\xa1\xf2\xff\0\0\0\0\x55\x01\x11\0\x02\
 +\0\0\0\x71\xa1\xf1\xff\0\0\0\0\x55\x01\x0f\0\x02\0\0\0\x71\xa1\xf3\xff\0\0\0\0\
 +\x55\x01\x0d\0\x01\0\0\0\x79\xa2\x40\xff\0\0\0\0\x07\x02\0\0\x08\0\0\0\xbf\x71\
 +\0\0\0\0\0\0\x79\xa3\x18\xff\0\0\0\0\xb7\x04\0\0\x10\0\0\0\xb7\x05\0\0\x01\0\0\
 +\0\x85\0\0\0\x44\0\0\0\xbf\x01\0\0\0\0\0\0\x67\x01\0\0\x20\0\0\0\x77\x01\0\0\
 +\x20\0\0\0\x55\x01\xf5\0\0\0\0\0\xb7\x01\0\0\x01\0\0\0\x73\x1a\x55\xff\0\0\0\0\
 +\x18\x07\0\0\x01\0\0\0\0\0\0\0\0\x18\0\x1c\x05\0\x3c\0\0\0\0\0\xb7\x08\0\0\x02\
 +\0\0\0\xb7\x07\0\0\0\0\0\0\x6b\x7a\xf0\xff\0\0\0\0\x05\0\x13\0\0\0\0\0\x0f\x81\
 +\0\0\0\0\0\0\xbf\x12\0\0\0\0\0\0\x07\x02\0\0\x01\0\0\0\x71\xa3\xf9\xff\0\0\0\0\
 +\x67\x03\0\0\x03\0\0\0\x3d\x32\x09\0\0\0\0\0\xbf\x72\0\0\0\0\0\0\x07\x02\0\0\
 +\x01\0\0\0\x67\x07\0\0\x20\0\0\0\xbf\x73\0\0\0\0\0\0\x77\x03\0\0\x20\0\0\0\xbf\
 +\x27\0\0\0\0\0\0\xbf\x18\0\0\0\0\0\0\xb7\x01\0\0\x1d\0\0\0\x2d\x31\x04\0\0\0\0\
 +\0\x79\xa8\x20\xff\0\0\0\0\x18\x07\0\0\x01\0\0\0\0\0\0\0\0\x18\0\x1c\x05\0\x25\
 +\0\0\0\0\0\xbf\x89\0\0\0\0\0\0\x79\xa1\x40\xff\0\0\0\0\x0f\x19\0\0\0\0\0\0\xbf\
 +\xa3\0\0\0\0\0\0\x07\x03\0\0\xf0\xff\xff\xff\x79\xa1\x38\xff\0\0\0\0\xbf\x92\0\
 +\0\0\0\0\0\xb7\x04\0\0\x02\0\0\0\xb7\x05\0\0\x01\0\0\0\x85\0\0\0\x44\0\0\0\xbf\
 +\x01\0\0\0\0\0\0\x67\x01\0\0\x20\0\0\0\x77\x01\0\0\x20\0\0\0\x55\x01\x79\0\0\0\
 +\0\0\x71\xa2\xf0\xff\0\0\0\0\x55\x02\x0e\0\xc9\0\0\0\x07\x09\0\0\x02\0\0\0\x79\
 +\xa1\x38\xff\0\0\0\0\xbf\x92\0\0\0\0\0\0\x79\xa3\x10\xff\0\0\0\0\xb7\x04\0\0\
 +\x10\0\0\0\xb7\x05\0\0\x01\0\0\0\x85\0\0\0\x44\0\0\0\xbf\x01\0\0\0\0\0\0\x67\
 +\x01\0\0\x20\0\0\0\x77\x01\0\0\x20\0\0\0\x55\x01\x6c\0\0\0\0\0\xb7\x01\0\0\x01\
 +\0\0\0\x73\x1a\x54\xff\0\0\0\0\x05\0\xde\xff\0\0\0\0\xb7\x01\0\0\x01\0\0\0\x15\
 +\x02\xcd\xff\0\0\0\0\x71\xa1\xf1\xff\0\0\0\0\x07\x01\0\0\x02\0\0\0\x05\0\xca\
 +\xff\0\0\0\0\xb7\x01\0\0\x01\0\0\0\x73\x1a\x56\xff\0\0\0\0\x71\xa1\xf9\xff\0\0\
 +\0\0\x67\x01\0\0\x03\0\0\0\x79\xa2\x40\xff\0\0\0\0\x0f\x12\0\0\0\0\0\0\x07\x02\
 +\0\0\x08\0\0\0\x7b\x2a\x40\xff\0\0\0\0\x71\xa9\xf8\xff\0\0\0\0\x79\xa2\x28\xff\
 +\0\0\0\0\x25\x09\x0c\0\x3c\0\0\0\xb7\x01\0\0\x01\0\0\0\x6f\x91\0\0\0\0\0\0\x5f\
 +\x71\0\0\0\0\0\0\x55\x01\x01\0\0\0\0\0\x05\0\x07\0\0\0\0\0\x07\x02\0\0\x01\0\0\
 +\0\xbf\x21\0\0\0\0\0\0\x67\x01\0\0\x20\0\0\0\x77\x01\0\0\x20\0\0\0\x55\x01\x7d\
 +\xff\x0b\0\0\0\x71\xa7\x56\xff\0\0\0\0\x05\0\x09\xff\0\0\0\0\x15\x09\xf8\xff\
 +\x87\0\0\0\x05\0\xfc\xff\0\0\0\0\x71\xa1\x51\xff\0\0\0\0\x79\xa0\x30\xff\0\0\0\
 +\0\x15\x01\x10\x01\0\0\0\0\x71\x62\x03\0\0\0\0\0\x67\x02\0\0\x08\0\0\0\x71\x61\
  \x02\0\0\0\0\0\x4f\x12\0\0\0\0\0\0\x71\x63\x04\0\0\0\0\0\x71\x61\x05\0\0\0\0\0\
  \x67\x01\0\0\x08\0\0\0\x4f\x31\0\0\0\0\0\0\x67\x01\0\0\x10\0\0\0\x4f\x21\0\0\0\
 -\0\0\0\x71\xa2\x5b\xff\0\0\0\0\x15\x02\x42\0\0\0\0\0\xbf\x12\0\0\0\0\0\0\x57\
 +\0\0\0\x71\xa2\x53\xff\0\0\0\0\x15\x02\x42\0\0\0\0\0\xbf\x12\0\0\0\0\0\0\x57\
  \x02\0\0\x10\0\0\0\x15\x02\x3f\0\0\0\0\0\x57\x01\0\0\x80\0\0\0\xb7\x02\0\0\x10\
  \0\0\0\xb7\x03\0\0\x10\0\0\0\x15\x01\x01\0\0\0\0\0\xb7\x03\0\0\x30\0\0\0\x71\
 -\xa4\x5d\xff\0\0\0\0\x15\x04\x01\0\0\0\0\0\xbf\x32\0\0\0\0\0\0\xbf\xa3\0\0\0\0\
 -\0\0\x07\x03\0\0\x64\xff\xff\xff\xbf\x34\0\0\0\0\0\0\x15\x01\x02\0\0\0\0\0\xbf\
 -\xa4\0\0\0\0\0\0\x07\x04\0\0\x84\xff\xff\xff\x71\xa5\x5c\xff\0\0\0\0\xbf\x31\0\
 +\xa4\x55\xff\0\0\0\0\x15\x04\x01\0\0\0\0\0\xbf\x32\0\0\0\0\0\0\xbf\xa3\0\0\0\0\
 +\0\0\x07\x03\0\0\x5c\xff\xff\xff\xbf\x34\0\0\0\0\0\0\x15\x01\x02\0\0\0\0\0\xbf\
 +\xa4\0\0\0\0\0\0\x07\x04\0\0\x7c\xff\xff\xff\x71\xa5\x54\xff\0\0\0\0\xbf\x31\0\
  \0\0\0\0\0\x15\x05\x01\0\0\0\0\0\xbf\x41\0\0\0\0\0\0\x61\x14\x04\0\0\0\0\0\x67\
 -\x04\0\0\x20\0\0\0\x61\x15\0\0\0\0\0\0\x4f\x54\0\0\0\0\0\0\x7b\x4a\xa8\xff\0\0\
 +\x04\0\0\x20\0\0\0\x61\x15\0\0\0\0\0\0\x4f\x54\0\0\0\0\0\0\x7b\x4a\xa0\xff\0\0\
  \0\0\x61\x14\x08\0\0\0\0\0\x61\x11\x0c\0\0\0\0\0\x67\x01\0\0\x20\0\0\0\x4f\x41\
 -\0\0\0\0\0\0\x7b\x1a\xb0\xff\0\0\0\0\x0f\x23\0\0\0\0\0\0\x61\x31\0\0\0\0\0\0\
 -\x61\x32\x04\0\0\0\0\0\x61\x34\x08\0\0\0\0\0\x61\x33\x0c\0\0\0\0\0\x69\xa5\x62\
 -\xff\0\0\0\0\x6b\x5a\xca\xff\0\0\0\0\x69\xa5\x60\xff\0\0\0\0\x6b\x5a\xc8\xff\0\
 -\0\0\0\x67\x03\0\0\x20\0\0\0\x4f\x43\0\0\0\0\0\0\x7b\x3a\xc0\xff\0\0\0\0\x67\
 -\x02\0\0\x20\0\0\0\x4f\x12\0\0\0\0\0\0\x7b\x2a\xb8\xff\0\0\0\0\x05\0\x6b\0\0\0\
 -\0\0\x71\xa2\x5a\xff\0\0\0\0\x15\x02\x04\0\0\0\0\0\xbf\x12\0\0\0\0\0\0\x57\x02\
 +\0\0\0\0\0\0\x7b\x1a\xa8\xff\0\0\0\0\x0f\x23\0\0\0\0\0\0\x61\x31\0\0\0\0\0\0\
 +\x61\x32\x04\0\0\0\0\0\x61\x34\x08\0\0\0\0\0\x61\x33\x0c\0\0\0\0\0\x69\xa5\x5a\
 +\xff\0\0\0\0\x6b\x5a\xc2\xff\0\0\0\0\x69\xa5\x58\xff\0\0\0\0\x6b\x5a\xc0\xff\0\
 +\0\0\0\x67\x03\0\0\x20\0\0\0\x4f\x43\0\0\0\0\0\0\x7b\x3a\xb8\xff\0\0\0\0\x67\
 +\x02\0\0\x20\0\0\0\x4f\x12\0\0\0\0\0\0\x7b\x2a\xb0\xff\0\0\0\0\x05\0\x6b\0\0\0\
 +\0\0\x71\xa2\x52\xff\0\0\0\0\x15\x02\x04\0\0\0\0\0\xbf\x12\0\0\0\0\0\0\x57\x02\
  \0\0\x04\0\0\0\x15\x02\x01\0\0\0\0\0\x05\0\xf1\xfe\0\0\0\0\x57\x01\0\0\x01\0\0\
 -\0\x15\x01\xd0\0\0\0\0\0\x61\xa1\x64\xff\0\0\0\0\x63\x1a\xa8\xff\0\0\0\0\x61\
 -\xa1\x68\xff\0\0\0\0\x63\x1a\xac\xff\0\0\0\0\x05\0\x5e\0\0\0\0\0\xb7\x09\0\0\
 -\x3c\0\0\0\x79\xa8\x28\xff\0\0\0\0\x67\0\0\0\x20\0\0\0\x77\0\0\0\x20\0\0\0\x15\
 -\0\xac\xff\0\0\0\0\x05\0\xc5\0\0\0\0\0\x71\xa2\x5a\xff\0\0\0\0\x15\x02\x26\0\0\
 +\0\x15\x01\xcd\0\0\0\0\0\x61\xa1\x5c\xff\0\0\0\0\x63\x1a\xa0\xff\0\0\0\0\x61\
 +\xa1\x60\xff\0\0\0\0\x63\x1a\xa4\xff\0\0\0\0\x05\0\x5e\0\0\0\0\0\xb7\x09\0\0\
 +\x3c\0\0\0\x79\xa8\x20\xff\0\0\0\0\x67\0\0\0\x20\0\0\0\x77\0\0\0\x20\0\0\0\x15\
 +\0\xac\xff\0\0\0\0\x05\0\xc2\0\0\0\0\0\x71\xa2\x52\xff\0\0\0\0\x15\x02\x26\0\0\
  \0\0\0\xbf\x12\0\0\0\0\0\0\x57\x02\0\0\x20\0\0\0\x15\x02\x23\0\0\0\0\0\x57\x01\
  \0\0\0\x01\0\0\xb7\x02\0\0\x10\0\0\0\xb7\x03\0\0\x10\0\0\0\x15\x01\x01\0\0\0\0\
 -\0\xb7\x03\0\0\x30\0\0\0\x71\xa4\x5d\xff\0\0\0\0\x15\x04\x01\0\0\0\0\0\xbf\x32\
 -\0\0\0\0\0\0\xbf\xa3\0\0\0\0\0\0\x07\x03\0\0\x64\xff\xff\xff\xbf\x34\0\0\0\0\0\
 -\0\x15\x01\x02\0\0\0\0\0\xbf\xa4\0\0\0\0\0\0\x07\x04\0\0\x84\xff\xff\xff\x71\
 -\xa5\x5c\xff\0\0\0\0\xbf\x31\0\0\0\0\0\0\x15\x05\xbd\xff\0\0\0\0\x05\0\xbb\xff\
 -\0\0\0\0\xb7\x01\0\0\x01\0\0\0\x73\x1a\x5a\xff\0\0\0\0\xb7\x01\0\0\0\0\0\0\x7b\
 -\x1a\xd0\xff\0\0\0\0\xbf\xa3\0\0\0\0\0\0\x07\x03\0\0\xd0\xff\xff\xff\x79\xa1\
 -\x40\xff\0\0\0\0\x79\xa2\x48\xff\0\0\0\0\xb7\x04\0\0\x08\0\0\0\xb7\x05\0\0\x01\
 -\0\0\0\x85\0\0\0\x44\0\0\0\x67\0\0\0\x20\0\0\0\x77\0\0\0\x20\0\0\0\x55\0\xa0\0\
 +\0\xb7\x03\0\0\x30\0\0\0\x71\xa4\x55\xff\0\0\0\0\x15\x04\x01\0\0\0\0\0\xbf\x32\
 +\0\0\0\0\0\0\xbf\xa3\0\0\0\0\0\0\x07\x03\0\0\x5c\xff\xff\xff\xbf\x34\0\0\0\0\0\
 +\0\x15\x01\x02\0\0\0\0\0\xbf\xa4\0\0\0\0\0\0\x07\x04\0\0\x7c\xff\xff\xff\x71\
 +\xa5\x54\xff\0\0\0\0\xbf\x31\0\0\0\0\0\0\x15\x05\xbd\xff\0\0\0\0\x05\0\xbb\xff\
 +\0\0\0\0\xb7\x01\0\0\x01\0\0\0\x73\x1a\x52\xff\0\0\0\0\xb7\x01\0\0\0\0\0\0\x7b\
 +\x1a\xc8\xff\0\0\0\0\xbf\xa3\0\0\0\0\0\0\x07\x03\0\0\xc8\xff\xff\xff\x79\xa1\
 +\x38\xff\0\0\0\0\x79\xa2\x40\xff\0\0\0\0\xb7\x04\0\0\x08\0\0\0\xb7\x05\0\0\x01\
 +\0\0\0\x85\0\0\0\x44\0\0\0\x67\0\0\0\x20\0\0\0\x77\0\0\0\x20\0\0\0\x55\0\x9d\0\
  \0\0\0\0\x05\0\xa8\xfe\0\0\0\0\x15\x09\xf3\xfe\x87\0\0\0\x05\0\x83\xff\0\0\0\0\
 -\xbf\x12\0\0\0\0\0\0\x57\x02\0\0\x08\0\0\0\x15\x02\x9a\0\0\0\0\0\x57\x01\0\0\
 +\xbf\x12\0\0\0\0\0\0\x57\x02\0\0\x08\0\0\0\x15\x02\x97\0\0\0\0\0\x57\x01\0\0\
  \x40\0\0\0\xb7\x02\0\0\x0c\0\0\0\xb7\x03\0\0\x0c\0\0\0\x15\x01\x01\0\0\0\0\0\
 -\xb7\x03\0\0\x2c\0\0\0\x71\xa4\x5c\xff\0\0\0\0\x15\x04\x01\0\0\0\0\0\xbf\x32\0\
 -\0\0\0\0\0\xbf\xa3\0\0\0\0\0\0\x07\x03\0\0\x58\xff\xff\xff\x0f\x23\0\0\0\0\0\0\
 +\xb7\x03\0\0\x2c\0\0\0\x71\xa4\x54\xff\0\0\0\0\x15\x04\x01\0\0\0\0\0\xbf\x32\0\
 +\0\0\0\0\0\xbf\xa3\0\0\0\0\0\0\x07\x03\0\0\x50\xff\xff\xff\x0f\x23\0\0\0\0\0\0\
  \x61\x32\x04\0\0\0\0\0\x67\x02\0\0\x20\0\0\0\x61\x34\0\0\0\0\0\0\x4f\x42\0\0\0\
 -\0\0\0\x7b\x2a\xa8\xff\0\0\0\0\x61\x32\x08\0\0\0\0\0\x61\x33\x0c\0\0\0\0\0\x67\
 -\x03\0\0\x20\0\0\0\x4f\x23\0\0\0\0\0\0\x7b\x3a\xb0\xff\0\0\0\0\x71\xa2\x5d\xff\
 -\0\0\0\0\x15\x02\x0c\0\0\0\0\0\x15\x01\x0b\0\0\0\0\0\x61\xa1\xa0\xff\0\0\0\0\
 -\x67\x01\0\0\x20\0\0\0\x61\xa2\x9c\xff\0\0\0\0\x4f\x21\0\0\0\0\0\0\x7b\x1a\xc0\
 -\xff\0\0\0\0\x61\xa1\x98\xff\0\0\0\0\x67\x01\0\0\x20\0\0\0\x61\xa2\x94\xff\0\0\
 +\0\0\0\x7b\x2a\xa0\xff\0\0\0\0\x61\x32\x08\0\0\0\0\0\x61\x33\x0c\0\0\0\0\0\x67\
 +\x03\0\0\x20\0\0\0\x4f\x23\0\0\0\0\0\0\x7b\x3a\xa8\xff\0\0\0\0\x71\xa2\x55\xff\
 +\0\0\0\0\x15\x02\x0c\0\0\0\0\0\x15\x01\x0b\0\0\0\0\0\x61\xa1\x98\xff\0\0\0\0\
 +\x67\x01\0\0\x20\0\0\0\x61\xa2\x94\xff\0\0\0\0\x4f\x21\0\0\0\0\0\0\x7b\x1a\xb8\
 +\xff\0\0\0\0\x61\xa1\x90\xff\0\0\0\0\x67\x01\0\0\x20\0\0\0\x61\xa2\x8c\xff\0\0\
  \0\0\x05\0\x0a\0\0\0\0\0\xb7\x09\0\0\x2b\0\0\0\x05\0\xae\xff\0\0\0\0\x61\xa1\
 -\x80\xff\0\0\0\0\x67\x01\0\0\x20\0\0\0\x61\xa2\x7c\xff\0\0\0\0\x4f\x21\0\0\0\0\
 -\0\0\x7b\x1a\xc0\xff\0\0\0\0\x61\xa1\x78\xff\0\0\0\0\x67\x01\0\0\x20\0\0\0\x61\
 -\xa2\x74\xff\0\0\0\0\x4f\x21\0\0\0\0\0\0\x7b\x1a\xb8\xff\0\0\0\0\xb7\x02\0\0\0\
 +\x78\xff\0\0\0\0\x67\x01\0\0\x20\0\0\0\x61\xa2\x74\xff\0\0\0\0\x4f\x21\0\0\0\0\
 +\0\0\x7b\x1a\xb8\xff\0\0\0\0\x61\xa1\x70\xff\0\0\0\0\x67\x01\0\0\x20\0\0\0\x61\
 +\xa2\x6c\xff\0\0\0\0\x4f\x21\0\0\0\0\0\0\x7b\x1a\xb0\xff\0\0\0\0\xb7\x02\0\0\0\
  \0\0\0\x07\x08\0\0\x04\0\0\0\x61\x03\0\0\0\0\0\0\xb7\x05\0\0\0\0\0\0\xbf\xa1\0\
 -\0\0\0\0\0\x07\x01\0\0\xa8\xff\xff\xff\x0f\x21\0\0\0\0\0\0\x71\x14\0\0\0\0\0\0\
 +\0\0\0\0\0\x07\x01\0\0\xa0\xff\xff\xff\x0f\x21\0\0\0\0\0\0\x71\x14\0\0\0\0\0\0\
  \xbf\x41\0\0\0\0\0\0\x67\x01\0\0\x38\0\0\0\xc7\x01\0\0\x3f\0\0\0\x5f\x31\0\0\0\
  \0\0\0\xaf\x51\0\0\0\0\0\0\xbf\x85\0\0\0\0\0\0\x0f\x25\0\0\0\0\0\0\x71\x55\0\0\
  \0\0\0\0\x67\x03\0\0\x01\0\0\0\xbf\x50\0\0\0\0\0\0\x77\0\0\0\x07\0\0\0\x4f\x03\
@@ -XXX,XX +XXX,XX @@ static inline const void *rss_bpf__elf_bytes(size_t *sz)
  \0\x57\0\0\0\x01\0\0\0\x67\x03\0\0\x01\0\0\0\x4f\x03\0\0\0\0\0\0\x57\x04\0\0\
  \x01\0\0\0\x87\x04\0\0\0\0\0\0\x5f\x34\0\0\0\0\0\0\xaf\x41\0\0\0\0\0\0\x57\x05\
  \0\0\x01\0\0\0\x67\x03\0\0\x01\0\0\0\x4f\x53\0\0\0\0\0\0\x07\x02\0\0\x01\0\0\0\
 -\xbf\x15\0\0\0\0\0\0\x15\x02\x01\0\x24\0\0\0\x05\0\xa9\xff\0\0\0\0\xbf\x12\0\0\
 -\0\0\0\0\x67\x02\0\0\x20\0\0\0\x77\x02\0\0\x20\0\0\0\x15\x02\x0e\0\0\0\0\0\x71\
 -\x63\x06\0\0\0\0\0\x71\x64\x07\0\0\0\0\0\x67\x04\0\0\x08\0\0\0\x4f\x34\0\0\0\0\
 -\0\0\x3f\x42\0\0\0\0\0\0\x2f\x42\0\0\0\0\0\0\x1f\x21\0\0\0\0\0\0\x63\x1a\x58\
 -\xff\0\0\0\0\xbf\xa2\0\0\0\0\0\0\x07\x02\0\0\x58\xff\xff\xff\x18\x01\0\0\0\0\0\
 -\0\0\0\0\0\0\0\0\0\x85\0\0\0\x01\0\0\0\x55\0\x05\0\0\0\0\0\x71\x61\x08\0\0\0\0\
 -\0\x71\x60\x09\0\0\0\0\0\x67\0\0\0\x08\0\0\0\x4f\x10\0\0\0\0\0\0\x95\0\0\0\0\0\
 -\0\0\x69\0\0\0\0\0\0\0\x05\0\xfd\xff\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\
 +\xbf\x15\0\0\0\0\0\0\x15\x02\x01\0\x24\0\0\0\x05\0\xa9\xff\0\0\0\0\x67\x01\0\0\
 +\x20\0\0\0\x77\x01\0\0\x20\0\0\0\x15\x01\x0c\0\0\0\0\0\x71\x62\x06\0\0\0\0\0\
 +\x71\x63\x07\0\0\0\0\0\x67\x03\0\0\x08\0\0\0\x4f\x23\0\0\0\0\0\0\x9f\x31\0\0\0\
 +\0\0\0\x63\x1a\x50\xff\0\0\0\0\xbf\xa2\0\0\0\0\0\0\x07\x02\0\0\x50\xff\xff\xff\
 +\x18\x01\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x85\0\0\0\x01\0\0\0\x55\0\x05\0\0\0\0\0\
 +\x71\x61\x08\0\0\0\0\0\x71\x60\x09\0\0\0\0\0\x67\0\0\0\x08\0\0\0\x4f\x10\0\0\0\
 +\0\0\0\x95\0\0\0\0\0\0\0\x69\0\0\0\0\0\0\0\x05\0\xfd\xff\0\0\0\0\0\0\0\0\0\0\0\
  \0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\
  \0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\
 -\0\x47\x50\x4c\x20\x76\x32\0\0\x9f\xeb\x01\0\x18\0\0\0\0\0\0\0\x10\x05\0\0\x10\
 -\x05\0\0\x65\x11\0\0\0\0\0\0\0\0\0\x02\x03\0\0\0\x01\0\0\0\0\0\0\x01\x04\0\0\0\
 -\x20\0\0\x01\0\0\0\0\0\0\0\x03\0\0\0\0\x02\0\0\0\x04\0\0\0\x02\0\0\0\x05\0\0\0\
 -\0\0\0\x01\x04\0\0\0\x20\0\0\0\0\0\0\0\0\0\0\x02\x06\0\0\0\0\0\0\0\0\0\0\x03\0\
 -\0\0\0\x02\0\0\0\x04\0\0\0\x04\0\0\0\0\0\0\0\0\0\0\x02\x08\0\0\0\0\0\0\0\0\0\0\
 -\x03\0\0\0\0\x02\0\0\0\x04\0\0\0\x0a\0\0\0\0\0\0\0\0\0\0\x02\x0a\0\0\0\0\0\0\0\
 -\0\0\0\x03\0\0\0\0\x02\0\0\0\x04\0\0\0\x01\0\0\0\0\0\0\0\x04\0\0\x04\x20\0\0\0\
 +\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x47\x50\
 +\x4c\x20\x76\x32\0\0\x9f\xeb\x01\0\x18\0\0\0\0\0\0\0\x58\x05\0\0\x58\x05\0\0\
 +\x6b\x11\0\0\0\0\0\0\0\0\0\x02\x03\0\0\0\x01\0\0\0\0\0\0\x01\x04\0\0\0\x20\0\0\
 +\x01\0\0\0\0\0\0\0\x03\0\0\0\0\x02\0\0\0\x04\0\0\0\x02\0\0\0\x05\0\0\0\0\0\0\
 +\x01\x04\0\0\0\x20\0\0\0\0\0\0\0\0\0\0\x02\x06\0\0\0\0\0\0\0\0\0\0\x03\0\0\0\0\
 +\x02\0\0\0\x04\0\0\0\x04\0\0\0\0\0\0\0\0\0\0\x02\x08\0\0\0\0\0\0\0\0\0\0\x03\0\
 +\0\0\0\x02\0\0\0\x04\0\0\0\x0a\0\0\0\0\0\0\0\0\0\0\x02\x0a\0\0\0\0\0\0\0\0\0\0\
 +\x03\0\0\0\0\x02\0\0\0\x04\0\0\0\x01\0\0\0\0\0\0\0\0\0\0\x02\x0c\0\0\0\0\0\0\0\
 +\0\0\0\x03\0\0\0\0\x02\0\0\0\x04\0\0\0\0\x04\0\0\0\0\0\0\x05\0\0\x04\x28\0\0\0\
  \x19\0\0\0\x01\0\0\0\0\0\0\0\x1e\0\0\0\x05\0\0\0\x40\0\0\0\x27\0\0\0\x07\0\0\0\
 -\x80\0\0\0\x32\0\0\0\x09\0\0\0\xc0\0\0\0\x3e\0\0\0\0\0\0\x0e\x0b\0\0\0\x01\0\0\
 -\0\0\0\0\0\0\0\0\x02\x0e\0\0\0\0\0\0\0\0\0\0\x03\0\0\0\0\x02\0\0\0\x04\0\0\0\
 -\x28\0\0\0\0\0\0\0\x04\0\0\x04\x20\0\0\0\x19\0\0\0\x01\0\0\0\0\0\0\0\x1e\0\0\0\
 -\x05\0\0\0\x40\0\0\0\x27\0\0\0\x0d\0\0\0\x80\0\0\0\x32\0\0\0\x09\0\0\0\xc0\0\0\
 -\0\x59\0\0\0\0\0\0\x0e\x0f\0\0\0\x01\0\0\0\0\0\0\0\0\0\0\x02\x12\0\0\0\0\0\0\0\
 -\0\0\0\x03\0\0\0\0\x02\0\0\0\x04\0\0\0\x80\0\0\0\0\0\0\0\x04\0\0\x04\x20\0\0\0\
 -\x19\0\0\0\x01\0\0\0\0\0\0\0\x1e\0\0\0\x05\0\0\0\x40\0\0\0\x27\0\0\0\x01\0\0\0\
 -\x80\0\0\0\x32\0\0\0\x11\0\0\0\xc0\0\0\0\x72\0\0\0\0\0\0\x0e\x13\0\0\0\x01\0\0\
 -\0\0\0\0\0\0\0\0\x02\x16\0\0\0\x90\0\0\0\x22\0\0\x04\xc0\0\0\0\x9a\0\0\0\x17\0\
 -\0\0\0\0\0\0\x9e\0\0\0\x17\0\0\0\x20\0\0\0\xa7\0\0\0\x17\0\0\0\x40\0\0\0\xac\0\
 -\0\0\x17\0\0\0\x60\0\0\0\xba\0\0\0\x17\0\0\0\x80\0\0\0\xc3\0\0\0\x17\0\0\0\xa0\
 -\0\0\0\xd0\0\0\0\x17\0\0\0\xc0\0\0\0\xd9\0\0\0\x17\0\0\0\xe0\0\0\0\xe4\0\0\0\
 -\x17\0\0\0\0\x01\0\0\xed\0\0\0\x17\0\0\0\x20\x01\0\0\xfd\0\0\0\x17\0\0\0\x40\
 -\x01\0\0\x05\x01\0\0\x17\0\0\0\x60\x01\0\0\x0e\x01\0\0\x19\0\0\0\x80\x01\0\0\
 -\x11\x01\0\0\x17\0\0\0\x20\x02\0\0\x16\x01\0\0\x17\0\0\0\x40\x02\0\0\x21\x01\0\
 -\0\x17\0\0\0\x60\x02\0\0\x26\x01\0\0\x17\0\0\0\x80\x02\0\0\x2f\x01\0\0\x17\0\0\
 -\0\xa0\x02\0\0\x37\x01\0\0\x17\0\0\0\xc0\x02\0\0\x3e\x01\0\0\x17\0\0\0\xe0\x02\
 -\0\0\x49\x01\0\0\x17\0\0\0\0\x03\0\0\x53\x01\0\0\x1a\0\0\0\x20\x03\0\0\x5e\x01\
 -\0\0\x1a\0\0\0\xa0\x03\0\0\x68\x01\0\0\x17\0\0\0\x20\x04\0\0\x74\x01\0\0\x17\0\
 -\0\0\x40\x04\0\0\x7f\x01\0\0\x17\0\0\0\x60\x04\0\0\0\0\0\0\x1b\0\0\0\x80\x04\0\
 -\0\x89\x01\0\0\x1d\0\0\0\xc0\x04\0\0\x90\x01\0\0\x17\0\0\0\0\x05\0\0\x99\x01\0\
 -\0\x17\0\0\0\x20\x05\0\0\0\0\0\0\x1f\0\0\0\x40\x05\0\0\xa2\x01\0\0\x17\0\0\0\
 -\x80\x05\0\0\xab\x01\0\0\x21\0\0\0\xa0\x05\0\0\xb7\x01\0\0\x1d\0\0\0\xc0\x05\0\
 -\0\xc0\x01\0\0\0\0\0\x08\x18\0\0\0\xc6\x01\0\0\0\0\0\x01\x04\0\0\0\x20\0\0\0\0\
 -\0\0\0\0\0\0\x03\0\0\0\0\x17\0\0\0\x04\0\0\0\x05\0\0\0\0\0\0\0\0\0\0\x03\0\0\0\
 -\0\x17\0\0\0\x04\0\0\0\x04\0\0\0\0\0\0\0\x01\0\0\x05\x08\0\0\0\xd3\x01\0\0\x1c\
 -\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x02\x2a\0\0\0\xdd\x01\0\0\0\0\0\x08\x1e\0\0\0\xe3\
 -\x01\0\0\0\0\0\x01\x08\0\0\0\x40\0\0\0\0\0\0\0\x01\0\0\x05\x08\0\0\0\xf6\x01\0\
 -\0\x20\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x02\x2b\0\0\0\xf9\x01\0\0\0\0\0\x08\x22\0\0\
 -\0\xfe\x01\0\0\0\0\0\x01\x01\0\0\0\x08\0\0\0\0\0\0\0\x01\0\0\x0d\x02\0\0\0\x0c\
 -\x02\0\0\x15\0\0\0\x10\x02\0\0\x01\0\0\x0c\x23\0\0\0\x32\x11\0\0\0\0\0\x01\x01\
 -\0\0\0\x08\0\0\x01\0\0\0\0\0\0\0\x03\0\0\0\0\x25\0\0\0\x04\0\0\0\x07\0\0\0\x37\
 -\x11\0\0\0\0\0\x0e\x26\0\0\0\x01\0\0\0\x40\x11\0\0\x03\0\0\x0f\0\0\0\0\x0c\0\0\
 -\0\0\0\0\0\x20\0\0\0\x10\0\0\0\0\0\0\0\x20\0\0\0\x14\0\0\0\0\0\0\0\x20\0\0\0\
 -\x46\x11\0\0\x01\0\0\x0f\0\0\0\0\x27\0\0\0\0\0\0\0\x07\0\0\0\x4e\x11\0\0\0\0\0\
 -\x07\0\0\0\0\x5c\x11\0\0\0\0\0\x07\0\0\0\0\0\x69\x6e\x74\0\x5f\x5f\x41\x52\x52\
 -\x41\x59\x5f\x53\x49\x5a\x45\x5f\x54\x59\x50\x45\x5f\x5f\0\x74\x79\x70\x65\0\
 -\x6b\x65\x79\x5f\x73\x69\x7a\x65\0\x76\x61\x6c\x75\x65\x5f\x73\x69\x7a\x65\0\
 -\x6d\x61\x78\x5f\x65\x6e\x74\x72\x69\x65\x73\0\x74\x61\x70\x5f\x72\x73\x73\x5f\
 -\x6d\x61\x70\x5f\x63\x6f\x6e\x66\x69\x67\x75\x72\x61\x74\x69\x6f\x6e\x73\0\x74\
 +\x80\0\0\0\x32\0\0\0\x09\0\0\0\xc0\0\0\0\x3e\0\0\0\x0b\0\0\0\0\x01\0\0\x48\0\0\
 +\0\0\0\0\x0e\x0d\0\0\0\x01\0\0\0\0\0\0\0\0\0\0\x02\x10\0\0\0\0\0\0\0\0\0\0\x03\
 +\0\0\0\0\x02\0\0\0\x04\0\0\0\x28\0\0\0\0\0\0\0\x05\0\0\x04\x28\0\0\0\x19\0\0\0\
 +\x01\0\0\0\0\0\0\0\x1e\0\0\0\x05\0\0\0\x40\0\0\0\x27\0\0\0\x0f\0\0\0\x80\0\0\0\
 +\x32\0\0\0\x09\0\0\0\xc0\0\0\0\x3e\0\0\0\x0b\0\0\0\0\x01\0\0\x63\0\0\0\0\0\0\
 +\x0e\x11\0\0\0\x01\0\0\0\0\0\0\0\0\0\0\x02\x14\0\0\0\0\0\0\0\0\0\0\x03\0\0\0\0\
 +\x02\0\0\0\x04\0\0\0\x80\0\0\0\0\0\0\0\x05\0\0\x04\x28\0\0\0\x19\0\0\0\x01\0\0\
 +\0\0\0\0\0\x1e\0\0\0\x05\0\0\0\x40\0\0\0\x27\0\0\0\x01\0\0\0\x80\0\0\0\x32\0\0\
 +\0\x13\0\0\0\xc0\0\0\0\x3e\0\0\0\x0b\0\0\0\0\x01\0\0\x7c\0\0\0\0\0\0\x0e\x15\0\
 +\0\0\x01\0\0\0\0\0\0\0\0\0\0\x02\x18\0\0\0\x9a\0\0\0\x22\0\0\x04\xc0\0\0\0\xa4\
 +\0\0\0\x19\0\0\0\0\0\0\0\xa8\0\0\0\x19\0\0\0\x20\0\0\0\xb1\0\0\0\x19\0\0\0\x40\
 +\0\0\0\xb6\0\0\0\x19\0\0\0\x60\0\0\0\xc4\0\0\0\x19\0\0\0\x80\0\0\0\xcd\0\0\0\
 +\x19\0\0\0\xa0\0\0\0\xda\0\0\0\x19\0\0\0\xc0\0\0\0\xe3\0\0\0\x19\0\0\0\xe0\0\0\
 +\0\xee\0\0\0\x19\0\0\0\0\x01\0\0\xf7\0\0\0\x19\0\0\0\x20\x01\0\0\x07\x01\0\0\
 +\x19\0\0\0\x40\x01\0\0\x0f\x01\0\0\x19\0\0\0\x60\x01\0\0\x18\x01\0\0\x1b\0\0\0\
 +\x80\x01\0\0\x1b\x01\0\0\x19\0\0\0\x20\x02\0\0\x20\x01\0\0\x19\0\0\0\x40\x02\0\
 +\0\x2b\x01\0\0\x19\0\0\0\x60\x02\0\0\x30\x01\0\0\x19\0\0\0\x80\x02\0\0\x39\x01\
 +\0\0\x19\0\0\0\xa0\x02\0\0\x41\x01\0\0\x19\0\0\0\xc0\x02\0\0\x48\x01\0\0\x19\0\
 +\0\0\xe0\x02\0\0\x53\x01\0\0\x19\0\0\0\0\x03\0\0\x5d\x01\0\0\x1c\0\0\0\x20\x03\
 +\0\0\x68\x01\0\0\x1c\0\0\0\xa0\x03\0\0\x72\x01\0\0\x19\0\0\0\x20\x04\0\0\x7e\
 +\x01\0\0\x19\0\0\0\x40\x04\0\0\x89\x01\0\0\x19\0\0\0\x60\x04\0\0\0\0\0\0\x1d\0\
 +\0\0\x80\x04\0\0\x93\x01\0\0\x1f\0\0\0\xc0\x04\0\0\x9a\x01\0\0\x19\0\0\0\0\x05\
 +\0\0\xa3\x01\0\0\x19\0\0\0\x20\x05\0\0\0\0\0\0\x21\0\0\0\x40\x05\0\0\xac\x01\0\
 +\0\x19\0\0\0\x80\x05\0\0\xb5\x01\0\0\x23\0\0\0\xa0\x05\0\0\xc1\x01\0\0\x1f\0\0\
 +\0\xc0\x05\0\0\xca\x01\0\0\0\0\0\x08\x1a\0\0\0\xd0\x01\0\0\0\0\0\x01\x04\0\0\0\
 +\x20\0\0\0\0\0\0\0\0\0\0\x03\0\0\0\0\x19\0\0\0\x04\0\0\0\x05\0\0\0\0\0\0\0\0\0\
 +\0\x03\0\0\0\0\x19\0\0\0\x04\0\0\0\x04\0\0\0\0\0\0\0\x01\0\0\x05\x08\0\0\0\xdd\
 +\x01\0\0\x1e\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x02\x2c\0\0\0\xe7\x01\0\0\0\0\0\x08\
 +\x20\0\0\0\xed\x01\0\0\0\0\0\x01\x08\0\0\0\x40\0\0\0\0\0\0\0\x01\0\0\x05\x08\0\
 +\0\0\0\x02\0\0\x22\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x02\x2d\0\0\0\x03\x02\0\0\0\0\0\
 +\x08\x24\0\0\0\x08\x02\0\0\0\0\0\x01\x01\0\0\0\x08\0\0\0\0\0\0\0\x01\0\0\x0d\
 +\x02\0\0\0\x16\x02\0\0\x17\0\0\0\x1a\x02\0\0\x01\0\0\x0c\x25\0\0\0\x38\x11\0\0\
 +\0\0\0\x01\x01\0\0\0\x08\0\0\x01\0\0\0\0\0\0\0\x03\0\0\0\0\x27\0\0\0\x04\0\0\0\
 +\x07\0\0\0\x3d\x11\0\0\0\0\0\x0e\x28\0\0\0\x01\0\0\0\x46\x11\0\0\x03\0\0\x0f\0\
 +\0\0\0\x0e\0\0\0\0\0\0\0\x28\0\0\0\x12\0\0\0\0\0\0\0\x28\0\0\0\x16\0\0\0\0\0\0\
 +\0\x28\0\0\0\x4c\x11\0\0\x01\0\0\x0f\0\0\0\0\x29\0\0\0\0\0\0\0\x07\0\0\0\x54\
 +\x11\0\0\0\0\0\x07\0\0\0\0\x62\x11\0\0\0\0\0\x07\0\0\0\0\0\x69\x6e\x74\0\x5f\
 +\x5f\x41\x52\x52\x41\x59\x5f\x53\x49\x5a\x45\x5f\x54\x59\x50\x45\x5f\x5f\0\x74\
 +\x79\x70\x65\0\x6b\x65\x79\x5f\x73\x69\x7a\x65\0\x76\x61\x6c\x75\x65\x5f\x73\
 +\x69\x7a\x65\0\x6d\x61\x78\x5f\x65\x6e\x74\x72\x69\x65\x73\0\x6d\x61\x70\x5f\
 +\x66\x6c\x61\x67\x73\0\x74\x61\x70\x5f\x72\x73\x73\x5f\x6d\x61\x70\x5f\x63\x6f\
 +\x6e\x66\x69\x67\x75\x72\x61\x74\x69\x6f\x6e\x73\0\x74\x61\x70\x5f\x72\x73\x73\
 +\x5f\x6d\x61\x70\x5f\x74\x6f\x65\x70\x6c\x69\x74\x7a\x5f\x6b\x65\x79\0\x74\x61\
 +\x70\x5f\x72\x73\x73\x5f\x6d\x61\x70\x5f\x69\x6e\x64\x69\x72\x65\x63\x74\x69\
 +\x6f\x6e\x5f\x74\x61\x62\x6c\x65\0\x5f\x5f\x73\x6b\x5f\x62\x75\x66\x66\0\x6c\
 +\x65\x6e\0\x70\x6b\x74\x5f\x74\x79\x70\x65\0\x6d\x61\x72\x6b\0\x71\x75\x65\x75\
 +\x65\x5f\x6d\x61\x70\x70\x69\x6e\x67\0\x70\x72\x6f\x74\x6f\x63\x6f\x6c\0\x76\
 +\x6c\x61\x6e\x5f\x70\x72\x65\x73\x65\x6e\x74\0\x76\x6c\x61\x6e\x5f\x74\x63\x69\
 +\0\x76\x6c\x61\x6e\x5f\x70\x72\x6f\x74\x6f\0\x70\x72\x69\x6f\x72\x69\x74\x79\0\
 +\x69\x6e\x67\x72\x65\x73\x73\x5f\x69\x66\x69\x6e\x64\x65\x78\0\x69\x66\x69\x6e\
 +\x64\x65\x78\0\x74\x63\x5f\x69\x6e\x64\x65\x78\0\x63\x62\0\x68\x61\x73\x68\0\
 +\x74\x63\x5f\x63\x6c\x61\x73\x73\x69\x64\0\x64\x61\x74\x61\0\x64\x61\x74\x61\
 +\x5f\x65\x6e\x64\0\x6e\x61\x70\x69\x5f\x69\x64\0\x66\x61\x6d\x69\x6c\x79\0\x72\
 +\x65\x6d\x6f\x74\x65\x5f\x69\x70\x34\0\x6c\x6f\x63\x61\x6c\x5f\x69\x70\x34\0\
 +\x72\x65\x6d\x6f\x74\x65\x5f\x69\x70\x36\0\x6c\x6f\x63\x61\x6c\x5f\x69\x70\x36\
 +\0\x72\x65\x6d\x6f\x74\x65\x5f\x70\x6f\x72\x74\0\x6c\x6f\x63\x61\x6c\x5f\x70\
 +\x6f\x72\x74\0\x64\x61\x74\x61\x5f\x6d\x65\x74\x61\0\x74\x73\x74\x61\x6d\x70\0\
 +\x77\x69\x72\x65\x5f\x6c\x65\x6e\0\x67\x73\x6f\x5f\x73\x65\x67\x73\0\x67\x73\
 +\x6f\x5f\x73\x69\x7a\x65\0\x74\x73\x74\x61\x6d\x70\x5f\x74\x79\x70\x65\0\x68\
 +\x77\x74\x73\x74\x61\x6d\x70\0\x5f\x5f\x75\x33\x32\0\x75\x6e\x73\x69\x67\x6e\
 +\x65\x64\x20\x69\x6e\x74\0\x66\x6c\x6f\x77\x5f\x6b\x65\x79\x73\0\x5f\x5f\x75\
 +\x36\x34\0\x75\x6e\x73\x69\x67\x6e\x65\x64\x20\x6c\x6f\x6e\x67\x20\x6c\x6f\x6e\
 +\x67\0\x73\x6b\0\x5f\x5f\x75\x38\0\x75\x6e\x73\x69\x67\x6e\x65\x64\x20\x63\x68\
 +\x61\x72\0\x73\x6b\x62\0\x74\x75\x6e\x5f\x72\x73\x73\x5f\x73\x74\x65\x65\x72\
 +\x69\x6e\x67\x5f\x70\x72\x6f\x67\0\x73\x6f\x63\x6b\x65\x74\0\x2f\x68\x6f\x6d\
 +\x65\x2f\x61\x6e\x64\x2f\x57\x6f\x72\x6b\x2f\x44\x61\x79\x6e\x69\x78\x2f\x71\
 +\x65\x6d\x75\x2f\x74\x6f\x6f\x6c\x73\x2f\x65\x62\x70\x66\x2f\x72\x73\x73\x2e\
 +\x62\x70\x66\x2e\x63\0\x69\x6e\x74\x20\x74\x75\x6e\x5f\x72\x73\x73\x5f\x73\x74\
 +\x65\x65\x72\x69\x6e\x67\x5f\x70\x72\x6f\x67\x28\x73\x74\x72\x75\x63\x74\x20\
 +\x5f\x5f\x73\x6b\x5f\x62\x75\x66\x66\x20\x2a\x73\x6b\x62\x29\0\x20\x20\x20\x20\
 +\x5f\x5f\x75\x33\x32\x20\x6b\x65\x79\x20\x3d\x20\x30\x3b\0\x20\x20\x20\x20\x63\
 +\x6f\x6e\x66\x69\x67\x20\x3d\x20\x62\x70\x66\x5f\x6d\x61\x70\x5f\x6c\x6f\x6f\
 +\x6b\x75\x70\x5f\x65\x6c\x65\x6d\x28\x26\x74\x61\x70\x5f\x72\x73\x73\x5f\x6d\
 +\x61\x70\x5f\x63\x6f\x6e\x66\x69\x67\x75\x72\x61\x74\x69\x6f\x6e\x73\x2c\x20\
 +\x26\x6b\x65\x79\x29\x3b\0\x20\x20\x20\x20\x74\x6f\x65\x20\x3d\x20\x62\x70\x66\
 +\x5f\x6d\x61\x70\x5f\x6c\x6f\x6f\x6b\x75\x70\x5f\x65\x6c\x65\x6d\x28\x26\x74\
  \x61\x70\x5f\x72\x73\x73\x5f\x6d\x61\x70\x5f\x74\x6f\x65\x70\x6c\x69\x74\x7a\
 -\x5f\x6b\x65\x79\0\x74\x61\x70\x5f\x72\x73\x73\x5f\x6d\x61\x70\x5f\x69\x6e\x64\
 -\x69\x72\x65\x63\x74\x69\x6f\x6e\x5f\x74\x61\x62\x6c\x65\0\x5f\x5f\x73\x6b\x5f\
 -\x62\x75\x66\x66\0\x6c\x65\x6e\0\x70\x6b\x74\x5f\x74\x79\x70\x65\0\x6d\x61\x72\
 -\x6b\0\x71\x75\x65\x75\x65\x5f\x6d\x61\x70\x70\x69\x6e\x67\0\x70\x72\x6f\x74\
 -\x6f\x63\x6f\x6c\0\x76\x6c\x61\x6e\x5f\x70\x72\x65\x73\x65\x6e\x74\0\x76\x6c\
 -\x61\x6e\x5f\x74\x63\x69\0\x76\x6c\x61\x6e\x5f\x70\x72\x6f\x74\x6f\0\x70\x72\
 -\x69\x6f\x72\x69\x74\x79\0\x69\x6e\x67\x72\x65\x73\x73\x5f\x69\x66\x69\x6e\x64\
 -\x65\x78\0\x69\x66\x69\x6e\x64\x65\x78\0\x74\x63\x5f\x69\x6e\x64\x65\x78\0\x63\
 -\x62\0\x68\x61\x73\x68\0\x74\x63\x5f\x63\x6c\x61\x73\x73\x69\x64\0\x64\x61\x74\
 -\x61\0\x64\x61\x74\x61\x5f\x65\x6e\x64\0\x6e\x61\x70\x69\x5f\x69\x64\0\x66\x61\
 -\x6d\x69\x6c\x79\0\x72\x65\x6d\x6f\x74\x65\x5f\x69\x70\x34\0\x6c\x6f\x63\x61\
 -\x6c\x5f\x69\x70\x34\0\x72\x65\x6d\x6f\x74\x65\x5f\x69\x70\x36\0\x6c\x6f\x63\
 -\x61\x6c\x5f\x69\x70\x36\0\x72\x65\x6d\x6f\x74\x65\x5f\x70\x6f\x72\x74\0\x6c\
 -\x6f\x63\x61\x6c\x5f\x70\x6f\x72\x74\0\x64\x61\x74\x61\x5f\x6d\x65\x74\x61\0\
 -\x74\x73\x74\x61\x6d\x70\0\x77\x69\x72\x65\x5f\x6c\x65\x6e\0\x67\x73\x6f\x5f\
 -\x73\x65\x67\x73\0\x67\x73\x6f\x5f\x73\x69\x7a\x65\0\x74\x73\x74\x61\x6d\x70\
 -\x5f\x74\x79\x70\x65\0\x68\x77\x74\x73\x74\x61\x6d\x70\0\x5f\x5f\x75\x33\x32\0\
 -\x75\x6e\x73\x69\x67\x6e\x65\x64\x20\x69\x6e\x74\0\x66\x6c\x6f\x77\x5f\x6b\x65\
 -\x79\x73\0\x5f\x5f\x75\x36\x34\0\x75\x6e\x73\x69\x67\x6e\x65\x64\x20\x6c\x6f\
 -\x6e\x67\x20\x6c\x6f\x6e\x67\0\x73\x6b\0\x5f\x5f\x75\x38\0\x75\x6e\x73\x69\x67\
 -\x6e\x65\x64\x20\x63\x68\x61\x72\0\x73\x6b\x62\0\x74\x75\x6e\x5f\x72\x73\x73\
 -\x5f\x73\x74\x65\x65\x72\x69\x6e\x67\x5f\x70\x72\x6f\x67\0\x74\x75\x6e\x5f\x72\
 -\x73\x73\x5f\x73\x74\x65\x65\x72\x69\x6e\x67\0\x2f\x68\x6f\x6d\x65\x2f\x73\x68\
 -\x72\x65\x65\x73\x68\x2f\x63\x2f\x71\x65\x6d\x75\x2f\x74\x6f\x6f\x6c\x73\x2f\
 -\x65\x62\x70\x66\x2f\x72\x73\x73\x2e\x62\x70\x66\x2e\x63\0\x69\x6e\x74\x20\x74\
 -\x75\x6e\x5f\x72\x73\x73\x5f\x73\x74\x65\x65\x72\x69\x6e\x67\x5f\x70\x72\x6f\
 -\x67\x28\x73\x74\x72\x75\x63\x74\x20\x5f\x5f\x73\x6b\x5f\x62\x75\x66\x66\x20\
 -\x2a\x73\x6b\x62\x29\0\x20\x20\x20\x20\x5f\x5f\x75\x33\x32\x20\x6b\x65\x79\x20\
 -\x3d\x20\x30\x3b\0\x20\x20\x20\x20\x63\x6f\x6e\x66\x69\x67\x20\x3d\x20\x62\x70\
 -\x66\x5f\x6d\x61\x70\x5f\x6c\x6f\x6f\x6b\x75\x70\x5f\x65\x6c\x65\x6d\x28\x26\
 -\x74\x61\x70\x5f\x72\x73\x73\x5f\x6d\x61\x70\x5f\x63\x6f\x6e\x66\x69\x67\x75\
 -\x72\x61\x74\x69\x6f\x6e\x73\x2c\x20\x26\x6b\x65\x79\x29\x3b\0\x20\x20\x20\x20\
 -\x74\x6f\x65\x20\x3d\x20\x62\x70\x66\x5f\x6d\x61\x70\x5f\x6c\x6f\x6f\x6b\x75\
 -\x70\x5f\x65\x6c\x65\x6d\x28\x26\x74\x61\x70\x5f\x72\x73\x73\x5f\x6d\x61\x70\
 -\x5f\x74\x6f\x65\x70\x6c\x69\x74\x7a\x5f\x6b\x65\x79\x2c\x20\x26\x6b\x65\x79\
 -\x29\x3b\0\x20\x20\x20\x20\x69\x66\x20\x28\x63\x6f\x6e\x66\x69\x67\x20\x26\x26\
 -\x20\x74\x6f\x65\x29\x20\x7b\0\x20\x20\x20\x20\x20\x20\x20\x20\x69\x66\x20\x28\
 -\x21\x63\x6f\x6e\x66\x69\x67\x2d\x3e\x72\x65\x64\x69\x72\x65\x63\x74\x29\x20\
 -\x7b\0\x20\x20\x20\x20\x5f\x5f\x75\x38\x20\x72\x73\x73\x5f\x69\x6e\x70\x75\x74\
 -\x5b\x48\x41\x53\x48\x5f\x43\x41\x4c\x43\x55\x4c\x41\x54\x49\x4f\x4e\x5f\x42\
 -\x55\x46\x46\x45\x52\x5f\x53\x49\x5a\x45\x5d\x20\x3d\x20\x7b\x7d\x3b\0\x20\x20\
 -\x20\x20\x73\x74\x72\x75\x63\x74\x20\x70\x61\x63\x6b\x65\x74\x5f\x68\x61\x73\
 -\x68\x5f\x69\x6e\x66\x6f\x5f\x74\x20\x70\x61\x63\x6b\x65\x74\x5f\x69\x6e\x66\
 -\x6f\x20\x3d\x20\x7b\x7d\x3b\0\x20\x20\x20\x20\x69\x66\x20\x28\x21\x69\x6e\x66\
 -\x6f\x20\x7c\x7c\x20\x21\x73\x6b\x62\x29\x20\x7b\0\x20\x20\x20\x20\x5f\x5f\x62\
 -\x65\x31\x36\x20\x72\x65\x74\x20\x3d\x20\x30\x3b\0\x20\x20\x20\x20\x65\x72\x72\
 -\x20\x3d\x20\x62\x70\x66\x5f\x73\x6b\x62\x5f\x6c\x6f\x61\x64\x5f\x62\x79\x74\
 -\x65\x73\x5f\x72\x65\x6c\x61\x74\x69\x76\x65\x28\x73\x6b\x62\x2c\x20\x6f\x66\
 -\x66\x73\x65\x74\x2c\x20\x26\x72\x65\x74\x2c\x20\x73\x69\x7a\x65\x6f\x66\x28\
 -\x72\x65\x74\x29\x2c\0\x20\x20\x20\x20\x69\x66\x20\x28\x65\x72\x72\x29\x20\x7b\
 -\0\x20\x20\x20\x20\x73\x77\x69\x74\x63\x68\x20\x28\x62\x70\x66\x5f\x6e\x74\x6f\
 -\x68\x73\x28\x72\x65\x74\x29\x29\x20\x7b\0\x20\x20\x20\x20\x20\x20\x20\x20\x65\
 -\x72\x72\x20\x3d\x20\x62\x70\x66\x5f\x73\x6b\x62\x5f\x6c\x6f\x61\x64\x5f\x62\
 -\x79\x74\x65\x73\x5f\x72\x65\x6c\x61\x74\x69\x76\x65\x28\x73\x6b\x62\x2c\x20\
 -\x6f\x66\x66\x73\x65\x74\x2c\x20\x26\x72\x65\x74\x2c\x20\x73\x69\x7a\x65\x6f\
 -\x66\x28\x72\x65\x74\x29\x2c\0\x20\x20\x20\x20\x72\x65\x74\x75\x72\x6e\x20\x72\
 -\x65\x74\x3b\0\x20\x20\x20\x20\x69\x66\x20\x28\x6c\x33\x5f\x70\x72\x6f\x74\x6f\
 -\x63\x6f\x6c\x20\x3d\x3d\x20\x30\x29\x20\x7b\0\x20\x20\x20\x20\x20\x20\x20\x20\
 -\x69\x6e\x66\x6f\x2d\x3e\x69\x73\x5f\x69\x70\x76\x34\x20\x3d\x20\x31\x3b\0\x20\
 -\x20\x20\x20\x20\x20\x20\x20\x73\x74\x72\x75\x63\x74\x20\x69\x70\x68\x64\x72\
 -\x20\x69\x70\x20\x3d\x20\x7b\x7d\x3b\0\x20\x20\x20\x20\x20\x20\x20\x20\x65\x72\
 -\x72\x20\x3d\x20\x62\x70\x66\x5f\x73\x6b\x62\x5f\x6c\x6f\x61\x64\x5f\x62\x79\
 -\x74\x65\x73\x5f\x72\x65\x6c\x61\x74\x69\x76\x65\x28\x73\x6b\x62\x2c\x20\x30\
 -\x2c\x20\x26\x69\x70\x2c\x20\x73\x69\x7a\x65\x6f\x66\x28\x69\x70\x29\x2c\0\x20\
 -\x20\x20\x20\x20\x20\x20\x20\x69\x66\x20\x28\x65\x72\x72\x29\x20\x7b\0\x20\x20\
 -\x20\x20\x20\x20\x20\x20\x69\x6e\x66\x6f\x2d\x3e\x69\x73\x5f\x66\x72\x61\x67\
 -\x6d\x65\x6e\x74\x65\x64\x20\x3d\x20\x21\x21\x69\x70\x2e\x66\x72\x61\x67\x5f\
 -\x6f\x66\x66\x3b\0\x20\x20\x20\x20\x20\x20\x20\x20\x69\x6e\x66\x6f\x2d\x3e\x69\
 -\x6e\x5f\x73\x72\x63\x20\x3d\x20\x69\x70\x2e\x73\x61\x64\x64\x72\x3b\0\x20\x20\
 -\x20\x20\x20\x20\x20\x20\x69\x6e\x66\x6f\x2d\x3e\x69\x6e\x5f\x64\x73\x74\x20\
 -\x3d\x20\x69\x70\x2e\x64\x61\x64\x64\x72\x3b\0\x20\x20\x20\x20\x20\x20\x20\x20\
 -\x6c\x34\x5f\x70\x72\x6f\x74\x6f\x63\x6f\x6c\x20\x3d\x20\x69\x70\x2e\x70\x72\
 -\x6f\x74\x6f\x63\x6f\x6c\x3b\0\x20\x20\x20\x20\x20\x20\x20\x20\x6c\x34\x5f\x6f\
 -\x66\x66\x73\x65\x74\x20\x3d\x20\x69\x70\x2e\x69\x68\x6c\x20\x2a\x20\x34\x3b\0\
 -\x20\x20\x20\x20\x69\x66\x20\x28\x6c\x34\x5f\x70\x72\x6f\x74\x6f\x63\x6f\x6c\
 -\x20\x21\x3d\x20\x30\x20\x26\x26\x20\x21\x69\x6e\x66\x6f\x2d\x3e\x69\x73\x5f\
 -\x66\x72\x61\x67\x6d\x65\x6e\x74\x65\x64\x29\x20\x7b\0\x20\x20\x20\x20\x20\x20\
 -\x20\x20\x69\x66\x20\x28\x6c\x34\x5f\x70\x72\x6f\x74\x6f\x63\x6f\x6c\x20\x3d\
 -\x3d\x20\x49\x50\x50\x52\x4f\x54\x4f\x5f\x54\x43\x50\x29\x20\x7b\0\x20\x20\x20\
 -\x20\x20\x20\x20\x20\x20\x20\x20\x20\x69\x6e\x66\x6f\x2d\x3e\x69\x73\x5f\x74\
 -\x63\x70\x20\x3d\x20\x31\x3b\0\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\
 -\x73\x74\x72\x75\x63\x74\x20\x74\x63\x70\x68\x64\x72\x20\x74\x63\x70\x20\x3d\
 -\x20\x7b\x7d\x3b\0\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x65\x72\x72\
 -\x20\x3d\x20\x62\x70\x66\x5f\x73\x6b\x62\x5f\x6c\x6f\x61\x64\x5f\x62\x79\x74\
 -\x65\x73\x5f\x72\x65\x6c\x61\x74\x69\x76\x65\x28\x73\x6b\x62\x2c\x20\x6c\x34\
 -\x5f\x6f\x66\x66\x73\x65\x74\x2c\x20\x26\x74\x63\x70\x2c\x20\x73\x69\x7a\x65\
 -\x6f\x66\x28\x74\x63\x70\x29\x2c\0\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\
 -\x20\x69\x66\x20\x28\x65\x72\x72\x29\x20\x7b\0\x20\x20\x20\x20\x69\x66\x20\x28\
 -\x70\x61\x63\x6b\x65\x74\x5f\x69\x6e\x66\x6f\x2e\x69\x73\x5f\x69\x70\x76\x34\
 -\x29\x20\x7b\0\x20\x20\x20\x20\x20\x20\x20\x20\x69\x66\x20\x28\x70\x61\x63\x6b\
 -\x65\x74\x5f\x69\x6e\x66\x6f\x2e\x69\x73\x5f\x74\x63\x70\x20\x26\x26\0\x20\x20\
 -\x20\x20\x20\x20\x20\x20\x69\x6e\x66\x6f\x2d\x3e\x69\x73\x5f\x69\x70\x76\x36\
 -\x20\x3d\x20\x31\x3b\0\x20\x20\x20\x20\x20\x20\x20\x20\x73\x74\x72\x75\x63\x74\
 -\x20\x69\x70\x76\x36\x68\x64\x72\x20\x69\x70\x36\x20\x3d\x20\x7b\x7d\x3b\0\x20\
 -\x20\x20\x20\x20\x20\x20\x20\x65\x72\x72\x20\x3d\x20\x62\x70\x66\x5f\x73\x6b\
 +\x5f\x6b\x65\x79\x2c\x20\x26\x6b\x65\x79\x29\x3b\0\x20\x20\x20\x20\x69\x66\x20\
 +\x28\x63\x6f\x6e\x66\x69\x67\x20\x26\x26\x20\x74\x6f\x65\x29\x20\x7b\0\x20\x20\
 +\x20\x20\x20\x20\x20\x20\x69\x66\x20\x28\x21\x63\x6f\x6e\x66\x69\x67\x2d\x3e\
 +\x72\x65\x64\x69\x72\x65\x63\x74\x29\x20\x7b\0\x20\x20\x20\x20\x5f\x5f\x75\x38\
 +\x20\x72\x73\x73\x5f\x69\x6e\x70\x75\x74\x5b\x48\x41\x53\x48\x5f\x43\x41\x4c\
 +\x43\x55\x4c\x41\x54\x49\x4f\x4e\x5f\x42\x55\x46\x46\x45\x52\x5f\x53\x49\x5a\
 +\x45\x5d\x20\x3d\x20\x7b\x7d\x3b\0\x20\x20\x20\x20\x73\x74\x72\x75\x63\x74\x20\
 +\x70\x61\x63\x6b\x65\x74\x5f\x68\x61\x73\x68\x5f\x69\x6e\x66\x6f\x5f\x74\x20\
 +\x70\x61\x63\x6b\x65\x74\x5f\x69\x6e\x66\x6f\x20\x3d\x20\x7b\x7d\x3b\0\x20\x20\
 +\x20\x20\x69\x66\x20\x28\x21\x69\x6e\x66\x6f\x20\x7c\x7c\x20\x21\x73\x6b\x62\
 +\x29\x20\x7b\0\x20\x20\x20\x20\x5f\x5f\x62\x65\x31\x36\x20\x72\x65\x74\x20\x3d\
 +\x20\x30\x3b\0\x20\x20\x20\x20\x65\x72\x72\x20\x3d\x20\x62\x70\x66\x5f\x73\x6b\
  \x62\x5f\x6c\x6f\x61\x64\x5f\x62\x79\x74\x65\x73\x5f\x72\x65\x6c\x61\x74\x69\
 -\x76\x65\x28\x73\x6b\x62\x2c\x20\x30\x2c\x20\x26\x69\x70\x36\x2c\x20\x73\x69\
 -\x7a\x65\x6f\x66\x28\x69\x70\x36\x29\x2c\0\x20\x20\x20\x20\x20\x20\x20\x20\x69\
 -\x6e\x66\x6f\x2d\x3e\x69\x6e\x36\x5f\x73\x72\x63\x20\x3d\x20\x69\x70\x36\x2e\
 -\x73\x61\x64\x64\x72\x3b\0\x20\x20\x20\x20\x20\x20\x20\x20\x69\x6e\x66\x6f\x2d\
 -\x3e\x69\x6e\x36\x5f\x64\x73\x74\x20\x3d\x20\x69\x70\x36\x2e\x64\x61\x64\x64\
 +\x76\x65\x28\x73\x6b\x62\x2c\x20\x6f\x66\x66\x73\x65\x74\x2c\x20\x26\x72\x65\
 +\x74\x2c\x20\x73\x69\x7a\x65\x6f\x66\x28\x72\x65\x74\x29\x2c\0\x20\x20\x20\x20\
 +\x69\x66\x20\x28\x65\x72\x72\x29\x20\x7b\0\x20\x20\x20\x20\x73\x77\x69\x74\x63\
 +\x68\x20\x28\x62\x70\x66\x5f\x6e\x74\x6f\x68\x73\x28\x72\x65\x74\x29\x29\x20\
 +\x7b\0\x20\x20\x20\x20\x20\x20\x20\x20\x65\x72\x72\x20\x3d\x20\x62\x70\x66\x5f\
 +\x73\x6b\x62\x5f\x6c\x6f\x61\x64\x5f\x62\x79\x74\x65\x73\x5f\x72\x65\x6c\x61\
 +\x74\x69\x76\x65\x28\x73\x6b\x62\x2c\x20\x6f\x66\x66\x73\x65\x74\x2c\x20\x26\
 +\x72\x65\x74\x2c\x20\x73\x69\x7a\x65\x6f\x66\x28\x72\x65\x74\x29\x2c\0\x20\x20\
 +\x20\x20\x72\x65\x74\x75\x72\x6e\x20\x72\x65\x74\x3b\0\x20\x20\x20\x20\x69\x66\
 +\x20\x28\x6c\x33\x5f\x70\x72\x6f\x74\x6f\x63\x6f\x6c\x20\x3d\x3d\x20\x30\x29\
 +\x20\x7b\0\x20\x20\x20\x20\x20\x20\x20\x20\x69\x6e\x66\x6f\x2d\x3e\x69\x73\x5f\
 +\x69\x70\x76\x34\x20\x3d\x20\x31\x3b\0\x20\x20\x20\x20\x20\x20\x20\x20\x73\x74\
 +\x72\x75\x63\x74\x20\x69\x70\x68\x64\x72\x20\x69\x70\x20\x3d\x20\x7b\x7d\x3b\0\
 +\x20\x20\x20\x20\x20\x20\x20\x20\x65\x72\x72\x20\x3d\x20\x62\x70\x66\x5f\x73\
 +\x6b\x62\x5f\x6c\x6f\x61\x64\x5f\x62\x79\x74\x65\x73\x5f\x72\x65\x6c\x61\x74\
 +\x69\x76\x65\x28\x73\x6b\x62\x2c\x20\x30\x2c\x20\x26\x69\x70\x2c\x20\x73\x69\
 +\x7a\x65\x6f\x66\x28\x69\x70\x29\x2c\0\x20\x20\x20\x20\x20\x20\x20\x20\x69\x66\
 +\x20\x28\x65\x72\x72\x29\x20\x7b\0\x20\x20\x20\x20\x20\x20\x20\x20\x69\x6e\x66\
 +\x6f\x2d\x3e\x69\x73\x5f\x66\x72\x61\x67\x6d\x65\x6e\x74\x65\x64\x20\x3d\x20\
 +\x21\x21\x69\x70\x2e\x66\x72\x61\x67\x5f\x6f\x66\x66\x3b\0\x20\x20\x20\x20\x20\
 +\x20\x20\x20\x69\x6e\x66\x6f\x2d\x3e\x69\x6e\x5f\x73\x72\x63\x20\x3d\x20\x69\
 +\x70\x2e\x73\x61\x64\x64\x72\x3b\0\x20\x20\x20\x20\x20\x20\x20\x20\x69\x6e\x66\
 +\x6f\x2d\x3e\x69\x6e\x5f\x64\x73\x74\x20\x3d\x20\x69\x70\x2e\x64\x61\x64\x64\
  \x72\x3b\0\x20\x20\x20\x20\x20\x20\x20\x20\x6c\x34\x5f\x70\x72\x6f\x74\x6f\x63\
 -\x6f\x6c\x20\x3d\x20\x69\x70\x36\x2e\x6e\x65\x78\x74\x68\x64\x72\x3b\0\x20\x20\
 -\x20\x20\x73\x77\x69\x74\x63\x68\x20\x28\x68\x64\x72\x5f\x74\x79\x70\x65\x29\
 -\x20\x7b\0\x20\x20\x20\x20\x73\x74\x72\x75\x63\x74\x20\x69\x70\x76\x36\x5f\x6f\
 -\x70\x74\x5f\x68\x64\x72\x20\x65\x78\x74\x5f\x68\x64\x72\x20\x3d\x20\x7b\x7d\
 -\x3b\0\x20\x20\x20\x20\x20\x20\x20\x20\x65\x72\x72\x20\x3d\x20\x62\x70\x66\x5f\
 +\x6f\x6c\x20\x3d\x20\x69\x70\x2e\x70\x72\x6f\x74\x6f\x63\x6f\x6c\x3b\0\x20\x20\
 +\x20\x20\x20\x20\x20\x20\x6c\x34\x5f\x6f\x66\x66\x73\x65\x74\x20\x3d\x20\x69\
 +\x70\x2e\x69\x68\x6c\x20\x2a\x20\x34\x3b\0\x20\x20\x20\x20\x69\x66\x20\x28\x6c\
 +\x34\x5f\x70\x72\x6f\x74\x6f\x63\x6f\x6c\x20\x21\x3d\x20\x30\x20\x26\x26\x20\
 +\x21\x69\x6e\x66\x6f\x2d\x3e\x69\x73\x5f\x66\x72\x61\x67\x6d\x65\x6e\x74\x65\
 +\x64\x29\x20\x7b\0\x20\x20\x20\x20\x20\x20\x20\x20\x69\x66\x20\x28\x6c\x34\x5f\
 +\x70\x72\x6f\x74\x6f\x63\x6f\x6c\x20\x3d\x3d\x20\x49\x50\x50\x52\x4f\x54\x4f\
 +\x5f\x54\x43\x50\x29\x20\x7b\0\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\
 +\x69\x6e\x66\x6f\x2d\x3e\x69\x73\x5f\x74\x63\x70\x20\x3d\x20\x31\x3b\0\x20\x20\
 +\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x73\x74\x72\x75\x63\x74\x20\x74\x63\
 +\x70\x68\x64\x72\x20\x74\x63\x70\x20\x3d\x20\x7b\x7d\x3b\0\x20\x20\x20\x20\x20\
 +\x20\x20\x20\x20\x20\x20\x20\x65\x72\x72\x20\x3d\x20\x62\x70\x66\x5f\x73\x6b\
 +\x62\x5f\x6c\x6f\x61\x64\x5f\x62\x79\x74\x65\x73\x5f\x72\x65\x6c\x61\x74\x69\
 +\x76\x65\x28\x73\x6b\x62\x2c\x20\x6c\x34\x5f\x6f\x66\x66\x73\x65\x74\x2c\x20\
 +\x26\x74\x63\x70\x2c\x20\x73\x69\x7a\x65\x6f\x66\x28\x74\x63\x70\x29\x2c\0\x20\
 +\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x69\x66\x20\x28\x65\x72\x72\x29\
 +\x20\x7b\0\x20\x20\x20\x20\x69\x66\x20\x28\x70\x61\x63\x6b\x65\x74\x5f\x69\x6e\
 +\x66\x6f\x2e\x69\x73\x5f\x69\x70\x76\x34\x29\x20\x7b\0\x20\x20\x20\x20\x20\x20\
 +\x20\x20\x69\x66\x20\x28\x70\x61\x63\x6b\x65\x74\x5f\x69\x6e\x66\x6f\x2e\x69\
 +\x73\x5f\x74\x63\x70\x20\x26\x26\0\x20\x20\x20\x20\x20\x20\x20\x20\x69\x6e\x66\
 +\x6f\x2d\x3e\x69\x73\x5f\x69\x70\x76\x36\x20\x3d\x20\x31\x3b\0\x20\x20\x20\x20\
 +\x20\x20\x20\x20\x73\x74\x72\x75\x63\x74\x20\x69\x70\x76\x36\x68\x64\x72\x20\
 +\x69\x70\x36\x20\x3d\x20\x7b\x7d\x3b\0\x20\x20\x20\x20\x20\x20\x20\x20\x65\x72\
 +\x72\x20\x3d\x20\x62\x70\x66\x5f\x73\x6b\x62\x5f\x6c\x6f\x61\x64\x5f\x62\x79\
 +\x74\x65\x73\x5f\x72\x65\x6c\x61\x74\x69\x76\x65\x28\x73\x6b\x62\x2c\x20\x30\
 +\x2c\x20\x26\x69\x70\x36\x2c\x20\x73\x69\x7a\x65\x6f\x66\x28\x69\x70\x36\x29\
 +\x2c\0\x20\x20\x20\x20\x20\x20\x20\x20\x69\x6e\x66\x6f\x2d\x3e\x69\x6e\x36\x5f\
 +\x73\x72\x63\x20\x3d\x20\x69\x70\x36\x2e\x73\x61\x64\x64\x72\x3b\0\x20\x20\x20\
 +\x20\x20\x20\x20\x20\x69\x6e\x66\x6f\x2d\x3e\x69\x6e\x36\x5f\x64\x73\x74\x20\
 +\x3d\x20\x69\x70\x36\x2e\x64\x61\x64\x64\x72\x3b\0\x20\x20\x20\x20\x20\x20\x20\
 +\x20\x6c\x34\x5f\x70\x72\x6f\x74\x6f\x63\x6f\x6c\x20\x3d\x20\x69\x70\x36\x2e\
 +\x6e\x65\x78\x74\x68\x64\x72\x3b\0\x20\x20\x20\x20\x73\x77\x69\x74\x63\x68\x20\
 +\x28\x68\x64\x72\x5f\x74\x79\x70\x65\x29\x20\x7b\0\x20\x20\x20\x20\x73\x74\x72\
 +\x75\x63\x74\x20\x69\x70\x76\x36\x5f\x6f\x70\x74\x5f\x68\x64\x72\x20\x65\x78\
 +\x74\x5f\x68\x64\x72\x20\x3d\x20\x7b\x7d\x3b\0\x20\x20\x20\x20\x20\x20\x20\x20\
 +\x65\x72\x72\x20\x3d\x20\x62\x70\x66\x5f\x73\x6b\x62\x5f\x6c\x6f\x61\x64\x5f\
 +\x62\x79\x74\x65\x73\x5f\x72\x65\x6c\x61\x74\x69\x76\x65\x28\x73\x6b\x62\x2c\
 +\x20\x2a\x6c\x34\x5f\x6f\x66\x66\x73\x65\x74\x2c\x20\x26\x65\x78\x74\x5f\x68\
 +\x64\x72\x2c\0\x20\x20\x20\x20\x20\x20\x20\x20\x69\x66\x20\x28\x2a\x6c\x34\x5f\
 +\x70\x72\x6f\x74\x6f\x63\x6f\x6c\x20\x3d\x3d\x20\x49\x50\x50\x52\x4f\x54\x4f\
 +\x5f\x52\x4f\x55\x54\x49\x4e\x47\x29\x20\x7b\0\x20\x20\x20\x20\x20\x20\x20\x20\
 +\x20\x20\x20\x20\x73\x74\x72\x75\x63\x74\x20\x69\x70\x76\x36\x5f\x72\x74\x5f\
 +\x68\x64\x72\x20\x65\x78\x74\x5f\x72\x74\x20\x3d\x20\x7b\x7d\x3b\0\x20\x20\x20\
 +\x20\x20\x20\x20\x20\x20\x20\x20\x20\x65\x72\x72\x20\x3d\x20\x62\x70\x66\x5f\
  \x73\x6b\x62\x5f\x6c\x6f\x61\x64\x5f\x62\x79\x74\x65\x73\x5f\x72\x65\x6c\x61\
  \x74\x69\x76\x65\x28\x73\x6b\x62\x2c\x20\x2a\x6c\x34\x5f\x6f\x66\x66\x73\x65\
 -\x74\x2c\x20\x26\x65\x78\x74\x5f\x68\x64\x72\x2c\0\x20\x20\x20\x20\x20\x20\x20\
 -\x20\x69\x66\x20\x28\x2a\x6c\x34\x5f\x70\x72\x6f\x74\x6f\x63\x6f\x6c\x20\x3d\
 -\x3d\x20\x49\x50\x50\x52\x4f\x54\x4f\x5f\x52\x4f\x55\x54\x49\x4e\x47\x29\x20\
 -\x7b\0\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x73\x74\x72\x75\x63\x74\
 -\x20\x69\x70\x76\x36\x5f\x72\x74\x5f\x68\x64\x72\x20\x65\x78\x74\x5f\x72\x74\
 -\x20\x3d\x20\x7b\x7d\x3b\0\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x65\
 -\x72\x72\x20\x3d\x20\x62\x70\x66\x5f\x73\x6b\x62\x5f\x6c\x6f\x61\x64\x5f\x62\
 -\x79\x74\x65\x73\x5f\x72\x65\x6c\x61\x74\x69\x76\x65\x28\x73\x6b\x62\x2c\x20\
 -\x2a\x6c\x34\x5f\x6f\x66\x66\x73\x65\x74\x2c\x20\x26\x65\x78\x74\x5f\x72\x74\
 -\x2c\0\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x69\x66\x20\x28\x28\x65\
 -\x78\x74\x5f\x72\x74\x2e\x74\x79\x70\x65\x20\x3d\x3d\x20\x49\x50\x56\x36\x5f\
 -\x53\x52\x43\x52\x54\x5f\x54\x59\x50\x45\x5f\x32\x29\x20\x26\x26\0\x20\x20\x20\
 -\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x2a\x6c\
 -\x34\x5f\x6f\x66\x66\x73\x65\x74\x20\x2b\x20\x6f\x66\x66\x73\x65\x74\x6f\x66\
 -\x28\x73\x74\x72\x75\x63\x74\x20\x72\x74\x32\x5f\x68\x64\x72\x2c\x20\x61\x64\
 -\x64\x72\x29\x2c\0\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\
 -\x20\x65\x72\x72\x20\x3d\x20\x62\x70\x66\x5f\x73\x6b\x62\x5f\x6c\x6f\x61\x64\
 -\x5f\x62\x79\x74\x65\x73\x5f\x72\x65\x6c\x61\x74\x69\x76\x65\x28\x73\x6b\x62\
 -\x2c\0\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x69\x66\
 -\x20\x28\x65\x72\x72\x29\x20\x7b\0\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\
 -\x20\x20\x20\x20\x20\x69\x6e\x66\x6f\x2d\x3e\x69\x73\x5f\x69\x70\x76\x36\x5f\
 -\x65\x78\x74\x5f\x64\x73\x74\x20\x3d\x20\x31\x3b\0\x20\x20\x20\x20\x20\x20\x20\
 -\x20\x20\x20\x20\x20\x7d\x20\x5f\x5f\x61\x74\x74\x72\x69\x62\x75\x74\x65\x5f\
 -\x5f\x28\x28\x70\x61\x63\x6b\x65\x64\x29\x29\x20\x6f\x70\x74\x20\x3d\x20\x7b\
 -\x7d\x3b\0\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x6f\
 -\x70\x74\x5f\x6f\x66\x66\x73\x65\x74\x20\x2b\x3d\x20\x28\x6f\x70\x74\x2e\x74\
 -\x79\x70\x65\x20\x3d\x3d\x20\x49\x50\x56\x36\x5f\x54\x4c\x56\x5f\x50\x41\x44\
 -\x31\x29\x20\x3f\0\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\
 -\x20\x69\x66\x20\x28\x6f\x70\x74\x5f\x6f\x66\x66\x73\x65\x74\x20\x2b\x20\x31\
 -\x20\x3e\x3d\x20\x65\x78\x74\x5f\x68\x64\x72\x2e\x68\x64\x72\x6c\x65\x6e\x20\
 -\x2a\x20\x38\x29\x20\x7b\0\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\
 -\x20\x20\x20\x65\x72\x72\x20\x3d\x20\x62\x70\x66\x5f\x73\x6b\x62\x5f\x6c\x6f\
 -\x61\x64\x5f\x62\x79\x74\x65\x73\x5f\x72\x65\x6c\x61\x74\x69\x76\x65\x28\x73\
 -\x6b\x62\x2c\x20\x2a\x6c\x34\x5f\x6f\x66\x66\x73\x65\x74\x20\x2b\x20\x6f\x70\
 -\x74\x5f\x6f\x66\x66\x73\x65\x74\x2c\0\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\
 -\x20\x20\x20\x20\x20\x20\x69\x66\x20\x28\x6f\x70\x74\x2e\x74\x79\x70\x65\x20\
 -\x3d\x3d\x20\x49\x50\x56\x36\x5f\x54\x4c\x56\x5f\x48\x41\x4f\x29\x20\x7b\0\x20\
 +\x74\x2c\x20\x26\x65\x78\x74\x5f\x72\x74\x2c\0\x20\x20\x20\x20\x20\x20\x20\x20\
 +\x20\x20\x20\x20\x69\x66\x20\x28\x28\x65\x78\x74\x5f\x72\x74\x2e\x74\x79\x70\
 +\x65\x20\x3d\x3d\x20\x49\x50\x56\x36\x5f\x53\x52\x43\x52\x54\x5f\x54\x59\x50\
 +\x45\x5f\x32\x29\x20\x26\x26\0\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\
 +\x20\x20\x20\x20\x20\x20\x20\x20\x2a\x6c\x34\x5f\x6f\x66\x66\x73\x65\x74\x20\
 +\x2b\x20\x6f\x66\x66\x73\x65\x74\x6f\x66\x28\x73\x74\x72\x75\x63\x74\x20\x72\
 +\x74\x32\x5f\x68\x64\x72\x2c\x20\x61\x64\x64\x72\x29\x2c\0\x20\x20\x20\x20\x20\
 +\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x65\x72\x72\x20\x3d\x20\x62\x70\
 +\x66\x5f\x73\x6b\x62\x5f\x6c\x6f\x61\x64\x5f\x62\x79\x74\x65\x73\x5f\x72\x65\
 +\x6c\x61\x74\x69\x76\x65\x28\x73\x6b\x62\x2c\0\x20\x20\x20\x20\x20\x20\x20\x20\
 +\x20\x20\x20\x20\x20\x20\x20\x20\x69\x66\x20\x28\x65\x72\x72\x29\x20\x7b\0\x20\
 +\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x69\x6e\x66\x6f\
 +\x2d\x3e\x69\x73\x5f\x69\x70\x76\x36\x5f\x65\x78\x74\x5f\x64\x73\x74\x20\x3d\
 +\x20\x31\x3b\0\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x7d\x20\x5f\x5f\
 +\x61\x74\x74\x72\x69\x62\x75\x74\x65\x5f\x5f\x28\x28\x70\x61\x63\x6b\x65\x64\
 +\x29\x29\x20\x6f\x70\x74\x20\x3d\x20\x7b\x7d\x3b\0\x20\x20\x20\x20\x20\x20\x20\
 +\x20\x20\x20\x20\x20\x20\x20\x20\x20\x6f\x70\x74\x5f\x6f\x66\x66\x73\x65\x74\
 +\x20\x2b\x3d\x20\x28\x6f\x70\x74\x2e\x74\x79\x70\x65\x20\x3d\x3d\x20\x49\x50\
 +\x56\x36\x5f\x54\x4c\x56\x5f\x50\x41\x44\x31\x29\x20\x3f\0\x20\x20\x20\x20\x20\
 +\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x69\x66\x20\x28\x6f\x70\x74\x5f\
 +\x6f\x66\x66\x73\x65\x74\x20\x2b\x20\x31\x20\x3e\x3d\x20\x65\x78\x74\x5f\x68\
 +\x64\x72\x2e\x68\x64\x72\x6c\x65\x6e\x20\x2a\x20\x38\x29\x20\x7b\0\x20\x20\x20\
 +\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x65\x72\x72\x20\x3d\x20\
 +\x62\x70\x66\x5f\x73\x6b\x62\x5f\x6c\x6f\x61\x64\x5f\x62\x79\x74\x65\x73\x5f\
 +\x72\x65\x6c\x61\x74\x69\x76\x65\x28\x73\x6b\x62\x2c\x20\x2a\x6c\x34\x5f\x6f\
 +\x66\x66\x73\x65\x74\x20\x2b\x20\x6f\x70\x74\x5f\x6f\x66\x66\x73\x65\x74\x2c\0\
 +\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x69\x66\x20\
 +\x28\x6f\x70\x74\x2e\x74\x79\x70\x65\x20\x3d\x3d\x20\x49\x50\x56\x36\x5f\x54\
 +\x4c\x56\x5f\x48\x41\x4f\x29\x20\x7b\0\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\
 +\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x2a\x6c\x34\x5f\x6f\
 +\x66\x66\x73\x65\x74\x20\x2b\x20\x6f\x70\x74\x5f\x6f\x66\x66\x73\x65\x74\0\x20\
  \x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\
 -\x20\x20\x20\x20\x2a\x6c\x34\x5f\x6f\x66\x66\x73\x65\x74\x20\x2b\x20\x6f\x70\
 -\x74\x5f\x6f\x66\x66\x73\x65\x74\0\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\
 -\x20\x20\x20\x20\x20\x20\x20\x20\x20\x65\x72\x72\x20\x3d\x20\x62\x70\x66\x5f\
 -\x73\x6b\x62\x5f\x6c\x6f\x61\x64\x5f\x62\x79\x74\x65\x73\x5f\x72\x65\x6c\x61\
 -\x74\x69\x76\x65\x28\x73\x6b\x62\x2c\0\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\
 -\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x69\x66\x20\x28\x65\x72\x72\x29\x20\
 -\x7b\0\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\
 -\x20\x20\x69\x6e\x66\x6f\x2d\x3e\x69\x73\x5f\x69\x70\x76\x36\x5f\x65\x78\x74\
 -\x5f\x73\x72\x63\x20\x3d\x20\x31\x3b\0\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\
 -\x20\x20\x69\x6e\x66\x6f\x2d\x3e\x69\x73\x5f\x66\x72\x61\x67\x6d\x65\x6e\x74\
 -\x65\x64\x20\x3d\x20\x74\x72\x75\x65\x3b\0\x20\x20\x20\x20\x20\x20\x20\x20\x2a\
 -\x6c\x34\x5f\x6f\x66\x66\x73\x65\x74\x20\x2b\x3d\x20\x28\x65\x78\x74\x5f\x68\
 -\x64\x72\x2e\x68\x64\x72\x6c\x65\x6e\x20\x2b\x20\x31\x29\x20\x2a\x20\x38\x3b\0\
 -\x20\x20\x20\x20\x20\x20\x20\x20\x2a\x6c\x34\x5f\x70\x72\x6f\x74\x6f\x63\x6f\
 -\x6c\x20\x3d\x20\x65\x78\x74\x5f\x68\x64\x72\x2e\x6e\x65\x78\x74\x68\x64\x72\
 -\x3b\0\x20\x20\x20\x20\x66\x6f\x72\x20\x28\x75\x6e\x73\x69\x67\x6e\x65\x64\x20\
 -\x69\x6e\x74\x20\x69\x20\x3d\x20\x30\x3b\x20\x69\x20\x3c\x20\x49\x50\x36\x5f\
 -\x45\x58\x54\x45\x4e\x53\x49\x4f\x4e\x53\x5f\x43\x4f\x55\x4e\x54\x3b\x20\x2b\
 -\x2b\x69\x29\x20\x7b\0\x20\x20\x20\x20\x7d\x20\x65\x6c\x73\x65\x20\x69\x66\x20\
 -\x28\x70\x61\x63\x6b\x65\x74\x5f\x69\x6e\x66\x6f\x2e\x69\x73\x5f\x69\x70\x76\
 -\x36\x29\x20\x7b\0\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x69\x66\x20\
 -\x28\x70\x61\x63\x6b\x65\x74\x5f\x69\x6e\x66\x6f\x2e\x69\x73\x5f\x69\x70\x76\
 -\x36\x5f\x65\x78\x74\x5f\x64\x73\x74\x20\x26\x26\0\x20\x20\x20\x20\x20\x20\x20\
 -\x20\x20\x20\x20\x20\x69\x66\x20\x28\x70\x61\x63\x6b\x65\x74\x5f\x69\x6e\x66\
 -\x6f\x2e\x69\x73\x5f\x69\x70\x76\x36\x5f\x65\x78\x74\x5f\x73\x72\x63\x20\x26\
 -\x26\0\x20\x20\x20\x20\x20\x20\x20\x20\x7d\x20\x65\x6c\x73\x65\x20\x69\x66\x20\
 -\x28\x70\x61\x63\x6b\x65\x74\x5f\x69\x6e\x66\x6f\x2e\x69\x73\x5f\x75\x64\x70\
 -\x20\x26\x26\0\x20\x20\x20\x20\x20\x20\x20\x20\x7d\x20\x65\x6c\x73\x65\x20\x69\
 -\x66\x20\x28\x63\x6f\x6e\x66\x69\x67\x2d\x3e\x68\x61\x73\x68\x5f\x74\x79\x70\
 -\x65\x73\x20\x26\x20\x56\x49\x52\x54\x49\x4f\x5f\x4e\x45\x54\x5f\x52\x53\x53\
 -\x5f\x48\x41\x53\x48\x5f\x54\x59\x50\x45\x5f\x49\x50\x76\x34\x29\x20\x7b\0\x20\
 -\x20\x20\x20\x5f\x5f\x62\x75\x69\x6c\x74\x69\x6e\x5f\x6d\x65\x6d\x63\x70\x79\
 -\x28\x26\x72\x73\x73\x5f\x69\x6e\x70\x75\x74\x5b\x2a\x62\x79\x74\x65\x73\x5f\
 -\x77\x72\x69\x74\x74\x65\x6e\x5d\x2c\x20\x70\x74\x72\x2c\x20\x73\x69\x7a\x65\
 -\x29\x3b\0\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x69\x6e\x66\x6f\x2d\
 -\x3e\x69\x73\x5f\x75\x64\x70\x20\x3d\x20\x31\x3b\0\x20\x20\x20\x20\x20\x20\x20\
 -\x20\x20\x20\x20\x20\x73\x74\x72\x75\x63\x74\x20\x75\x64\x70\x68\x64\x72\x20\
 -\x75\x64\x70\x20\x3d\x20\x7b\x7d\x3b\0\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\
 -\x20\x20\x65\x72\x72\x20\x3d\x20\x62\x70\x66\x5f\x73\x6b\x62\x5f\x6c\x6f\x61\
 -\x64\x5f\x62\x79\x74\x65\x73\x5f\x72\x65\x6c\x61\x74\x69\x76\x65\x28\x73\x6b\
 -\x62\x2c\x20\x6c\x34\x5f\x6f\x66\x66\x73\x65\x74\x2c\x20\x26\x75\x64\x70\x2c\
 -\x20\x73\x69\x7a\x65\x6f\x66\x28\x75\x64\x70\x29\x2c\0\x20\x20\x20\x20\x20\x20\
 +\x65\x72\x72\x20\x3d\x20\x62\x70\x66\x5f\x73\x6b\x62\x5f\x6c\x6f\x61\x64\x5f\
 +\x62\x79\x74\x65\x73\x5f\x72\x65\x6c\x61\x74\x69\x76\x65\x28\x73\x6b\x62\x2c\0\
 +\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\
 +\x20\x69\x66\x20\x28\x65\x72\x72\x29\x20\x7b\0\x20\x20\x20\x20\x20\x20\x20\x20\
 +\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x69\x6e\x66\x6f\x2d\x3e\x69\
 +\x73\x5f\x69\x70\x76\x36\x5f\x65\x78\x74\x5f\x73\x72\x63\x20\x3d\x20\x31\x3b\0\
 +\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x69\x6e\x66\x6f\x2d\x3e\x69\
 +\x73\x5f\x66\x72\x61\x67\x6d\x65\x6e\x74\x65\x64\x20\x3d\x20\x74\x72\x75\x65\
 +\x3b\0\x20\x20\x20\x20\x20\x20\x20\x20\x2a\x6c\x34\x5f\x6f\x66\x66\x73\x65\x74\
 +\x20\x2b\x3d\x20\x28\x65\x78\x74\x5f\x68\x64\x72\x2e\x68\x64\x72\x6c\x65\x6e\
 +\x20\x2b\x20\x31\x29\x20\x2a\x20\x38\x3b\0\x20\x20\x20\x20\x20\x20\x20\x20\x2a\
 +\x6c\x34\x5f\x70\x72\x6f\x74\x6f\x63\x6f\x6c\x20\x3d\x20\x65\x78\x74\x5f\x68\
 +\x64\x72\x2e\x6e\x65\x78\x74\x68\x64\x72\x3b\0\x20\x20\x20\x20\x66\x6f\x72\x20\
 +\x28\x75\x6e\x73\x69\x67\x6e\x65\x64\x20\x69\x6e\x74\x20\x69\x20\x3d\x20\x30\
 +\x3b\x20\x69\x20\x3c\x20\x49\x50\x36\x5f\x45\x58\x54\x45\x4e\x53\x49\x4f\x4e\
 +\x53\x5f\x43\x4f\x55\x4e\x54\x3b\x20\x2b\x2b\x69\x29\x20\x7b\0\x20\x20\x20\x20\
 +\x7d\x20\x65\x6c\x73\x65\x20\x69\x66\x20\x28\x70\x61\x63\x6b\x65\x74\x5f\x69\
 +\x6e\x66\x6f\x2e\x69\x73\x5f\x69\x70\x76\x36\x29\x20\x7b\0\x20\x20\x20\x20\x20\
 +\x20\x20\x20\x20\x20\x20\x20\x69\x66\x20\x28\x70\x61\x63\x6b\x65\x74\x5f\x69\
 +\x6e\x66\x6f\x2e\x69\x73\x5f\x69\x70\x76\x36\x5f\x65\x78\x74\x5f\x64\x73\x74\
 +\x20\x26\x26\0\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x69\x66\x20\x28\
 +\x70\x61\x63\x6b\x65\x74\x5f\x69\x6e\x66\x6f\x2e\x69\x73\x5f\x69\x70\x76\x36\
 +\x5f\x65\x78\x74\x5f\x73\x72\x63\x20\x26\x26\0\x20\x20\x20\x20\x20\x20\x20\x20\
 +\x7d\x20\x65\x6c\x73\x65\x20\x69\x66\x20\x28\x70\x61\x63\x6b\x65\x74\x5f\x69\
 +\x6e\x66\x6f\x2e\x69\x73\x5f\x75\x64\x70\x20\x26\x26\0\x20\x20\x20\x20\x20\x20\
  \x20\x20\x7d\x20\x65\x6c\x73\x65\x20\x69\x66\x20\x28\x63\x6f\x6e\x66\x69\x67\
  \x2d\x3e\x68\x61\x73\x68\x5f\x74\x79\x70\x65\x73\x20\x26\x20\x56\x49\x52\x54\
  \x49\x4f\x5f\x4e\x45\x54\x5f\x52\x53\x53\x5f\x48\x41\x53\x48\x5f\x54\x59\x50\
 -\x45\x5f\x49\x50\x76\x36\x29\x20\x7b\0\x20\x20\x20\x20\x66\x6f\x72\x20\x28\x62\
 -\x79\x74\x65\x20\x3d\x20\x30\x3b\x20\x62\x79\x74\x65\x20\x3c\x20\x48\x41\x53\
 -\x48\x5f\x43\x41\x4c\x43\x55\x4c\x41\x54\x49\x4f\x4e\x5f\x42\x55\x46\x46\x45\
 -\x52\x5f\x53\x49\x5a\x45\x3b\x20\x62\x79\x74\x65\x2b\x2b\x29\x20\x7b\0\x20\x20\
 -\x20\x20\x5f\x5f\x75\x33\x32\x20\x6c\x65\x66\x74\x6d\x6f\x73\x74\x5f\x33\x32\
 -\x5f\x62\x69\x74\x73\x20\x3d\x20\x6b\x65\x79\x2d\x3e\x6c\x65\x66\x74\x6d\x6f\
 -\x73\x74\x5f\x33\x32\x5f\x62\x69\x74\x73\x3b\0\x20\x20\x20\x20\x20\x20\x20\x20\
 -\x5f\x5f\x75\x38\x20\x69\x6e\x70\x75\x74\x5f\x62\x79\x74\x65\x20\x3d\x20\x69\
 -\x6e\x70\x75\x74\x5b\x62\x79\x74\x65\x5d\x3b\0\x20\x20\x20\x20\x20\x20\x20\x20\
 -\x20\x20\x20\x20\x69\x66\x20\x28\x69\x6e\x70\x75\x74\x5f\x62\x79\x74\x65\x20\
 -\x26\x20\x28\x31\x20\x3c\x3c\x20\x37\x29\x29\x20\x7b\0\x20\x20\x20\x20\x20\x20\
 -\x20\x20\x5f\x5f\x75\x38\x20\x6b\x65\x79\x5f\x62\x79\x74\x65\x20\x3d\x20\x6b\
 -\x65\x79\x2d\x3e\x6e\x65\x78\x74\x5f\x62\x79\x74\x65\x5b\x62\x79\x74\x65\x5d\
 -\x3b\0\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\
 -\x20\x20\x28\x6c\x65\x66\x74\x6d\x6f\x73\x74\x5f\x33\x32\x5f\x62\x69\x74\x73\
 -\x20\x3c\x3c\x20\x31\x29\x20\x7c\x20\x28\x28\x6b\x65\x79\x5f\x62\x79\x74\x65\
 -\x20\x26\x20\x28\x31\x20\x3c\x3c\x20\x37\x29\x29\x20\x3e\x3e\x20\x37\x29\x3b\0\
 -\x20\x20\x20\x20\x20\x20\x20\x20\x69\x66\x20\x28\x68\x61\x73\x68\x29\x20\x7b\0\
 -\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x5f\x5f\x75\x33\x32\x20\x74\
 -\x61\x62\x6c\x65\x5f\x69\x64\x78\x20\x3d\x20\x68\x61\x73\x68\x20\x25\x20\x63\
 -\x6f\x6e\x66\x69\x67\x2d\x3e\x69\x6e\x64\x69\x72\x65\x63\x74\x69\x6f\x6e\x73\
 -\x5f\x6c\x65\x6e\x3b\0\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x71\x75\
 -\x65\x75\x65\x20\x3d\x20\x62\x70\x66\x5f\x6d\x61\x70\x5f\x6c\x6f\x6f\x6b\x75\
 -\x70\x5f\x65\x6c\x65\x6d\x28\x26\x74\x61\x70\x5f\x72\x73\x73\x5f\x6d\x61\x70\
 -\x5f\x69\x6e\x64\x69\x72\x65\x63\x74\x69\x6f\x6e\x5f\x74\x61\x62\x6c\x65\x2c\0\
 -\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x69\x66\x20\x28\x71\x75\x65\
 -\x75\x65\x29\x20\x7b\0\x7d\0\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\
 -\x20\x20\x20\x20\x72\x65\x74\x75\x72\x6e\x20\x2a\x71\x75\x65\x75\x65\x3b\0\x63\
 -\x68\x61\x72\0\x5f\x6c\x69\x63\x65\x6e\x73\x65\0\x2e\x6d\x61\x70\x73\0\x6c\x69\
 -\x63\x65\x6e\x73\x65\0\x62\x70\x66\x5f\x66\x6c\x6f\x77\x5f\x6b\x65\x79\x73\0\
 -\x62\x70\x66\x5f\x73\x6f\x63\x6b\0\0\0\0\x9f\xeb\x01\0\x20\0\0\0\0\0\0\0\x14\0\
 -\0\0\x14\0\0\0\x6c\x0c\0\0\x80\x0c\0\0\0\0\0\0\x08\0\0\0\x26\x02\0\0\x01\0\0\0\
 -\0\0\0\0\x24\0\0\0\x10\0\0\0\x26\x02\0\0\xc6\0\0\0\0\0\0\0\x37\x02\0\0\x61\x02\
 -\0\0\0\x50\x08\0\x10\0\0\0\x37\x02\0\0\x92\x02\0\0\x0b\x68\x08\0\x20\0\0\0\x37\
 -\x02\0\0\0\0\0\0\0\0\0\0\x28\0\0\0\x37\x02\0\0\xa5\x02\0\0\x0e\x74\x08\0\x50\0\
 -\0\0\x37\x02\0\0\xea\x02\0\0\x0b\x78\x08\0\x88\0\0\0\x37\x02\0\0\x2a\x03\0\0\
 -\x10\x80\x08\0\x90\0\0\0\x37\x02\0\0\0\0\0\0\0\0\0\0\x98\0\0\0\x37\x02\0\0\x2a\
 -\x03\0\0\x10\x80\x08\0\xa0\0\0\0\x37\x02\0\0\x43\x03\0\0\x16\x84\x08\0\xa8\0\0\
 -\0\x37\x02\0\0\x43\x03\0\0\x0d\x84\x08\0\xc0\0\0\0\x37\x02\0\0\x64\x03\0\0\x0a\
 -\xfc\x05\0\xe8\0\0\0\x37\x02\0\0\x9b\x03\0\0\x1f\x0c\x06\0\x38\x01\0\0\x37\x02\
 -\0\0\xcb\x03\0\0\x0f\xa0\x04\0\x40\x01\0\0\x37\x02\0\0\xe4\x03\0\0\x0c\x20\x04\
 -\0\x50\x01\0\0\x37\x02\0\0\0\0\0\0\0\0\0\0\x58\x01\0\0\x37\x02\0\0\xf8\x03\0\0\
 -\x0b\x2c\x04\0\x90\x01\0\0\x37\x02\0\0\x3e\x04\0\0\x09\x34\x04\0\xa0\x01\0\0\
 -\x37\x02\0\0\x4d\x04\0\0\x0d\x44\x04\0\xb8\x01\0\0\x37\x02\0\0\x4d\x04\0\0\x05\
 -\x44\x04\0\xd8\x01\0\0\x37\x02\0\0\0\0\0\0\0\0\0\0\xe0\x01\0\0\x37\x02\0\0\x6b\
 -\x04\0\0\x0f\x58\x04\0\x10\x02\0\0\x37\x02\0\0\x3e\x04\0\0\x09\x70\x04\0\x18\
 -\x02\0\0\x37\x02\0\0\xb5\x04\0\0\x0c\x80\x04\0\x20\x02\0\0\x37\x02\0\0\xc5\x04\
 -\0\0\x09\xbc\x04\0\x50\x02\0\0\x37\x02\0\0\xe1\x04\0\0\x17\xd4\x04\0\x60\x02\0\
 -\0\x37\x02\0\0\xfc\x04\0\0\x16\xdc\x04\0\x80\x02\0\0\x37\x02\0\0\xe1\x04\0\0\
 -\x17\xd4\x04\0\x88\x02\0\0\x37\x02\0\0\x1a\x05\0\0\x0f\xe0\x04\0\xc0\x02\0\0\
 -\x37\x02\0\0\x5d\x05\0\0\x0d\xe8\x04\0\xc8\x02\0\0\x37\x02\0\0\x70\x05\0\0\x24\
 -\0\x05\0\xd0\x02\0\0\x37\x02\0\0\x70\x05\0\0\x20\0\x05\0\xe0\x02\0\0\x37\x02\0\
 -\0\x9d\x05\0\0\x1b\xf8\x04\0\xe8\x02\0\0\x37\x02\0\0\x9d\x05\0\0\x16\xf8\x04\0\
 -\xf0\x02\0\0\x37\x02\0\0\xbe\x05\0\0\x1b\xfc\x04\0\xf8\x02\0\0\x37\x02\0\0\xbe\
 -\x05\0\0\x16\xfc\x04\0\0\x03\0\0\x37\x02\0\0\xdf\x05\0\0\x1a\x08\x05\0\x08\x03\
 -\0\0\x37\x02\0\0\x70\x05\0\0\x1d\0\x05\0\x10\x03\0\0\x37\x02\0\0\x02\x06\0\0\
 -\x18\x0c\x05\0\x18\x03\0\0\x37\x02\0\0\x02\x06\0\0\x1c\x0c\x05\0\x30\x03\0\0\
 -\x37\x02\0\0\x22\x06\0\0\x15\x68\x05\0\x40\x03\0\0\x37\x02\0\0\x22\x06\0\0\x1a\
 -\x68\x05\0\x58\x03\0\0\x37\x02\0\0\x56\x06\0\0\x0d\x6c\x05\0\x78\x03\0\0\x37\
 -\x02\0\0\x80\x06\0\0\x1a\x70\x05\0\x88\x03\0\0\x37\x02\0\0\x9e\x06\0\0\x1b\x78\
 -\x05\0\xa8\x03\0\0\x37\x02\0\0\x80\x06\0\0\x1a\x70\x05\0\xb0\x03\0\0\x37\x02\0\
 -\0\xc2\x06\0\0\x13\x7c\x05\0\xe8\x03\0\0\x37\x02\0\0\x13\x07\0\0\x11\x84\x05\0\
 -\xf0\x03\0\0\x37\x02\0\0\0\0\0\0\0\0\0\0\x10\x04\0\0\x37\x02\0\0\x2a\x07\0\0\
 -\x15\x28\x06\0\x18\x04\0\0\x37\x02\0\0\x2a\x07\0\0\x09\x28\x06\0\x20\x04\0\0\
 -\x37\x02\0\0\0\0\0\0\0\0\0\0\x70\x04\0\0\x37\x02\0\0\x49\x07\0\0\x19\x2c\x06\0\
 -\x80\x04\0\0\x37\x02\0\0\x49\x07\0\0\x20\x2c\x06\0\xa0\x04\0\0\x37\x02\0\0\0\0\
 -\0\0\0\0\0\0\xf0\x04\0\0\x37\x02\0\0\x6b\x07\0\0\x17\x14\x05\0\0\x05\0\0\x37\
 -\x02\0\0\x86\x07\0\0\x18\x1c\x05\0\x30\x05\0\0\x37\x02\0\0\x6b\x07\0\0\x17\x14\
 -\x05\0\x48\x05\0\0\x37\x02\0\0\xa7\x07\0\0\x0f\x20\x05\0\x80\x05\0\0\x37\x02\0\
 -\0\x5d\x05\0\0\x0d\x28\x05\0\x88\x05\0\0\x37\x02\0\0\xec\x07\0\0\x1d\x38\x05\0\
 -\xc8\x05\0\0\x37\x02\0\0\x0f\x08\0\0\x1d\x3c\x05\0\x08\x06\0\0\x37\x02\0\0\x32\
 -\x08\0\0\x1b\x44\x05\0\x10\x06\0\0\x37\x02\0\0\x55\x08\0\0\x05\x30\x02\0\x58\
 -\x06\0\0\x37\x02\0\0\x6d\x08\0\0\x19\xb8\x02\0\xd0\x06\0\0\x37\x02\0\0\0\0\0\0\
 -\0\0\0\0\xd8\x06\0\0\x37\x02\0\0\x93\x08\0\0\x0f\xc8\x02\0\x10\x07\0\0\x37\x02\
 -\0\0\x5d\x05\0\0\x0d\xd0\x02\0\x20\x07\0\0\x37\x02\0\0\xd8\x08\0\0\x0d\xe0\x02\
 -\0\x40\x07\0\0\x37\x02\0\0\x07\x09\0\0\x20\xe4\x02\0\x68\x07\0\0\x37\x02\0\0\
 -\x33\x09\0\0\x13\xec\x02\0\xa8\x07\0\0\x37\x02\0\0\x13\x07\0\0\x11\xf4\x02\0\
 -\xb0\x07\0\0\x37\x02\0\0\x7b\x09\0\0\x19\x04\x03\0\xb8\x07\0\0\x37\x02\0\0\x7b\
 -\x09\0\0\x34\x04\x03\0\xe0\x07\0\0\x37\x02\0\0\xb1\x09\0\0\x15\x18\x03\0\xf0\
 -\x07\0\0\x37\x02\0\0\xf2\x09\0\0\x17\x14\x03\0\x30\x08\0\0\x37\x02\0\0\x29\x0a\
 -\0\0\x15\x24\x03\0\x38\x08\0\0\x37\x02\0\0\x44\x0a\0\0\x27\x34\x03\0\x70\x08\0\
 -\0\x37\x02\0\0\x6f\x0a\0\0\x27\x50\x03\0\x80\x08\0\0\x37\x02\0\0\x9f\x0a\0\0\
 -\x1c\xb4\x03\0\x88\x08\0\0\x37\x02\0\0\xdb\x0a\0\0\x20\xc0\x03\0\x98\x08\0\0\
 -\x37\x02\0\0\xdb\x0a\0\0\x2f\xc0\x03\0\xa0\x08\0\0\x37\x02\0\0\xdb\x0a\0\0\x36\
 -\xc0\x03\0\xa8\x08\0\0\x37\x02\0\0\xdb\x0a\0\0\x15\xc0\x03\0\x18\x09\0\0\x37\
 -\x02\0\0\x17\x0b\0\0\x43\x64\x03\0\x38\x09\0\0\x37\x02\0\0\0\0\0\0\0\0\0\0\x40\
 -\x09\0\0\x37\x02\0\0\x17\x0b\0\0\x17\x64\x03\0\x80\x09\0\0\x37\x02\0\0\x29\x0a\
 -\0\0\x15\x6c\x03\0\x88\x09\0\0\x37\x02\0\0\x67\x0b\0\0\x19\x7c\x03\0\x90\x09\0\
 -\0\x37\x02\0\0\x67\x0b\0\0\x15\x7c\x03\0\x98\x09\0\0\x37\x02\0\0\x97\x0b\0\0\
 -\x19\x84\x03\0\xa0\x09\0\0\x37\x02\0\0\xc7\x0b\0\0\x1b\x80\x03\0\xe8\x09\0\0\
 -\x37\x02\0\0\x02\x0c\0\0\x19\x94\x03\0\xf0\x09\0\0\x37\x02\0\0\x21\x0c\0\0\x2b\
 -\xa4\x03\0\x10\x0a\0\0\x37\x02\0\0\x9f\x0a\0\0\x1f\xb4\x03\0\x30\x0a\0\0\x37\
 -\x02\0\0\x50\x0c\0\0\x21\xd4\x03\0\x40\x0a\0\0\x37\x02\0\0\x78\x0c\0\0\x20\xe4\
 -\x03\0\x48\x0a\0\0\x37\x02\0\0\x78\x0c\0\0\x2c\xe4\x03\0\x60\x0a\0\0\x37\x02\0\
 -\0\x78\x0c\0\0\x14\xe4\x03\0\x70\x0a\0\0\x37\x02\0\0\xa8\x0c\0\0\x20\xe0\x03\0\
 -\x80\x0a\0\0\x37\x02\0\0\x55\x08\0\0\x05\x30\x02\0\xb0\x0a\0\0\x37\x02\0\0\xd0\
 -\x0c\0\0\x38\xc0\x02\0\xd0\x0a\0\0\x37\x02\0\0\xd0\x0c\0\0\x05\xc0\x02\0\xe8\
 -\x0a\0\0\x37\x02\0\0\x55\x08\0\0\x05\x30\x02\0\xf8\x0a\0\0\x37\x02\0\0\x0e\x0d\
 -\0\0\x1c\xc4\x06\0\x08\x0b\0\0\x37\x02\0\0\x0e\x0d\0\0\x10\xc4\x06\0\x10\x0b\0\
 -\0\x37\x02\0\0\0\0\0\0\0\0\0\0\x60\x0b\0\0\x37\x02\0\0\x49\x07\0\0\x19\xc8\x06\
 -\0\x68\x0b\0\0\x37\x02\0\0\x49\x07\0\0\x20\xc8\x06\0\xa0\x0b\0\0\x37\x02\0\0\
 -\x34\x0d\0\0\x2d\0\x07\0\xb0\x0b\0\0\x37\x02\0\0\x34\x0d\0\0\x1d\0\x07\0\xb8\
 -\x0b\0\0\x37\x02\0\0\x34\x0d\0\0\x2d\0\x07\0\xc8\x0b\0\0\x37\x02\0\0\x63\x0d\0\
 -\0\x2d\xd4\x06\0\xf8\x0b\0\0\x37\x02\0\0\x63\x0d\0\0\x1d\xd4\x06\0\x08\x0c\0\0\
 -\x37\x02\0\0\x63\x0d\0\0\x2d\xd4\x06\0\x18\x0c\0\0\x37\x02\0\0\0\0\0\0\0\0\0\0\
 -\xe8\x0c\0\0\x37\x02\0\0\x92\x0d\0\0\x20\x68\x06\0\xf0\x0c\0\0\x37\x02\0\0\x92\
 -\x0d\0\0\x27\x68\x06\0\x18\x0d\0\0\x37\x02\0\0\xbb\x0d\0\0\x27\xa4\x06\0\x20\
 -\x0d\0\0\x37\x02\0\0\xbb\x0d\0\0\x14\xa4\x06\0\x28\x0d\0\0\x37\x02\0\0\x04\x0e\
 -\0\0\x05\x98\x01\0\x38\x0d\0\0\x37\x02\0\0\x04\x0e\0\0\x05\x98\x01\0\x60\x0d\0\
 -\0\x37\x02\0\0\0\0\0\0\0\0\0\0\x70\x0d\0\0\x37\x02\0\0\0\0\0\0\0\0\0\0\x80\x0d\
 -\0\0\x37\x02\0\0\x92\x0d\0\0\x20\x44\x07\0\x88\x0d\0\0\x37\x02\0\0\x92\x0d\0\0\
 -\x27\x44\x07\0\xc0\x0d\0\0\x37\x02\0\0\x34\x0d\0\0\x2d\x7c\x07\0\xd0\x0d\0\0\
 -\x37\x02\0\0\x34\x0d\0\0\x1d\x7c\x07\0\xd8\x0d\0\0\x37\x02\0\0\x34\x0d\0\0\x2d\
 -\x7c\x07\0\xe8\x0d\0\0\x37\x02\0\0\x63\x0d\0\0\x2d\x50\x07\0\x18\x0e\0\0\x37\
 -\x02\0\0\x63\x0d\0\0\x1d\x50\x07\0\x28\x0e\0\0\x37\x02\0\0\x63\x0d\0\0\x2d\x50\
 -\x07\0\x40\x0e\0\0\x37\x02\0\0\x41\x0e\0\0\x1a\xa0\x05\0\x50\x0e\0\0\x37\x02\0\
 -\0\x5f\x0e\0\0\x1b\xa8\x05\0\x60\x0e\0\0\x37\x02\0\0\x41\x0e\0\0\x1a\xa0\x05\0\
 -\x68\x0e\0\0\x37\x02\0\0\x83\x0e\0\0\x13\xac\x05\0\xa0\x0e\0\0\x37\x02\0\0\x13\
 -\x07\0\0\x11\xb4\x05\0\xb0\x0e\0\0\x37\x02\0\0\x55\x08\0\0\x05\x30\x02\0\xc0\
 -\x0e\0\0\x37\x02\0\0\xd4\x0e\0\0\x27\xc8\x07\0\xd0\x0e\0\0\x37\x02\0\0\xd4\x0e\
 -\0\0\x14\xc8\x07\0\xf0\x0e\0\0\x37\x02\0\0\x63\x0d\0\0\x2d\xcc\x07\0\0\x0f\0\0\
 -\x37\x02\0\0\x63\x0d\0\0\x1d\xcc\x07\0\x08\x0f\0\0\x37\x02\0\0\x63\x0d\0\0\x2d\
 -\xcc\x07\0\x30\x0f\0\0\x37\x02\0\0\0\0\0\0\0\0\0\0\x80\x0f\0\0\x37\x02\0\0\x34\
 -\x0d\0\0\x1d\xf8\x07\0\x88\x0f\0\0\x37\x02\0\0\x34\x0d\0\0\x2d\xf8\x07\0\x98\
 -\x0f\0\0\x37\x02\0\0\x04\x0e\0\0\x05\x98\x01\0\xf0\x0f\0\0\x37\x02\0\0\x04\x0e\
 -\0\0\x05\x98\x01\0\x30\x10\0\0\x37\x02\0\0\0\0\0\0\0\0\0\0\x48\x10\0\0\x37\x02\
 -\0\0\x1d\x0f\0\0\x05\xd0\x01\0\x50\x10\0\0\x37\x02\0\0\x5f\x0f\0\0\x23\xc4\x01\
 -\0\x68\x10\0\0\x37\x02\0\0\0\0\0\0\0\0\0\0\x70\x10\0\0\x37\x02\0\0\x93\x0f\0\0\
 -\x1b\xd4\x01\0\x90\x10\0\0\x37\x02\0\0\xba\x0f\0\0\x11\xe8\x01\0\xa8\x10\0\0\
 -\x37\x02\0\0\xe3\x0f\0\0\x19\xd8\x01\0\xc0\x10\0\0\x37\x02\0\0\x11\x10\0\0\x27\
 -\xfc\x01\0\xc8\x10\0\0\x37\x02\0\0\x11\x10\0\0\x46\xfc\x01\0\xd8\x10\0\0\x37\
 -\x02\0\0\x11\x10\0\0\x2d\xfc\x01\0\xe0\x10\0\0\x37\x02\0\0\xba\x0f\0\0\x11\xe8\
 -\x01\0\x08\x11\0\0\x37\x02\0\0\x11\x10\0\0\x46\xfc\x01\0\x20\x11\0\0\x37\x02\0\
 -\0\x11\x10\0\0\x27\xfc\x01\0\x28\x11\0\0\x37\x02\0\0\x11\x10\0\0\x2d\xfc\x01\0\
 -\x30\x11\0\0\x37\x02\0\0\xba\x0f\0\0\x11\xe8\x01\0\x58\x11\0\0\x37\x02\0\0\x11\
 -\x10\0\0\x27\xfc\x01\0\x60\x11\0\0\x37\x02\0\0\x11\x10\0\0\x46\xfc\x01\0\x78\
 -\x11\0\0\x37\x02\0\0\x11\x10\0\0\x2d\xfc\x01\0\x80\x11\0\0\x37\x02\0\0\xba\x0f\
 -\0\0\x11\xe8\x01\0\xa8\x11\0\0\x37\x02\0\0\x11\x10\0\0\x27\xfc\x01\0\xb0\x11\0\
 -\0\x37\x02\0\0\x11\x10\0\0\x46\xfc\x01\0\xc8\x11\0\0\x37\x02\0\0\x11\x10\0\0\
 -\x2d\xfc\x01\0\xd0\x11\0\0\x37\x02\0\0\xba\x0f\0\0\x11\xe8\x01\0\xf8\x11\0\0\
 -\x37\x02\0\0\x11\x10\0\0\x46\xfc\x01\0\x10\x12\0\0\x37\x02\0\0\x11\x10\0\0\x27\
 -\xfc\x01\0\x18\x12\0\0\x37\x02\0\0\x11\x10\0\0\x2d\xfc\x01\0\x20\x12\0\0\x37\
 -\x02\0\0\xba\x0f\0\0\x11\xe8\x01\0\x48\x12\0\0\x37\x02\0\0\x11\x10\0\0\x46\xfc\
 -\x01\0\x60\x12\0\0\x37\x02\0\0\x11\x10\0\0\x27\xfc\x01\0\x68\x12\0\0\x37\x02\0\
 -\0\x11\x10\0\0\x2d\xfc\x01\0\x70\x12\0\0\x37\x02\0\0\xba\x0f\0\0\x11\xe8\x01\0\
 -\x98\x12\0\0\x37\x02\0\0\x11\x10\0\0\x46\xfc\x01\0\xb0\x12\0\0\x37\x02\0\0\x11\
 -\x10\0\0\x27\xfc\x01\0\xb8\x12\0\0\x37\x02\0\0\x11\x10\0\0\x2d\xfc\x01\0\xc0\
 -\x12\0\0\x37\x02\0\0\xba\x0f\0\0\x11\xe8\x01\0\xe0\x12\0\0\x37\x02\0\0\x11\x10\
 -\0\0\x46\xfc\x01\0\xe8\x12\0\0\x37\x02\0\0\x11\x10\0\0\x27\xfc\x01\0\xf0\x12\0\
 -\0\x37\x02\0\0\x11\x10\0\0\x2d\xfc\x01\0\xf8\x12\0\0\x37\x02\0\0\x1d\x0f\0\0\
 -\x3d\xd0\x01\0\x08\x13\0\0\x37\x02\0\0\x1d\x0f\0\0\x05\xd0\x01\0\x18\x13\0\0\
 -\x37\x02\0\0\x5d\x10\0\0\x0d\x98\x08\0\x30\x13\0\0\x37\x02\0\0\x5d\x10\0\0\x0d\
 -\x98\x08\0\x38\x13\0\0\x37\x02\0\0\x71\x10\0\0\x2e\x9c\x08\0\x58\x13\0\0\x37\
 -\x02\0\0\x71\x10\0\0\x24\x9c\x08\0\x70\x13\0\0\x37\x02\0\0\x71\x10\0\0\x13\x9c\
 -\x08\0\x80\x13\0\0\x37\x02\0\0\x71\x10\0\0\x2e\x9c\x08\0\x88\x13\0\0\x37\x02\0\
 -\0\xb0\x10\0\0\x15\xa8\x08\0\xa0\x13\0\0\x37\x02\0\0\xf8\x10\0\0\x11\xb4\x08\0\
 -\xa8\x13\0\0\x37\x02\0\0\0\0\0\0\0\0\0\0\xc8\x13\0\0\x37\x02\0\0\x11\x11\0\0\
 -\x01\xd8\x08\0\xd0\x13\0\0\x37\x02\0\0\x13\x11\0\0\x18\xb8\x08\0\0\0\0\0\0\0\0\
 -\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x03\0\x03\0\0\0\0\0\0\0\0\0\0\0\0\0\
 -\0\0\0\0\xde\0\0\0\0\0\x03\0\xc8\x13\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x7a\x01\0\0\0\
 -\0\x03\0\xb8\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xff\0\0\0\0\0\x03\0\xa8\x13\0\0\0\0\
 -\0\0\0\0\0\0\0\0\0\0\xc7\0\0\0\0\0\x03\0\xd0\x01\0\0\0\0\0\0\0\0\0\0\0\0\0\0\
 -\x2c\x02\0\0\0\0\x03\0\x20\x02\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xf7\0\0\0\0\0\x03\0\
 -\xe8\x04\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x1c\x02\0\0\0\0\x03\0\x10\x04\0\0\0\0\0\0\
 -\0\0\0\0\0\0\0\0\x28\x01\0\0\0\0\x03\0\xe0\x02\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xf3\
 -\x01\0\0\0\0\x03\0\x30\x03\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xeb\x01\0\0\0\0\x03\0\
 -\x38\x0e\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x44\x02\0\0\0\0\x03\0\xf0\x03\0\0\0\0\0\0\
 -\0\0\0\0\0\0\0\0\xe3\x01\0\0\0\0\x03\0\xf8\x0a\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x18\
 -\x01\0\0\0\0\x03\0\xe8\x0c\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x30\x01\0\0\0\0\x03\0\
 -\xa0\x04\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xa9\x01\0\0\0\0\x03\0\x40\x10\0\0\0\0\0\0\
 -\0\0\0\0\0\0\0\0\x51\x01\0\0\0\0\x03\0\x78\x0d\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x5c\
 -\x02\0\0\0\0\x03\0\xb0\x0e\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x03\x02\0\0\0\0\x03\0\
 -\x50\x06\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xc2\x01\0\0\0\0\x03\0\xc0\x06\0\0\0\0\0\0\
 -\0\0\0\0\0\0\0\0\x69\x01\0\0\0\0\x03\0\x20\x07\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x61\
 -\x01\0\0\0\0\x03\0\x60\x08\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x38\x01\0\0\0\0\x03\0\
 -\x30\x0a\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x20\x01\0\0\0\0\x03\0\x40\x0a\0\0\0\0\0\0\
 -\0\0\0\0\0\0\0\0\xba\x01\0\0\0\0\x03\0\xe0\x0f\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xa1\
 -\x01\0\0\0\0\x03\0\x48\x08\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x40\x01\0\0\0\0\x03\0\
 -\x18\x09\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xfb\x01\0\0\0\0\x03\0\x80\x08\0\0\0\0\0\0\
 -\0\0\0\0\0\0\0\0\x99\x01\0\0\0\0\x03\0\xf8\x08\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x59\
 -\x01\0\0\0\0\x03\0\x50\x0d\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x54\x02\0\0\0\0\x03\0\
 -\x08\x0a\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xef\0\0\0\0\0\x03\0\xe8\x0a\0\0\0\0\0\0\0\
 -\0\0\0\0\0\0\0\x4c\x02\0\0\0\0\x03\0\xb0\x0a\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x24\
 -\x02\0\0\0\0\x03\0\xd8\x0a\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x89\x01\0\0\0\0\x03\0\
 -\x80\x0d\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x10\x01\0\0\0\0\x03\0\xb0\x0b\0\0\0\0\0\0\
 -\0\0\0\0\0\0\0\0\xd6\0\0\0\0\0\x03\0\xc8\x0b\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x14\
 -\x02\0\0\0\0\x03\0\xf8\x0b\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xb2\x01\0\0\0\0\x03\0\
 -\x18\x0c\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xdb\x01\0\0\0\0\x03\0\x10\x0c\0\0\0\0\0\0\
 -\0\0\0\0\0\0\0\0\x3c\x02\0\0\0\0\x03\0\x18\x0d\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x91\
 -\x01\0\0\0\0\x03\0\x60\x0d\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x81\x01\0\0\0\0\x03\0\
 -\xc0\x0e\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xe7\0\0\0\0\0\x03\0\xd0\x0d\0\0\0\0\0\0\0\
 -\0\0\0\0\0\0\0\x34\x02\0\0\0\0\x03\0\xe8\x0d\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xd3\
 -\x01\0\0\0\0\x03\0\x18\x0e\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x08\x01\0\0\0\0\x03\0\0\
 -\x0f\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xce\0\0\0\0\0\x03\0\x18\x0f\0\0\0\0\0\0\0\0\0\
 -\0\0\0\0\0\x0b\x02\0\0\0\0\x03\0\xf0\x0f\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xca\x01\0\
 -\0\0\0\x03\0\x30\x10\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x71\x01\0\0\0\0\x03\0\x60\x10\
 -\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x48\x01\0\0\0\0\x03\0\x18\x13\0\0\0\0\0\0\0\0\0\0\
 -\0\0\0\0\x64\x02\0\0\0\0\x03\0\xd0\x13\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x4e\0\0\0\
 -\x12\0\x03\0\0\0\0\0\0\0\0\0\xe0\x13\0\0\0\0\0\0\x33\0\0\0\x11\0\x05\0\0\0\0\0\
 -\0\0\0\0\x20\0\0\0\0\0\0\0\x01\0\0\0\x11\0\x05\0\x20\0\0\0\0\0\0\0\x20\0\0\0\0\
 -\0\0\0\x90\0\0\0\x11\0\x05\0\x40\0\0\0\0\0\0\0\x20\0\0\0\0\0\0\0\x87\0\0\0\x11\
 -\0\x06\0\0\0\0\0\0\0\0\0\x07\0\0\0\0\0\0\0\x28\0\0\0\0\0\0\0\x01\0\0\0\x37\0\0\
 -\0\x50\0\0\0\0\0\0\0\x01\0\0\0\x38\0\0\0\x88\x13\0\0\0\0\0\0\x01\0\0\0\x39\0\0\
 -\0\xd8\x04\0\0\0\0\0\0\x04\0\0\0\x37\0\0\0\xe4\x04\0\0\0\0\0\0\x04\0\0\0\x38\0\
 -\0\0\xf0\x04\0\0\0\0\0\0\x04\0\0\0\x39\0\0\0\x08\x05\0\0\0\0\0\0\x04\0\0\0\x3a\
 -\0\0\0\x2c\0\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x40\0\0\0\0\0\0\0\x04\0\0\0\x01\0\
 -\0\0\x50\0\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x60\0\0\0\0\0\0\0\x04\0\0\0\x01\0\0\
 -\0\x70\0\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x80\0\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\
 -\x90\0\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xa0\0\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\
 -\xb0\0\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xc0\0\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\
 -\xd0\0\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xe0\0\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\
 -\xf0\0\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\0\x01\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\
 -\x10\x01\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x20\x01\0\0\0\0\0\0\x04\0\0\0\x01\0\0\
 -\0\x30\x01\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x40\x01\0\0\0\0\0\0\x04\0\0\0\x01\0\
 -\0\0\x50\x01\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x60\x01\0\0\0\0\0\0\x04\0\0\0\x01\
 -\0\0\0\x70\x01\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x80\x01\0\0\0\0\0\0\x04\0\0\0\
 -\x01\0\0\0\x90\x01\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xa0\x01\0\0\0\0\0\0\x04\0\0\
 -\0\x01\0\0\0\xb0\x01\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xc0\x01\0\0\0\0\0\0\x04\0\
 -\0\0\x01\0\0\0\xd0\x01\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xe0\x01\0\0\0\0\0\0\x04\
 -\0\0\0\x01\0\0\0\xf0\x01\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\0\x02\0\0\0\0\0\0\x04\
 -\0\0\0\x01\0\0\0\x10\x02\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x20\x02\0\0\0\0\0\0\
 -\x04\0\0\0\x01\0\0\0\x30\x02\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x40\x02\0\0\0\0\0\
 -\0\x04\0\0\0\x01\0\0\0\x50\x02\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x60\x02\0\0\0\0\
 -\0\0\x04\0\0\0\x01\0\0\0\x70\x02\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x80\x02\0\0\0\
 -\0\0\0\x04\0\0\0\x01\0\0\0\x90\x02\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xa0\x02\0\0\
 -\0\0\0\0\x04\0\0\0\x01\0\0\0\xb0\x02\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xc0\x02\0\
 -\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xd0\x02\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xe0\x02\
 -\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xf0\x02\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\0\x03\
 -\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x10\x03\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x20\
 -\x03\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x30\x03\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\
 -\x40\x03\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x50\x03\0\0\0\0\0\0\x04\0\0\0\x01\0\0\
 -\0\x60\x03\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x70\x03\0\0\0\0\0\0\x04\0\0\0\x01\0\
 -\0\0\x80\x03\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x90\x03\0\0\0\0\0\0\x04\0\0\0\x01\
 -\0\0\0\xa0\x03\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xb0\x03\0\0\0\0\0\0\x04\0\0\0\
 -\x01\0\0\0\xc0\x03\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xd0\x03\0\0\0\0\0\0\x04\0\0\
 -\0\x01\0\0\0\xe0\x03\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xf0\x03\0\0\0\0\0\0\x04\0\
 -\0\0\x01\0\0\0\0\x04\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x10\x04\0\0\0\0\0\0\x04\0\
 -\0\0\x01\0\0\0\x20\x04\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x30\x04\0\0\0\0\0\0\x04\
 -\0\0\0\x01\0\0\0\x40\x04\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x50\x04\0\0\0\0\0\0\
 -\x04\0\0\0\x01\0\0\0\x60\x04\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x70\x04\0\0\0\0\0\
 -\0\x04\0\0\0\x01\0\0\0\x80\x04\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x90\x04\0\0\0\0\
 -\0\0\x04\0\0\0\x01\0\0\0\xa0\x04\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xb0\x04\0\0\0\
 -\0\0\0\x04\0\0\0\x01\0\0\0\xc0\x04\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xd0\x04\0\0\
 -\0\0\0\0\x04\0\0\0\x01\0\0\0\xe0\x04\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xf0\x04\0\
 -\0\0\0\0\0\x04\0\0\0\x01\0\0\0\0\x05\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x10\x05\0\
 -\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x20\x05\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x30\x05\
 -\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x40\x05\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x50\
 -\x05\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x60\x05\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\
 -\x70\x05\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x80\x05\0\0\0\0\0\0\x04\0\0\0\x01\0\0\
 -\0\x90\x05\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xa0\x05\0\0\0\0\0\0\x04\0\0\0\x01\0\
 -\0\0\xb0\x05\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xc0\x05\0\0\0\0\0\0\x04\0\0\0\x01\
 -\0\0\0\xd0\x05\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xe0\x05\0\0\0\0\0\0\x04\0\0\0\
 -\x01\0\0\0\xf0\x05\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\0\x06\0\0\0\0\0\0\x04\0\0\0\
 -\x01\0\0\0\x10\x06\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x20\x06\0\0\0\0\0\0\x04\0\0\
 -\0\x01\0\0\0\x30\x06\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x40\x06\0\0\0\0\0\0\x04\0\
 -\0\0\x01\0\0\0\x50\x06\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x60\x06\0\0\0\0\0\0\x04\
 -\0\0\0\x01\0\0\0\x70\x06\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x80\x06\0\0\0\0\0\0\
 -\x04\0\0\0\x01\0\0\0\x90\x06\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xa0\x06\0\0\0\0\0\
 -\0\x04\0\0\0\x01\0\0\0\xb0\x06\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xc0\x06\0\0\0\0\
 -\0\0\x04\0\0\0\x01\0\0\0\xd0\x06\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xe0\x06\0\0\0\
 -\0\0\0\x04\0\0\0\x01\0\0\0\xf0\x06\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\0\x07\0\0\0\
 -\0\0\0\x04\0\0\0\x01\0\0\0\x10\x07\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x20\x07\0\0\
 -\0\0\0\0\x04\0\0\0\x01\0\0\0\x30\x07\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x40\x07\0\
 -\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x50\x07\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x60\x07\
 -\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x70\x07\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x80\
 -\x07\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x90\x07\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\
 -\xa0\x07\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xb0\x07\0\0\0\0\0\0\x04\0\0\0\x01\0\0\
 -\0\xc0\x07\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xd0\x07\0\0\0\0\0\0\x04\0\0\0\x01\0\
 -\0\0\xe0\x07\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xf0\x07\0\0\0\0\0\0\x04\0\0\0\x01\
 -\0\0\0\0\x08\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x10\x08\0\0\0\0\0\0\x04\0\0\0\x01\
 -\0\0\0\x20\x08\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x30\x08\0\0\0\0\0\0\x04\0\0\0\
 -\x01\0\0\0\x40\x08\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x50\x08\0\0\0\0\0\0\x04\0\0\
 -\0\x01\0\0\0\x60\x08\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x70\x08\0\0\0\0\0\0\x04\0\
 -\0\0\x01\0\0\0\x80\x08\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x90\x08\0\0\0\0\0\0\x04\
 -\0\0\0\x01\0\0\0\xa0\x08\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xb0\x08\0\0\0\0\0\0\
 -\x04\0\0\0\x01\0\0\0\xc0\x08\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xd0\x08\0\0\0\0\0\
 -\0\x04\0\0\0\x01\0\0\0\xe0\x08\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xf0\x08\0\0\0\0\
 -\0\0\x04\0\0\0\x01\0\0\0\0\x09\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x10\x09\0\0\0\0\
 -\0\0\x04\0\0\0\x01\0\0\0\x20\x09\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x30\x09\0\0\0\
 -\0\0\0\x04\0\0\0\x01\0\0\0\x40\x09\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x50\x09\0\0\
 -\0\0\0\0\x04\0\0\0\x01\0\0\0\x60\x09\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x70\x09\0\
 -\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x80\x09\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x90\x09\
 -\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xa0\x09\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xb0\
 -\x09\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xc0\x09\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\
 -\xd0\x09\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xe0\x09\0\0\0\0\0\0\x04\0\0\0\x01\0\0\
 -\0\xf0\x09\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\0\x0a\0\0\0\0\0\0\x04\0\0\0\x01\0\0\
 -\0\x10\x0a\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x20\x0a\0\0\0\0\0\0\x04\0\0\0\x01\0\
 -\0\0\x30\x0a\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x40\x0a\0\0\0\0\0\0\x04\0\0\0\x01\
 -\0\0\0\x50\x0a\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x60\x0a\0\0\0\0\0\0\x04\0\0\0\
 -\x01\0\0\0\x70\x0a\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x80\x0a\0\0\0\0\0\0\x04\0\0\
 -\0\x01\0\0\0\x90\x0a\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xa0\x0a\0\0\0\0\0\0\x04\0\
 -\0\0\x01\0\0\0\xb0\x0a\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xc0\x0a\0\0\0\0\0\0\x04\
 -\0\0\0\x01\0\0\0\xd0\x0a\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xe0\x0a\0\0\0\0\0\0\
 -\x04\0\0\0\x01\0\0\0\xf0\x0a\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\0\x0b\0\0\0\0\0\0\
 -\x04\0\0\0\x01\0\0\0\x10\x0b\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x20\x0b\0\0\0\0\0\
 -\0\x04\0\0\0\x01\0\0\0\x30\x0b\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x40\x0b\0\0\0\0\
 -\0\0\x04\0\0\0\x01\0\0\0\x50\x0b\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x60\x0b\0\0\0\
 -\0\0\0\x04\0\0\0\x01\0\0\0\x70\x0b\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x80\x0b\0\0\
 -\0\0\0\0\x04\0\0\0\x01\0\0\0\x90\x0b\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xa0\x0b\0\
 -\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xb0\x0b\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xc0\x0b\
 -\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xd0\x0b\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xe0\
 -\x0b\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xf0\x0b\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\0\
 -\x0c\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x10\x0c\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\
 -\x20\x0c\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x30\x0c\0\0\0\0\0\0\x04\0\0\0\x01\0\0\
 -\0\x40\x0c\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x50\x0c\0\0\0\0\0\0\x04\0\0\0\x01\0\
 -\0\0\x60\x0c\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x70\x0c\0\0\0\0\0\0\x04\0\0\0\x01\
 -\0\0\0\x80\x0c\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x90\x0c\0\0\0\0\0\0\x04\0\0\0\
 -\x01\0\0\0\x40\x41\x42\x43\x44\0\x74\x61\x70\x5f\x72\x73\x73\x5f\x6d\x61\x70\
 -\x5f\x74\x6f\x65\x70\x6c\x69\x74\x7a\x5f\x6b\x65\x79\0\x2e\x74\x65\x78\x74\0\
 -\x2e\x72\x65\x6c\x2e\x42\x54\x46\x2e\x65\x78\x74\0\x2e\x6d\x61\x70\x73\0\x74\
 -\x61\x70\x5f\x72\x73\x73\x5f\x6d\x61\x70\x5f\x63\x6f\x6e\x66\x69\x67\x75\x72\
 -\x61\x74\x69\x6f\x6e\x73\0\x74\x75\x6e\x5f\x72\x73\x73\x5f\x73\x74\x65\x65\x72\
 -\x69\x6e\x67\x5f\x70\x72\x6f\x67\0\x2e\x72\x65\x6c\x74\x75\x6e\x5f\x72\x73\x73\
 -\x5f\x73\x74\x65\x65\x72\x69\x6e\x67\0\x2e\x6c\x6c\x76\x6d\x5f\x61\x64\x64\x72\
 -\x73\x69\x67\0\x5f\x6c\x69\x63\x65\x6e\x73\x65\0\x74\x61\x70\x5f\x72\x73\x73\
 -\x5f\x6d\x61\x70\x5f\x69\x6e\x64\x69\x72\x65\x63\x74\x69\x6f\x6e\x5f\x74\x61\
 -\x62\x6c\x65\0\x2e\x73\x74\x72\x74\x61\x62\0\x2e\x73\x79\x6d\x74\x61\x62\0\x2e\
 -\x72\x65\x6c\x2e\x42\x54\x46\0\x4c\x42\x42\x30\x5f\x39\0\x4c\x42\x42\x30\x5f\
 -\x39\x39\0\x4c\x42\x42\x30\x5f\x37\x39\0\x4c\x42\x42\x30\x5f\x31\x30\x39\0\x4c\
 -\x42\x42\x30\x5f\x38\x38\0\x4c\x42\x42\x30\x5f\x34\x38\0\x4c\x42\x42\x30\x5f\
 -\x31\x38\0\x4c\x42\x42\x30\x5f\x31\x30\x38\0\x4c\x42\x42\x30\x5f\x39\x37\0\x4c\
 -\x42\x42\x30\x5f\x37\x37\0\x4c\x42\x42\x30\x5f\x36\x37\0\x4c\x42\x42\x30\x5f\
 -\x34\x37\0\x4c\x42\x42\x30\x5f\x31\x37\0\x4c\x42\x42\x30\x5f\x36\x36\0\x4c\x42\
 -\x42\x30\x5f\x34\x36\0\x4c\x42\x42\x30\x5f\x33\x36\0\x4c\x42\x42\x30\x5f\x31\
 -\x30\x36\0\x4c\x42\x42\x30\x5f\x35\x35\0\x4c\x42\x42\x30\x5f\x34\x35\0\x4c\x42\
 -\x42\x30\x5f\x33\x35\0\x4c\x42\x42\x30\x5f\x32\x35\0\x4c\x42\x42\x30\x5f\x31\
 -\x30\x35\0\x4c\x42\x42\x30\x5f\x34\0\x4c\x42\x42\x30\x5f\x39\x34\0\x4c\x42\x42\
 -\x30\x5f\x38\x34\0\x4c\x42\x42\x30\x5f\x35\x34\0\x4c\x42\x42\x30\x5f\x34\x34\0\
 +\x45\x5f\x49\x50\x76\x34\x29\x20\x7b\0\x20\x20\x20\x20\x5f\x5f\x62\x75\x69\x6c\
 +\x74\x69\x6e\x5f\x6d\x65\x6d\x63\x70\x79\x28\x26\x72\x73\x73\x5f\x69\x6e\x70\
 +\x75\x74\x5b\x2a\x62\x79\x74\x65\x73\x5f\x77\x72\x69\x74\x74\x65\x6e\x5d\x2c\
 +\x20\x70\x74\x72\x2c\x20\x73\x69\x7a\x65\x29\x3b\0\x20\x20\x20\x20\x20\x20\x20\
 +\x20\x20\x20\x20\x20\x69\x6e\x66\x6f\x2d\x3e\x69\x73\x5f\x75\x64\x70\x20\x3d\
 +\x20\x31\x3b\0\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x73\x74\x72\x75\
 +\x63\x74\x20\x75\x64\x70\x68\x64\x72\x20\x75\x64\x70\x20\x3d\x20\x7b\x7d\x3b\0\
 +\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x65\x72\x72\x20\x3d\x20\x62\
 +\x70\x66\x5f\x73\x6b\x62\x5f\x6c\x6f\x61\x64\x5f\x62\x79\x74\x65\x73\x5f\x72\
 +\x65\x6c\x61\x74\x69\x76\x65\x28\x73\x6b\x62\x2c\x20\x6c\x34\x5f\x6f\x66\x66\
 +\x73\x65\x74\x2c\x20\x26\x75\x64\x70\x2c\x20\x73\x69\x7a\x65\x6f\x66\x28\x75\
 +\x64\x70\x29\x2c\0\x20\x20\x20\x20\x20\x20\x20\x20\x7d\x20\x65\x6c\x73\x65\x20\
 +\x69\x66\x20\x28\x63\x6f\x6e\x66\x69\x67\x2d\x3e\x68\x61\x73\x68\x5f\x74\x79\
 +\x70\x65\x73\x20\x26\x20\x56\x49\x52\x54\x49\x4f\x5f\x4e\x45\x54\x5f\x52\x53\
 +\x53\x5f\x48\x41\x53\x48\x5f\x54\x59\x50\x45\x5f\x49\x50\x76\x36\x29\x20\x7b\0\
 +\x20\x20\x20\x20\x66\x6f\x72\x20\x28\x62\x79\x74\x65\x20\x3d\x20\x30\x3b\x20\
 +\x62\x79\x74\x65\x20\x3c\x20\x48\x41\x53\x48\x5f\x43\x41\x4c\x43\x55\x4c\x41\
 +\x54\x49\x4f\x4e\x5f\x42\x55\x46\x46\x45\x52\x5f\x53\x49\x5a\x45\x3b\x20\x62\
 +\x79\x74\x65\x2b\x2b\x29\x20\x7b\0\x20\x20\x20\x20\x5f\x5f\x75\x33\x32\x20\x6c\
 +\x65\x66\x74\x6d\x6f\x73\x74\x5f\x33\x32\x5f\x62\x69\x74\x73\x20\x3d\x20\x6b\
 +\x65\x79\x2d\x3e\x6c\x65\x66\x74\x6d\x6f\x73\x74\x5f\x33\x32\x5f\x62\x69\x74\
 +\x73\x3b\0\x20\x20\x20\x20\x20\x20\x20\x20\x5f\x5f\x75\x38\x20\x69\x6e\x70\x75\
 +\x74\x5f\x62\x79\x74\x65\x20\x3d\x20\x69\x6e\x70\x75\x74\x5b\x62\x79\x74\x65\
 +\x5d\x3b\0\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x69\x66\x20\x28\x69\
 +\x6e\x70\x75\x74\x5f\x62\x79\x74\x65\x20\x26\x20\x28\x31\x20\x3c\x3c\x20\x37\
 +\x29\x29\x20\x7b\0\x20\x20\x20\x20\x20\x20\x20\x20\x5f\x5f\x75\x38\x20\x6b\x65\
 +\x79\x5f\x62\x79\x74\x65\x20\x3d\x20\x6b\x65\x79\x2d\x3e\x6e\x65\x78\x74\x5f\
 +\x62\x79\x74\x65\x5b\x62\x79\x74\x65\x5d\x3b\0\x20\x20\x20\x20\x20\x20\x20\x20\
 +\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x28\x6c\x65\x66\x74\x6d\x6f\
 +\x73\x74\x5f\x33\x32\x5f\x62\x69\x74\x73\x20\x3c\x3c\x20\x31\x29\x20\x7c\x20\
 +\x28\x28\x6b\x65\x79\x5f\x62\x79\x74\x65\x20\x26\x20\x28\x31\x20\x3c\x3c\x20\
 +\x37\x29\x29\x20\x3e\x3e\x20\x37\x29\x3b\0\x20\x20\x20\x20\x20\x20\x20\x20\x69\
 +\x66\x20\x28\x68\x61\x73\x68\x29\x20\x7b\0\x20\x20\x20\x20\x20\x20\x20\x20\x20\
 +\x20\x20\x20\x5f\x5f\x75\x33\x32\x20\x74\x61\x62\x6c\x65\x5f\x69\x64\x78\x20\
 +\x3d\x20\x68\x61\x73\x68\x20\x25\x20\x63\x6f\x6e\x66\x69\x67\x2d\x3e\x69\x6e\
 +\x64\x69\x72\x65\x63\x74\x69\x6f\x6e\x73\x5f\x6c\x65\x6e\x3b\0\x20\x20\x20\x20\
 +\x20\x20\x20\x20\x20\x20\x20\x20\x71\x75\x65\x75\x65\x20\x3d\x20\x62\x70\x66\
 +\x5f\x6d\x61\x70\x5f\x6c\x6f\x6f\x6b\x75\x70\x5f\x65\x6c\x65\x6d\x28\x26\x74\
 +\x61\x70\x5f\x72\x73\x73\x5f\x6d\x61\x70\x5f\x69\x6e\x64\x69\x72\x65\x63\x74\
 +\x69\x6f\x6e\x5f\x74\x61\x62\x6c\x65\x2c\0\x20\x20\x20\x20\x20\x20\x20\x20\x20\
 +\x20\x20\x20\x69\x66\x20\x28\x71\x75\x65\x75\x65\x29\x20\x7b\0\x7d\0\x20\x20\
 +\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x72\x65\x74\x75\x72\
 +\x6e\x20\x2a\x71\x75\x65\x75\x65\x3b\0\x63\x68\x61\x72\0\x5f\x6c\x69\x63\x65\
 +\x6e\x73\x65\0\x2e\x6d\x61\x70\x73\0\x6c\x69\x63\x65\x6e\x73\x65\0\x62\x70\x66\
 +\x5f\x66\x6c\x6f\x77\x5f\x6b\x65\x79\x73\0\x62\x70\x66\x5f\x73\x6f\x63\x6b\0\0\
 +\x9f\xeb\x01\0\x20\0\0\0\0\0\0\0\x14\0\0\0\x14\0\0\0\x1c\x0d\0\0\x30\x0d\0\0\0\
 +\0\0\0\x08\0\0\0\x30\x02\0\0\x01\0\0\0\0\0\0\0\x26\0\0\0\x10\0\0\0\x30\x02\0\0\
 +\xd1\0\0\0\0\0\0\0\x37\x02\0\0\x67\x02\0\0\0\x5c\x08\0\x10\0\0\0\x37\x02\0\0\
 +\x98\x02\0\0\x0b\x74\x08\0\x20\0\0\0\x37\x02\0\0\0\0\0\0\0\0\0\0\x28\0\0\0\x37\
 +\x02\0\0\xab\x02\0\0\x0e\x80\x08\0\x50\0\0\0\x37\x02\0\0\xf0\x02\0\0\x0b\x84\
 +\x08\0\x88\0\0\0\x37\x02\0\0\x30\x03\0\0\x10\x8c\x08\0\x90\0\0\0\x37\x02\0\0\0\
 +\0\0\0\0\0\0\0\x98\0\0\0\x37\x02\0\0\x30\x03\0\0\x10\x8c\x08\0\xa0\0\0\0\x37\
 +\x02\0\0\x49\x03\0\0\x16\x90\x08\0\xa8\0\0\0\x37\x02\0\0\x49\x03\0\0\x0d\x90\
 +\x08\0\xc0\0\0\0\x37\x02\0\0\x6a\x03\0\0\x0a\x08\x06\0\xe8\0\0\0\x37\x02\0\0\
 +\xa1\x03\0\0\x1f\x18\x06\0\x38\x01\0\0\x37\x02\0\0\xd1\x03\0\0\x0f\xac\x04\0\
 +\x40\x01\0\0\x37\x02\0\0\xea\x03\0\0\x0c\x2c\x04\0\x50\x01\0\0\x37\x02\0\0\0\0\
 +\0\0\0\0\0\0\x58\x01\0\0\x37\x02\0\0\xfe\x03\0\0\x0b\x38\x04\0\x80\x01\0\0\x37\
 +\x02\0\0\x44\x04\0\0\x09\x40\x04\0\x90\x01\0\0\x37\x02\0\0\x44\x04\0\0\x09\x40\
 +\x04\0\xa0\x01\0\0\x37\x02\0\0\x53\x04\0\0\x0d\x50\x04\0\xb8\x01\0\0\x37\x02\0\
 +\0\x53\x04\0\0\x05\x50\x04\0\xd8\x01\0\0\x37\x02\0\0\0\0\0\0\0\0\0\0\xe0\x01\0\
 +\0\x37\x02\0\0\x71\x04\0\0\x0f\x64\x04\0\0\x02\0\0\x37\x02\0\0\x71\x04\0\0\x09\
 +\x64\x04\0\x10\x02\0\0\x37\x02\0\0\x44\x04\0\0\x09\x7c\x04\0\x18\x02\0\0\x37\
 +\x02\0\0\xbb\x04\0\0\x0c\x8c\x04\0\x20\x02\0\0\x37\x02\0\0\xcb\x04\0\0\x09\xc8\
 +\x04\0\x50\x02\0\0\x37\x02\0\0\xe7\x04\0\0\x17\xe0\x04\0\x60\x02\0\0\x37\x02\0\
 +\0\x02\x05\0\0\x16\xe8\x04\0\x80\x02\0\0\x37\x02\0\0\xe7\x04\0\0\x17\xe0\x04\0\
 +\x88\x02\0\0\x37\x02\0\0\x20\x05\0\0\x0f\xec\x04\0\xb0\x02\0\0\x37\x02\0\0\x63\
 +\x05\0\0\x0d\xf4\x04\0\xc0\x02\0\0\x37\x02\0\0\x63\x05\0\0\x0d\xf4\x04\0\xc8\
 +\x02\0\0\x37\x02\0\0\x76\x05\0\0\x24\x0c\x05\0\xd0\x02\0\0\x37\x02\0\0\x76\x05\
 +\0\0\x20\x0c\x05\0\xe0\x02\0\0\x37\x02\0\0\xa3\x05\0\0\x1b\x04\x05\0\xe8\x02\0\
 +\0\x37\x02\0\0\xa3\x05\0\0\x16\x04\x05\0\xf0\x02\0\0\x37\x02\0\0\xc4\x05\0\0\
 +\x1b\x08\x05\0\xf8\x02\0\0\x37\x02\0\0\xc4\x05\0\0\x16\x08\x05\0\0\x03\0\0\x37\
 +\x02\0\0\xe5\x05\0\0\x1a\x14\x05\0\x08\x03\0\0\x37\x02\0\0\x76\x05\0\0\x1d\x0c\
 +\x05\0\x10\x03\0\0\x37\x02\0\0\x08\x06\0\0\x18\x18\x05\0\x18\x03\0\0\x37\x02\0\
 +\0\x08\x06\0\0\x1c\x18\x05\0\x30\x03\0\0\x37\x02\0\0\x28\x06\0\0\x15\x74\x05\0\
 +\x40\x03\0\0\x37\x02\0\0\x28\x06\0\0\x1a\x74\x05\0\x58\x03\0\0\x37\x02\0\0\x5c\
 +\x06\0\0\x0d\x78\x05\0\x78\x03\0\0\x37\x02\0\0\x86\x06\0\0\x1a\x7c\x05\0\x88\
 +\x03\0\0\x37\x02\0\0\xa4\x06\0\0\x1b\x84\x05\0\xa8\x03\0\0\x37\x02\0\0\x86\x06\
 +\0\0\x1a\x7c\x05\0\xb0\x03\0\0\x37\x02\0\0\xc8\x06\0\0\x13\x88\x05\0\xd8\x03\0\
 +\0\x37\x02\0\0\x19\x07\0\0\x11\x90\x05\0\xe8\x03\0\0\x37\x02\0\0\x19\x07\0\0\
 +\x11\x90\x05\0\xf0\x03\0\0\x37\x02\0\0\0\0\0\0\0\0\0\0\x10\x04\0\0\x37\x02\0\0\
 +\x30\x07\0\0\x15\x34\x06\0\x18\x04\0\0\x37\x02\0\0\x30\x07\0\0\x09\x34\x06\0\
 +\x20\x04\0\0\x37\x02\0\0\0\0\0\0\0\0\0\0\x70\x04\0\0\x37\x02\0\0\x4f\x07\0\0\
 +\x19\x38\x06\0\x80\x04\0\0\x37\x02\0\0\x4f\x07\0\0\x20\x38\x06\0\xa0\x04\0\0\
 +\x37\x02\0\0\0\0\0\0\0\0\0\0\xf0\x04\0\0\x37\x02\0\0\x71\x07\0\0\x17\x20\x05\0\
 +\0\x05\0\0\x37\x02\0\0\x8c\x07\0\0\x18\x28\x05\0\x30\x05\0\0\x37\x02\0\0\x71\
 +\x07\0\0\x17\x20\x05\0\x48\x05\0\0\x37\x02\0\0\xad\x07\0\0\x0f\x2c\x05\0\x70\
 +\x05\0\0\x37\x02\0\0\x63\x05\0\0\x0d\x34\x05\0\x80\x05\0\0\x37\x02\0\0\x63\x05\
 +\0\0\x0d\x34\x05\0\x88\x05\0\0\x37\x02\0\0\xf2\x07\0\0\x1d\x44\x05\0\xc8\x05\0\
 +\0\x37\x02\0\0\x15\x08\0\0\x1d\x48\x05\0\x08\x06\0\0\x37\x02\0\0\x38\x08\0\0\
 +\x1b\x50\x05\0\x10\x06\0\0\x37\x02\0\0\x5b\x08\0\0\x05\x3c\x02\0\x58\x06\0\0\
 +\x37\x02\0\0\x73\x08\0\0\x19\xc4\x02\0\xd0\x06\0\0\x37\x02\0\0\0\0\0\0\0\0\0\0\
 +\xd8\x06\0\0\x37\x02\0\0\x99\x08\0\0\x0f\xd4\x02\0\0\x07\0\0\x37\x02\0\0\x63\
 +\x05\0\0\x0d\xdc\x02\0\x18\x07\0\0\x37\x02\0\0\x63\x05\0\0\x0d\xdc\x02\0\x20\
 +\x07\0\0\x37\x02\0\0\xde\x08\0\0\x0d\xec\x02\0\x40\x07\0\0\x37\x02\0\0\x0d\x09\
 +\0\0\x20\xf0\x02\0\x68\x07\0\0\x37\x02\0\0\x39\x09\0\0\x13\xf8\x02\0\x90\x07\0\
 +\0\x37\x02\0\0\x19\x07\0\0\x11\0\x03\0\xa8\x07\0\0\x37\x02\0\0\x19\x07\0\0\x11\
 +\0\x03\0\xb0\x07\0\0\x37\x02\0\0\x81\x09\0\0\x19\x10\x03\0\xb8\x07\0\0\x37\x02\
 +\0\0\x81\x09\0\0\x34\x10\x03\0\xe0\x07\0\0\x37\x02\0\0\xb7\x09\0\0\x15\x24\x03\
 +\0\xf0\x07\0\0\x37\x02\0\0\xf8\x09\0\0\x17\x20\x03\0\x18\x08\0\0\x37\x02\0\0\
 +\x2f\x0a\0\0\x15\x30\x03\0\x30\x08\0\0\x37\x02\0\0\x2f\x0a\0\0\x15\x30\x03\0\
 +\x38\x08\0\0\x37\x02\0\0\x4a\x0a\0\0\x27\x40\x03\0\x70\x08\0\0\x37\x02\0\0\x75\
 +\x0a\0\0\x27\x5c\x03\0\x80\x08\0\0\x37\x02\0\0\xa5\x0a\0\0\x1c\xc0\x03\0\x88\
 +\x08\0\0\x37\x02\0\0\xe1\x0a\0\0\x20\xcc\x03\0\x98\x08\0\0\x37\x02\0\0\xe1\x0a\
 +\0\0\x2f\xcc\x03\0\xa0\x08\0\0\x37\x02\0\0\xe1\x0a\0\0\x36\xcc\x03\0\xa8\x08\0\
 +\0\x37\x02\0\0\xe1\x0a\0\0\x15\xcc\x03\0\x18\x09\0\0\x37\x02\0\0\x1d\x0b\0\0\
 +\x43\x70\x03\0\x38\x09\0\0\x37\x02\0\0\0\0\0\0\0\0\0\0\x40\x09\0\0\x37\x02\0\0\
 +\x1d\x0b\0\0\x17\x70\x03\0\x68\x09\0\0\x37\x02\0\0\x2f\x0a\0\0\x15\x78\x03\0\
 +\x80\x09\0\0\x37\x02\0\0\x2f\x0a\0\0\x15\x78\x03\0\x88\x09\0\0\x37\x02\0\0\x6d\
 +\x0b\0\0\x19\x88\x03\0\x90\x09\0\0\x37\x02\0\0\x6d\x0b\0\0\x15\x88\x03\0\x98\
 +\x09\0\0\x37\x02\0\0\x9d\x0b\0\0\x19\x90\x03\0\xa0\x09\0\0\x37\x02\0\0\xcd\x0b\
 +\0\0\x1b\x8c\x03\0\xd0\x09\0\0\x37\x02\0\0\x08\x0c\0\0\x19\xa0\x03\0\xe8\x09\0\
 +\0\x37\x02\0\0\x08\x0c\0\0\x19\xa0\x03\0\xf0\x09\0\0\x37\x02\0\0\x27\x0c\0\0\
 +\x2b\xb0\x03\0\x10\x0a\0\0\x37\x02\0\0\xa5\x0a\0\0\x1f\xc0\x03\0\x30\x0a\0\0\
 +\x37\x02\0\0\x56\x0c\0\0\x21\xe0\x03\0\x40\x0a\0\0\x37\x02\0\0\x7e\x0c\0\0\x20\
 +\xf0\x03\0\x48\x0a\0\0\x37\x02\0\0\x7e\x0c\0\0\x2c\xf0\x03\0\x60\x0a\0\0\x37\
 +\x02\0\0\x7e\x0c\0\0\x14\xf0\x03\0\x70\x0a\0\0\x37\x02\0\0\xae\x0c\0\0\x20\xec\
 +\x03\0\x80\x0a\0\0\x37\x02\0\0\x5b\x08\0\0\x05\x3c\x02\0\xb0\x0a\0\0\x37\x02\0\
 +\0\xd6\x0c\0\0\x38\xcc\x02\0\xd0\x0a\0\0\x37\x02\0\0\xd6\x0c\0\0\x05\xcc\x02\0\
 +\xe8\x0a\0\0\x37\x02\0\0\x5b\x08\0\0\x05\x3c\x02\0\xf8\x0a\0\0\x37\x02\0\0\x14\
 +\x0d\0\0\x1c\xd0\x06\0\x08\x0b\0\0\x37\x02\0\0\x14\x0d\0\0\x10\xd0\x06\0\x10\
 +\x0b\0\0\x37\x02\0\0\0\0\0\0\0\0\0\0\x60\x0b\0\0\x37\x02\0\0\x4f\x07\0\0\x19\
 +\xd4\x06\0\x68\x0b\0\0\x37\x02\0\0\x4f\x07\0\0\x20\xd4\x06\0\xa0\x0b\0\0\x37\
 +\x02\0\0\x3a\x0d\0\0\x2d\x0c\x07\0\xb0\x0b\0\0\x37\x02\0\0\x3a\x0d\0\0\x1d\x0c\
 +\x07\0\xb8\x0b\0\0\x37\x02\0\0\x3a\x0d\0\0\x2d\x0c\x07\0\xc8\x0b\0\0\x37\x02\0\
 +\0\x69\x0d\0\0\x2d\xe0\x06\0\xf8\x0b\0\0\x37\x02\0\0\x69\x0d\0\0\x1d\xe0\x06\0\
 +\x08\x0c\0\0\x37\x02\0\0\x69\x0d\0\0\x2d\xe0\x06\0\x18\x0c\0\0\x37\x02\0\0\0\0\
 +\0\0\0\0\0\0\xe8\x0c\0\0\x37\x02\0\0\x98\x0d\0\0\x20\x74\x06\0\xf0\x0c\0\0\x37\
 +\x02\0\0\x98\x0d\0\0\x27\x74\x06\0\x18\x0d\0\0\x37\x02\0\0\xc1\x0d\0\0\x27\xb0\
 +\x06\0\x20\x0d\0\0\x37\x02\0\0\xc1\x0d\0\0\x14\xb0\x06\0\x28\x0d\0\0\x37\x02\0\
 +\0\x0a\x0e\0\0\x05\xa4\x01\0\x38\x0d\0\0\x37\x02\0\0\x0a\x0e\0\0\x05\xa4\x01\0\
 +\x60\x0d\0\0\x37\x02\0\0\x63\x05\0\0\x0d\x60\x05\0\x70\x0d\0\0\x37\x02\0\0\0\0\
 +\0\0\0\0\0\0\x80\x0d\0\0\x37\x02\0\0\x98\x0d\0\0\x20\x50\x07\0\x88\x0d\0\0\x37\
 +\x02\0\0\x98\x0d\0\0\x27\x50\x07\0\xc0\x0d\0\0\x37\x02\0\0\x3a\x0d\0\0\x2d\x88\
 +\x07\0\xd0\x0d\0\0\x37\x02\0\0\x3a\x0d\0\0\x1d\x88\x07\0\xd8\x0d\0\0\x37\x02\0\
 +\0\x3a\x0d\0\0\x2d\x88\x07\0\xe8\x0d\0\0\x37\x02\0\0\x69\x0d\0\0\x2d\x5c\x07\0\
 +\x18\x0e\0\0\x37\x02\0\0\x69\x0d\0\0\x1d\x5c\x07\0\x28\x0e\0\0\x37\x02\0\0\x69\
 +\x0d\0\0\x2d\x5c\x07\0\x40\x0e\0\0\x37\x02\0\0\x47\x0e\0\0\x1a\xac\x05\0\x50\
 +\x0e\0\0\x37\x02\0\0\x65\x0e\0\0\x1b\xb4\x05\0\x60\x0e\0\0\x37\x02\0\0\x47\x0e\
 +\0\0\x1a\xac\x05\0\x68\x0e\0\0\x37\x02\0\0\x89\x0e\0\0\x13\xb8\x05\0\x90\x0e\0\
 +\0\x37\x02\0\0\x19\x07\0\0\x11\xc0\x05\0\xa0\x0e\0\0\x37\x02\0\0\x19\x07\0\0\
 +\x11\xc0\x05\0\xb0\x0e\0\0\x37\x02\0\0\x5b\x08\0\0\x05\x3c\x02\0\xc0\x0e\0\0\
 +\x37\x02\0\0\xda\x0e\0\0\x27\xd4\x07\0\xd0\x0e\0\0\x37\x02\0\0\xda\x0e\0\0\x14\
 +\xd4\x07\0\xf0\x0e\0\0\x37\x02\0\0\x69\x0d\0\0\x2d\xd8\x07\0\0\x0f\0\0\x37\x02\
 +\0\0\x69\x0d\0\0\x1d\xd8\x07\0\x08\x0f\0\0\x37\x02\0\0\x69\x0d\0\0\x2d\xd8\x07\
 +\0\x30\x0f\0\0\x37\x02\0\0\0\0\0\0\0\0\0\0\x80\x0f\0\0\x37\x02\0\0\x3a\x0d\0\0\
 +\x1d\x04\x08\0\x88\x0f\0\0\x37\x02\0\0\x3a\x0d\0\0\x2d\x04\x08\0\x98\x0f\0\0\
 +\x37\x02\0\0\x0a\x0e\0\0\x05\xa4\x01\0\xf0\x0f\0\0\x37\x02\0\0\x0a\x0e\0\0\x05\
 +\xa4\x01\0\x30\x10\0\0\x37\x02\0\0\0\0\0\0\0\0\0\0\x48\x10\0\0\x37\x02\0\0\x23\
 +\x0f\0\0\x05\xdc\x01\0\x50\x10\0\0\x37\x02\0\0\x65\x0f\0\0\x23\xd0\x01\0\x68\
 +\x10\0\0\x37\x02\0\0\0\0\0\0\0\0\0\0\x70\x10\0\0\x37\x02\0\0\x99\x0f\0\0\x1b\
 +\xe0\x01\0\x90\x10\0\0\x37\x02\0\0\xc0\x0f\0\0\x11\xf4\x01\0\xa8\x10\0\0\x37\
 +\x02\0\0\xe9\x0f\0\0\x19\xe4\x01\0\xc0\x10\0\0\x37\x02\0\0\x17\x10\0\0\x27\x08\
 +\x02\0\xc8\x10\0\0\x37\x02\0\0\x17\x10\0\0\x46\x08\x02\0\xd8\x10\0\0\x37\x02\0\
 +\0\x17\x10\0\0\x2d\x08\x02\0\xe0\x10\0\0\x37\x02\0\0\xc0\x0f\0\0\x11\xf4\x01\0\
 +\x08\x11\0\0\x37\x02\0\0\x17\x10\0\0\x46\x08\x02\0\x20\x11\0\0\x37\x02\0\0\x17\
 +\x10\0\0\x27\x08\x02\0\x28\x11\0\0\x37\x02\0\0\x17\x10\0\0\x2d\x08\x02\0\x30\
 +\x11\0\0\x37\x02\0\0\xc0\x0f\0\0\x11\xf4\x01\0\x58\x11\0\0\x37\x02\0\0\x17\x10\
 +\0\0\x27\x08\x02\0\x60\x11\0\0\x37\x02\0\0\x17\x10\0\0\x46\x08\x02\0\x78\x11\0\
 +\0\x37\x02\0\0\x17\x10\0\0\x2d\x08\x02\0\x80\x11\0\0\x37\x02\0\0\xc0\x0f\0\0\
 +\x11\xf4\x01\0\xa8\x11\0\0\x37\x02\0\0\x17\x10\0\0\x27\x08\x02\0\xb0\x11\0\0\
 +\x37\x02\0\0\x17\x10\0\0\x46\x08\x02\0\xc8\x11\0\0\x37\x02\0\0\x17\x10\0\0\x2d\
 +\x08\x02\0\xd0\x11\0\0\x37\x02\0\0\xc0\x0f\0\0\x11\xf4\x01\0\xf8\x11\0\0\x37\
 +\x02\0\0\x17\x10\0\0\x46\x08\x02\0\x10\x12\0\0\x37\x02\0\0\x17\x10\0\0\x27\x08\
 +\x02\0\x18\x12\0\0\x37\x02\0\0\x17\x10\0\0\x2d\x08\x02\0\x20\x12\0\0\x37\x02\0\
 +\0\xc0\x0f\0\0\x11\xf4\x01\0\x48\x12\0\0\x37\x02\0\0\x17\x10\0\0\x46\x08\x02\0\
 +\x60\x12\0\0\x37\x02\0\0\x17\x10\0\0\x27\x08\x02\0\x68\x12\0\0\x37\x02\0\0\x17\
 +\x10\0\0\x2d\x08\x02\0\x70\x12\0\0\x37\x02\0\0\xc0\x0f\0\0\x11\xf4\x01\0\x98\
 +\x12\0\0\x37\x02\0\0\x17\x10\0\0\x46\x08\x02\0\xb0\x12\0\0\x37\x02\0\0\x17\x10\
 +\0\0\x27\x08\x02\0\xb8\x12\0\0\x37\x02\0\0\x17\x10\0\0\x2d\x08\x02\0\xc0\x12\0\
 +\0\x37\x02\0\0\xc0\x0f\0\0\x11\xf4\x01\0\xe0\x12\0\0\x37\x02\0\0\x17\x10\0\0\
 +\x46\x08\x02\0\xe8\x12\0\0\x37\x02\0\0\x17\x10\0\0\x27\x08\x02\0\xf0\x12\0\0\
 +\x37\x02\0\0\x17\x10\0\0\x2d\x08\x02\0\xf8\x12\0\0\x37\x02\0\0\x23\x0f\0\0\x3d\
 +\xdc\x01\0\x08\x13\0\0\x37\x02\0\0\x23\x0f\0\0\x05\xdc\x01\0\x18\x13\0\0\x37\
 +\x02\0\0\x63\x10\0\0\x0d\xa4\x08\0\x28\x13\0\0\x37\x02\0\0\x63\x10\0\0\x0d\xa4\
 +\x08\0\x30\x13\0\0\x37\x02\0\0\x77\x10\0\0\x2e\xa8\x08\0\x50\x13\0\0\x37\x02\0\
 +\0\x77\x10\0\0\x24\xa8\x08\0\x58\x13\0\0\x37\x02\0\0\x77\x10\0\0\x13\xa8\x08\0\
 +\x68\x13\0\0\x37\x02\0\0\x77\x10\0\0\x2e\xa8\x08\0\x70\x13\0\0\x37\x02\0\0\xb6\
 +\x10\0\0\x15\xb4\x08\0\x88\x13\0\0\x37\x02\0\0\xfe\x10\0\0\x11\xc0\x08\0\x90\
 +\x13\0\0\x37\x02\0\0\0\0\0\0\0\0\0\0\xb0\x13\0\0\x37\x02\0\0\x17\x11\0\0\x01\
 +\xe4\x08\0\xb8\x13\0\0\x37\x02\0\0\x19\x11\0\0\x18\xc4\x08\0\0\0\0\0\0\0\0\0\0\
 +\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x03\0\x03\0\0\0\0\0\0\0\0\0\0\0\
 +\0\0\0\0\0\0\x0d\x01\0\0\0\0\x03\0\xb0\x13\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x78\x01\
 +\0\0\0\0\x03\0\xb8\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x2e\x01\0\0\0\0\x03\0\x90\x13\
 +\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xbd\0\0\0\0\0\x03\0\xd0\x01\0\0\0\0\0\0\0\0\0\0\0\
 +\0\0\0\x2a\x02\0\0\0\0\x03\0\x20\x02\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x05\x01\0\0\0\
 +\0\x03\0\xe8\x04\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x43\x02\0\0\0\0\x03\0\x10\x04\0\0\
 +\0\0\0\0\0\0\0\0\0\0\0\0\x26\x01\0\0\0\0\x03\0\xe0\x02\0\0\0\0\0\0\0\0\0\0\0\0\
 +\0\0\x1a\x02\0\0\0\0\x03\0\x30\x03\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x12\x02\0\0\0\0\
 +\x03\0\x38\x0e\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xd4\0\0\0\0\0\x03\0\xf0\x03\0\0\0\0\
 +\0\0\0\0\0\0\0\0\0\0\x0a\x02\0\0\0\0\x03\0\xf8\x0a\0\0\0\0\0\0\0\0\0\0\0\0\0\0\
 +\x47\x01\0\0\0\0\x03\0\xe8\x0c\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x57\x01\0\0\0\0\x03\
 +\0\xa0\x04\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xd0\x01\0\0\0\0\x03\0\x40\x10\0\0\0\0\0\
 +\0\0\0\0\0\0\0\0\0\x7f\x01\0\0\0\0\x03\0\x78\x0d\0\0\0\0\0\0\0\0\0\0\0\0\0\0\
 +\x53\x02\0\0\0\0\x03\0\xb0\x0e\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xe9\x01\0\0\0\0\x03\
 +\0\x50\x06\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xc8\x01\0\0\0\0\x03\0\xc0\x06\0\0\0\0\0\
 +\0\0\0\0\0\0\0\0\0\xb0\x01\0\0\0\0\x03\0\x60\x0d\0\0\0\0\0\0\0\0\0\0\0\0\0\0\
 +\x8f\x01\0\0\0\0\x03\0\x60\x08\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x5f\x01\0\0\0\0\x03\
 +\0\x30\x0a\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x4f\x01\0\0\0\0\x03\0\x40\x0a\0\0\0\0\0\
 +\0\0\0\0\0\0\0\0\0\xe1\x01\0\0\0\0\x03\0\xe0\x0f\0\0\0\0\0\0\0\0\0\0\0\0\0\0\
 +\xc0\x01\0\0\0\0\x03\0\x48\x08\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x67\x01\0\0\0\0\x03\
 +\0\x18\x09\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x22\x02\0\0\0\0\x03\0\x80\x08\0\0\0\0\0\
 +\0\0\0\0\0\0\0\0\0\xb8\x01\0\0\0\0\x03\0\xf8\x08\0\0\0\0\0\0\0\0\0\0\0\0\0\0\
 +\x87\x01\0\0\0\0\x03\0\x50\x0d\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xe4\0\0\0\0\0\x03\0\
 +\x08\x0a\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x1e\x01\0\0\0\0\x03\0\xe8\x0a\0\0\0\0\0\0\
 +\0\0\0\0\0\0\0\0\xdc\0\0\0\0\0\x03\0\xb0\x0a\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x4b\
 +\x02\0\0\0\0\x03\0\xd8\x0a\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xa8\x01\0\0\0\0\x03\0\
 +\x80\x0d\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x3f\x01\0\0\0\0\x03\0\xb0\x0b\0\0\0\0\0\0\
 +\0\0\0\0\0\0\0\0\xfd\0\0\0\0\0\x03\0\xc8\x0b\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x3b\
 +\x02\0\0\0\0\x03\0\xf8\x0b\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xd9\x01\0\0\0\0\x03\0\
 +\x18\x0c\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x02\x02\0\0\0\0\x03\0\x10\x0c\0\0\0\0\0\0\
 +\0\0\0\0\0\0\0\0\xcc\0\0\0\0\0\x03\0\x18\x0d\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xa0\
 +\x01\0\0\0\0\x03\0\xc0\x0e\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x16\x01\0\0\0\0\x03\0\
 +\xd0\x0d\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xc4\0\0\0\0\0\x03\0\xe8\x0d\0\0\0\0\0\0\0\
 +\0\0\0\0\0\0\0\xfa\x01\0\0\0\0\x03\0\x18\x0e\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x37\
 +\x01\0\0\0\0\x03\0\0\x0f\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xf5\0\0\0\0\0\x03\0\x18\
 +\x0f\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x32\x02\0\0\0\0\x03\0\xf0\x0f\0\0\0\0\0\0\0\0\
 +\0\0\0\0\0\0\xf1\x01\0\0\0\0\x03\0\x30\x10\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x97\x01\
 +\0\0\0\0\x03\0\x60\x10\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x6f\x01\0\0\0\0\x03\0\x18\
 +\x13\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xec\0\0\0\0\0\x03\0\xb8\x13\0\0\0\0\0\0\0\0\0\
 +\0\0\0\0\0\x59\0\0\0\x12\0\x03\0\0\0\0\0\0\0\0\0\xc8\x13\0\0\0\0\0\0\x3e\0\0\0\
 +\x11\0\x05\0\0\0\0\0\0\0\0\0\x28\0\0\0\0\0\0\0\x01\0\0\0\x11\0\x05\0\x28\0\0\0\
 +\0\0\0\0\x28\0\0\0\0\0\0\0\x86\0\0\0\x11\0\x05\0\x50\0\0\0\0\0\0\0\x28\0\0\0\0\
 +\0\0\0\x7d\0\0\0\x11\0\x06\0\0\0\0\0\0\0\0\0\x07\0\0\0\0\0\0\0\x28\0\0\0\0\0\0\
 +\0\x01\0\0\0\x36\0\0\0\x50\0\0\0\0\0\0\0\x01\0\0\0\x37\0\0\0\x70\x13\0\0\0\0\0\
 +\0\x01\0\0\0\x38\0\0\0\x20\x05\0\0\0\0\0\0\x04\0\0\0\x36\0\0\0\x2c\x05\0\0\0\0\
 +\0\0\x04\0\0\0\x37\0\0\0\x38\x05\0\0\0\0\0\0\x04\0\0\0\x38\0\0\0\x50\x05\0\0\0\
 +\0\0\0\x04\0\0\0\x39\0\0\0\x2c\0\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x40\0\0\0\0\0\
 +\0\0\x04\0\0\0\x01\0\0\0\x50\0\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x60\0\0\0\0\0\0\
 +\0\x04\0\0\0\x01\0\0\0\x70\0\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x80\0\0\0\0\0\0\0\
 +\x04\0\0\0\x01\0\0\0\x90\0\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xa0\0\0\0\0\0\0\0\
 +\x04\0\0\0\x01\0\0\0\xb0\0\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xc0\0\0\0\0\0\0\0\
 +\x04\0\0\0\x01\0\0\0\xd0\0\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xe0\0\0\0\0\0\0\0\
 +\x04\0\0\0\x01\0\0\0\xf0\0\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\0\x01\0\0\0\0\0\0\
 +\x04\0\0\0\x01\0\0\0\x10\x01\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x20\x01\0\0\0\0\0\
 +\0\x04\0\0\0\x01\0\0\0\x30\x01\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x40\x01\0\0\0\0\
 +\0\0\x04\0\0\0\x01\0\0\0\x50\x01\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x60\x01\0\0\0\
 +\0\0\0\x04\0\0\0\x01\0\0\0\x70\x01\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x80\x01\0\0\
 +\0\0\0\0\x04\0\0\0\x01\0\0\0\x90\x01\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xa0\x01\0\
 +\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xb0\x01\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xc0\x01\
 +\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xd0\x01\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xe0\
 +\x01\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xf0\x01\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\0\
 +\x02\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x10\x02\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\
 +\x20\x02\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x30\x02\0\0\0\0\0\0\x04\0\0\0\x01\0\0\
 +\0\x40\x02\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x50\x02\0\0\0\0\0\0\x04\0\0\0\x01\0\
 +\0\0\x60\x02\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x70\x02\0\0\0\0\0\0\x04\0\0\0\x01\
 +\0\0\0\x80\x02\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x90\x02\0\0\0\0\0\0\x04\0\0\0\
 +\x01\0\0\0\xa0\x02\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xb0\x02\0\0\0\0\0\0\x04\0\0\
 +\0\x01\0\0\0\xc0\x02\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xd0\x02\0\0\0\0\0\0\x04\0\
 +\0\0\x01\0\0\0\xe0\x02\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xf0\x02\0\0\0\0\0\0\x04\
 +\0\0\0\x01\0\0\0\0\x03\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x10\x03\0\0\0\0\0\0\x04\
 +\0\0\0\x01\0\0\0\x20\x03\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x30\x03\0\0\0\0\0\0\
 +\x04\0\0\0\x01\0\0\0\x40\x03\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x50\x03\0\0\0\0\0\
 +\0\x04\0\0\0\x01\0\0\0\x60\x03\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x70\x03\0\0\0\0\
 +\0\0\x04\0\0\0\x01\0\0\0\x80\x03\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x90\x03\0\0\0\
 +\0\0\0\x04\0\0\0\x01\0\0\0\xa0\x03\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xb0\x03\0\0\
 +\0\0\0\0\x04\0\0\0\x01\0\0\0\xc0\x03\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xd0\x03\0\
 +\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xe0\x03\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xf0\x03\
 +\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\0\x04\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x10\x04\
 +\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x20\x04\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x30\
 +\x04\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x40\x04\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\
 +\x50\x04\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x60\x04\0\0\0\0\0\0\x04\0\0\0\x01\0\0\
 +\0\x70\x04\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x80\x04\0\0\0\0\0\0\x04\0\0\0\x01\0\
 +\0\0\x90\x04\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xa0\x04\0\0\0\0\0\0\x04\0\0\0\x01\
 +\0\0\0\xb0\x04\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xc0\x04\0\0\0\0\0\0\x04\0\0\0\
 +\x01\0\0\0\xd0\x04\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xe0\x04\0\0\0\0\0\0\x04\0\0\
 +\0\x01\0\0\0\xf0\x04\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\0\x05\0\0\0\0\0\0\x04\0\0\
 +\0\x01\0\0\0\x10\x05\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x20\x05\0\0\0\0\0\0\x04\0\
 +\0\0\x01\0\0\0\x30\x05\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x40\x05\0\0\0\0\0\0\x04\
 +\0\0\0\x01\0\0\0\x50\x05\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x60\x05\0\0\0\0\0\0\
 +\x04\0\0\0\x01\0\0\0\x70\x05\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x80\x05\0\0\0\0\0\
 +\0\x04\0\0\0\x01\0\0\0\x90\x05\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xa0\x05\0\0\0\0\
 +\0\0\x04\0\0\0\x01\0\0\0\xb0\x05\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xc0\x05\0\0\0\
 +\0\0\0\x04\0\0\0\x01\0\0\0\xd0\x05\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xe0\x05\0\0\
 +\0\0\0\0\x04\0\0\0\x01\0\0\0\xf0\x05\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\0\x06\0\0\
 +\0\0\0\0\x04\0\0\0\x01\0\0\0\x10\x06\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x20\x06\0\
 +\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x30\x06\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x40\x06\
 +\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x50\x06\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x60\
 +\x06\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x70\x06\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\
 +\x80\x06\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x90\x06\0\0\0\0\0\0\x04\0\0\0\x01\0\0\
 +\0\xa0\x06\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xb0\x06\0\0\0\0\0\0\x04\0\0\0\x01\0\
 +\0\0\xc0\x06\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xd0\x06\0\0\0\0\0\0\x04\0\0\0\x01\
 +\0\0\0\xe0\x06\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xf0\x06\0\0\0\0\0\0\x04\0\0\0\
 +\x01\0\0\0\0\x07\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x10\x07\0\0\0\0\0\0\x04\0\0\0\
 +\x01\0\0\0\x20\x07\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x30\x07\0\0\0\0\0\0\x04\0\0\
 +\0\x01\0\0\0\x40\x07\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x50\x07\0\0\0\0\0\0\x04\0\
 +\0\0\x01\0\0\0\x60\x07\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x70\x07\0\0\0\0\0\0\x04\
 +\0\0\0\x01\0\0\0\x80\x07\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x90\x07\0\0\0\0\0\0\
 +\x04\0\0\0\x01\0\0\0\xa0\x07\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xb0\x07\0\0\0\0\0\
 +\0\x04\0\0\0\x01\0\0\0\xc0\x07\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xd0\x07\0\0\0\0\
 +\0\0\x04\0\0\0\x01\0\0\0\xe0\x07\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xf0\x07\0\0\0\
 +\0\0\0\x04\0\0\0\x01\0\0\0\0\x08\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x10\x08\0\0\0\
 +\0\0\0\x04\0\0\0\x01\0\0\0\x20\x08\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x30\x08\0\0\
 +\0\0\0\0\x04\0\0\0\x01\0\0\0\x40\x08\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x50\x08\0\
 +\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x60\x08\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x70\x08\
 +\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x80\x08\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x90\
 +\x08\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xa0\x08\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\
 +\xb0\x08\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xc0\x08\0\0\0\0\0\0\x04\0\0\0\x01\0\0\
 +\0\xd0\x08\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xe0\x08\0\0\0\0\0\0\x04\0\0\0\x01\0\
 +\0\0\xf0\x08\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\0\x09\0\0\0\0\0\0\x04\0\0\0\x01\0\
 +\0\0\x10\x09\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x20\x09\0\0\0\0\0\0\x04\0\0\0\x01\
 +\0\0\0\x30\x09\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x40\x09\0\0\0\0\0\0\x04\0\0\0\
 +\x01\0\0\0\x50\x09\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x60\x09\0\0\0\0\0\0\x04\0\0\
 +\0\x01\0\0\0\x70\x09\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x80\x09\0\0\0\0\0\0\x04\0\
 +\0\0\x01\0\0\0\x90\x09\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xa0\x09\0\0\0\0\0\0\x04\
 +\0\0\0\x01\0\0\0\xb0\x09\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xc0\x09\0\0\0\0\0\0\
 +\x04\0\0\0\x01\0\0\0\xd0\x09\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xe0\x09\0\0\0\0\0\
 +\0\x04\0\0\0\x01\0\0\0\xf0\x09\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\0\x0a\0\0\0\0\0\
 +\0\x04\0\0\0\x01\0\0\0\x10\x0a\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x20\x0a\0\0\0\0\
 +\0\0\x04\0\0\0\x01\0\0\0\x30\x0a\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x40\x0a\0\0\0\
 +\0\0\0\x04\0\0\0\x01\0\0\0\x50\x0a\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x60\x0a\0\0\
 +\0\0\0\0\x04\0\0\0\x01\0\0\0\x70\x0a\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x80\x0a\0\
 +\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x90\x0a\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xa0\x0a\
 +\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xb0\x0a\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xc0\
 +\x0a\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xd0\x0a\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\
 +\xe0\x0a\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xf0\x0a\0\0\0\0\0\0\x04\0\0\0\x01\0\0\
 +\0\0\x0b\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x10\x0b\0\0\0\0\0\0\x04\0\0\0\x01\0\0\
 +\0\x20\x0b\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x30\x0b\0\0\0\0\0\0\x04\0\0\0\x01\0\
 +\0\0\x40\x0b\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x50\x0b\0\0\0\0\0\0\x04\0\0\0\x01\
 +\0\0\0\x60\x0b\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x70\x0b\0\0\0\0\0\0\x04\0\0\0\
 +\x01\0\0\0\x80\x0b\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x90\x0b\0\0\0\0\0\0\x04\0\0\
 +\0\x01\0\0\0\xa0\x0b\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xb0\x0b\0\0\0\0\0\0\x04\0\
 +\0\0\x01\0\0\0\xc0\x0b\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xd0\x0b\0\0\0\0\0\0\x04\
 +\0\0\0\x01\0\0\0\xe0\x0b\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xf0\x0b\0\0\0\0\0\0\
 +\x04\0\0\0\x01\0\0\0\0\x0c\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x10\x0c\0\0\0\0\0\0\
 +\x04\0\0\0\x01\0\0\0\x20\x0c\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x30\x0c\0\0\0\0\0\
 +\0\x04\0\0\0\x01\0\0\0\x40\x0c\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x50\x0c\0\0\0\0\
 +\0\0\x04\0\0\0\x01\0\0\0\x60\x0c\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x70\x0c\0\0\0\
 +\0\0\0\x04\0\0\0\x01\0\0\0\x80\x0c\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x90\x0c\0\0\
 +\0\0\0\0\x04\0\0\0\x01\0\0\0\xa0\x0c\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xb0\x0c\0\
 +\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xc0\x0c\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xd0\x0c\
 +\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xe0\x0c\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xf0\
 +\x0c\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\0\x0d\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x10\
 +\x0d\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x20\x0d\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\
 +\x30\x0d\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x40\x0d\0\0\0\0\0\0\x04\0\0\0\x01\0\0\
 +\0\x3f\x40\x41\x42\x43\0\x74\x61\x70\x5f\x72\x73\x73\x5f\x6d\x61\x70\x5f\x74\
 +\x6f\x65\x70\x6c\x69\x74\x7a\x5f\x6b\x65\x79\0\x2e\x74\x65\x78\x74\0\x2e\x72\
 +\x65\x6c\x2e\x42\x54\x46\x2e\x65\x78\x74\0\x2e\x72\x65\x6c\x73\x6f\x63\x6b\x65\
 +\x74\0\x2e\x6d\x61\x70\x73\0\x74\x61\x70\x5f\x72\x73\x73\x5f\x6d\x61\x70\x5f\
 +\x63\x6f\x6e\x66\x69\x67\x75\x72\x61\x74\x69\x6f\x6e\x73\0\x74\x75\x6e\x5f\x72\
 +\x73\x73\x5f\x73\x74\x65\x65\x72\x69\x6e\x67\x5f\x70\x72\x6f\x67\0\x2e\x6c\x6c\
 +\x76\x6d\x5f\x61\x64\x64\x72\x73\x69\x67\0\x5f\x6c\x69\x63\x65\x6e\x73\x65\0\
 +\x74\x61\x70\x5f\x72\x73\x73\x5f\x6d\x61\x70\x5f\x69\x6e\x64\x69\x72\x65\x63\
 +\x74\x69\x6f\x6e\x5f\x74\x61\x62\x6c\x65\0\x2e\x73\x74\x72\x74\x61\x62\0\x2e\
 +\x73\x79\x6d\x74\x61\x62\0\x2e\x72\x65\x6c\x2e\x42\x54\x46\0\x4c\x42\x42\x30\
 +\x5f\x39\0\x4c\x42\x42\x30\x5f\x38\x39\0\x4c\x42\x42\x30\x5f\x36\x39\0\x4c\x42\
 +\x42\x30\x5f\x35\x39\0\x4c\x42\x42\x30\x5f\x34\x39\0\x4c\x42\x42\x30\x5f\x33\
 +\x39\0\x4c\x42\x42\x30\x5f\x31\x30\x39\0\x4c\x42\x42\x30\x5f\x39\x38\0\x4c\x42\
 +\x42\x30\x5f\x37\x38\0\x4c\x42\x42\x30\x5f\x31\x38\0\x4c\x42\x42\x30\x5f\x31\
 +\x30\x38\0\x4c\x42\x42\x30\x5f\x38\x37\0\x4c\x42\x42\x30\x5f\x34\x37\0\x4c\x42\
 +\x42\x30\x5f\x31\x37\0\x4c\x42\x42\x30\x5f\x31\x30\x37\0\x4c\x42\x42\x30\x5f\
 +\x39\x36\0\x4c\x42\x42\x30\x5f\x37\x36\0\x4c\x42\x42\x30\x5f\x36\x36\0\x4c\x42\
 +\x42\x30\x5f\x34\x36\0\x4c\x42\x42\x30\x5f\x36\x35\0\x4c\x42\x42\x30\x5f\x34\
 +\x35\0\x4c\x42\x42\x30\x5f\x33\x35\0\x4c\x42\x42\x30\x5f\x31\x30\x35\0\x4c\x42\
 +\x42\x30\x5f\x34\0\x4c\x42\x42\x30\x5f\x35\x34\0\x4c\x42\x42\x30\x5f\x34\x34\0\
  \x4c\x42\x42\x30\x5f\x33\x34\0\x4c\x42\x42\x30\x5f\x31\x30\x34\0\x4c\x42\x42\
 -\x30\x5f\x38\x33\0\x4c\x42\x42\x30\x5f\x35\x33\0\x4c\x42\x42\x30\x5f\x32\x33\0\
 -\x4c\x42\x42\x30\x5f\x31\x30\x33\0\x4c\x42\x42\x30\x5f\x39\x32\0\x4c\x42\x42\
 -\x30\x5f\x38\x32\0\x4c\x42\x42\x30\x5f\x37\x32\0\x4c\x42\x42\x30\x5f\x36\x32\0\
 -\x4c\x42\x42\x30\x5f\x35\x32\0\x4c\x42\x42\x30\x5f\x34\x32\0\x4c\x42\x42\x30\
 -\x5f\x32\x32\0\x4c\x42\x42\x30\x5f\x31\x30\x32\0\x4c\x42\x42\x30\x5f\x38\x31\0\
 -\x4c\x42\x42\x30\x5f\x36\x31\0\x4c\x42\x42\x30\x5f\x35\x31\0\x4c\x42\x42\x30\
 -\x5f\x31\x31\0\x4c\x42\x42\x30\x5f\x39\x30\0\x4c\x42\x42\x30\x5f\x37\x30\0\x4c\
 -\x42\x42\x30\x5f\x36\x30\0\x4c\x42\x42\x30\x5f\x35\x30\0\x4c\x42\x42\x30\x5f\
 -\x34\x30\0\x4c\x42\x42\x30\x5f\x32\x30\0\x4c\x42\x42\x30\x5f\x31\x31\x30\0\0\0\
 +\x30\x5f\x39\x33\0\x4c\x42\x42\x30\x5f\x38\x33\0\x4c\x42\x42\x30\x5f\x35\x33\0\
 +\x4c\x42\x42\x30\x5f\x34\x33\0\x4c\x42\x42\x30\x5f\x33\x33\0\x4c\x42\x42\x30\
 +\x5f\x32\x33\0\x4c\x42\x42\x30\x5f\x31\x30\x33\0\x4c\x42\x42\x30\x5f\x38\x32\0\
 +\x4c\x42\x42\x30\x5f\x35\x32\0\x4c\x42\x42\x30\x5f\x32\x32\0\x4c\x42\x42\x30\
 +\x5f\x31\x30\x32\0\x4c\x42\x42\x30\x5f\x39\x31\0\x4c\x42\x42\x30\x5f\x38\x31\0\
 +\x4c\x42\x42\x30\x5f\x37\x31\0\x4c\x42\x42\x30\x5f\x36\x31\0\x4c\x42\x42\x30\
 +\x5f\x35\x31\0\x4c\x42\x42\x30\x5f\x34\x31\0\x4c\x42\x42\x30\x5f\x31\x31\0\x4c\
 +\x42\x42\x30\x5f\x31\x30\x31\0\x4c\x42\x42\x30\x5f\x38\x30\0\x4c\x42\x42\x30\
 +\x5f\x36\x30\0\x4c\x42\x42\x30\x5f\x35\x30\0\x4c\x42\x42\x30\x5f\x32\x30\0\0\0\
  \0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\
 -\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xae\0\0\0\x03\0\0\0\
 -\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x25\x4a\0\0\0\0\0\0\x6d\x02\0\0\0\0\0\0\0\0\0\
 -\0\0\0\0\0\x01\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x1a\0\0\0\x01\0\0\0\x06\0\0\0\0\0\
 -\0\0\0\0\0\0\0\0\0\0\x40\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x04\0\0\
 -\0\0\0\0\0\0\0\0\0\0\0\0\0\x68\0\0\0\x01\0\0\0\x06\0\0\0\0\0\0\0\0\0\0\0\0\0\0\
 -\0\x40\0\0\0\0\0\0\0\xe0\x13\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x08\0\0\0\0\0\0\0\0\0\
 -\0\0\0\0\0\0\x64\0\0\0\x09\0\0\0\x40\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x40\x3d\0\0\
 -\0\0\0\0\x30\0\0\0\0\0\0\0\x0c\0\0\0\x03\0\0\0\x08\0\0\0\0\0\0\0\x10\0\0\0\0\0\
 -\0\0\x2d\0\0\0\x01\0\0\0\x03\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x20\x14\0\0\0\0\0\0\
 -\x60\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x08\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x88\0\0\0\
 -\x01\0\0\0\x03\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x80\x14\0\0\0\0\0\0\x07\0\0\0\0\0\
 -\0\0\0\0\0\0\0\0\0\0\x01\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xc2\0\0\0\x01\0\0\0\0\0\
 -\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x88\x14\0\0\0\0\0\0\x8d\x16\0\0\0\0\0\0\0\0\0\0\0\
 -\0\0\0\x04\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xbe\0\0\0\x09\0\0\0\x40\0\0\0\0\0\0\0\
 -\0\0\0\0\0\0\0\0\x70\x3d\0\0\0\0\0\0\x40\0\0\0\0\0\0\0\x0c\0\0\0\x07\0\0\0\x08\
 -\0\0\0\0\0\0\0\x10\0\0\0\0\0\0\0\x24\0\0\0\x01\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\
 -\0\0\0\x18\x2b\0\0\0\0\0\0\xa0\x0c\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x04\0\0\0\0\0\0\
 -\0\0\0\0\0\0\0\0\0\x20\0\0\0\x09\0\0\0\x40\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xb0\
 -\x3d\0\0\0\0\0\0\x70\x0c\0\0\0\0\0\0\x0c\0\0\0\x09\0\0\0\x08\0\0\0\0\0\0\0\x10\
 -\0\0\0\0\0\0\0\x79\0\0\0\x03\x4c\xff\x6f\0\0\0\x80\0\0\0\0\0\0\0\0\0\0\0\0\x20\
 -\x4a\0\0\0\0\0\0\x05\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x01\0\0\0\0\0\0\0\0\0\0\0\0\
 -\0\0\0\xb6\0\0\0\x02\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xb8\x37\0\0\0\0\0\0\
 -\x88\x05\0\0\0\0\0\0\x01\0\0\0\x36\0\0\0\x08\0\0\0\0\0\0\0\x18\0\0\0\0\0\0\0";
 +\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xa4\0\0\0\x03\0\0\0\0\0\0\0\0\0\
 +\0\0\0\0\0\0\0\0\0\0\xbd\x4b\0\0\0\0\0\0\x5b\x02\0\0\0\0\0\0\0\0\0\0\0\0\0\0\
 +\x01\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x1a\0\0\0\x01\0\0\0\x06\0\0\0\0\0\0\0\0\0\0\
 +\0\0\0\0\0\x40\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x04\0\0\0\0\0\0\0\
 +\0\0\0\0\0\0\0\0\x31\0\0\0\x01\0\0\0\x06\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x40\0\0\
 +\0\0\0\0\0\xc8\x13\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x08\0\0\0\0\0\0\0\0\0\0\0\0\0\0\
 +\0\x2d\0\0\0\x09\0\0\0\x40\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x28\x3e\0\0\0\0\0\0\
 +\x30\0\0\0\0\0\0\0\x0c\0\0\0\x03\0\0\0\x08\0\0\0\0\0\0\0\x10\0\0\0\0\0\0\0\x38\
 +\0\0\0\x01\0\0\0\x03\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x08\x14\0\0\0\0\0\0\x78\0\0\
 +\0\0\0\0\0\0\0\0\0\0\0\0\0\x08\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x7e\0\0\0\x01\0\0\
 +\0\x03\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x80\x14\0\0\0\0\0\0\x07\0\0\0\0\0\0\0\0\0\
 +\0\0\0\0\0\0\x01\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xb8\0\0\0\x01\0\0\0\0\0\0\0\0\0\
 +\0\0\0\0\0\0\0\0\0\0\x88\x14\0\0\0\0\0\0\xdb\x16\0\0\0\0\0\0\0\0\0\0\0\0\0\0\
 +\x04\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xb4\0\0\0\x09\0\0\0\x40\0\0\0\0\0\0\0\0\0\0\
 +\0\0\0\0\0\x58\x3e\0\0\0\0\0\0\x40\0\0\0\0\0\0\0\x0c\0\0\0\x07\0\0\0\x08\0\0\0\
 +\0\0\0\0\x10\0\0\0\0\0\0\0\x24\0\0\0\x01\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\
 +\x64\x2b\0\0\0\0\0\0\x50\x0d\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x04\0\0\0\0\0\0\0\0\0\
 +\0\0\0\0\0\0\x20\0\0\0\x09\0\0\0\x40\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x98\x3e\0\0\
 +\0\0\0\0\x20\x0d\0\0\0\0\0\0\x0c\0\0\0\x09\0\0\0\x08\0\0\0\0\0\0\0\x10\0\0\0\0\
 +\0\0\0\x6f\0\0\0\x03\x4c\xff\x6f\0\0\0\x80\0\0\0\0\0\0\0\0\0\0\0\0\xb8\x4b\0\0\
 +\0\0\0\0\x05\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x01\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\
 +\xac\0\0\0\x02\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xb8\x38\0\0\0\0\0\0\x70\
 +\x05\0\0\0\0\0\0\x01\0\0\0\x35\0\0\0\x08\0\0\0\0\0\0\0\x18\0\0\0\0\0\0\0";
  }
- #ifdef __cplusplus
+ /**
-diff --git a/meson.build b/meson.build
++ * Get the shadow vq vring address.
 + * @svq: Shadow virtqueue
 + * @addr: Destination to store address
 + */
 +void vhost_svq_get_vring_addr(const VhostShadowVirtqueue *svq,
 +                              struct vhost_vring_addr *addr)
 +{
 +    addr->desc_user_addr = (uint64_t)(intptr_t)svq->vring.desc;
 +    addr->avail_user_addr = (uint64_t)(intptr_t)svq->vring.avail;
 +    addr->used_user_addr = (uint64_t)(intptr_t)svq->vring.used;
 +}
 +
 +size_t vhost_svq_driver_area_size(const VhostShadowVirtqueue *svq)
 +{
 +    size_t desc_size = sizeof(vring_desc_t) * svq->vring.num;
 +    size_t avail_size = offsetof(vring_avail_t, ring) +
 +                                             sizeof(uint16_t) * svq->vring.num;
 +
 +    return ROUND_UP(desc_size + avail_size, qemu_real_host_page_size);
 +}
 +
 +size_t vhost_svq_device_area_size(const VhostShadowVirtqueue *svq)
 +{
 +    size_t used_size = offsetof(vring_used_t, ring) +
 +                                    sizeof(vring_used_elem_t) * svq->vring.num;
 +    return ROUND_UP(used_size, qemu_real_host_page_size);
 +}
 +
 +/**
   * Set a new file descriptor for the guest to kick the SVQ and notify for avail
   *
   * @svq: The svq
 diff --git a/hw/virtio/vhost-shadow-virtqueue.h b/hw/virtio/vhost-shadow-virtqueue.h
 index XXXXXXX..XXXXXXX 100644
---- a/meson.build
+--- a/hw/virtio/vhost-shadow-virtqueue.h
-+++ b/meson.build
++++ b/hw/virtio/vhost-shadow-virtqueue.h
-@@ -XXX,XX +XXX,XX @@ elif get_option('vduse_blk_export').disabled()
+@@ -XXX,XX +XXX,XX @@
- endif
+ #define VHOST_SHADOW_VIRTQUEUE_H
- # libbpf
+ #include "qemu/event_notifier.h"
--libbpf = dependency('libbpf', required: get_option('bpf'), method: 'pkg-config')
++#include "hw/virtio/virtio.h"
-+bpf_version = '1.1.0'
++#include "standard-headers/linux/vhost_types.h"
-+libbpf = dependency('libbpf', version: '>=' + bpf_version, required: get_option('bpf'), method: 'pkg-config')
- if libbpf.found() and not cc.links('''
+ /* Shadow virtqueue to relay notifications */
-    #include <bpf/libbpf.h>
+ typedef struct VhostShadowVirtqueue {
-+   #include <linux/bpf.h>
++    /* Shadow vring */
-    int main(void)
++    struct vring vring;
-    {
++
-+     // check flag availability
+     /* Shadow kick notifier, sent to vhost */
-+     int flag = BPF_F_MMAPABLE;
+     EventNotifier hdev_kick;
-      bpf_object__destroy_skeleton(NULL);
+     /* Shadow call notifier, sent to vhost */
-      return 0;
+@@ -XXX,XX +XXX,XX @@ bool vhost_svq_valid_features(uint64_t features, Error **errp);
-    }''', dependencies: libbpf)
-   libbpf = not_found
+ void vhost_svq_set_svq_kick_fd(VhostShadowVirtqueue *svq, int svq_kick_fd);
-   if get_option('bpf').enabled()
+ void vhost_svq_set_svq_call_fd(VhostShadowVirtqueue *svq, int call_fd);
--    error('libbpf skeleton test failed')
++void vhost_svq_get_vring_addr(const VhostShadowVirtqueue *svq,
-+    error('libbpf skeleton/mmaping test failed')
++                              struct vhost_vring_addr *addr);
-   else
++size_t vhost_svq_driver_area_size(const VhostShadowVirtqueue *svq);
--    warning('libbpf skeleton test failed, disabling')
++size_t vhost_svq_device_area_size(const VhostShadowVirtqueue *svq);
-+    warning('libbpf skeleton/mmaping test failed, disabling')
-   endif
+ void vhost_svq_stop(VhostShadowVirtqueue *svq);
  endif
 diff --git a/tools/ebpf/rss.bpf.c b/tools/ebpf/rss.bpf.c
 index XXXXXXX..XXXXXXX 100644
 --- a/tools/ebpf/rss.bpf.c
 +++ b/tools/ebpf/rss.bpf.c
@@ -XXX,XX +XXX,XX @@ struct {
      __uint(key_size, sizeof(__u32));
      __uint(value_size, sizeof(struct rss_config_t));
      __uint(max_entries, 1);
 +    __uint(map_flags, BPF_F_MMAPABLE);
  } tap_rss_map_configurations SEC(".maps");
  struct {
@@ -XXX,XX +XXX,XX @@ struct {
      __uint(key_size, sizeof(__u32));
      __uint(value_size, sizeof(struct toeplitz_key_data_t));
      __uint(max_entries, 1);
 +    __uint(map_flags, BPF_F_MMAPABLE);
  } tap_rss_map_toeplitz_key SEC(".maps");
  struct {
@@ -XXX,XX +XXX,XX @@ struct {
      __uint(key_size, sizeof(__u32));
      __uint(value_size, sizeof(__u16));
      __uint(max_entries, INDIRECTION_TABLE_SIZE);
 +    __uint(map_flags, BPF_F_MMAPABLE);
  } tap_rss_map_indirection_table SEC(".maps");
  static inline void net_rx_rss_add_chunk(__u8 *rss_input, size_t *bytes_written,
@@ -XXX,XX +XXX,XX @@ static inline __u32 calculate_rss_hash(struct __sk_buff *skb,
      return result;
  }
 -SEC("tun_rss_steering")
 +SEC("socket")
  int tun_rss_steering_prog(struct __sk_buff *skb)
  {
 --
 .7.4

-[PULL 08/17] igb: RX payload guest writting refactoring
+[PULL V3 07/15] vdpa: adapt vhost_ops callbacks to svq
-From: Tomasz Dzieciol <t.dzieciol@partner.samsung.com>
+From: Eugenio Pérez <eperezma@redhat.com>
-Refactoring is done in preparation for support of multiple advanced
+First half of the buffers forwarding part, preparing vhost-vdpa
-descriptors RX modes, especially packet-split modes.
+callbacks to SVQ to offer it. QEMU cannot enable it at this moment, so
 this is effectively dead code at the moment, but it helps to reduce
 patch size.
-Signed-off-by: Tomasz Dzieciol <t.dzieciol@partner.samsung.com>
+Signed-off-by: Eugenio Pérez <eperezma@redhat.com>
-Reviewed-by: Akihiko Odaki <akihiko.odaki@daynix.com>
+Acked-by: Michael S. Tsirkin <mst@redhat.com>
 Tested-by: Akihiko Odaki <akihiko.odaki@daynix.com>
 Signed-off-by: Jason Wang <jasowang@redhat.com>
 ---
- hw/net/e1000e_core.c     |  18 ++--
+ hw/virtio/vhost-vdpa.c | 48 +++++++++++++++++++++++++++++++++++++++++-------
- hw/net/igb_core.c        | 213 ++++++++++++++++++++++++++++++-----------------
+file changed, 41 insertions(+), 7 deletions(-)
  tests/qtest/libqos/igb.c |   5 ++
 files changed, 150 insertions(+), 86 deletions(-)
-diff --git a/hw/net/e1000e_core.c b/hw/net/e1000e_core.c
+diff --git a/hw/virtio/vhost-vdpa.c b/hw/virtio/vhost-vdpa.c
 index XXXXXXX..XXXXXXX 100644
---- a/hw/net/e1000e_core.c
+--- a/hw/virtio/vhost-vdpa.c
-+++ b/hw/net/e1000e_core.c
++++ b/hw/virtio/vhost-vdpa.c
-@@ -XXX,XX +XXX,XX @@ e1000e_write_hdr_to_rx_buffers(E1000ECore *core,
+@@ -XXX,XX +XXX,XX @@ static int vhost_vdpa_get_config(struct vhost_dev *dev, uint8_t *config,
- }
+     return ret;
+  }
- static void
--e1000e_write_to_rx_buffers(E1000ECore *core,
++static int vhost_vdpa_set_dev_vring_base(struct vhost_dev *dev,
--                           hwaddr ba[MAX_PS_BUFFERS],
++                                         struct vhost_vring_state *ring)
 -                           e1000e_ba_state *bastate,
 -                           const char *data,
 -                           dma_addr_t data_len)
 +e1000e_write_payload_frag_to_rx_buffers(E1000ECore *core,
 +                                        hwaddr ba[MAX_PS_BUFFERS],
 +                                        e1000e_ba_state *bastate,
 +                                        const char *data,
 +                                        dma_addr_t data_len)
  {
      while (data_len > 0) {
          uint32_t cur_buf_len = core->rxbuf_sizes[bastate->cur_idx];
@@ -XXX,XX +XXX,XX @@ e1000e_write_packet_to_guest(E1000ECore *core, struct NetRxPkt *pkt,
                  while (copy_size) {
                      iov_copy = MIN(copy_size, iov->iov_len - iov_ofs);
 -                    e1000e_write_to_rx_buffers(core, ba, &bastate,
 -                                            iov->iov_base + iov_ofs, iov_copy);
 +                    e1000e_write_payload_frag_to_rx_buffers(core, ba, &bastate,
 +                                                            iov->iov_base +
 +                                                            iov_ofs,
 +                                                            iov_copy);
                      copy_size -= iov_copy;
                      iov_ofs += iov_copy;
@@ -XXX,XX +XXX,XX @@ e1000e_write_packet_to_guest(E1000ECore *core, struct NetRxPkt *pkt,
                  if (desc_offset + desc_size >= total_size) {
                      /* Simulate FCS checksum presence in the last descriptor */
 -                    e1000e_write_to_rx_buffers(core, ba, &bastate,
 +                    e1000e_write_payload_frag_to_rx_buffers(core, ba, &bastate,
                            (const char *) &fcs_pad, e1000x_fcs_len(core->mac));
                  }
              }
 diff --git a/hw/net/igb_core.c b/hw/net/igb_core.c
 index XXXXXXX..XXXXXXX 100644
 --- a/hw/net/igb_core.c
 +++ b/hw/net/igb_core.c
@@ -XXX,XX +XXX,XX @@ igb_has_rxbufs(IGBCore *core, const E1000ERingInfo *r, size_t total_size)
                           bufsize;
  }
 +static uint32_t
 +igb_rxhdrbufsize(IGBCore *core, const E1000ERingInfo *r)
 +{
-+    uint32_t srrctl = core->mac[E1000_SRRCTL(r->idx) >> 2];
++    trace_vhost_vdpa_set_vring_base(dev, ring->index, ring->num);
-+    return (srrctl & E1000_SRRCTL_BSIZEHDRSIZE_MASK) >>
++    return vhost_vdpa_call(dev, VHOST_SET_VRING_BASE, ring);
 +           E1000_SRRCTL_BSIZEHDRSIZE_SHIFT;
 +}
 +
- void
+ static int vhost_vdpa_set_vring_dev_kick(struct vhost_dev *dev,
- igb_start_recv(IGBCore *core)
+                                          struct vhost_vring_file *file)
  {
-@@ -XXX,XX +XXX,XX @@ igb_read_adv_rx_descr(IGBCore *core, union e1000_adv_rx_desc *desc,
+@@ -XXX,XX +XXX,XX @@ static int vhost_vdpa_set_vring_dev_call(struct vhost_dev *dev,
-     *buff_addr = le64_to_cpu(desc->read.pkt_addr);
+     return vhost_vdpa_call(dev, VHOST_SET_VRING_CALL, file);
  }
-+typedef struct IGBPacketRxDMAState {
++static int vhost_vdpa_set_vring_dev_addr(struct vhost_dev *dev,
-+    size_t size;
++                                         struct vhost_vring_addr *addr)
-+    size_t total_size;
++{
-+    size_t ps_hdr_len;
++    trace_vhost_vdpa_set_vring_addr(dev, addr->index, addr->flags,
-+    size_t desc_size;
++                                addr->desc_user_addr, addr->used_user_addr,
-+    size_t desc_offset;
++                                addr->avail_user_addr,
-+    uint32_t rx_desc_packet_buf_size;
++                                addr->log_guest_addr);
 +    uint32_t rx_desc_header_buf_size;
 +    struct iovec *iov;
 +    size_t iov_ofs;
 +    bool is_first;
 +    uint16_t written;
 +    hwaddr ba;
 +} IGBPacketRxDMAState;
 +
- static inline void
++    return vhost_vdpa_call(dev, VHOST_SET_VRING_ADDR, addr);
- igb_read_rx_descr(IGBCore *core, union e1000_rx_desc_union *desc,
++
                    hwaddr *buff_addr)
@@ -XXX,XX +XXX,XX @@ igb_pci_dma_write_rx_desc(IGBCore *core, PCIDevice *dev, dma_addr_t addr,
  }
  static void
 -igb_write_to_rx_buffers(IGBCore *core,
 -                        PCIDevice *d,
 -                        hwaddr ba,
 -                        uint16_t *written,
 -                        const char *data,
 -                        dma_addr_t data_len)
 -{
 -    trace_igb_rx_desc_buff_write(ba, *written, data, data_len);
 -    pci_dma_write(d, ba + *written, data, data_len);
 -    *written += data_len;
 -}
 -
 -static void
  igb_update_rx_stats(IGBCore *core, const E1000ERingInfo *rxi,
                      size_t pkt_size, size_t pkt_fcs_size)
  {
@@ -XXX,XX +XXX,XX @@ igb_rx_descr_threshold_hit(IGBCore *core, const E1000ERingInfo *rxi)
  }
  static void
 +igb_truncate_to_descriptor_size(IGBPacketRxDMAState *pdma_st, size_t *size)
 +{
 +    if (*size > pdma_st->rx_desc_packet_buf_size) {
 +        *size = pdma_st->rx_desc_packet_buf_size;
 +    }
 +}
 +
-+static void
+ /**
-+igb_write_payload_frag_to_rx_buffers(IGBCore *core,
+  * Set the shadow virtqueue descriptors to the device
-+                                     PCIDevice *d,
+  *
-+                                     hwaddr ba,
+@@ -XXX,XX +XXX,XX @@ static int vhost_vdpa_set_log_base(struct vhost_dev *dev, uint64_t base,
-+                                     uint16_t *written,
+ static int vhost_vdpa_set_vring_addr(struct vhost_dev *dev,
-+                                     uint32_t cur_buf_len,
+                                        struct vhost_vring_addr *addr)
-+                                     const char *data,
+ {
-+                                     dma_addr_t data_len)
+-    trace_vhost_vdpa_set_vring_addr(dev, addr->index, addr->flags,
-+{
+-                                    addr->desc_user_addr, addr->used_user_addr,
-+    trace_igb_rx_desc_buff_write(ba, *written, data, data_len);
+-                                    addr->avail_user_addr,
-+    pci_dma_write(d, ba + *written, data, data_len);
+-                                    addr->log_guest_addr);
-+    *written += data_len;
+-    return vhost_vdpa_call(dev, VHOST_SET_VRING_ADDR, addr);
-+}
++    struct vhost_vdpa *v = dev->opaque;
 +
-+static void
++    if (v->shadow_vqs_enabled) {
-+igb_write_payload_to_rx_buffers(IGBCore *core,
++        /*
-+                                struct NetRxPkt *pkt,
++         * Device vring addr was set at device start. SVQ base is handled by
-+                                PCIDevice *d,
++         * VirtQueue code.
-+                                IGBPacketRxDMAState *pdma_st,
++         */
-+                                size_t *copy_size)
++        return 0;
 +{
 +    static const uint32_t fcs_pad;
 +    size_t iov_copy;
 +
 +    /* Copy packet payload */
 +    while (*copy_size) {
 +        iov_copy = MIN(*copy_size, pdma_st->iov->iov_len - pdma_st->iov_ofs);
 +        igb_write_payload_frag_to_rx_buffers(core, d,
 +                                             pdma_st->ba,
 +                                             &pdma_st->written,
 +                                             pdma_st->rx_desc_packet_buf_size,
 +                                             pdma_st->iov->iov_base +
 +                                             pdma_st->iov_ofs,
 +                                             iov_copy);
 +
 +        *copy_size -= iov_copy;
 +        pdma_st->iov_ofs += iov_copy;
 +        if (pdma_st->iov_ofs == pdma_st->iov->iov_len) {
 +            pdma_st->iov++;
 +            pdma_st->iov_ofs = 0;
 +        }
 +    }
 +
-+    if (pdma_st->desc_offset + pdma_st->desc_size >= pdma_st->total_size) {
++    return vhost_vdpa_set_vring_dev_addr(dev, addr);
-+        /* Simulate FCS checksum presence in the last descriptor */
+ }
-+        igb_write_payload_frag_to_rx_buffers(core, d,
-+                                             pdma_st->ba,
+ static int vhost_vdpa_set_vring_num(struct vhost_dev *dev,
-+                                             &pdma_st->written,
+@@ -XXX,XX +XXX,XX @@ static int vhost_vdpa_set_vring_num(struct vhost_dev *dev,
-+                                             pdma_st->rx_desc_packet_buf_size,
+ static int vhost_vdpa_set_vring_base(struct vhost_dev *dev,
-+                                             (const char *) &fcs_pad,
+                                        struct vhost_vring_state *ring)
-+                                             e1000x_fcs_len(core->mac));
+ {
-+    }
+-    trace_vhost_vdpa_set_vring_base(dev, ring->index, ring->num);
-+}
+-    return vhost_vdpa_call(dev, VHOST_SET_VRING_BASE, ring);
 +    struct vhost_vdpa *v = dev->opaque;
 +
-+static void
++    if (v->shadow_vqs_enabled) {
-+igb_write_to_rx_buffers(IGBCore *core,
++        /*
-+                        struct NetRxPkt *pkt,
++         * Device vring base was set at device start. SVQ base is handled by
-+                        PCIDevice *d,
++         * VirtQueue code.
-+                        IGBPacketRxDMAState *pdma_st)
++         */
-+{
++        return 0;
 +    size_t copy_size;
 +
 +    if (!pdma_st->ba) {
 +        /* as per intel docs; skip descriptors with null buf addr */
 +        trace_e1000e_rx_null_descriptor();
 +        return;
 +    }
 +
-+    if (pdma_st->desc_offset >= pdma_st->size) {
++    return vhost_vdpa_set_dev_vring_base(dev, ring);
 +        return;
 +    }
 +
 +    pdma_st->desc_size = pdma_st->total_size - pdma_st->desc_offset;
 +    igb_truncate_to_descriptor_size(pdma_st, &pdma_st->desc_size);
 +    copy_size = pdma_st->size - pdma_st->desc_offset;
 +    igb_truncate_to_descriptor_size(pdma_st, &copy_size);
 +    igb_write_payload_to_rx_buffers(core, pkt, d, pdma_st, &copy_size);
 +}
 +
 +static void
  igb_write_packet_to_guest(IGBCore *core, struct NetRxPkt *pkt,
                            const E1000E_RxRing *rxr,
                            const E1000E_RSSInfo *rss_info,
@@ -XXX,XX +XXX,XX @@ igb_write_packet_to_guest(IGBCore *core, struct NetRxPkt *pkt,
      PCIDevice *d;
      dma_addr_t base;
      union e1000_rx_desc_union desc;
 -    size_t desc_size;
 -    size_t desc_offset = 0;
 -    size_t iov_ofs = 0;
 -
 -    struct iovec *iov = net_rx_pkt_get_iovec(pkt);
 -    size_t size = net_rx_pkt_get_total_len(pkt);
 -    size_t total_size = size + e1000x_fcs_len(core->mac);
 -    const E1000ERingInfo *rxi = rxr->i;
 -    size_t bufsize = igb_rxbufsize(core, rxi);
 -
 +    const E1000ERingInfo *rxi;
 +    size_t rx_desc_len;
 +
 +    IGBPacketRxDMAState pdma_st = {0};
 +    pdma_st.is_first = true;
 +    pdma_st.size = net_rx_pkt_get_total_len(pkt);
 +    pdma_st.total_size = pdma_st.size + e1000x_fcs_len(core->mac);
 +
 +    rxi = rxr->i;
 +    rx_desc_len = core->rx_desc_len;
 +    pdma_st.rx_desc_packet_buf_size = igb_rxbufsize(core, rxi);
 +    pdma_st.rx_desc_header_buf_size = igb_rxhdrbufsize(core, rxi);
 +    pdma_st.iov = net_rx_pkt_get_iovec(pkt);
      d = pcie_sriov_get_vf_at_index(core->owner, rxi->idx % 8);
      if (!d) {
          d = core->owner;
      }
      do {
 -        hwaddr ba;
 -        uint16_t written = 0;
 +        pdma_st.written = 0;
          bool is_last = false;
 -        desc_size = total_size - desc_offset;
 -
 -        if (desc_size > bufsize) {
 -            desc_size = bufsize;
 -        }
 -
          if (igb_ring_empty(core, rxi)) {
              return;
          }
          base = igb_ring_head_descr(core, rxi);
 +        pci_dma_read(d, base, &desc, rx_desc_len);
 +        trace_e1000e_rx_descr(rxi->idx, base, rx_desc_len);
 -        pci_dma_read(d, base, &desc, core->rx_desc_len);
 -
 -        trace_e1000e_rx_descr(rxi->idx, base, core->rx_desc_len);
 -
 -        igb_read_rx_descr(core, &desc, &ba);
 -
 -        if (ba) {
 -            if (desc_offset < size) {
 -                static const uint32_t fcs_pad;
 -                size_t iov_copy;
 -                size_t copy_size = size - desc_offset;
 -                if (copy_size > bufsize) {
 -                    copy_size = bufsize;
 -                }
 -
 -                /* Copy packet payload */
 -                while (copy_size) {
 -                    iov_copy = MIN(copy_size, iov->iov_len - iov_ofs);
 -
 -                    igb_write_to_rx_buffers(core, d, ba, &written,
 -                                            iov->iov_base + iov_ofs, iov_copy);
 +        igb_read_rx_descr(core, &desc, &pdma_st.ba);
 -                    copy_size -= iov_copy;
 -                    iov_ofs += iov_copy;
 -                    if (iov_ofs == iov->iov_len) {
 -                        iov++;
 -                        iov_ofs = 0;
 -                    }
 -                }
 -
 -                if (desc_offset + desc_size >= total_size) {
 -                    /* Simulate FCS checksum presence in the last descriptor */
 -                    igb_write_to_rx_buffers(core, d, ba, &written,
 -                          (const char *) &fcs_pad, e1000x_fcs_len(core->mac));
 -                }
 -            }
 -        } else { /* as per intel docs; skip descriptors with null buf addr */
 -            trace_e1000e_rx_null_descriptor();
 -        }
 -        desc_offset += desc_size;
 -        if (desc_offset >= total_size) {
 +        igb_write_to_rx_buffers(core, pkt, d, &pdma_st);
 +        pdma_st.desc_offset += pdma_st.desc_size;
 +        if (pdma_st.desc_offset >= pdma_st.total_size) {
              is_last = true;
          }
          igb_write_rx_descr(core, &desc, is_last ? core->rx_pkt : NULL,
 -                           rss_info, etqf, ts, written);
 -        igb_pci_dma_write_rx_desc(core, d, base, &desc, core->rx_desc_len);
 -
 -        igb_ring_advance(core, rxi, core->rx_desc_len / E1000_MIN_RX_DESC_LEN);
 -
 -    } while (desc_offset < total_size);
 +                           rss_info, etqf, ts, pdma_st.written);
 +        igb_pci_dma_write_rx_desc(core, d, base, &desc, rx_desc_len);
 +        igb_ring_advance(core, rxi, rx_desc_len / E1000_MIN_RX_DESC_LEN);
 +    } while (pdma_st.desc_offset < pdma_st.total_size);
 -    igb_update_rx_stats(core, rxi, size, total_size);
 +    igb_update_rx_stats(core, rxi, pdma_st.size, pdma_st.total_size);
  }
- static bool
+ static int vhost_vdpa_get_vring_base(struct vhost_dev *dev,
 diff --git a/tests/qtest/libqos/igb.c b/tests/qtest/libqos/igb.c
 index XXXXXXX..XXXXXXX 100644
 --- a/tests/qtest/libqos/igb.c
 +++ b/tests/qtest/libqos/igb.c
@@ -XXX,XX +XXX,XX @@ static void igb_pci_start_hw(QOSGraphObject *obj)
                          E1000_RAH_AV | E1000_RAH_POOL_1 |
                          le16_to_cpu(*(uint16_t *)(address + 4)));
 +    /* Set supported receive descriptor mode */
 +    e1000e_macreg_write(&d->e1000e,
 +                        E1000_SRRCTL(0),
 +                        E1000_SRRCTL_DESCTYPE_ADV_ONEBUF);
 +
      /* Enable receive */
      e1000e_macreg_write(&d->e1000e, E1000_RFCTL, E1000_RFCTL_EXTEN);
      e1000e_macreg_write(&d->e1000e, E1000_RCTL, E1000_RCTL_EN);
 --
 .7.4

-[PULL 13/17] ebpf: Added eBPF map update through mmap.
+[PULL V3 08/15] vhost: Shadow virtqueue buffers forwarding
-From: Andrew Melnychenko <andrew@daynix.com>
+From: Eugenio Pérez <eperezma@redhat.com>
-Changed eBPF map updates through mmaped array.
+Initial version of shadow virtqueue that actually forward buffers. There
-Mmaped arrays provide direct access to map data.
+is no iommu support at the moment, and that will be addressed in future
-It should omit using bpf_map_update_elem() call,
+patches of this series. Since all vhost-vdpa devices use forced IOMMU,
-which may require capabilities that are not present.
+this means that SVQ is not usable at this point of the series on any
+device.
-Signed-off-by: Andrew Melnychenko <andrew@daynix.com>
 For simplicity it only supports modern devices, that expects vring
 in little endian, with split ring and no event idx or indirect
 descriptors. Support for them will not be added in this series.
 It reuses the VirtQueue code for the device part. The driver part is
 based on Linux's virtio_ring driver, but with stripped functionality
 and optimizations so it's easier to review.
 However, forwarding buffers have some particular pieces: One of the most
 unexpected ones is that a guest's buffer can expand through more than
 one descriptor in SVQ. While this is handled gracefully by qemu's
 emulated virtio devices, it may cause unexpected SVQ queue full. This
 patch also solves it by checking for this condition at both guest's
 kicks and device's calls. The code may be more elegant in the future if
 SVQ code runs in its own iocontext.
 Signed-off-by: Eugenio Pérez <eperezma@redhat.com>
 Acked-by: Michael S. Tsirkin <mst@redhat.com>
 Signed-off-by: Jason Wang <jasowang@redhat.com>
 ---
- ebpf/ebpf_rss.c | 117 +++++++++++++++++++++++++++++++++++++++++++++-----------
+ hw/virtio/vhost-shadow-virtqueue.c | 352 ++++++++++++++++++++++++++++++++++++-
- ebpf/ebpf_rss.h |   5 +++
+ hw/virtio/vhost-shadow-virtqueue.h |  26 +++
-files changed, 99 insertions(+), 23 deletions(-)
+ hw/virtio/vhost-vdpa.c             | 155 +++++++++++++++-
+files changed, 522 insertions(+), 11 deletions(-)
-diff --git a/ebpf/ebpf_rss.c b/ebpf/ebpf_rss.c
 diff --git a/hw/virtio/vhost-shadow-virtqueue.c b/hw/virtio/vhost-shadow-virtqueue.c
 index XXXXXXX..XXXXXXX 100644
---- a/ebpf/ebpf_rss.c
+--- a/hw/virtio/vhost-shadow-virtqueue.c
-+++ b/ebpf/ebpf_rss.c
++++ b/hw/virtio/vhost-shadow-virtqueue.c
-@@ -XXX,XX +XXX,XX @@ void ebpf_rss_init(struct EBPFRSSContext *ctx)
+@@ -XXX,XX +XXX,XX @@
  #include "qemu/error-report.h"
  #include "qapi/error.h"
  #include "qemu/main-loop.h"
 +#include "qemu/log.h"
 +#include "qemu/memalign.h"
  #include "linux-headers/linux/vhost.h"
  /**
@@ -XXX,XX +XXX,XX @@ bool vhost_svq_valid_features(uint64_t features, Error **errp)
  }
  /**
 - * Forward guest notifications.
 + * Number of descriptors that the SVQ can make available from the guest.
 + *
 + * @svq: The svq
 + */
 +static uint16_t vhost_svq_available_slots(const VhostShadowVirtqueue *svq)
 +{
 +    return svq->vring.num - (svq->shadow_avail_idx - svq->shadow_used_idx);
 +}
 +
 +static void vhost_vring_write_descs(VhostShadowVirtqueue *svq,
 +                                    const struct iovec *iovec, size_t num,
 +                                    bool more_descs, bool write)
 +{
 +    uint16_t i = svq->free_head, last = svq->free_head;
 +    unsigned n;
 +    uint16_t flags = write ? cpu_to_le16(VRING_DESC_F_WRITE) : 0;
 +    vring_desc_t *descs = svq->vring.desc;
 +
 +    if (num == 0) {
 +        return;
 +    }
 +
 +    for (n = 0; n < num; n++) {
 +        if (more_descs || (n + 1 < num)) {
 +            descs[i].flags = flags | cpu_to_le16(VRING_DESC_F_NEXT);
 +        } else {
 +            descs[i].flags = flags;
 +        }
 +        descs[i].addr = cpu_to_le64((hwaddr)(intptr_t)iovec[n].iov_base);
 +        descs[i].len = cpu_to_le32(iovec[n].iov_len);
 +
 +        last = i;
 +        i = cpu_to_le16(descs[i].next);
 +    }
 +
 +    svq->free_head = le16_to_cpu(descs[last].next);
 +}
 +
 +static bool vhost_svq_add_split(VhostShadowVirtqueue *svq,
 +                                VirtQueueElement *elem, unsigned *head)
 +{
 +    unsigned avail_idx;
 +    vring_avail_t *avail = svq->vring.avail;
 +
 +    *head = svq->free_head;
 +
 +    /* We need some descriptors here */
 +    if (unlikely(!elem->out_num && !elem->in_num)) {
 +        qemu_log_mask(LOG_GUEST_ERROR,
 +                      "Guest provided element with no descriptors");
 +        return false;
 +    }
 +
 +    vhost_vring_write_descs(svq, elem->out_sg, elem->out_num, elem->in_num > 0,
 +                            false);
 +    vhost_vring_write_descs(svq, elem->in_sg, elem->in_num, false, true);
 +
 +    /*
 +     * Put the entry in the available array (but don't update avail->idx until
 +     * they do sync).
 +     */
 +    avail_idx = svq->shadow_avail_idx & (svq->vring.num - 1);
 +    avail->ring[avail_idx] = cpu_to_le16(*head);
 +    svq->shadow_avail_idx++;
 +
 +    /* Update the avail index after write the descriptor */
 +    smp_wmb();
 +    avail->idx = cpu_to_le16(svq->shadow_avail_idx);
 +
 +    return true;
 +}
 +
 +static bool vhost_svq_add(VhostShadowVirtqueue *svq, VirtQueueElement *elem)
 +{
 +    unsigned qemu_head;
 +    bool ok = vhost_svq_add_split(svq, elem, &qemu_head);
 +    if (unlikely(!ok)) {
 +        return false;
 +    }
 +
 +    svq->ring_id_maps[qemu_head] = elem;
 +    return true;
 +}
 +
 +static void vhost_svq_kick(VhostShadowVirtqueue *svq)
 +{
 +    /*
 +     * We need to expose the available array entries before checking the used
 +     * flags
 +     */
 +    smp_mb();
 +    if (svq->vring.used->flags & VRING_USED_F_NO_NOTIFY) {
 +        return;
 +    }
 +
 +    event_notifier_set(&svq->hdev_kick);
 +}
 +
 +/**
 + * Forward available buffers.
 + *
 + * @svq: Shadow VirtQueue
 + *
 + * Note that this function does not guarantee that all guest's available
 + * buffers are available to the device in SVQ avail ring. The guest may have
 + * exposed a GPA / GIOVA contiguous buffer, but it may not be contiguous in
 + * qemu vaddr.
 + *
 + * If that happens, guest's kick notifications will be disabled until the
 + * device uses some buffers.
 + */
 +static void vhost_handle_guest_kick(VhostShadowVirtqueue *svq)
 +{
 +    /* Clear event notifier */
 +    event_notifier_test_and_clear(&svq->svq_kick);
 +
 +    /* Forward to the device as many available buffers as possible */
 +    do {
 +        virtio_queue_set_notification(svq->vq, false);
 +
 +        while (true) {
 +            VirtQueueElement *elem;
 +            bool ok;
 +
 +            if (svq->next_guest_avail_elem) {
 +                elem = g_steal_pointer(&svq->next_guest_avail_elem);
 +            } else {
 +                elem = virtqueue_pop(svq->vq, sizeof(*elem));
 +            }
 +
 +            if (!elem) {
 +                break;
 +            }
 +
 +            if (elem->out_num + elem->in_num > vhost_svq_available_slots(svq)) {
 +                /*
 +                 * This condition is possible since a contiguous buffer in GPA
 +                 * does not imply a contiguous buffer in qemu's VA
 +                 * scatter-gather segments. If that happens, the buffer exposed
 +                 * to the device needs to be a chain of descriptors at this
 +                 * moment.
 +                 *
 +                 * SVQ cannot hold more available buffers if we are here:
 +                 * queue the current guest descriptor and ignore further kicks
 +                 * until some elements are used.
 +                 */
 +                svq->next_guest_avail_elem = elem;
 +                return;
 +            }
 +
 +            ok = vhost_svq_add(svq, elem);
 +            if (unlikely(!ok)) {
 +                /* VQ is broken, just return and ignore any other kicks */
 +                return;
 +            }
 +            vhost_svq_kick(svq);
 +        }
 +
 +        virtio_queue_set_notification(svq->vq, true);
 +    } while (!virtio_queue_empty(svq->vq));
 +}
 +
 +/**
 + * Handle guest's kick.
   *
   * @n: guest kick event notifier, the one that guest set to notify svq.
   */
 -static void vhost_handle_guest_kick(EventNotifier *n)
 +static void vhost_handle_guest_kick_notifier(EventNotifier *n)
  {
-     if (ctx != NULL) {
+     VhostShadowVirtqueue *svq = container_of(n, VhostShadowVirtqueue, svq_kick);
-         ctx->obj = NULL;
+     event_notifier_test_and_clear(n);
-+        ctx->program_fd = -1;
+-    event_notifier_set(&svq->hdev_kick);
-+        ctx->map_configuration = -1;
++    vhost_handle_guest_kick(svq);
-+        ctx->map_toeplitz_key = -1;
++}
-+        ctx->map_indirections_table = -1;
++
-+
++static bool vhost_svq_more_used(VhostShadowVirtqueue *svq)
-+        ctx->mmap_configuration = NULL;
++{
-+        ctx->mmap_toeplitz_key = NULL;
++    if (svq->last_used_idx != svq->shadow_used_idx) {
-+        ctx->mmap_indirections_table = NULL;
++        return true;
 +    }
 +
 +    svq->shadow_used_idx = cpu_to_le16(svq->vring.used->idx);
 +
 +    return svq->last_used_idx != svq->shadow_used_idx;
  }
  /**
 - * Forward vhost notifications
 + * Enable vhost device calls after disable them.
 + *
 + * @svq: The svq
 + *
 + * It returns false if there are pending used buffers from the vhost device,
 + * avoiding the possible races between SVQ checking for more work and enabling
 + * callbacks. True if SVQ used vring has no more pending buffers.
 + */
 +static bool vhost_svq_enable_notification(VhostShadowVirtqueue *svq)
 +{
 +    svq->vring.avail->flags &= ~cpu_to_le16(VRING_AVAIL_F_NO_INTERRUPT);
 +    /* Make sure the flag is written before the read of used_idx */
 +    smp_mb();
 +    return !vhost_svq_more_used(svq);
 +}
 +
 +static void vhost_svq_disable_notification(VhostShadowVirtqueue *svq)
 +{
 +    svq->vring.avail->flags |= cpu_to_le16(VRING_AVAIL_F_NO_INTERRUPT);
 +}
 +
 +static VirtQueueElement *vhost_svq_get_buf(VhostShadowVirtqueue *svq,
 +                                           uint32_t *len)
 +{
 +    vring_desc_t *descs = svq->vring.desc;
 +    const vring_used_t *used = svq->vring.used;
 +    vring_used_elem_t used_elem;
 +    uint16_t last_used;
 +
 +    if (!vhost_svq_more_used(svq)) {
 +        return NULL;
 +    }
 +
 +    /* Only get used array entries after they have been exposed by dev */
 +    smp_rmb();
 +    last_used = svq->last_used_idx & (svq->vring.num - 1);
 +    used_elem.id = le32_to_cpu(used->ring[last_used].id);
 +    used_elem.len = le32_to_cpu(used->ring[last_used].len);
 +
 +    svq->last_used_idx++;
 +    if (unlikely(used_elem.id >= svq->vring.num)) {
 +        qemu_log_mask(LOG_GUEST_ERROR, "Device %s says index %u is used",
 +                      svq->vdev->name, used_elem.id);
 +        return NULL;
 +    }
 +
 +    if (unlikely(!svq->ring_id_maps[used_elem.id])) {
 +        qemu_log_mask(LOG_GUEST_ERROR,
 +            "Device %s says index %u is used, but it was not available",
 +            svq->vdev->name, used_elem.id);
 +        return NULL;
 +    }
 +
 +    descs[used_elem.id].next = svq->free_head;
 +    svq->free_head = used_elem.id;
 +
 +    *len = used_elem.len;
 +    return g_steal_pointer(&svq->ring_id_maps[used_elem.id]);
 +}
 +
 +static void vhost_svq_flush(VhostShadowVirtqueue *svq,
 +                            bool check_for_avail_queue)
 +{
 +    VirtQueue *vq = svq->vq;
 +
 +    /* Forward as many used buffers as possible. */
 +    do {
 +        unsigned i = 0;
 +
 +        vhost_svq_disable_notification(svq);
 +        while (true) {
 +            uint32_t len;
 +            g_autofree VirtQueueElement *elem = vhost_svq_get_buf(svq, &len);
 +            if (!elem) {
 +                break;
 +            }
 +
 +            if (unlikely(i >= svq->vring.num)) {
 +                qemu_log_mask(LOG_GUEST_ERROR,
 +                         "More than %u used buffers obtained in a %u size SVQ",
 +                         i, svq->vring.num);
 +                virtqueue_fill(vq, elem, len, i);
 +                virtqueue_flush(vq, i);
 +                return;
 +            }
 +            virtqueue_fill(vq, elem, len, i++);
 +        }
 +
 +        virtqueue_flush(vq, i);
 +        event_notifier_set(&svq->svq_call);
 +
 +        if (check_for_avail_queue && svq->next_guest_avail_elem) {
 +            /*
 +             * Avail ring was full when vhost_svq_flush was called, so it's a
 +             * good moment to make more descriptors available if possible.
 +             */
 +            vhost_handle_guest_kick(svq);
 +        }
 +    } while (!vhost_svq_enable_notification(svq));
 +}
 +
 +/**
 + * Forward used buffers.
   *
   * @n: hdev call event notifier, the one that device set to notify svq.
 + *
 + * Note that we are not making any buffers available in the loop, there is no
 + * way that it runs more than virtqueue size times.
   */
  static void vhost_svq_handle_call(EventNotifier *n)
  {
      VhostShadowVirtqueue *svq = container_of(n, VhostShadowVirtqueue,
                                               hdev_call);
      event_notifier_test_and_clear(n);
 -    event_notifier_set(&svq->svq_call);
 +    vhost_svq_flush(svq, true);
  }
  /**
@@ -XXX,XX +XXX,XX @@ void vhost_svq_set_svq_kick_fd(VhostShadowVirtqueue *svq, int svq_kick_fd)
      if (poll_start) {
          event_notifier_init_fd(svq_kick, svq_kick_fd);
          event_notifier_set(svq_kick);
 -        event_notifier_set_handler(svq_kick, vhost_handle_guest_kick);
 +        event_notifier_set_handler(svq_kick, vhost_handle_guest_kick_notifier);
 +    }
 +}
 +
 +/**
 + * Start the shadow virtqueue operation.
 + *
 + * @svq: Shadow Virtqueue
 + * @vdev: VirtIO device
 + * @vq: Virtqueue to shadow
 + */
 +void vhost_svq_start(VhostShadowVirtqueue *svq, VirtIODevice *vdev,
 +                     VirtQueue *vq)
 +{
 +    size_t desc_size, driver_size, device_size;
 +
 +    svq->next_guest_avail_elem = NULL;
 +    svq->shadow_avail_idx = 0;
 +    svq->shadow_used_idx = 0;
 +    svq->last_used_idx = 0;
 +    svq->vdev = vdev;
 +    svq->vq = vq;
 +
 +    svq->vring.num = virtio_queue_get_num(vdev, virtio_get_queue_index(vq));
 +    driver_size = vhost_svq_driver_area_size(svq);
 +    device_size = vhost_svq_device_area_size(svq);
 +    svq->vring.desc = qemu_memalign(qemu_real_host_page_size, driver_size);
 +    desc_size = sizeof(vring_desc_t) * svq->vring.num;
 +    svq->vring.avail = (void *)((char *)svq->vring.desc + desc_size);
 +    memset(svq->vring.desc, 0, driver_size);
 +    svq->vring.used = qemu_memalign(qemu_real_host_page_size, device_size);
 +    memset(svq->vring.used, 0, device_size);
 +    svq->ring_id_maps = g_new0(VirtQueueElement *, svq->vring.num);
 +    for (unsigned i = 0; i < svq->vring.num - 1; i++) {
 +        svq->vring.desc[i].next = cpu_to_le16(i + 1);
      }
  }
- bool ebpf_rss_is_loaded(struct EBPFRSSContext *ctx)
+@@ -XXX,XX +XXX,XX @@ void vhost_svq_set_svq_kick_fd(VhostShadowVirtqueue *svq, int svq_kick_fd)
  void vhost_svq_stop(VhostShadowVirtqueue *svq)
  {
--    return ctx != NULL && ctx->obj != NULL;
+     event_notifier_set_handler(&svq->svq_kick, NULL);
-+    return ctx != NULL && (ctx->obj != NULL || ctx->program_fd != -1);
++    g_autofree VirtQueueElement *next_avail_elem = NULL;
-+}
++
-+
++    if (!svq->vq) {
-+static bool ebpf_rss_mmap(struct EBPFRSSContext *ctx)
++        return;
-+{
++    }
-+    if (!ebpf_rss_is_loaded(ctx)) {
++
 +    /* Send all pending used descriptors to guest */
 +    vhost_svq_flush(svq, false);
 +
 +    for (unsigned i = 0; i < svq->vring.num; ++i) {
 +        g_autofree VirtQueueElement *elem = NULL;
 +        elem = g_steal_pointer(&svq->ring_id_maps[i]);
 +        if (elem) {
 +            virtqueue_detach_element(svq->vq, elem, 0);
 +        }
 +    }
 +
 +    next_avail_elem = g_steal_pointer(&svq->next_guest_avail_elem);
 +    if (next_avail_elem) {
 +        virtqueue_detach_element(svq->vq, next_avail_elem, 0);
 +    }
 +    svq->vq = NULL;
 +    g_free(svq->ring_id_maps);
 +    qemu_vfree(svq->vring.desc);
 +    qemu_vfree(svq->vring.used);
  }
  /**
 diff --git a/hw/virtio/vhost-shadow-virtqueue.h b/hw/virtio/vhost-shadow-virtqueue.h
 index XXXXXXX..XXXXXXX 100644
 --- a/hw/virtio/vhost-shadow-virtqueue.h
 +++ b/hw/virtio/vhost-shadow-virtqueue.h
@@ -XXX,XX +XXX,XX @@ typedef struct VhostShadowVirtqueue {
      /* Guest's call notifier, where the SVQ calls guest. */
      EventNotifier svq_call;
 +
 +    /* Virtio queue shadowing */
 +    VirtQueue *vq;
 +
 +    /* Virtio device */
 +    VirtIODevice *vdev;
 +
 +    /* Map for use the guest's descriptors */
 +    VirtQueueElement **ring_id_maps;
 +
 +    /* Next VirtQueue element that guest made available */
 +    VirtQueueElement *next_guest_avail_elem;
 +
 +    /* Next head to expose to the device */
 +    uint16_t shadow_avail_idx;
 +
 +    /* Next free descriptor */
 +    uint16_t free_head;
 +
 +    /* Last seen used idx */
 +    uint16_t shadow_used_idx;
 +
 +    /* Next head to consume from the device */
 +    uint16_t last_used_idx;
  } VhostShadowVirtqueue;
  bool vhost_svq_valid_features(uint64_t features, Error **errp);
@@ -XXX,XX +XXX,XX @@ void vhost_svq_get_vring_addr(const VhostShadowVirtqueue *svq,
  size_t vhost_svq_driver_area_size(const VhostShadowVirtqueue *svq);
  size_t vhost_svq_device_area_size(const VhostShadowVirtqueue *svq);
 +void vhost_svq_start(VhostShadowVirtqueue *svq, VirtIODevice *vdev,
 +                     VirtQueue *vq);
  void vhost_svq_stop(VhostShadowVirtqueue *svq);
  VhostShadowVirtqueue *vhost_svq_new(void);
 diff --git a/hw/virtio/vhost-vdpa.c b/hw/virtio/vhost-vdpa.c
 index XXXXXXX..XXXXXXX 100644
 --- a/hw/virtio/vhost-vdpa.c
 +++ b/hw/virtio/vhost-vdpa.c
@@ -XXX,XX +XXX,XX @@ static int vhost_vdpa_set_vring_dev_addr(struct vhost_dev *dev,
   * Note that this function does not rewind kick file descriptor if cannot set
   * call one.
   */
 -static bool vhost_vdpa_svq_setup(struct vhost_dev *dev,
 -                                 VhostShadowVirtqueue *svq, unsigned idx,
 -                                 Error **errp)
 +static int vhost_vdpa_svq_set_fds(struct vhost_dev *dev,
 +                                  VhostShadowVirtqueue *svq, unsigned idx,
 +                                  Error **errp)
  {
      struct vhost_vring_file file = {
          .index = dev->vq_index + idx,
@@ -XXX,XX +XXX,XX @@ static bool vhost_vdpa_svq_setup(struct vhost_dev *dev,
      r = vhost_vdpa_set_vring_dev_kick(dev, &file);
      if (unlikely(r != 0)) {
          error_setg_errno(errp, -r, "Can't set device kick fd");
 -        return false;
 +        return r;
      }
      event_notifier = &svq->hdev_call;
@@ -XXX,XX +XXX,XX @@ static bool vhost_vdpa_svq_setup(struct vhost_dev *dev,
          error_setg_errno(errp, -r, "Can't set device call fd");
      }
 +    return r;
 +}
 +
 +/**
 + * Unmap a SVQ area in the device
 + */
 +static bool vhost_vdpa_svq_unmap_ring(struct vhost_vdpa *v, hwaddr iova,
 +                                      hwaddr size)
 +{
 +    int r;
 +
 +    size = ROUND_UP(size, qemu_real_host_page_size);
 +    r = vhost_vdpa_dma_unmap(v, iova, size);
 +    return r == 0;
 +}
 +
 +static bool vhost_vdpa_svq_unmap_rings(struct vhost_dev *dev,
 +                                       const VhostShadowVirtqueue *svq)
 +{
 +    struct vhost_vdpa *v = dev->opaque;
 +    struct vhost_vring_addr svq_addr;
 +    size_t device_size = vhost_svq_device_area_size(svq);
 +    size_t driver_size = vhost_svq_driver_area_size(svq);
 +    bool ok;
 +
 +    vhost_svq_get_vring_addr(svq, &svq_addr);
 +
 +    ok = vhost_vdpa_svq_unmap_ring(v, svq_addr.desc_user_addr, driver_size);
 +    if (unlikely(!ok)) {
 +        return false;
 +    }
 +
-+    ctx->mmap_configuration = mmap(NULL, qemu_real_host_page_size(),
++    return vhost_vdpa_svq_unmap_ring(v, svq_addr.used_user_addr, device_size);
-+                                   PROT_READ | PROT_WRITE, MAP_SHARED,
++}
-+                                   ctx->map_configuration, 0);
++
-+    if (ctx->mmap_configuration == MAP_FAILED) {
++/**
-+        trace_ebpf_error("eBPF RSS", "can not mmap eBPF configuration array");
++ * Map the shadow virtqueue rings in the device
 + *
 + * @dev: The vhost device
 + * @svq: The shadow virtqueue
 + * @addr: Assigned IOVA addresses
 + * @errp: Error pointer
 + */
 +static bool vhost_vdpa_svq_map_rings(struct vhost_dev *dev,
 +                                     const VhostShadowVirtqueue *svq,
 +                                     struct vhost_vring_addr *addr,
 +                                     Error **errp)
 +{
 +    struct vhost_vdpa *v = dev->opaque;
 +    size_t device_size = vhost_svq_device_area_size(svq);
 +    size_t driver_size = vhost_svq_driver_area_size(svq);
 +    int r;
 +
 +    ERRP_GUARD();
 +    vhost_svq_get_vring_addr(svq, addr);
 +
 +    r = vhost_vdpa_dma_map(v, addr->desc_user_addr, driver_size,
 +                           (void *)(uintptr_t)addr->desc_user_addr, true);
 +    if (unlikely(r != 0)) {
 +        error_setg_errno(errp, -r, "Cannot create vq driver region: ");
 +        return false;
 +    }
-+    ctx->mmap_toeplitz_key = mmap(NULL, qemu_real_host_page_size(),
++
-+                                   PROT_READ | PROT_WRITE, MAP_SHARED,
++    r = vhost_vdpa_dma_map(v, addr->used_user_addr, device_size,
-+                                   ctx->map_toeplitz_key, 0);
++                           (void *)(intptr_t)addr->used_user_addr, false);
-+    if (ctx->mmap_toeplitz_key == MAP_FAILED) {
++    if (unlikely(r != 0)) {
-+        trace_ebpf_error("eBPF RSS", "can not mmap eBPF toeplitz key");
++        error_setg_errno(errp, -r, "Cannot create vq device region: ");
-+        goto toeplitz_fail;
++    }
-+    }
++
-+    ctx->mmap_indirections_table = mmap(NULL, qemu_real_host_page_size(),
++    return r == 0;
-+                                   PROT_READ | PROT_WRITE, MAP_SHARED,
++}
-+                                   ctx->map_indirections_table, 0);
++
-+    if (ctx->mmap_indirections_table == MAP_FAILED) {
++static bool vhost_vdpa_svq_setup(struct vhost_dev *dev,
-+        trace_ebpf_error("eBPF RSS", "can not mmap eBPF indirection table");
++                                 VhostShadowVirtqueue *svq, unsigned idx,
-+        goto indirection_fail;
++                                 Error **errp)
 +{
 +    uint16_t vq_index = dev->vq_index + idx;
 +    struct vhost_vring_state s = {
 +        .index = vq_index,
 +    };
 +    int r;
 +
 +    r = vhost_vdpa_set_dev_vring_base(dev, &s);
 +    if (unlikely(r)) {
 +        error_setg_errno(errp, -r, "Cannot set vring base");
 +        return false;
 +    }
 +
 +    r = vhost_vdpa_svq_set_fds(dev, svq, idx, errp);
      return r == 0;
  }
@@ -XXX,XX +XXX,XX @@ static bool vhost_vdpa_svqs_start(struct vhost_dev *dev)
      }
      for (i = 0; i < v->shadow_vqs->len; ++i) {
 +        VirtQueue *vq = virtio_get_queue(dev->vdev, dev->vq_index + i);
          VhostShadowVirtqueue *svq = g_ptr_array_index(v->shadow_vqs, i);
 +        struct vhost_vring_addr addr = {
 +            .index = i,
 +        };
 +        int r;
          bool ok = vhost_vdpa_svq_setup(dev, svq, i, &err);
          if (unlikely(!ok)) {
 -            error_reportf_err(err, "Cannot setup SVQ %u: ", i);
 +            goto err;
 +        }
 +
 +        vhost_svq_start(svq, dev->vdev, vq);
 +        ok = vhost_vdpa_svq_map_rings(dev, svq, &addr, &err);
 +        if (unlikely(!ok)) {
 +            goto err_map;
 +        }
 +
 +        /* Override vring GPA set by vhost subsystem */
 +        r = vhost_vdpa_set_vring_dev_addr(dev, &addr);
 +        if (unlikely(r != 0)) {
 +            error_setg_errno(&err, -r, "Cannot set device address");
 +            goto err_set_addr;
 +        }
 +    }
 +
 +    return true;
 +
-+indirection_fail:
++err_set_addr:
-+    munmap(ctx->mmap_toeplitz_key, qemu_real_host_page_size());
++    vhost_vdpa_svq_unmap_rings(dev, g_ptr_array_index(v->shadow_vqs, i));
-+toeplitz_fail:
++
-+    munmap(ctx->mmap_configuration, qemu_real_host_page_size());
++err_map:
-+
++    vhost_svq_stop(g_ptr_array_index(v->shadow_vqs, i));
-+    ctx->mmap_configuration = NULL;
++
-+    ctx->mmap_toeplitz_key = NULL;
++err:
-+    ctx->mmap_indirections_table = NULL;
++    error_reportf_err(err, "Cannot setup SVQ %u: ", i);
 +    for (unsigned j = 0; j < i; ++j) {
 +        VhostShadowVirtqueue *svq = g_ptr_array_index(v->shadow_vqs, j);
 +        vhost_vdpa_svq_unmap_rings(dev, svq);
 +        vhost_svq_stop(svq);
 +    }
 +
 +    return false;
 +}
 +
-+static void ebpf_rss_munmap(struct EBPFRSSContext *ctx)
++static bool vhost_vdpa_svqs_stop(struct vhost_dev *dev)
 +{
-+    if (!ebpf_rss_is_loaded(ctx)) {
++    struct vhost_vdpa *v = dev->opaque;
-+        return;
++
-+    }
++    if (!v->shadow_vqs) {
-+
++        return true;
-+    munmap(ctx->mmap_indirections_table, qemu_real_host_page_size());
++    }
-+    munmap(ctx->mmap_toeplitz_key, qemu_real_host_page_size());
++
-+    munmap(ctx->mmap_configuration, qemu_real_host_page_size());
++    for (unsigned i = 0; i < v->shadow_vqs->len; ++i) {
-+
++        VhostShadowVirtqueue *svq = g_ptr_array_index(v->shadow_vqs, i);
-+    ctx->mmap_configuration = NULL;
++        bool ok = vhost_vdpa_svq_unmap_rings(dev, svq);
-+    ctx->mmap_toeplitz_key = NULL;
++        if (unlikely(!ok)) {
-+    ctx->mmap_indirections_table = NULL;
+             return false;
- }
+         }
  bool ebpf_rss_load(struct EBPFRSSContext *ctx)
  {
      struct rss_bpf *rss_bpf_ctx;
 -    if (ctx == NULL) {
 +    if (ctx == NULL || ebpf_rss_is_loaded(ctx)) {
          return false;
      }
+@@ -XXX,XX +XXX,XX @@ static int vhost_vdpa_dev_start(struct vhost_dev *dev, bool started)
-@@ -XXX,XX +XXX,XX @@ bool ebpf_rss_load(struct EBPFRSSContext *ctx)
+         }
-     ctx->map_toeplitz_key = bpf_map__fd(
+         vhost_vdpa_set_vring_ready(dev);
-             rss_bpf_ctx->maps.tap_rss_map_toeplitz_key);
+     } else {
++        ok = vhost_vdpa_svqs_stop(dev);
-+    if (!ebpf_rss_mmap(ctx)) {
++        if (unlikely(!ok)) {
-+        goto error;
++            return -1;
-+    }
++        }
-+
+         vhost_vdpa_host_notifiers_uninit(dev, dev->nvqs);
      return true;
  error:
      rss_bpf__destroy(rss_bpf_ctx);
      ctx->obj = NULL;
 +    ctx->program_fd = -1;
 +    ctx->map_configuration = -1;
 +    ctx->map_toeplitz_key = -1;
 +    ctx->map_indirections_table = -1;
      return false;
  }
@@ -XXX,XX +XXX,XX @@ error:
  static bool ebpf_rss_set_config(struct EBPFRSSContext *ctx,
                                  struct EBPFRSSConfig *config)
  {
 -    uint32_t map_key = 0;
 -
      if (!ebpf_rss_is_loaded(ctx)) {
          return false;
      }
--    if (bpf_map_update_elem(ctx->map_configuration,
 -                            &map_key, config, 0) < 0) {
 -        return false;
 -    }
 +
 +    memcpy(ctx->mmap_configuration, config, sizeof(*config));
      return true;
  }
@@ -XXX,XX +XXX,XX @@ static bool ebpf_rss_set_indirections_table(struct EBPFRSSContext *ctx,
                                              uint16_t *indirections_table,
                                              size_t len)
  {
 -    uint32_t i = 0;
 -
      if (!ebpf_rss_is_loaded(ctx) || indirections_table == NULL ||
         len > VIRTIO_NET_RSS_MAX_TABLE_LEN) {
          return false;
      }
 -    for (; i < len; ++i) {
 -        if (bpf_map_update_elem(ctx->map_indirections_table, &i,
 -                                indirections_table + i, 0) < 0) {
 -            return false;
 -        }
 -    }
 +    memcpy(ctx->mmap_indirections_table, indirections_table,
 +            sizeof(*indirections_table) * len);
      return true;
  }
  static bool ebpf_rss_set_toepliz_key(struct EBPFRSSContext *ctx,
                                       uint8_t *toeplitz_key)
  {
 -    uint32_t map_key = 0;
 -
      /* prepare toeplitz key */
      uint8_t toe[VIRTIO_NET_RSS_MAX_KEY_SIZE] = {};
@@ -XXX,XX +XXX,XX @@ static bool ebpf_rss_set_toepliz_key(struct EBPFRSSContext *ctx,
      memcpy(toe, toeplitz_key, VIRTIO_NET_RSS_MAX_KEY_SIZE);
      *(uint32_t *)toe = ntohl(*(uint32_t *)toe);
 -    if (bpf_map_update_elem(ctx->map_toeplitz_key, &map_key, toe,
 -                            0) < 0) {
 -        return false;
 -    }
 +    memcpy(ctx->mmap_toeplitz_key, toe, VIRTIO_NET_RSS_MAX_KEY_SIZE);
      return true;
  }
@@ -XXX,XX +XXX,XX @@ void ebpf_rss_unload(struct EBPFRSSContext *ctx)
          return;
      }
 -    rss_bpf__destroy(ctx->obj);
 +    ebpf_rss_munmap(ctx);
 +
 +    if (ctx->obj) {
 +        rss_bpf__destroy(ctx->obj);
 +    } else {
 +        close(ctx->program_fd);
 +        close(ctx->map_configuration);
 +        close(ctx->map_toeplitz_key);
 +        close(ctx->map_indirections_table);
 +    }
 +
      ctx->obj = NULL;
 +    ctx->program_fd = -1;
 +    ctx->map_configuration = -1;
 +    ctx->map_toeplitz_key = -1;
 +    ctx->map_indirections_table = -1;
  }
 diff --git a/ebpf/ebpf_rss.h b/ebpf/ebpf_rss.h
 index XXXXXXX..XXXXXXX 100644
 --- a/ebpf/ebpf_rss.h
 +++ b/ebpf/ebpf_rss.h
@@ -XXX,XX +XXX,XX @@ struct EBPFRSSContext {
      int map_configuration;
      int map_toeplitz_key;
      int map_indirections_table;
 +
 +    /* mapped eBPF maps for direct access to omit bpf_map_update_elem() */
 +    void *mmap_configuration;
 +    void *mmap_toeplitz_key;
 +    void *mmap_indirections_table;
  };
  struct EBPFRSSConfig {
 --
 .7.4

-[PULL 11/17] e1000e: rename e1000e_ba_state and e1000e_write_hdr_to_rx_buffers
+[PULL V3 09/15] util: Add iova_tree_alloc_map
-From: Tomasz Dzieciol <t.dzieciol@partner.samsung.com>
+From: Eugenio Pérez <eperezma@redhat.com>
-Rename e1000e_ba_state according and e1000e_write_hdr_to_rx_buffers for
+This iova tree function allows it to look for a hole in allocated
-consistency with IGB.
+regions and return a totally new translation for a given translated
+address.
-Signed-off-by: Tomasz Dzieciol <t.dzieciol@partner.samsung.com>
-Reviewed-by: Akihiko Odaki <akihiko.odaki@daynix.com>
+It's usage is mainly to allow devices to access qemu address space,
-Tested-by: Akihiko Odaki <akihiko.odaki@daynix.com>
+remapping guest's one into a new iova space where qemu can add chunks of
 addresses.
 Signed-off-by: Eugenio Pérez <eperezma@redhat.com>
 Reviewed-by: Peter Xu <peterx@redhat.com>
 Acked-by: Michael S. Tsirkin <mst@redhat.com>
 Signed-off-by: Jason Wang <jasowang@redhat.com>
 ---
- hw/net/e1000e_core.c | 28 +++++++++++++++-------------
+ include/qemu/iova-tree.h |  18 +++++++
-file changed, 15 insertions(+), 13 deletions(-)
+ util/iova-tree.c         | 136 +++++++++++++++++++++++++++++++++++++++++++++++
+files changed, 154 insertions(+)
-diff --git a/hw/net/e1000e_core.c b/hw/net/e1000e_core.c
 diff --git a/include/qemu/iova-tree.h b/include/qemu/iova-tree.h
 index XXXXXXX..XXXXXXX 100644
---- a/hw/net/e1000e_core.c
+--- a/include/qemu/iova-tree.h
-+++ b/hw/net/e1000e_core.c
++++ b/include/qemu/iova-tree.h
-@@ -XXX,XX +XXX,XX @@ e1000e_pci_dma_write_rx_desc(E1000ECore *core, dma_addr_t addr,
+@@ -XXX,XX +XXX,XX @@
-     }
+ #define  IOVA_OK           (0)
  #define  IOVA_ERR_INVALID  (-1) /* Invalid parameters */
  #define  IOVA_ERR_OVERLAP  (-2) /* IOVA range overlapped */
 +#define  IOVA_ERR_NOMEM    (-3) /* Cannot allocate */
  typedef struct IOVATree IOVATree;
  typedef struct DMAMap {
@@ -XXX,XX +XXX,XX @@ const DMAMap *iova_tree_find_address(const IOVATree *tree, hwaddr iova);
  void iova_tree_foreach(IOVATree *tree, iova_tree_iterator iterator);
  /**
 + * iova_tree_alloc_map:
 + *
 + * @tree: the iova tree to allocate from
 + * @map: the new map (as translated addr & size) to allocate in the iova region
 + * @iova_begin: the minimum address of the allocation
 + * @iova_end: the maximum addressable direction of the allocation
 + *
 + * Allocates a new region of a given size, between iova_min and iova_max.
 + *
 + * Return: Same as iova_tree_insert, but cannot overlap and can return error if
 + * iova tree is out of free contiguous range. The caller gets the assigned iova
 + * in map->iova.
 + */
 +int iova_tree_alloc_map(IOVATree *tree, DMAMap *map, hwaddr iova_begin,
 +                        hwaddr iova_end);
 +
 +/**
   * iova_tree_destroy:
   *
   * @tree: the iova tree to destroy
 diff --git a/util/iova-tree.c b/util/iova-tree.c
 index XXXXXXX..XXXXXXX 100644
 --- a/util/iova-tree.c
 +++ b/util/iova-tree.c
@@ -XXX,XX +XXX,XX @@ struct IOVATree {
      GTree *tree;
  };
 +/* Args to pass to iova_tree_alloc foreach function. */
 +struct IOVATreeAllocArgs {
 +    /* Size of the desired allocation */
 +    size_t new_size;
 +
 +    /* The minimum address allowed in the allocation */
 +    hwaddr iova_begin;
 +
 +    /* Map at the left of the hole, can be NULL if "this" is first one */
 +    const DMAMap *prev;
 +
 +    /* Map at the right of the hole, can be NULL if "prev" is the last one */
 +    const DMAMap *this;
 +
 +    /* If found, we fill in the IOVA here */
 +    hwaddr iova_result;
 +
 +    /* Whether have we found a valid IOVA */
 +    bool iova_found;
 +};
 +
 +/**
 + * Iterate args to the next hole
 + *
 + * @args: The alloc arguments
 + * @next: The next mapping in the tree. Can be NULL to signal the last one
 + */
 +static void iova_tree_alloc_args_iterate(struct IOVATreeAllocArgs *args,
 +                                         const DMAMap *next)
 +{
 +    args->prev = args->this;
 +    args->this = next;
 +}
 +
  static int iova_tree_compare(gconstpointer a, gconstpointer b, gpointer data)
  {
      const DMAMap *m1 = a, *m2 = b;
@@ -XXX,XX +XXX,XX @@ int iova_tree_remove(IOVATree *tree, const DMAMap *map)
      return IOVA_OK;
  }
--typedef struct e1000e_ba_state_st {
++/**
-+typedef struct E1000EBAState {
++ * Try to find an unallocated IOVA range between prev and this elements.
-     uint16_t written[MAX_PS_BUFFERS];
++ *
-     uint8_t cur_idx;
++ * @args: Arguments to allocation
--} e1000e_ba_state;
++ *
-+} E1000EBAState;
++ * Cases:
++ *
- static inline void
++ * (1) !prev, !this: No entries allocated, always succeed
--e1000e_write_hdr_to_rx_buffers(E1000ECore *core,
++ *
--                               hwaddr ba[MAX_PS_BUFFERS],
++ * (2) !prev, this: We're iterating at the 1st element.
--                               e1000e_ba_state *bastate,
++ *
--                               const char *data,
++ * (3) prev, !this: We're iterating at the last element.
--                               dma_addr_t data_len)
++ *
-+e1000e_write_hdr_frag_to_rx_buffers(E1000ECore *core,
++ * (4) prev, this: this is the most common case, we'll try to find a hole
-+                                    hwaddr ba[MAX_PS_BUFFERS],
++ * between "prev" and "this" mapping.
-+                                    E1000EBAState *bastate,
++ *
-+                                    const char *data,
++ * Note that this function assumes the last valid iova is HWADDR_MAX, but it
-+                                    dma_addr_t data_len)
++ * searches linearly so it's easy to discard the result if it's not the case.
 + */
 +static void iova_tree_alloc_map_in_hole(struct IOVATreeAllocArgs *args)
 +{
 +    const DMAMap *prev = args->prev, *this = args->this;
 +    uint64_t hole_start, hole_last;
 +
 +    if (this && this->iova + this->size < args->iova_begin) {
 +        return;
 +    }
 +
 +    hole_start = MAX(prev ? prev->iova + prev->size + 1 : 0, args->iova_begin);
 +    hole_last = this ? this->iova : HWADDR_MAX;
 +
 +    if (hole_last - hole_start > args->new_size) {
 +        args->iova_result = hole_start;
 +        args->iova_found = true;
 +    }
 +}
 +
 +/**
 + * Foreach dma node in the tree, compare if there is a hole with its previous
 + * node (or minimum iova address allowed) and the node.
 + *
 + * @key: Node iterating
 + * @value: Node iterating
 + * @pargs: Struct to communicate with the outside world
 + *
 + * Return: false to keep iterating, true if needs break.
 + */
 +static gboolean iova_tree_alloc_traverse(gpointer key, gpointer value,
 +                                         gpointer pargs)
 +{
 +    struct IOVATreeAllocArgs *args = pargs;
 +    DMAMap *node = value;
 +
 +    assert(key == value);
 +
 +    iova_tree_alloc_args_iterate(args, node);
 +    iova_tree_alloc_map_in_hole(args);
 +    return args->iova_found;
 +}
 +
 +int iova_tree_alloc_map(IOVATree *tree, DMAMap *map, hwaddr iova_begin,
 +                        hwaddr iova_last)
 +{
 +    struct IOVATreeAllocArgs args = {
 +        .new_size = map->size,
 +        .iova_begin = iova_begin,
 +    };
 +
 +    if (unlikely(iova_last < iova_begin)) {
 +        return IOVA_ERR_INVALID;
 +    }
 +
 +    /*
 +     * Find a valid hole for the mapping
 +     *
 +     * Assuming low iova_begin, so no need to do a binary search to
 +     * locate the first node.
 +     *
 +     * TODO: Replace all this with g_tree_node_first/next/last when available
 +     * (from glib since 2.68). To do it with g_tree_foreach complicates the
 +     * code a lot.
 +     *
 +     */
 +    g_tree_foreach(tree->tree, iova_tree_alloc_traverse, &args);
 +    if (!args.iova_found) {
 +        /*
 +         * Either tree is empty or the last hole is still not checked.
 +         * g_tree_foreach does not compare (last, iova_last] range, so we check
 +         * it here.
 +         */
 +        iova_tree_alloc_args_iterate(&args, NULL);
 +        iova_tree_alloc_map_in_hole(&args);
 +    }
 +
 +    if (!args.iova_found || args.iova_result + map->size > iova_last) {
 +        return IOVA_ERR_NOMEM;
 +    }
 +
 +    map->iova = args.iova_result;
 +    return iova_tree_insert(tree, map);
 +}
 +
  void iova_tree_destroy(IOVATree *tree)
  {
-     assert(data_len <= core->rxbuf_sizes[0] - bastate->written[0]);
+     g_tree_destroy(tree->tree);
@@ -XXX,XX +XXX,XX @@ e1000e_write_hdr_to_rx_buffers(E1000ECore *core,
  static void
  e1000e_write_payload_frag_to_rx_buffers(E1000ECore *core,
                                          hwaddr ba[MAX_PS_BUFFERS],
 -                                        e1000e_ba_state *bastate,
 +                                        E1000EBAState *bastate,
                                          const char *data,
                                          dma_addr_t data_len)
  {
@@ -XXX,XX +XXX,XX @@ e1000e_write_packet_to_guest(E1000ECore *core, struct NetRxPkt *pkt,
      do {
          hwaddr ba[MAX_PS_BUFFERS];
 -        e1000e_ba_state bastate = { { 0 } };
 +        E1000EBAState bastate = { { 0 } };
          bool is_last = false;
          desc_size = total_size - desc_offset;
@@ -XXX,XX +XXX,XX @@ e1000e_write_packet_to_guest(E1000ECore *core, struct NetRxPkt *pkt,
                              iov_copy = MIN(ps_hdr_len - ps_hdr_copied,
                                             iov->iov_len - iov_ofs);
 -                            e1000e_write_hdr_to_rx_buffers(core, ba, &bastate,
 -                                                      iov->iov_base, iov_copy);
 +                            e1000e_write_hdr_frag_to_rx_buffers(core, ba,
 +                                                                &bastate,
 +                                                                iov->iov_base,
 +                                                                iov_copy);
                              copy_size -= iov_copy;
                              ps_hdr_copied += iov_copy;
@@ -XXX,XX +XXX,XX @@ e1000e_write_packet_to_guest(E1000ECore *core, struct NetRxPkt *pkt,
                      } else {
                          /* Leave buffer 0 of each descriptor except first */
                          /* empty as per spec 7.1.5.1                      */
 -                        e1000e_write_hdr_to_rx_buffers(core, ba, &bastate,
 -                                                       NULL, 0);
 +                        e1000e_write_hdr_frag_to_rx_buffers(core, ba, &bastate,
 +                                                            NULL, 0);
                      }
                  }
 --
 .7.4

-[PULL 02/17] tap: Add check for USO features
+[PULL V3 10/15] util: add iova_tree_find_iova
-From: Yuri Benditovich <yuri.benditovich@daynix.com>
+From: Eugenio Pérez <eperezma@redhat.com>
-Tap indicates support for USO features according to
+This function does the reverse operation of iova_tree_find: To look for
-capabilities of current kernel module.
+a mapping that match a translated address so we can do the reverse.
-Signed-off-by: Yuri Benditovich <yuri.benditovich@daynix.com>
+This have linear complexity instead of logarithmic, but it supports
-Signed-off-by: Andrew Melnychecnko <andrew@daynix.com>
+overlapping HVA. Future developments could reduce it.
 Signed-off-by: Eugenio Pérez <eperezma@redhat.com>
 Acked-by: Michael S. Tsirkin <mst@redhat.com>
 Signed-off-by: Jason Wang <jasowang@redhat.com>
 ---
- include/net/net.h |  3 +++
+ include/qemu/iova-tree.h | 20 +++++++++++++++++++-
- net/net.c         |  9 +++++++++
+ util/iova-tree.c         | 34 ++++++++++++++++++++++++++++++++++
- net/tap-bsd.c     |  5 +++++
+files changed, 53 insertions(+), 1 deletion(-)
  net/tap-linux.c   | 12 ++++++++++++
  net/tap-solaris.c |  5 +++++
  net/tap-stub.c    |  5 +++++
  net/tap.c         | 12 ++++++++++++
  net/tap_int.h     |  1 +
 files changed, 52 insertions(+)
-diff --git a/include/net/net.h b/include/net/net.h
+diff --git a/include/qemu/iova-tree.h b/include/qemu/iova-tree.h
 index XXXXXXX..XXXXXXX 100644
---- a/include/net/net.h
+--- a/include/qemu/iova-tree.h
-+++ b/include/net/net.h
++++ b/include/qemu/iova-tree.h
-@@ -XXX,XX +XXX,XX @@ typedef void (LinkStatusChanged)(NetClientState *);
+@@ -XXX,XX +XXX,XX @@ int iova_tree_remove(IOVATree *tree, const DMAMap *map);
- typedef void (NetClientDestructor)(NetClientState *);
+  * @tree: the iova tree to search from
- typedef RxFilterInfo *(QueryRxFilter)(NetClientState *);
+  * @map: the mapping to search
- typedef bool (HasUfo)(NetClientState *);
+  *
-+typedef bool (HasUso)(NetClientState *);
+- * Search for a mapping in the iova tree that overlaps with the
- typedef bool (HasVnetHdr)(NetClientState *);
++ * Search for a mapping in the iova tree that iova overlaps with the
- typedef bool (HasVnetHdrLen)(NetClientState *, int);
+  * mapping range specified.  Only the first found mapping will be
- typedef bool (GetUsingVnetHdr)(NetClientState *);
+  * returned.
-@@ -XXX,XX +XXX,XX @@ typedef struct NetClientInfo {
+  *
-     QueryRxFilter *query_rx_filter;
+@@ -XXX,XX +XXX,XX @@ int iova_tree_remove(IOVATree *tree, const DMAMap *map);
-     NetPoll *poll;
+ const DMAMap *iova_tree_find(const IOVATree *tree, const DMAMap *map);
-     HasUfo *has_ufo;
-+    HasUso *has_uso;
+ /**
-     HasVnetHdr *has_vnet_hdr;
++ * iova_tree_find_iova:
-     HasVnetHdrLen *has_vnet_hdr_len;
++ *
-     GetUsingVnetHdr *get_using_vnet_hdr;
++ * @tree: the iova tree to search from
-@@ -XXX,XX +XXX,XX @@ void qemu_set_info_str(NetClientState *nc,
++ * @map: the mapping to search
-                        const char *fmt, ...) G_GNUC_PRINTF(2, 3);
++ *
- void qemu_format_nic_info_str(NetClientState *nc, uint8_t macaddr[6]);
++ * Search for a mapping in the iova tree that translated_addr overlaps with the
- bool qemu_has_ufo(NetClientState *nc);
++ * mapping range specified.  Only the first found mapping will be
-+bool qemu_has_uso(NetClientState *nc);
++ * returned.
- bool qemu_has_vnet_hdr(NetClientState *nc);
++ *
- bool qemu_has_vnet_hdr_len(NetClientState *nc, int len);
++ * Return: DMAMap pointer if found, or NULL if not found.  Note that
- bool qemu_get_using_vnet_hdr(NetClientState *nc);
++ * the returned DMAMap pointer is maintained internally.  User should
-diff --git a/net/net.c b/net/net.c
++ * only read the content but never modify or free the content.  Also,
 + * user is responsible to make sure the pointer is valid (say, no
 + * concurrent deletion in progress).
 + */
 +const DMAMap *iova_tree_find_iova(const IOVATree *tree, const DMAMap *map);
 +
 +/**
   * iova_tree_find_address:
   *
   * @tree: the iova tree to search from
 diff --git a/util/iova-tree.c b/util/iova-tree.c
 index XXXXXXX..XXXXXXX 100644
---- a/net/net.c
+--- a/util/iova-tree.c
-+++ b/net/net.c
++++ b/util/iova-tree.c
-@@ -XXX,XX +XXX,XX @@ bool qemu_has_ufo(NetClientState *nc)
+@@ -XXX,XX +XXX,XX @@ struct IOVATreeAllocArgs {
-     return nc->info->has_ufo(nc);
+     bool iova_found;
  };
 +typedef struct IOVATreeFindIOVAArgs {
 +    const DMAMap *needle;
 +    const DMAMap *result;
 +} IOVATreeFindIOVAArgs;
 +
  /**
   * Iterate args to the next hole
   *
@@ -XXX,XX +XXX,XX @@ const DMAMap *iova_tree_find(const IOVATree *tree, const DMAMap *map)
      return g_tree_lookup(tree->tree, map);
  }
-+bool qemu_has_uso(NetClientState *nc)
++static gboolean iova_tree_find_address_iterator(gpointer key, gpointer value,
 +                                                gpointer data)
 +{
-+    if (!nc || !nc->info->has_uso) {
++    const DMAMap *map = key;
 +    IOVATreeFindIOVAArgs *args = data;
 +    const DMAMap *needle;
 +
 +    g_assert(key == value);
 +
 +    needle = args->needle;
 +    if (map->translated_addr + map->size < needle->translated_addr ||
 +        needle->translated_addr + needle->size < map->translated_addr) {
 +        return false;
 +    }
 +
-+    return nc->info->has_uso(nc);
++    args->result = map;
 +    return true;
 +}
 +
- bool qemu_has_vnet_hdr(NetClientState *nc)
++const DMAMap *iova_tree_find_iova(const IOVATree *tree, const DMAMap *map)
  {
      if (!nc || !nc->info->has_vnet_hdr) {
 diff --git a/net/tap-bsd.c b/net/tap-bsd.c
 index XXXXXXX..XXXXXXX 100644
 --- a/net/tap-bsd.c
 +++ b/net/tap-bsd.c
@@ -XXX,XX +XXX,XX @@ int tap_probe_has_ufo(int fd)
      return 0;
  }
 +int tap_probe_has_uso(int fd)
 +{
-+    return 0;
++    IOVATreeFindIOVAArgs args = {
 +        .needle = map,
 +    };
 +
 +    g_tree_foreach(tree->tree, iova_tree_find_address_iterator, &args);
 +    return args.result;
 +}
 +
- int tap_probe_vnet_hdr_len(int fd, int len)
+ const DMAMap *iova_tree_find_address(const IOVATree *tree, hwaddr iova)
  {
-     return 0;
+     const DMAMap map = { .iova = iova, .size = 0 };
 diff --git a/net/tap-linux.c b/net/tap-linux.c
 index XXXXXXX..XXXXXXX 100644
 --- a/net/tap-linux.c
 +++ b/net/tap-linux.c
@@ -XXX,XX +XXX,XX @@ int tap_probe_has_ufo(int fd)
      return 1;
  }
 +int tap_probe_has_uso(int fd)
 +{
 +    unsigned offload;
 +
 +    offload = TUN_F_CSUM | TUN_F_USO4 | TUN_F_USO6;
 +
 +    if (ioctl(fd, TUNSETOFFLOAD, offload) < 0) {
 +        return 0;
 +    }
 +    return 1;
 +}
 +
  /* Verify that we can assign given length */
  int tap_probe_vnet_hdr_len(int fd, int len)
  {
 diff --git a/net/tap-solaris.c b/net/tap-solaris.c
 index XXXXXXX..XXXXXXX 100644
 --- a/net/tap-solaris.c
 +++ b/net/tap-solaris.c
@@ -XXX,XX +XXX,XX @@ int tap_probe_has_ufo(int fd)
      return 0;
  }
 +int tap_probe_has_uso(int fd)
 +{
 +    return 0;
 +}
 +
  int tap_probe_vnet_hdr_len(int fd, int len)
  {
      return 0;
 diff --git a/net/tap-stub.c b/net/tap-stub.c
 index XXXXXXX..XXXXXXX 100644
 --- a/net/tap-stub.c
 +++ b/net/tap-stub.c
@@ -XXX,XX +XXX,XX @@ int tap_probe_has_ufo(int fd)
      return 0;
  }
 +int tap_probe_has_uso(int fd)
 +{
 +    return 0;
 +}
 +
  int tap_probe_vnet_hdr_len(int fd, int len)
  {
      return 0;
 diff --git a/net/tap.c b/net/tap.c
 index XXXXXXX..XXXXXXX 100644
 --- a/net/tap.c
 +++ b/net/tap.c
@@ -XXX,XX +XXX,XX @@ typedef struct TAPState {
      bool write_poll;
      bool using_vnet_hdr;
      bool has_ufo;
 +    bool has_uso;
      bool enabled;
      VHostNetState *vhost_net;
      unsigned host_vnet_hdr_len;
@@ -XXX,XX +XXX,XX @@ static bool tap_has_ufo(NetClientState *nc)
      return s->has_ufo;
  }
 +static bool tap_has_uso(NetClientState *nc)
 +{
 +    TAPState *s = DO_UPCAST(TAPState, nc, nc);
 +
 +    assert(nc->info->type == NET_CLIENT_DRIVER_TAP);
 +
 +    return s->has_uso;
 +}
 +
  static bool tap_has_vnet_hdr(NetClientState *nc)
  {
      TAPState *s = DO_UPCAST(TAPState, nc, nc);
@@ -XXX,XX +XXX,XX @@ static NetClientInfo net_tap_info = {
      .poll = tap_poll,
      .cleanup = tap_cleanup,
      .has_ufo = tap_has_ufo,
 +    .has_uso = tap_has_uso,
      .has_vnet_hdr = tap_has_vnet_hdr,
      .has_vnet_hdr_len = tap_has_vnet_hdr_len,
      .get_using_vnet_hdr = tap_get_using_vnet_hdr,
@@ -XXX,XX +XXX,XX @@ static TAPState *net_tap_fd_init(NetClientState *peer,
      s->host_vnet_hdr_len = vnet_hdr ? sizeof(struct virtio_net_hdr) : 0;
      s->using_vnet_hdr = false;
      s->has_ufo = tap_probe_has_ufo(s->fd);
 +    s->has_uso = tap_probe_has_uso(s->fd);
      s->enabled = true;
      tap_set_offload(&s->nc, 0, 0, 0, 0, 0, 0, 0);
      /*
 diff --git a/net/tap_int.h b/net/tap_int.h
 index XXXXXXX..XXXXXXX 100644
 --- a/net/tap_int.h
 +++ b/net/tap_int.h
@@ -XXX,XX +XXX,XX @@ void tap_set_sndbuf(int fd, const NetdevTapOptions *tap, Error **errp);
  int tap_probe_vnet_hdr(int fd, Error **errp);
  int tap_probe_vnet_hdr_len(int fd, int len);
  int tap_probe_has_ufo(int fd);
 +int tap_probe_has_uso(int fd);
  void tap_fd_set_offload(int fd, int csum, int tso4, int tso6, int ecn, int ufo,
                          int uso4, int uso6);
  void tap_fd_set_vnet_hdr_len(int fd, int len);
 --
 .7.4

-[PULL 03/17] virtio-net: Add USO flags to vhost support.
+Deleted patch
-From: Andrew Melnychenko <andrew@daynix.com>
-New features are subject to check with vhost-user and vdpa.
-Signed-off-by: Yuri Benditovich <yuri.benditovich@daynix.com>
-Signed-off-by: Andrew Melnychenko <andrew@daynix.com>
-Signed-off-by: Jason Wang <jasowang@redhat.com>
----
- hw/net/vhost_net.c | 3 +++
- net/vhost-vdpa.c   | 3 +++
-files changed, 6 insertions(+)
-diff --git a/hw/net/vhost_net.c b/hw/net/vhost_net.c
-index XXXXXXX..XXXXXXX 100644
---- a/hw/net/vhost_net.c
-+++ b/hw/net/vhost_net.c
-@@ -XXX,XX +XXX,XX @@ static const int user_feature_bits[] = {
-     VIRTIO_F_RING_RESET,
-     VIRTIO_NET_F_RSS,
-     VIRTIO_NET_F_HASH_REPORT,
-+    VIRTIO_NET_F_GUEST_USO4,
-+    VIRTIO_NET_F_GUEST_USO6,
-+    VIRTIO_NET_F_HOST_USO,
-     /* This bit implies RARP isn't sent by QEMU out of band */
-     VIRTIO_NET_F_GUEST_ANNOUNCE,
-diff --git a/net/vhost-vdpa.c b/net/vhost-vdpa.c
-index XXXXXXX..XXXXXXX 100644
---- a/net/vhost-vdpa.c
-+++ b/net/vhost-vdpa.c
-@@ -XXX,XX +XXX,XX @@ const int vdpa_feature_bits[] = {
-     VIRTIO_NET_F_GUEST_TSO4,
-     VIRTIO_NET_F_GUEST_TSO6,
-     VIRTIO_NET_F_GUEST_UFO,
-+    VIRTIO_NET_F_GUEST_USO4,
-+    VIRTIO_NET_F_GUEST_USO6,
-     VIRTIO_NET_F_HASH_REPORT,
-     VIRTIO_NET_F_HOST_ECN,
-     VIRTIO_NET_F_HOST_TSO4,
-     VIRTIO_NET_F_HOST_TSO6,
-     VIRTIO_NET_F_HOST_UFO,
-+    VIRTIO_NET_F_HOST_USO,
-     VIRTIO_NET_F_MQ,
-     VIRTIO_NET_F_MRG_RXBUF,
-     VIRTIO_NET_F_MTU,
---
-.7.4

-[PULL 12/17] net: add initial support for AF_XDP network backend
+[PULL V3 11/15] vhost: Add VhostIOVATree
-From: Ilya Maximets <i.maximets@ovn.org>
+From: Eugenio Pérez <eperezma@redhat.com>
-AF_XDP is a network socket family that allows communication directly
+This tree is able to look for a translated address from an IOVA address.
 with the network device driver in the kernel, bypassing most or all
 of the kernel networking stack.  In the essence, the technology is
 pretty similar to netmap.  But, unlike netmap, AF_XDP is Linux-native
 and works with any network interfaces without driver modifications.
 Unlike vhost-based backends (kernel, user, vdpa), AF_XDP doesn't
 require access to character devices or unix sockets.  Only access to
 the network interface itself is necessary.
-This patch implements a network backend that communicates with the
+At first glance it is similar to util/iova-tree. However, SVQ working on
-kernel by creating an AF_XDP socket.  A chunk of userspace memory
+devices with limited IOVA space need more capabilities, like allocating
-is shared between QEMU and the host kernel.  4 ring buffers (Tx, Rx,
+IOVA chunks or performing reverse translations (qemu addresses to iova).
 Fill and Completion) are placed in that memory along with a pool of
 memory buffers for the packet data.  Data transmission is done by
 allocating one of the buffers, copying packet data into it and
 placing the pointer into Tx ring.  After transmission, device will
 return the buffer via Completion ring.  On Rx, device will take
 a buffer form a pre-populated Fill ring, write the packet data into
 it and place the buffer into Rx ring.
-AF_XDP network backend takes on the communication with the host
+The allocation capability, as "assign a free IOVA address to this chunk
-kernel and the network interface and forwards packets to/from the
+of memory in qemu's address space" allows shadow virtqueue to create a
-peer device in QEMU.
+new address space that is not restricted by guest's addressable one, so
 we can allocate shadow vqs vrings outside of it.
-Usage example:
+It duplicates the tree so it can search efficiently in both directions,
 and it will signal overlap if iova or the translated address is present
 in any tree.
-  -device virtio-net-pci,netdev=guest1,mac=00:16:35:AF:AA:5C
+Signed-off-by: Eugenio Pérez <eperezma@redhat.com>
-  -netdev af-xdp,ifname=ens6f1np1,id=guest1,mode=native,queues=1
+Acked-by: Michael S. Tsirkin <mst@redhat.com>
 XDP program bridges the socket with a network interface.  It can be
 attached to the interface in 2 different modes:
 . skb - this mode should work for any interface and doesn't require
          driver support.  With a caveat of lower performance.
 . native - this does require support from the driver and allows to
             bypass skb allocation in the kernel and potentially use
             zero-copy while getting packets in/out userspace.
 By default, QEMU will try to use native mode and fall back to skb.
 Mode can be forced via 'mode' option.  To force 'copy' even in native
 mode, use 'force-copy=on' option.  This might be useful if there is
 some issue with the driver.
 Option 'queues=N' allows to specify how many device queues should
 be open.  Note that all the queues that are not open are still
 functional and can receive traffic, but it will not be delivered to
 QEMU.  So, the number of device queues should generally match the
 QEMU configuration, unless the device is shared with something
 else and the traffic re-direction to appropriate queues is correctly
 configured on a device level (e.g. with ethtool -N).
 'start-queue=M' option can be used to specify from which queue id
 QEMU should start configuring 'N' queues.  It might also be necessary
 to use this option with certain NICs, e.g. MLX5 NICs.  See the docs
 for examples.
 In a general case QEMU will need CAP_NET_ADMIN and CAP_SYS_ADMIN
 or CAP_BPF capabilities in order to load default XSK/XDP programs to
 the network interface and configure BPF maps.  It is possible, however,
 to run with no capabilities.  For that to work, an external process
 with enough capabilities will need to pre-load default XSK program,
 create AF_XDP sockets and pass their file descriptors to QEMU process
 on startup via 'sock-fds' option.  Network backend will need to be
 configured with 'inhibit=on' to avoid loading of the program.
 QEMU will need 32 MB of locked memory (RLIMIT_MEMLOCK) per queue
 or CAP_IPC_LOCK.
 There are few performance challenges with the current network backends.
 First is that they do not support IO threads.  This means that data
 path is handled by the main thread in QEMU and may slow down other
 work or may be slowed down by some other work.  This also means that
 taking advantage of multi-queue is generally not possible today.
 Another thing is that data path is going through the device emulation
 code, which is not really optimized for performance.  The fastest
 "frontend" device is virtio-net.  But it's not optimized for heavy
 traffic either, because it expects such use-cases to be handled via
 some implementation of vhost (user, kernel, vdpa).  In practice, we
 have virtio notifications and rcu lock/unlock on a per-packet basis
 and not very efficient accesses to the guest memory.  Communication
 channels between backend and frontend devices do not allow passing
 more than one packet at a time as well.
 Some of these challenges can be avoided in the future by adding better
 batching into device emulation or by implementing vhost-af-xdp variant.
 There are also a few kernel limitations.  AF_XDP sockets do not
 support any kinds of checksum or segmentation offloading.  Buffers
 are limited to a page size (4K), i.e. MTU is limited.  Multi-buffer
 support implementation for AF_XDP is in progress, but not ready yet.
 Also, transmission in all non-zero-copy modes is synchronous, i.e.
 done in a syscall.  That doesn't allow high packet rates on virtual
 interfaces.
 However, keeping in mind all of these challenges, current implementation
 of the AF_XDP backend shows a decent performance while running on top
 of a physical NIC with zero-copy support.
 Test setup:
 VMs running on 2 physical hosts connected via ConnectX6-Dx card.
 Network backend is configured to open the NIC directly in native mode.
 The driver supports zero-copy.  NIC is configured to use 1 queue.
 Inside a VM - iperf3 for basic TCP performance testing and dpdk-testpmd
 for PPS testing.
 iperf3 result:
  TCP stream      : 19.1 Gbps
 dpdk-testpmd (single queue, single CPU core, 64 B packets) results:
  Tx only         : 3.4 Mpps
  Rx only         : 2.0 Mpps
  L2 FWD Loopback : 1.5 Mpps
 In skb mode the same setup shows much lower performance, similar to
 the setup where pair of physical NICs is replaced with veth pair:
 iperf3 result:
   TCP stream      : 9 Gbps
 dpdk-testpmd (single queue, single CPU core, 64 B packets) results:
   Tx only         : 1.2 Mpps
   Rx only         : 1.0 Mpps
   L2 FWD Loopback : 0.7 Mpps
 Results in skb mode or over the veth are close to results of a tap
 backend with vhost=on and disabled segmentation offloading bridged
 with a NIC.
 Signed-off-by: Ilya Maximets <i.maximets@ovn.org>
 Signed-off-by: Jason Wang <jasowang@redhat.com>
 ---
- MAINTAINERS                                     |   4 +
+ hw/virtio/meson.build       |   2 +-
- hmp-commands.hx                                 |   3 +
+ hw/virtio/vhost-iova-tree.c | 110 ++++++++++++++++++++++++++++++++++++++++++++
- meson.build                                     |   9 +
+ hw/virtio/vhost-iova-tree.h |  27 +++++++++++
- meson_options.txt                               |   2 +
+files changed, 138 insertions(+), 1 deletion(-)
- net/af-xdp.c                                    | 526 ++++++++++++++++++++++++
+ create mode 100644 hw/virtio/vhost-iova-tree.c
- net/clients.h                                   |   5 +
+ create mode 100644 hw/virtio/vhost-iova-tree.h
  net/meson.build                                 |   3 +
  net/net.c                                       |   6 +
  qapi/net.json                                   |  58 +++
  qemu-options.hx                                 |  70 +++-
  scripts/ci/org.centos/stream/8/x86_64/configure |   1 +
  scripts/meson-buildoptions.sh                   |   3 +
  tests/docker/dockerfiles/debian-amd64.docker    |   1 +
 files changed, 690 insertions(+), 1 deletion(-)
  create mode 100644 net/af-xdp.c
-diff --git a/MAINTAINERS b/MAINTAINERS
+diff --git a/hw/virtio/meson.build b/hw/virtio/meson.build
 index XXXXXXX..XXXXXXX 100644
---- a/MAINTAINERS
+--- a/hw/virtio/meson.build
-+++ b/MAINTAINERS
++++ b/hw/virtio/meson.build
-@@ -XXX,XX +XXX,XX @@ W: http://info.iet.unipi.it/~luigi/netmap/
+@@ -XXX,XX +XXX,XX @@ softmmu_ss.add(when: 'CONFIG_ALL', if_true: files('vhost-stub.c'))
- S: Maintained
- F: net/netmap.c
+ virtio_ss = ss.source_set()
+ virtio_ss.add(files('virtio.c'))
-+AF_XDP network backend
+-virtio_ss.add(when: 'CONFIG_VHOST', if_true: files('vhost.c', 'vhost-backend.c', 'vhost-shadow-virtqueue.c'))
-+R: Ilya Maximets <i.maximets@ovn.org>
++virtio_ss.add(when: 'CONFIG_VHOST', if_true: files('vhost.c', 'vhost-backend.c', 'vhost-shadow-virtqueue.c', 'vhost-iova-tree.c'))
-+F: net/af-xdp.c
+ virtio_ss.add(when: 'CONFIG_VHOST_USER', if_true: files('vhost-user.c'))
-+
+ virtio_ss.add(when: 'CONFIG_VHOST_VDPA', if_true: files('vhost-vdpa.c'))
- Host Memory Backends
+ virtio_ss.add(when: 'CONFIG_VIRTIO_BALLOON', if_true: files('virtio-balloon.c'))
- M: David Hildenbrand <david@redhat.com>
+diff --git a/hw/virtio/vhost-iova-tree.c b/hw/virtio/vhost-iova-tree.c
  M: Igor Mammedov <imammedo@redhat.com>
 diff --git a/hmp-commands.hx b/hmp-commands.hx
 index XXXXXXX..XXXXXXX 100644
 --- a/hmp-commands.hx
 +++ b/hmp-commands.hx
@@ -XXX,XX +XXX,XX @@ ERST
          .name       = "netdev_add",
          .args_type  = "netdev:O",
          .params     = "[user|tap|socket|stream|dgram|vde|bridge|hubport|netmap|vhost-user"
 +#ifdef CONFIG_AF_XDP
 +                      "|af-xdp"
 +#endif
  #ifdef CONFIG_VMNET
                        "|vmnet-host|vmnet-shared|vmnet-bridged"
  #endif
 diff --git a/meson.build b/meson.build
 index XXXXXXX..XXXXXXX 100644
 --- a/meson.build
 +++ b/meson.build
@@ -XXX,XX +XXX,XX @@ if libbpf.found() and not cc.links('''
    endif
  endif
 +# libxdp
 +libxdp = not_found
 +if not get_option('af_xdp').auto() or have_system
 +    libxdp = dependency('libxdp', required: get_option('af_xdp'),
 +                        version: '>=1.4.0', method: 'pkg-config')
 +endif
 +
  # libdw
  libdw = not_found
  if not get_option('libdw').auto() or \
@@ -XXX,XX +XXX,XX @@ config_host_data.set('CONFIG_HEXAGON_IDEF_PARSER', get_option('hexagon_idef_pars
  config_host_data.set('CONFIG_LIBATTR', have_old_libattr)
  config_host_data.set('CONFIG_LIBCAP_NG', libcap_ng.found())
  config_host_data.set('CONFIG_EBPF', libbpf.found())
 +config_host_data.set('CONFIG_AF_XDP', libxdp.found())
  config_host_data.set('CONFIG_LIBDAXCTL', libdaxctl.found())
  config_host_data.set('CONFIG_LIBISCSI', libiscsi.found())
  config_host_data.set('CONFIG_LIBNFS', libnfs.found())
@@ -XXX,XX +XXX,XX @@ summary_info = {}
  if targetos == 'darwin'
    summary_info += {'vmnet.framework support': vmnet}
  endif
 +summary_info += {'AF_XDP support':    libxdp}
  summary_info += {'slirp support':     slirp}
  summary_info += {'vde support':       vde}
  summary_info += {'netmap support':    have_netmap}
 diff --git a/meson_options.txt b/meson_options.txt
 index XXXXXXX..XXXXXXX 100644
 --- a/meson_options.txt
 +++ b/meson_options.txt
@@ -XXX,XX +XXX,XX @@ option('avx512bw', type: 'feature', value: 'auto',
  option('keyring', type: 'feature', value: 'auto',
         description: 'Linux keyring support')
 +option('af_xdp', type : 'feature', value : 'auto',
 +       description: 'AF_XDP network backend support')
  option('attr', type : 'feature', value : 'auto',
         description: 'attr/xattr support')
  option('auth_pam', type : 'feature', value : 'auto',
 diff --git a/net/af-xdp.c b/net/af-xdp.c
 new file mode 100644
 index XXXXXXX..XXXXXXX
 --- /dev/null
-+++ b/net/af-xdp.c
++++ b/hw/virtio/vhost-iova-tree.c
 @@ -XXX,XX +XXX,XX @@
 +/*
-+ * AF_XDP network backend.
++ * vhost software live migration iova tree
 + *
-+ * Copyright (c) 2023 Red Hat, Inc.
++ * SPDX-FileCopyrightText: Red Hat, Inc. 2021
 + * SPDX-FileContributor: Author: Eugenio Pérez <eperezma@redhat.com>
 + *
-+ * Authors:
++ * SPDX-License-Identifier: GPL-2.0-or-later
 + *  Ilya Maximets <i.maximets@ovn.org>
 + *
 + * This work is licensed under the terms of the GNU GPL, version 2 or later.
 + * See the COPYING file in the top-level directory.
 + */
 +
++#include "qemu/osdep.h"
++#include "qemu/iova-tree.h"
++#include "vhost-iova-tree.h"
 +
-+#include "qemu/osdep.h"
++#define iova_min_addr qemu_real_host_page_size
 +#include <bpf/bpf.h>
 +#include <inttypes.h>
 +#include <linux/if_link.h>
 +#include <linux/if_xdp.h>
 +#include <net/if.h>
 +#include <xdp/xsk.h>
 +
-+#include "clients.h"
++/**
-+#include "monitor/monitor.h"
++ * VhostIOVATree, able to:
-+#include "net/net.h"
++ * - Translate iova address
-+#include "qapi/error.h"
++ * - Reverse translate iova address (from translated to iova)
-+#include "qemu/cutils.h"
++ * - Allocate IOVA regions for translated range (linear operation)
-+#include "qemu/error-report.h"
++ */
-+#include "qemu/iov.h"
++struct VhostIOVATree {
-+#include "qemu/main-loop.h"
++    /* First addressable iova address in the device */
-+#include "qemu/memalign.h"
++    uint64_t iova_first;
 +
++    /* Last addressable iova address in the device */
++    uint64_t iova_last;
 +
-+typedef struct AFXDPState {
++    /* IOVA address to qemu memory maps. */
-+    NetClientState       nc;
++    IOVATree *iova_taddr_map;
 +};
 +
-+    struct xsk_socket    *xsk;
++/**
-+    struct xsk_ring_cons rx;
++ * Create a new IOVA tree
-+    struct xsk_ring_prod tx;
++ *
-+    struct xsk_ring_cons cq;
++ * Returns the new IOVA tree
-+    struct xsk_ring_prod fq;
++ */
 +VhostIOVATree *vhost_iova_tree_new(hwaddr iova_first, hwaddr iova_last)
 +{
 +    VhostIOVATree *tree = g_new(VhostIOVATree, 1);
 +
-+    char                 ifname[IFNAMSIZ];
++    /* Some devices do not like 0 addresses */
-+    int                  ifindex;
++    tree->iova_first = MAX(iova_first, iova_min_addr);
-+    bool                 read_poll;
++    tree->iova_last = iova_last;
 +    bool                 write_poll;
 +    uint32_t             outstanding_tx;
 +
-+    uint64_t             *pool;
++    tree->iova_taddr_map = iova_tree_new();
-+    uint32_t             n_pool;
++    return tree;
 +    char                 *buffer;
 +    struct xsk_umem      *umem;
 +
 +    uint32_t             n_queues;
 +    uint32_t             xdp_flags;
 +    bool                 inhibit;
 +} AFXDPState;
 +
 +#define AF_XDP_BATCH_SIZE 64
 +
 +static void af_xdp_send(void *opaque);
 +static void af_xdp_writable(void *opaque);
 +
 +/* Set the event-loop handlers for the af-xdp backend. */
 +static void af_xdp_update_fd_handler(AFXDPState *s)
 +{
 +    qemu_set_fd_handler(xsk_socket__fd(s->xsk),
 +                        s->read_poll ? af_xdp_send : NULL,
 +                        s->write_poll ? af_xdp_writable : NULL,
 +                        s);
 +}
 +
-+/* Update the read handler. */
++/**
-+static void af_xdp_read_poll(AFXDPState *s, bool enable)
++ * Delete an iova tree
 + */
 +void vhost_iova_tree_delete(VhostIOVATree *iova_tree)
 +{
-+    if (s->read_poll != enable) {
++    iova_tree_destroy(iova_tree->iova_taddr_map);
-+        s->read_poll = enable;
++    g_free(iova_tree);
 +        af_xdp_update_fd_handler(s);
 +    }
 +}
 +
-+/* Update the write handler. */
++/**
-+static void af_xdp_write_poll(AFXDPState *s, bool enable)
++ * Find the IOVA address stored from a memory address
 + *
 + * @tree: The iova tree
 + * @map: The map with the memory address
 + *
 + * Return the stored mapping, or NULL if not found.
 + */
 +const DMAMap *vhost_iova_tree_find_iova(const VhostIOVATree *tree,
 +                                        const DMAMap *map)
 +{
-+    if (s->write_poll != enable) {
++    return iova_tree_find_iova(tree->iova_taddr_map, map);
 +        s->write_poll = enable;
 +        af_xdp_update_fd_handler(s);
 +    }
 +}
 +
-+static void af_xdp_poll(NetClientState *nc, bool enable)
++/**
 + * Allocate a new mapping
 + *
 + * @tree: The iova tree
 + * @map: The iova map
 + *
 + * Returns:
 + * - IOVA_OK if the map fits in the container
 + * - IOVA_ERR_INVALID if the map does not make sense (like size overflow)
 + * - IOVA_ERR_NOMEM if tree cannot allocate more space.
 + *
 + * It returns assignated iova in map->iova if return value is VHOST_DMA_MAP_OK.
 + */
 +int vhost_iova_tree_map_alloc(VhostIOVATree *tree, DMAMap *map)
 +{
-+    AFXDPState *s = DO_UPCAST(AFXDPState, nc, nc);
++    /* Some vhost devices do not like addr 0. Skip first page */
 +    hwaddr iova_first = tree->iova_first ?: qemu_real_host_page_size;
 +
-+    if (s->read_poll != enable || s->write_poll != enable) {
++    if (map->translated_addr + map->size < map->translated_addr ||
-+        s->write_poll = enable;
++        map->perm == IOMMU_NONE) {
-+        s->read_poll  = enable;
++        return IOVA_ERR_INVALID;
 +        af_xdp_update_fd_handler(s);
 +    }
++
++    /* Allocate a node in IOVA address */
++    return iova_tree_alloc_map(tree->iova_taddr_map, map, iova_first,
++                               tree->iova_last);
 +}
 +
-+static void af_xdp_complete_tx(AFXDPState *s)
++/**
 + * Remove existing mappings from iova tree
 + *
 + * @iova_tree: The vhost iova tree
 + * @map: The map to remove
 + */
 +void vhost_iova_tree_remove(VhostIOVATree *iova_tree, const DMAMap *map)
 +{
-+    uint32_t idx = 0;
++    iova_tree_remove(iova_tree->iova_taddr_map, map);
-+    uint32_t done, i;
++}
-+    uint64_t *addr;
+diff --git a/hw/virtio/vhost-iova-tree.h b/hw/virtio/vhost-iova-tree.h
 new file mode 100644
 index XXXXXXX..XXXXXXX
 --- /dev/null
 +++ b/hw/virtio/vhost-iova-tree.h
@@ -XXX,XX +XXX,XX @@
 +/*
 + * vhost software live migration iova tree
 + *
 + * SPDX-FileCopyrightText: Red Hat, Inc. 2021
 + * SPDX-FileContributor: Author: Eugenio Pérez <eperezma@redhat.com>
 + *
 + * SPDX-License-Identifier: GPL-2.0-or-later
 + */
 +
-+    done = xsk_ring_cons__peek(&s->cq, XSK_RING_CONS__DEFAULT_NUM_DESCS, &idx);
++#ifndef HW_VIRTIO_VHOST_IOVA_TREE_H
 +#define HW_VIRTIO_VHOST_IOVA_TREE_H
 +
-+    for (i = 0; i < done; i++) {
++#include "qemu/iova-tree.h"
-+        addr = (void *) xsk_ring_cons__comp_addr(&s->cq, idx++);
++#include "exec/memory.h"
 +        s->pool[s->n_pool++] = *addr;
 +        s->outstanding_tx--;
 +    }
 +
-+    if (done) {
++typedef struct VhostIOVATree VhostIOVATree;
 +        xsk_ring_cons__release(&s->cq, done);
 +    }
 +}
 +
-+/*
++VhostIOVATree *vhost_iova_tree_new(uint64_t iova_first, uint64_t iova_last);
-+ * The fd_write() callback, invoked if the fd is marked as writable
++void vhost_iova_tree_delete(VhostIOVATree *iova_tree);
-+ * after a poll.
++G_DEFINE_AUTOPTR_CLEANUP_FUNC(VhostIOVATree, vhost_iova_tree_delete);
 + */
 +static void af_xdp_writable(void *opaque)
 +{
 +    AFXDPState *s = opaque;
 +
-+    /* Try to recover buffers that are already sent. */
++const DMAMap *vhost_iova_tree_find_iova(const VhostIOVATree *iova_tree,
-+    af_xdp_complete_tx(s);
++                                        const DMAMap *map);
 +int vhost_iova_tree_map_alloc(VhostIOVATree *iova_tree, DMAMap *map);
 +void vhost_iova_tree_remove(VhostIOVATree *iova_tree, const DMAMap *map);
 +
-+    /*
-+     * Unregister the handler, unless we still have packets to transmit
-+     * and kernel needs a wake up.
-+     */
-+    if (!s->outstanding_tx || !xsk_ring_prod__needs_wakeup(&s->tx)) {
-+        af_xdp_write_poll(s, false);
-+    }
-+
-+    /* Flush any buffered packets. */
-+    qemu_flush_queued_packets(&s->nc);
-+}
-+
-+static ssize_t af_xdp_receive(NetClientState *nc,
-+                              const uint8_t *buf, size_t size)
-+{
-+    AFXDPState *s = DO_UPCAST(AFXDPState, nc, nc);
-+    struct xdp_desc *desc;
-+    uint32_t idx;
-+    void *data;
-+
-+    /* Try to recover buffers that are already sent. */
-+    af_xdp_complete_tx(s);
-+
-+    if (size > XSK_UMEM__DEFAULT_FRAME_SIZE) {
-+        /* We can't transmit packet this size... */
-+        return size;
-+    }
-+
-+    if (!s->n_pool || !xsk_ring_prod__reserve(&s->tx, 1, &idx)) {
-+        /*
-+         * Out of buffers or space in tx ring.  Poll until we can write.
-+         * This will also kick the Tx, if it was waiting on CQ.
-+         */
-+        af_xdp_write_poll(s, true);
-+        return 0;
-+    }
-+
-+    desc = xsk_ring_prod__tx_desc(&s->tx, idx);
-+    desc->addr = s->pool[--s->n_pool];
-+    desc->len = size;
-+
-+    data = xsk_umem__get_data(s->buffer, desc->addr);
-+    memcpy(data, buf, size);
-+
-+    xsk_ring_prod__submit(&s->tx, 1);
-+    s->outstanding_tx++;
-+
-+    if (xsk_ring_prod__needs_wakeup(&s->tx)) {
-+        af_xdp_write_poll(s, true);
-+    }
-+
-+    return size;
-+}
-+
-+/*
-+ * Complete a previous send (backend --> guest) and enable the
-+ * fd_read callback.
-+ */
-+static void af_xdp_send_completed(NetClientState *nc, ssize_t len)
-+{
-+    AFXDPState *s = DO_UPCAST(AFXDPState, nc, nc);
-+
-+    af_xdp_read_poll(s, true);
-+}
-+
-+static void af_xdp_fq_refill(AFXDPState *s, uint32_t n)
-+{
-+    uint32_t i, idx = 0;
-+
-+    /* Leave one packet for Tx, just in case. */
-+    if (s->n_pool < n + 1) {
-+        n = s->n_pool;
-+    }
-+
-+    if (!n || !xsk_ring_prod__reserve(&s->fq, n, &idx)) {
-+        return;
-+    }
-+
-+    for (i = 0; i < n; i++) {
-+        *xsk_ring_prod__fill_addr(&s->fq, idx++) = s->pool[--s->n_pool];
-+    }
-+    xsk_ring_prod__submit(&s->fq, n);
-+
-+    if (xsk_ring_prod__needs_wakeup(&s->fq)) {
-+        /* Receive was blocked by not having enough buffers.  Wake it up. */
-+        af_xdp_read_poll(s, true);
-+    }
-+}
-+
-+static void af_xdp_send(void *opaque)
-+{
-+    uint32_t i, n_rx, idx = 0;
-+    AFXDPState *s = opaque;
-+
-+    n_rx = xsk_ring_cons__peek(&s->rx, AF_XDP_BATCH_SIZE, &idx);
-+    if (!n_rx) {
-+        return;
-+    }
-+
-+    for (i = 0; i < n_rx; i++) {
-+        const struct xdp_desc *desc;
-+        struct iovec iov;
-+
-+        desc = xsk_ring_cons__rx_desc(&s->rx, idx++);
-+
-+        iov.iov_base = xsk_umem__get_data(s->buffer, desc->addr);
-+        iov.iov_len = desc->len;
-+
-+        s->pool[s->n_pool++] = desc->addr;
-+
-+        if (!qemu_sendv_packet_async(&s->nc, &iov, 1,
-+                                     af_xdp_send_completed)) {
-+            /*
-+             * The peer does not receive anymore.  Packet is queued, stop
-+             * reading from the backend until af_xdp_send_completed().
-+             */
-+            af_xdp_read_poll(s, false);
-+
-+            /* Return unused descriptors to not break the ring cache. */
-+            xsk_ring_cons__cancel(&s->rx, n_rx - i - 1);
-+            n_rx = i + 1;
-+            break;
-+        }
-+    }
-+
-+    /* Release actually sent descriptors and try to re-fill. */
-+    xsk_ring_cons__release(&s->rx, n_rx);
-+    af_xdp_fq_refill(s, AF_XDP_BATCH_SIZE);
-+}
-+
-+/* Flush and close. */
-+static void af_xdp_cleanup(NetClientState *nc)
-+{
-+    AFXDPState *s = DO_UPCAST(AFXDPState, nc, nc);
-+
-+    qemu_purge_queued_packets(nc);
-+
-+    af_xdp_poll(nc, false);
-+
-+    xsk_socket__delete(s->xsk);
-+    s->xsk = NULL;
-+    g_free(s->pool);
-+    s->pool = NULL;
-+    xsk_umem__delete(s->umem);
-+    s->umem = NULL;
-+    qemu_vfree(s->buffer);
-+    s->buffer = NULL;
-+
-+    /* Remove the program if it's the last open queue. */
-+    if (!s->inhibit && nc->queue_index == s->n_queues - 1 && s->xdp_flags
-+        && bpf_xdp_detach(s->ifindex, s->xdp_flags, NULL) != 0) {
-+        fprintf(stderr,
-+                "af-xdp: unable to remove XDP program from '%s', ifindex: %d\n",
-+                s->ifname, s->ifindex);
-+    }
-+}
-+
-+static int af_xdp_umem_create(AFXDPState *s, int sock_fd, Error **errp)
-+{
-+    struct xsk_umem_config config = {
-+        .fill_size = XSK_RING_PROD__DEFAULT_NUM_DESCS,
-+        .comp_size = XSK_RING_CONS__DEFAULT_NUM_DESCS,
-+        .frame_size = XSK_UMEM__DEFAULT_FRAME_SIZE,
-+        .frame_headroom = 0,
-+    };
-+    uint64_t n_descs;
-+    uint64_t size;
-+    int64_t i;
-+    int ret;
-+
-+    /* Number of descriptors if all 4 queues (rx, tx, cq, fq) are full. */
-+    n_descs = (XSK_RING_PROD__DEFAULT_NUM_DESCS
-+               + XSK_RING_CONS__DEFAULT_NUM_DESCS) * 2;
-+    size = n_descs * XSK_UMEM__DEFAULT_FRAME_SIZE;
-+
-+    s->buffer = qemu_memalign(qemu_real_host_page_size(), size);
-+    memset(s->buffer, 0, size);
-+
-+    if (sock_fd < 0) {
-+        ret = xsk_umem__create(&s->umem, s->buffer, size,
-+                               &s->fq, &s->cq, &config);
-+    } else {
-+        ret = xsk_umem__create_with_fd(&s->umem, sock_fd, s->buffer, size,
-+                                       &s->fq, &s->cq, &config);
-+    }
-+
-+    if (ret) {
-+        qemu_vfree(s->buffer);
-+        error_setg_errno(errp, errno,
-+                         "failed to create umem for %s queue_index: %d",
-+                         s->ifname, s->nc.queue_index);
-+        return -1;
-+    }
-+
-+    s->pool = g_new(uint64_t, n_descs);
-+    /* Fill the pool in the opposite order, because it's a LIFO queue. */
-+    for (i = n_descs; i >= 0; i--) {
-+        s->pool[i] = i * XSK_UMEM__DEFAULT_FRAME_SIZE;
-+    }
-+    s->n_pool = n_descs;
-+
-+    af_xdp_fq_refill(s, XSK_RING_PROD__DEFAULT_NUM_DESCS);
-+
-+    return 0;
-+}
-+
-+static int af_xdp_socket_create(AFXDPState *s,
-+                                const NetdevAFXDPOptions *opts, Error **errp)
-+{
-+    struct xsk_socket_config cfg = {
-+        .rx_size = XSK_RING_CONS__DEFAULT_NUM_DESCS,
-+        .tx_size = XSK_RING_PROD__DEFAULT_NUM_DESCS,
-+        .libxdp_flags = 0,
-+        .bind_flags = XDP_USE_NEED_WAKEUP,
-+        .xdp_flags = XDP_FLAGS_UPDATE_IF_NOEXIST,
-+    };
-+    int queue_id, error = 0;
-+
-+    s->inhibit = opts->has_inhibit && opts->inhibit;
-+    if (s->inhibit) {
-+        cfg.libxdp_flags |= XSK_LIBXDP_FLAGS__INHIBIT_PROG_LOAD;
-+    }
-+
-+    if (opts->has_force_copy && opts->force_copy) {
-+        cfg.bind_flags |= XDP_COPY;
-+    }
-+
-+    queue_id = s->nc.queue_index;
-+    if (opts->has_start_queue && opts->start_queue > 0) {
-+        queue_id += opts->start_queue;
-+    }
-+
-+    if (opts->has_mode) {
-+        /* Specific mode requested. */
-+        cfg.xdp_flags |= (opts->mode == AFXDP_MODE_NATIVE)
-+                         ? XDP_FLAGS_DRV_MODE : XDP_FLAGS_SKB_MODE;
-+        if (xsk_socket__create(&s->xsk, s->ifname, queue_id,
-+                               s->umem, &s->rx, &s->tx, &cfg)) {
-+            error = errno;
-+        }
-+    } else {
-+        /* No mode requested, try native first. */
-+        cfg.xdp_flags |= XDP_FLAGS_DRV_MODE;
-+
-+        if (xsk_socket__create(&s->xsk, s->ifname, queue_id,
-+                               s->umem, &s->rx, &s->tx, &cfg)) {
-+            /* Can't use native mode, try skb. */
-+            cfg.xdp_flags &= ~XDP_FLAGS_DRV_MODE;
-+            cfg.xdp_flags |= XDP_FLAGS_SKB_MODE;
-+
-+            if (xsk_socket__create(&s->xsk, s->ifname, queue_id,
-+                                   s->umem, &s->rx, &s->tx, &cfg)) {
-+                error = errno;
-+            }
-+        }
-+    }
-+
-+    if (error) {
-+        error_setg_errno(errp, error,
-+                         "failed to create AF_XDP socket for %s queue_id: %d",
-+                         s->ifname, queue_id);
-+        return -1;
-+    }
-+
-+    s->xdp_flags = cfg.xdp_flags;
-+
-+    return 0;
-+}
-+
-+/* NetClientInfo methods. */
-+static NetClientInfo net_af_xdp_info = {
-+    .type = NET_CLIENT_DRIVER_AF_XDP,
-+    .size = sizeof(AFXDPState),
-+    .receive = af_xdp_receive,
-+    .poll = af_xdp_poll,
-+    .cleanup = af_xdp_cleanup,
-+};
-+
-+static int *parse_socket_fds(const char *sock_fds_str,
-+                             int64_t n_expected, Error **errp)
-+{
-+    gchar **substrings = g_strsplit(sock_fds_str, ":", -1);
-+    int64_t i, n_sock_fds = g_strv_length(substrings);
-+    int *sock_fds = NULL;
-+
-+    if (n_sock_fds != n_expected) {
-+        error_setg(errp, "expected %"PRIi64" socket fds, got %"PRIi64,
-+                   n_expected, n_sock_fds);
-+        goto exit;
-+    }
-+
-+    sock_fds = g_new(int, n_sock_fds);
-+
-+    for (i = 0; i < n_sock_fds; i++) {
-+        sock_fds[i] = monitor_fd_param(monitor_cur(), substrings[i], errp);
-+        if (sock_fds[i] < 0) {
-+            g_free(sock_fds);
-+            sock_fds = NULL;
-+            goto exit;
-+        }
-+    }
-+
-+exit:
-+    g_strfreev(substrings);
-+    return sock_fds;
-+}
-+
-+/*
-+ * The exported init function.
-+ *
-+ * ... -netdev af-xdp,ifname="..."
-+ */
-+int net_init_af_xdp(const Netdev *netdev,
-+                    const char *name, NetClientState *peer, Error **errp)
-+{
-+    const NetdevAFXDPOptions *opts = &netdev->u.af_xdp;
-+    NetClientState *nc, *nc0 = NULL;
-+    unsigned int ifindex;
-+    uint32_t prog_id = 0;
-+    int *sock_fds = NULL;
-+    int64_t i, queues;
-+    Error *err = NULL;
-+    AFXDPState *s;
-+
-+    ifindex = if_nametoindex(opts->ifname);
-+    if (!ifindex) {
-+        error_setg_errno(errp, errno, "failed to get ifindex for '%s'",
-+                         opts->ifname);
-+        return -1;
-+    }
-+
-+    queues = opts->has_queues ? opts->queues : 1;
-+    if (queues < 1) {
-+        error_setg(errp, "invalid number of queues (%" PRIi64 ") for '%s'",
-+                   queues, opts->ifname);
-+        return -1;
-+    }
-+
-+    if ((opts->has_inhibit && opts->inhibit) != !!opts->sock_fds) {
-+        error_setg(errp, "'inhibit=on' requires 'sock-fds' and vice versa");
-+        return -1;
-+    }
-+
-+    if (opts->sock_fds) {
-+        sock_fds = parse_socket_fds(opts->sock_fds, queues, errp);
-+        if (!sock_fds) {
-+            return -1;
-+        }
-+    }
-+
-+    for (i = 0; i < queues; i++) {
-+        nc = qemu_new_net_client(&net_af_xdp_info, peer, "af-xdp", name);
-+        qemu_set_info_str(nc, "af-xdp%"PRIi64" to %s", i, opts->ifname);
-+        nc->queue_index = i;
-+
-+        if (!nc0) {
-+            nc0 = nc;
-+        }
-+
-+        s = DO_UPCAST(AFXDPState, nc, nc);
-+
-+        pstrcpy(s->ifname, sizeof(s->ifname), opts->ifname);
-+        s->ifindex = ifindex;
-+        s->n_queues = queues;
-+
-+        if (af_xdp_umem_create(s, sock_fds ? sock_fds[i] : -1, errp)
-+            || af_xdp_socket_create(s, opts, errp)) {
-+            /* Make sure the XDP program will be removed. */
-+            s->n_queues = i;
-+            error_propagate(errp, err);
-+            goto err;
-+        }
-+    }
-+
-+    if (nc0) {
-+        s = DO_UPCAST(AFXDPState, nc, nc0);
-+        if (bpf_xdp_query_id(s->ifindex, s->xdp_flags, &prog_id) || !prog_id) {
-+            error_setg_errno(errp, errno,
-+                             "no XDP program loaded on '%s', ifindex: %d",
-+                             s->ifname, s->ifindex);
-+            goto err;
-+        }
-+    }
-+
-+    af_xdp_read_poll(s, true); /* Initially only poll for reads. */
-+
-+    return 0;
-+
-+err:
-+    g_free(sock_fds);
-+    if (nc0) {
-+        qemu_del_net_client(nc0);
-+    }
-+
-+    return -1;
-+}
-diff --git a/net/clients.h b/net/clients.h
-index XXXXXXX..XXXXXXX 100644
---- a/net/clients.h
-+++ b/net/clients.h
-@@ -XXX,XX +XXX,XX @@ int net_init_netmap(const Netdev *netdev, const char *name,
-                     NetClientState *peer, Error **errp);
- #endif
-+#ifdef CONFIG_AF_XDP
-+int net_init_af_xdp(const Netdev *netdev, const char *name,
-+                    NetClientState *peer, Error **errp);
 +#endif
-+
- int net_init_vhost_user(const Netdev *netdev, const char *name,
-                         NetClientState *peer, Error **errp);
-diff --git a/net/meson.build b/net/meson.build
-index XXXXXXX..XXXXXXX 100644
---- a/net/meson.build
-+++ b/net/meson.build
-@@ -XXX,XX +XXX,XX @@ system_ss.add(when: vde, if_true: files('vde.c'))
- if have_netmap
-   system_ss.add(files('netmap.c'))
- endif
-+
-+system_ss.add(when: libxdp, if_true: files('af-xdp.c'))
-+
- if have_vhost_net_user
-   system_ss.add(when: 'CONFIG_VIRTIO_NET', if_true: files('vhost-user.c'), if_false: files('vhost-user-stub.c'))
-   system_ss.add(when: 'CONFIG_ALL', if_true: files('vhost-user-stub.c'))
-diff --git a/net/net.c b/net/net.c
-index XXXXXXX..XXXXXXX 100644
---- a/net/net.c
-+++ b/net/net.c
-@@ -XXX,XX +XXX,XX @@ static int (* const net_client_init_fun[NET_CLIENT_DRIVER__MAX])(
- #ifdef CONFIG_NETMAP
-         [NET_CLIENT_DRIVER_NETMAP]    = net_init_netmap,
- #endif
-+#ifdef CONFIG_AF_XDP
-+        [NET_CLIENT_DRIVER_AF_XDP]    = net_init_af_xdp,
-+#endif
- #ifdef CONFIG_NET_BRIDGE
-         [NET_CLIENT_DRIVER_BRIDGE]    = net_init_bridge,
- #endif
-@@ -XXX,XX +XXX,XX @@ void show_netdevs(void)
- #ifdef CONFIG_NETMAP
-         "netmap",
- #endif
-+#ifdef CONFIG_AF_XDP
-+        "af-xdp",
-+#endif
- #ifdef CONFIG_POSIX
-         "vhost-user",
- #endif
-diff --git a/qapi/net.json b/qapi/net.json
-index XXXXXXX..XXXXXXX 100644
---- a/qapi/net.json
-+++ b/qapi/net.json
-@@ -XXX,XX +XXX,XX @@
-     '*devname':    'str' } }
- ##
-+# @AFXDPMode:
-+#
-+# Attach mode for a default XDP program
-+#
-+# @skb: generic mode, no driver support necessary
-+#
-+# @native: DRV mode, program is attached to a driver, packets are passed to
-+#     the socket without allocation of skb.
-+#
-+# Since: 8.2
-+##
-+{ 'enum': 'AFXDPMode',
-+  'data': [ 'native', 'skb' ],
-+  'if': 'CONFIG_AF_XDP' }
-+
-+##
-+# @NetdevAFXDPOptions:
-+#
-+# AF_XDP network backend
-+#
-+# @ifname: The name of an existing network interface.
-+#
-+# @mode: Attach mode for a default XDP program.  If not specified, then
-+#     'native' will be tried first, then 'skb'.
-+#
-+# @force-copy: Force XDP copy mode even if device supports zero-copy.
-+#     (default: false)
-+#
-+# @queues: number of queues to be used for multiqueue interfaces (default: 1).
-+#
-+# @start-queue: Use @queues starting from this queue number (default: 0).
-+#
-+# @inhibit: Don't load a default XDP program, use one already loaded to
-+#     the interface (default: false).  Requires @sock-fds.
-+#
-+# @sock-fds: A colon (:) separated list of file descriptors for already open
-+#     but not bound AF_XDP sockets in the queue order.  One fd per queue.
-+#     These descriptors should already be added into XDP socket map for
-+#     corresponding queues.  Requires @inhibit.
-+#
-+# Since: 8.2
-+##
-+{ 'struct': 'NetdevAFXDPOptions',
-+  'data': {
-+    'ifname':       'str',
-+    '*mode':        'AFXDPMode',
-+    '*force-copy':  'bool',
-+    '*queues':      'int',
-+    '*start-queue': 'int',
-+    '*inhibit':     'bool',
-+    '*sock-fds':    'str' },
-+  'if': 'CONFIG_AF_XDP' }
-+
-+##
- # @NetdevVhostUserOptions:
- #
- # Vhost-user network backend
-@@ -XXX,XX +XXX,XX @@
- # @vmnet-bridged: since 7.1
- # @stream: since 7.2
- # @dgram: since 7.2
-+# @af-xdp: since 8.2
- #
- # Since: 2.7
- ##
-@@ -XXX,XX +XXX,XX @@
-   'data': [ 'none', 'nic', 'user', 'tap', 'l2tpv3', 'socket', 'stream',
-             'dgram', 'vde', 'bridge', 'hubport', 'netmap', 'vhost-user',
-             'vhost-vdpa',
-+            { 'name': 'af-xdp', 'if': 'CONFIG_AF_XDP' },
-             { 'name': 'vmnet-host', 'if': 'CONFIG_VMNET' },
-             { 'name': 'vmnet-shared', 'if': 'CONFIG_VMNET' },
-             { 'name': 'vmnet-bridged', 'if': 'CONFIG_VMNET' }] }
-@@ -XXX,XX +XXX,XX @@
-     'bridge':   'NetdevBridgeOptions',
-     'hubport':  'NetdevHubPortOptions',
-     'netmap':   'NetdevNetmapOptions',
-+    'af-xdp':   { 'type': 'NetdevAFXDPOptions',
-+                  'if': 'CONFIG_AF_XDP' },
-     'vhost-user': 'NetdevVhostUserOptions',
-     'vhost-vdpa': 'NetdevVhostVDPAOptions',
-     'vmnet-host': { 'type': 'NetdevVmnetHostOptions',
-diff --git a/qemu-options.hx b/qemu-options.hx
-index XXXXXXX..XXXXXXX 100644
---- a/qemu-options.hx
-+++ b/qemu-options.hx
-@@ -XXX,XX +XXX,XX @@ DEF("netdev", HAS_ARG, QEMU_OPTION_netdev,
-     "                VALE port (created on the fly) called 'name' ('nmname' is name of the \n"
-     "                netmap device, defaults to '/dev/netmap')\n"
- #endif
-+#ifdef CONFIG_AF_XDP
-+    "-netdev af-xdp,id=str,ifname=name[,mode=native|skb][,force-copy=on|off]\n"
-+    "         [,queues=n][,start-queue=m][,inhibit=on|off][,sock-fds=x:y:...:z]\n"
-+    "                attach to the existing network interface 'name' with AF_XDP socket\n"
-+    "                use 'mode=MODE' to specify an XDP program attach mode\n"
-+    "                use 'force-copy=on|off' to force XDP copy mode even if device supports zero-copy (default: off)\n"
-+    "                use 'inhibit=on|off' to inhibit loading of a default XDP program (default: off)\n"
-+    "                with inhibit=on,\n"
-+    "                  use 'sock-fds' to provide file descriptors for already open AF_XDP sockets\n"
-+    "                  added to a socket map in XDP program.  One socket per queue.\n"
-+    "                use 'queues=n' to specify how many queues of a multiqueue interface should be used\n"
-+    "                use 'start-queue=m' to specify the first queue that should be used\n"
-+#endif
- #ifdef CONFIG_POSIX
-     "-netdev vhost-user,id=str,chardev=dev[,vhostforce=on|off]\n"
-     "                configure a vhost-user network, backed by a chardev 'dev'\n"
-@@ -XXX,XX +XXX,XX @@ DEF("nic", HAS_ARG, QEMU_OPTION_nic,
- #ifdef CONFIG_NETMAP
-     "netmap|"
- #endif
-+#ifdef CONFIG_AF_XDP
-+    "af-xdp|"
-+#endif
- #ifdef CONFIG_POSIX
-     "vhost-user|"
- #endif
-@@ -XXX,XX +XXX,XX @@ DEF("net", HAS_ARG, QEMU_OPTION_net,
- #ifdef CONFIG_NETMAP
-     "netmap|"
- #endif
-+#ifdef CONFIG_AF_XDP
-+    "af-xdp|"
-+#endif
- #ifdef CONFIG_VMNET
-     "vmnet-host|vmnet-shared|vmnet-bridged|"
- #endif
-@@ -XXX,XX +XXX,XX @@ DEF("net", HAS_ARG, QEMU_OPTION_net,
-     "                old way to initialize a host network interface\n"
-     "                (use the -netdev option if possible instead)\n", QEMU_ARCH_ALL)
- SRST
--``-nic [tap|bridge|user|l2tpv3|vde|netmap|vhost-user|socket][,...][,mac=macaddr][,model=mn]``
-+``-nic [tap|bridge|user|l2tpv3|vde|netmap|af-xdp|vhost-user|socket][,...][,mac=macaddr][,model=mn]``
-     This option is a shortcut for configuring both the on-board
-     (default) guest NIC hardware and the host network backend in one go.
-     The host backend options are the same as with the corresponding
-@@ -XXX,XX +XXX,XX @@ SRST
-         # launch QEMU instance
-         |qemu_system| linux.img -nic vde,sock=/tmp/myswitch
-+``-netdev af-xdp,id=str,ifname=name[,mode=native|skb][,force-copy=on|off][,queues=n][,start-queue=m][,inhibit=on|off][,sock-fds=x:y:...:z]``
-+    Configure AF_XDP backend to connect to a network interface 'name'
-+    using AF_XDP socket.  A specific program attach mode for a default
-+    XDP program can be forced with 'mode', defaults to best-effort,
-+    where the likely most performant mode will be in use.  Number of queues
-+    'n' should generally match the number or queues in the interface,
-+    defaults to 1.  Traffic arriving on non-configured device queues will
-+    not be delivered to the network backend.
-+
-+    .. parsed-literal::
-+
-+        # set number of queues to 4
-+        ethtool -L eth0 combined 4
-+        # launch QEMU instance
-+        |qemu_system| linux.img -device virtio-net-pci,netdev=n1 \\
-+            -netdev af-xdp,id=n1,ifname=eth0,queues=4
-+
-+    'start-queue' option can be specified if a particular range of queues
-+    [m, m + n] should be in use.  For example, this is may be necessary in
-+    order to use certain NICs in native mode.  Kernel allows the driver to
-+    create a separate set of XDP queues on top of regular ones, and only
-+    these queues can be used for AF_XDP sockets.  NICs that work this way
-+    may also require an additional traffic redirection with ethtool to these
-+    special queues.
-+
-+    .. parsed-literal::
-+
-+        # set number of queues to 1
-+        ethtool -L eth0 combined 1
-+        # redirect all the traffic to the second queue (id: 1)
-+        # note: drivers may require non-empty key/mask pair.
-+        ethtool -N eth0 flow-type ether \\
-+            dst 00:00:00:00:00:00 m FF:FF:FF:FF:FF:FE action 1
-+        ethtool -N eth0 flow-type ether \\
-+            dst 00:00:00:00:00:01 m FF:FF:FF:FF:FF:FE action 1
-+        # launch QEMU instance
-+        |qemu_system| linux.img -device virtio-net-pci,netdev=n1 \\
-+            -netdev af-xdp,id=n1,ifname=eth0,queues=1,start-queue=1
-+
-+    XDP program can also be loaded externally.  In this case 'inhibit' option
-+    should be set to 'on' and 'sock-fds' provided with file descriptors for
-+    already open but not bound XDP sockets already added to a socket map for
-+    corresponding queues.  One socket per queue.
-+
-+    .. parsed-literal::
-+
-+        |qemu_system| linux.img -device virtio-net-pci,netdev=n1 \\
-+            -netdev af-xdp,id=n1,ifname=eth0,queues=3,inhibit=on,sock-fds=15:16:17
-+
- ``-netdev vhost-user,chardev=id[,vhostforce=on|off][,queues=n]``
-     Establish a vhost-user netdev, backed by a chardev id. The chardev
-     should be a unix domain socket backed one. The vhost-user uses a
-diff --git a/scripts/ci/org.centos/stream/8/x86_64/configure b/scripts/ci/org.centos/stream/8/x86_64/configure
-index XXXXXXX..XXXXXXX 100755
---- a/scripts/ci/org.centos/stream/8/x86_64/configure
-+++ b/scripts/ci/org.centos/stream/8/x86_64/configure
-@@ -XXX,XX +XXX,XX @@
- --block-drv-ro-whitelist="vmdk,vhdx,vpc,https,ssh" \
- --with-coroutine=ucontext \
- --tls-priority=@QEMU,SYSTEM \
-+--disable-af-xdp \
- --disable-attr \
- --disable-auth-pam \
- --disable-avx2 \
-diff --git a/scripts/meson-buildoptions.sh b/scripts/meson-buildoptions.sh
-index XXXXXXX..XXXXXXX 100644
---- a/scripts/meson-buildoptions.sh
-+++ b/scripts/meson-buildoptions.sh
-@@ -XXX,XX +XXX,XX @@ meson_options_help() {
-   printf "%s\n" 'disabled with --disable-FEATURE, default is enabled if available'
-   printf "%s\n" '(unless built with --without-default-features):'
-   printf "%s\n" ''
-+  printf "%s\n" '  af-xdp          AF_XDP network backend support'
-   printf "%s\n" '  alsa            ALSA sound support'
-   printf "%s\n" '  attr            attr/xattr support'
-   printf "%s\n" '  auth-pam        PAM access control'
-@@ -XXX,XX +XXX,XX @@ meson_options_help() {
- }
- _meson_option_parse() {
-   case $1 in
-+    --enable-af-xdp) printf "%s" -Daf_xdp=enabled ;;
-+    --disable-af-xdp) printf "%s" -Daf_xdp=disabled ;;
-     --enable-alsa) printf "%s" -Dalsa=enabled ;;
-     --disable-alsa) printf "%s" -Dalsa=disabled ;;
-     --enable-attr) printf "%s" -Dattr=enabled ;;
-diff --git a/tests/docker/dockerfiles/debian-amd64.docker b/tests/docker/dockerfiles/debian-amd64.docker
-index XXXXXXX..XXXXXXX 100644
---- a/tests/docker/dockerfiles/debian-amd64.docker
-+++ b/tests/docker/dockerfiles/debian-amd64.docker
-@@ -XXX,XX +XXX,XX @@ RUN export DEBIAN_FRONTEND=noninteractive && \
-                       libvirglrenderer-dev \
-                       libvte-2.91-dev \
-                       libxen-dev \
-+                      libxdp-dev \
-                       libzstd-dev \
-                       llvm \
-                       locales \
 --
 .7.4

-[PULL 06/17] igb: rename E1000E_RingInfo_st
+[PULL V3 12/15] vdpa: Add custom IOTLB translations to SVQ
-From: Tomasz Dzieciol <t.dzieciol@partner.samsung.com>
+From: Eugenio Pérez <eperezma@redhat.com>
-Rename E1000E_RingInfo_st and E1000E_RingInfo according to qemu typdefs guide.
+Use translations added in VhostIOVATree in SVQ.
-Signed-off-by: Tomasz Dzieciol <t.dzieciol@partner.samsung.com>
+Only introduce usage here, not allocation and deallocation. As with
-Reviewed-by: Akihiko Odaki <akihiko.odaki@daynix.com>
+previous patches, we use the dead code paths of shadow_vqs_enabled to
-Tested-by: Akihiko Odaki <akihiko.odaki@daynix.com>
+avoid commiting too many changes at once. These are impossible to take
 at the moment.
 Signed-off-by: Eugenio Pérez <eperezma@redhat.com>
 Acked-by: Michael S. Tsirkin <mst@redhat.com>
 Signed-off-by: Jason Wang <jasowang@redhat.com>
 ---
- hw/net/e1000e_core.c | 34 +++++++++++++++++-----------------
+ hw/virtio/vhost-shadow-virtqueue.c |  86 +++++++++++++++++++++++---
- hw/net/igb_core.c    | 42 +++++++++++++++++++++---------------------
+ hw/virtio/vhost-shadow-virtqueue.h |   6 +-
-files changed, 38 insertions(+), 38 deletions(-)
+ hw/virtio/vhost-vdpa.c             | 122 +++++++++++++++++++++++++++++++------
+ include/hw/virtio/vhost-vdpa.h     |   3 +
-diff --git a/hw/net/e1000e_core.c b/hw/net/e1000e_core.c
+files changed, 187 insertions(+), 30 deletions(-)
 diff --git a/hw/virtio/vhost-shadow-virtqueue.c b/hw/virtio/vhost-shadow-virtqueue.c
 index XXXXXXX..XXXXXXX 100644
---- a/hw/net/e1000e_core.c
+--- a/hw/virtio/vhost-shadow-virtqueue.c
-+++ b/hw/net/e1000e_core.c
++++ b/hw/virtio/vhost-shadow-virtqueue.c
-@@ -XXX,XX +XXX,XX @@ e1000e_txdesc_writeback(E1000ECore *core, dma_addr_t base,
+@@ -XXX,XX +XXX,XX @@ static uint16_t vhost_svq_available_slots(const VhostShadowVirtqueue *svq)
-     return e1000e_tx_wb_interrupt_cause(core, queue_idx);
+     return svq->vring.num - (svq->shadow_avail_idx - svq->shadow_used_idx);
  }
--typedef struct E1000E_RingInfo_st {
+-static void vhost_vring_write_descs(VhostShadowVirtqueue *svq,
-+typedef struct E1000ERingInfo {
++/**
-     int dbah;
++ * Translate addresses between the qemu's virtual address and the SVQ IOVA
-     int dbal;
++ *
-     int dlen;
++ * @svq: Shadow VirtQueue
-     int dh;
++ * @vaddr: Translated IOVA addresses
-     int dt;
++ * @iovec: Source qemu's VA addresses
-     int idx;
++ * @num: Length of iovec and minimum length of vaddr
--} E1000E_RingInfo;
++ */
-+} E1000ERingInfo;
++static bool vhost_svq_translate_addr(const VhostShadowVirtqueue *svq,
++                                     hwaddr *addrs, const struct iovec *iovec,
- static inline bool
++                                     size_t num)
--e1000e_ring_empty(E1000ECore *core, const E1000E_RingInfo *r)
++{
-+e1000e_ring_empty(E1000ECore *core, const E1000ERingInfo *r)
++    if (num == 0) {
- {
++        return true;
-     return core->mac[r->dh] == core->mac[r->dt] ||
++    }
-                 core->mac[r->dt] >= core->mac[r->dlen] / E1000_RING_DESC_LEN;
++
 +    for (size_t i = 0; i < num; ++i) {
 +        DMAMap needle = {
 +            .translated_addr = (hwaddr)(uintptr_t)iovec[i].iov_base,
 +            .size = iovec[i].iov_len,
 +        };
 +        Int128 needle_last, map_last;
 +        size_t off;
 +
 +        const DMAMap *map = vhost_iova_tree_find_iova(svq->iova_tree, &needle);
 +        /*
 +         * Map cannot be NULL since iova map contains all guest space and
 +         * qemu already has a physical address mapped
 +         */
 +        if (unlikely(!map)) {
 +            qemu_log_mask(LOG_GUEST_ERROR,
 +                          "Invalid address 0x%"HWADDR_PRIx" given by guest",
 +                          needle.translated_addr);
 +            return false;
 +        }
 +
 +        off = needle.translated_addr - map->translated_addr;
 +        addrs[i] = map->iova + off;
 +
 +        needle_last = int128_add(int128_make64(needle.translated_addr),
 +                                 int128_make64(iovec[i].iov_len));
 +        map_last = int128_make64(map->translated_addr + map->size);
 +        if (unlikely(int128_gt(needle_last, map_last))) {
 +            qemu_log_mask(LOG_GUEST_ERROR,
 +                          "Guest buffer expands over iova range");
 +            return false;
 +        }
 +    }
 +
 +    return true;
 +}
 +
 +static void vhost_vring_write_descs(VhostShadowVirtqueue *svq, hwaddr *sg,
                                      const struct iovec *iovec, size_t num,
                                      bool more_descs, bool write)
  {
@@ -XXX,XX +XXX,XX @@ static void vhost_vring_write_descs(VhostShadowVirtqueue *svq,
          } else {
              descs[i].flags = flags;
          }
 -        descs[i].addr = cpu_to_le64((hwaddr)(intptr_t)iovec[n].iov_base);
 +        descs[i].addr = cpu_to_le64(sg[n]);
          descs[i].len = cpu_to_le32(iovec[n].iov_len);
          last = i;
@@ -XXX,XX +XXX,XX @@ static bool vhost_svq_add_split(VhostShadowVirtqueue *svq,
  {
      unsigned avail_idx;
      vring_avail_t *avail = svq->vring.avail;
 +    bool ok;
 +    g_autofree hwaddr *sgs = g_new(hwaddr, MAX(elem->out_num, elem->in_num));
      *head = svq->free_head;
@@ -XXX,XX +XXX,XX @@ static bool vhost_svq_add_split(VhostShadowVirtqueue *svq,
          return false;
      }
 -    vhost_vring_write_descs(svq, elem->out_sg, elem->out_num, elem->in_num > 0,
 -                            false);
 -    vhost_vring_write_descs(svq, elem->in_sg, elem->in_num, false, true);
 +    ok = vhost_svq_translate_addr(svq, sgs, elem->out_sg, elem->out_num);
 +    if (unlikely(!ok)) {
 +        return false;
 +    }
 +    vhost_vring_write_descs(svq, sgs, elem->out_sg, elem->out_num,
 +                            elem->in_num > 0, false);
 +
 +
 +    ok = vhost_svq_translate_addr(svq, sgs, elem->in_sg, elem->in_num);
 +    if (unlikely(!ok)) {
 +        return false;
 +    }
 +
 +    vhost_vring_write_descs(svq, sgs, elem->in_sg, elem->in_num, false, true);
      /*
       * Put the entry in the available array (but don't update avail->idx until
@@ -XXX,XX +XXX,XX @@ void vhost_svq_set_svq_call_fd(VhostShadowVirtqueue *svq, int call_fd)
  void vhost_svq_get_vring_addr(const VhostShadowVirtqueue *svq,
                                struct vhost_vring_addr *addr)
  {
 -    addr->desc_user_addr = (uint64_t)(intptr_t)svq->vring.desc;
 -    addr->avail_user_addr = (uint64_t)(intptr_t)svq->vring.avail;
 -    addr->used_user_addr = (uint64_t)(intptr_t)svq->vring.used;
 +    addr->desc_user_addr = (uint64_t)(uintptr_t)svq->vring.desc;
 +    addr->avail_user_addr = (uint64_t)(uintptr_t)svq->vring.avail;
 +    addr->used_user_addr = (uint64_t)(uintptr_t)svq->vring.used;
  }
- static inline uint64_t
+ size_t vhost_svq_driver_area_size(const VhostShadowVirtqueue *svq)
--e1000e_ring_base(E1000ECore *core, const E1000E_RingInfo *r)
+@@ -XXX,XX +XXX,XX @@ void vhost_svq_stop(VhostShadowVirtqueue *svq)
-+e1000e_ring_base(E1000ECore *core, const E1000ERingInfo *r)
+  * Creates vhost shadow virtqueue, and instructs the vhost device to use the
- {
+  * shadow methods and file descriptors.
-     uint64_t bah = core->mac[r->dbah];
+  *
-     uint64_t bal = core->mac[r->dbal];
++ * @iova_tree: Tree to perform descriptors translations
-@@ -XXX,XX +XXX,XX @@ e1000e_ring_base(E1000ECore *core, const E1000E_RingInfo *r)
++ *
   * Returns the new virtqueue or NULL.
   *
   * In case of error, reason is reported through error_report.
   */
 -VhostShadowVirtqueue *vhost_svq_new(void)
 +VhostShadowVirtqueue *vhost_svq_new(VhostIOVATree *iova_tree)
  {
      g_autofree VhostShadowVirtqueue *svq = g_new0(VhostShadowVirtqueue, 1);
      int r;
@@ -XXX,XX +XXX,XX @@ VhostShadowVirtqueue *vhost_svq_new(void)
      event_notifier_init_fd(&svq->svq_kick, VHOST_FILE_UNBIND);
      event_notifier_set_handler(&svq->hdev_call, vhost_svq_handle_call);
 +    svq->iova_tree = iova_tree;
      return g_steal_pointer(&svq);
  err_init_hdev_call:
 diff --git a/hw/virtio/vhost-shadow-virtqueue.h b/hw/virtio/vhost-shadow-virtqueue.h
 index XXXXXXX..XXXXXXX 100644
 --- a/hw/virtio/vhost-shadow-virtqueue.h
 +++ b/hw/virtio/vhost-shadow-virtqueue.h
@@ -XXX,XX +XXX,XX @@
  #include "qemu/event_notifier.h"
  #include "hw/virtio/virtio.h"
  #include "standard-headers/linux/vhost_types.h"
 +#include "hw/virtio/vhost-iova-tree.h"
  /* Shadow virtqueue to relay notifications */
  typedef struct VhostShadowVirtqueue {
@@ -XXX,XX +XXX,XX @@ typedef struct VhostShadowVirtqueue {
      /* Virtio device */
      VirtIODevice *vdev;
 +    /* IOVA mapping */
 +    VhostIOVATree *iova_tree;
 +
      /* Map for use the guest's descriptors */
      VirtQueueElement **ring_id_maps;
@@ -XXX,XX +XXX,XX @@ void vhost_svq_start(VhostShadowVirtqueue *svq, VirtIODevice *vdev,
                       VirtQueue *vq);
  void vhost_svq_stop(VhostShadowVirtqueue *svq);
 -VhostShadowVirtqueue *vhost_svq_new(void);
 +VhostShadowVirtqueue *vhost_svq_new(VhostIOVATree *iova_tree);
  void vhost_svq_free(gpointer vq);
  G_DEFINE_AUTOPTR_CLEANUP_FUNC(VhostShadowVirtqueue, vhost_svq_free);
 diff --git a/hw/virtio/vhost-vdpa.c b/hw/virtio/vhost-vdpa.c
 index XXXXXXX..XXXXXXX 100644
 --- a/hw/virtio/vhost-vdpa.c
 +++ b/hw/virtio/vhost-vdpa.c
@@ -XXX,XX +XXX,XX @@ static void vhost_vdpa_listener_region_add(MemoryListener *listener,
                                           vaddr, section->readonly);
      llsize = int128_sub(llend, int128_make64(iova));
 +    if (v->shadow_vqs_enabled) {
 +        DMAMap mem_region = {
 +            .translated_addr = (hwaddr)(uintptr_t)vaddr,
 +            .size = int128_get64(llsize) - 1,
 +            .perm = IOMMU_ACCESS_FLAG(true, section->readonly),
 +        };
 +
 +        int r = vhost_iova_tree_map_alloc(v->iova_tree, &mem_region);
 +        if (unlikely(r != IOVA_OK)) {
 +            error_report("Can't allocate a mapping (%d)", r);
 +            goto fail;
 +        }
 +
 +        iova = mem_region.iova;
 +    }
      vhost_vdpa_iotlb_batch_begin_once(v);
      ret = vhost_vdpa_dma_map(v, iova, int128_get64(llsize),
@@ -XXX,XX +XXX,XX @@ static void vhost_vdpa_listener_region_del(MemoryListener *listener,
      llsize = int128_sub(llend, int128_make64(iova));
 +    if (v->shadow_vqs_enabled) {
 +        const DMAMap *result;
 +        const void *vaddr = memory_region_get_ram_ptr(section->mr) +
 +            section->offset_within_region +
 +            (iova - section->offset_within_address_space);
 +        DMAMap mem_region = {
 +            .translated_addr = (hwaddr)(uintptr_t)vaddr,
 +            .size = int128_get64(llsize) - 1,
 +        };
 +
 +        result = vhost_iova_tree_find_iova(v->iova_tree, &mem_region);
 +        iova = result->iova;
 +        vhost_iova_tree_remove(v->iova_tree, &mem_region);
 +    }
      vhost_vdpa_iotlb_batch_begin_once(v);
      ret = vhost_vdpa_dma_unmap(v, iova, int128_get64(llsize));
      if (ret) {
@@ -XXX,XX +XXX,XX @@ static int vhost_vdpa_init_svq(struct vhost_dev *hdev, struct vhost_vdpa *v,
      shadow_vqs = g_ptr_array_new_full(hdev->nvqs, vhost_svq_free);
      for (unsigned n = 0; n < hdev->nvqs; ++n) {
 -        g_autoptr(VhostShadowVirtqueue) svq = vhost_svq_new();
 +        g_autoptr(VhostShadowVirtqueue) svq = vhost_svq_new(v->iova_tree);
          if (unlikely(!svq)) {
              error_setg(errp, "Cannot create svq %u", n);
@@ -XXX,XX +XXX,XX @@ static int vhost_vdpa_svq_set_fds(struct vhost_dev *dev,
  /**
   * Unmap a SVQ area in the device
   */
 -static bool vhost_vdpa_svq_unmap_ring(struct vhost_vdpa *v, hwaddr iova,
 -                                      hwaddr size)
 +static bool vhost_vdpa_svq_unmap_ring(struct vhost_vdpa *v,
 +                                      const DMAMap *needle)
  {
 +    const DMAMap *result = vhost_iova_tree_find_iova(v->iova_tree, needle);
 +    hwaddr size;
      int r;
 -    size = ROUND_UP(size, qemu_real_host_page_size);
 -    r = vhost_vdpa_dma_unmap(v, iova, size);
 +    if (unlikely(!result)) {
 +        error_report("Unable to find SVQ address to unmap");
 +        return false;
 +    }
 +
 +    size = ROUND_UP(result->size, qemu_real_host_page_size);
 +    r = vhost_vdpa_dma_unmap(v, result->iova, size);
      return r == 0;
  }
- static inline uint64_t
+ static bool vhost_vdpa_svq_unmap_rings(struct vhost_dev *dev,
--e1000e_ring_head_descr(E1000ECore *core, const E1000E_RingInfo *r)
+                                        const VhostShadowVirtqueue *svq)
-+e1000e_ring_head_descr(E1000ECore *core, const E1000ERingInfo *r)
+ {
- {
++    DMAMap needle = {};
-     return e1000e_ring_base(core, r) + E1000_RING_DESC_LEN * core->mac[r->dh];
+     struct vhost_vdpa *v = dev->opaque;
      struct vhost_vring_addr svq_addr;
 -    size_t device_size = vhost_svq_device_area_size(svq);
 -    size_t driver_size = vhost_svq_driver_area_size(svq);
      bool ok;
      vhost_svq_get_vring_addr(svq, &svq_addr);
 -    ok = vhost_vdpa_svq_unmap_ring(v, svq_addr.desc_user_addr, driver_size);
 +    needle.translated_addr = svq_addr.desc_user_addr;
 +    ok = vhost_vdpa_svq_unmap_ring(v, &needle);
      if (unlikely(!ok)) {
          return false;
      }
 -    return vhost_vdpa_svq_unmap_ring(v, svq_addr.used_user_addr, device_size);
 +    needle.translated_addr = svq_addr.used_user_addr;
 +    return vhost_vdpa_svq_unmap_ring(v, &needle);
 +}
 +
 +/**
 + * Map the SVQ area in the device
 + *
 + * @v: Vhost-vdpa device
 + * @needle: The area to search iova
 + * @errorp: Error pointer
 + */
 +static bool vhost_vdpa_svq_map_ring(struct vhost_vdpa *v, DMAMap *needle,
 +                                    Error **errp)
 +{
 +    int r;
 +
 +    r = vhost_iova_tree_map_alloc(v->iova_tree, needle);
 +    if (unlikely(r != IOVA_OK)) {
 +        error_setg(errp, "Cannot allocate iova (%d)", r);
 +        return false;
 +    }
 +
 +    r = vhost_vdpa_dma_map(v, needle->iova, needle->size + 1,
 +                           (void *)(uintptr_t)needle->translated_addr,
 +                           needle->perm == IOMMU_RO);
 +    if (unlikely(r != 0)) {
 +        error_setg_errno(errp, -r, "Cannot map region to device");
 +        vhost_iova_tree_remove(v->iova_tree, needle);
 +    }
 +
 +    return r == 0;
  }
- static inline void
+ /**
--e1000e_ring_advance(E1000ECore *core, const E1000E_RingInfo *r, uint32_t count)
+@@ -XXX,XX +XXX,XX @@ static bool vhost_vdpa_svq_map_rings(struct vhost_dev *dev,
-+e1000e_ring_advance(E1000ECore *core, const E1000ERingInfo *r, uint32_t count)
+                                      struct vhost_vring_addr *addr,
- {
+                                      Error **errp)
-     core->mac[r->dh] += count;
+ {
++    DMAMap device_region, driver_region;
-@@ -XXX,XX +XXX,XX @@ e1000e_ring_advance(E1000ECore *core, const E1000E_RingInfo *r, uint32_t count)
++    struct vhost_vring_addr svq_addr;
      struct vhost_vdpa *v = dev->opaque;
      size_t device_size = vhost_svq_device_area_size(svq);
      size_t driver_size = vhost_svq_driver_area_size(svq);
 -    int r;
 +    size_t avail_offset;
 +    bool ok;
      ERRP_GUARD();
 -    vhost_svq_get_vring_addr(svq, addr);
 +    vhost_svq_get_vring_addr(svq, &svq_addr);
 -    r = vhost_vdpa_dma_map(v, addr->desc_user_addr, driver_size,
 -                           (void *)(uintptr_t)addr->desc_user_addr, true);
 -    if (unlikely(r != 0)) {
 -        error_setg_errno(errp, -r, "Cannot create vq driver region: ");
 +    driver_region = (DMAMap) {
 +        .translated_addr = svq_addr.desc_user_addr,
 +        .size = driver_size - 1,
 +        .perm = IOMMU_RO,
 +    };
 +    ok = vhost_vdpa_svq_map_ring(v, &driver_region, errp);
 +    if (unlikely(!ok)) {
 +        error_prepend(errp, "Cannot create vq driver region: ");
          return false;
      }
 +    addr->desc_user_addr = driver_region.iova;
 +    avail_offset = svq_addr.avail_user_addr - svq_addr.desc_user_addr;
 +    addr->avail_user_addr = driver_region.iova + avail_offset;
 -    r = vhost_vdpa_dma_map(v, addr->used_user_addr, device_size,
 -                           (void *)(intptr_t)addr->used_user_addr, false);
 -    if (unlikely(r != 0)) {
 -        error_setg_errno(errp, -r, "Cannot create vq device region: ");
 +    device_region = (DMAMap) {
 +        .translated_addr = svq_addr.used_user_addr,
 +        .size = device_size - 1,
 +        .perm = IOMMU_RW,
 +    };
 +    ok = vhost_vdpa_svq_map_ring(v, &device_region, errp);
 +    if (unlikely(!ok)) {
 +        error_prepend(errp, "Cannot create vq device region: ");
 +        vhost_vdpa_svq_unmap_ring(v, &driver_region);
      }
 +    addr->used_user_addr = device_region.iova;
 -    return r == 0;
 +    return ok;
  }
- static inline uint32_t
+ static bool vhost_vdpa_svq_setup(struct vhost_dev *dev,
--e1000e_ring_free_descr_num(E1000ECore *core, const E1000E_RingInfo *r)
+diff --git a/include/hw/virtio/vhost-vdpa.h b/include/hw/virtio/vhost-vdpa.h
 +e1000e_ring_free_descr_num(E1000ECore *core, const E1000ERingInfo *r)
  {
      trace_e1000e_ring_free_space(r->idx, core->mac[r->dlen],
                                   core->mac[r->dh],  core->mac[r->dt]);
@@ -XXX,XX +XXX,XX @@ e1000e_ring_free_descr_num(E1000ECore *core, const E1000E_RingInfo *r)
  }
  static inline bool
 -e1000e_ring_enabled(E1000ECore *core, const E1000E_RingInfo *r)
 +e1000e_ring_enabled(E1000ECore *core, const E1000ERingInfo *r)
  {
      return core->mac[r->dlen] > 0;
  }
  static inline uint32_t
 -e1000e_ring_len(E1000ECore *core, const E1000E_RingInfo *r)
 +e1000e_ring_len(E1000ECore *core, const E1000ERingInfo *r)
  {
      return core->mac[r->dlen];
  }
  typedef struct E1000E_TxRing_st {
 -    const E1000E_RingInfo *i;
 +    const E1000ERingInfo *i;
      struct e1000e_tx *tx;
  } E1000E_TxRing;
@@ -XXX,XX +XXX,XX @@ e1000e_mq_queue_idx(int base_reg_idx, int reg_idx)
  static inline void
  e1000e_tx_ring_init(E1000ECore *core, E1000E_TxRing *txr, int idx)
  {
 -    static const E1000E_RingInfo i[E1000E_NUM_QUEUES] = {
 +    static const E1000ERingInfo i[E1000E_NUM_QUEUES] = {
          { TDBAH,  TDBAL,  TDLEN,  TDH,  TDT, 0 },
          { TDBAH1, TDBAL1, TDLEN1, TDH1, TDT1, 1 }
      };
@@ -XXX,XX +XXX,XX @@ e1000e_tx_ring_init(E1000ECore *core, E1000E_TxRing *txr, int idx)
  }
  typedef struct E1000E_RxRing_st {
 -    const E1000E_RingInfo *i;
 +    const E1000ERingInfo *i;
  } E1000E_RxRing;
  static inline void
  e1000e_rx_ring_init(E1000ECore *core, E1000E_RxRing *rxr, int idx)
  {
 -    static const E1000E_RingInfo i[E1000E_NUM_QUEUES] = {
 +    static const E1000ERingInfo i[E1000E_NUM_QUEUES] = {
          { RDBAH0, RDBAL0, RDLEN0, RDH0, RDT0, 0 },
          { RDBAH1, RDBAL1, RDLEN1, RDH1, RDT1, 1 }
      };
@@ -XXX,XX +XXX,XX @@ e1000e_start_xmit(E1000ECore *core, const E1000E_TxRing *txr)
      dma_addr_t base;
      struct e1000_tx_desc desc;
      bool ide = false;
 -    const E1000E_RingInfo *txi = txr->i;
 +    const E1000ERingInfo *txi = txr->i;
      uint32_t cause = E1000_ICS_TXQE;
      if (!(core->mac[TCTL] & E1000_TCTL_EN)) {
@@ -XXX,XX +XXX,XX @@ e1000e_start_xmit(E1000ECore *core, const E1000E_TxRing *txr)
  }
  static bool
 -e1000e_has_rxbufs(E1000ECore *core, const E1000E_RingInfo *r,
 +e1000e_has_rxbufs(E1000ECore *core, const E1000ERingInfo *r,
                    size_t total_size)
  {
      uint32_t bufs = e1000e_ring_free_descr_num(core, r);
@@ -XXX,XX +XXX,XX @@ e1000e_update_rx_stats(E1000ECore *core, size_t pkt_size, size_t pkt_fcs_size)
  }
  static inline bool
 -e1000e_rx_descr_threshold_hit(E1000ECore *core, const E1000E_RingInfo *rxi)
 +e1000e_rx_descr_threshold_hit(E1000ECore *core, const E1000ERingInfo *rxi)
  {
      return e1000e_ring_free_descr_num(core, rxi) ==
             e1000e_ring_len(core, rxi) >> core->rxbuf_min_shift;
@@ -XXX,XX +XXX,XX @@ e1000e_write_packet_to_guest(E1000ECore *core, struct NetRxPkt *pkt,
      struct iovec *iov = net_rx_pkt_get_iovec(pkt);
      size_t size = net_rx_pkt_get_total_len(pkt);
      size_t total_size = size + e1000x_fcs_len(core->mac);
 -    const E1000E_RingInfo *rxi;
 +    const E1000ERingInfo *rxi;
      size_t ps_hdr_len = 0;
      bool do_ps = e1000e_do_ps(core, pkt, &ps_hdr_len);
      bool is_first = true;
 diff --git a/hw/net/igb_core.c b/hw/net/igb_core.c
 index XXXXXXX..XXXXXXX 100644
---- a/hw/net/igb_core.c
+--- a/include/hw/virtio/vhost-vdpa.h
-+++ b/hw/net/igb_core.c
++++ b/include/hw/virtio/vhost-vdpa.h
-@@ -XXX,XX +XXX,XX @@ static uint32_t igb_rx_wb_eic(IGBCore *core, int queue_idx)
+@@ -XXX,XX +XXX,XX @@
-     return (ent & E1000_IVAR_VALID) ? BIT(ent & 0x1f) : 0;
- }
+ #include <gmodule.h>
--typedef struct E1000E_RingInfo_st {
++#include "hw/virtio/vhost-iova-tree.h"
-+typedef struct E1000ERingInfo {
+ #include "hw/virtio/virtio.h"
-     int dbah;
+ #include "standard-headers/linux/vhost_types.h"
-     int dbal;
-     int dlen;
+@@ -XXX,XX +XXX,XX @@ typedef struct vhost_vdpa {
-     int dh;
+     MemoryListener listener;
-     int dt;
+     struct vhost_vdpa_iova_range iova_range;
-     int idx;
+     bool shadow_vqs_enabled;
--} E1000E_RingInfo;
++    /* IOVA mapping used by the Shadow Virtqueue */
-+} E1000ERingInfo;
++    VhostIOVATree *iova_tree;
+     GPtrArray *shadow_vqs;
- static inline bool
+     struct vhost_dev *dev;
--igb_ring_empty(IGBCore *core, const E1000E_RingInfo *r)
+     VhostVDPAHostNotifier notifier[VIRTIO_QUEUE_MAX];
 +igb_ring_empty(IGBCore *core, const E1000ERingInfo *r)
  {
      return core->mac[r->dh] == core->mac[r->dt] ||
                  core->mac[r->dt] >= core->mac[r->dlen] / E1000_RING_DESC_LEN;
  }
  static inline uint64_t
 -igb_ring_base(IGBCore *core, const E1000E_RingInfo *r)
 +igb_ring_base(IGBCore *core, const E1000ERingInfo *r)
  {
      uint64_t bah = core->mac[r->dbah];
      uint64_t bal = core->mac[r->dbal];
@@ -XXX,XX +XXX,XX @@ igb_ring_base(IGBCore *core, const E1000E_RingInfo *r)
  }
  static inline uint64_t
 -igb_ring_head_descr(IGBCore *core, const E1000E_RingInfo *r)
 +igb_ring_head_descr(IGBCore *core, const E1000ERingInfo *r)
  {
      return igb_ring_base(core, r) + E1000_RING_DESC_LEN * core->mac[r->dh];
  }
  static inline void
 -igb_ring_advance(IGBCore *core, const E1000E_RingInfo *r, uint32_t count)
 +igb_ring_advance(IGBCore *core, const E1000ERingInfo *r, uint32_t count)
  {
      core->mac[r->dh] += count;
@@ -XXX,XX +XXX,XX @@ igb_ring_advance(IGBCore *core, const E1000E_RingInfo *r, uint32_t count)
  }
  static inline uint32_t
 -igb_ring_free_descr_num(IGBCore *core, const E1000E_RingInfo *r)
 +igb_ring_free_descr_num(IGBCore *core, const E1000ERingInfo *r)
  {
      trace_e1000e_ring_free_space(r->idx, core->mac[r->dlen],
                                   core->mac[r->dh],  core->mac[r->dt]);
@@ -XXX,XX +XXX,XX @@ igb_ring_free_descr_num(IGBCore *core, const E1000E_RingInfo *r)
  }
  static inline bool
 -igb_ring_enabled(IGBCore *core, const E1000E_RingInfo *r)
 +igb_ring_enabled(IGBCore *core, const E1000ERingInfo *r)
  {
      return core->mac[r->dlen] > 0;
  }
  typedef struct IGB_TxRing_st {
 -    const E1000E_RingInfo *i;
 +    const E1000ERingInfo *i;
      struct igb_tx *tx;
  } IGB_TxRing;
@@ -XXX,XX +XXX,XX @@ igb_mq_queue_idx(int base_reg_idx, int reg_idx)
  static inline void
  igb_tx_ring_init(IGBCore *core, IGB_TxRing *txr, int idx)
  {
 -    static const E1000E_RingInfo i[IGB_NUM_QUEUES] = {
 +    static const E1000ERingInfo i[IGB_NUM_QUEUES] = {
          { TDBAH0, TDBAL0, TDLEN0, TDH0, TDT0, 0 },
          { TDBAH1, TDBAL1, TDLEN1, TDH1, TDT1, 1 },
          { TDBAH2, TDBAL2, TDLEN2, TDH2, TDT2, 2 },
@@ -XXX,XX +XXX,XX @@ igb_tx_ring_init(IGBCore *core, IGB_TxRing *txr, int idx)
  }
  typedef struct E1000E_RxRing_st {
 -    const E1000E_RingInfo *i;
 +    const E1000ERingInfo *i;
  } E1000E_RxRing;
  static inline void
  igb_rx_ring_init(IGBCore *core, E1000E_RxRing *rxr, int idx)
  {
 -    static const E1000E_RingInfo i[IGB_NUM_QUEUES] = {
 +    static const E1000ERingInfo i[IGB_NUM_QUEUES] = {
          { RDBAH0, RDBAL0, RDLEN0, RDH0, RDT0, 0 },
          { RDBAH1, RDBAL1, RDLEN1, RDH1, RDT1, 1 },
          { RDBAH2, RDBAL2, RDLEN2, RDH2, RDT2, 2 },
@@ -XXX,XX +XXX,XX @@ igb_rx_ring_init(IGBCore *core, E1000E_RxRing *rxr, int idx)
  static uint32_t
  igb_txdesc_writeback(IGBCore *core, dma_addr_t base,
                       union e1000_adv_tx_desc *tx_desc,
 -                     const E1000E_RingInfo *txi)
 +                     const E1000ERingInfo *txi)
  {
      PCIDevice *d;
      uint32_t cmd_type_len = le32_to_cpu(tx_desc->read.cmd_type_len);
@@ -XXX,XX +XXX,XX @@ igb_txdesc_writeback(IGBCore *core, dma_addr_t base,
  }
  static inline bool
 -igb_tx_enabled(IGBCore *core, const E1000E_RingInfo *txi)
 +igb_tx_enabled(IGBCore *core, const E1000ERingInfo *txi)
  {
      bool vmdq = core->mac[MRQC] & 1;
      uint16_t qn = txi->idx;
@@ -XXX,XX +XXX,XX @@ igb_start_xmit(IGBCore *core, const IGB_TxRing *txr)
      PCIDevice *d;
      dma_addr_t base;
      union e1000_adv_tx_desc desc;
 -    const E1000E_RingInfo *txi = txr->i;
 +    const E1000ERingInfo *txi = txr->i;
      uint32_t eic = 0;
      if (!igb_tx_enabled(core, txi)) {
@@ -XXX,XX +XXX,XX @@ igb_start_xmit(IGBCore *core, const IGB_TxRing *txr)
  }
  static uint32_t
 -igb_rxbufsize(IGBCore *core, const E1000E_RingInfo *r)
 +igb_rxbufsize(IGBCore *core, const E1000ERingInfo *r)
  {
      uint32_t srrctl = core->mac[E1000_SRRCTL(r->idx) >> 2];
      uint32_t bsizepkt = srrctl & E1000_SRRCTL_BSIZEPKT_MASK;
@@ -XXX,XX +XXX,XX @@ igb_rxbufsize(IGBCore *core, const E1000E_RingInfo *r)
  }
  static bool
 -igb_has_rxbufs(IGBCore *core, const E1000E_RingInfo *r, size_t total_size)
 +igb_has_rxbufs(IGBCore *core, const E1000ERingInfo *r, size_t total_size)
  {
      uint32_t bufs = igb_ring_free_descr_num(core, r);
      uint32_t bufsize = igb_rxbufsize(core, r);
@@ -XXX,XX +XXX,XX @@ igb_write_to_rx_buffers(IGBCore *core,
  }
  static void
 -igb_update_rx_stats(IGBCore *core, const E1000E_RingInfo *rxi,
 +igb_update_rx_stats(IGBCore *core, const E1000ERingInfo *rxi,
                      size_t pkt_size, size_t pkt_fcs_size)
  {
      eth_pkt_types_e pkt_type = net_rx_pkt_get_packet_type(core->rx_pkt);
@@ -XXX,XX +XXX,XX @@ igb_update_rx_stats(IGBCore *core, const E1000E_RingInfo *rxi,
  }
  static inline bool
 -igb_rx_descr_threshold_hit(IGBCore *core, const E1000E_RingInfo *rxi)
 +igb_rx_descr_threshold_hit(IGBCore *core, const E1000ERingInfo *rxi)
  {
      return igb_ring_free_descr_num(core, rxi) ==
             ((core->mac[E1000_SRRCTL(rxi->idx) >> 2] >> 20) & 31) * 16;
@@ -XXX,XX +XXX,XX @@ igb_write_packet_to_guest(IGBCore *core, struct NetRxPkt *pkt,
      struct iovec *iov = net_rx_pkt_get_iovec(pkt);
      size_t size = net_rx_pkt_get_total_len(pkt);
      size_t total_size = size + e1000x_fcs_len(core->mac);
 -    const E1000E_RingInfo *rxi = rxr->i;
 +    const E1000ERingInfo *rxi = rxr->i;
      size_t bufsize = igb_rxbufsize(core, rxi);
      d = pcie_sriov_get_vf_at_index(core->owner, rxi->idx % 8);
@@ -XXX,XX +XXX,XX @@ igb_write_packet_to_guest(IGBCore *core, struct NetRxPkt *pkt,
  }
  static bool
 -igb_rx_strip_vlan(IGBCore *core, const E1000E_RingInfo *rxi)
 +igb_rx_strip_vlan(IGBCore *core, const E1000ERingInfo *rxi)
  {
      if (core->mac[MRQC] & 1) {
          uint16_t pool = rxi->idx % IGB_NUM_VM_POOLS;
 --
 .7.4

-[PULL 09/17] igb: add IPv6 extended headers traffic detection
+[PULL V3 13/15] vdpa: Adapt vhost_vdpa_get_vring_base to SVQ
-From: Tomasz Dzieciol <t.dzieciol@partner.samsung.com>
+From: Eugenio Pérez <eperezma@redhat.com>
-Signed-off-by: Tomasz Dzieciol <t.dzieciol@partner.samsung.com>
+This is needed to achieve migration, so the destination can restore its
-Reviewed-by: Akihiko Odaki <akihiko.odaki@daynix.com>
+index.
-Tested-by: Akihiko Odaki <akihiko.odaki@daynix.com>
 Setting base as last used idx, so destination will see as available all
 the entries that the device did not use, including the in-flight
 processing ones.
 This is ok for networking, but other kinds of devices might have
 problems with these retransmissions.
 Signed-off-by: Eugenio Pérez <eperezma@redhat.com>
 Acked-by: Michael S. Tsirkin <mst@redhat.com>
 Signed-off-by: Jason Wang <jasowang@redhat.com>
 ---
- hw/net/igb_core.c | 4 +++-
+ hw/virtio/vhost-vdpa.c | 17 +++++++++++++++++
- hw/net/igb_regs.h | 1 +
+file changed, 17 insertions(+)
 files changed, 4 insertions(+), 1 deletion(-)
-diff --git a/hw/net/igb_core.c b/hw/net/igb_core.c
+diff --git a/hw/virtio/vhost-vdpa.c b/hw/virtio/vhost-vdpa.c
 index XXXXXXX..XXXXXXX 100644
---- a/hw/net/igb_core.c
+--- a/hw/virtio/vhost-vdpa.c
-+++ b/hw/net/igb_core.c
++++ b/hw/virtio/vhost-vdpa.c
-@@ -XXX,XX +XXX,XX @@ igb_rx_desc_get_packet_type(IGBCore *core, struct NetRxPkt *pkt, uint16_t etqf)
+@@ -XXX,XX +XXX,XX @@ static int vhost_vdpa_set_vring_base(struct vhost_dev *dev,
-     net_rx_pkt_get_protocols(pkt, &hasip4, &hasip6, &l4hdr_proto);
+ static int vhost_vdpa_get_vring_base(struct vhost_dev *dev,
+                                        struct vhost_vring_state *ring)
-     if (hasip6 && !(core->mac[RFCTL] & E1000_RFCTL_IPV6_DIS)) {
+ {
--        pkt_type = E1000_ADVRXD_PKT_IP6;
++    struct vhost_vdpa *v = dev->opaque;
-+        eth_ip6_hdr_info *ip6hdr_info = net_rx_pkt_get_ip6_info(pkt);
+     int ret;
-+        pkt_type = ip6hdr_info->has_ext_hdrs ? E1000_ADVRXD_PKT_IP6E :
-+                                               E1000_ADVRXD_PKT_IP6;
++    if (v->shadow_vqs_enabled) {
-     } else if (hasip4) {
++        VhostShadowVirtqueue *svq = g_ptr_array_index(v->shadow_vqs,
-         pkt_type = E1000_ADVRXD_PKT_IP4;
++                                                      ring->index);
-     } else {
++
-diff --git a/hw/net/igb_regs.h b/hw/net/igb_regs.h
++        /*
-index XXXXXXX..XXXXXXX 100644
++         * Setting base as last used idx, so destination will see as available
---- a/hw/net/igb_regs.h
++         * all the entries that the device did not use, including the in-flight
-+++ b/hw/net/igb_regs.h
++         * processing ones.
-@@ -XXX,XX +XXX,XX @@ union e1000_adv_rx_desc {
++         *
++         * TODO: This is ok for networking, but other kinds of devices might
- #define E1000_ADVRXD_PKT_IP4  BIT(0)
++         * have problems with these retransmissions.
- #define E1000_ADVRXD_PKT_IP6  BIT(2)
++         */
-+#define E1000_ADVRXD_PKT_IP6E BIT(3)
++        ring->num = svq->last_used_idx;
- #define E1000_ADVRXD_PKT_TCP  BIT(4)
++        return 0;
- #define E1000_ADVRXD_PKT_UDP  BIT(5)
++    }
- #define E1000_ADVRXD_PKT_SCTP BIT(6)
++
      ret = vhost_vdpa_call(dev, VHOST_GET_VRING_BASE, ring);
      trace_vhost_vdpa_get_vring_base(dev, ring->index, ring->num);
      return ret;
 --
 .7.4

-[PULL 05/17] igb: remove TCP ACK detection
+[PULL V3 14/15] vdpa: Never set log_base addr if SVQ is enabled
-From: Tomasz Dzieciol <t.dzieciol@partner.samsung.com>
+From: Eugenio Pérez <eperezma@redhat.com>
-TCP ACK detection is no longer present in igb.
+Setting the log address would make the device start reporting invalid
 dirty memory because the SVQ vrings are located in qemu's memory.
-Signed-off-by: Tomasz Dzieciol <t.dzieciol@partner.samsung.com>
+Signed-off-by: Eugenio Pérez <eperezma@redhat.com>
-Reviewed-by: Akihiko Odaki <akihiko.odaki@daynix.com>
+Acked-by: Michael S. Tsirkin <mst@redhat.com>
 Tested-by: Akihiko Odaki <akihiko.odaki@daynix.com>
 Signed-off-by: Jason Wang <jasowang@redhat.com>
 ---
- hw/net/igb_core.c | 5 -----
+ hw/virtio/vhost-vdpa.c | 3 ++-
-file changed, 5 deletions(-)
+file changed, 2 insertions(+), 1 deletion(-)
-diff --git a/hw/net/igb_core.c b/hw/net/igb_core.c
+diff --git a/hw/virtio/vhost-vdpa.c b/hw/virtio/vhost-vdpa.c
 index XXXXXXX..XXXXXXX 100644
---- a/hw/net/igb_core.c
+--- a/hw/virtio/vhost-vdpa.c
-+++ b/hw/net/igb_core.c
++++ b/hw/virtio/vhost-vdpa.c
-@@ -XXX,XX +XXX,XX @@ igb_build_rx_metadata(IGBCore *core,
+@@ -XXX,XX +XXX,XX @@ static int vhost_vdpa_dev_start(struct vhost_dev *dev, bool started)
-             trace_e1000e_rx_metadata_ip_id(*ip_id);
+ static int vhost_vdpa_set_log_base(struct vhost_dev *dev, uint64_t base,
                                       struct vhost_log *log)
  {
 -    if (vhost_vdpa_one_time_request(dev)) {
 +    struct vhost_vdpa *v = dev->opaque;
 +    if (v->shadow_vqs_enabled || vhost_vdpa_one_time_request(dev)) {
          return 0;
      }
--    if (l4hdr_proto == ETH_L4_HDR_PROTO_TCP && net_rx_pkt_is_tcp_ack(pkt)) {
--        *status_flags |= E1000_RXD_STAT_ACK;
--        trace_e1000e_rx_metadata_ack();
--    }
--
-     if (pkt_info) {
-         *pkt_info = rss_info->enabled ? rss_info->type : 0;
 --
 .7.4

-[PULL 07/17] igb: RX descriptors guest writting refactoring
+[PULL V3 15/15] vdpa: Expose VHOST_F_LOG_ALL on SVQ
-From: Tomasz Dzieciol <t.dzieciol@partner.samsung.com>
+From: Eugenio Pérez <eperezma@redhat.com>
-Refactoring is done in preparation for support of multiple advanced
+SVQ is able to log the dirty bits by itself, so let's use it to not
-descriptors RX modes, especially packet-split modes.
+block migration.
-Signed-off-by: Tomasz Dzieciol <t.dzieciol@partner.samsung.com>
+Also, ignore set and clear of VHOST_F_LOG_ALL on set_features if SVQ is
-Reviewed-by: Akihiko Odaki <akihiko.odaki@daynix.com>
+enabled. Even if the device supports it, the reports would be nonsense
-Tested-by: Akihiko Odaki <akihiko.odaki@daynix.com>
+because SVQ memory is in the qemu region.
 The log region is still allocated. Future changes might skip that, but
 this series is already long enough.
 Signed-off-by: Eugenio Pérez <eperezma@redhat.com>
 Acked-by: Michael S. Tsirkin <mst@redhat.com>
 Signed-off-by: Jason Wang <jasowang@redhat.com>
 ---
- hw/net/igb_core.c   | 170 +++++++++++++++++++++++++++-------------------------
+ hw/virtio/vhost-vdpa.c         | 39 +++++++++++++++++++++++++++++++++++----
- hw/net/igb_regs.h   |  10 ++--
+ include/hw/virtio/vhost-vdpa.h |  1 +
- hw/net/trace-events |   4 +-
+files changed, 36 insertions(+), 4 deletions(-)
 files changed, 96 insertions(+), 88 deletions(-)
-diff --git a/hw/net/igb_core.c b/hw/net/igb_core.c
+diff --git a/hw/virtio/vhost-vdpa.c b/hw/virtio/vhost-vdpa.c
 index XXXXXXX..XXXXXXX 100644
---- a/hw/net/igb_core.c
+--- a/hw/virtio/vhost-vdpa.c
-+++ b/hw/net/igb_core.c
++++ b/hw/virtio/vhost-vdpa.c
-@@ -XXX,XX +XXX,XX @@ igb_verify_csum_in_sw(IGBCore *core,
+@@ -XXX,XX +XXX,XX @@ static bool vhost_vdpa_one_time_request(struct vhost_dev *dev)
      return v->index != 0;
  }
- static void
++static int vhost_vdpa_get_dev_features(struct vhost_dev *dev,
--igb_build_rx_metadata(IGBCore *core,
++                                       uint64_t *features)
--                      struct NetRxPkt *pkt,
++{
--                      bool is_eop,
++    int ret;
--                      const E1000E_RSSInfo *rss_info, uint16_t etqf, bool ts,
++
--                      uint16_t *pkt_info, uint16_t *hdr_info,
++    ret = vhost_vdpa_call(dev, VHOST_GET_FEATURES, features);
--                      uint32_t *rss,
++    trace_vhost_vdpa_get_features(dev, *features);
--                      uint32_t *status_flags,
++    return ret;
--                      uint16_t *ip_id,
++}
--                      uint16_t *vlan_tag)
++
-+igb_build_rx_metadata_common(IGBCore *core,
+ static int vhost_vdpa_init_svq(struct vhost_dev *hdev, struct vhost_vdpa *v,
-+                             struct NetRxPkt *pkt,
+                                Error **errp)
 +                             bool is_eop,
 +                             uint32_t *status_flags,
 +                             uint16_t *vlan_tag)
  {
-     struct virtio_net_hdr *vhdr;
+@@ -XXX,XX +XXX,XX @@ static int vhost_vdpa_init_svq(struct vhost_dev *hdev, struct vhost_vdpa *v,
-     bool hasip4, hasip6, csum_valid;
+         return 0;
@@ -XXX,XX +XXX,XX @@ igb_build_rx_metadata(IGBCore *core,
      *status_flags = E1000_RXD_STAT_DD;
      /* No additional metadata needed for non-EOP descriptors */
 -    /* TODO: EOP apply only to status so don't skip whole function. */
      if (!is_eop) {
          goto func_exit;
      }
-@@ -XXX,XX +XXX,XX @@ igb_build_rx_metadata(IGBCore *core,
-         trace_e1000e_rx_metadata_vlan(*vlan_tag);
+-    r = hdev->vhost_ops->vhost_get_features(hdev, &dev_features);
 +    r = vhost_vdpa_get_dev_features(hdev, &dev_features);
      if (r != 0) {
          error_setg_errno(errp, -r, "Can't get vdpa device features");
          return r;
@@ -XXX,XX +XXX,XX @@ static int vhost_vdpa_set_mem_table(struct vhost_dev *dev,
  static int vhost_vdpa_set_features(struct vhost_dev *dev,
                                     uint64_t features)
  {
 +    struct vhost_vdpa *v = dev->opaque;
      int ret;
      if (vhost_vdpa_one_time_request(dev)) {
          return 0;
      }
--    /* Packet parsing results */
++    if (v->shadow_vqs_enabled) {
--    if ((core->mac[RXCSUM] & E1000_RXCSUM_PCSD) != 0) {
++        if ((v->acked_features ^ features) == BIT_ULL(VHOST_F_LOG_ALL)) {
--        if (rss_info->enabled) {
++            /*
--            *rss = cpu_to_le32(rss_info->hash);
++             * QEMU is just trying to enable or disable logging. SVQ handles
--            trace_igb_rx_metadata_rss(*rss);
++             * this sepparately, so no need to forward this.
--        }
++             */
--    } else if (hasip4) {
++            v->acked_features = features;
--            *status_flags |= E1000_RXD_STAT_IPIDV;
++            return 0;
--            *ip_id = cpu_to_le16(net_rx_pkt_get_ip_id(pkt));
++        }
 -            trace_e1000e_rx_metadata_ip_id(*ip_id);
 -    }
 -
 -    if (pkt_info) {
 -        *pkt_info = rss_info->enabled ? rss_info->type : 0;
 -
 -        if (etqf < 8) {
 -            *pkt_info |= (BIT(11) | etqf) << 4;
 -        } else {
 -            if (hasip4) {
 -                *pkt_info |= E1000_ADVRXD_PKT_IP4;
 -            }
 -
 -            if (hasip6) {
 -                *pkt_info |= E1000_ADVRXD_PKT_IP6;
 -            }
 -
 -            switch (l4hdr_proto) {
 -            case ETH_L4_HDR_PROTO_TCP:
 -                *pkt_info |= E1000_ADVRXD_PKT_TCP;
 -                break;
 -
 -            case ETH_L4_HDR_PROTO_UDP:
 -                *pkt_info |= E1000_ADVRXD_PKT_UDP;
 -                break;
 -
 -            case ETH_L4_HDR_PROTO_SCTP:
 -                *pkt_info |= E1000_ADVRXD_PKT_SCTP;
 -                break;
 -
 -            default:
 -                break;
 -            }
 -        }
 -    }
 -
 -    if (hdr_info) {
 -        *hdr_info = 0;
 -    }
 -
 -    if (ts) {
 -        *status_flags |= BIT(16);
 -    }
 -
      /* RX CSO information */
      if (hasip6 && (core->mac[RFCTL] & E1000_RFCTL_IPV6_XSUM_DIS)) {
          trace_e1000e_rx_metadata_ipv6_sum_disabled();
@@ -XXX,XX +XXX,XX @@ func_exit:
  static inline void
  igb_write_lgcy_rx_descr(IGBCore *core, struct e1000_rx_desc *desc,
                          struct NetRxPkt *pkt,
 -                        const E1000E_RSSInfo *rss_info, uint16_t etqf, bool ts,
 +                        const E1000E_RSSInfo *rss_info,
                          uint16_t length)
  {
 -    uint32_t status_flags, rss;
 -    uint16_t ip_id;
 +    uint32_t status_flags;
      assert(!rss_info->enabled);
 +
-+    memset(desc, 0, sizeof(*desc));
++        v->acked_features = features;
      desc->length = cpu_to_le16(length);
 -    desc->csum = 0;
 +    igb_build_rx_metadata_common(core, pkt, pkt != NULL,
 +                                 &status_flags,
 +                                 &desc->special);
 -    igb_build_rx_metadata(core, pkt, pkt != NULL,
 -                          rss_info, etqf, ts,
 -                          NULL, NULL, &rss,
 -                          &status_flags, &ip_id,
 -                          &desc->special);
      desc->errors = (uint8_t) (le32_to_cpu(status_flags) >> 24);
      desc->status = (uint8_t) le32_to_cpu(status_flags);
  }
 +static uint16_t
 +igb_rx_desc_get_packet_type(IGBCore *core, struct NetRxPkt *pkt, uint16_t etqf)
 +{
 +    uint16_t pkt_type;
 +    bool hasip4, hasip6;
 +    EthL4HdrProto l4hdr_proto;
 +
-+    if (etqf < 8) {
++        /* We must not ack _F_LOG if SVQ is enabled */
-+        pkt_type = BIT(11) | etqf;
++        features &= ~BIT_ULL(VHOST_F_LOG_ALL);
 +        return pkt_type;
 +    }
 +
-+    net_rx_pkt_get_protocols(pkt, &hasip4, &hasip6, &l4hdr_proto);
+     trace_vhost_vdpa_set_features(dev, features);
      ret = vhost_vdpa_call(dev, VHOST_SET_FEATURES, &features);
      if (ret) {
@@ -XXX,XX +XXX,XX @@ static int vhost_vdpa_set_vring_call(struct vhost_dev *dev,
  static int vhost_vdpa_get_features(struct vhost_dev *dev,
                                       uint64_t *features)
  {
 -    int ret;
 +    struct vhost_vdpa *v = dev->opaque;
 +    int ret = vhost_vdpa_get_dev_features(dev, features);
 +
-+    if (hasip6 && !(core->mac[RFCTL] & E1000_RFCTL_IPV6_DIS)) {
++    if (ret == 0 && v->shadow_vqs_enabled) {
-+        pkt_type = E1000_ADVRXD_PKT_IP6;
++        /* Add SVQ logging capabilities */
-+    } else if (hasip4) {
++        *features |= BIT_ULL(VHOST_F_LOG_ALL);
 +        pkt_type = E1000_ADVRXD_PKT_IP4;
 +    } else {
 +        pkt_type = 0;
 +    }
-+
-+    switch (l4hdr_proto) {
+-    ret = vhost_vdpa_call(dev, VHOST_GET_FEATURES, features);
-+    case ETH_L4_HDR_PROTO_TCP:
+-    trace_vhost_vdpa_get_features(dev, *features);
-+        pkt_type |= E1000_ADVRXD_PKT_TCP;
+     return ret;
 +        break;
 +    case ETH_L4_HDR_PROTO_UDP:
 +        pkt_type |= E1000_ADVRXD_PKT_UDP;
 +        break;
 +    case ETH_L4_HDR_PROTO_SCTP:
 +        pkt_type |= E1000_ADVRXD_PKT_SCTP;
 +        break;
 +    default:
 +        break;
 +    }
 +
 +    return pkt_type;
 +}
 +
  static inline void
  igb_write_adv_rx_descr(IGBCore *core, union e1000_adv_rx_desc *desc,
                         struct NetRxPkt *pkt,
                         const E1000E_RSSInfo *rss_info, uint16_t etqf, bool ts,
                         uint16_t length)
  {
 +    bool hasip4, hasip6;
 +    EthL4HdrProto l4hdr_proto;
 +    uint16_t rss_type = 0, pkt_type;
 +    bool eop = (pkt != NULL);
 +    uint32_t adv_desc_status_error = 0;
      memset(&desc->wb, 0, sizeof(desc->wb));
      desc->wb.upper.length = cpu_to_le16(length);
 +    igb_build_rx_metadata_common(core, pkt, eop,
 +                                 &desc->wb.upper.status_error,
 +                                 &desc->wb.upper.vlan);
 +
 +    if (!eop) {
 +        return;
 +    }
 +
 +    net_rx_pkt_get_protocols(pkt, &hasip4, &hasip6, &l4hdr_proto);
 +
 +    if ((core->mac[RXCSUM] & E1000_RXCSUM_PCSD) != 0) {
 +        if (rss_info->enabled) {
 +            desc->wb.lower.hi_dword.rss = cpu_to_le32(rss_info->hash);
 +            rss_type = rss_info->type;
 +            trace_igb_rx_metadata_rss(desc->wb.lower.hi_dword.rss, rss_type);
 +        }
 +    } else if (hasip4) {
 +            adv_desc_status_error |= E1000_RXD_STAT_IPIDV;
 +            desc->wb.lower.hi_dword.csum_ip.ip_id =
 +                cpu_to_le16(net_rx_pkt_get_ip_id(pkt));
 +            trace_e1000e_rx_metadata_ip_id(
 +                desc->wb.lower.hi_dword.csum_ip.ip_id);
 +    }
 +
 +    if (ts) {
 +        adv_desc_status_error |= BIT(16);
 +    }
 -    igb_build_rx_metadata(core, pkt, pkt != NULL,
 -                          rss_info, etqf, ts,
 -                          &desc->wb.lower.lo_dword.pkt_info,
 -                          &desc->wb.lower.lo_dword.hdr_info,
 -                          &desc->wb.lower.hi_dword.rss,
 -                          &desc->wb.upper.status_error,
 -                          &desc->wb.lower.hi_dword.csum_ip.ip_id,
 -                          &desc->wb.upper.vlan);
 +    pkt_type = igb_rx_desc_get_packet_type(core, pkt, etqf);
 +    trace_e1000e_rx_metadata_pkt_type(pkt_type);
 +    desc->wb.lower.lo_dword.pkt_info = cpu_to_le16(rss_type | (pkt_type << 4));
 +    desc->wb.upper.status_error |= cpu_to_le32(adv_desc_status_error);
  }
- static inline void
+diff --git a/include/hw/virtio/vhost-vdpa.h b/include/hw/virtio/vhost-vdpa.h
@@ -XXX,XX +XXX,XX @@ igb_write_rx_descr(IGBCore *core, union e1000_rx_desc_union *desc,
                     uint16_t etqf, bool ts, uint16_t length)
  {
      if (igb_rx_use_legacy_descriptor(core)) {
 -        igb_write_lgcy_rx_descr(core, &desc->legacy, pkt, rss_info,
 -                                etqf, ts, length);
 +        igb_write_lgcy_rx_descr(core, &desc->legacy, pkt, rss_info, length);
      } else {
          igb_write_adv_rx_descr(core, &desc->adv, pkt, rss_info,
                                 etqf, ts, length);
 diff --git a/hw/net/igb_regs.h b/hw/net/igb_regs.h
 index XXXXXXX..XXXXXXX 100644
---- a/hw/net/igb_regs.h
+--- a/include/hw/virtio/vhost-vdpa.h
-+++ b/hw/net/igb_regs.h
++++ b/include/hw/virtio/vhost-vdpa.h
-@@ -XXX,XX +XXX,XX @@ union e1000_adv_rx_desc {
+@@ -XXX,XX +XXX,XX @@ typedef struct vhost_vdpa {
+     bool iotlb_batch_begin_sent;
- #define E1000_STATUS_NUM_VFS_SHIFT 14
+     MemoryListener listener;
+     struct vhost_vdpa_iova_range iova_range;
--#define E1000_ADVRXD_PKT_IP4 BIT(4)
++    uint64_t acked_features;
--#define E1000_ADVRXD_PKT_IP6 BIT(6)
+     bool shadow_vqs_enabled;
--#define E1000_ADVRXD_PKT_TCP BIT(8)
+     /* IOVA mapping used by the Shadow Virtqueue */
--#define E1000_ADVRXD_PKT_UDP BIT(9)
+     VhostIOVATree *iova_tree;
 -#define E1000_ADVRXD_PKT_SCTP BIT(10)
 +#define E1000_ADVRXD_PKT_IP4  BIT(0)
 +#define E1000_ADVRXD_PKT_IP6  BIT(2)
 +#define E1000_ADVRXD_PKT_TCP  BIT(4)
 +#define E1000_ADVRXD_PKT_UDP  BIT(5)
 +#define E1000_ADVRXD_PKT_SCTP BIT(6)
  static inline uint8_t igb_ivar_entry_rx(uint8_t i)
  {
 diff --git a/hw/net/trace-events b/hw/net/trace-events
 index XXXXXXX..XXXXXXX 100644
 --- a/hw/net/trace-events
 +++ b/hw/net/trace-events
@@ -XXX,XX +XXX,XX @@ igb_link_set_ext_params(bool asd_check, bool speed_select_bypass, bool pfrstd) "
  igb_rx_desc_buff_size(uint32_t b) "buffer size: %u"
  igb_rx_desc_buff_write(uint64_t addr, uint16_t offset, const void* source, uint32_t len) "addr: 0x%"PRIx64", offset: %u, from: %p, length: %u"
 -igb_rx_metadata_rss(uint32_t rss) "RSS data: 0x%X"
 +igb_rx_metadata_rss(uint32_t rss, uint16_t rss_pkt_type) "RSS data: rss: 0x%X, rss_pkt_type: 0x%X"
  igb_irq_icr_clear_gpie_nsicr(void) "Clearing ICR on read due to GPIE.NSICR enabled"
  igb_irq_set_iam(uint32_t icr) "Update IAM: 0x%x"
@@ -XXX,XX +XXX,XX @@ igb_irq_eitr_set(uint32_t eitr_num, uint32_t val) "EITR[%u] = 0x%x"
  igb_set_pfmailbox(uint32_t vf_num, uint32_t val) "PFMailbox[%d]: 0x%x"
  igb_set_vfmailbox(uint32_t vf_num, uint32_t val) "VFMailbox[%d]: 0x%x"
 +igb_wrn_rx_desc_modes_not_supp(int desc_type) "Not supported descriptor type: %d"
 +
  # igbvf.c
  igbvf_wrn_io_addr_unknown(uint64_t addr) "IO unknown register 0x%"PRIx64
 --
 .7.4

The following changes since commit 03a3a62fbd0aa5227e978eef3c67d3978aec9e5f:

Merge tag 'for-upstream' of https://gitlab.com/bonzini/qemu into staging (2023-09-07 10:29:06 -0400)

are available in the git repository at:

https://github.com/jasowang/qemu.git tags/net-pull-request

for you to fetch changes up to 049cfda145e96b2605cdf9739f1bcf9ebf3a83e1:

ebpf: Updated eBPF program and skeleton. (2023-09-08 14:33:46 +0800)

----------------------------------------------------------------

----------------------------------------------------------------
Andrew Melnychenko (7):
      tap: Add USO support to tap device.
      virtio-net: Add USO flags to vhost support.
      ebpf: Added eBPF map update through mmap.
      ebpf: Added eBPF initialization by fds.
      virtio-net: Added property to load eBPF RSS with fds.
      qmp: Added new command to retrieve eBPF blob.
      ebpf: Updated eBPF program and skeleton.

Ilya Maximets (1):
      net: add initial support for AF_XDP network backend

Tomasz Dzieciol (7):
      igb: remove TCP ACK detection
      igb: rename E1000E_RingInfo_st
      igb: RX descriptors guest writting refactoring
      igb: RX payload guest writting refactoring
      igb: add IPv6 extended headers traffic detection
      igb: packet-split descriptors support
      e1000e: rename e1000e_ba_state and e1000e_write_hdr_to_rx_buffers

Yuri Benditovich (2):
      tap: Add check for USO features
      virtio-net: Add support for USO features

From: Andrew Melnychenko <andrew@daynix.com>

Passing additional parameters (USOv4 and USOv6 offloads) when
setting TAP offloads

diff --git a/hw/net/e1000e_core.c b/hw/net/e1000e_core.c
index XXXXXXX..XXXXXXX 100644
--- a/hw/net/e1000e_core.c
+++ b/hw/net/e1000e_core.c
@@ -XXX,XX +XXX,XX @@ e1000e_update_rx_offloads(E1000ECore *core)
 
     if (core->has_vnet) {
         qemu_set_offload(qemu_get_queue(core->owner_nic)->peer,
-                         cso_state, 0, 0, 0, 0);
+                         cso_state, 0, 0, 0, 0, 0, 0);
     }
 }
 
diff --git a/hw/net/igb_core.c b/hw/net/igb_core.c
index XXXXXXX..XXXXXXX 100644
--- a/hw/net/igb_core.c
+++ b/hw/net/igb_core.c
@@ -XXX,XX +XXX,XX @@ igb_update_rx_offloads(IGBCore *core)
 
     if (core->has_vnet) {
         qemu_set_offload(qemu_get_queue(core->owner_nic)->peer,
-                         cso_state, 0, 0, 0, 0);
+                         cso_state, 0, 0, 0, 0, 0, 0);
     }
 }
 
diff --git a/hw/net/virtio-net.c b/hw/net/virtio-net.c
index XXXXXXX..XXXXXXX 100644
--- a/hw/net/virtio-net.c
+++ b/hw/net/virtio-net.c
@@ -XXX,XX +XXX,XX @@ static void virtio_net_apply_guest_offloads(VirtIONet *n)
             !!(n->curr_guest_offloads & (1ULL << VIRTIO_NET_F_GUEST_TSO4)),
             !!(n->curr_guest_offloads & (1ULL << VIRTIO_NET_F_GUEST_TSO6)),
             !!(n->curr_guest_offloads & (1ULL << VIRTIO_NET_F_GUEST_ECN)),
-            !!(n->curr_guest_offloads & (1ULL << VIRTIO_NET_F_GUEST_UFO)));
+            !!(n->curr_guest_offloads & (1ULL << VIRTIO_NET_F_GUEST_UFO)),
+            !!(n->curr_guest_offloads & (1ULL << VIRTIO_NET_F_GUEST_USO4)),
+            !!(n->curr_guest_offloads & (1ULL << VIRTIO_NET_F_GUEST_USO6)));
 }
 
 static uint64_t virtio_net_guest_offloads_by_features(uint32_t features)
diff --git a/hw/net/vmxnet3.c b/hw/net/vmxnet3.c
index XXXXXXX..XXXXXXX 100644
--- a/hw/net/vmxnet3.c
+++ b/hw/net/vmxnet3.c
@@ -XXX,XX +XXX,XX @@ static void vmxnet3_update_features(VMXNET3State *s)
                          s->lro_supported,
                          s->lro_supported,
                          0,
+                         0,
+                         0,
                          0);
     }
 }
diff --git a/include/net/net.h b/include/net/net.h
index XXXXXXX..XXXXXXX 100644
--- a/include/net/net.h
+++ b/include/net/net.h
@@ -XXX,XX +XXX,XX @@ typedef bool (HasVnetHdr)(NetClientState *);
 typedef bool (HasVnetHdrLen)(NetClientState *, int);
 typedef bool (GetUsingVnetHdr)(NetClientState *);
 typedef void (UsingVnetHdr)(NetClientState *, bool);
-typedef void (SetOffload)(NetClientState *, int, int, int, int, int);
+typedef void (SetOffload)(NetClientState *, int, int, int, int, int, int, int);
 typedef int (GetVnetHdrLen)(NetClientState *);
 typedef void (SetVnetHdrLen)(NetClientState *, int);
 typedef int (SetVnetLE)(NetClientState *, bool);
@@ -XXX,XX +XXX,XX @@ bool qemu_has_vnet_hdr_len(NetClientState *nc, int len);
 bool qemu_get_using_vnet_hdr(NetClientState *nc);
 void qemu_using_vnet_hdr(NetClientState *nc, bool enable);
 void qemu_set_offload(NetClientState *nc, int csum, int tso4, int tso6,
-                      int ecn, int ufo);
+                      int ecn, int ufo, int uso4, int uso6);
 int qemu_get_vnet_hdr_len(NetClientState *nc);
 void qemu_set_vnet_hdr_len(NetClientState *nc, int len);
 int qemu_set_vnet_le(NetClientState *nc, bool is_le);
diff --git a/net/net.c b/net/net.c
index XXXXXXX..XXXXXXX 100644
--- a/net/net.c
+++ b/net/net.c
@@ -XXX,XX +XXX,XX @@ void qemu_using_vnet_hdr(NetClientState *nc, bool enable)
 }
 
 void qemu_set_offload(NetClientState *nc, int csum, int tso4, int tso6,
-                          int ecn, int ufo)
+                          int ecn, int ufo, int uso4, int uso6)
 {
     if (!nc || !nc->info->set_offload) {
         return;
     }
 
-    nc->info->set_offload(nc, csum, tso4, tso6, ecn, ufo);
+    nc->info->set_offload(nc, csum, tso4, tso6, ecn, ufo, uso4, uso6);
 }
 
 int qemu_get_vnet_hdr_len(NetClientState *nc)
diff --git a/net/tap-bsd.c b/net/tap-bsd.c
index XXXXXXX..XXXXXXX 100644
--- a/net/tap-bsd.c
+++ b/net/tap-bsd.c
@@ -XXX,XX +XXX,XX @@ int tap_fd_set_vnet_be(int fd, int is_be)
 }
 
 void tap_fd_set_offload(int fd, int csum, int tso4,
-                        int tso6, int ecn, int ufo)
+                        int tso6, int ecn, int ufo, int uso4, int uso6)
 {
 }
 
diff --git a/net/tap-linux.c b/net/tap-linux.c
index XXXXXXX..XXXXXXX 100644
--- a/net/tap-linux.c
+++ b/net/tap-linux.c
@@ -XXX,XX +XXX,XX @@ int tap_fd_set_vnet_be(int fd, int is_be)
 }
 
 void tap_fd_set_offload(int fd, int csum, int tso4,
-                        int tso6, int ecn, int ufo)
+                        int tso6, int ecn, int ufo, int uso4, int uso6)
 {
     unsigned int offload = 0;
 
@@ -XXX,XX +XXX,XX @@ void tap_fd_set_offload(int fd, int csum, int tso4,
             offload |= TUN_F_TSO_ECN;
         if (ufo)
             offload |= TUN_F_UFO;
+        if (uso4) {
+            offload |= TUN_F_USO4;
+        }
+        if (uso6) {
+            offload |= TUN_F_USO6;
+        }
     }
 
     if (ioctl(fd, TUNSETOFFLOAD, offload) != 0) {
-        offload &= ~TUN_F_UFO;
+        offload &= ~(TUN_F_USO4 | TUN_F_USO6);
         if (ioctl(fd, TUNSETOFFLOAD, offload) != 0) {
-            fprintf(stderr, "TUNSETOFFLOAD ioctl() failed: %s\n",
+            offload &= ~TUN_F_UFO;
+            if (ioctl(fd, TUNSETOFFLOAD, offload) != 0) {
+                fprintf(stderr, "TUNSETOFFLOAD ioctl() failed: %s\n",
                     strerror(errno));
+            }
         }
     }
 }
diff --git a/net/tap-linux.h b/net/tap-linux.h
index XXXXXXX..XXXXXXX 100644
--- a/net/tap-linux.h
+++ b/net/tap-linux.h
@@ -XXX,XX +XXX,XX @@
 #define TUN_F_TSO6    0x04    /* I can handle TSO for IPv6 packets */
 #define TUN_F_TSO_ECN 0x08    /* I can handle TSO with ECN bits. */
 #define TUN_F_UFO     0x10    /* I can handle UFO packets */
+#define TUN_F_USO4    0x20    /* I can handle USO for IPv4 packets */
+#define TUN_F_USO6    0x40    /* I can handle USO for IPv6 packets */
 
 #endif /* QEMU_TAP_LINUX_H */
diff --git a/net/tap-solaris.c b/net/tap-solaris.c
index XXXXXXX..XXXXXXX 100644
--- a/net/tap-solaris.c
+++ b/net/tap-solaris.c
@@ -XXX,XX +XXX,XX @@ int tap_fd_set_vnet_be(int fd, int is_be)
 }
 
 void tap_fd_set_offload(int fd, int csum, int tso4,
-                        int tso6, int ecn, int ufo)
+                        int tso6, int ecn, int ufo, int uso4, int uso6)
 {
 }
 
diff --git a/net/tap-stub.c b/net/tap-stub.c
index XXXXXXX..XXXXXXX 100644
--- a/net/tap-stub.c
+++ b/net/tap-stub.c
@@ -XXX,XX +XXX,XX @@ int tap_fd_set_vnet_be(int fd, int is_be)
 }
 
 void tap_fd_set_offload(int fd, int csum, int tso4,
-                        int tso6, int ecn, int ufo)
+                        int tso6, int ecn, int ufo, int uso4, int uso6)
 {
 }
 
diff --git a/net/tap-win32.c b/net/tap-win32.c
index XXXXXXX..XXXXXXX 100644
--- a/net/tap-win32.c
+++ b/net/tap-win32.c
@@ -XXX,XX +XXX,XX @@ static void tap_using_vnet_hdr(NetClientState *nc, bool using_vnet_hdr)
 }
 
 static void tap_set_offload(NetClientState *nc, int csum, int tso4,
-                     int tso6, int ecn, int ufo)
+                     int tso6, int ecn, int ufo, int uso4, int uso6)
 {
 }
 
diff --git a/net/tap.c b/net/tap.c
index XXXXXXX..XXXXXXX 100644
--- a/net/tap.c
+++ b/net/tap.c
@@ -XXX,XX +XXX,XX @@ static int tap_set_vnet_be(NetClientState *nc, bool is_be)
 }
 
 static void tap_set_offload(NetClientState *nc, int csum, int tso4,
-                     int tso6, int ecn, int ufo)
+                     int tso6, int ecn, int ufo, int uso4, int uso6)
 {
     TAPState *s = DO_UPCAST(TAPState, nc, nc);
     if (s->fd < 0) {
         return;
     }
 
-    tap_fd_set_offload(s->fd, csum, tso4, tso6, ecn, ufo);
+    tap_fd_set_offload(s->fd, csum, tso4, tso6, ecn, ufo, uso4, uso6);
 }
 
 static void tap_exit_notify(Notifier *notifier, void *data)
@@ -XXX,XX +XXX,XX @@ static TAPState *net_tap_fd_init(NetClientState *peer,
     s->using_vnet_hdr = false;
     s->has_ufo = tap_probe_has_ufo(s->fd);
     s->enabled = true;
-    tap_set_offload(&s->nc, 0, 0, 0, 0, 0);
+    tap_set_offload(&s->nc, 0, 0, 0, 0, 0, 0, 0);
     /*
      * Make sure host header length is set correctly in tap:
      * it might have been modified by another instance of qemu.
diff --git a/net/tap_int.h b/net/tap_int.h
index XXXXXXX..XXXXXXX 100644
--- a/net/tap_int.h
+++ b/net/tap_int.h
@@ -XXX,XX +XXX,XX @@ void tap_set_sndbuf(int fd, const NetdevTapOptions *tap, Error **errp);
 int tap_probe_vnet_hdr(int fd, Error **errp);
 int tap_probe_vnet_hdr_len(int fd, int len);
 int tap_probe_has_ufo(int fd);
-void tap_fd_set_offload(int fd, int csum, int tso4, int tso6, int ecn, int ufo);
+void tap_fd_set_offload(int fd, int csum, int tso4, int tso6, int ecn, int ufo,
+                        int uso4, int uso6);
 void tap_fd_set_vnet_hdr_len(int fd, int len);
 int tap_fd_set_vnet_le(int fd, int vnet_is_le);
 int tap_fd_set_vnet_be(int fd, int vnet_is_be);
-- 
2.7.4

From: Yuri Benditovich <yuri.benditovich@daynix.com>

Tap indicates support for USO features according to
capabilities of current kernel module.

Signed-off-by: Yuri Benditovich <yuri.benditovich@daynix.com>
Signed-off-by: Andrew Melnychecnko <andrew@daynix.com>
Signed-off-by: Jason Wang <jasowang@redhat.com>
---
 include/net/net.h |  3 +++
 net/net.c         |  9 +++++++++
 net/tap-bsd.c     |  5 +++++
 net/tap-linux.c   | 12 ++++++++++++
 net/tap-solaris.c |  5 +++++
 net/tap-stub.c    |  5 +++++
 net/tap.c         | 12 ++++++++++++
 net/tap_int.h     |  1 +
 8 files changed, 52 insertions(+)

diff --git a/include/net/net.h b/include/net/net.h
index XXXXXXX..XXXXXXX 100644
--- a/include/net/net.h
+++ b/include/net/net.h
@@ -XXX,XX +XXX,XX @@ typedef void (LinkStatusChanged)(NetClientState *);
 typedef void (NetClientDestructor)(NetClientState *);
 typedef RxFilterInfo *(QueryRxFilter)(NetClientState *);
 typedef bool (HasUfo)(NetClientState *);
+typedef bool (HasUso)(NetClientState *);
 typedef bool (HasVnetHdr)(NetClientState *);
 typedef bool (HasVnetHdrLen)(NetClientState *, int);
 typedef bool (GetUsingVnetHdr)(NetClientState *);
@@ -XXX,XX +XXX,XX @@ typedef struct NetClientInfo {
     QueryRxFilter *query_rx_filter;
     NetPoll *poll;
     HasUfo *has_ufo;
+    HasUso *has_uso;
     HasVnetHdr *has_vnet_hdr;
     HasVnetHdrLen *has_vnet_hdr_len;
     GetUsingVnetHdr *get_using_vnet_hdr;
@@ -XXX,XX +XXX,XX @@ void qemu_set_info_str(NetClientState *nc,
                        const char *fmt, ...) G_GNUC_PRINTF(2, 3);
 void qemu_format_nic_info_str(NetClientState *nc, uint8_t macaddr[6]);
 bool qemu_has_ufo(NetClientState *nc);
+bool qemu_has_uso(NetClientState *nc);
 bool qemu_has_vnet_hdr(NetClientState *nc);
 bool qemu_has_vnet_hdr_len(NetClientState *nc, int len);
 bool qemu_get_using_vnet_hdr(NetClientState *nc);
diff --git a/net/net.c b/net/net.c
index XXXXXXX..XXXXXXX 100644
--- a/net/net.c
+++ b/net/net.c
@@ -XXX,XX +XXX,XX @@ bool qemu_has_ufo(NetClientState *nc)
     return nc->info->has_ufo(nc);
 }
 
+bool qemu_has_uso(NetClientState *nc)
+{
+    if (!nc || !nc->info->has_uso) {
+        return false;
+    }
+
+    return nc->info->has_uso(nc);
+}
+
 bool qemu_has_vnet_hdr(NetClientState *nc)
 {
     if (!nc || !nc->info->has_vnet_hdr) {
diff --git a/net/tap-bsd.c b/net/tap-bsd.c
index XXXXXXX..XXXXXXX 100644
--- a/net/tap-bsd.c
+++ b/net/tap-bsd.c
@@ -XXX,XX +XXX,XX @@ int tap_probe_has_ufo(int fd)
     return 0;
 }
 
+int tap_probe_has_uso(int fd)
+{
+    return 0;
+}
+
 int tap_probe_vnet_hdr_len(int fd, int len)
 {
     return 0;
diff --git a/net/tap-linux.c b/net/tap-linux.c
index XXXXXXX..XXXXXXX 100644
--- a/net/tap-linux.c
+++ b/net/tap-linux.c
@@ -XXX,XX +XXX,XX @@ int tap_probe_has_ufo(int fd)
     return 1;
 }
 
+int tap_probe_has_uso(int fd)
+{
+    unsigned offload;
+
+    offload = TUN_F_CSUM | TUN_F_USO4 | TUN_F_USO6;
+
+    if (ioctl(fd, TUNSETOFFLOAD, offload) < 0) {
+        return 0;
+    }
+    return 1;
+}
+
 /* Verify that we can assign given length */
 int tap_probe_vnet_hdr_len(int fd, int len)
 {
diff --git a/net/tap-solaris.c b/net/tap-solaris.c
index XXXXXXX..XXXXXXX 100644
--- a/net/tap-solaris.c
+++ b/net/tap-solaris.c
@@ -XXX,XX +XXX,XX @@ int tap_probe_has_ufo(int fd)
     return 0;
 }
 
+int tap_probe_has_uso(int fd)
+{
+    return 0;
+}
+
 int tap_probe_vnet_hdr_len(int fd, int len)
 {
     return 0;
diff --git a/net/tap-stub.c b/net/tap-stub.c
index XXXXXXX..XXXXXXX 100644
--- a/net/tap-stub.c
+++ b/net/tap-stub.c
@@ -XXX,XX +XXX,XX @@ int tap_probe_has_ufo(int fd)
     return 0;
 }
 
+int tap_probe_has_uso(int fd)
+{
+    return 0;
+}
+
 int tap_probe_vnet_hdr_len(int fd, int len)
 {
     return 0;
diff --git a/net/tap.c b/net/tap.c
index XXXXXXX..XXXXXXX 100644
--- a/net/tap.c
+++ b/net/tap.c
@@ -XXX,XX +XXX,XX @@ typedef struct TAPState {
     bool write_poll;
     bool using_vnet_hdr;
     bool has_ufo;
+    bool has_uso;
     bool enabled;
     VHostNetState *vhost_net;
     unsigned host_vnet_hdr_len;
@@ -XXX,XX +XXX,XX @@ static bool tap_has_ufo(NetClientState *nc)
     return s->has_ufo;
 }
 
+static bool tap_has_uso(NetClientState *nc)
+{
+    TAPState *s = DO_UPCAST(TAPState, nc, nc);
+
+    assert(nc->info->type == NET_CLIENT_DRIVER_TAP);
+
+    return s->has_uso;
+}
+
 static bool tap_has_vnet_hdr(NetClientState *nc)
 {
     TAPState *s = DO_UPCAST(TAPState, nc, nc);
@@ -XXX,XX +XXX,XX @@ static NetClientInfo net_tap_info = {
     .poll = tap_poll,
     .cleanup = tap_cleanup,
     .has_ufo = tap_has_ufo,
+    .has_uso = tap_has_uso,
     .has_vnet_hdr = tap_has_vnet_hdr,
     .has_vnet_hdr_len = tap_has_vnet_hdr_len,
     .get_using_vnet_hdr = tap_get_using_vnet_hdr,
@@ -XXX,XX +XXX,XX @@ static TAPState *net_tap_fd_init(NetClientState *peer,
     s->host_vnet_hdr_len = vnet_hdr ? sizeof(struct virtio_net_hdr) : 0;
     s->using_vnet_hdr = false;
     s->has_ufo = tap_probe_has_ufo(s->fd);
+    s->has_uso = tap_probe_has_uso(s->fd);
     s->enabled = true;
     tap_set_offload(&s->nc, 0, 0, 0, 0, 0, 0, 0);
     /*
diff --git a/net/tap_int.h b/net/tap_int.h
index XXXXXXX..XXXXXXX 100644
--- a/net/tap_int.h
+++ b/net/tap_int.h
@@ -XXX,XX +XXX,XX @@ void tap_set_sndbuf(int fd, const NetdevTapOptions *tap, Error **errp);
 int tap_probe_vnet_hdr(int fd, Error **errp);
 int tap_probe_vnet_hdr_len(int fd, int len);
 int tap_probe_has_ufo(int fd);
+int tap_probe_has_uso(int fd);
 void tap_fd_set_offload(int fd, int csum, int tso4, int tso6, int ecn, int ufo,
                         int uso4, int uso6);
 void tap_fd_set_vnet_hdr_len(int fd, int len);
-- 
2.7.4

From: Andrew Melnychenko <andrew@daynix.com>

New features are subject to check with vhost-user and vdpa.

diff --git a/hw/net/vhost_net.c b/hw/net/vhost_net.c
index XXXXXXX..XXXXXXX 100644
--- a/hw/net/vhost_net.c
+++ b/hw/net/vhost_net.c
@@ -XXX,XX +XXX,XX @@ static const int user_feature_bits[] = {
     VIRTIO_F_RING_RESET,
     VIRTIO_NET_F_RSS,
     VIRTIO_NET_F_HASH_REPORT,
+    VIRTIO_NET_F_GUEST_USO4,
+    VIRTIO_NET_F_GUEST_USO6,
+    VIRTIO_NET_F_HOST_USO,
 
     /* This bit implies RARP isn't sent by QEMU out of band */
     VIRTIO_NET_F_GUEST_ANNOUNCE,
diff --git a/net/vhost-vdpa.c b/net/vhost-vdpa.c
index XXXXXXX..XXXXXXX 100644
--- a/net/vhost-vdpa.c
+++ b/net/vhost-vdpa.c
@@ -XXX,XX +XXX,XX @@ const int vdpa_feature_bits[] = {
     VIRTIO_NET_F_GUEST_TSO4,
     VIRTIO_NET_F_GUEST_TSO6,
     VIRTIO_NET_F_GUEST_UFO,
+    VIRTIO_NET_F_GUEST_USO4,
+    VIRTIO_NET_F_GUEST_USO6,
     VIRTIO_NET_F_HASH_REPORT,
     VIRTIO_NET_F_HOST_ECN,
     VIRTIO_NET_F_HOST_TSO4,
     VIRTIO_NET_F_HOST_TSO6,
     VIRTIO_NET_F_HOST_UFO,
+    VIRTIO_NET_F_HOST_USO,
     VIRTIO_NET_F_MQ,
     VIRTIO_NET_F_MRG_RXBUF,
     VIRTIO_NET_F_MTU,
-- 
2.7.4

From: Yuri Benditovich <yuri.benditovich@daynix.com>

USO features of virtio-net device depend on kernel ability
to support them, for backward compatibility by default the
features are disabled on 8.0 and earlier.

Signed-off-by: Yuri Benditovich <yuri.benditovich@daynix.com>
Signed-off-by: Andrew Melnychecnko <andrew@daynix.com>
Signed-off-by: Jason Wang <jasowang@redhat.com>
---
 hw/core/machine.c   |  4 ++++
 hw/net/virtio-net.c | 31 +++++++++++++++++++++++++++++--
 2 files changed, 33 insertions(+), 2 deletions(-)

diff --git a/hw/core/machine.c b/hw/core/machine.c
index XXXXXXX..XXXXXXX 100644
--- a/hw/core/machine.c
+++ b/hw/core/machine.c
@@ -XXX,XX +XXX,XX @@
 #include "exec/confidential-guest-support.h"
 #include "hw/virtio/virtio.h"
 #include "hw/virtio/virtio-pci.h"
+#include "hw/virtio/virtio-net.h"
 
 GlobalProperty hw_compat_8_1[] = {};
 const size_t hw_compat_8_1_len = G_N_ELEMENTS(hw_compat_8_1);
@@ -XXX,XX +XXX,XX @@ const size_t hw_compat_8_1_len = G_N_ELEMENTS(hw_compat_8_1);
 GlobalProperty hw_compat_8_0[] = {
     { "migration", "multifd-flush-after-each-section", "on"},
     { TYPE_PCI_DEVICE, "x-pcie-ari-nextfn-1", "on" },
+    { TYPE_VIRTIO_NET, "host_uso", "off"},
+    { TYPE_VIRTIO_NET, "guest_uso4", "off"},
+    { TYPE_VIRTIO_NET, "guest_uso6", "off"},
 };
 const size_t hw_compat_8_0_len = G_N_ELEMENTS(hw_compat_8_0);
 
diff --git a/hw/net/virtio-net.c b/hw/net/virtio-net.c
index XXXXXXX..XXXXXXX 100644
--- a/hw/net/virtio-net.c
+++ b/hw/net/virtio-net.c
@@ -XXX,XX +XXX,XX @@ static int peer_has_ufo(VirtIONet *n)
     return n->has_ufo;
 }
 
+static int peer_has_uso(VirtIONet *n)
+{
+    if (!peer_has_vnet_hdr(n)) {
+        return 0;
+    }
+
+    return qemu_has_uso(qemu_get_queue(n->nic)->peer);
+}
+
 static void virtio_net_set_mrg_rx_bufs(VirtIONet *n, int mergeable_rx_bufs,
                                        int version_1, int hash_report)
 {
@@ -XXX,XX +XXX,XX @@ static uint64_t virtio_net_get_features(VirtIODevice *vdev, uint64_t features,
         virtio_clear_feature(&features, VIRTIO_NET_F_GUEST_TSO6);
         virtio_clear_feature(&features, VIRTIO_NET_F_GUEST_ECN);
 
+        virtio_clear_feature(&features, VIRTIO_NET_F_HOST_USO);
+        virtio_clear_feature(&features, VIRTIO_NET_F_GUEST_USO4);
+        virtio_clear_feature(&features, VIRTIO_NET_F_GUEST_USO6);
+
         virtio_clear_feature(&features, VIRTIO_NET_F_HASH_REPORT);
     }
 
@@ -XXX,XX +XXX,XX @@ static uint64_t virtio_net_get_features(VirtIODevice *vdev, uint64_t features,
         virtio_clear_feature(&features, VIRTIO_NET_F_HOST_UFO);
     }
 
+    if (!peer_has_uso(n)) {
+        virtio_clear_feature(&features, VIRTIO_NET_F_HOST_USO);
+        virtio_clear_feature(&features, VIRTIO_NET_F_GUEST_USO4);
+        virtio_clear_feature(&features, VIRTIO_NET_F_GUEST_USO6);
+    }
+
     if (!get_vhost_net(nc->peer)) {
         return features;
     }
@@ -XXX,XX +XXX,XX @@ static void virtio_net_apply_guest_offloads(VirtIONet *n)
             !!(n->curr_guest_offloads & (1ULL << VIRTIO_NET_F_GUEST_USO6)));
 }
 
-static uint64_t virtio_net_guest_offloads_by_features(uint32_t features)
+static uint64_t virtio_net_guest_offloads_by_features(uint64_t features)
 {
     static const uint64_t guest_offloads_mask =
         (1ULL << VIRTIO_NET_F_GUEST_CSUM) |
         (1ULL << VIRTIO_NET_F_GUEST_TSO4) |
         (1ULL << VIRTIO_NET_F_GUEST_TSO6) |
         (1ULL << VIRTIO_NET_F_GUEST_ECN)  |
-        (1ULL << VIRTIO_NET_F_GUEST_UFO);
+        (1ULL << VIRTIO_NET_F_GUEST_UFO)  |
+        (1ULL << VIRTIO_NET_F_GUEST_USO4) |
+        (1ULL << VIRTIO_NET_F_GUEST_USO6);
 
     return guest_offloads_mask & features;
 }
@@ -XXX,XX +XXX,XX @@ static Property virtio_net_properties[] = {
     DEFINE_PROP_INT32("speed", VirtIONet, net_conf.speed, SPEED_UNKNOWN),
     DEFINE_PROP_STRING("duplex", VirtIONet, net_conf.duplex_str),
     DEFINE_PROP_BOOL("failover", VirtIONet, failover, false),
+    DEFINE_PROP_BIT64("guest_uso4", VirtIONet, host_features,
+                      VIRTIO_NET_F_GUEST_USO4, true),
+    DEFINE_PROP_BIT64("guest_uso6", VirtIONet, host_features,
+                      VIRTIO_NET_F_GUEST_USO6, true),
+    DEFINE_PROP_BIT64("host_uso", VirtIONet, host_features,
+                      VIRTIO_NET_F_HOST_USO, true),
     DEFINE_PROP_END_OF_LIST(),
 };
 
-- 
2.7.4

From: Tomasz Dzieciol <t.dzieciol@partner.samsung.com>

Rename E1000E_RingInfo_st and E1000E_RingInfo according to qemu typdefs guide.

Signed-off-by: Tomasz Dzieciol <t.dzieciol@partner.samsung.com>
Reviewed-by: Akihiko Odaki <akihiko.odaki@daynix.com>
Tested-by: Akihiko Odaki <akihiko.odaki@daynix.com>
Signed-off-by: Jason Wang <jasowang@redhat.com>
---
 hw/net/e1000e_core.c | 34 +++++++++++++++++-----------------
 hw/net/igb_core.c    | 42 +++++++++++++++++++++---------------------
 2 files changed, 38 insertions(+), 38 deletions(-)

diff --git a/hw/net/e1000e_core.c b/hw/net/e1000e_core.c
index XXXXXXX..XXXXXXX 100644
--- a/hw/net/e1000e_core.c
+++ b/hw/net/e1000e_core.c
@@ -XXX,XX +XXX,XX @@ e1000e_txdesc_writeback(E1000ECore *core, dma_addr_t base,
     return e1000e_tx_wb_interrupt_cause(core, queue_idx);
 }
 
-typedef struct E1000E_RingInfo_st {
+typedef struct E1000ERingInfo {
     int dbah;
     int dbal;
     int dlen;
     int dh;
     int dt;
     int idx;
-} E1000E_RingInfo;
+} E1000ERingInfo;
 
 static inline bool
-e1000e_ring_empty(E1000ECore *core, const E1000E_RingInfo *r)
+e1000e_ring_empty(E1000ECore *core, const E1000ERingInfo *r)
 {
     return core->mac[r->dh] == core->mac[r->dt] ||
                 core->mac[r->dt] >= core->mac[r->dlen] / E1000_RING_DESC_LEN;
 }
 
 static inline uint64_t
-e1000e_ring_base(E1000ECore *core, const E1000E_RingInfo *r)
+e1000e_ring_base(E1000ECore *core, const E1000ERingInfo *r)
 {
     uint64_t bah = core->mac[r->dbah];
     uint64_t bal = core->mac[r->dbal];
@@ -XXX,XX +XXX,XX @@ e1000e_ring_base(E1000ECore *core, const E1000E_RingInfo *r)
 }
 
 static inline uint64_t
-e1000e_ring_head_descr(E1000ECore *core, const E1000E_RingInfo *r)
+e1000e_ring_head_descr(E1000ECore *core, const E1000ERingInfo *r)
 {
     return e1000e_ring_base(core, r) + E1000_RING_DESC_LEN * core->mac[r->dh];
 }
 
 static inline void
-e1000e_ring_advance(E1000ECore *core, const E1000E_RingInfo *r, uint32_t count)
+e1000e_ring_advance(E1000ECore *core, const E1000ERingInfo *r, uint32_t count)
 {
     core->mac[r->dh] += count;
 
@@ -XXX,XX +XXX,XX @@ e1000e_ring_advance(E1000ECore *core, const E1000E_RingInfo *r, uint32_t count)
 }
 
 static inline uint32_t
-e1000e_ring_free_descr_num(E1000ECore *core, const E1000E_RingInfo *r)
+e1000e_ring_free_descr_num(E1000ECore *core, const E1000ERingInfo *r)
 {
     trace_e1000e_ring_free_space(r->idx, core->mac[r->dlen],
                                  core->mac[r->dh],  core->mac[r->dt]);
@@ -XXX,XX +XXX,XX @@ e1000e_ring_free_descr_num(E1000ECore *core, const E1000E_RingInfo *r)
 }
 
 static inline bool
-e1000e_ring_enabled(E1000ECore *core, const E1000E_RingInfo *r)
+e1000e_ring_enabled(E1000ECore *core, const E1000ERingInfo *r)
 {
     return core->mac[r->dlen] > 0;
 }
 
 static inline uint32_t
-e1000e_ring_len(E1000ECore *core, const E1000E_RingInfo *r)
+e1000e_ring_len(E1000ECore *core, const E1000ERingInfo *r)
 {
     return core->mac[r->dlen];
 }
 
 typedef struct E1000E_TxRing_st {
-    const E1000E_RingInfo *i;
+    const E1000ERingInfo *i;
     struct e1000e_tx *tx;
 } E1000E_TxRing;
 
@@ -XXX,XX +XXX,XX @@ e1000e_mq_queue_idx(int base_reg_idx, int reg_idx)
 static inline void
 e1000e_tx_ring_init(E1000ECore *core, E1000E_TxRing *txr, int idx)
 {
-    static const E1000E_RingInfo i[E1000E_NUM_QUEUES] = {
+    static const E1000ERingInfo i[E1000E_NUM_QUEUES] = {
         { TDBAH,  TDBAL,  TDLEN,  TDH,  TDT, 0 },
         { TDBAH1, TDBAL1, TDLEN1, TDH1, TDT1, 1 }
     };
@@ -XXX,XX +XXX,XX @@ e1000e_tx_ring_init(E1000ECore *core, E1000E_TxRing *txr, int idx)
 }
 
 typedef struct E1000E_RxRing_st {
-    const E1000E_RingInfo *i;
+    const E1000ERingInfo *i;
 } E1000E_RxRing;
 
 static inline void
 e1000e_rx_ring_init(E1000ECore *core, E1000E_RxRing *rxr, int idx)
 {
-    static const E1000E_RingInfo i[E1000E_NUM_QUEUES] = {
+    static const E1000ERingInfo i[E1000E_NUM_QUEUES] = {
         { RDBAH0, RDBAL0, RDLEN0, RDH0, RDT0, 0 },
         { RDBAH1, RDBAL1, RDLEN1, RDH1, RDT1, 1 }
     };
@@ -XXX,XX +XXX,XX @@ e1000e_start_xmit(E1000ECore *core, const E1000E_TxRing *txr)
     dma_addr_t base;
     struct e1000_tx_desc desc;
     bool ide = false;
-    const E1000E_RingInfo *txi = txr->i;
+    const E1000ERingInfo *txi = txr->i;
     uint32_t cause = E1000_ICS_TXQE;
 
     if (!(core->mac[TCTL] & E1000_TCTL_EN)) {
@@ -XXX,XX +XXX,XX @@ e1000e_start_xmit(E1000ECore *core, const E1000E_TxRing *txr)
 }
 
 static bool
-e1000e_has_rxbufs(E1000ECore *core, const E1000E_RingInfo *r,
+e1000e_has_rxbufs(E1000ECore *core, const E1000ERingInfo *r,
                   size_t total_size)
 {
     uint32_t bufs = e1000e_ring_free_descr_num(core, r);
@@ -XXX,XX +XXX,XX @@ e1000e_update_rx_stats(E1000ECore *core, size_t pkt_size, size_t pkt_fcs_size)
 }
 
 static inline bool
-e1000e_rx_descr_threshold_hit(E1000ECore *core, const E1000E_RingInfo *rxi)
+e1000e_rx_descr_threshold_hit(E1000ECore *core, const E1000ERingInfo *rxi)
 {
     return e1000e_ring_free_descr_num(core, rxi) ==
            e1000e_ring_len(core, rxi) >> core->rxbuf_min_shift;
@@ -XXX,XX +XXX,XX @@ e1000e_write_packet_to_guest(E1000ECore *core, struct NetRxPkt *pkt,
     struct iovec *iov = net_rx_pkt_get_iovec(pkt);
     size_t size = net_rx_pkt_get_total_len(pkt);
     size_t total_size = size + e1000x_fcs_len(core->mac);
-    const E1000E_RingInfo *rxi;
+    const E1000ERingInfo *rxi;
     size_t ps_hdr_len = 0;
     bool do_ps = e1000e_do_ps(core, pkt, &ps_hdr_len);
     bool is_first = true;
diff --git a/hw/net/igb_core.c b/hw/net/igb_core.c
index XXXXXXX..XXXXXXX 100644
--- a/hw/net/igb_core.c
+++ b/hw/net/igb_core.c
@@ -XXX,XX +XXX,XX @@ static uint32_t igb_rx_wb_eic(IGBCore *core, int queue_idx)
     return (ent & E1000_IVAR_VALID) ? BIT(ent & 0x1f) : 0;
 }
 
-typedef struct E1000E_RingInfo_st {
+typedef struct E1000ERingInfo {
     int dbah;
     int dbal;
     int dlen;
     int dh;
     int dt;
     int idx;
-} E1000E_RingInfo;
+} E1000ERingInfo;
 
 static inline bool
-igb_ring_empty(IGBCore *core, const E1000E_RingInfo *r)
+igb_ring_empty(IGBCore *core, const E1000ERingInfo *r)
 {
     return core->mac[r->dh] == core->mac[r->dt] ||
                 core->mac[r->dt] >= core->mac[r->dlen] / E1000_RING_DESC_LEN;
 }
 
 static inline uint64_t
-igb_ring_base(IGBCore *core, const E1000E_RingInfo *r)
+igb_ring_base(IGBCore *core, const E1000ERingInfo *r)
 {
     uint64_t bah = core->mac[r->dbah];
     uint64_t bal = core->mac[r->dbal];
@@ -XXX,XX +XXX,XX @@ igb_ring_base(IGBCore *core, const E1000E_RingInfo *r)
 }
 
 static inline uint64_t
-igb_ring_head_descr(IGBCore *core, const E1000E_RingInfo *r)
+igb_ring_head_descr(IGBCore *core, const E1000ERingInfo *r)
 {
     return igb_ring_base(core, r) + E1000_RING_DESC_LEN * core->mac[r->dh];
 }
 
 static inline void
-igb_ring_advance(IGBCore *core, const E1000E_RingInfo *r, uint32_t count)
+igb_ring_advance(IGBCore *core, const E1000ERingInfo *r, uint32_t count)
 {
     core->mac[r->dh] += count;
 
@@ -XXX,XX +XXX,XX @@ igb_ring_advance(IGBCore *core, const E1000E_RingInfo *r, uint32_t count)
 }
 
 static inline uint32_t
-igb_ring_free_descr_num(IGBCore *core, const E1000E_RingInfo *r)
+igb_ring_free_descr_num(IGBCore *core, const E1000ERingInfo *r)
 {
     trace_e1000e_ring_free_space(r->idx, core->mac[r->dlen],
                                  core->mac[r->dh],  core->mac[r->dt]);
@@ -XXX,XX +XXX,XX @@ igb_ring_free_descr_num(IGBCore *core, const E1000E_RingInfo *r)
 }
 
 static inline bool
-igb_ring_enabled(IGBCore *core, const E1000E_RingInfo *r)
+igb_ring_enabled(IGBCore *core, const E1000ERingInfo *r)
 {
     return core->mac[r->dlen] > 0;
 }
 
 typedef struct IGB_TxRing_st {
-    const E1000E_RingInfo *i;
+    const E1000ERingInfo *i;
     struct igb_tx *tx;
 } IGB_TxRing;
 
@@ -XXX,XX +XXX,XX @@ igb_mq_queue_idx(int base_reg_idx, int reg_idx)
 static inline void
 igb_tx_ring_init(IGBCore *core, IGB_TxRing *txr, int idx)
 {
-    static const E1000E_RingInfo i[IGB_NUM_QUEUES] = {
+    static const E1000ERingInfo i[IGB_NUM_QUEUES] = {
         { TDBAH0, TDBAL0, TDLEN0, TDH0, TDT0, 0 },
         { TDBAH1, TDBAL1, TDLEN1, TDH1, TDT1, 1 },
         { TDBAH2, TDBAL2, TDLEN2, TDH2, TDT2, 2 },
@@ -XXX,XX +XXX,XX @@ igb_tx_ring_init(IGBCore *core, IGB_TxRing *txr, int idx)
 }
 
 typedef struct E1000E_RxRing_st {
-    const E1000E_RingInfo *i;
+    const E1000ERingInfo *i;
 } E1000E_RxRing;
 
 static inline void
 igb_rx_ring_init(IGBCore *core, E1000E_RxRing *rxr, int idx)
 {
-    static const E1000E_RingInfo i[IGB_NUM_QUEUES] = {
+    static const E1000ERingInfo i[IGB_NUM_QUEUES] = {
         { RDBAH0, RDBAL0, RDLEN0, RDH0, RDT0, 0 },
         { RDBAH1, RDBAL1, RDLEN1, RDH1, RDT1, 1 },
         { RDBAH2, RDBAL2, RDLEN2, RDH2, RDT2, 2 },
@@ -XXX,XX +XXX,XX @@ igb_rx_ring_init(IGBCore *core, E1000E_RxRing *rxr, int idx)
 static uint32_t
 igb_txdesc_writeback(IGBCore *core, dma_addr_t base,
                      union e1000_adv_tx_desc *tx_desc,
-                     const E1000E_RingInfo *txi)
+                     const E1000ERingInfo *txi)
 {
     PCIDevice *d;
     uint32_t cmd_type_len = le32_to_cpu(tx_desc->read.cmd_type_len);
@@ -XXX,XX +XXX,XX @@ igb_txdesc_writeback(IGBCore *core, dma_addr_t base,
 }
 
 static inline bool
-igb_tx_enabled(IGBCore *core, const E1000E_RingInfo *txi)
+igb_tx_enabled(IGBCore *core, const E1000ERingInfo *txi)
 {
     bool vmdq = core->mac[MRQC] & 1;
     uint16_t qn = txi->idx;
@@ -XXX,XX +XXX,XX @@ igb_start_xmit(IGBCore *core, const IGB_TxRing *txr)
     PCIDevice *d;
     dma_addr_t base;
     union e1000_adv_tx_desc desc;
-    const E1000E_RingInfo *txi = txr->i;
+    const E1000ERingInfo *txi = txr->i;
     uint32_t eic = 0;
 
     if (!igb_tx_enabled(core, txi)) {
@@ -XXX,XX +XXX,XX @@ igb_start_xmit(IGBCore *core, const IGB_TxRing *txr)
 }
 
 static uint32_t
-igb_rxbufsize(IGBCore *core, const E1000E_RingInfo *r)
+igb_rxbufsize(IGBCore *core, const E1000ERingInfo *r)
 {
     uint32_t srrctl = core->mac[E1000_SRRCTL(r->idx) >> 2];
     uint32_t bsizepkt = srrctl & E1000_SRRCTL_BSIZEPKT_MASK;
@@ -XXX,XX +XXX,XX @@ igb_rxbufsize(IGBCore *core, const E1000E_RingInfo *r)
 }
 
 static bool
-igb_has_rxbufs(IGBCore *core, const E1000E_RingInfo *r, size_t total_size)
+igb_has_rxbufs(IGBCore *core, const E1000ERingInfo *r, size_t total_size)
 {
     uint32_t bufs = igb_ring_free_descr_num(core, r);
     uint32_t bufsize = igb_rxbufsize(core, r);
@@ -XXX,XX +XXX,XX @@ igb_write_to_rx_buffers(IGBCore *core,
 }
 
 static void
-igb_update_rx_stats(IGBCore *core, const E1000E_RingInfo *rxi,
+igb_update_rx_stats(IGBCore *core, const E1000ERingInfo *rxi,
                     size_t pkt_size, size_t pkt_fcs_size)
 {
     eth_pkt_types_e pkt_type = net_rx_pkt_get_packet_type(core->rx_pkt);
@@ -XXX,XX +XXX,XX @@ igb_update_rx_stats(IGBCore *core, const E1000E_RingInfo *rxi,
 }
 
 static inline bool
-igb_rx_descr_threshold_hit(IGBCore *core, const E1000E_RingInfo *rxi)
+igb_rx_descr_threshold_hit(IGBCore *core, const E1000ERingInfo *rxi)
 {
     return igb_ring_free_descr_num(core, rxi) ==
            ((core->mac[E1000_SRRCTL(rxi->idx) >> 2] >> 20) & 31) * 16;
@@ -XXX,XX +XXX,XX @@ igb_write_packet_to_guest(IGBCore *core, struct NetRxPkt *pkt,
     struct iovec *iov = net_rx_pkt_get_iovec(pkt);
     size_t size = net_rx_pkt_get_total_len(pkt);
     size_t total_size = size + e1000x_fcs_len(core->mac);
-    const E1000E_RingInfo *rxi = rxr->i;
+    const E1000ERingInfo *rxi = rxr->i;
     size_t bufsize = igb_rxbufsize(core, rxi);
 
     d = pcie_sriov_get_vf_at_index(core->owner, rxi->idx % 8);
@@ -XXX,XX +XXX,XX @@ igb_write_packet_to_guest(IGBCore *core, struct NetRxPkt *pkt,
 }
 
 static bool
-igb_rx_strip_vlan(IGBCore *core, const E1000E_RingInfo *rxi)
+igb_rx_strip_vlan(IGBCore *core, const E1000ERingInfo *rxi)
 {
     if (core->mac[MRQC] & 1) {
         uint16_t pool = rxi->idx % IGB_NUM_VM_POOLS;
-- 
2.7.4

From: Tomasz Dzieciol <t.dzieciol@partner.samsung.com>

Refactoring is done in preparation for support of multiple advanced
descriptors RX modes, especially packet-split modes.

Signed-off-by: Tomasz Dzieciol <t.dzieciol@partner.samsung.com>
Reviewed-by: Akihiko Odaki <akihiko.odaki@daynix.com>
Tested-by: Akihiko Odaki <akihiko.odaki@daynix.com>
Signed-off-by: Jason Wang <jasowang@redhat.com>
---
 hw/net/igb_core.c   | 170 +++++++++++++++++++++++++++-------------------------
 hw/net/igb_regs.h   |  10 ++--
 hw/net/trace-events |   4 +-
 3 files changed, 96 insertions(+), 88 deletions(-)

diff --git a/hw/net/igb_core.c b/hw/net/igb_core.c
index XXXXXXX..XXXXXXX 100644
--- a/hw/net/igb_core.c
+++ b/hw/net/igb_core.c
@@ -XXX,XX +XXX,XX @@ igb_verify_csum_in_sw(IGBCore *core,
 }
 
 static void
-igb_build_rx_metadata(IGBCore *core,
-                      struct NetRxPkt *pkt,
-                      bool is_eop,
-                      const E1000E_RSSInfo *rss_info, uint16_t etqf, bool ts,
-                      uint16_t *pkt_info, uint16_t *hdr_info,
-                      uint32_t *rss,
-                      uint32_t *status_flags,
-                      uint16_t *ip_id,
-                      uint16_t *vlan_tag)
+igb_build_rx_metadata_common(IGBCore *core,
+                             struct NetRxPkt *pkt,
+                             bool is_eop,
+                             uint32_t *status_flags,
+                             uint16_t *vlan_tag)
 {
     struct virtio_net_hdr *vhdr;
     bool hasip4, hasip6, csum_valid;
@@ -XXX,XX +XXX,XX @@ igb_build_rx_metadata(IGBCore *core,
     *status_flags = E1000_RXD_STAT_DD;
 
     /* No additional metadata needed for non-EOP descriptors */
-    /* TODO: EOP apply only to status so don't skip whole function. */
     if (!is_eop) {
         goto func_exit;
     }
@@ -XXX,XX +XXX,XX @@ igb_build_rx_metadata(IGBCore *core,
         trace_e1000e_rx_metadata_vlan(*vlan_tag);
     }
 
-    /* Packet parsing results */
-    if ((core->mac[RXCSUM] & E1000_RXCSUM_PCSD) != 0) {
-        if (rss_info->enabled) {
-            *rss = cpu_to_le32(rss_info->hash);
-            trace_igb_rx_metadata_rss(*rss);
-        }
-    } else if (hasip4) {
-            *status_flags |= E1000_RXD_STAT_IPIDV;
-            *ip_id = cpu_to_le16(net_rx_pkt_get_ip_id(pkt));
-            trace_e1000e_rx_metadata_ip_id(*ip_id);
-    }
-
-    if (pkt_info) {
-        *pkt_info = rss_info->enabled ? rss_info->type : 0;
-
-        if (etqf < 8) {
-            *pkt_info |= (BIT(11) | etqf) << 4;
-        } else {
-            if (hasip4) {
-                *pkt_info |= E1000_ADVRXD_PKT_IP4;
-            }
-
-            if (hasip6) {
-                *pkt_info |= E1000_ADVRXD_PKT_IP6;
-            }
-
-            switch (l4hdr_proto) {
-            case ETH_L4_HDR_PROTO_TCP:
-                *pkt_info |= E1000_ADVRXD_PKT_TCP;
-                break;
-
-            case ETH_L4_HDR_PROTO_UDP:
-                *pkt_info |= E1000_ADVRXD_PKT_UDP;
-                break;
-
-            case ETH_L4_HDR_PROTO_SCTP:
-                *pkt_info |= E1000_ADVRXD_PKT_SCTP;
-                break;
-
-            default:
-                break;
-            }
-        }
-    }
-
-    if (hdr_info) {
-        *hdr_info = 0;
-    }
-
-    if (ts) {
-        *status_flags |= BIT(16);
-    }
-
     /* RX CSO information */
     if (hasip6 && (core->mac[RFCTL] & E1000_RFCTL_IPV6_XSUM_DIS)) {
         trace_e1000e_rx_metadata_ipv6_sum_disabled();
@@ -XXX,XX +XXX,XX @@ func_exit:
 static inline void
 igb_write_lgcy_rx_descr(IGBCore *core, struct e1000_rx_desc *desc,
                         struct NetRxPkt *pkt,
-                        const E1000E_RSSInfo *rss_info, uint16_t etqf, bool ts,
+                        const E1000E_RSSInfo *rss_info,
                         uint16_t length)
 {
-    uint32_t status_flags, rss;
-    uint16_t ip_id;
+    uint32_t status_flags;
 
     assert(!rss_info->enabled);
+
+    memset(desc, 0, sizeof(*desc));
     desc->length = cpu_to_le16(length);
-    desc->csum = 0;
+    igb_build_rx_metadata_common(core, pkt, pkt != NULL,
+                                 &status_flags,
+                                 &desc->special);
 
-    igb_build_rx_metadata(core, pkt, pkt != NULL,
-                          rss_info, etqf, ts,
-                          NULL, NULL, &rss,
-                          &status_flags, &ip_id,
-                          &desc->special);
     desc->errors = (uint8_t) (le32_to_cpu(status_flags) >> 24);
     desc->status = (uint8_t) le32_to_cpu(status_flags);
 }
 
+static uint16_t
+igb_rx_desc_get_packet_type(IGBCore *core, struct NetRxPkt *pkt, uint16_t etqf)
+{
+    uint16_t pkt_type;
+    bool hasip4, hasip6;
+    EthL4HdrProto l4hdr_proto;
+
+    if (etqf < 8) {
+        pkt_type = BIT(11) | etqf;
+        return pkt_type;
+    }
+
+    net_rx_pkt_get_protocols(pkt, &hasip4, &hasip6, &l4hdr_proto);
+
+    if (hasip6 && !(core->mac[RFCTL] & E1000_RFCTL_IPV6_DIS)) {
+        pkt_type = E1000_ADVRXD_PKT_IP6;
+    } else if (hasip4) {
+        pkt_type = E1000_ADVRXD_PKT_IP4;
+    } else {
+        pkt_type = 0;
+    }
+
+    switch (l4hdr_proto) {
+    case ETH_L4_HDR_PROTO_TCP:
+        pkt_type |= E1000_ADVRXD_PKT_TCP;
+        break;
+    case ETH_L4_HDR_PROTO_UDP:
+        pkt_type |= E1000_ADVRXD_PKT_UDP;
+        break;
+    case ETH_L4_HDR_PROTO_SCTP:
+        pkt_type |= E1000_ADVRXD_PKT_SCTP;
+        break;
+    default:
+        break;
+    }
+
+    return pkt_type;
+}
+
 static inline void
 igb_write_adv_rx_descr(IGBCore *core, union e1000_adv_rx_desc *desc,
                        struct NetRxPkt *pkt,
                        const E1000E_RSSInfo *rss_info, uint16_t etqf, bool ts,
                        uint16_t length)
 {
+    bool hasip4, hasip6;
+    EthL4HdrProto l4hdr_proto;
+    uint16_t rss_type = 0, pkt_type;
+    bool eop = (pkt != NULL);
+    uint32_t adv_desc_status_error = 0;
     memset(&desc->wb, 0, sizeof(desc->wb));
 
     desc->wb.upper.length = cpu_to_le16(length);
+    igb_build_rx_metadata_common(core, pkt, eop,
+                                 &desc->wb.upper.status_error,
+                                 &desc->wb.upper.vlan);
+
+    if (!eop) {
+        return;
+    }
+
+    net_rx_pkt_get_protocols(pkt, &hasip4, &hasip6, &l4hdr_proto);
+
+    if ((core->mac[RXCSUM] & E1000_RXCSUM_PCSD) != 0) {
+        if (rss_info->enabled) {
+            desc->wb.lower.hi_dword.rss = cpu_to_le32(rss_info->hash);
+            rss_type = rss_info->type;
+            trace_igb_rx_metadata_rss(desc->wb.lower.hi_dword.rss, rss_type);
+        }
+    } else if (hasip4) {
+            adv_desc_status_error |= E1000_RXD_STAT_IPIDV;
+            desc->wb.lower.hi_dword.csum_ip.ip_id =
+                cpu_to_le16(net_rx_pkt_get_ip_id(pkt));
+            trace_e1000e_rx_metadata_ip_id(
+                desc->wb.lower.hi_dword.csum_ip.ip_id);
+    }
+
+    if (ts) {
+        adv_desc_status_error |= BIT(16);
+    }
 
-    igb_build_rx_metadata(core, pkt, pkt != NULL,
-                          rss_info, etqf, ts,
-                          &desc->wb.lower.lo_dword.pkt_info,
-                          &desc->wb.lower.lo_dword.hdr_info,
-                          &desc->wb.lower.hi_dword.rss,
-                          &desc->wb.upper.status_error,
-                          &desc->wb.lower.hi_dword.csum_ip.ip_id,
-                          &desc->wb.upper.vlan);
+    pkt_type = igb_rx_desc_get_packet_type(core, pkt, etqf);
+    trace_e1000e_rx_metadata_pkt_type(pkt_type);
+    desc->wb.lower.lo_dword.pkt_info = cpu_to_le16(rss_type | (pkt_type << 4));
+    desc->wb.upper.status_error |= cpu_to_le32(adv_desc_status_error);
 }
 
 static inline void
@@ -XXX,XX +XXX,XX @@ igb_write_rx_descr(IGBCore *core, union e1000_rx_desc_union *desc,
                    uint16_t etqf, bool ts, uint16_t length)
 {
     if (igb_rx_use_legacy_descriptor(core)) {
-        igb_write_lgcy_rx_descr(core, &desc->legacy, pkt, rss_info,
-                                etqf, ts, length);
+        igb_write_lgcy_rx_descr(core, &desc->legacy, pkt, rss_info, length);
     } else {
         igb_write_adv_rx_descr(core, &desc->adv, pkt, rss_info,
                                etqf, ts, length);
diff --git a/hw/net/igb_regs.h b/hw/net/igb_regs.h
index XXXXXXX..XXXXXXX 100644
--- a/hw/net/igb_regs.h
+++ b/hw/net/igb_regs.h
@@ -XXX,XX +XXX,XX @@ union e1000_adv_rx_desc {
 
 #define E1000_STATUS_NUM_VFS_SHIFT 14
 
-#define E1000_ADVRXD_PKT_IP4 BIT(4)
-#define E1000_ADVRXD_PKT_IP6 BIT(6)
-#define E1000_ADVRXD_PKT_TCP BIT(8)
-#define E1000_ADVRXD_PKT_UDP BIT(9)
-#define E1000_ADVRXD_PKT_SCTP BIT(10)
+#define E1000_ADVRXD_PKT_IP4  BIT(0)
+#define E1000_ADVRXD_PKT_IP6  BIT(2)
+#define E1000_ADVRXD_PKT_TCP  BIT(4)
+#define E1000_ADVRXD_PKT_UDP  BIT(5)
+#define E1000_ADVRXD_PKT_SCTP BIT(6)
 
 static inline uint8_t igb_ivar_entry_rx(uint8_t i)
 {
diff --git a/hw/net/trace-events b/hw/net/trace-events
index XXXXXXX..XXXXXXX 100644
--- a/hw/net/trace-events
+++ b/hw/net/trace-events
@@ -XXX,XX +XXX,XX @@ igb_link_set_ext_params(bool asd_check, bool speed_select_bypass, bool pfrstd) "
 igb_rx_desc_buff_size(uint32_t b) "buffer size: %u"
 igb_rx_desc_buff_write(uint64_t addr, uint16_t offset, const void* source, uint32_t len) "addr: 0x%"PRIx64", offset: %u, from: %p, length: %u"
 
-igb_rx_metadata_rss(uint32_t rss) "RSS data: 0x%X"
+igb_rx_metadata_rss(uint32_t rss, uint16_t rss_pkt_type) "RSS data: rss: 0x%X, rss_pkt_type: 0x%X"
 
 igb_irq_icr_clear_gpie_nsicr(void) "Clearing ICR on read due to GPIE.NSICR enabled"
 igb_irq_set_iam(uint32_t icr) "Update IAM: 0x%x"
@@ -XXX,XX +XXX,XX @@ igb_irq_eitr_set(uint32_t eitr_num, uint32_t val) "EITR[%u] = 0x%x"
 igb_set_pfmailbox(uint32_t vf_num, uint32_t val) "PFMailbox[%d]: 0x%x"
 igb_set_vfmailbox(uint32_t vf_num, uint32_t val) "VFMailbox[%d]: 0x%x"
 
+igb_wrn_rx_desc_modes_not_supp(int desc_type) "Not supported descriptor type: %d"
+
 # igbvf.c
 igbvf_wrn_io_addr_unknown(uint64_t addr) "IO unknown register 0x%"PRIx64
 
-- 
2.7.4

From: Tomasz Dzieciol <t.dzieciol@partner.samsung.com>

Refactoring is done in preparation for support of multiple advanced
descriptors RX modes, especially packet-split modes.

Signed-off-by: Tomasz Dzieciol <t.dzieciol@partner.samsung.com>
Reviewed-by: Akihiko Odaki <akihiko.odaki@daynix.com>
Tested-by: Akihiko Odaki <akihiko.odaki@daynix.com>
Signed-off-by: Jason Wang <jasowang@redhat.com>
---
 hw/net/e1000e_core.c     |  18 ++--
 hw/net/igb_core.c        | 213 ++++++++++++++++++++++++++++++-----------------
 tests/qtest/libqos/igb.c |   5 ++
 3 files changed, 150 insertions(+), 86 deletions(-)

diff --git a/hw/net/e1000e_core.c b/hw/net/e1000e_core.c
index XXXXXXX..XXXXXXX 100644
--- a/hw/net/e1000e_core.c
+++ b/hw/net/e1000e_core.c
@@ -XXX,XX +XXX,XX @@ e1000e_write_hdr_to_rx_buffers(E1000ECore *core,
 }
 
 static void
-e1000e_write_to_rx_buffers(E1000ECore *core,
-                           hwaddr ba[MAX_PS_BUFFERS],
-                           e1000e_ba_state *bastate,
-                           const char *data,
-                           dma_addr_t data_len)
+e1000e_write_payload_frag_to_rx_buffers(E1000ECore *core,
+                                        hwaddr ba[MAX_PS_BUFFERS],
+                                        e1000e_ba_state *bastate,
+                                        const char *data,
+                                        dma_addr_t data_len)
 {
     while (data_len > 0) {
         uint32_t cur_buf_len = core->rxbuf_sizes[bastate->cur_idx];
@@ -XXX,XX +XXX,XX @@ e1000e_write_packet_to_guest(E1000ECore *core, struct NetRxPkt *pkt,
                 while (copy_size) {
                     iov_copy = MIN(copy_size, iov->iov_len - iov_ofs);
 
-                    e1000e_write_to_rx_buffers(core, ba, &bastate,
-                                            iov->iov_base + iov_ofs, iov_copy);
+                    e1000e_write_payload_frag_to_rx_buffers(core, ba, &bastate,
+                                                            iov->iov_base +
+                                                            iov_ofs,
+                                                            iov_copy);
 
                     copy_size -= iov_copy;
                     iov_ofs += iov_copy;
@@ -XXX,XX +XXX,XX @@ e1000e_write_packet_to_guest(E1000ECore *core, struct NetRxPkt *pkt,
 
                 if (desc_offset + desc_size >= total_size) {
                     /* Simulate FCS checksum presence in the last descriptor */
-                    e1000e_write_to_rx_buffers(core, ba, &bastate,
+                    e1000e_write_payload_frag_to_rx_buffers(core, ba, &bastate,
                           (const char *) &fcs_pad, e1000x_fcs_len(core->mac));
                 }
             }
diff --git a/hw/net/igb_core.c b/hw/net/igb_core.c
index XXXXXXX..XXXXXXX 100644
--- a/hw/net/igb_core.c
+++ b/hw/net/igb_core.c
@@ -XXX,XX +XXX,XX @@ igb_has_rxbufs(IGBCore *core, const E1000ERingInfo *r, size_t total_size)
                          bufsize;
 }
 
+static uint32_t
+igb_rxhdrbufsize(IGBCore *core, const E1000ERingInfo *r)
+{
+    uint32_t srrctl = core->mac[E1000_SRRCTL(r->idx) >> 2];
+    return (srrctl & E1000_SRRCTL_BSIZEHDRSIZE_MASK) >>
+           E1000_SRRCTL_BSIZEHDRSIZE_SHIFT;
+}
+
 void
 igb_start_recv(IGBCore *core)
 {
@@ -XXX,XX +XXX,XX @@ igb_read_adv_rx_descr(IGBCore *core, union e1000_adv_rx_desc *desc,
     *buff_addr = le64_to_cpu(desc->read.pkt_addr);
 }
 
+typedef struct IGBPacketRxDMAState {
+    size_t size;
+    size_t total_size;
+    size_t ps_hdr_len;
+    size_t desc_size;
+    size_t desc_offset;
+    uint32_t rx_desc_packet_buf_size;
+    uint32_t rx_desc_header_buf_size;
+    struct iovec *iov;
+    size_t iov_ofs;
+    bool is_first;
+    uint16_t written;
+    hwaddr ba;
+} IGBPacketRxDMAState;
+
 static inline void
 igb_read_rx_descr(IGBCore *core, union e1000_rx_desc_union *desc,
                   hwaddr *buff_addr)
@@ -XXX,XX +XXX,XX @@ igb_pci_dma_write_rx_desc(IGBCore *core, PCIDevice *dev, dma_addr_t addr,
 }
 
 static void
-igb_write_to_rx_buffers(IGBCore *core,
-                        PCIDevice *d,
-                        hwaddr ba,
-                        uint16_t *written,
-                        const char *data,
-                        dma_addr_t data_len)
-{
-    trace_igb_rx_desc_buff_write(ba, *written, data, data_len);
-    pci_dma_write(d, ba + *written, data, data_len);
-    *written += data_len;
-}
-
-static void
 igb_update_rx_stats(IGBCore *core, const E1000ERingInfo *rxi,
                     size_t pkt_size, size_t pkt_fcs_size)
 {
@@ -XXX,XX +XXX,XX @@ igb_rx_descr_threshold_hit(IGBCore *core, const E1000ERingInfo *rxi)
 }
 
 static void
+igb_truncate_to_descriptor_size(IGBPacketRxDMAState *pdma_st, size_t *size)
+{
+    if (*size > pdma_st->rx_desc_packet_buf_size) {
+        *size = pdma_st->rx_desc_packet_buf_size;
+    }
+}
+
+static void
+igb_write_payload_frag_to_rx_buffers(IGBCore *core,
+                                     PCIDevice *d,
+                                     hwaddr ba,
+                                     uint16_t *written,
+                                     uint32_t cur_buf_len,
+                                     const char *data,
+                                     dma_addr_t data_len)
+{
+    trace_igb_rx_desc_buff_write(ba, *written, data, data_len);
+    pci_dma_write(d, ba + *written, data, data_len);
+    *written += data_len;
+}
+
+static void
+igb_write_payload_to_rx_buffers(IGBCore *core,
+                                struct NetRxPkt *pkt,
+                                PCIDevice *d,
+                                IGBPacketRxDMAState *pdma_st,
+                                size_t *copy_size)
+{
+    static const uint32_t fcs_pad;
+    size_t iov_copy;
+
+    /* Copy packet payload */
+    while (*copy_size) {
+        iov_copy = MIN(*copy_size, pdma_st->iov->iov_len - pdma_st->iov_ofs);
+        igb_write_payload_frag_to_rx_buffers(core, d,
+                                             pdma_st->ba,
+                                             &pdma_st->written,
+                                             pdma_st->rx_desc_packet_buf_size,
+                                             pdma_st->iov->iov_base +
+                                             pdma_st->iov_ofs,
+                                             iov_copy);
+
+        *copy_size -= iov_copy;
+        pdma_st->iov_ofs += iov_copy;
+        if (pdma_st->iov_ofs == pdma_st->iov->iov_len) {
+            pdma_st->iov++;
+            pdma_st->iov_ofs = 0;
+        }
+    }
+
+    if (pdma_st->desc_offset + pdma_st->desc_size >= pdma_st->total_size) {
+        /* Simulate FCS checksum presence in the last descriptor */
+        igb_write_payload_frag_to_rx_buffers(core, d,
+                                             pdma_st->ba,
+                                             &pdma_st->written,
+                                             pdma_st->rx_desc_packet_buf_size,
+                                             (const char *) &fcs_pad,
+                                             e1000x_fcs_len(core->mac));
+    }
+}
+
+static void
+igb_write_to_rx_buffers(IGBCore *core,
+                        struct NetRxPkt *pkt,
+                        PCIDevice *d,
+                        IGBPacketRxDMAState *pdma_st)
+{
+    size_t copy_size;
+
+    if (!pdma_st->ba) {
+        /* as per intel docs; skip descriptors with null buf addr */
+        trace_e1000e_rx_null_descriptor();
+        return;
+    }
+
+    if (pdma_st->desc_offset >= pdma_st->size) {
+        return;
+    }
+
+    pdma_st->desc_size = pdma_st->total_size - pdma_st->desc_offset;
+    igb_truncate_to_descriptor_size(pdma_st, &pdma_st->desc_size);
+    copy_size = pdma_st->size - pdma_st->desc_offset;
+    igb_truncate_to_descriptor_size(pdma_st, &copy_size);
+    igb_write_payload_to_rx_buffers(core, pkt, d, pdma_st, &copy_size);
+}
+
+static void
 igb_write_packet_to_guest(IGBCore *core, struct NetRxPkt *pkt,
                           const E1000E_RxRing *rxr,
                           const E1000E_RSSInfo *rss_info,
@@ -XXX,XX +XXX,XX @@ igb_write_packet_to_guest(IGBCore *core, struct NetRxPkt *pkt,
     PCIDevice *d;
     dma_addr_t base;
     union e1000_rx_desc_union desc;
-    size_t desc_size;
-    size_t desc_offset = 0;
-    size_t iov_ofs = 0;
-
-    struct iovec *iov = net_rx_pkt_get_iovec(pkt);
-    size_t size = net_rx_pkt_get_total_len(pkt);
-    size_t total_size = size + e1000x_fcs_len(core->mac);
-    const E1000ERingInfo *rxi = rxr->i;
-    size_t bufsize = igb_rxbufsize(core, rxi);
-
+    const E1000ERingInfo *rxi;
+    size_t rx_desc_len;
+
+    IGBPacketRxDMAState pdma_st = {0};
+    pdma_st.is_first = true;
+    pdma_st.size = net_rx_pkt_get_total_len(pkt);
+    pdma_st.total_size = pdma_st.size + e1000x_fcs_len(core->mac);
+
+    rxi = rxr->i;
+    rx_desc_len = core->rx_desc_len;
+    pdma_st.rx_desc_packet_buf_size = igb_rxbufsize(core, rxi);
+    pdma_st.rx_desc_header_buf_size = igb_rxhdrbufsize(core, rxi);
+    pdma_st.iov = net_rx_pkt_get_iovec(pkt);
     d = pcie_sriov_get_vf_at_index(core->owner, rxi->idx % 8);
     if (!d) {
         d = core->owner;
     }
 
     do {
-        hwaddr ba;
-        uint16_t written = 0;
+        pdma_st.written = 0;
         bool is_last = false;
 
-        desc_size = total_size - desc_offset;
-
-        if (desc_size > bufsize) {
-            desc_size = bufsize;
-        }
-
         if (igb_ring_empty(core, rxi)) {
             return;
         }
 
         base = igb_ring_head_descr(core, rxi);
+        pci_dma_read(d, base, &desc, rx_desc_len);
+        trace_e1000e_rx_descr(rxi->idx, base, rx_desc_len);
 
-        pci_dma_read(d, base, &desc, core->rx_desc_len);
-
-        trace_e1000e_rx_descr(rxi->idx, base, core->rx_desc_len);
-
-        igb_read_rx_descr(core, &desc, &ba);
-
-        if (ba) {
-            if (desc_offset < size) {
-                static const uint32_t fcs_pad;
-                size_t iov_copy;
-                size_t copy_size = size - desc_offset;
-                if (copy_size > bufsize) {
-                    copy_size = bufsize;
-                }
-
-                /* Copy packet payload */
-                while (copy_size) {
-                    iov_copy = MIN(copy_size, iov->iov_len - iov_ofs);
-
-                    igb_write_to_rx_buffers(core, d, ba, &written,
-                                            iov->iov_base + iov_ofs, iov_copy);
+        igb_read_rx_descr(core, &desc, &pdma_st.ba);
 
-                    copy_size -= iov_copy;
-                    iov_ofs += iov_copy;
-                    if (iov_ofs == iov->iov_len) {
-                        iov++;
-                        iov_ofs = 0;
-                    }
-                }
-
-                if (desc_offset + desc_size >= total_size) {
-                    /* Simulate FCS checksum presence in the last descriptor */
-                    igb_write_to_rx_buffers(core, d, ba, &written,
-                          (const char *) &fcs_pad, e1000x_fcs_len(core->mac));
-                }
-            }
-        } else { /* as per intel docs; skip descriptors with null buf addr */
-            trace_e1000e_rx_null_descriptor();
-        }
-        desc_offset += desc_size;
-        if (desc_offset >= total_size) {
+        igb_write_to_rx_buffers(core, pkt, d, &pdma_st);
+        pdma_st.desc_offset += pdma_st.desc_size;
+        if (pdma_st.desc_offset >= pdma_st.total_size) {
             is_last = true;
         }
 
         igb_write_rx_descr(core, &desc, is_last ? core->rx_pkt : NULL,
-                           rss_info, etqf, ts, written);
-        igb_pci_dma_write_rx_desc(core, d, base, &desc, core->rx_desc_len);
-
-        igb_ring_advance(core, rxi, core->rx_desc_len / E1000_MIN_RX_DESC_LEN);
-
-    } while (desc_offset < total_size);
+                           rss_info, etqf, ts, pdma_st.written);
+        igb_pci_dma_write_rx_desc(core, d, base, &desc, rx_desc_len);
+        igb_ring_advance(core, rxi, rx_desc_len / E1000_MIN_RX_DESC_LEN);
+    } while (pdma_st.desc_offset < pdma_st.total_size);
 
-    igb_update_rx_stats(core, rxi, size, total_size);
+    igb_update_rx_stats(core, rxi, pdma_st.size, pdma_st.total_size);
 }
 
 static bool
diff --git a/tests/qtest/libqos/igb.c b/tests/qtest/libqos/igb.c
index XXXXXXX..XXXXXXX 100644
--- a/tests/qtest/libqos/igb.c
+++ b/tests/qtest/libqos/igb.c
@@ -XXX,XX +XXX,XX @@ static void igb_pci_start_hw(QOSGraphObject *obj)
                         E1000_RAH_AV | E1000_RAH_POOL_1 |
                         le16_to_cpu(*(uint16_t *)(address + 4)));
 
+    /* Set supported receive descriptor mode */
+    e1000e_macreg_write(&d->e1000e,
+                        E1000_SRRCTL(0),
+                        E1000_SRRCTL_DESCTYPE_ADV_ONEBUF);
+
     /* Enable receive */
     e1000e_macreg_write(&d->e1000e, E1000_RFCTL, E1000_RFCTL_EXTEN);
     e1000e_macreg_write(&d->e1000e, E1000_RCTL, E1000_RCTL_EN);
-- 
2.7.4

From: Tomasz Dzieciol <t.dzieciol@partner.samsung.com>

Signed-off-by: Tomasz Dzieciol <t.dzieciol@partner.samsung.com>
Reviewed-by: Akihiko Odaki <akihiko.odaki@daynix.com>
Tested-by: Akihiko Odaki <akihiko.odaki@daynix.com>
Signed-off-by: Jason Wang <jasowang@redhat.com>
---
 hw/net/igb_core.c | 4 +++-
 hw/net/igb_regs.h | 1 +
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/hw/net/igb_core.c b/hw/net/igb_core.c
index XXXXXXX..XXXXXXX 100644
--- a/hw/net/igb_core.c
+++ b/hw/net/igb_core.c
@@ -XXX,XX +XXX,XX @@ igb_rx_desc_get_packet_type(IGBCore *core, struct NetRxPkt *pkt, uint16_t etqf)
     net_rx_pkt_get_protocols(pkt, &hasip4, &hasip6, &l4hdr_proto);
 
     if (hasip6 && !(core->mac[RFCTL] & E1000_RFCTL_IPV6_DIS)) {
-        pkt_type = E1000_ADVRXD_PKT_IP6;
+        eth_ip6_hdr_info *ip6hdr_info = net_rx_pkt_get_ip6_info(pkt);
+        pkt_type = ip6hdr_info->has_ext_hdrs ? E1000_ADVRXD_PKT_IP6E :
+                                               E1000_ADVRXD_PKT_IP6;
     } else if (hasip4) {
         pkt_type = E1000_ADVRXD_PKT_IP4;
     } else {
diff --git a/hw/net/igb_regs.h b/hw/net/igb_regs.h
index XXXXXXX..XXXXXXX 100644
--- a/hw/net/igb_regs.h
+++ b/hw/net/igb_regs.h
@@ -XXX,XX +XXX,XX @@ union e1000_adv_rx_desc {
 
 #define E1000_ADVRXD_PKT_IP4  BIT(0)
 #define E1000_ADVRXD_PKT_IP6  BIT(2)
+#define E1000_ADVRXD_PKT_IP6E BIT(3)
 #define E1000_ADVRXD_PKT_TCP  BIT(4)
 #define E1000_ADVRXD_PKT_UDP  BIT(5)
 #define E1000_ADVRXD_PKT_SCTP BIT(6)
-- 
2.7.4

From: Tomasz Dzieciol <t.dzieciol@partner.samsung.com>

Packet-split descriptors are used by Linux VF driver for MTU values from 2048

Signed-off-by: Tomasz Dzieciol <t.dzieciol@partner.samsung.com>
Reviewed-by: Akihiko Odaki <akihiko.odaki@daynix.com>
Tested-by: Akihiko Odaki <akihiko.odaki@daynix.com>
Signed-off-by: Jason Wang <jasowang@redhat.com>
---
 hw/net/igb_core.c   | 348 +++++++++++++++++++++++++++++++++++++++++++++-------
 hw/net/igb_regs.h   |   9 ++
 hw/net/trace-events |   2 +-
 3 files changed, 316 insertions(+), 43 deletions(-)

diff --git a/hw/net/igb_core.c b/hw/net/igb_core.c
index XXXXXXX..XXXXXXX 100644
--- a/hw/net/igb_core.c
+++ b/hw/net/igb_core.c
@@ -XXX,XX +XXX,XX @@ igb_rx_use_legacy_descriptor(IGBCore *core)
     return false;
 }
 
+typedef struct E1000ERingInfo {
+    int dbah;
+    int dbal;
+    int dlen;
+    int dh;
+    int dt;
+    int idx;
+} E1000ERingInfo;
+
+static uint32_t
+igb_rx_queue_desctyp_get(IGBCore *core, const E1000ERingInfo *r)
+{
+    return core->mac[E1000_SRRCTL(r->idx) >> 2] & E1000_SRRCTL_DESCTYPE_MASK;
+}
+
+static bool
+igb_rx_use_ps_descriptor(IGBCore *core, const E1000ERingInfo *r)
+{
+    uint32_t desctyp = igb_rx_queue_desctyp_get(core, r);
+    return desctyp == E1000_SRRCTL_DESCTYPE_HDR_SPLIT ||
+           desctyp == E1000_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS;
+}
+
 static inline bool
 igb_rss_enabled(IGBCore *core)
 {
@@ -XXX,XX +XXX,XX @@ static uint32_t igb_rx_wb_eic(IGBCore *core, int queue_idx)
     return (ent & E1000_IVAR_VALID) ? BIT(ent & 0x1f) : 0;
 }
 
-typedef struct E1000ERingInfo {
-    int dbah;
-    int dbal;
-    int dlen;
-    int dh;
-    int dt;
-    int idx;
-} E1000ERingInfo;
-
 static inline bool
 igb_ring_empty(IGBCore *core, const E1000ERingInfo *r)
 {
@@ -XXX,XX +XXX,XX @@ igb_read_lgcy_rx_descr(IGBCore *core, struct e1000_rx_desc *desc,
 }
 
 static inline void
-igb_read_adv_rx_descr(IGBCore *core, union e1000_adv_rx_desc *desc,
-                      hwaddr *buff_addr)
+igb_read_adv_rx_single_buf_descr(IGBCore *core, union e1000_adv_rx_desc *desc,
+                                 hwaddr *buff_addr)
 {
     *buff_addr = le64_to_cpu(desc->read.pkt_addr);
 }
 
+static inline void
+igb_read_adv_rx_split_buf_descr(IGBCore *core, union e1000_adv_rx_desc *desc,
+                                hwaddr *buff_addr)
+{
+    buff_addr[0] = le64_to_cpu(desc->read.hdr_addr);
+    buff_addr[1] = le64_to_cpu(desc->read.pkt_addr);
+}
+
+typedef struct IGBBAState {
+    uint16_t written[IGB_MAX_PS_BUFFERS];
+    uint8_t cur_idx;
+} IGBBAState;
+
+typedef struct IGBSplitDescriptorData {
+    bool sph;
+    bool hbo;
+    size_t hdr_len;
+} IGBSplitDescriptorData;
+
 typedef struct IGBPacketRxDMAState {
     size_t size;
     size_t total_size;
@@ -XXX,XX +XXX,XX @@ typedef struct IGBPacketRxDMAState {
     uint32_t rx_desc_header_buf_size;
     struct iovec *iov;
     size_t iov_ofs;
+    bool do_ps;
     bool is_first;
-    uint16_t written;
-    hwaddr ba;
+    IGBBAState bastate;
+    hwaddr ba[IGB_MAX_PS_BUFFERS];
+    IGBSplitDescriptorData ps_desc_data;
 } IGBPacketRxDMAState;
 
 static inline void
-igb_read_rx_descr(IGBCore *core, union e1000_rx_desc_union *desc,
-                  hwaddr *buff_addr)
+igb_read_rx_descr(IGBCore *core,
+                  union e1000_rx_desc_union *desc,
+                  IGBPacketRxDMAState *pdma_st,
+                  const E1000ERingInfo *r)
 {
+    uint32_t desc_type;
+
     if (igb_rx_use_legacy_descriptor(core)) {
-        igb_read_lgcy_rx_descr(core, &desc->legacy, buff_addr);
-    } else {
-        igb_read_adv_rx_descr(core, &desc->adv, buff_addr);
+        igb_read_lgcy_rx_descr(core, &desc->legacy, &pdma_st->ba[1]);
+        pdma_st->ba[0] = 0;
+        return;
+    }
+
+    /* advanced header split descriptor */
+    if (igb_rx_use_ps_descriptor(core, r)) {
+        igb_read_adv_rx_split_buf_descr(core, &desc->adv, &pdma_st->ba[0]);
+        return;
+    }
+
+    /* descriptor replication modes not supported */
+    desc_type = igb_rx_queue_desctyp_get(core, r);
+    if (desc_type != E1000_SRRCTL_DESCTYPE_ADV_ONEBUF) {
+        trace_igb_wrn_rx_desc_modes_not_supp(desc_type);
     }
+
+    /* advanced single buffer descriptor */
+    igb_read_adv_rx_single_buf_descr(core, &desc->adv, &pdma_st->ba[1]);
+    pdma_st->ba[0] = 0;
 }
 
 static void
@@ -XXX,XX +XXX,XX @@ igb_write_lgcy_rx_descr(IGBCore *core, struct e1000_rx_desc *desc,
     desc->status = (uint8_t) le32_to_cpu(status_flags);
 }
 
+static bool
+igb_rx_ps_descriptor_split_always(IGBCore *core, const E1000ERingInfo *r)
+{
+    uint32_t desctyp = igb_rx_queue_desctyp_get(core, r);
+    return desctyp == E1000_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS;
+}
+
 static uint16_t
 igb_rx_desc_get_packet_type(IGBCore *core, struct NetRxPkt *pkt, uint16_t etqf)
 {
@@ -XXX,XX +XXX,XX @@ igb_write_adv_rx_descr(IGBCore *core, union e1000_adv_rx_desc *desc,
 }
 
 static inline void
-igb_write_rx_descr(IGBCore *core, union e1000_rx_desc_union *desc,
-                   struct NetRxPkt *pkt, const E1000E_RSSInfo *rss_info,
-                   uint16_t etqf, bool ts, uint16_t length)
+igb_write_adv_ps_rx_descr(IGBCore *core,
+                          union e1000_adv_rx_desc *desc,
+                          struct NetRxPkt *pkt,
+                          const E1000E_RSSInfo *rss_info,
+                          const E1000ERingInfo *r,
+                          uint16_t etqf,
+                          bool ts,
+                          IGBPacketRxDMAState *pdma_st)
+{
+    size_t pkt_len;
+    uint16_t hdr_info = 0;
+
+    if (pdma_st->do_ps) {
+        pkt_len = pdma_st->bastate.written[1];
+    } else {
+        pkt_len = pdma_st->bastate.written[0] + pdma_st->bastate.written[1];
+    }
+
+    igb_write_adv_rx_descr(core, desc, pkt, rss_info, etqf, ts, pkt_len);
+
+    hdr_info = (pdma_st->ps_desc_data.hdr_len << E1000_ADVRXD_HDR_LEN_OFFSET) &
+               E1000_ADVRXD_ADV_HDR_LEN_MASK;
+    hdr_info |= pdma_st->ps_desc_data.sph ? E1000_ADVRXD_HDR_SPH : 0;
+    desc->wb.lower.lo_dword.hdr_info = cpu_to_le16(hdr_info);
+
+    desc->wb.upper.status_error |= cpu_to_le32(
+        pdma_st->ps_desc_data.hbo ? E1000_ADVRXD_ST_ERR_HBO_OFFSET : 0);
+}
+
+static inline void
+igb_write_rx_descr(IGBCore *core,
+                   union e1000_rx_desc_union *desc,
+                   struct NetRxPkt *pkt,
+                   const E1000E_RSSInfo *rss_info,
+                   uint16_t etqf,
+                   bool ts,
+                   IGBPacketRxDMAState *pdma_st,
+                   const E1000ERingInfo *r)
 {
     if (igb_rx_use_legacy_descriptor(core)) {
-        igb_write_lgcy_rx_descr(core, &desc->legacy, pkt, rss_info, length);
+        igb_write_lgcy_rx_descr(core, &desc->legacy, pkt, rss_info,
+                                pdma_st->bastate.written[1]);
+    } else if (igb_rx_use_ps_descriptor(core, r)) {
+        igb_write_adv_ps_rx_descr(core, &desc->adv, pkt, rss_info, r, etqf, ts,
+                                  pdma_st);
     } else {
         igb_write_adv_rx_descr(core, &desc->adv, pkt, rss_info,
-                               etqf, ts, length);
+                               etqf, ts, pdma_st->bastate.written[1]);
     }
 }
 
@@ -XXX,XX +XXX,XX @@ igb_rx_descr_threshold_hit(IGBCore *core, const E1000ERingInfo *rxi)
            ((core->mac[E1000_SRRCTL(rxi->idx) >> 2] >> 20) & 31) * 16;
 }
 
+static bool
+igb_do_ps(IGBCore *core,
+          const E1000ERingInfo *r,
+          struct NetRxPkt *pkt,
+          IGBPacketRxDMAState *pdma_st)
+{
+    bool hasip4, hasip6;
+    EthL4HdrProto l4hdr_proto;
+    bool fragment;
+    bool split_always;
+    size_t bheader_size;
+    size_t total_pkt_len;
+
+    if (!igb_rx_use_ps_descriptor(core, r)) {
+        return false;
+    }
+
+    total_pkt_len = net_rx_pkt_get_total_len(pkt);
+    bheader_size = igb_rxhdrbufsize(core, r);
+    split_always = igb_rx_ps_descriptor_split_always(core, r);
+    if (split_always && total_pkt_len <= bheader_size) {
+        pdma_st->ps_hdr_len = total_pkt_len;
+        pdma_st->ps_desc_data.hdr_len = total_pkt_len;
+        return true;
+    }
+
+    net_rx_pkt_get_protocols(pkt, &hasip4, &hasip6, &l4hdr_proto);
+
+    if (hasip4) {
+        fragment = net_rx_pkt_get_ip4_info(pkt)->fragment;
+    } else if (hasip6) {
+        fragment = net_rx_pkt_get_ip6_info(pkt)->fragment;
+    } else {
+        pdma_st->ps_desc_data.hdr_len = bheader_size;
+        goto header_not_handled;
+    }
+
+    if (fragment && (core->mac[RFCTL] & E1000_RFCTL_IPFRSP_DIS)) {
+        pdma_st->ps_desc_data.hdr_len = bheader_size;
+        goto header_not_handled;
+    }
+
+    /* no header splitting for SCTP */
+    if (!fragment && (l4hdr_proto == ETH_L4_HDR_PROTO_UDP ||
+                      l4hdr_proto == ETH_L4_HDR_PROTO_TCP)) {
+        pdma_st->ps_hdr_len = net_rx_pkt_get_l5_hdr_offset(pkt);
+    } else {
+        pdma_st->ps_hdr_len = net_rx_pkt_get_l4_hdr_offset(pkt);
+    }
+
+    pdma_st->ps_desc_data.sph = true;
+    pdma_st->ps_desc_data.hdr_len = pdma_st->ps_hdr_len;
+
+    if (pdma_st->ps_hdr_len > bheader_size) {
+        pdma_st->ps_desc_data.hbo = true;
+        goto header_not_handled;
+    }
+
+    return true;
+
+header_not_handled:
+    if (split_always) {
+        pdma_st->ps_hdr_len = bheader_size;
+        return true;
+    }
+
+    return false;
+}
+
 static void
 igb_truncate_to_descriptor_size(IGBPacketRxDMAState *pdma_st, size_t *size)
 {
-    if (*size > pdma_st->rx_desc_packet_buf_size) {
-        *size = pdma_st->rx_desc_packet_buf_size;
+    if (pdma_st->do_ps && pdma_st->is_first) {
+        if (*size > pdma_st->rx_desc_packet_buf_size + pdma_st->ps_hdr_len) {
+            *size = pdma_st->rx_desc_packet_buf_size + pdma_st->ps_hdr_len;
+        }
+    } else {
+        if (*size > pdma_st->rx_desc_packet_buf_size) {
+            *size = pdma_st->rx_desc_packet_buf_size;
+        }
+    }
+}
+
+static inline void
+igb_write_hdr_frag_to_rx_buffers(IGBCore *core,
+                                 PCIDevice *d,
+                                 IGBPacketRxDMAState *pdma_st,
+                                 const char *data,
+                                 dma_addr_t data_len)
+{
+    assert(data_len <= pdma_st->rx_desc_header_buf_size -
+                       pdma_st->bastate.written[0]);
+    pci_dma_write(d,
+                  pdma_st->ba[0] + pdma_st->bastate.written[0],
+                  data, data_len);
+    pdma_st->bastate.written[0] += data_len;
+    pdma_st->bastate.cur_idx = 1;
+}
+
+static void
+igb_write_header_to_rx_buffers(IGBCore *core,
+                               struct NetRxPkt *pkt,
+                               PCIDevice *d,
+                               IGBPacketRxDMAState *pdma_st,
+                               size_t *copy_size)
+{
+    size_t iov_copy;
+    size_t ps_hdr_copied = 0;
+
+    if (!pdma_st->is_first) {
+        /* Leave buffer 0 of each descriptor except first */
+        /* empty                                          */
+        pdma_st->bastate.cur_idx = 1;
+        return;
     }
+
+    do {
+        iov_copy = MIN(pdma_st->ps_hdr_len - ps_hdr_copied,
+                       pdma_st->iov->iov_len - pdma_st->iov_ofs);
+
+        igb_write_hdr_frag_to_rx_buffers(core, d, pdma_st,
+                                         pdma_st->iov->iov_base,
+                                         iov_copy);
+
+        *copy_size -= iov_copy;
+        ps_hdr_copied += iov_copy;
+
+        pdma_st->iov_ofs += iov_copy;
+        if (pdma_st->iov_ofs == pdma_st->iov->iov_len) {
+            pdma_st->iov++;
+            pdma_st->iov_ofs = 0;
+        }
+    } while (ps_hdr_copied < pdma_st->ps_hdr_len);
+
+    pdma_st->is_first = false;
 }
 
 static void
 igb_write_payload_frag_to_rx_buffers(IGBCore *core,
                                      PCIDevice *d,
-                                     hwaddr ba,
-                                     uint16_t *written,
-                                     uint32_t cur_buf_len,
+                                     IGBPacketRxDMAState *pdma_st,
                                      const char *data,
                                      dma_addr_t data_len)
 {
-    trace_igb_rx_desc_buff_write(ba, *written, data, data_len);
-    pci_dma_write(d, ba + *written, data, data_len);
-    *written += data_len;
+    while (data_len > 0) {
+        assert(pdma_st->bastate.cur_idx < IGB_MAX_PS_BUFFERS);
+
+        uint32_t cur_buf_bytes_left =
+            pdma_st->rx_desc_packet_buf_size -
+            pdma_st->bastate.written[pdma_st->bastate.cur_idx];
+        uint32_t bytes_to_write = MIN(data_len, cur_buf_bytes_left);
+
+        trace_igb_rx_desc_buff_write(
+            pdma_st->bastate.cur_idx,
+            pdma_st->ba[pdma_st->bastate.cur_idx],
+            pdma_st->bastate.written[pdma_st->bastate.cur_idx],
+            data,
+            bytes_to_write);
+
+        pci_dma_write(d,
+                      pdma_st->ba[pdma_st->bastate.cur_idx] +
+                      pdma_st->bastate.written[pdma_st->bastate.cur_idx],
+                      data, bytes_to_write);
+
+        pdma_st->bastate.written[pdma_st->bastate.cur_idx] += bytes_to_write;
+        data += bytes_to_write;
+        data_len -= bytes_to_write;
+
+        if (pdma_st->bastate.written[pdma_st->bastate.cur_idx] ==
+            pdma_st->rx_desc_packet_buf_size) {
+            pdma_st->bastate.cur_idx++;
+        }
+    }
 }
 
 static void
@@ -XXX,XX +XXX,XX @@ igb_write_payload_to_rx_buffers(IGBCore *core,
     while (*copy_size) {
         iov_copy = MIN(*copy_size, pdma_st->iov->iov_len - pdma_st->iov_ofs);
         igb_write_payload_frag_to_rx_buffers(core, d,
-                                             pdma_st->ba,
-                                             &pdma_st->written,
-                                             pdma_st->rx_desc_packet_buf_size,
+                                             pdma_st,
                                              pdma_st->iov->iov_base +
                                              pdma_st->iov_ofs,
                                              iov_copy);
@@ -XXX,XX +XXX,XX @@ igb_write_payload_to_rx_buffers(IGBCore *core,
     if (pdma_st->desc_offset + pdma_st->desc_size >= pdma_st->total_size) {
         /* Simulate FCS checksum presence in the last descriptor */
         igb_write_payload_frag_to_rx_buffers(core, d,
-                                             pdma_st->ba,
-                                             &pdma_st->written,
-                                             pdma_st->rx_desc_packet_buf_size,
+                                             pdma_st,
                                              (const char *) &fcs_pad,
                                              e1000x_fcs_len(core->mac));
     }
@@ -XXX,XX +XXX,XX @@ igb_write_to_rx_buffers(IGBCore *core,
 {
     size_t copy_size;
 
-    if (!pdma_st->ba) {
+    if (!(pdma_st->ba)[1] || (pdma_st->do_ps && !(pdma_st->ba[0]))) {
         /* as per intel docs; skip descriptors with null buf addr */
         trace_e1000e_rx_null_descriptor();
         return;
@@ -XXX,XX +XXX,XX @@ igb_write_to_rx_buffers(IGBCore *core,
     igb_truncate_to_descriptor_size(pdma_st, &pdma_st->desc_size);
     copy_size = pdma_st->size - pdma_st->desc_offset;
     igb_truncate_to_descriptor_size(pdma_st, &copy_size);
+
+    /* For PS mode copy the packet header first */
+    if (pdma_st->do_ps) {
+        igb_write_header_to_rx_buffers(core, pkt, d, pdma_st, &copy_size);
+    } else {
+        pdma_st->bastate.cur_idx = 1;
+    }
+
     igb_write_payload_to_rx_buffers(core, pkt, d, pdma_st, &copy_size);
 }
 
@@ -XXX,XX +XXX,XX @@ igb_write_packet_to_guest(IGBCore *core, struct NetRxPkt *pkt,
         d = core->owner;
     }
 
+    pdma_st.do_ps = igb_do_ps(core, rxi, pkt, &pdma_st);
+
     do {
-        pdma_st.written = 0;
+        memset(&pdma_st.bastate, 0, sizeof(IGBBAState));
         bool is_last = false;
 
         if (igb_ring_empty(core, rxi)) {
@@ -XXX,XX +XXX,XX @@ igb_write_packet_to_guest(IGBCore *core, struct NetRxPkt *pkt,
         pci_dma_read(d, base, &desc, rx_desc_len);
         trace_e1000e_rx_descr(rxi->idx, base, rx_desc_len);
 
-        igb_read_rx_descr(core, &desc, &pdma_st.ba);
+        igb_read_rx_descr(core, &desc, &pdma_st, rxi);
 
         igb_write_to_rx_buffers(core, pkt, d, &pdma_st);
         pdma_st.desc_offset += pdma_st.desc_size;
@@ -XXX,XX +XXX,XX @@ igb_write_packet_to_guest(IGBCore *core, struct NetRxPkt *pkt,
             is_last = true;
         }
 
-        igb_write_rx_descr(core, &desc, is_last ? core->rx_pkt : NULL,
-                           rss_info, etqf, ts, pdma_st.written);
+        igb_write_rx_descr(core, &desc,
+                           is_last ? pkt : NULL,
+                           rss_info,
+                           etqf, ts,
+                           &pdma_st,
+                           rxi);
         igb_pci_dma_write_rx_desc(core, d, base, &desc, rx_desc_len);
         igb_ring_advance(core, rxi, rx_desc_len / E1000_MIN_RX_DESC_LEN);
     } while (pdma_st.desc_offset < pdma_st.total_size);
diff --git a/hw/net/igb_regs.h b/hw/net/igb_regs.h
index XXXXXXX..XXXXXXX 100644
--- a/hw/net/igb_regs.h
+++ b/hw/net/igb_regs.h
@@ -XXX,XX +XXX,XX @@ union e1000_adv_rx_desc {
 #define E1000_SRRCTL_BSIZEHDRSIZE_MASK         0x00000F00
 #define E1000_SRRCTL_BSIZEHDRSIZE_SHIFT        2  /* Shift _left_ */
 #define E1000_SRRCTL_DESCTYPE_ADV_ONEBUF       0x02000000
+#define E1000_SRRCTL_DESCTYPE_HDR_SPLIT        0x04000000
 #define E1000_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS 0x0A000000
 #define E1000_SRRCTL_DESCTYPE_MASK             0x0E000000
 #define E1000_SRRCTL_DROP_EN                   0x80000000
@@ -XXX,XX +XXX,XX @@ union e1000_adv_rx_desc {
 #define E1000_ADVRXD_PKT_UDP  BIT(5)
 #define E1000_ADVRXD_PKT_SCTP BIT(6)
 
+#define IGB_MAX_PS_BUFFERS 2
+
+#define E1000_ADVRXD_HDR_LEN_OFFSET    (21 - 16)
+#define E1000_ADVRXD_ADV_HDR_LEN_MASK  ((BIT(10) - 1) << \
+                                        E1000_ADVRXD_HDR_LEN_OFFSET)
+#define E1000_ADVRXD_HDR_SPH           BIT(15)
+#define E1000_ADVRXD_ST_ERR_HBO_OFFSET BIT(3 + 20)
+
 static inline uint8_t igb_ivar_entry_rx(uint8_t i)
 {
     return i < 8 ? i * 4 : (i - 8) * 4 + 2;
diff --git a/hw/net/trace-events b/hw/net/trace-events
index XXXXXXX..XXXXXXX 100644
--- a/hw/net/trace-events
+++ b/hw/net/trace-events
@@ -XXX,XX +XXX,XX @@ igb_core_mdic_write_unhandled(uint32_t addr) "MDIC WRITE: PHY[%u] UNHANDLED"
 igb_link_set_ext_params(bool asd_check, bool speed_select_bypass, bool pfrstd) "Set extended link params: ASD check: %d, Speed select bypass: %d, PF reset done: %d"
 
 igb_rx_desc_buff_size(uint32_t b) "buffer size: %u"
-igb_rx_desc_buff_write(uint64_t addr, uint16_t offset, const void* source, uint32_t len) "addr: 0x%"PRIx64", offset: %u, from: %p, length: %u"
+igb_rx_desc_buff_write(uint8_t idx, uint64_t addr, uint16_t offset, const void* source, uint32_t len) "buffer %u, addr: 0x%"PRIx64", offset: %u, from: %p, length: %u"
 
 igb_rx_metadata_rss(uint32_t rss, uint16_t rss_pkt_type) "RSS data: rss: 0x%X, rss_pkt_type: 0x%X"
 
-- 
2.7.4

From: Tomasz Dzieciol <t.dzieciol@partner.samsung.com>

Rename e1000e_ba_state according and e1000e_write_hdr_to_rx_buffers for
consistency with IGB.

Signed-off-by: Tomasz Dzieciol <t.dzieciol@partner.samsung.com>
Reviewed-by: Akihiko Odaki <akihiko.odaki@daynix.com>
Tested-by: Akihiko Odaki <akihiko.odaki@daynix.com>
Signed-off-by: Jason Wang <jasowang@redhat.com>
---
 hw/net/e1000e_core.c | 28 +++++++++++++++-------------
 1 file changed, 15 insertions(+), 13 deletions(-)

diff --git a/hw/net/e1000e_core.c b/hw/net/e1000e_core.c
index XXXXXXX..XXXXXXX 100644
--- a/hw/net/e1000e_core.c
+++ b/hw/net/e1000e_core.c
@@ -XXX,XX +XXX,XX @@ e1000e_pci_dma_write_rx_desc(E1000ECore *core, dma_addr_t addr,
     }
 }
 
-typedef struct e1000e_ba_state_st {
+typedef struct E1000EBAState {
     uint16_t written[MAX_PS_BUFFERS];
     uint8_t cur_idx;
-} e1000e_ba_state;
+} E1000EBAState;
 
 static inline void
-e1000e_write_hdr_to_rx_buffers(E1000ECore *core,
-                               hwaddr ba[MAX_PS_BUFFERS],
-                               e1000e_ba_state *bastate,
-                               const char *data,
-                               dma_addr_t data_len)
+e1000e_write_hdr_frag_to_rx_buffers(E1000ECore *core,
+                                    hwaddr ba[MAX_PS_BUFFERS],
+                                    E1000EBAState *bastate,
+                                    const char *data,
+                                    dma_addr_t data_len)
 {
     assert(data_len <= core->rxbuf_sizes[0] - bastate->written[0]);
 
@@ -XXX,XX +XXX,XX @@ e1000e_write_hdr_to_rx_buffers(E1000ECore *core,
 static void
 e1000e_write_payload_frag_to_rx_buffers(E1000ECore *core,
                                         hwaddr ba[MAX_PS_BUFFERS],
-                                        e1000e_ba_state *bastate,
+                                        E1000EBAState *bastate,
                                         const char *data,
                                         dma_addr_t data_len)
 {
@@ -XXX,XX +XXX,XX @@ e1000e_write_packet_to_guest(E1000ECore *core, struct NetRxPkt *pkt,
 
     do {
         hwaddr ba[MAX_PS_BUFFERS];
-        e1000e_ba_state bastate = { { 0 } };
+        E1000EBAState bastate = { { 0 } };
         bool is_last = false;
 
         desc_size = total_size - desc_offset;
@@ -XXX,XX +XXX,XX @@ e1000e_write_packet_to_guest(E1000ECore *core, struct NetRxPkt *pkt,
                             iov_copy = MIN(ps_hdr_len - ps_hdr_copied,
                                            iov->iov_len - iov_ofs);
 
-                            e1000e_write_hdr_to_rx_buffers(core, ba, &bastate,
-                                                      iov->iov_base, iov_copy);
+                            e1000e_write_hdr_frag_to_rx_buffers(core, ba,
+                                                                &bastate,
+                                                                iov->iov_base,
+                                                                iov_copy);
 
                             copy_size -= iov_copy;
                             ps_hdr_copied += iov_copy;
@@ -XXX,XX +XXX,XX @@ e1000e_write_packet_to_guest(E1000ECore *core, struct NetRxPkt *pkt,
                     } else {
                         /* Leave buffer 0 of each descriptor except first */
                         /* empty as per spec 7.1.5.1                      */
-                        e1000e_write_hdr_to_rx_buffers(core, ba, &bastate,
-                                                       NULL, 0);
+                        e1000e_write_hdr_frag_to_rx_buffers(core, ba, &bastate,
+                                                            NULL, 0);
                     }
                 }
 
-- 
2.7.4

From: Ilya Maximets <i.maximets@ovn.org>

AF_XDP is a network socket family that allows communication directly
with the network device driver in the kernel, bypassing most or all
of the kernel networking stack.  In the essence, the technology is
pretty similar to netmap.  But, unlike netmap, AF_XDP is Linux-native
and works with any network interfaces without driver modifications.
Unlike vhost-based backends (kernel, user, vdpa), AF_XDP doesn't
require access to character devices or unix sockets.  Only access to
the network interface itself is necessary.

This patch implements a network backend that communicates with the
kernel by creating an AF_XDP socket.  A chunk of userspace memory
is shared between QEMU and the host kernel.  4 ring buffers (Tx, Rx,
Fill and Completion) are placed in that memory along with a pool of
memory buffers for the packet data.  Data transmission is done by
allocating one of the buffers, copying packet data into it and
placing the pointer into Tx ring.  After transmission, device will
return the buffer via Completion ring.  On Rx, device will take
a buffer form a pre-populated Fill ring, write the packet data into
it and place the buffer into Rx ring.

AF_XDP network backend takes on the communication with the host
kernel and the network interface and forwards packets to/from the
peer device in QEMU.

Usage example:

-device virtio-net-pci,netdev=guest1,mac=00:16:35:AF:AA:5C
  -netdev af-xdp,ifname=ens6f1np1,id=guest1,mode=native,queues=1

XDP program bridges the socket with a network interface.  It can be
attached to the interface in 2 different modes:

1. skb - this mode should work for any interface and doesn't require
         driver support.  With a caveat of lower performance.

2. native - this does require support from the driver and allows to
            bypass skb allocation in the kernel and potentially use
            zero-copy while getting packets in/out userspace.

By default, QEMU will try to use native mode and fall back to skb.
Mode can be forced via 'mode' option.  To force 'copy' even in native
mode, use 'force-copy=on' option.  This might be useful if there is
some issue with the driver.

Option 'queues=N' allows to specify how many device queues should
be open.  Note that all the queues that are not open are still
functional and can receive traffic, but it will not be delivered to
QEMU.  So, the number of device queues should generally match the
QEMU configuration, unless the device is shared with something
else and the traffic re-direction to appropriate queues is correctly
configured on a device level (e.g. with ethtool -N).
'start-queue=M' option can be used to specify from which queue id
QEMU should start configuring 'N' queues.  It might also be necessary
to use this option with certain NICs, e.g. MLX5 NICs.  See the docs
for examples.

In a general case QEMU will need CAP_NET_ADMIN and CAP_SYS_ADMIN
or CAP_BPF capabilities in order to load default XSK/XDP programs to
the network interface and configure BPF maps.  It is possible, however,
to run with no capabilities.  For that to work, an external process
with enough capabilities will need to pre-load default XSK program,
create AF_XDP sockets and pass their file descriptors to QEMU process
on startup via 'sock-fds' option.  Network backend will need to be
configured with 'inhibit=on' to avoid loading of the program.
QEMU will need 32 MB of locked memory (RLIMIT_MEMLOCK) per queue
or CAP_IPC_LOCK.

There are few performance challenges with the current network backends.

First is that they do not support IO threads.  This means that data
path is handled by the main thread in QEMU and may slow down other
work or may be slowed down by some other work.  This also means that
taking advantage of multi-queue is generally not possible today.

Another thing is that data path is going through the device emulation
code, which is not really optimized for performance.  The fastest
"frontend" device is virtio-net.  But it's not optimized for heavy
traffic either, because it expects such use-cases to be handled via
some implementation of vhost (user, kernel, vdpa).  In practice, we
have virtio notifications and rcu lock/unlock on a per-packet basis
and not very efficient accesses to the guest memory.  Communication
channels between backend and frontend devices do not allow passing
more than one packet at a time as well.

Some of these challenges can be avoided in the future by adding better
batching into device emulation or by implementing vhost-af-xdp variant.

There are also a few kernel limitations.  AF_XDP sockets do not
support any kinds of checksum or segmentation offloading.  Buffers
are limited to a page size (4K), i.e. MTU is limited.  Multi-buffer
support implementation for AF_XDP is in progress, but not ready yet.
Also, transmission in all non-zero-copy modes is synchronous, i.e.
done in a syscall.  That doesn't allow high packet rates on virtual
interfaces.

However, keeping in mind all of these challenges, current implementation
of the AF_XDP backend shows a decent performance while running on top
of a physical NIC with zero-copy support.

Test setup:

2 VMs running on 2 physical hosts connected via ConnectX6-Dx card.
Network backend is configured to open the NIC directly in native mode.
The driver supports zero-copy.  NIC is configured to use 1 queue.

Inside a VM - iperf3 for basic TCP performance testing and dpdk-testpmd
for PPS testing.

iperf3 result:
 TCP stream      : 19.1 Gbps

dpdk-testpmd (single queue, single CPU core, 64 B packets) results:
 Tx only         : 3.4 Mpps
 Rx only         : 2.0 Mpps
 L2 FWD Loopback : 1.5 Mpps

In skb mode the same setup shows much lower performance, similar to
the setup where pair of physical NICs is replaced with veth pair:

iperf3 result:
  TCP stream      : 9 Gbps

dpdk-testpmd (single queue, single CPU core, 64 B packets) results:
  Tx only         : 1.2 Mpps
  Rx only         : 1.0 Mpps
  L2 FWD Loopback : 0.7 Mpps

Results in skb mode or over the veth are close to results of a tap
backend with vhost=on and disabled segmentation offloading bridged
with a NIC.

diff --git a/MAINTAINERS b/MAINTAINERS
index XXXXXXX..XXXXXXX 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -XXX,XX +XXX,XX @@ W: http://info.iet.unipi.it/~luigi/netmap/
 S: Maintained
 F: net/netmap.c
 
+AF_XDP network backend
+R: Ilya Maximets <i.maximets@ovn.org>
+F: net/af-xdp.c
+
 Host Memory Backends
 M: David Hildenbrand <david@redhat.com>
 M: Igor Mammedov <imammedo@redhat.com>
diff --git a/hmp-commands.hx b/hmp-commands.hx
index XXXXXXX..XXXXXXX 100644
--- a/hmp-commands.hx
+++ b/hmp-commands.hx
@@ -XXX,XX +XXX,XX @@ ERST
         .name       = "netdev_add",
         .args_type  = "netdev:O",
         .params     = "[user|tap|socket|stream|dgram|vde|bridge|hubport|netmap|vhost-user"
+#ifdef CONFIG_AF_XDP
+                      "|af-xdp"
+#endif
 #ifdef CONFIG_VMNET
                       "|vmnet-host|vmnet-shared|vmnet-bridged"
 #endif
diff --git a/meson.build b/meson.build
index XXXXXXX..XXXXXXX 100644
--- a/meson.build
+++ b/meson.build
@@ -XXX,XX +XXX,XX @@ if libbpf.found() and not cc.links('''
   endif
 endif
 
+# libxdp
+libxdp = not_found
+if not get_option('af_xdp').auto() or have_system
+    libxdp = dependency('libxdp', required: get_option('af_xdp'),
+                        version: '>=1.4.0', method: 'pkg-config')
+endif
+
 # libdw
 libdw = not_found
 if not get_option('libdw').auto() or \
@@ -XXX,XX +XXX,XX @@ config_host_data.set('CONFIG_HEXAGON_IDEF_PARSER', get_option('hexagon_idef_pars
 config_host_data.set('CONFIG_LIBATTR', have_old_libattr)
 config_host_data.set('CONFIG_LIBCAP_NG', libcap_ng.found())
 config_host_data.set('CONFIG_EBPF', libbpf.found())
+config_host_data.set('CONFIG_AF_XDP', libxdp.found())
 config_host_data.set('CONFIG_LIBDAXCTL', libdaxctl.found())
 config_host_data.set('CONFIG_LIBISCSI', libiscsi.found())
 config_host_data.set('CONFIG_LIBNFS', libnfs.found())
@@ -XXX,XX +XXX,XX @@ summary_info = {}
 if targetos == 'darwin'
   summary_info += {'vmnet.framework support': vmnet}
 endif
+summary_info += {'AF_XDP support':    libxdp}
 summary_info += {'slirp support':     slirp}
 summary_info += {'vde support':       vde}
 summary_info += {'netmap support':    have_netmap}
diff --git a/meson_options.txt b/meson_options.txt
index XXXXXXX..XXXXXXX 100644
--- a/meson_options.txt
+++ b/meson_options.txt
@@ -XXX,XX +XXX,XX @@ option('avx512bw', type: 'feature', value: 'auto',
 option('keyring', type: 'feature', value: 'auto',
        description: 'Linux keyring support')
 
+option('af_xdp', type : 'feature', value : 'auto',
+       description: 'AF_XDP network backend support')
 option('attr', type : 'feature', value : 'auto',
        description: 'attr/xattr support')
 option('auth_pam', type : 'feature', value : 'auto',
diff --git a/net/af-xdp.c b/net/af-xdp.c
new file mode 100644
index XXXXXXX..XXXXXXX
--- /dev/null
+++ b/net/af-xdp.c
@@ -XXX,XX +XXX,XX @@
+/*
+ * AF_XDP network backend.
+ *
+ * Copyright (c) 2023 Red Hat, Inc.
+ *
+ * Authors:
+ *  Ilya Maximets <i.maximets@ovn.org>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+
+
+#include "qemu/osdep.h"
+#include <bpf/bpf.h>
+#include <inttypes.h>
+#include <linux/if_link.h>
+#include <linux/if_xdp.h>
+#include <net/if.h>
+#include <xdp/xsk.h>
+
+#include "clients.h"
+#include "monitor/monitor.h"
+#include "net/net.h"
+#include "qapi/error.h"
+#include "qemu/cutils.h"
+#include "qemu/error-report.h"
+#include "qemu/iov.h"
+#include "qemu/main-loop.h"
+#include "qemu/memalign.h"
+
+
+typedef struct AFXDPState {
+    NetClientState       nc;
+
+    struct xsk_socket    *xsk;
+    struct xsk_ring_cons rx;
+    struct xsk_ring_prod tx;
+    struct xsk_ring_cons cq;
+    struct xsk_ring_prod fq;
+
+    char                 ifname[IFNAMSIZ];
+    int                  ifindex;
+    bool                 read_poll;
+    bool                 write_poll;
+    uint32_t             outstanding_tx;
+
+    uint64_t             *pool;
+    uint32_t             n_pool;
+    char                 *buffer;
+    struct xsk_umem      *umem;
+
+    uint32_t             n_queues;
+    uint32_t             xdp_flags;
+    bool                 inhibit;
+} AFXDPState;
+
+#define AF_XDP_BATCH_SIZE 64
+
+static void af_xdp_send(void *opaque);
+static void af_xdp_writable(void *opaque);
+
+/* Set the event-loop handlers for the af-xdp backend. */
+static void af_xdp_update_fd_handler(AFXDPState *s)
+{
+    qemu_set_fd_handler(xsk_socket__fd(s->xsk),
+                        s->read_poll ? af_xdp_send : NULL,
+                        s->write_poll ? af_xdp_writable : NULL,
+                        s);
+}
+
+/* Update the read handler. */
+static void af_xdp_read_poll(AFXDPState *s, bool enable)
+{
+    if (s->read_poll != enable) {
+        s->read_poll = enable;
+        af_xdp_update_fd_handler(s);
+    }
+}
+
+/* Update the write handler. */
+static void af_xdp_write_poll(AFXDPState *s, bool enable)
+{
+    if (s->write_poll != enable) {
+        s->write_poll = enable;
+        af_xdp_update_fd_handler(s);
+    }
+}
+
+static void af_xdp_poll(NetClientState *nc, bool enable)
+{
+    AFXDPState *s = DO_UPCAST(AFXDPState, nc, nc);
+
+    if (s->read_poll != enable || s->write_poll != enable) {
+        s->write_poll = enable;
+        s->read_poll  = enable;
+        af_xdp_update_fd_handler(s);
+    }
+}
+
+static void af_xdp_complete_tx(AFXDPState *s)
+{
+    uint32_t idx = 0;
+    uint32_t done, i;
+    uint64_t *addr;
+
+    done = xsk_ring_cons__peek(&s->cq, XSK_RING_CONS__DEFAULT_NUM_DESCS, &idx);
+
+    for (i = 0; i < done; i++) {
+        addr = (void *) xsk_ring_cons__comp_addr(&s->cq, idx++);
+        s->pool[s->n_pool++] = *addr;
+        s->outstanding_tx--;
+    }
+
+    if (done) {
+        xsk_ring_cons__release(&s->cq, done);
+    }
+}
+
+/*
+ * The fd_write() callback, invoked if the fd is marked as writable
+ * after a poll.
+ */
+static void af_xdp_writable(void *opaque)
+{
+    AFXDPState *s = opaque;
+
+    /* Try to recover buffers that are already sent. */
+    af_xdp_complete_tx(s);
+
+    /*
+     * Unregister the handler, unless we still have packets to transmit
+     * and kernel needs a wake up.
+     */
+    if (!s->outstanding_tx || !xsk_ring_prod__needs_wakeup(&s->tx)) {
+        af_xdp_write_poll(s, false);
+    }
+
+    /* Flush any buffered packets. */
+    qemu_flush_queued_packets(&s->nc);
+}
+
+static ssize_t af_xdp_receive(NetClientState *nc,
+                              const uint8_t *buf, size_t size)
+{
+    AFXDPState *s = DO_UPCAST(AFXDPState, nc, nc);
+    struct xdp_desc *desc;
+    uint32_t idx;
+    void *data;
+
+    /* Try to recover buffers that are already sent. */
+    af_xdp_complete_tx(s);
+
+    if (size > XSK_UMEM__DEFAULT_FRAME_SIZE) {
+        /* We can't transmit packet this size... */
+        return size;
+    }
+
+    if (!s->n_pool || !xsk_ring_prod__reserve(&s->tx, 1, &idx)) {
+        /*
+         * Out of buffers or space in tx ring.  Poll until we can write.
+         * This will also kick the Tx, if it was waiting on CQ.
+         */
+        af_xdp_write_poll(s, true);
+        return 0;
+    }
+
+    desc = xsk_ring_prod__tx_desc(&s->tx, idx);
+    desc->addr = s->pool[--s->n_pool];
+    desc->len = size;
+
+    data = xsk_umem__get_data(s->buffer, desc->addr);
+    memcpy(data, buf, size);
+
+    xsk_ring_prod__submit(&s->tx, 1);
+    s->outstanding_tx++;
+
+    if (xsk_ring_prod__needs_wakeup(&s->tx)) {
+        af_xdp_write_poll(s, true);
+    }
+
+    return size;
+}
+
+/*
+ * Complete a previous send (backend --> guest) and enable the
+ * fd_read callback.
+ */
+static void af_xdp_send_completed(NetClientState *nc, ssize_t len)
+{
+    AFXDPState *s = DO_UPCAST(AFXDPState, nc, nc);
+
+    af_xdp_read_poll(s, true);
+}
+
+static void af_xdp_fq_refill(AFXDPState *s, uint32_t n)
+{
+    uint32_t i, idx = 0;
+
+    /* Leave one packet for Tx, just in case. */
+    if (s->n_pool < n + 1) {
+        n = s->n_pool;
+    }
+
+    if (!n || !xsk_ring_prod__reserve(&s->fq, n, &idx)) {
+        return;
+    }
+
+    for (i = 0; i < n; i++) {
+        *xsk_ring_prod__fill_addr(&s->fq, idx++) = s->pool[--s->n_pool];
+    }
+    xsk_ring_prod__submit(&s->fq, n);
+
+    if (xsk_ring_prod__needs_wakeup(&s->fq)) {
+        /* Receive was blocked by not having enough buffers.  Wake it up. */
+        af_xdp_read_poll(s, true);
+    }
+}
+
+static void af_xdp_send(void *opaque)
+{
+    uint32_t i, n_rx, idx = 0;
+    AFXDPState *s = opaque;
+
+    n_rx = xsk_ring_cons__peek(&s->rx, AF_XDP_BATCH_SIZE, &idx);
+    if (!n_rx) {
+        return;
+    }
+
+    for (i = 0; i < n_rx; i++) {
+        const struct xdp_desc *desc;
+        struct iovec iov;
+
+        desc = xsk_ring_cons__rx_desc(&s->rx, idx++);
+
+        iov.iov_base = xsk_umem__get_data(s->buffer, desc->addr);
+        iov.iov_len = desc->len;
+
+        s->pool[s->n_pool++] = desc->addr;
+
+        if (!qemu_sendv_packet_async(&s->nc, &iov, 1,
+                                     af_xdp_send_completed)) {
+            /*
+             * The peer does not receive anymore.  Packet is queued, stop
+             * reading from the backend until af_xdp_send_completed().
+             */
+            af_xdp_read_poll(s, false);
+
+            /* Return unused descriptors to not break the ring cache. */
+            xsk_ring_cons__cancel(&s->rx, n_rx - i - 1);
+            n_rx = i + 1;
+            break;
+        }
+    }
+
+    /* Release actually sent descriptors and try to re-fill. */
+    xsk_ring_cons__release(&s->rx, n_rx);
+    af_xdp_fq_refill(s, AF_XDP_BATCH_SIZE);
+}
+
+/* Flush and close. */
+static void af_xdp_cleanup(NetClientState *nc)
+{
+    AFXDPState *s = DO_UPCAST(AFXDPState, nc, nc);
+
+    qemu_purge_queued_packets(nc);
+
+    af_xdp_poll(nc, false);
+
+    xsk_socket__delete(s->xsk);
+    s->xsk = NULL;
+    g_free(s->pool);
+    s->pool = NULL;
+    xsk_umem__delete(s->umem);
+    s->umem = NULL;
+    qemu_vfree(s->buffer);
+    s->buffer = NULL;
+
+    /* Remove the program if it's the last open queue. */
+    if (!s->inhibit && nc->queue_index == s->n_queues - 1 && s->xdp_flags
+        && bpf_xdp_detach(s->ifindex, s->xdp_flags, NULL) != 0) {
+        fprintf(stderr,
+                "af-xdp: unable to remove XDP program from '%s', ifindex: %d\n",
+                s->ifname, s->ifindex);
+    }
+}
+
+static int af_xdp_umem_create(AFXDPState *s, int sock_fd, Error **errp)
+{
+    struct xsk_umem_config config = {
+        .fill_size = XSK_RING_PROD__DEFAULT_NUM_DESCS,
+        .comp_size = XSK_RING_CONS__DEFAULT_NUM_DESCS,
+        .frame_size = XSK_UMEM__DEFAULT_FRAME_SIZE,
+        .frame_headroom = 0,
+    };
+    uint64_t n_descs;
+    uint64_t size;
+    int64_t i;
+    int ret;
+
+    /* Number of descriptors if all 4 queues (rx, tx, cq, fq) are full. */
+    n_descs = (XSK_RING_PROD__DEFAULT_NUM_DESCS
+               + XSK_RING_CONS__DEFAULT_NUM_DESCS) * 2;
+    size = n_descs * XSK_UMEM__DEFAULT_FRAME_SIZE;
+
+    s->buffer = qemu_memalign(qemu_real_host_page_size(), size);
+    memset(s->buffer, 0, size);
+
+    if (sock_fd < 0) {
+        ret = xsk_umem__create(&s->umem, s->buffer, size,
+                               &s->fq, &s->cq, &config);
+    } else {
+        ret = xsk_umem__create_with_fd(&s->umem, sock_fd, s->buffer, size,
+                                       &s->fq, &s->cq, &config);
+    }
+
+    if (ret) {
+        qemu_vfree(s->buffer);
+        error_setg_errno(errp, errno,
+                         "failed to create umem for %s queue_index: %d",
+                         s->ifname, s->nc.queue_index);
+        return -1;
+    }
+
+    s->pool = g_new(uint64_t, n_descs);
+    /* Fill the pool in the opposite order, because it's a LIFO queue. */
+    for (i = n_descs; i >= 0; i--) {
+        s->pool[i] = i * XSK_UMEM__DEFAULT_FRAME_SIZE;
+    }
+    s->n_pool = n_descs;
+
+    af_xdp_fq_refill(s, XSK_RING_PROD__DEFAULT_NUM_DESCS);
+
+    return 0;
+}
+
+static int af_xdp_socket_create(AFXDPState *s,
+                                const NetdevAFXDPOptions *opts, Error **errp)
+{
+    struct xsk_socket_config cfg = {
+        .rx_size = XSK_RING_CONS__DEFAULT_NUM_DESCS,
+        .tx_size = XSK_RING_PROD__DEFAULT_NUM_DESCS,
+        .libxdp_flags = 0,
+        .bind_flags = XDP_USE_NEED_WAKEUP,
+        .xdp_flags = XDP_FLAGS_UPDATE_IF_NOEXIST,
+    };
+    int queue_id, error = 0;
+
+    s->inhibit = opts->has_inhibit && opts->inhibit;
+    if (s->inhibit) {
+        cfg.libxdp_flags |= XSK_LIBXDP_FLAGS__INHIBIT_PROG_LOAD;
+    }
+
+    if (opts->has_force_copy && opts->force_copy) {
+        cfg.bind_flags |= XDP_COPY;
+    }
+
+    queue_id = s->nc.queue_index;
+    if (opts->has_start_queue && opts->start_queue > 0) {
+        queue_id += opts->start_queue;
+    }
+
+    if (opts->has_mode) {
+        /* Specific mode requested. */
+        cfg.xdp_flags |= (opts->mode == AFXDP_MODE_NATIVE)
+                         ? XDP_FLAGS_DRV_MODE : XDP_FLAGS_SKB_MODE;
+        if (xsk_socket__create(&s->xsk, s->ifname, queue_id,
+                               s->umem, &s->rx, &s->tx, &cfg)) {
+            error = errno;
+        }
+    } else {
+        /* No mode requested, try native first. */
+        cfg.xdp_flags |= XDP_FLAGS_DRV_MODE;
+
+        if (xsk_socket__create(&s->xsk, s->ifname, queue_id,
+                               s->umem, &s->rx, &s->tx, &cfg)) {
+            /* Can't use native mode, try skb. */
+            cfg.xdp_flags &= ~XDP_FLAGS_DRV_MODE;
+            cfg.xdp_flags |= XDP_FLAGS_SKB_MODE;
+
+            if (xsk_socket__create(&s->xsk, s->ifname, queue_id,
+                                   s->umem, &s->rx, &s->tx, &cfg)) {
+                error = errno;
+            }
+        }
+    }
+
+    if (error) {
+        error_setg_errno(errp, error,
+                         "failed to create AF_XDP socket for %s queue_id: %d",
+                         s->ifname, queue_id);
+        return -1;
+    }
+
+    s->xdp_flags = cfg.xdp_flags;
+
+    return 0;
+}
+
+/* NetClientInfo methods. */
+static NetClientInfo net_af_xdp_info = {
+    .type = NET_CLIENT_DRIVER_AF_XDP,
+    .size = sizeof(AFXDPState),
+    .receive = af_xdp_receive,
+    .poll = af_xdp_poll,
+    .cleanup = af_xdp_cleanup,
+};
+
+static int *parse_socket_fds(const char *sock_fds_str,
+                             int64_t n_expected, Error **errp)
+{
+    gchar **substrings = g_strsplit(sock_fds_str, ":", -1);
+    int64_t i, n_sock_fds = g_strv_length(substrings);
+    int *sock_fds = NULL;
+
+    if (n_sock_fds != n_expected) {
+        error_setg(errp, "expected %"PRIi64" socket fds, got %"PRIi64,
+                   n_expected, n_sock_fds);
+        goto exit;
+    }
+
+    sock_fds = g_new(int, n_sock_fds);
+
+    for (i = 0; i < n_sock_fds; i++) {
+        sock_fds[i] = monitor_fd_param(monitor_cur(), substrings[i], errp);
+        if (sock_fds[i] < 0) {
+            g_free(sock_fds);
+            sock_fds = NULL;
+            goto exit;
+        }
+    }
+
+exit:
+    g_strfreev(substrings);
+    return sock_fds;
+}
+
+/*
+ * The exported init function.
+ *
+ * ... -netdev af-xdp,ifname="..."
+ */
+int net_init_af_xdp(const Netdev *netdev,
+                    const char *name, NetClientState *peer, Error **errp)
+{
+    const NetdevAFXDPOptions *opts = &netdev->u.af_xdp;
+    NetClientState *nc, *nc0 = NULL;
+    unsigned int ifindex;
+    uint32_t prog_id = 0;
+    int *sock_fds = NULL;
+    int64_t i, queues;
+    Error *err = NULL;
+    AFXDPState *s;
+
+    ifindex = if_nametoindex(opts->ifname);
+    if (!ifindex) {
+        error_setg_errno(errp, errno, "failed to get ifindex for '%s'",
+                         opts->ifname);
+        return -1;
+    }
+
+    queues = opts->has_queues ? opts->queues : 1;
+    if (queues < 1) {
+        error_setg(errp, "invalid number of queues (%" PRIi64 ") for '%s'",
+                   queues, opts->ifname);
+        return -1;
+    }
+
+    if ((opts->has_inhibit && opts->inhibit) != !!opts->sock_fds) {
+        error_setg(errp, "'inhibit=on' requires 'sock-fds' and vice versa");
+        return -1;
+    }
+
+    if (opts->sock_fds) {
+        sock_fds = parse_socket_fds(opts->sock_fds, queues, errp);
+        if (!sock_fds) {
+            return -1;
+        }
+    }
+
+    for (i = 0; i < queues; i++) {
+        nc = qemu_new_net_client(&net_af_xdp_info, peer, "af-xdp", name);
+        qemu_set_info_str(nc, "af-xdp%"PRIi64" to %s", i, opts->ifname);
+        nc->queue_index = i;
+
+        if (!nc0) {
+            nc0 = nc;
+        }
+
+        s = DO_UPCAST(AFXDPState, nc, nc);
+
+        pstrcpy(s->ifname, sizeof(s->ifname), opts->ifname);
+        s->ifindex = ifindex;
+        s->n_queues = queues;
+
+        if (af_xdp_umem_create(s, sock_fds ? sock_fds[i] : -1, errp)
+            || af_xdp_socket_create(s, opts, errp)) {
+            /* Make sure the XDP program will be removed. */
+            s->n_queues = i;
+            error_propagate(errp, err);
+            goto err;
+        }
+    }
+
+    if (nc0) {
+        s = DO_UPCAST(AFXDPState, nc, nc0);
+        if (bpf_xdp_query_id(s->ifindex, s->xdp_flags, &prog_id) || !prog_id) {
+            error_setg_errno(errp, errno,
+                             "no XDP program loaded on '%s', ifindex: %d",
+                             s->ifname, s->ifindex);
+            goto err;
+        }
+    }
+
+    af_xdp_read_poll(s, true); /* Initially only poll for reads. */
+
+    return 0;
+
+err:
+    g_free(sock_fds);
+    if (nc0) {
+        qemu_del_net_client(nc0);
+    }
+
+    return -1;
+}
diff --git a/net/clients.h b/net/clients.h
index XXXXXXX..XXXXXXX 100644
--- a/net/clients.h
+++ b/net/clients.h
@@ -XXX,XX +XXX,XX @@ int net_init_netmap(const Netdev *netdev, const char *name,
                     NetClientState *peer, Error **errp);
 #endif
 
+#ifdef CONFIG_AF_XDP
+int net_init_af_xdp(const Netdev *netdev, const char *name,
+                    NetClientState *peer, Error **errp);
+#endif
+
 int net_init_vhost_user(const Netdev *netdev, const char *name,
                         NetClientState *peer, Error **errp);
 
diff --git a/net/meson.build b/net/meson.build
index XXXXXXX..XXXXXXX 100644
--- a/net/meson.build
+++ b/net/meson.build
@@ -XXX,XX +XXX,XX @@ system_ss.add(when: vde, if_true: files('vde.c'))
 if have_netmap
   system_ss.add(files('netmap.c'))
 endif
+
+system_ss.add(when: libxdp, if_true: files('af-xdp.c'))
+
 if have_vhost_net_user
   system_ss.add(when: 'CONFIG_VIRTIO_NET', if_true: files('vhost-user.c'), if_false: files('vhost-user-stub.c'))
   system_ss.add(when: 'CONFIG_ALL', if_true: files('vhost-user-stub.c'))
diff --git a/net/net.c b/net/net.c
index XXXXXXX..XXXXXXX 100644
--- a/net/net.c
+++ b/net/net.c
@@ -XXX,XX +XXX,XX @@ static int (* const net_client_init_fun[NET_CLIENT_DRIVER__MAX])(
 #ifdef CONFIG_NETMAP
         [NET_CLIENT_DRIVER_NETMAP]    = net_init_netmap,
 #endif
+#ifdef CONFIG_AF_XDP
+        [NET_CLIENT_DRIVER_AF_XDP]    = net_init_af_xdp,
+#endif
 #ifdef CONFIG_NET_BRIDGE
         [NET_CLIENT_DRIVER_BRIDGE]    = net_init_bridge,
 #endif
@@ -XXX,XX +XXX,XX @@ void show_netdevs(void)
 #ifdef CONFIG_NETMAP
         "netmap",
 #endif
+#ifdef CONFIG_AF_XDP
+        "af-xdp",
+#endif
 #ifdef CONFIG_POSIX
         "vhost-user",
 #endif
diff --git a/qapi/net.json b/qapi/net.json
index XXXXXXX..XXXXXXX 100644
--- a/qapi/net.json
+++ b/qapi/net.json
@@ -XXX,XX +XXX,XX @@
     '*devname':    'str' } }
 
 ##
+# @AFXDPMode:
+#
+# Attach mode for a default XDP program
+#
+# @skb: generic mode, no driver support necessary
+#
+# @native: DRV mode, program is attached to a driver, packets are passed to
+#     the socket without allocation of skb.
+#
+# Since: 8.2
+##
+{ 'enum': 'AFXDPMode',
+  'data': [ 'native', 'skb' ],
+  'if': 'CONFIG_AF_XDP' }
+
+##
+# @NetdevAFXDPOptions:
+#
+# AF_XDP network backend
+#
+# @ifname: The name of an existing network interface.
+#
+# @mode: Attach mode for a default XDP program.  If not specified, then
+#     'native' will be tried first, then 'skb'.
+#
+# @force-copy: Force XDP copy mode even if device supports zero-copy.
+#     (default: false)
+#
+# @queues: number of queues to be used for multiqueue interfaces (default: 1).
+#
+# @start-queue: Use @queues starting from this queue number (default: 0).
+#
+# @inhibit: Don't load a default XDP program, use one already loaded to
+#     the interface (default: false).  Requires @sock-fds.
+#
+# @sock-fds: A colon (:) separated list of file descriptors for already open
+#     but not bound AF_XDP sockets in the queue order.  One fd per queue.
+#     These descriptors should already be added into XDP socket map for
+#     corresponding queues.  Requires @inhibit.
+#
+# Since: 8.2
+##
+{ 'struct': 'NetdevAFXDPOptions',
+  'data': {
+    'ifname':       'str',
+    '*mode':        'AFXDPMode',
+    '*force-copy':  'bool',
+    '*queues':      'int',
+    '*start-queue': 'int',
+    '*inhibit':     'bool',
+    '*sock-fds':    'str' },
+  'if': 'CONFIG_AF_XDP' }
+
+##
 # @NetdevVhostUserOptions:
 #
 # Vhost-user network backend
@@ -XXX,XX +XXX,XX @@
 # @vmnet-bridged: since 7.1
 # @stream: since 7.2
 # @dgram: since 7.2
+# @af-xdp: since 8.2
 #
 # Since: 2.7
 ##
@@ -XXX,XX +XXX,XX @@
   'data': [ 'none', 'nic', 'user', 'tap', 'l2tpv3', 'socket', 'stream',
             'dgram', 'vde', 'bridge', 'hubport', 'netmap', 'vhost-user',
             'vhost-vdpa',
+            { 'name': 'af-xdp', 'if': 'CONFIG_AF_XDP' },
             { 'name': 'vmnet-host', 'if': 'CONFIG_VMNET' },
             { 'name': 'vmnet-shared', 'if': 'CONFIG_VMNET' },
             { 'name': 'vmnet-bridged', 'if': 'CONFIG_VMNET' }] }
@@ -XXX,XX +XXX,XX @@
     'bridge':   'NetdevBridgeOptions',
     'hubport':  'NetdevHubPortOptions',
     'netmap':   'NetdevNetmapOptions',
+    'af-xdp':   { 'type': 'NetdevAFXDPOptions',
+                  'if': 'CONFIG_AF_XDP' },
     'vhost-user': 'NetdevVhostUserOptions',
     'vhost-vdpa': 'NetdevVhostVDPAOptions',
     'vmnet-host': { 'type': 'NetdevVmnetHostOptions',
diff --git a/qemu-options.hx b/qemu-options.hx
index XXXXXXX..XXXXXXX 100644
--- a/qemu-options.hx
+++ b/qemu-options.hx
@@ -XXX,XX +XXX,XX @@ DEF("netdev", HAS_ARG, QEMU_OPTION_netdev,
     "                VALE port (created on the fly) called 'name' ('nmname' is name of the \n"
     "                netmap device, defaults to '/dev/netmap')\n"
 #endif
+#ifdef CONFIG_AF_XDP
+    "-netdev af-xdp,id=str,ifname=name[,mode=native|skb][,force-copy=on|off]\n"
+    "         [,queues=n][,start-queue=m][,inhibit=on|off][,sock-fds=x:y:...:z]\n"
+    "                attach to the existing network interface 'name' with AF_XDP socket\n"
+    "                use 'mode=MODE' to specify an XDP program attach mode\n"
+    "                use 'force-copy=on|off' to force XDP copy mode even if device supports zero-copy (default: off)\n"
+    "                use 'inhibit=on|off' to inhibit loading of a default XDP program (default: off)\n"
+    "                with inhibit=on,\n"
+    "                  use 'sock-fds' to provide file descriptors for already open AF_XDP sockets\n"
+    "                  added to a socket map in XDP program.  One socket per queue.\n"
+    "                use 'queues=n' to specify how many queues of a multiqueue interface should be used\n"
+    "                use 'start-queue=m' to specify the first queue that should be used\n"
+#endif
 #ifdef CONFIG_POSIX
     "-netdev vhost-user,id=str,chardev=dev[,vhostforce=on|off]\n"
     "                configure a vhost-user network, backed by a chardev 'dev'\n"
@@ -XXX,XX +XXX,XX @@ DEF("nic", HAS_ARG, QEMU_OPTION_nic,
 #ifdef CONFIG_NETMAP
     "netmap|"
 #endif
+#ifdef CONFIG_AF_XDP
+    "af-xdp|"
+#endif
 #ifdef CONFIG_POSIX
     "vhost-user|"
 #endif
@@ -XXX,XX +XXX,XX @@ DEF("net", HAS_ARG, QEMU_OPTION_net,
 #ifdef CONFIG_NETMAP
     "netmap|"
 #endif
+#ifdef CONFIG_AF_XDP
+    "af-xdp|"
+#endif
 #ifdef CONFIG_VMNET
     "vmnet-host|vmnet-shared|vmnet-bridged|"
 #endif
@@ -XXX,XX +XXX,XX @@ DEF("net", HAS_ARG, QEMU_OPTION_net,
     "                old way to initialize a host network interface\n"
     "                (use the -netdev option if possible instead)\n", QEMU_ARCH_ALL)
 SRST
-``-nic [tap|bridge|user|l2tpv3|vde|netmap|vhost-user|socket][,...][,mac=macaddr][,model=mn]``
+``-nic [tap|bridge|user|l2tpv3|vde|netmap|af-xdp|vhost-user|socket][,...][,mac=macaddr][,model=mn]``
     This option is a shortcut for configuring both the on-board
     (default) guest NIC hardware and the host network backend in one go.
     The host backend options are the same as with the corresponding
@@ -XXX,XX +XXX,XX @@ SRST
         # launch QEMU instance
         |qemu_system| linux.img -nic vde,sock=/tmp/myswitch
 
+``-netdev af-xdp,id=str,ifname=name[,mode=native|skb][,force-copy=on|off][,queues=n][,start-queue=m][,inhibit=on|off][,sock-fds=x:y:...:z]``
+    Configure AF_XDP backend to connect to a network interface 'name'
+    using AF_XDP socket.  A specific program attach mode for a default
+    XDP program can be forced with 'mode', defaults to best-effort,
+    where the likely most performant mode will be in use.  Number of queues
+    'n' should generally match the number or queues in the interface,
+    defaults to 1.  Traffic arriving on non-configured device queues will
+    not be delivered to the network backend.
+
+    .. parsed-literal::
+
+        # set number of queues to 4
+        ethtool -L eth0 combined 4
+        # launch QEMU instance
+        |qemu_system| linux.img -device virtio-net-pci,netdev=n1 \\
+            -netdev af-xdp,id=n1,ifname=eth0,queues=4
+
+    'start-queue' option can be specified if a particular range of queues
+    [m, m + n] should be in use.  For example, this is may be necessary in
+    order to use certain NICs in native mode.  Kernel allows the driver to
+    create a separate set of XDP queues on top of regular ones, and only
+    these queues can be used for AF_XDP sockets.  NICs that work this way
+    may also require an additional traffic redirection with ethtool to these
+    special queues.
+
+    .. parsed-literal::
+
+        # set number of queues to 1
+        ethtool -L eth0 combined 1
+        # redirect all the traffic to the second queue (id: 1)
+        # note: drivers may require non-empty key/mask pair.
+        ethtool -N eth0 flow-type ether \\
+            dst 00:00:00:00:00:00 m FF:FF:FF:FF:FF:FE action 1
+        ethtool -N eth0 flow-type ether \\
+            dst 00:00:00:00:00:01 m FF:FF:FF:FF:FF:FE action 1
+        # launch QEMU instance
+        |qemu_system| linux.img -device virtio-net-pci,netdev=n1 \\
+            -netdev af-xdp,id=n1,ifname=eth0,queues=1,start-queue=1
+
+    XDP program can also be loaded externally.  In this case 'inhibit' option
+    should be set to 'on' and 'sock-fds' provided with file descriptors for
+    already open but not bound XDP sockets already added to a socket map for
+    corresponding queues.  One socket per queue.
+
+    .. parsed-literal::
+
+        |qemu_system| linux.img -device virtio-net-pci,netdev=n1 \\
+            -netdev af-xdp,id=n1,ifname=eth0,queues=3,inhibit=on,sock-fds=15:16:17
+
 ``-netdev vhost-user,chardev=id[,vhostforce=on|off][,queues=n]``
     Establish a vhost-user netdev, backed by a chardev id. The chardev
     should be a unix domain socket backed one. The vhost-user uses a
diff --git a/scripts/ci/org.centos/stream/8/x86_64/configure b/scripts/ci/org.centos/stream/8/x86_64/configure
index XXXXXXX..XXXXXXX 100755
--- a/scripts/ci/org.centos/stream/8/x86_64/configure
+++ b/scripts/ci/org.centos/stream/8/x86_64/configure
@@ -XXX,XX +XXX,XX @@
 --block-drv-ro-whitelist="vmdk,vhdx,vpc,https,ssh" \
 --with-coroutine=ucontext \
 --tls-priority=@QEMU,SYSTEM \
+--disable-af-xdp \
 --disable-attr \
 --disable-auth-pam \
 --disable-avx2 \
diff --git a/scripts/meson-buildoptions.sh b/scripts/meson-buildoptions.sh
index XXXXXXX..XXXXXXX 100644
--- a/scripts/meson-buildoptions.sh
+++ b/scripts/meson-buildoptions.sh
@@ -XXX,XX +XXX,XX @@ meson_options_help() {
   printf "%s\n" 'disabled with --disable-FEATURE, default is enabled if available'
   printf "%s\n" '(unless built with --without-default-features):'
   printf "%s\n" ''
+  printf "%s\n" '  af-xdp          AF_XDP network backend support'
   printf "%s\n" '  alsa            ALSA sound support'
   printf "%s\n" '  attr            attr/xattr support'
   printf "%s\n" '  auth-pam        PAM access control'
@@ -XXX,XX +XXX,XX @@ meson_options_help() {
 }
 _meson_option_parse() {
   case $1 in
+    --enable-af-xdp) printf "%s" -Daf_xdp=enabled ;;
+    --disable-af-xdp) printf "%s" -Daf_xdp=disabled ;;
     --enable-alsa) printf "%s" -Dalsa=enabled ;;
     --disable-alsa) printf "%s" -Dalsa=disabled ;;
     --enable-attr) printf "%s" -Dattr=enabled ;;
diff --git a/tests/docker/dockerfiles/debian-amd64.docker b/tests/docker/dockerfiles/debian-amd64.docker
index XXXXXXX..XXXXXXX 100644
--- a/tests/docker/dockerfiles/debian-amd64.docker
+++ b/tests/docker/dockerfiles/debian-amd64.docker
@@ -XXX,XX +XXX,XX @@ RUN export DEBIAN_FRONTEND=noninteractive && \
                       libvirglrenderer-dev \
                       libvte-2.91-dev \
                       libxen-dev \
+                      libxdp-dev \
                       libzstd-dev \
                       llvm \
                       locales \
-- 
2.7.4

From: Andrew Melnychenko <andrew@daynix.com>

Changed eBPF map updates through mmaped array.
Mmaped arrays provide direct access to map data.
It should omit using bpf_map_update_elem() call,
which may require capabilities that are not present.

Signed-off-by: Andrew Melnychenko <andrew@daynix.com>
Signed-off-by: Jason Wang <jasowang@redhat.com>
---
 ebpf/ebpf_rss.c | 117 +++++++++++++++++++++++++++++++++++++++++++++-----------
 ebpf/ebpf_rss.h |   5 +++
 2 files changed, 99 insertions(+), 23 deletions(-)

diff --git a/ebpf/ebpf_rss.c b/ebpf/ebpf_rss.c
index XXXXXXX..XXXXXXX 100644
--- a/ebpf/ebpf_rss.c
+++ b/ebpf/ebpf_rss.c
@@ -XXX,XX +XXX,XX @@ void ebpf_rss_init(struct EBPFRSSContext *ctx)
 {
     if (ctx != NULL) {
         ctx->obj = NULL;
+        ctx->program_fd = -1;
+        ctx->map_configuration = -1;
+        ctx->map_toeplitz_key = -1;
+        ctx->map_indirections_table = -1;
+
+        ctx->mmap_configuration = NULL;
+        ctx->mmap_toeplitz_key = NULL;
+        ctx->mmap_indirections_table = NULL;
     }
 }
 
 bool ebpf_rss_is_loaded(struct EBPFRSSContext *ctx)
 {
-    return ctx != NULL && ctx->obj != NULL;
+    return ctx != NULL && (ctx->obj != NULL || ctx->program_fd != -1);
+}
+
+static bool ebpf_rss_mmap(struct EBPFRSSContext *ctx)
+{
+    if (!ebpf_rss_is_loaded(ctx)) {
+        return false;
+    }
+
+    ctx->mmap_configuration = mmap(NULL, qemu_real_host_page_size(),
+                                   PROT_READ | PROT_WRITE, MAP_SHARED,
+                                   ctx->map_configuration, 0);
+    if (ctx->mmap_configuration == MAP_FAILED) {
+        trace_ebpf_error("eBPF RSS", "can not mmap eBPF configuration array");
+        return false;
+    }
+    ctx->mmap_toeplitz_key = mmap(NULL, qemu_real_host_page_size(),
+                                   PROT_READ | PROT_WRITE, MAP_SHARED,
+                                   ctx->map_toeplitz_key, 0);
+    if (ctx->mmap_toeplitz_key == MAP_FAILED) {
+        trace_ebpf_error("eBPF RSS", "can not mmap eBPF toeplitz key");
+        goto toeplitz_fail;
+    }
+    ctx->mmap_indirections_table = mmap(NULL, qemu_real_host_page_size(),
+                                   PROT_READ | PROT_WRITE, MAP_SHARED,
+                                   ctx->map_indirections_table, 0);
+    if (ctx->mmap_indirections_table == MAP_FAILED) {
+        trace_ebpf_error("eBPF RSS", "can not mmap eBPF indirection table");
+        goto indirection_fail;
+    }
+
+    return true;
+
+indirection_fail:
+    munmap(ctx->mmap_toeplitz_key, qemu_real_host_page_size());
+toeplitz_fail:
+    munmap(ctx->mmap_configuration, qemu_real_host_page_size());
+
+    ctx->mmap_configuration = NULL;
+    ctx->mmap_toeplitz_key = NULL;
+    ctx->mmap_indirections_table = NULL;
+    return false;
+}
+
+static void ebpf_rss_munmap(struct EBPFRSSContext *ctx)
+{
+    if (!ebpf_rss_is_loaded(ctx)) {
+        return;
+    }
+
+    munmap(ctx->mmap_indirections_table, qemu_real_host_page_size());
+    munmap(ctx->mmap_toeplitz_key, qemu_real_host_page_size());
+    munmap(ctx->mmap_configuration, qemu_real_host_page_size());
+
+    ctx->mmap_configuration = NULL;
+    ctx->mmap_toeplitz_key = NULL;
+    ctx->mmap_indirections_table = NULL;
 }
 
 bool ebpf_rss_load(struct EBPFRSSContext *ctx)
 {
     struct rss_bpf *rss_bpf_ctx;
 
-    if (ctx == NULL) {
+    if (ctx == NULL || ebpf_rss_is_loaded(ctx)) {
         return false;
     }
 
@@ -XXX,XX +XXX,XX @@ bool ebpf_rss_load(struct EBPFRSSContext *ctx)
     ctx->map_toeplitz_key = bpf_map__fd(
             rss_bpf_ctx->maps.tap_rss_map_toeplitz_key);
 
+    if (!ebpf_rss_mmap(ctx)) {
+        goto error;
+    }
+
     return true;
 error:
     rss_bpf__destroy(rss_bpf_ctx);
     ctx->obj = NULL;
+    ctx->program_fd = -1;
+    ctx->map_configuration = -1;
+    ctx->map_toeplitz_key = -1;
+    ctx->map_indirections_table = -1;
 
     return false;
 }
@@ -XXX,XX +XXX,XX @@ error:
 static bool ebpf_rss_set_config(struct EBPFRSSContext *ctx,
                                 struct EBPFRSSConfig *config)
 {
-    uint32_t map_key = 0;
-
     if (!ebpf_rss_is_loaded(ctx)) {
         return false;
     }
-    if (bpf_map_update_elem(ctx->map_configuration,
-                            &map_key, config, 0) < 0) {
-        return false;
-    }
+
+    memcpy(ctx->mmap_configuration, config, sizeof(*config));
     return true;
 }
 
@@ -XXX,XX +XXX,XX @@ static bool ebpf_rss_set_indirections_table(struct EBPFRSSContext *ctx,
                                             uint16_t *indirections_table,
                                             size_t len)
 {
-    uint32_t i = 0;
-
     if (!ebpf_rss_is_loaded(ctx) || indirections_table == NULL ||
        len > VIRTIO_NET_RSS_MAX_TABLE_LEN) {
         return false;
     }
 
-    for (; i < len; ++i) {
-        if (bpf_map_update_elem(ctx->map_indirections_table, &i,
-                                indirections_table + i, 0) < 0) {
-            return false;
-        }
-    }
+    memcpy(ctx->mmap_indirections_table, indirections_table,
+            sizeof(*indirections_table) * len);
     return true;
 }
 
 static bool ebpf_rss_set_toepliz_key(struct EBPFRSSContext *ctx,
                                      uint8_t *toeplitz_key)
 {
-    uint32_t map_key = 0;
-
     /* prepare toeplitz key */
     uint8_t toe[VIRTIO_NET_RSS_MAX_KEY_SIZE] = {};
 
@@ -XXX,XX +XXX,XX @@ static bool ebpf_rss_set_toepliz_key(struct EBPFRSSContext *ctx,
     memcpy(toe, toeplitz_key, VIRTIO_NET_RSS_MAX_KEY_SIZE);
     *(uint32_t *)toe = ntohl(*(uint32_t *)toe);
 
-    if (bpf_map_update_elem(ctx->map_toeplitz_key, &map_key, toe,
-                            0) < 0) {
-        return false;
-    }
+    memcpy(ctx->mmap_toeplitz_key, toe, VIRTIO_NET_RSS_MAX_KEY_SIZE);
     return true;
 }
 
@@ -XXX,XX +XXX,XX @@ void ebpf_rss_unload(struct EBPFRSSContext *ctx)
         return;
     }
 
-    rss_bpf__destroy(ctx->obj);
+    ebpf_rss_munmap(ctx);
+
+    if (ctx->obj) {
+        rss_bpf__destroy(ctx->obj);
+    } else {
+        close(ctx->program_fd);
+        close(ctx->map_configuration);
+        close(ctx->map_toeplitz_key);
+        close(ctx->map_indirections_table);
+    }
+
     ctx->obj = NULL;
+    ctx->program_fd = -1;
+    ctx->map_configuration = -1;
+    ctx->map_toeplitz_key = -1;
+    ctx->map_indirections_table = -1;
 }
diff --git a/ebpf/ebpf_rss.h b/ebpf/ebpf_rss.h
index XXXXXXX..XXXXXXX 100644
--- a/ebpf/ebpf_rss.h
+++ b/ebpf/ebpf_rss.h
@@ -XXX,XX +XXX,XX @@ struct EBPFRSSContext {
     int map_configuration;
     int map_toeplitz_key;
     int map_indirections_table;
+
+    /* mapped eBPF maps for direct access to omit bpf_map_update_elem() */
+    void *mmap_configuration;
+    void *mmap_toeplitz_key;
+    void *mmap_indirections_table;
 };
 
 struct EBPFRSSConfig {
-- 
2.7.4

From: Andrew Melnychenko <andrew@daynix.com>

It allows using file descriptors of eBPF provided
outside of QEMU.
QEMU may be run without capabilities for eBPF and run
RSS program provided by management tool(g.e. libvirt).

Signed-off-by: Andrew Melnychenko <andrew@daynix.com>
Signed-off-by: Jason Wang <jasowang@redhat.com>
---
 ebpf/ebpf_rss-stub.c |  6 ++++++
 ebpf/ebpf_rss.c      | 27 +++++++++++++++++++++++++++
 ebpf/ebpf_rss.h      |  5 +++++
 3 files changed, 38 insertions(+)

diff --git a/ebpf/ebpf_rss-stub.c b/ebpf/ebpf_rss-stub.c
index XXXXXXX..XXXXXXX 100644
--- a/ebpf/ebpf_rss-stub.c
+++ b/ebpf/ebpf_rss-stub.c
@@ -XXX,XX +XXX,XX @@ bool ebpf_rss_load(struct EBPFRSSContext *ctx)
     return false;
 }
 
+bool ebpf_rss_load_fds(struct EBPFRSSContext *ctx, int program_fd,
+                       int config_fd, int toeplitz_fd, int table_fd)
+{
+    return false;
+}
+
 bool ebpf_rss_set_all(struct EBPFRSSContext *ctx, struct EBPFRSSConfig *config,
                       uint16_t *indirections_table, uint8_t *toeplitz_key)
 {
diff --git a/ebpf/ebpf_rss.c b/ebpf/ebpf_rss.c
index XXXXXXX..XXXXXXX 100644
--- a/ebpf/ebpf_rss.c
+++ b/ebpf/ebpf_rss.c
@@ -XXX,XX +XXX,XX @@ error:
     return false;
 }
 
+bool ebpf_rss_load_fds(struct EBPFRSSContext *ctx, int program_fd,
+                       int config_fd, int toeplitz_fd, int table_fd)
+{
+    if (ctx == NULL || ebpf_rss_is_loaded(ctx)) {
+        return false;
+    }
+
+    if (program_fd < 0 || config_fd < 0 || toeplitz_fd < 0 || table_fd < 0) {
+        return false;
+    }
+
+    ctx->program_fd = program_fd;
+    ctx->map_configuration = config_fd;
+    ctx->map_toeplitz_key = toeplitz_fd;
+    ctx->map_indirections_table = table_fd;
+
+    if (!ebpf_rss_mmap(ctx)) {
+        ctx->program_fd = -1;
+        ctx->map_configuration = -1;
+        ctx->map_toeplitz_key = -1;
+        ctx->map_indirections_table = -1;
+        return false;
+    }
+
+    return true;
+}
+
 static bool ebpf_rss_set_config(struct EBPFRSSContext *ctx,
                                 struct EBPFRSSConfig *config)
 {
diff --git a/ebpf/ebpf_rss.h b/ebpf/ebpf_rss.h
index XXXXXXX..XXXXXXX 100644
--- a/ebpf/ebpf_rss.h
+++ b/ebpf/ebpf_rss.h
@@ -XXX,XX +XXX,XX @@
 #ifndef QEMU_EBPF_RSS_H
 #define QEMU_EBPF_RSS_H
 
+#define EBPF_RSS_MAX_FDS 4
+
 struct EBPFRSSContext {
     void *obj;
     int program_fd;
@@ -XXX,XX +XXX,XX @@ bool ebpf_rss_is_loaded(struct EBPFRSSContext *ctx);
 
 bool ebpf_rss_load(struct EBPFRSSContext *ctx);
 
+bool ebpf_rss_load_fds(struct EBPFRSSContext *ctx, int program_fd,
+                       int config_fd, int toeplitz_fd, int table_fd);
+
 bool ebpf_rss_set_all(struct EBPFRSSContext *ctx, struct EBPFRSSConfig *config,
                       uint16_t *indirections_table, uint8_t *toeplitz_key);
 
-- 
2.7.4

From: Andrew Melnychenko <andrew@daynix.com>

eBPF RSS program and maps may now be passed during initialization.
Initially was implemented for libvirt to launch qemu without permissions,
and initialized eBPF program through the helper.

Signed-off-by: Andrew Melnychenko <andrew@daynix.com>
Signed-off-by: Jason Wang <jasowang@redhat.com>
---
 hw/net/virtio-net.c            | 55 +++++++++++++++++++++++++++++++++++++-----
 include/hw/virtio/virtio-net.h |  1 +
 2 files changed, 50 insertions(+), 6 deletions(-)

diff --git a/hw/net/virtio-net.c b/hw/net/virtio-net.c
index XXXXXXX..XXXXXXX 100644
--- a/hw/net/virtio-net.c
+++ b/hw/net/virtio-net.c
@@ -XXX,XX +XXX,XX @@
 #include "sysemu/sysemu.h"
 #include "trace.h"
 #include "monitor/qdev.h"
+#include "monitor/monitor.h"
 #include "hw/pci/pci_device.h"
 #include "net_rx_pkt.h"
 #include "hw/virtio/vhost.h"
@@ -XXX,XX +XXX,XX @@ static void virtio_net_detach_epbf_rss(VirtIONet *n)
     virtio_net_attach_ebpf_to_backend(n->nic, -1);
 }
 
-static bool virtio_net_load_ebpf(VirtIONet *n)
+static bool virtio_net_load_ebpf_fds(VirtIONet *n, Error **errp)
 {
-    if (!virtio_net_attach_ebpf_to_backend(n->nic, -1)) {
-        /* backend does't support steering ebpf */
-        return false;
+    int fds[EBPF_RSS_MAX_FDS] = { [0 ... EBPF_RSS_MAX_FDS - 1] = -1};
+    int nfds = 0;
+    int ret = true;
+    int i = 0;
+    g_auto(GStrv) fds_strs = g_strsplit(n->ebpf_rss_fds, ":", 0);
+
+    ERRP_GUARD();
+
+    if (g_strv_length(fds_strs) != EBPF_RSS_MAX_FDS) {
+        error_setg(errp,
+                  "Expected %d file descriptors but got %d",
+                  EBPF_RSS_MAX_FDS, g_strv_length(fds_strs));
+       return false;
+   }
+
+    for (i = 0; i < nfds; i++) {
+        fds[i] = monitor_fd_param(monitor_cur(), fds_strs[i], errp);
+        if (*errp) {
+            ret = false;
+            goto exit;
+        }
+    }
+
+    ret = ebpf_rss_load_fds(&n->ebpf_rss, fds[0], fds[1], fds[2], fds[3]);
+
+exit:
+    if (!ret || *errp) {
+        for (i = 0; i < nfds && fds[i] != -1; i++) {
+            close(fds[i]);
+        }
     }
 
-    return ebpf_rss_load(&n->ebpf_rss);
+    return ret;
+}
+
+static bool virtio_net_load_ebpf(VirtIONet *n, Error **errp)
+{
+    bool ret = false;
+
+    if (virtio_net_attach_ebpf_to_backend(n->nic, -1)) {
+        if (!(n->ebpf_rss_fds
+                && virtio_net_load_ebpf_fds(n, errp))) {
+            ret = ebpf_rss_load(&n->ebpf_rss);
+        }
+    }
+
+    return ret;
 }
 
 static void virtio_net_unload_ebpf(VirtIONet *n)
@@ -XXX,XX +XXX,XX @@ static void virtio_net_device_realize(DeviceState *dev, Error **errp)
     net_rx_pkt_init(&n->rx_pkt);
 
     if (virtio_has_feature(n->host_features, VIRTIO_NET_F_RSS)) {
-        virtio_net_load_ebpf(n);
+        virtio_net_load_ebpf(n, errp);
     }
 }
 
@@ -XXX,XX +XXX,XX @@ static Property virtio_net_properties[] = {
                     VIRTIO_NET_F_RSS, false),
     DEFINE_PROP_BIT64("hash", VirtIONet, host_features,
                     VIRTIO_NET_F_HASH_REPORT, false),
+    DEFINE_PROP_STRING("ebpf_rss_fds", VirtIONet, ebpf_rss_fds),
     DEFINE_PROP_BIT64("guest_rsc_ext", VirtIONet, host_features,
                     VIRTIO_NET_F_RSC_EXT, false),
     DEFINE_PROP_UINT32("rsc_interval", VirtIONet, rsc_timeout,
diff --git a/include/hw/virtio/virtio-net.h b/include/hw/virtio/virtio-net.h
index XXXXXXX..XXXXXXX 100644
--- a/include/hw/virtio/virtio-net.h
+++ b/include/hw/virtio/virtio-net.h
@@ -XXX,XX +XXX,XX @@ struct VirtIONet {
     VirtioNetRssData rss_data;
     struct NetRxPkt *rx_pkt;
     struct EBPFRSSContext ebpf_rss;
+    char *ebpf_rss_fds;
 };
 
 size_t virtio_net_handle_ctrl_iov(VirtIODevice *vdev,
-- 
2.7.4

From: Andrew Melnychenko <andrew@daynix.com>

Now, the binary objects may be retrieved by id.
It would require for future qmp commands that may require specific
eBPF blob.

Added command "request-ebpf". This command returns
eBPF program encoded base64. The program taken from the
skeleton and essentially is an ELF object that can be
loaded in the future with libbpf.

The reason to use the command to provide the eBPF object
instead of a separate artifact was to avoid issues related
to finding the eBPF itself. eBPF object is an ELF binary
that contains the eBPF program and eBPF map description(BTF).
Overall, eBPF object should contain the program and enough
metadata to create/load eBPF with libbpf. As the eBPF
maps/program should correspond to QEMU, the eBPF can't
be used from different QEMU build.

The first solution was a helper that comes with QEMU
and loads appropriate eBPF objects. And the issue is
to find a proper helper if the system has several
different QEMUs installed and/or built from the source,
which helpers may not be compatible.

Another issue is QEMU updating while there is a running
QEMU instance. With an updated helper, it may not be
possible to hotplug virtio-net device to the already
running QEMU. Overall, requesting the eBPF object from
QEMU itself solves possible failures with acceptable effort.

Links:
[PATCH 3/5] qmp: Added the helper stamp check.
https://lore.kernel.org/all/20230219162100.174318-4-andrew@daynix.com/

Signed-off-by: Andrew Melnychenko <andrew@daynix.com>
Signed-off-by: Jason Wang <jasowang@redhat.com>
---
 ebpf/ebpf.c           | 70 +++++++++++++++++++++++++++++++++++++++++++++++++++
 ebpf/ebpf.h           | 31 +++++++++++++++++++++++
 ebpf/ebpf_rss.c       |  6 +++++
 ebpf/meson.build      |  2 +-
 qapi/ebpf.json        | 66 ++++++++++++++++++++++++++++++++++++++++++++++++
 qapi/meson.build      |  1 +
 qapi/qapi-schema.json |  1 +
 7 files changed, 176 insertions(+), 1 deletion(-)
 create mode 100644 ebpf/ebpf.c
 create mode 100644 ebpf/ebpf.h
 create mode 100644 qapi/ebpf.json

diff --git a/ebpf/ebpf.c b/ebpf/ebpf.c
new file mode 100644
index XXXXXXX..XXXXXXX
--- /dev/null
+++ b/ebpf/ebpf.c
@@ -XXX,XX +XXX,XX @@
+/*
+ * QEMU eBPF binary declaration routine.
+ *
+ * Developed by Daynix Computing LTD (http://www.daynix.com)
+ *
+ * Authors:
+ *  Andrew Melnychenko <andrew@daynix.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or
+ * later.  See the COPYING file in the top-level directory.
+ */
+
+#include "qemu/osdep.h"
+#include "qemu/queue.h"
+#include "qapi/error.h"
+#include "qapi/qapi-commands-ebpf.h"
+#include "ebpf/ebpf.h"
+
+struct ElfBinaryDataEntry {
+    int id;
+    const void *data;
+    size_t datalen;
+
+    QSLIST_ENTRY(ElfBinaryDataEntry) node;
+};
+
+static QSLIST_HEAD(, ElfBinaryDataEntry) ebpf_elf_obj_list =
+                                            QSLIST_HEAD_INITIALIZER();
+
+void ebpf_register_binary_data(int id, const void *data, size_t datalen)
+{
+    struct ElfBinaryDataEntry *dataentry = NULL;
+
+    dataentry = g_new0(struct ElfBinaryDataEntry, 1);
+    dataentry->data = data;
+    dataentry->datalen = datalen;
+    dataentry->id = id;
+
+    QSLIST_INSERT_HEAD(&ebpf_elf_obj_list, dataentry, node);
+}
+
+const void *ebpf_find_binary_by_id(int id, size_t *sz, Error **errp)
+{
+    struct ElfBinaryDataEntry *it = NULL;
+    QSLIST_FOREACH(it, &ebpf_elf_obj_list, node) {
+        if (id == it->id) {
+            *sz = it->datalen;
+            return it->data;
+        }
+    }
+
+    error_setg(errp, "can't find eBPF object with id: %d", id);
+
+    return NULL;
+}
+
+EbpfObject *qmp_request_ebpf(EbpfProgramID id, Error **errp)
+{
+    EbpfObject *ret = NULL;
+    size_t size = 0;
+    const void *data = ebpf_find_binary_by_id(id, &size, errp);
+    if (!data) {
+        return NULL;
+    }
+
+    ret = g_new0(EbpfObject, 1);
+    ret->object = g_base64_encode(data, size);
+
+    return ret;
+}
diff --git a/ebpf/ebpf.h b/ebpf/ebpf.h
new file mode 100644
index XXXXXXX..XXXXXXX
--- /dev/null
+++ b/ebpf/ebpf.h
@@ -XXX,XX +XXX,XX @@
+/*
+ * QEMU eBPF binary declaration routine.
+ *
+ * Developed by Daynix Computing LTD (http://www.daynix.com)
+ *
+ * Authors:
+ *  Andrew Melnychenko <andrew@daynix.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or
+ * later.  See the COPYING file in the top-level directory.
+ */
+
+#ifndef EBPF_H
+#define EBPF_H
+
+struct Error;
+
+void ebpf_register_binary_data(int id, const void *data,
+                               size_t datalen);
+const void *ebpf_find_binary_by_id(int id, size_t *sz,
+                                   struct Error **errp);
+
+#define ebpf_binary_init(id, fn)                                           \
+static void __attribute__((constructor)) ebpf_binary_init_ ## fn(void)     \
+{                                                                          \
+    size_t datalen = 0;                                                    \
+    const void *data = fn(&datalen);                                       \
+    ebpf_register_binary_data(id, data, datalen);                          \
+}
+
+#endif /* EBPF_H */
diff --git a/ebpf/ebpf_rss.c b/ebpf/ebpf_rss.c
index XXXXXXX..XXXXXXX 100644
--- a/ebpf/ebpf_rss.c
+++ b/ebpf/ebpf_rss.c
@@ -XXX,XX +XXX,XX @@
 
 #include "qemu/osdep.h"
 #include "qemu/error-report.h"
+#include "qapi/qapi-types-misc.h"
+#include "qapi/qapi-commands-ebpf.h"
 
 #include <bpf/libbpf.h>
 #include <bpf/bpf.h>
@@ -XXX,XX +XXX,XX @@
 
 #include "ebpf/ebpf_rss.h"
 #include "ebpf/rss.bpf.skeleton.h"
+#include "ebpf/ebpf.h"
+
 #include "trace.h"
 
 void ebpf_rss_init(struct EBPFRSSContext *ctx)
@@ -XXX,XX +XXX,XX @@ void ebpf_rss_unload(struct EBPFRSSContext *ctx)
     ctx->map_toeplitz_key = -1;
     ctx->map_indirections_table = -1;
 }
+
+ebpf_binary_init(EBPF_PROGRAMID_RSS, rss_bpf__elf_bytes)
diff --git a/ebpf/meson.build b/ebpf/meson.build
index XXXXXXX..XXXXXXX 100644
--- a/ebpf/meson.build
+++ b/ebpf/meson.build
@@ -1 +1 @@
-system_ss.add(when: libbpf, if_true: files('ebpf_rss.c'), if_false: files('ebpf_rss-stub.c'))
+common_ss.add(when: libbpf, if_true: files('ebpf.c', 'ebpf_rss.c'), if_false: files('ebpf_rss-stub.c'))
diff --git a/qapi/ebpf.json b/qapi/ebpf.json
new file mode 100644
index XXXXXXX..XXXXXXX
--- /dev/null
+++ b/qapi/ebpf.json
@@ -XXX,XX +XXX,XX @@
+# -*- Mode: Python -*-
+# vim: filetype=python
+#
+# This work is licensed under the terms of the GNU GPL, version 2 or later.
+# See the COPYING file in the top-level directory.
+
+##
+# = eBPF Objects
+#
+# eBPF object is an ELF binary that contains the eBPF
+# program and eBPF map description(BTF). Overall, eBPF
+# object should contain the program and enough metadata
+# to create/load eBPF with libbpf. As the eBPF maps/program
+# should correspond to QEMU, the eBPF can't be used from
+# different QEMU build.
+#
+# Currently, there is a possible eBPF for receive-side scaling (RSS).
+#
+##
+
+##
+# @EbpfObject:
+#
+# An eBPF ELF object.
+#
+# @object: the eBPF object encoded in base64
+#
+# Since: 8.2
+##
+{ 'struct': 'EbpfObject',
+  'data': {'object': 'str'},
+  'if': 'CONFIG_EBPF' }
+
+##
+# @EbpfProgramID:
+#
+# The eBPF programs that can be gotten with request-ebpf.
+#
+# @rss: Receive side scaling, technology that allows steering traffic
+#     between queues by calculation hash.  Users may set up
+#     indirection table and hash/packet types configurations.  Used
+#     with virtio-net.
+#
+# Since: 8.2
+##
+{ 'enum': 'EbpfProgramID',
+  'if': 'CONFIG_EBPF',
+  'data': [ { 'name': 'rss' } ] }
+
+##
+# @request-ebpf:
+#
+# Retrieve an eBPF object that can be loaded with libbpf.  Management
+# applications (g.e. libvirt) may load it and pass file descriptors to
+# QEMU, so they can run running QEMU without BPF capabilities.
+#
+# @id: The ID of the program to return.
+#
+# Returns: eBPF object encoded in base64.
+#
+# Since: 8.2
+##
+{ 'command': 'request-ebpf',
+  'data': { 'id': 'EbpfProgramID' },
+  'returns': 'EbpfObject',
+  'if': 'CONFIG_EBPF' }
diff --git a/qapi/meson.build b/qapi/meson.build
index XXXXXXX..XXXXXXX 100644
--- a/qapi/meson.build
+++ b/qapi/meson.build
@@ -XXX,XX +XXX,XX @@ qapi_all_modules = [
   'crypto',
   'cxl',
   'dump',
+  'ebpf',
   'error',
   'introspect',
   'job',
diff --git a/qapi/qapi-schema.json b/qapi/qapi-schema.json
index XXXXXXX..XXXXXXX 100644
--- a/qapi/qapi-schema.json
+++ b/qapi/qapi-schema.json
@@ -XXX,XX +XXX,XX @@
 { 'include': 'char.json' }
 { 'include': 'dump.json' }
 { 'include': 'net.json' }
+{ 'include': 'ebpf.json' }
 { 'include': 'rdma.json' }
 { 'include': 'rocker.json' }
 { 'include': 'tpm.json' }
-- 
2.7.4

From: Andrew Melnychenko <andrew@daynix.com>

Updated section name, so libbpf should init/gues proper
program type without specifications during open/load.
Also, added map_flags with explicitly declared BPF_F_MMAPABLE.
Added check for BPF_F_MMAPABLE flag to meson script and
requirements to libbpf version.

Signed-off-by: Andrew Melnychenko <andrew@daynix.com>
Signed-off-by: Jason Wang <jasowang@redhat.com>
---
 ebpf/rss.bpf.skeleton.h | 1460 ++++++++++++++++++++++++-----------------------
 meson.build             |   10 +-
 tools/ebpf/rss.bpf.c    |    5 +-
 3 files changed, 748 insertions(+), 727 deletions(-)

diff --git a/ebpf/rss.bpf.skeleton.h b/ebpf/rss.bpf.skeleton.h
index XXXXXXX..XXXXXXX 100644
--- a/ebpf/rss.bpf.skeleton.h
+++ b/ebpf/rss.bpf.skeleton.h
@@ -XXX,XX +XXX,XX @@ err:
 
 static inline const void *rss_bpf__elf_bytes(size_t *sz)
 {
-	*sz = 20440;
+	*sz = 20824;
 	return (const void *)"\
 \x7f\x45\x4c\x46\x02\x01\x01\0\0\0\0\0\0\0\0\0\x01\0\xf7\0\x01\0\0\0\0\0\0\0\0\
-\0\0\0\0\0\0\0\0\0\0\0\x98\x4c\0\0\0\0\0\0\0\0\0\0\x40\0\0\0\0\0\x40\0\x0d\0\
-\x01\0\xbf\x19\0\0\0\0\0\0\xb7\x01\0\0\0\0\0\0\x63\x1a\x54\xff\0\0\0\0\xbf\xa7\
-\0\0\0\0\0\0\x07\x07\0\0\x54\xff\xff\xff\x18\x01\0\0\0\0\0\0\0\0\0\0\0\0\0\0\
+\0\0\0\0\0\0\0\0\0\0\0\x18\x4e\0\0\0\0\0\0\0\0\0\0\x40\0\0\0\0\0\x40\0\x0d\0\
+\x01\0\xbf\x19\0\0\0\0\0\0\xb7\x01\0\0\0\0\0\0\x63\x1a\x4c\xff\0\0\0\0\xbf\xa7\
+\0\0\0\0\0\0\x07\x07\0\0\x4c\xff\xff\xff\x18\x01\0\0\0\0\0\0\0\0\0\0\0\0\0\0\
 \xbf\x72\0\0\0\0\0\0\x85\0\0\0\x01\0\0\0\xbf\x06\0\0\0\0\0\0\x18\x01\0\0\0\0\0\
 \0\0\0\0\0\0\0\0\0\xbf\x72\0\0\0\0\0\0\x85\0\0\0\x01\0\0\0\xbf\x08\0\0\0\0\0\0\
-\x18\0\0\0\xff\xff\xff\xff\0\0\0\0\0\0\0\0\x15\x06\x67\x02\0\0\0\0\xbf\x87\0\0\
-\0\0\0\0\x15\x07\x65\x02\0\0\0\0\x71\x61\0\0\0\0\0\0\x55\x01\x01\0\0\0\0\0\x05\
-\0\x5e\x02\0\0\0\0\xb7\x01\0\0\0\0\0\0\x63\x1a\xc8\xff\0\0\0\0\x7b\x1a\xc0\xff\
-\0\0\0\0\x7b\x1a\xb8\xff\0\0\0\0\x7b\x1a\xb0\xff\0\0\0\0\x7b\x1a\xa8\xff\0\0\0\
-\0\x63\x1a\xa0\xff\0\0\0\0\x7b\x1a\x98\xff\0\0\0\0\x7b\x1a\x90\xff\0\0\0\0\x7b\
-\x1a\x88\xff\0\0\0\0\x7b\x1a\x80\xff\0\0\0\0\x7b\x1a\x78\xff\0\0\0\0\x7b\x1a\
-\x70\xff\0\0\0\0\x7b\x1a\x68\xff\0\0\0\0\x7b\x1a\x60\xff\0\0\0\0\x7b\x1a\x58\
-\xff\0\0\0\0\x15\x09\x4d\x02\0\0\0\0\x6b\x1a\xd0\xff\0\0\0\0\xbf\xa3\0\0\0\0\0\
-\0\x07\x03\0\0\xd0\xff\xff\xff\xbf\x91\0\0\0\0\0\0\xb7\x02\0\0\x0c\0\0\0\xb7\
+\x18\0\0\0\xff\xff\xff\xff\0\0\0\0\0\0\0\0\x15\x06\x64\x02\0\0\0\0\xbf\x87\0\0\
+\0\0\0\0\x15\x07\x62\x02\0\0\0\0\x71\x61\0\0\0\0\0\0\x55\x01\x01\0\0\0\0\0\x05\
+\0\x5b\x02\0\0\0\0\xb7\x01\0\0\0\0\0\0\x63\x1a\xc0\xff\0\0\0\0\x7b\x1a\xb8\xff\
+\0\0\0\0\x7b\x1a\xb0\xff\0\0\0\0\x7b\x1a\xa8\xff\0\0\0\0\x7b\x1a\xa0\xff\0\0\0\
+\0\x63\x1a\x98\xff\0\0\0\0\x7b\x1a\x90\xff\0\0\0\0\x7b\x1a\x88\xff\0\0\0\0\x7b\
+\x1a\x80\xff\0\0\0\0\x7b\x1a\x78\xff\0\0\0\0\x7b\x1a\x70\xff\0\0\0\0\x7b\x1a\
+\x68\xff\0\0\0\0\x7b\x1a\x60\xff\0\0\0\0\x7b\x1a\x58\xff\0\0\0\0\x7b\x1a\x50\
+\xff\0\0\0\0\x15\x09\x4a\x02\0\0\0\0\x6b\x1a\xc8\xff\0\0\0\0\xbf\xa3\0\0\0\0\0\
+\0\x07\x03\0\0\xc8\xff\xff\xff\xbf\x91\0\0\0\0\0\0\xb7\x02\0\0\x0c\0\0\0\xb7\
 \x04\0\0\x02\0\0\0\xb7\x05\0\0\0\0\0\0\x85\0\0\0\x44\0\0\0\x67\0\0\0\x20\0\0\0\
-\x77\0\0\0\x20\0\0\0\x55\0\x42\x02\0\0\0\0\xb7\x02\0\0\x10\0\0\0\x69\xa1\xd0\
+\x77\0\0\0\x20\0\0\0\x55\0\x3f\x02\0\0\0\0\xb7\x02\0\0\x10\0\0\0\x69\xa1\xc8\
 \xff\0\0\0\0\xbf\x13\0\0\0\0\0\0\xdc\x03\0\0\x10\0\0\0\x15\x03\x02\0\0\x81\0\0\
 \x55\x03\x0b\0\xa8\x88\0\0\xb7\x02\0\0\x14\0\0\0\xbf\xa3\0\0\0\0\0\0\x07\x03\0\
-\0\xd0\xff\xff\xff\xbf\x91\0\0\0\0\0\0\xb7\x04\0\0\x02\0\0\0\xb7\x05\0\0\0\0\0\
-\0\x85\0\0\0\x44\0\0\0\x67\0\0\0\x20\0\0\0\x77\0\0\0\x20\0\0\0\x55\0\x32\x02\0\
-\0\0\0\x69\xa1\xd0\xff\0\0\0\0\x15\x01\x30\x02\0\0\0\0\x7b\x7a\x38\xff\0\0\0\0\
-\x7b\x9a\x40\xff\0\0\0\0\x15\x01\x55\0\x86\xdd\0\0\x55\x01\x39\0\x08\0\0\0\xb7\
-\x07\0\0\x01\0\0\0\x73\x7a\x58\xff\0\0\0\0\xb7\x01\0\0\0\0\0\0\x63\x1a\xe0\xff\
-\0\0\0\0\x7b\x1a\xd8\xff\0\0\0\0\x7b\x1a\xd0\xff\0\0\0\0\xbf\xa3\0\0\0\0\0\0\
-\x07\x03\0\0\xd0\xff\xff\xff\x79\xa1\x40\xff\0\0\0\0\xb7\x02\0\0\0\0\0\0\xb7\
+\0\xc8\xff\xff\xff\xbf\x91\0\0\0\0\0\0\xb7\x04\0\0\x02\0\0\0\xb7\x05\0\0\0\0\0\
+\0\x85\0\0\0\x44\0\0\0\x67\0\0\0\x20\0\0\0\x77\0\0\0\x20\0\0\0\x55\0\x2f\x02\0\
+\0\0\0\x69\xa1\xc8\xff\0\0\0\0\x15\x01\x2d\x02\0\0\0\0\x7b\x7a\x30\xff\0\0\0\0\
+\x7b\x9a\x38\xff\0\0\0\0\x15\x01\x55\0\x86\xdd\0\0\x55\x01\x39\0\x08\0\0\0\xb7\
+\x07\0\0\x01\0\0\0\x73\x7a\x50\xff\0\0\0\0\xb7\x01\0\0\0\0\0\0\x63\x1a\xd8\xff\
+\0\0\0\0\x7b\x1a\xd0\xff\0\0\0\0\x7b\x1a\xc8\xff\0\0\0\0\xbf\xa3\0\0\0\0\0\0\
+\x07\x03\0\0\xc8\xff\xff\xff\x79\xa1\x38\xff\0\0\0\0\xb7\x02\0\0\0\0\0\0\xb7\
 \x04\0\0\x14\0\0\0\xb7\x05\0\0\x01\0\0\0\x85\0\0\0\x44\0\0\0\x67\0\0\0\x20\0\0\
-\0\x77\0\0\0\x20\0\0\0\x55\0\x1c\x02\0\0\0\0\x69\xa1\xd6\xff\0\0\0\0\x55\x01\
-\x01\0\0\0\0\0\xb7\x07\0\0\0\0\0\0\x61\xa1\xdc\xff\0\0\0\0\x63\x1a\x64\xff\0\0\
-\0\0\x61\xa1\xe0\xff\0\0\0\0\x63\x1a\x68\xff\0\0\0\0\x71\xa9\xd9\xff\0\0\0\0\
-\x73\x7a\x5e\xff\0\0\0\0\x71\xa1\xd0\xff\0\0\0\0\x67\x01\0\0\x02\0\0\0\x57\x01\
-\0\0\x3c\0\0\0\x7b\x1a\x48\xff\0\0\0\0\xbf\x91\0\0\0\0\0\0\x57\x01\0\0\xff\0\0\
+\0\x77\0\0\0\x20\0\0\0\x55\0\x19\x02\0\0\0\0\x69\xa1\xce\xff\0\0\0\0\x55\x01\
+\x01\0\0\0\0\0\xb7\x07\0\0\0\0\0\0\x61\xa1\xd4\xff\0\0\0\0\x63\x1a\x5c\xff\0\0\
+\0\0\x61\xa1\xd8\xff\0\0\0\0\x63\x1a\x60\xff\0\0\0\0\x71\xa9\xd1\xff\0\0\0\0\
+\x73\x7a\x56\xff\0\0\0\0\x71\xa1\xc8\xff\0\0\0\0\x67\x01\0\0\x02\0\0\0\x57\x01\
+\0\0\x3c\0\0\0\x7b\x1a\x40\xff\0\0\0\0\xbf\x91\0\0\0\0\0\0\x57\x01\0\0\xff\0\0\
 \0\x15\x01\x19\0\0\0\0\0\x57\x07\0\0\xff\0\0\0\x55\x07\x17\0\0\0\0\0\x57\x09\0\
 \0\xff\0\0\0\x15\x09\x5a\x01\x11\0\0\0\x55\x09\x14\0\x06\0\0\0\xb7\x01\0\0\x01\
-\0\0\0\x73\x1a\x5b\xff\0\0\0\0\xb7\x01\0\0\0\0\0\0\x63\x1a\xe0\xff\0\0\0\0\x7b\
-\x1a\xd8\xff\0\0\0\0\x7b\x1a\xd0\xff\0\0\0\0\xbf\xa3\0\0\0\0\0\0\x07\x03\0\0\
-\xd0\xff\xff\xff\x79\xa1\x40\xff\0\0\0\0\x79\xa2\x48\xff\0\0\0\0\xb7\x04\0\0\
+\0\0\0\x73\x1a\x53\xff\0\0\0\0\xb7\x01\0\0\0\0\0\0\x63\x1a\xd8\xff\0\0\0\0\x7b\
+\x1a\xd0\xff\0\0\0\0\x7b\x1a\xc8\xff\0\0\0\0\xbf\xa3\0\0\0\0\0\0\x07\x03\0\0\
+\xc8\xff\xff\xff\x79\xa1\x38\xff\0\0\0\0\x79\xa2\x40\xff\0\0\0\0\xb7\x04\0\0\
 \x14\0\0\0\xb7\x05\0\0\x01\0\0\0\x85\0\0\0\x44\0\0\0\x67\0\0\0\x20\0\0\0\x77\0\
-\0\0\x20\0\0\0\x55\0\xf7\x01\0\0\0\0\x69\xa1\xd0\xff\0\0\0\0\x6b\x1a\x60\xff\0\
-\0\0\0\x69\xa1\xd2\xff\0\0\0\0\x6b\x1a\x62\xff\0\0\0\0\x71\xa1\x58\xff\0\0\0\0\
+\0\0\x20\0\0\0\x55\0\xf4\x01\0\0\0\0\x69\xa1\xc8\xff\0\0\0\0\x6b\x1a\x58\xff\0\
+\0\0\0\x69\xa1\xca\xff\0\0\0\0\x6b\x1a\x5a\xff\0\0\0\0\x71\xa1\x50\xff\0\0\0\0\
 \x15\x01\xdb\0\0\0\0\0\x71\x62\x03\0\0\0\0\0\x67\x02\0\0\x08\0\0\0\x71\x61\x02\
 \0\0\0\0\0\x4f\x12\0\0\0\0\0\0\x71\x63\x04\0\0\0\0\0\x71\x61\x05\0\0\0\0\0\x67\
 \x01\0\0\x08\0\0\0\x4f\x31\0\0\0\0\0\0\x67\x01\0\0\x10\0\0\0\x4f\x21\0\0\0\0\0\
-\0\x71\xa2\x5b\xff\0\0\0\0\x79\xa0\x38\xff\0\0\0\0\x15\x02\x0c\x01\0\0\0\0\xbf\
-\x12\0\0\0\0\0\0\x57\x02\0\0\x02\0\0\0\x15\x02\x09\x01\0\0\0\0\x61\xa1\x64\xff\
-\0\0\0\0\x63\x1a\xa8\xff\0\0\0\0\x61\xa1\x68\xff\0\0\0\0\x63\x1a\xac\xff\0\0\0\
-\0\x69\xa1\x60\xff\0\0\0\0\x6b\x1a\xb0\xff\0\0\0\0\x69\xa1\x62\xff\0\0\0\0\x6b\
-\x1a\xb2\xff\0\0\0\0\x05\0\x6b\x01\0\0\0\0\xb7\x01\0\0\x01\0\0\0\x73\x1a\x59\
-\xff\0\0\0\0\xb7\x01\0\0\0\0\0\0\x7b\x1a\xf0\xff\0\0\0\0\x7b\x1a\xe8\xff\0\0\0\
-\0\x7b\x1a\xe0\xff\0\0\0\0\x7b\x1a\xd8\xff\0\0\0\0\x7b\x1a\xd0\xff\0\0\0\0\xbf\
-\xa3\0\0\0\0\0\0\x07\x03\0\0\xd0\xff\xff\xff\xb7\x01\0\0\x28\0\0\0\x7b\x1a\x48\
+\0\x71\xa2\x53\xff\0\0\0\0\x79\xa0\x30\xff\0\0\0\0\x15\x02\x0c\x01\0\0\0\0\xbf\
+\x12\0\0\0\0\0\0\x57\x02\0\0\x02\0\0\0\x15\x02\x09\x01\0\0\0\0\x61\xa1\x5c\xff\
+\0\0\0\0\x63\x1a\xa0\xff\0\0\0\0\x61\xa1\x60\xff\0\0\0\0\x63\x1a\xa4\xff\0\0\0\
+\0\x69\xa1\x58\xff\0\0\0\0\x6b\x1a\xa8\xff\0\0\0\0\x69\xa1\x5a\xff\0\0\0\0\x6b\
+\x1a\xaa\xff\0\0\0\0\x05\0\x6b\x01\0\0\0\0\xb7\x01\0\0\x01\0\0\0\x73\x1a\x51\
+\xff\0\0\0\0\xb7\x01\0\0\0\0\0\0\x7b\x1a\xe8\xff\0\0\0\0\x7b\x1a\xe0\xff\0\0\0\
+\0\x7b\x1a\xd8\xff\0\0\0\0\x7b\x1a\xd0\xff\0\0\0\0\x7b\x1a\xc8\xff\0\0\0\0\xbf\
+\xa3\0\0\0\0\0\0\x07\x03\0\0\xc8\xff\xff\xff\xb7\x01\0\0\x28\0\0\0\x7b\x1a\x40\
 \xff\0\0\0\0\xbf\x91\0\0\0\0\0\0\xb7\x02\0\0\0\0\0\0\xb7\x04\0\0\x28\0\0\0\xb7\
 \x05\0\0\x01\0\0\0\x85\0\0\0\x44\0\0\0\x67\0\0\0\x20\0\0\0\x77\0\0\0\x20\0\0\0\
-\x55\0\xfe\0\0\0\0\0\x79\xa1\xe0\xff\0\0\0\0\x63\x1a\x6c\xff\0\0\0\0\x77\x01\0\
-\0\x20\0\0\0\x63\x1a\x70\xff\0\0\0\0\x79\xa1\xd8\xff\0\0\0\0\x63\x1a\x64\xff\0\
-\0\0\0\x77\x01\0\0\x20\0\0\0\x63\x1a\x68\xff\0\0\0\0\x79\xa1\xe8\xff\0\0\0\0\
-\x63\x1a\x74\xff\0\0\0\0\x77\x01\0\0\x20\0\0\0\x63\x1a\x78\xff\0\0\0\0\x79\xa1\
-\xf0\xff\0\0\0\0\x63\x1a\x7c\xff\0\0\0\0\x77\x01\0\0\x20\0\0\0\x63\x1a\x80\xff\
-\0\0\0\0\x71\xa9\xd6\xff\0\0\0\0\x25\x09\x13\x01\x3c\0\0\0\xb7\x01\0\0\x01\0\0\
+\x55\0\xfe\0\0\0\0\0\x79\xa1\xd8\xff\0\0\0\0\x63\x1a\x64\xff\0\0\0\0\x77\x01\0\
+\0\x20\0\0\0\x63\x1a\x68\xff\0\0\0\0\x79\xa1\xd0\xff\0\0\0\0\x63\x1a\x5c\xff\0\
+\0\0\0\x77\x01\0\0\x20\0\0\0\x63\x1a\x60\xff\0\0\0\0\x79\xa1\xe0\xff\0\0\0\0\
+\x63\x1a\x6c\xff\0\0\0\0\x77\x01\0\0\x20\0\0\0\x63\x1a\x70\xff\0\0\0\0\x79\xa1\
+\xe8\xff\0\0\0\0\x63\x1a\x74\xff\0\0\0\0\x77\x01\0\0\x20\0\0\0\x63\x1a\x78\xff\
+\0\0\0\0\x71\xa9\xce\xff\0\0\0\0\x25\x09\x13\x01\x3c\0\0\0\xb7\x01\0\0\x01\0\0\
 \0\x6f\x91\0\0\0\0\0\0\x18\x02\0\0\x01\0\0\0\0\0\0\0\0\x18\0\x1c\x5f\x21\0\0\0\
 \0\0\0\x55\x01\x01\0\0\0\0\0\x05\0\x0c\x01\0\0\0\0\xb7\x01\0\0\0\0\0\0\x6b\x1a\
-\xfe\xff\0\0\0\0\xb7\x01\0\0\x28\0\0\0\x7b\x1a\x48\xff\0\0\0\0\xbf\xa1\0\0\0\0\
-\0\0\x07\x01\0\0\x94\xff\xff\xff\x7b\x1a\x20\xff\0\0\0\0\xbf\xa1\0\0\0\0\0\0\
-\x07\x01\0\0\x84\xff\xff\xff\x7b\x1a\x18\xff\0\0\0\0\x18\x07\0\0\x01\0\0\0\0\0\
-\0\0\0\x18\0\x1c\xb7\x02\0\0\0\0\0\0\x7b\x8a\x28\xff\0\0\0\0\x7b\x2a\x30\xff\0\
-\0\0\0\xbf\xa3\0\0\0\0\0\0\x07\x03\0\0\xfe\xff\xff\xff\x79\xa1\x40\xff\0\0\0\0\
-\x79\xa2\x48\xff\0\0\0\0\xb7\x04\0\0\x02\0\0\0\xb7\x05\0\0\x01\0\0\0\x85\0\0\0\
-\x44\0\0\0\x67\0\0\0\x20\0\0\0\x77\0\0\0\x20\0\0\0\x15\0\x01\0\0\0\0\0\x05\0\
-\x91\x01\0\0\0\0\xbf\x91\0\0\0\0\0\0\x15\x01\x26\0\x3c\0\0\0\x15\x01\x5f\0\x2c\
-\0\0\0\x55\x01\x60\0\x2b\0\0\0\xb7\x01\0\0\0\0\0\0\x63\x1a\xf8\xff\0\0\0\0\xbf\
-\xa3\0\0\0\0\0\0\x07\x03\0\0\xf8\xff\xff\xff\x79\xa7\x40\xff\0\0\0\0\xbf\x71\0\
-\0\0\0\0\0\x79\xa2\x48\xff\0\0\0\0\xb7\x04\0\0\x04\0\0\0\xb7\x05\0\0\x01\0\0\0\
-\x85\0\0\0\x44\0\0\0\xbf\x01\0\0\0\0\0\0\x67\x01\0\0\x20\0\0\0\x77\x01\0\0\x20\
-\0\0\0\x55\x01\x06\x01\0\0\0\0\x71\xa1\xfa\xff\0\0\0\0\x55\x01\x11\0\x02\0\0\0\
-\x71\xa1\xf9\xff\0\0\0\0\x55\x01\x0f\0\x02\0\0\0\x71\xa1\xfb\xff\0\0\0\0\x55\
-\x01\x0d\0\x01\0\0\0\x79\xa2\x48\xff\0\0\0\0\x07\x02\0\0\x08\0\0\0\xbf\x71\0\0\
-\0\0\0\0\x79\xa3\x20\xff\0\0\0\0\xb7\x04\0\0\x10\0\0\0\xb7\x05\0\0\x01\0\0\0\
-\x85\0\0\0\x44\0\0\0\xbf\x01\0\0\0\0\0\0\x67\x01\0\0\x20\0\0\0\x77\x01\0\0\x20\
-\0\0\0\x55\x01\xf5\0\0\0\0\0\xb7\x01\0\0\x01\0\0\0\x73\x1a\x5d\xff\0\0\0\0\x18\
-\x07\0\0\x01\0\0\0\0\0\0\0\0\x18\0\x1c\x05\0\x3c\0\0\0\0\0\xb7\x08\0\0\x02\0\0\
-\0\xb7\x07\0\0\0\0\0\0\x6b\x7a\xf8\xff\0\0\0\0\x05\0\x13\0\0\0\0\0\x0f\x81\0\0\
-\0\0\0\0\xbf\x12\0\0\0\0\0\0\x07\x02\0\0\x01\0\0\0\x71\xa3\xff\xff\0\0\0\0\x67\
-\x03\0\0\x03\0\0\0\x3d\x32\x09\0\0\0\0\0\xbf\x72\0\0\0\0\0\0\x07\x02\0\0\x01\0\
-\0\0\x67\x07\0\0\x20\0\0\0\xbf\x73\0\0\0\0\0\0\x77\x03\0\0\x20\0\0\0\xbf\x27\0\
-\0\0\0\0\0\xbf\x18\0\0\0\0\0\0\xb7\x01\0\0\x1d\0\0\0\x2d\x31\x04\0\0\0\0\0\x79\
-\xa8\x28\xff\0\0\0\0\x18\x07\0\0\x01\0\0\0\0\0\0\0\0\x18\0\x1c\x05\0\x25\0\0\0\
-\0\0\xbf\x89\0\0\0\0\0\0\x79\xa1\x48\xff\0\0\0\0\x0f\x19\0\0\0\0\0\0\xbf\xa3\0\
-\0\0\0\0\0\x07\x03\0\0\xf8\xff\xff\xff\x79\xa1\x40\xff\0\0\0\0\xbf\x92\0\0\0\0\
-\0\0\xb7\x04\0\0\x02\0\0\0\xb7\x05\0\0\x01\0\0\0\x85\0\0\0\x44\0\0\0\xbf\x01\0\
-\0\0\0\0\0\x67\x01\0\0\x20\0\0\0\x77\x01\0\0\x20\0\0\0\x55\x01\x79\0\0\0\0\0\
-\x71\xa2\xf8\xff\0\0\0\0\x55\x02\x0e\0\xc9\0\0\0\x07\x09\0\0\x02\0\0\0\x79\xa1\
-\x40\xff\0\0\0\0\xbf\x92\0\0\0\0\0\0\x79\xa3\x18\xff\0\0\0\0\xb7\x04\0\0\x10\0\
-\0\0\xb7\x05\0\0\x01\0\0\0\x85\0\0\0\x44\0\0\0\xbf\x01\0\0\0\0\0\0\x67\x01\0\0\
-\x20\0\0\0\x77\x01\0\0\x20\0\0\0\x55\x01\x6c\0\0\0\0\0\xb7\x01\0\0\x01\0\0\0\
-\x73\x1a\x5c\xff\0\0\0\0\x05\0\xde\xff\0\0\0\0\xb7\x01\0\0\x01\0\0\0\x15\x02\
-\xcd\xff\0\0\0\0\x71\xa1\xf9\xff\0\0\0\0\x07\x01\0\0\x02\0\0\0\x05\0\xca\xff\0\
-\0\0\0\xb7\x01\0\0\x01\0\0\0\x73\x1a\x5e\xff\0\0\0\0\x71\xa1\xff\xff\0\0\0\0\
-\x67\x01\0\0\x03\0\0\0\x79\xa2\x48\xff\0\0\0\0\x0f\x12\0\0\0\0\0\0\x07\x02\0\0\
-\x08\0\0\0\x7b\x2a\x48\xff\0\0\0\0\x71\xa9\xfe\xff\0\0\0\0\x79\xa2\x30\xff\0\0\
-\0\0\x25\x09\x0c\0\x3c\0\0\0\xb7\x01\0\0\x01\0\0\0\x6f\x91\0\0\0\0\0\0\x5f\x71\
-\0\0\0\0\0\0\x55\x01\x01\0\0\0\0\0\x05\0\x07\0\0\0\0\0\x07\x02\0\0\x01\0\0\0\
-\xbf\x21\0\0\0\0\0\0\x67\x01\0\0\x20\0\0\0\x77\x01\0\0\x20\0\0\0\x55\x01\x7d\
-\xff\x0b\0\0\0\x71\xa7\x5e\xff\0\0\0\0\x05\0\x09\xff\0\0\0\0\x15\x09\xf8\xff\
-\x87\0\0\0\x05\0\xfc\xff\0\0\0\0\x71\xa1\x59\xff\0\0\0\0\x79\xa0\x38\xff\0\0\0\
-\0\x15\x01\x13\x01\0\0\0\0\x71\x62\x03\0\0\0\0\0\x67\x02\0\0\x08\0\0\0\x71\x61\
+\xf8\xff\0\0\0\0\xb7\x01\0\0\x28\0\0\0\x7b\x1a\x40\xff\0\0\0\0\xbf\xa1\0\0\0\0\
+\0\0\x07\x01\0\0\x8c\xff\xff\xff\x7b\x1a\x18\xff\0\0\0\0\xbf\xa1\0\0\0\0\0\0\
+\x07\x01\0\0\x7c\xff\xff\xff\x7b\x1a\x10\xff\0\0\0\0\x18\x07\0\0\x01\0\0\0\0\0\
+\0\0\0\x18\0\x1c\xb7\x02\0\0\0\0\0\0\x7b\x8a\x20\xff\0\0\0\0\x7b\x2a\x28\xff\0\
+\0\0\0\xbf\xa3\0\0\0\0\0\0\x07\x03\0\0\xf8\xff\xff\xff\x79\xa1\x38\xff\0\0\0\0\
+\x79\xa2\x40\xff\0\0\0\0\xb7\x04\0\0\x02\0\0\0\xb7\x05\0\0\x01\0\0\0\x85\0\0\0\
+\x44\0\0\0\xbf\x01\0\0\0\0\0\0\x67\x01\0\0\x20\0\0\0\x77\x01\0\0\x20\0\0\0\x55\
+\x01\xc8\0\0\0\0\0\xbf\x91\0\0\0\0\0\0\x15\x01\x26\0\x3c\0\0\0\x15\x01\x5f\0\
+\x2c\0\0\0\x55\x01\x60\0\x2b\0\0\0\xb7\x01\0\0\0\0\0\0\x63\x1a\xf0\xff\0\0\0\0\
+\xbf\xa3\0\0\0\0\0\0\x07\x03\0\0\xf0\xff\xff\xff\x79\xa7\x38\xff\0\0\0\0\xbf\
+\x71\0\0\0\0\0\0\x79\xa2\x40\xff\0\0\0\0\xb7\x04\0\0\x04\0\0\0\xb7\x05\0\0\x01\
+\0\0\0\x85\0\0\0\x44\0\0\0\xbf\x01\0\0\0\0\0\0\x67\x01\0\0\x20\0\0\0\x77\x01\0\
+\0\x20\0\0\0\x55\x01\x06\x01\0\0\0\0\x71\xa1\xf2\xff\0\0\0\0\x55\x01\x11\0\x02\
+\0\0\0\x71\xa1\xf1\xff\0\0\0\0\x55\x01\x0f\0\x02\0\0\0\x71\xa1\xf3\xff\0\0\0\0\
+\x55\x01\x0d\0\x01\0\0\0\x79\xa2\x40\xff\0\0\0\0\x07\x02\0\0\x08\0\0\0\xbf\x71\
+\0\0\0\0\0\0\x79\xa3\x18\xff\0\0\0\0\xb7\x04\0\0\x10\0\0\0\xb7\x05\0\0\x01\0\0\
+\0\x85\0\0\0\x44\0\0\0\xbf\x01\0\0\0\0\0\0\x67\x01\0\0\x20\0\0\0\x77\x01\0\0\
+\x20\0\0\0\x55\x01\xf5\0\0\0\0\0\xb7\x01\0\0\x01\0\0\0\x73\x1a\x55\xff\0\0\0\0\
+\x18\x07\0\0\x01\0\0\0\0\0\0\0\0\x18\0\x1c\x05\0\x3c\0\0\0\0\0\xb7\x08\0\0\x02\
+\0\0\0\xb7\x07\0\0\0\0\0\0\x6b\x7a\xf0\xff\0\0\0\0\x05\0\x13\0\0\0\0\0\x0f\x81\
+\0\0\0\0\0\0\xbf\x12\0\0\0\0\0\0\x07\x02\0\0\x01\0\0\0\x71\xa3\xf9\xff\0\0\0\0\
+\x67\x03\0\0\x03\0\0\0\x3d\x32\x09\0\0\0\0\0\xbf\x72\0\0\0\0\0\0\x07\x02\0\0\
+\x01\0\0\0\x67\x07\0\0\x20\0\0\0\xbf\x73\0\0\0\0\0\0\x77\x03\0\0\x20\0\0\0\xbf\
+\x27\0\0\0\0\0\0\xbf\x18\0\0\0\0\0\0\xb7\x01\0\0\x1d\0\0\0\x2d\x31\x04\0\0\0\0\
+\0\x79\xa8\x20\xff\0\0\0\0\x18\x07\0\0\x01\0\0\0\0\0\0\0\0\x18\0\x1c\x05\0\x25\
+\0\0\0\0\0\xbf\x89\0\0\0\0\0\0\x79\xa1\x40\xff\0\0\0\0\x0f\x19\0\0\0\0\0\0\xbf\
+\xa3\0\0\0\0\0\0\x07\x03\0\0\xf0\xff\xff\xff\x79\xa1\x38\xff\0\0\0\0\xbf\x92\0\
+\0\0\0\0\0\xb7\x04\0\0\x02\0\0\0\xb7\x05\0\0\x01\0\0\0\x85\0\0\0\x44\0\0\0\xbf\
+\x01\0\0\0\0\0\0\x67\x01\0\0\x20\0\0\0\x77\x01\0\0\x20\0\0\0\x55\x01\x79\0\0\0\
+\0\0\x71\xa2\xf0\xff\0\0\0\0\x55\x02\x0e\0\xc9\0\0\0\x07\x09\0\0\x02\0\0\0\x79\
+\xa1\x38\xff\0\0\0\0\xbf\x92\0\0\0\0\0\0\x79\xa3\x10\xff\0\0\0\0\xb7\x04\0\0\
+\x10\0\0\0\xb7\x05\0\0\x01\0\0\0\x85\0\0\0\x44\0\0\0\xbf\x01\0\0\0\0\0\0\x67\
+\x01\0\0\x20\0\0\0\x77\x01\0\0\x20\0\0\0\x55\x01\x6c\0\0\0\0\0\xb7\x01\0\0\x01\
+\0\0\0\x73\x1a\x54\xff\0\0\0\0\x05\0\xde\xff\0\0\0\0\xb7\x01\0\0\x01\0\0\0\x15\
+\x02\xcd\xff\0\0\0\0\x71\xa1\xf1\xff\0\0\0\0\x07\x01\0\0\x02\0\0\0\x05\0\xca\
+\xff\0\0\0\0\xb7\x01\0\0\x01\0\0\0\x73\x1a\x56\xff\0\0\0\0\x71\xa1\xf9\xff\0\0\
+\0\0\x67\x01\0\0\x03\0\0\0\x79\xa2\x40\xff\0\0\0\0\x0f\x12\0\0\0\0\0\0\x07\x02\
+\0\0\x08\0\0\0\x7b\x2a\x40\xff\0\0\0\0\x71\xa9\xf8\xff\0\0\0\0\x79\xa2\x28\xff\
+\0\0\0\0\x25\x09\x0c\0\x3c\0\0\0\xb7\x01\0\0\x01\0\0\0\x6f\x91\0\0\0\0\0\0\x5f\
+\x71\0\0\0\0\0\0\x55\x01\x01\0\0\0\0\0\x05\0\x07\0\0\0\0\0\x07\x02\0\0\x01\0\0\
+\0\xbf\x21\0\0\0\0\0\0\x67\x01\0\0\x20\0\0\0\x77\x01\0\0\x20\0\0\0\x55\x01\x7d\
+\xff\x0b\0\0\0\x71\xa7\x56\xff\0\0\0\0\x05\0\x09\xff\0\0\0\0\x15\x09\xf8\xff\
+\x87\0\0\0\x05\0\xfc\xff\0\0\0\0\x71\xa1\x51\xff\0\0\0\0\x79\xa0\x30\xff\0\0\0\
+\0\x15\x01\x10\x01\0\0\0\0\x71\x62\x03\0\0\0\0\0\x67\x02\0\0\x08\0\0\0\x71\x61\
 \x02\0\0\0\0\0\x4f\x12\0\0\0\0\0\0\x71\x63\x04\0\0\0\0\0\x71\x61\x05\0\0\0\0\0\
 \x67\x01\0\0\x08\0\0\0\x4f\x31\0\0\0\0\0\0\x67\x01\0\0\x10\0\0\0\x4f\x21\0\0\0\
-\0\0\0\x71\xa2\x5b\xff\0\0\0\0\x15\x02\x42\0\0\0\0\0\xbf\x12\0\0\0\0\0\0\x57\
+\0\0\0\x71\xa2\x53\xff\0\0\0\0\x15\x02\x42\0\0\0\0\0\xbf\x12\0\0\0\0\0\0\x57\
 \x02\0\0\x10\0\0\0\x15\x02\x3f\0\0\0\0\0\x57\x01\0\0\x80\0\0\0\xb7\x02\0\0\x10\
 \0\0\0\xb7\x03\0\0\x10\0\0\0\x15\x01\x01\0\0\0\0\0\xb7\x03\0\0\x30\0\0\0\x71\
-\xa4\x5d\xff\0\0\0\0\x15\x04\x01\0\0\0\0\0\xbf\x32\0\0\0\0\0\0\xbf\xa3\0\0\0\0\
-\0\0\x07\x03\0\0\x64\xff\xff\xff\xbf\x34\0\0\0\0\0\0\x15\x01\x02\0\0\0\0\0\xbf\
-\xa4\0\0\0\0\0\0\x07\x04\0\0\x84\xff\xff\xff\x71\xa5\x5c\xff\0\0\0\0\xbf\x31\0\
+\xa4\x55\xff\0\0\0\0\x15\x04\x01\0\0\0\0\0\xbf\x32\0\0\0\0\0\0\xbf\xa3\0\0\0\0\
+\0\0\x07\x03\0\0\x5c\xff\xff\xff\xbf\x34\0\0\0\0\0\0\x15\x01\x02\0\0\0\0\0\xbf\
+\xa4\0\0\0\0\0\0\x07\x04\0\0\x7c\xff\xff\xff\x71\xa5\x54\xff\0\0\0\0\xbf\x31\0\
 \0\0\0\0\0\x15\x05\x01\0\0\0\0\0\xbf\x41\0\0\0\0\0\0\x61\x14\x04\0\0\0\0\0\x67\
-\x04\0\0\x20\0\0\0\x61\x15\0\0\0\0\0\0\x4f\x54\0\0\0\0\0\0\x7b\x4a\xa8\xff\0\0\
+\x04\0\0\x20\0\0\0\x61\x15\0\0\0\0\0\0\x4f\x54\0\0\0\0\0\0\x7b\x4a\xa0\xff\0\0\
 \0\0\x61\x14\x08\0\0\0\0\0\x61\x11\x0c\0\0\0\0\0\x67\x01\0\0\x20\0\0\0\x4f\x41\
-\0\0\0\0\0\0\x7b\x1a\xb0\xff\0\0\0\0\x0f\x23\0\0\0\0\0\0\x61\x31\0\0\0\0\0\0\
-\x61\x32\x04\0\0\0\0\0\x61\x34\x08\0\0\0\0\0\x61\x33\x0c\0\0\0\0\0\x69\xa5\x62\
-\xff\0\0\0\0\x6b\x5a\xca\xff\0\0\0\0\x69\xa5\x60\xff\0\0\0\0\x6b\x5a\xc8\xff\0\
-\0\0\0\x67\x03\0\0\x20\0\0\0\x4f\x43\0\0\0\0\0\0\x7b\x3a\xc0\xff\0\0\0\0\x67\
-\x02\0\0\x20\0\0\0\x4f\x12\0\0\0\0\0\0\x7b\x2a\xb8\xff\0\0\0\0\x05\0\x6b\0\0\0\
-\0\0\x71\xa2\x5a\xff\0\0\0\0\x15\x02\x04\0\0\0\0\0\xbf\x12\0\0\0\0\0\0\x57\x02\
+\0\0\0\0\0\0\x7b\x1a\xa8\xff\0\0\0\0\x0f\x23\0\0\0\0\0\0\x61\x31\0\0\0\0\0\0\
+\x61\x32\x04\0\0\0\0\0\x61\x34\x08\0\0\0\0\0\x61\x33\x0c\0\0\0\0\0\x69\xa5\x5a\
+\xff\0\0\0\0\x6b\x5a\xc2\xff\0\0\0\0\x69\xa5\x58\xff\0\0\0\0\x6b\x5a\xc0\xff\0\
+\0\0\0\x67\x03\0\0\x20\0\0\0\x4f\x43\0\0\0\0\0\0\x7b\x3a\xb8\xff\0\0\0\0\x67\
+\x02\0\0\x20\0\0\0\x4f\x12\0\0\0\0\0\0\x7b\x2a\xb0\xff\0\0\0\0\x05\0\x6b\0\0\0\
+\0\0\x71\xa2\x52\xff\0\0\0\0\x15\x02\x04\0\0\0\0\0\xbf\x12\0\0\0\0\0\0\x57\x02\
 \0\0\x04\0\0\0\x15\x02\x01\0\0\0\0\0\x05\0\xf1\xfe\0\0\0\0\x57\x01\0\0\x01\0\0\
-\0\x15\x01\xd0\0\0\0\0\0\x61\xa1\x64\xff\0\0\0\0\x63\x1a\xa8\xff\0\0\0\0\x61\
-\xa1\x68\xff\0\0\0\0\x63\x1a\xac\xff\0\0\0\0\x05\0\x5e\0\0\0\0\0\xb7\x09\0\0\
-\x3c\0\0\0\x79\xa8\x28\xff\0\0\0\0\x67\0\0\0\x20\0\0\0\x77\0\0\0\x20\0\0\0\x15\
-\0\xac\xff\0\0\0\0\x05\0\xc5\0\0\0\0\0\x71\xa2\x5a\xff\0\0\0\0\x15\x02\x26\0\0\
+\0\x15\x01\xcd\0\0\0\0\0\x61\xa1\x5c\xff\0\0\0\0\x63\x1a\xa0\xff\0\0\0\0\x61\
+\xa1\x60\xff\0\0\0\0\x63\x1a\xa4\xff\0\0\0\0\x05\0\x5e\0\0\0\0\0\xb7\x09\0\0\
+\x3c\0\0\0\x79\xa8\x20\xff\0\0\0\0\x67\0\0\0\x20\0\0\0\x77\0\0\0\x20\0\0\0\x15\
+\0\xac\xff\0\0\0\0\x05\0\xc2\0\0\0\0\0\x71\xa2\x52\xff\0\0\0\0\x15\x02\x26\0\0\
 \0\0\0\xbf\x12\0\0\0\0\0\0\x57\x02\0\0\x20\0\0\0\x15\x02\x23\0\0\0\0\0\x57\x01\
 \0\0\0\x01\0\0\xb7\x02\0\0\x10\0\0\0\xb7\x03\0\0\x10\0\0\0\x15\x01\x01\0\0\0\0\
-\0\xb7\x03\0\0\x30\0\0\0\x71\xa4\x5d\xff\0\0\0\0\x15\x04\x01\0\0\0\0\0\xbf\x32\
-\0\0\0\0\0\0\xbf\xa3\0\0\0\0\0\0\x07\x03\0\0\x64\xff\xff\xff\xbf\x34\0\0\0\0\0\
-\0\x15\x01\x02\0\0\0\0\0\xbf\xa4\0\0\0\0\0\0\x07\x04\0\0\x84\xff\xff\xff\x71\
-\xa5\x5c\xff\0\0\0\0\xbf\x31\0\0\0\0\0\0\x15\x05\xbd\xff\0\0\0\0\x05\0\xbb\xff\
-\0\0\0\0\xb7\x01\0\0\x01\0\0\0\x73\x1a\x5a\xff\0\0\0\0\xb7\x01\0\0\0\0\0\0\x7b\
-\x1a\xd0\xff\0\0\0\0\xbf\xa3\0\0\0\0\0\0\x07\x03\0\0\xd0\xff\xff\xff\x79\xa1\
-\x40\xff\0\0\0\0\x79\xa2\x48\xff\0\0\0\0\xb7\x04\0\0\x08\0\0\0\xb7\x05\0\0\x01\
-\0\0\0\x85\0\0\0\x44\0\0\0\x67\0\0\0\x20\0\0\0\x77\0\0\0\x20\0\0\0\x55\0\xa0\0\
+\0\xb7\x03\0\0\x30\0\0\0\x71\xa4\x55\xff\0\0\0\0\x15\x04\x01\0\0\0\0\0\xbf\x32\
+\0\0\0\0\0\0\xbf\xa3\0\0\0\0\0\0\x07\x03\0\0\x5c\xff\xff\xff\xbf\x34\0\0\0\0\0\
+\0\x15\x01\x02\0\0\0\0\0\xbf\xa4\0\0\0\0\0\0\x07\x04\0\0\x7c\xff\xff\xff\x71\
+\xa5\x54\xff\0\0\0\0\xbf\x31\0\0\0\0\0\0\x15\x05\xbd\xff\0\0\0\0\x05\0\xbb\xff\
+\0\0\0\0\xb7\x01\0\0\x01\0\0\0\x73\x1a\x52\xff\0\0\0\0\xb7\x01\0\0\0\0\0\0\x7b\
+\x1a\xc8\xff\0\0\0\0\xbf\xa3\0\0\0\0\0\0\x07\x03\0\0\xc8\xff\xff\xff\x79\xa1\
+\x38\xff\0\0\0\0\x79\xa2\x40\xff\0\0\0\0\xb7\x04\0\0\x08\0\0\0\xb7\x05\0\0\x01\
+\0\0\0\x85\0\0\0\x44\0\0\0\x67\0\0\0\x20\0\0\0\x77\0\0\0\x20\0\0\0\x55\0\x9d\0\
 \0\0\0\0\x05\0\xa8\xfe\0\0\0\0\x15\x09\xf3\xfe\x87\0\0\0\x05\0\x83\xff\0\0\0\0\
-\xbf\x12\0\0\0\0\0\0\x57\x02\0\0\x08\0\0\0\x15\x02\x9a\0\0\0\0\0\x57\x01\0\0\
+\xbf\x12\0\0\0\0\0\0\x57\x02\0\0\x08\0\0\0\x15\x02\x97\0\0\0\0\0\x57\x01\0\0\
 \x40\0\0\0\xb7\x02\0\0\x0c\0\0\0\xb7\x03\0\0\x0c\0\0\0\x15\x01\x01\0\0\0\0\0\
-\xb7\x03\0\0\x2c\0\0\0\x71\xa4\x5c\xff\0\0\0\0\x15\x04\x01\0\0\0\0\0\xbf\x32\0\
-\0\0\0\0\0\xbf\xa3\0\0\0\0\0\0\x07\x03\0\0\x58\xff\xff\xff\x0f\x23\0\0\0\0\0\0\
+\xb7\x03\0\0\x2c\0\0\0\x71\xa4\x54\xff\0\0\0\0\x15\x04\x01\0\0\0\0\0\xbf\x32\0\
+\0\0\0\0\0\xbf\xa3\0\0\0\0\0\0\x07\x03\0\0\x50\xff\xff\xff\x0f\x23\0\0\0\0\0\0\
 \x61\x32\x04\0\0\0\0\0\x67\x02\0\0\x20\0\0\0\x61\x34\0\0\0\0\0\0\x4f\x42\0\0\0\
-\0\0\0\x7b\x2a\xa8\xff\0\0\0\0\x61\x32\x08\0\0\0\0\0\x61\x33\x0c\0\0\0\0\0\x67\
-\x03\0\0\x20\0\0\0\x4f\x23\0\0\0\0\0\0\x7b\x3a\xb0\xff\0\0\0\0\x71\xa2\x5d\xff\
-\0\0\0\0\x15\x02\x0c\0\0\0\0\0\x15\x01\x0b\0\0\0\0\0\x61\xa1\xa0\xff\0\0\0\0\
-\x67\x01\0\0\x20\0\0\0\x61\xa2\x9c\xff\0\0\0\0\x4f\x21\0\0\0\0\0\0\x7b\x1a\xc0\
-\xff\0\0\0\0\x61\xa1\x98\xff\0\0\0\0\x67\x01\0\0\x20\0\0\0\x61\xa2\x94\xff\0\0\
+\0\0\0\x7b\x2a\xa0\xff\0\0\0\0\x61\x32\x08\0\0\0\0\0\x61\x33\x0c\0\0\0\0\0\x67\
+\x03\0\0\x20\0\0\0\x4f\x23\0\0\0\0\0\0\x7b\x3a\xa8\xff\0\0\0\0\x71\xa2\x55\xff\
+\0\0\0\0\x15\x02\x0c\0\0\0\0\0\x15\x01\x0b\0\0\0\0\0\x61\xa1\x98\xff\0\0\0\0\
+\x67\x01\0\0\x20\0\0\0\x61\xa2\x94\xff\0\0\0\0\x4f\x21\0\0\0\0\0\0\x7b\x1a\xb8\
+\xff\0\0\0\0\x61\xa1\x90\xff\0\0\0\0\x67\x01\0\0\x20\0\0\0\x61\xa2\x8c\xff\0\0\
 \0\0\x05\0\x0a\0\0\0\0\0\xb7\x09\0\0\x2b\0\0\0\x05\0\xae\xff\0\0\0\0\x61\xa1\
-\x80\xff\0\0\0\0\x67\x01\0\0\x20\0\0\0\x61\xa2\x7c\xff\0\0\0\0\x4f\x21\0\0\0\0\
-\0\0\x7b\x1a\xc0\xff\0\0\0\0\x61\xa1\x78\xff\0\0\0\0\x67\x01\0\0\x20\0\0\0\x61\
-\xa2\x74\xff\0\0\0\0\x4f\x21\0\0\0\0\0\0\x7b\x1a\xb8\xff\0\0\0\0\xb7\x02\0\0\0\
+\x78\xff\0\0\0\0\x67\x01\0\0\x20\0\0\0\x61\xa2\x74\xff\0\0\0\0\x4f\x21\0\0\0\0\
+\0\0\x7b\x1a\xb8\xff\0\0\0\0\x61\xa1\x70\xff\0\0\0\0\x67\x01\0\0\x20\0\0\0\x61\
+\xa2\x6c\xff\0\0\0\0\x4f\x21\0\0\0\0\0\0\x7b\x1a\xb0\xff\0\0\0\0\xb7\x02\0\0\0\
 \0\0\0\x07\x08\0\0\x04\0\0\0\x61\x03\0\0\0\0\0\0\xb7\x05\0\0\0\0\0\0\xbf\xa1\0\
-\0\0\0\0\0\x07\x01\0\0\xa8\xff\xff\xff\x0f\x21\0\0\0\0\0\0\x71\x14\0\0\0\0\0\0\
+\0\0\0\0\0\x07\x01\0\0\xa0\xff\xff\xff\x0f\x21\0\0\0\0\0\0\x71\x14\0\0\0\0\0\0\
 \xbf\x41\0\0\0\0\0\0\x67\x01\0\0\x38\0\0\0\xc7\x01\0\0\x3f\0\0\0\x5f\x31\0\0\0\
 \0\0\0\xaf\x51\0\0\0\0\0\0\xbf\x85\0\0\0\0\0\0\x0f\x25\0\0\0\0\0\0\x71\x55\0\0\
 \0\0\0\0\x67\x03\0\0\x01\0\0\0\xbf\x50\0\0\0\0\0\0\x77\0\0\0\x07\0\0\0\x4f\x03\
@@ -XXX,XX +XXX,XX @@ static inline const void *rss_bpf__elf_bytes(size_t *sz)
 \0\x57\0\0\0\x01\0\0\0\x67\x03\0\0\x01\0\0\0\x4f\x03\0\0\0\0\0\0\x57\x04\0\0\
 \x01\0\0\0\x87\x04\0\0\0\0\0\0\x5f\x34\0\0\0\0\0\0\xaf\x41\0\0\0\0\0\0\x57\x05\
 \0\0\x01\0\0\0\x67\x03\0\0\x01\0\0\0\x4f\x53\0\0\0\0\0\0\x07\x02\0\0\x01\0\0\0\
-\xbf\x15\0\0\0\0\0\0\x15\x02\x01\0\x24\0\0\0\x05\0\xa9\xff\0\0\0\0\xbf\x12\0\0\
-\0\0\0\0\x67\x02\0\0\x20\0\0\0\x77\x02\0\0\x20\0\0\0\x15\x02\x0e\0\0\0\0\0\x71\
-\x63\x06\0\0\0\0\0\x71\x64\x07\0\0\0\0\0\x67\x04\0\0\x08\0\0\0\x4f\x34\0\0\0\0\
-\0\0\x3f\x42\0\0\0\0\0\0\x2f\x42\0\0\0\0\0\0\x1f\x21\0\0\0\0\0\0\x63\x1a\x58\
-\xff\0\0\0\0\xbf\xa2\0\0\0\0\0\0\x07\x02\0\0\x58\xff\xff\xff\x18\x01\0\0\0\0\0\
-\0\0\0\0\0\0\0\0\0\x85\0\0\0\x01\0\0\0\x55\0\x05\0\0\0\0\0\x71\x61\x08\0\0\0\0\
-\0\x71\x60\x09\0\0\0\0\0\x67\0\0\0\x08\0\0\0\x4f\x10\0\0\0\0\0\0\x95\0\0\0\0\0\
-\0\0\x69\0\0\0\0\0\0\0\x05\0\xfd\xff\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\
+\xbf\x15\0\0\0\0\0\0\x15\x02\x01\0\x24\0\0\0\x05\0\xa9\xff\0\0\0\0\x67\x01\0\0\
+\x20\0\0\0\x77\x01\0\0\x20\0\0\0\x15\x01\x0c\0\0\0\0\0\x71\x62\x06\0\0\0\0\0\
+\x71\x63\x07\0\0\0\0\0\x67\x03\0\0\x08\0\0\0\x4f\x23\0\0\0\0\0\0\x9f\x31\0\0\0\
+\0\0\0\x63\x1a\x50\xff\0\0\0\0\xbf\xa2\0\0\0\0\0\0\x07\x02\0\0\x50\xff\xff\xff\
+\x18\x01\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x85\0\0\0\x01\0\0\0\x55\0\x05\0\0\0\0\0\
+\x71\x61\x08\0\0\0\0\0\x71\x60\x09\0\0\0\0\0\x67\0\0\0\x08\0\0\0\x4f\x10\0\0\0\
+\0\0\0\x95\0\0\0\0\0\0\0\x69\0\0\0\0\0\0\0\x05\0\xfd\xff\0\0\0\0\0\0\0\0\0\0\0\
 \0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\
 \0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\
-\0\x47\x50\x4c\x20\x76\x32\0\0\x9f\xeb\x01\0\x18\0\0\0\0\0\0\0\x10\x05\0\0\x10\
-\x05\0\0\x65\x11\0\0\0\0\0\0\0\0\0\x02\x03\0\0\0\x01\0\0\0\0\0\0\x01\x04\0\0\0\
-\x20\0\0\x01\0\0\0\0\0\0\0\x03\0\0\0\0\x02\0\0\0\x04\0\0\0\x02\0\0\0\x05\0\0\0\
-\0\0\0\x01\x04\0\0\0\x20\0\0\0\0\0\0\0\0\0\0\x02\x06\0\0\0\0\0\0\0\0\0\0\x03\0\
-\0\0\0\x02\0\0\0\x04\0\0\0\x04\0\0\0\0\0\0\0\0\0\0\x02\x08\0\0\0\0\0\0\0\0\0\0\
-\x03\0\0\0\0\x02\0\0\0\x04\0\0\0\x0a\0\0\0\0\0\0\0\0\0\0\x02\x0a\0\0\0\0\0\0\0\
-\0\0\0\x03\0\0\0\0\x02\0\0\0\x04\0\0\0\x01\0\0\0\0\0\0\0\x04\0\0\x04\x20\0\0\0\
+\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x47\x50\
+\x4c\x20\x76\x32\0\0\x9f\xeb\x01\0\x18\0\0\0\0\0\0\0\x58\x05\0\0\x58\x05\0\0\
+\x6b\x11\0\0\0\0\0\0\0\0\0\x02\x03\0\0\0\x01\0\0\0\0\0\0\x01\x04\0\0\0\x20\0\0\
+\x01\0\0\0\0\0\0\0\x03\0\0\0\0\x02\0\0\0\x04\0\0\0\x02\0\0\0\x05\0\0\0\0\0\0\
+\x01\x04\0\0\0\x20\0\0\0\0\0\0\0\0\0\0\x02\x06\0\0\0\0\0\0\0\0\0\0\x03\0\0\0\0\
+\x02\0\0\0\x04\0\0\0\x04\0\0\0\0\0\0\0\0\0\0\x02\x08\0\0\0\0\0\0\0\0\0\0\x03\0\
+\0\0\0\x02\0\0\0\x04\0\0\0\x0a\0\0\0\0\0\0\0\0\0\0\x02\x0a\0\0\0\0\0\0\0\0\0\0\
+\x03\0\0\0\0\x02\0\0\0\x04\0\0\0\x01\0\0\0\0\0\0\0\0\0\0\x02\x0c\0\0\0\0\0\0\0\
+\0\0\0\x03\0\0\0\0\x02\0\0\0\x04\0\0\0\0\x04\0\0\0\0\0\0\x05\0\0\x04\x28\0\0\0\
 \x19\0\0\0\x01\0\0\0\0\0\0\0\x1e\0\0\0\x05\0\0\0\x40\0\0\0\x27\0\0\0\x07\0\0\0\
-\x80\0\0\0\x32\0\0\0\x09\0\0\0\xc0\0\0\0\x3e\0\0\0\0\0\0\x0e\x0b\0\0\0\x01\0\0\
-\0\0\0\0\0\0\0\0\x02\x0e\0\0\0\0\0\0\0\0\0\0\x03\0\0\0\0\x02\0\0\0\x04\0\0\0\
-\x28\0\0\0\0\0\0\0\x04\0\0\x04\x20\0\0\0\x19\0\0\0\x01\0\0\0\0\0\0\0\x1e\0\0\0\
-\x05\0\0\0\x40\0\0\0\x27\0\0\0\x0d\0\0\0\x80\0\0\0\x32\0\0\0\x09\0\0\0\xc0\0\0\
-\0\x59\0\0\0\0\0\0\x0e\x0f\0\0\0\x01\0\0\0\0\0\0\0\0\0\0\x02\x12\0\0\0\0\0\0\0\
-\0\0\0\x03\0\0\0\0\x02\0\0\0\x04\0\0\0\x80\0\0\0\0\0\0\0\x04\0\0\x04\x20\0\0\0\
-\x19\0\0\0\x01\0\0\0\0\0\0\0\x1e\0\0\0\x05\0\0\0\x40\0\0\0\x27\0\0\0\x01\0\0\0\
-\x80\0\0\0\x32\0\0\0\x11\0\0\0\xc0\0\0\0\x72\0\0\0\0\0\0\x0e\x13\0\0\0\x01\0\0\
-\0\0\0\0\0\0\0\0\x02\x16\0\0\0\x90\0\0\0\x22\0\0\x04\xc0\0\0\0\x9a\0\0\0\x17\0\
-\0\0\0\0\0\0\x9e\0\0\0\x17\0\0\0\x20\0\0\0\xa7\0\0\0\x17\0\0\0\x40\0\0\0\xac\0\
-\0\0\x17\0\0\0\x60\0\0\0\xba\0\0\0\x17\0\0\0\x80\0\0\0\xc3\0\0\0\x17\0\0\0\xa0\
-\0\0\0\xd0\0\0\0\x17\0\0\0\xc0\0\0\0\xd9\0\0\0\x17\0\0\0\xe0\0\0\0\xe4\0\0\0\
-\x17\0\0\0\0\x01\0\0\xed\0\0\0\x17\0\0\0\x20\x01\0\0\xfd\0\0\0\x17\0\0\0\x40\
-\x01\0\0\x05\x01\0\0\x17\0\0\0\x60\x01\0\0\x0e\x01\0\0\x19\0\0\0\x80\x01\0\0\
-\x11\x01\0\0\x17\0\0\0\x20\x02\0\0\x16\x01\0\0\x17\0\0\0\x40\x02\0\0\x21\x01\0\
-\0\x17\0\0\0\x60\x02\0\0\x26\x01\0\0\x17\0\0\0\x80\x02\0\0\x2f\x01\0\0\x17\0\0\
-\0\xa0\x02\0\0\x37\x01\0\0\x17\0\0\0\xc0\x02\0\0\x3e\x01\0\0\x17\0\0\0\xe0\x02\
-\0\0\x49\x01\0\0\x17\0\0\0\0\x03\0\0\x53\x01\0\0\x1a\0\0\0\x20\x03\0\0\x5e\x01\
-\0\0\x1a\0\0\0\xa0\x03\0\0\x68\x01\0\0\x17\0\0\0\x20\x04\0\0\x74\x01\0\0\x17\0\
-\0\0\x40\x04\0\0\x7f\x01\0\0\x17\0\0\0\x60\x04\0\0\0\0\0\0\x1b\0\0\0\x80\x04\0\
-\0\x89\x01\0\0\x1d\0\0\0\xc0\x04\0\0\x90\x01\0\0\x17\0\0\0\0\x05\0\0\x99\x01\0\
-\0\x17\0\0\0\x20\x05\0\0\0\0\0\0\x1f\0\0\0\x40\x05\0\0\xa2\x01\0\0\x17\0\0\0\
-\x80\x05\0\0\xab\x01\0\0\x21\0\0\0\xa0\x05\0\0\xb7\x01\0\0\x1d\0\0\0\xc0\x05\0\
-\0\xc0\x01\0\0\0\0\0\x08\x18\0\0\0\xc6\x01\0\0\0\0\0\x01\x04\0\0\0\x20\0\0\0\0\
-\0\0\0\0\0\0\x03\0\0\0\0\x17\0\0\0\x04\0\0\0\x05\0\0\0\0\0\0\0\0\0\0\x03\0\0\0\
-\0\x17\0\0\0\x04\0\0\0\x04\0\0\0\0\0\0\0\x01\0\0\x05\x08\0\0\0\xd3\x01\0\0\x1c\
-\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x02\x2a\0\0\0\xdd\x01\0\0\0\0\0\x08\x1e\0\0\0\xe3\
-\x01\0\0\0\0\0\x01\x08\0\0\0\x40\0\0\0\0\0\0\0\x01\0\0\x05\x08\0\0\0\xf6\x01\0\
-\0\x20\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x02\x2b\0\0\0\xf9\x01\0\0\0\0\0\x08\x22\0\0\
-\0\xfe\x01\0\0\0\0\0\x01\x01\0\0\0\x08\0\0\0\0\0\0\0\x01\0\0\x0d\x02\0\0\0\x0c\
-\x02\0\0\x15\0\0\0\x10\x02\0\0\x01\0\0\x0c\x23\0\0\0\x32\x11\0\0\0\0\0\x01\x01\
-\0\0\0\x08\0\0\x01\0\0\0\0\0\0\0\x03\0\0\0\0\x25\0\0\0\x04\0\0\0\x07\0\0\0\x37\
-\x11\0\0\0\0\0\x0e\x26\0\0\0\x01\0\0\0\x40\x11\0\0\x03\0\0\x0f\0\0\0\0\x0c\0\0\
-\0\0\0\0\0\x20\0\0\0\x10\0\0\0\0\0\0\0\x20\0\0\0\x14\0\0\0\0\0\0\0\x20\0\0\0\
-\x46\x11\0\0\x01\0\0\x0f\0\0\0\0\x27\0\0\0\0\0\0\0\x07\0\0\0\x4e\x11\0\0\0\0\0\
-\x07\0\0\0\0\x5c\x11\0\0\0\0\0\x07\0\0\0\0\0\x69\x6e\x74\0\x5f\x5f\x41\x52\x52\
-\x41\x59\x5f\x53\x49\x5a\x45\x5f\x54\x59\x50\x45\x5f\x5f\0\x74\x79\x70\x65\0\
-\x6b\x65\x79\x5f\x73\x69\x7a\x65\0\x76\x61\x6c\x75\x65\x5f\x73\x69\x7a\x65\0\
-\x6d\x61\x78\x5f\x65\x6e\x74\x72\x69\x65\x73\0\x74\x61\x70\x5f\x72\x73\x73\x5f\
-\x6d\x61\x70\x5f\x63\x6f\x6e\x66\x69\x67\x75\x72\x61\x74\x69\x6f\x6e\x73\0\x74\
+\x80\0\0\0\x32\0\0\0\x09\0\0\0\xc0\0\0\0\x3e\0\0\0\x0b\0\0\0\0\x01\0\0\x48\0\0\
+\0\0\0\0\x0e\x0d\0\0\0\x01\0\0\0\0\0\0\0\0\0\0\x02\x10\0\0\0\0\0\0\0\0\0\0\x03\
+\0\0\0\0\x02\0\0\0\x04\0\0\0\x28\0\0\0\0\0\0\0\x05\0\0\x04\x28\0\0\0\x19\0\0\0\
+\x01\0\0\0\0\0\0\0\x1e\0\0\0\x05\0\0\0\x40\0\0\0\x27\0\0\0\x0f\0\0\0\x80\0\0\0\
+\x32\0\0\0\x09\0\0\0\xc0\0\0\0\x3e\0\0\0\x0b\0\0\0\0\x01\0\0\x63\0\0\0\0\0\0\
+\x0e\x11\0\0\0\x01\0\0\0\0\0\0\0\0\0\0\x02\x14\0\0\0\0\0\0\0\0\0\0\x03\0\0\0\0\
+\x02\0\0\0\x04\0\0\0\x80\0\0\0\0\0\0\0\x05\0\0\x04\x28\0\0\0\x19\0\0\0\x01\0\0\
+\0\0\0\0\0\x1e\0\0\0\x05\0\0\0\x40\0\0\0\x27\0\0\0\x01\0\0\0\x80\0\0\0\x32\0\0\
+\0\x13\0\0\0\xc0\0\0\0\x3e\0\0\0\x0b\0\0\0\0\x01\0\0\x7c\0\0\0\0\0\0\x0e\x15\0\
+\0\0\x01\0\0\0\0\0\0\0\0\0\0\x02\x18\0\0\0\x9a\0\0\0\x22\0\0\x04\xc0\0\0\0\xa4\
+\0\0\0\x19\0\0\0\0\0\0\0\xa8\0\0\0\x19\0\0\0\x20\0\0\0\xb1\0\0\0\x19\0\0\0\x40\
+\0\0\0\xb6\0\0\0\x19\0\0\0\x60\0\0\0\xc4\0\0\0\x19\0\0\0\x80\0\0\0\xcd\0\0\0\
+\x19\0\0\0\xa0\0\0\0\xda\0\0\0\x19\0\0\0\xc0\0\0\0\xe3\0\0\0\x19\0\0\0\xe0\0\0\
+\0\xee\0\0\0\x19\0\0\0\0\x01\0\0\xf7\0\0\0\x19\0\0\0\x20\x01\0\0\x07\x01\0\0\
+\x19\0\0\0\x40\x01\0\0\x0f\x01\0\0\x19\0\0\0\x60\x01\0\0\x18\x01\0\0\x1b\0\0\0\
+\x80\x01\0\0\x1b\x01\0\0\x19\0\0\0\x20\x02\0\0\x20\x01\0\0\x19\0\0\0\x40\x02\0\
+\0\x2b\x01\0\0\x19\0\0\0\x60\x02\0\0\x30\x01\0\0\x19\0\0\0\x80\x02\0\0\x39\x01\
+\0\0\x19\0\0\0\xa0\x02\0\0\x41\x01\0\0\x19\0\0\0\xc0\x02\0\0\x48\x01\0\0\x19\0\
+\0\0\xe0\x02\0\0\x53\x01\0\0\x19\0\0\0\0\x03\0\0\x5d\x01\0\0\x1c\0\0\0\x20\x03\
+\0\0\x68\x01\0\0\x1c\0\0\0\xa0\x03\0\0\x72\x01\0\0\x19\0\0\0\x20\x04\0\0\x7e\
+\x01\0\0\x19\0\0\0\x40\x04\0\0\x89\x01\0\0\x19\0\0\0\x60\x04\0\0\0\0\0\0\x1d\0\
+\0\0\x80\x04\0\0\x93\x01\0\0\x1f\0\0\0\xc0\x04\0\0\x9a\x01\0\0\x19\0\0\0\0\x05\
+\0\0\xa3\x01\0\0\x19\0\0\0\x20\x05\0\0\0\0\0\0\x21\0\0\0\x40\x05\0\0\xac\x01\0\
+\0\x19\0\0\0\x80\x05\0\0\xb5\x01\0\0\x23\0\0\0\xa0\x05\0\0\xc1\x01\0\0\x1f\0\0\
+\0\xc0\x05\0\0\xca\x01\0\0\0\0\0\x08\x1a\0\0\0\xd0\x01\0\0\0\0\0\x01\x04\0\0\0\
+\x20\0\0\0\0\0\0\0\0\0\0\x03\0\0\0\0\x19\0\0\0\x04\0\0\0\x05\0\0\0\0\0\0\0\0\0\
+\0\x03\0\0\0\0\x19\0\0\0\x04\0\0\0\x04\0\0\0\0\0\0\0\x01\0\0\x05\x08\0\0\0\xdd\
+\x01\0\0\x1e\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x02\x2c\0\0\0\xe7\x01\0\0\0\0\0\x08\
+\x20\0\0\0\xed\x01\0\0\0\0\0\x01\x08\0\0\0\x40\0\0\0\0\0\0\0\x01\0\0\x05\x08\0\
+\0\0\0\x02\0\0\x22\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x02\x2d\0\0\0\x03\x02\0\0\0\0\0\
+\x08\x24\0\0\0\x08\x02\0\0\0\0\0\x01\x01\0\0\0\x08\0\0\0\0\0\0\0\x01\0\0\x0d\
+\x02\0\0\0\x16\x02\0\0\x17\0\0\0\x1a\x02\0\0\x01\0\0\x0c\x25\0\0\0\x38\x11\0\0\
+\0\0\0\x01\x01\0\0\0\x08\0\0\x01\0\0\0\0\0\0\0\x03\0\0\0\0\x27\0\0\0\x04\0\0\0\
+\x07\0\0\0\x3d\x11\0\0\0\0\0\x0e\x28\0\0\0\x01\0\0\0\x46\x11\0\0\x03\0\0\x0f\0\
+\0\0\0\x0e\0\0\0\0\0\0\0\x28\0\0\0\x12\0\0\0\0\0\0\0\x28\0\0\0\x16\0\0\0\0\0\0\
+\0\x28\0\0\0\x4c\x11\0\0\x01\0\0\x0f\0\0\0\0\x29\0\0\0\0\0\0\0\x07\0\0\0\x54\
+\x11\0\0\0\0\0\x07\0\0\0\0\x62\x11\0\0\0\0\0\x07\0\0\0\0\0\x69\x6e\x74\0\x5f\
+\x5f\x41\x52\x52\x41\x59\x5f\x53\x49\x5a\x45\x5f\x54\x59\x50\x45\x5f\x5f\0\x74\
+\x79\x70\x65\0\x6b\x65\x79\x5f\x73\x69\x7a\x65\0\x76\x61\x6c\x75\x65\x5f\x73\
+\x69\x7a\x65\0\x6d\x61\x78\x5f\x65\x6e\x74\x72\x69\x65\x73\0\x6d\x61\x70\x5f\
+\x66\x6c\x61\x67\x73\0\x74\x61\x70\x5f\x72\x73\x73\x5f\x6d\x61\x70\x5f\x63\x6f\
+\x6e\x66\x69\x67\x75\x72\x61\x74\x69\x6f\x6e\x73\0\x74\x61\x70\x5f\x72\x73\x73\
+\x5f\x6d\x61\x70\x5f\x74\x6f\x65\x70\x6c\x69\x74\x7a\x5f\x6b\x65\x79\0\x74\x61\
+\x70\x5f\x72\x73\x73\x5f\x6d\x61\x70\x5f\x69\x6e\x64\x69\x72\x65\x63\x74\x69\
+\x6f\x6e\x5f\x74\x61\x62\x6c\x65\0\x5f\x5f\x73\x6b\x5f\x62\x75\x66\x66\0\x6c\
+\x65\x6e\0\x70\x6b\x74\x5f\x74\x79\x70\x65\0\x6d\x61\x72\x6b\0\x71\x75\x65\x75\
+\x65\x5f\x6d\x61\x70\x70\x69\x6e\x67\0\x70\x72\x6f\x74\x6f\x63\x6f\x6c\0\x76\
+\x6c\x61\x6e\x5f\x70\x72\x65\x73\x65\x6e\x74\0\x76\x6c\x61\x6e\x5f\x74\x63\x69\
+\0\x76\x6c\x61\x6e\x5f\x70\x72\x6f\x74\x6f\0\x70\x72\x69\x6f\x72\x69\x74\x79\0\
+\x69\x6e\x67\x72\x65\x73\x73\x5f\x69\x66\x69\x6e\x64\x65\x78\0\x69\x66\x69\x6e\
+\x64\x65\x78\0\x74\x63\x5f\x69\x6e\x64\x65\x78\0\x63\x62\0\x68\x61\x73\x68\0\
+\x74\x63\x5f\x63\x6c\x61\x73\x73\x69\x64\0\x64\x61\x74\x61\0\x64\x61\x74\x61\
+\x5f\x65\x6e\x64\0\x6e\x61\x70\x69\x5f\x69\x64\0\x66\x61\x6d\x69\x6c\x79\0\x72\
+\x65\x6d\x6f\x74\x65\x5f\x69\x70\x34\0\x6c\x6f\x63\x61\x6c\x5f\x69\x70\x34\0\
+\x72\x65\x6d\x6f\x74\x65\x5f\x69\x70\x36\0\x6c\x6f\x63\x61\x6c\x5f\x69\x70\x36\
+\0\x72\x65\x6d\x6f\x74\x65\x5f\x70\x6f\x72\x74\0\x6c\x6f\x63\x61\x6c\x5f\x70\
+\x6f\x72\x74\0\x64\x61\x74\x61\x5f\x6d\x65\x74\x61\0\x74\x73\x74\x61\x6d\x70\0\
+\x77\x69\x72\x65\x5f\x6c\x65\x6e\0\x67\x73\x6f\x5f\x73\x65\x67\x73\0\x67\x73\
+\x6f\x5f\x73\x69\x7a\x65\0\x74\x73\x74\x61\x6d\x70\x5f\x74\x79\x70\x65\0\x68\
+\x77\x74\x73\x74\x61\x6d\x70\0\x5f\x5f\x75\x33\x32\0\x75\x6e\x73\x69\x67\x6e\
+\x65\x64\x20\x69\x6e\x74\0\x66\x6c\x6f\x77\x5f\x6b\x65\x79\x73\0\x5f\x5f\x75\
+\x36\x34\0\x75\x6e\x73\x69\x67\x6e\x65\x64\x20\x6c\x6f\x6e\x67\x20\x6c\x6f\x6e\
+\x67\0\x73\x6b\0\x5f\x5f\x75\x38\0\x75\x6e\x73\x69\x67\x6e\x65\x64\x20\x63\x68\
+\x61\x72\0\x73\x6b\x62\0\x74\x75\x6e\x5f\x72\x73\x73\x5f\x73\x74\x65\x65\x72\
+\x69\x6e\x67\x5f\x70\x72\x6f\x67\0\x73\x6f\x63\x6b\x65\x74\0\x2f\x68\x6f\x6d\
+\x65\x2f\x61\x6e\x64\x2f\x57\x6f\x72\x6b\x2f\x44\x61\x79\x6e\x69\x78\x2f\x71\
+\x65\x6d\x75\x2f\x74\x6f\x6f\x6c\x73\x2f\x65\x62\x70\x66\x2f\x72\x73\x73\x2e\
+\x62\x70\x66\x2e\x63\0\x69\x6e\x74\x20\x74\x75\x6e\x5f\x72\x73\x73\x5f\x73\x74\
+\x65\x65\x72\x69\x6e\x67\x5f\x70\x72\x6f\x67\x28\x73\x74\x72\x75\x63\x74\x20\
+\x5f\x5f\x73\x6b\x5f\x62\x75\x66\x66\x20\x2a\x73\x6b\x62\x29\0\x20\x20\x20\x20\
+\x5f\x5f\x75\x33\x32\x20\x6b\x65\x79\x20\x3d\x20\x30\x3b\0\x20\x20\x20\x20\x63\
+\x6f\x6e\x66\x69\x67\x20\x3d\x20\x62\x70\x66\x5f\x6d\x61\x70\x5f\x6c\x6f\x6f\
+\x6b\x75\x70\x5f\x65\x6c\x65\x6d\x28\x26\x74\x61\x70\x5f\x72\x73\x73\x5f\x6d\
+\x61\x70\x5f\x63\x6f\x6e\x66\x69\x67\x75\x72\x61\x74\x69\x6f\x6e\x73\x2c\x20\
+\x26\x6b\x65\x79\x29\x3b\0\x20\x20\x20\x20\x74\x6f\x65\x20\x3d\x20\x62\x70\x66\
+\x5f\x6d\x61\x70\x5f\x6c\x6f\x6f\x6b\x75\x70\x5f\x65\x6c\x65\x6d\x28\x26\x74\
 \x61\x70\x5f\x72\x73\x73\x5f\x6d\x61\x70\x5f\x74\x6f\x65\x70\x6c\x69\x74\x7a\
-\x5f\x6b\x65\x79\0\x74\x61\x70\x5f\x72\x73\x73\x5f\x6d\x61\x70\x5f\x69\x6e\x64\
-\x69\x72\x65\x63\x74\x69\x6f\x6e\x5f\x74\x61\x62\x6c\x65\0\x5f\x5f\x73\x6b\x5f\
-\x62\x75\x66\x66\0\x6c\x65\x6e\0\x70\x6b\x74\x5f\x74\x79\x70\x65\0\x6d\x61\x72\
-\x6b\0\x71\x75\x65\x75\x65\x5f\x6d\x61\x70\x70\x69\x6e\x67\0\x70\x72\x6f\x74\
-\x6f\x63\x6f\x6c\0\x76\x6c\x61\x6e\x5f\x70\x72\x65\x73\x65\x6e\x74\0\x76\x6c\
-\x61\x6e\x5f\x74\x63\x69\0\x76\x6c\x61\x6e\x5f\x70\x72\x6f\x74\x6f\0\x70\x72\
-\x69\x6f\x72\x69\x74\x79\0\x69\x6e\x67\x72\x65\x73\x73\x5f\x69\x66\x69\x6e\x64\
-\x65\x78\0\x69\x66\x69\x6e\x64\x65\x78\0\x74\x63\x5f\x69\x6e\x64\x65\x78\0\x63\
-\x62\0\x68\x61\x73\x68\0\x74\x63\x5f\x63\x6c\x61\x73\x73\x69\x64\0\x64\x61\x74\
-\x61\0\x64\x61\x74\x61\x5f\x65\x6e\x64\0\x6e\x61\x70\x69\x5f\x69\x64\0\x66\x61\
-\x6d\x69\x6c\x79\0\x72\x65\x6d\x6f\x74\x65\x5f\x69\x70\x34\0\x6c\x6f\x63\x61\
-\x6c\x5f\x69\x70\x34\0\x72\x65\x6d\x6f\x74\x65\x5f\x69\x70\x36\0\x6c\x6f\x63\
-\x61\x6c\x5f\x69\x70\x36\0\x72\x65\x6d\x6f\x74\x65\x5f\x70\x6f\x72\x74\0\x6c\
-\x6f\x63\x61\x6c\x5f\x70\x6f\x72\x74\0\x64\x61\x74\x61\x5f\x6d\x65\x74\x61\0\
-\x74\x73\x74\x61\x6d\x70\0\x77\x69\x72\x65\x5f\x6c\x65\x6e\0\x67\x73\x6f\x5f\
-\x73\x65\x67\x73\0\x67\x73\x6f\x5f\x73\x69\x7a\x65\0\x74\x73\x74\x61\x6d\x70\
-\x5f\x74\x79\x70\x65\0\x68\x77\x74\x73\x74\x61\x6d\x70\0\x5f\x5f\x75\x33\x32\0\
-\x75\x6e\x73\x69\x67\x6e\x65\x64\x20\x69\x6e\x74\0\x66\x6c\x6f\x77\x5f\x6b\x65\
-\x79\x73\0\x5f\x5f\x75\x36\x34\0\x75\x6e\x73\x69\x67\x6e\x65\x64\x20\x6c\x6f\
-\x6e\x67\x20\x6c\x6f\x6e\x67\0\x73\x6b\0\x5f\x5f\x75\x38\0\x75\x6e\x73\x69\x67\
-\x6e\x65\x64\x20\x63\x68\x61\x72\0\x73\x6b\x62\0\x74\x75\x6e\x5f\x72\x73\x73\
-\x5f\x73\x74\x65\x65\x72\x69\x6e\x67\x5f\x70\x72\x6f\x67\0\x74\x75\x6e\x5f\x72\
-\x73\x73\x5f\x73\x74\x65\x65\x72\x69\x6e\x67\0\x2f\x68\x6f\x6d\x65\x2f\x73\x68\
-\x72\x65\x65\x73\x68\x2f\x63\x2f\x71\x65\x6d\x75\x2f\x74\x6f\x6f\x6c\x73\x2f\
-\x65\x62\x70\x66\x2f\x72\x73\x73\x2e\x62\x70\x66\x2e\x63\0\x69\x6e\x74\x20\x74\
-\x75\x6e\x5f\x72\x73\x73\x5f\x73\x74\x65\x65\x72\x69\x6e\x67\x5f\x70\x72\x6f\
-\x67\x28\x73\x74\x72\x75\x63\x74\x20\x5f\x5f\x73\x6b\x5f\x62\x75\x66\x66\x20\
-\x2a\x73\x6b\x62\x29\0\x20\x20\x20\x20\x5f\x5f\x75\x33\x32\x20\x6b\x65\x79\x20\
-\x3d\x20\x30\x3b\0\x20\x20\x20\x20\x63\x6f\x6e\x66\x69\x67\x20\x3d\x20\x62\x70\
-\x66\x5f\x6d\x61\x70\x5f\x6c\x6f\x6f\x6b\x75\x70\x5f\x65\x6c\x65\x6d\x28\x26\
-\x74\x61\x70\x5f\x72\x73\x73\x5f\x6d\x61\x70\x5f\x63\x6f\x6e\x66\x69\x67\x75\
-\x72\x61\x74\x69\x6f\x6e\x73\x2c\x20\x26\x6b\x65\x79\x29\x3b\0\x20\x20\x20\x20\
-\x74\x6f\x65\x20\x3d\x20\x62\x70\x66\x5f\x6d\x61\x70\x5f\x6c\x6f\x6f\x6b\x75\
-\x70\x5f\x65\x6c\x65\x6d\x28\x26\x74\x61\x70\x5f\x72\x73\x73\x5f\x6d\x61\x70\
-\x5f\x74\x6f\x65\x70\x6c\x69\x74\x7a\x5f\x6b\x65\x79\x2c\x20\x26\x6b\x65\x79\
-\x29\x3b\0\x20\x20\x20\x20\x69\x66\x20\x28\x63\x6f\x6e\x66\x69\x67\x20\x26\x26\
-\x20\x74\x6f\x65\x29\x20\x7b\0\x20\x20\x20\x20\x20\x20\x20\x20\x69\x66\x20\x28\
-\x21\x63\x6f\x6e\x66\x69\x67\x2d\x3e\x72\x65\x64\x69\x72\x65\x63\x74\x29\x20\
-\x7b\0\x20\x20\x20\x20\x5f\x5f\x75\x38\x20\x72\x73\x73\x5f\x69\x6e\x70\x75\x74\
-\x5b\x48\x41\x53\x48\x5f\x43\x41\x4c\x43\x55\x4c\x41\x54\x49\x4f\x4e\x5f\x42\
-\x55\x46\x46\x45\x52\x5f\x53\x49\x5a\x45\x5d\x20\x3d\x20\x7b\x7d\x3b\0\x20\x20\
-\x20\x20\x73\x74\x72\x75\x63\x74\x20\x70\x61\x63\x6b\x65\x74\x5f\x68\x61\x73\
-\x68\x5f\x69\x6e\x66\x6f\x5f\x74\x20\x70\x61\x63\x6b\x65\x74\x5f\x69\x6e\x66\
-\x6f\x20\x3d\x20\x7b\x7d\x3b\0\x20\x20\x20\x20\x69\x66\x20\x28\x21\x69\x6e\x66\
-\x6f\x20\x7c\x7c\x20\x21\x73\x6b\x62\x29\x20\x7b\0\x20\x20\x20\x20\x5f\x5f\x62\
-\x65\x31\x36\x20\x72\x65\x74\x20\x3d\x20\x30\x3b\0\x20\x20\x20\x20\x65\x72\x72\
-\x20\x3d\x20\x62\x70\x66\x5f\x73\x6b\x62\x5f\x6c\x6f\x61\x64\x5f\x62\x79\x74\
-\x65\x73\x5f\x72\x65\x6c\x61\x74\x69\x76\x65\x28\x73\x6b\x62\x2c\x20\x6f\x66\
-\x66\x73\x65\x74\x2c\x20\x26\x72\x65\x74\x2c\x20\x73\x69\x7a\x65\x6f\x66\x28\
-\x72\x65\x74\x29\x2c\0\x20\x20\x20\x20\x69\x66\x20\x28\x65\x72\x72\x29\x20\x7b\
-\0\x20\x20\x20\x20\x73\x77\x69\x74\x63\x68\x20\x28\x62\x70\x66\x5f\x6e\x74\x6f\
-\x68\x73\x28\x72\x65\x74\x29\x29\x20\x7b\0\x20\x20\x20\x20\x20\x20\x20\x20\x65\
-\x72\x72\x20\x3d\x20\x62\x70\x66\x5f\x73\x6b\x62\x5f\x6c\x6f\x61\x64\x5f\x62\
-\x79\x74\x65\x73\x5f\x72\x65\x6c\x61\x74\x69\x76\x65\x28\x73\x6b\x62\x2c\x20\
-\x6f\x66\x66\x73\x65\x74\x2c\x20\x26\x72\x65\x74\x2c\x20\x73\x69\x7a\x65\x6f\
-\x66\x28\x72\x65\x74\x29\x2c\0\x20\x20\x20\x20\x72\x65\x74\x75\x72\x6e\x20\x72\
-\x65\x74\x3b\0\x20\x20\x20\x20\x69\x66\x20\x28\x6c\x33\x5f\x70\x72\x6f\x74\x6f\
-\x63\x6f\x6c\x20\x3d\x3d\x20\x30\x29\x20\x7b\0\x20\x20\x20\x20\x20\x20\x20\x20\
-\x69\x6e\x66\x6f\x2d\x3e\x69\x73\x5f\x69\x70\x76\x34\x20\x3d\x20\x31\x3b\0\x20\
-\x20\x20\x20\x20\x20\x20\x20\x73\x74\x72\x75\x63\x74\x20\x69\x70\x68\x64\x72\
-\x20\x69\x70\x20\x3d\x20\x7b\x7d\x3b\0\x20\x20\x20\x20\x20\x20\x20\x20\x65\x72\
-\x72\x20\x3d\x20\x62\x70\x66\x5f\x73\x6b\x62\x5f\x6c\x6f\x61\x64\x5f\x62\x79\
-\x74\x65\x73\x5f\x72\x65\x6c\x61\x74\x69\x76\x65\x28\x73\x6b\x62\x2c\x20\x30\
-\x2c\x20\x26\x69\x70\x2c\x20\x73\x69\x7a\x65\x6f\x66\x28\x69\x70\x29\x2c\0\x20\
-\x20\x20\x20\x20\x20\x20\x20\x69\x66\x20\x28\x65\x72\x72\x29\x20\x7b\0\x20\x20\
-\x20\x20\x20\x20\x20\x20\x69\x6e\x66\x6f\x2d\x3e\x69\x73\x5f\x66\x72\x61\x67\
-\x6d\x65\x6e\x74\x65\x64\x20\x3d\x20\x21\x21\x69\x70\x2e\x66\x72\x61\x67\x5f\
-\x6f\x66\x66\x3b\0\x20\x20\x20\x20\x20\x20\x20\x20\x69\x6e\x66\x6f\x2d\x3e\x69\
-\x6e\x5f\x73\x72\x63\x20\x3d\x20\x69\x70\x2e\x73\x61\x64\x64\x72\x3b\0\x20\x20\
-\x20\x20\x20\x20\x20\x20\x69\x6e\x66\x6f\x2d\x3e\x69\x6e\x5f\x64\x73\x74\x20\
-\x3d\x20\x69\x70\x2e\x64\x61\x64\x64\x72\x3b\0\x20\x20\x20\x20\x20\x20\x20\x20\
-\x6c\x34\x5f\x70\x72\x6f\x74\x6f\x63\x6f\x6c\x20\x3d\x20\x69\x70\x2e\x70\x72\
-\x6f\x74\x6f\x63\x6f\x6c\x3b\0\x20\x20\x20\x20\x20\x20\x20\x20\x6c\x34\x5f\x6f\
-\x66\x66\x73\x65\x74\x20\x3d\x20\x69\x70\x2e\x69\x68\x6c\x20\x2a\x20\x34\x3b\0\
-\x20\x20\x20\x20\x69\x66\x20\x28\x6c\x34\x5f\x70\x72\x6f\x74\x6f\x63\x6f\x6c\
-\x20\x21\x3d\x20\x30\x20\x26\x26\x20\x21\x69\x6e\x66\x6f\x2d\x3e\x69\x73\x5f\
-\x66\x72\x61\x67\x6d\x65\x6e\x74\x65\x64\x29\x20\x7b\0\x20\x20\x20\x20\x20\x20\
-\x20\x20\x69\x66\x20\x28\x6c\x34\x5f\x70\x72\x6f\x74\x6f\x63\x6f\x6c\x20\x3d\
-\x3d\x20\x49\x50\x50\x52\x4f\x54\x4f\x5f\x54\x43\x50\x29\x20\x7b\0\x20\x20\x20\
-\x20\x20\x20\x20\x20\x20\x20\x20\x20\x69\x6e\x66\x6f\x2d\x3e\x69\x73\x5f\x74\
-\x63\x70\x20\x3d\x20\x31\x3b\0\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\
-\x73\x74\x72\x75\x63\x74\x20\x74\x63\x70\x68\x64\x72\x20\x74\x63\x70\x20\x3d\
-\x20\x7b\x7d\x3b\0\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x65\x72\x72\
-\x20\x3d\x20\x62\x70\x66\x5f\x73\x6b\x62\x5f\x6c\x6f\x61\x64\x5f\x62\x79\x74\
-\x65\x73\x5f\x72\x65\x6c\x61\x74\x69\x76\x65\x28\x73\x6b\x62\x2c\x20\x6c\x34\
-\x5f\x6f\x66\x66\x73\x65\x74\x2c\x20\x26\x74\x63\x70\x2c\x20\x73\x69\x7a\x65\
-\x6f\x66\x28\x74\x63\x70\x29\x2c\0\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\
-\x20\x69\x66\x20\x28\x65\x72\x72\x29\x20\x7b\0\x20\x20\x20\x20\x69\x66\x20\x28\
-\x70\x61\x63\x6b\x65\x74\x5f\x69\x6e\x66\x6f\x2e\x69\x73\x5f\x69\x70\x76\x34\
-\x29\x20\x7b\0\x20\x20\x20\x20\x20\x20\x20\x20\x69\x66\x20\x28\x70\x61\x63\x6b\
-\x65\x74\x5f\x69\x6e\x66\x6f\x2e\x69\x73\x5f\x74\x63\x70\x20\x26\x26\0\x20\x20\
-\x20\x20\x20\x20\x20\x20\x69\x6e\x66\x6f\x2d\x3e\x69\x73\x5f\x69\x70\x76\x36\
-\x20\x3d\x20\x31\x3b\0\x20\x20\x20\x20\x20\x20\x20\x20\x73\x74\x72\x75\x63\x74\
-\x20\x69\x70\x76\x36\x68\x64\x72\x20\x69\x70\x36\x20\x3d\x20\x7b\x7d\x3b\0\x20\
-\x20\x20\x20\x20\x20\x20\x20\x65\x72\x72\x20\x3d\x20\x62\x70\x66\x5f\x73\x6b\
+\x5f\x6b\x65\x79\x2c\x20\x26\x6b\x65\x79\x29\x3b\0\x20\x20\x20\x20\x69\x66\x20\
+\x28\x63\x6f\x6e\x66\x69\x67\x20\x26\x26\x20\x74\x6f\x65\x29\x20\x7b\0\x20\x20\
+\x20\x20\x20\x20\x20\x20\x69\x66\x20\x28\x21\x63\x6f\x6e\x66\x69\x67\x2d\x3e\
+\x72\x65\x64\x69\x72\x65\x63\x74\x29\x20\x7b\0\x20\x20\x20\x20\x5f\x5f\x75\x38\
+\x20\x72\x73\x73\x5f\x69\x6e\x70\x75\x74\x5b\x48\x41\x53\x48\x5f\x43\x41\x4c\
+\x43\x55\x4c\x41\x54\x49\x4f\x4e\x5f\x42\x55\x46\x46\x45\x52\x5f\x53\x49\x5a\
+\x45\x5d\x20\x3d\x20\x7b\x7d\x3b\0\x20\x20\x20\x20\x73\x74\x72\x75\x63\x74\x20\
+\x70\x61\x63\x6b\x65\x74\x5f\x68\x61\x73\x68\x5f\x69\x6e\x66\x6f\x5f\x74\x20\
+\x70\x61\x63\x6b\x65\x74\x5f\x69\x6e\x66\x6f\x20\x3d\x20\x7b\x7d\x3b\0\x20\x20\
+\x20\x20\x69\x66\x20\x28\x21\x69\x6e\x66\x6f\x20\x7c\x7c\x20\x21\x73\x6b\x62\
+\x29\x20\x7b\0\x20\x20\x20\x20\x5f\x5f\x62\x65\x31\x36\x20\x72\x65\x74\x20\x3d\
+\x20\x30\x3b\0\x20\x20\x20\x20\x65\x72\x72\x20\x3d\x20\x62\x70\x66\x5f\x73\x6b\
 \x62\x5f\x6c\x6f\x61\x64\x5f\x62\x79\x74\x65\x73\x5f\x72\x65\x6c\x61\x74\x69\
-\x76\x65\x28\x73\x6b\x62\x2c\x20\x30\x2c\x20\x26\x69\x70\x36\x2c\x20\x73\x69\
-\x7a\x65\x6f\x66\x28\x69\x70\x36\x29\x2c\0\x20\x20\x20\x20\x20\x20\x20\x20\x69\
-\x6e\x66\x6f\x2d\x3e\x69\x6e\x36\x5f\x73\x72\x63\x20\x3d\x20\x69\x70\x36\x2e\
-\x73\x61\x64\x64\x72\x3b\0\x20\x20\x20\x20\x20\x20\x20\x20\x69\x6e\x66\x6f\x2d\
-\x3e\x69\x6e\x36\x5f\x64\x73\x74\x20\x3d\x20\x69\x70\x36\x2e\x64\x61\x64\x64\
+\x76\x65\x28\x73\x6b\x62\x2c\x20\x6f\x66\x66\x73\x65\x74\x2c\x20\x26\x72\x65\
+\x74\x2c\x20\x73\x69\x7a\x65\x6f\x66\x28\x72\x65\x74\x29\x2c\0\x20\x20\x20\x20\
+\x69\x66\x20\x28\x65\x72\x72\x29\x20\x7b\0\x20\x20\x20\x20\x73\x77\x69\x74\x63\
+\x68\x20\x28\x62\x70\x66\x5f\x6e\x74\x6f\x68\x73\x28\x72\x65\x74\x29\x29\x20\
+\x7b\0\x20\x20\x20\x20\x20\x20\x20\x20\x65\x72\x72\x20\x3d\x20\x62\x70\x66\x5f\
+\x73\x6b\x62\x5f\x6c\x6f\x61\x64\x5f\x62\x79\x74\x65\x73\x5f\x72\x65\x6c\x61\
+\x74\x69\x76\x65\x28\x73\x6b\x62\x2c\x20\x6f\x66\x66\x73\x65\x74\x2c\x20\x26\
+\x72\x65\x74\x2c\x20\x73\x69\x7a\x65\x6f\x66\x28\x72\x65\x74\x29\x2c\0\x20\x20\
+\x20\x20\x72\x65\x74\x75\x72\x6e\x20\x72\x65\x74\x3b\0\x20\x20\x20\x20\x69\x66\
+\x20\x28\x6c\x33\x5f\x70\x72\x6f\x74\x6f\x63\x6f\x6c\x20\x3d\x3d\x20\x30\x29\
+\x20\x7b\0\x20\x20\x20\x20\x20\x20\x20\x20\x69\x6e\x66\x6f\x2d\x3e\x69\x73\x5f\
+\x69\x70\x76\x34\x20\x3d\x20\x31\x3b\0\x20\x20\x20\x20\x20\x20\x20\x20\x73\x74\
+\x72\x75\x63\x74\x20\x69\x70\x68\x64\x72\x20\x69\x70\x20\x3d\x20\x7b\x7d\x3b\0\
+\x20\x20\x20\x20\x20\x20\x20\x20\x65\x72\x72\x20\x3d\x20\x62\x70\x66\x5f\x73\
+\x6b\x62\x5f\x6c\x6f\x61\x64\x5f\x62\x79\x74\x65\x73\x5f\x72\x65\x6c\x61\x74\
+\x69\x76\x65\x28\x73\x6b\x62\x2c\x20\x30\x2c\x20\x26\x69\x70\x2c\x20\x73\x69\
+\x7a\x65\x6f\x66\x28\x69\x70\x29\x2c\0\x20\x20\x20\x20\x20\x20\x20\x20\x69\x66\
+\x20\x28\x65\x72\x72\x29\x20\x7b\0\x20\x20\x20\x20\x20\x20\x20\x20\x69\x6e\x66\
+\x6f\x2d\x3e\x69\x73\x5f\x66\x72\x61\x67\x6d\x65\x6e\x74\x65\x64\x20\x3d\x20\
+\x21\x21\x69\x70\x2e\x66\x72\x61\x67\x5f\x6f\x66\x66\x3b\0\x20\x20\x20\x20\x20\
+\x20\x20\x20\x69\x6e\x66\x6f\x2d\x3e\x69\x6e\x5f\x73\x72\x63\x20\x3d\x20\x69\
+\x70\x2e\x73\x61\x64\x64\x72\x3b\0\x20\x20\x20\x20\x20\x20\x20\x20\x69\x6e\x66\
+\x6f\x2d\x3e\x69\x6e\x5f\x64\x73\x74\x20\x3d\x20\x69\x70\x2e\x64\x61\x64\x64\
 \x72\x3b\0\x20\x20\x20\x20\x20\x20\x20\x20\x6c\x34\x5f\x70\x72\x6f\x74\x6f\x63\
-\x6f\x6c\x20\x3d\x20\x69\x70\x36\x2e\x6e\x65\x78\x74\x68\x64\x72\x3b\0\x20\x20\
-\x20\x20\x73\x77\x69\x74\x63\x68\x20\x28\x68\x64\x72\x5f\x74\x79\x70\x65\x29\
-\x20\x7b\0\x20\x20\x20\x20\x73\x74\x72\x75\x63\x74\x20\x69\x70\x76\x36\x5f\x6f\
-\x70\x74\x5f\x68\x64\x72\x20\x65\x78\x74\x5f\x68\x64\x72\x20\x3d\x20\x7b\x7d\
-\x3b\0\x20\x20\x20\x20\x20\x20\x20\x20\x65\x72\x72\x20\x3d\x20\x62\x70\x66\x5f\
+\x6f\x6c\x20\x3d\x20\x69\x70\x2e\x70\x72\x6f\x74\x6f\x63\x6f\x6c\x3b\0\x20\x20\
+\x20\x20\x20\x20\x20\x20\x6c\x34\x5f\x6f\x66\x66\x73\x65\x74\x20\x3d\x20\x69\
+\x70\x2e\x69\x68\x6c\x20\x2a\x20\x34\x3b\0\x20\x20\x20\x20\x69\x66\x20\x28\x6c\
+\x34\x5f\x70\x72\x6f\x74\x6f\x63\x6f\x6c\x20\x21\x3d\x20\x30\x20\x26\x26\x20\
+\x21\x69\x6e\x66\x6f\x2d\x3e\x69\x73\x5f\x66\x72\x61\x67\x6d\x65\x6e\x74\x65\
+\x64\x29\x20\x7b\0\x20\x20\x20\x20\x20\x20\x20\x20\x69\x66\x20\x28\x6c\x34\x5f\
+\x70\x72\x6f\x74\x6f\x63\x6f\x6c\x20\x3d\x3d\x20\x49\x50\x50\x52\x4f\x54\x4f\
+\x5f\x54\x43\x50\x29\x20\x7b\0\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\
+\x69\x6e\x66\x6f\x2d\x3e\x69\x73\x5f\x74\x63\x70\x20\x3d\x20\x31\x3b\0\x20\x20\
+\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x73\x74\x72\x75\x63\x74\x20\x74\x63\
+\x70\x68\x64\x72\x20\x74\x63\x70\x20\x3d\x20\x7b\x7d\x3b\0\x20\x20\x20\x20\x20\
+\x20\x20\x20\x20\x20\x20\x20\x65\x72\x72\x20\x3d\x20\x62\x70\x66\x5f\x73\x6b\
+\x62\x5f\x6c\x6f\x61\x64\x5f\x62\x79\x74\x65\x73\x5f\x72\x65\x6c\x61\x74\x69\
+\x76\x65\x28\x73\x6b\x62\x2c\x20\x6c\x34\x5f\x6f\x66\x66\x73\x65\x74\x2c\x20\
+\x26\x74\x63\x70\x2c\x20\x73\x69\x7a\x65\x6f\x66\x28\x74\x63\x70\x29\x2c\0\x20\
+\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x69\x66\x20\x28\x65\x72\x72\x29\
+\x20\x7b\0\x20\x20\x20\x20\x69\x66\x20\x28\x70\x61\x63\x6b\x65\x74\x5f\x69\x6e\
+\x66\x6f\x2e\x69\x73\x5f\x69\x70\x76\x34\x29\x20\x7b\0\x20\x20\x20\x20\x20\x20\
+\x20\x20\x69\x66\x20\x28\x70\x61\x63\x6b\x65\x74\x5f\x69\x6e\x66\x6f\x2e\x69\
+\x73\x5f\x74\x63\x70\x20\x26\x26\0\x20\x20\x20\x20\x20\x20\x20\x20\x69\x6e\x66\
+\x6f\x2d\x3e\x69\x73\x5f\x69\x70\x76\x36\x20\x3d\x20\x31\x3b\0\x20\x20\x20\x20\
+\x20\x20\x20\x20\x73\x74\x72\x75\x63\x74\x20\x69\x70\x76\x36\x68\x64\x72\x20\
+\x69\x70\x36\x20\x3d\x20\x7b\x7d\x3b\0\x20\x20\x20\x20\x20\x20\x20\x20\x65\x72\
+\x72\x20\x3d\x20\x62\x70\x66\x5f\x73\x6b\x62\x5f\x6c\x6f\x61\x64\x5f\x62\x79\
+\x74\x65\x73\x5f\x72\x65\x6c\x61\x74\x69\x76\x65\x28\x73\x6b\x62\x2c\x20\x30\
+\x2c\x20\x26\x69\x70\x36\x2c\x20\x73\x69\x7a\x65\x6f\x66\x28\x69\x70\x36\x29\
+\x2c\0\x20\x20\x20\x20\x20\x20\x20\x20\x69\x6e\x66\x6f\x2d\x3e\x69\x6e\x36\x5f\
+\x73\x72\x63\x20\x3d\x20\x69\x70\x36\x2e\x73\x61\x64\x64\x72\x3b\0\x20\x20\x20\
+\x20\x20\x20\x20\x20\x69\x6e\x66\x6f\x2d\x3e\x69\x6e\x36\x5f\x64\x73\x74\x20\
+\x3d\x20\x69\x70\x36\x2e\x64\x61\x64\x64\x72\x3b\0\x20\x20\x20\x20\x20\x20\x20\
+\x20\x6c\x34\x5f\x70\x72\x6f\x74\x6f\x63\x6f\x6c\x20\x3d\x20\x69\x70\x36\x2e\
+\x6e\x65\x78\x74\x68\x64\x72\x3b\0\x20\x20\x20\x20\x73\x77\x69\x74\x63\x68\x20\
+\x28\x68\x64\x72\x5f\x74\x79\x70\x65\x29\x20\x7b\0\x20\x20\x20\x20\x73\x74\x72\
+\x75\x63\x74\x20\x69\x70\x76\x36\x5f\x6f\x70\x74\x5f\x68\x64\x72\x20\x65\x78\
+\x74\x5f\x68\x64\x72\x20\x3d\x20\x7b\x7d\x3b\0\x20\x20\x20\x20\x20\x20\x20\x20\
+\x65\x72\x72\x20\x3d\x20\x62\x70\x66\x5f\x73\x6b\x62\x5f\x6c\x6f\x61\x64\x5f\
+\x62\x79\x74\x65\x73\x5f\x72\x65\x6c\x61\x74\x69\x76\x65\x28\x73\x6b\x62\x2c\
+\x20\x2a\x6c\x34\x5f\x6f\x66\x66\x73\x65\x74\x2c\x20\x26\x65\x78\x74\x5f\x68\
+\x64\x72\x2c\0\x20\x20\x20\x20\x20\x20\x20\x20\x69\x66\x20\x28\x2a\x6c\x34\x5f\
+\x70\x72\x6f\x74\x6f\x63\x6f\x6c\x20\x3d\x3d\x20\x49\x50\x50\x52\x4f\x54\x4f\
+\x5f\x52\x4f\x55\x54\x49\x4e\x47\x29\x20\x7b\0\x20\x20\x20\x20\x20\x20\x20\x20\
+\x20\x20\x20\x20\x73\x74\x72\x75\x63\x74\x20\x69\x70\x76\x36\x5f\x72\x74\x5f\
+\x68\x64\x72\x20\x65\x78\x74\x5f\x72\x74\x20\x3d\x20\x7b\x7d\x3b\0\x20\x20\x20\
+\x20\x20\x20\x20\x20\x20\x20\x20\x20\x65\x72\x72\x20\x3d\x20\x62\x70\x66\x5f\
 \x73\x6b\x62\x5f\x6c\x6f\x61\x64\x5f\x62\x79\x74\x65\x73\x5f\x72\x65\x6c\x61\
 \x74\x69\x76\x65\x28\x73\x6b\x62\x2c\x20\x2a\x6c\x34\x5f\x6f\x66\x66\x73\x65\
-\x74\x2c\x20\x26\x65\x78\x74\x5f\x68\x64\x72\x2c\0\x20\x20\x20\x20\x20\x20\x20\
-\x20\x69\x66\x20\x28\x2a\x6c\x34\x5f\x70\x72\x6f\x74\x6f\x63\x6f\x6c\x20\x3d\
-\x3d\x20\x49\x50\x50\x52\x4f\x54\x4f\x5f\x52\x4f\x55\x54\x49\x4e\x47\x29\x20\
-\x7b\0\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x73\x74\x72\x75\x63\x74\
-\x20\x69\x70\x76\x36\x5f\x72\x74\x5f\x68\x64\x72\x20\x65\x78\x74\x5f\x72\x74\
-\x20\x3d\x20\x7b\x7d\x3b\0\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x65\
-\x72\x72\x20\x3d\x20\x62\x70\x66\x5f\x73\x6b\x62\x5f\x6c\x6f\x61\x64\x5f\x62\
-\x79\x74\x65\x73\x5f\x72\x65\x6c\x61\x74\x69\x76\x65\x28\x73\x6b\x62\x2c\x20\
-\x2a\x6c\x34\x5f\x6f\x66\x66\x73\x65\x74\x2c\x20\x26\x65\x78\x74\x5f\x72\x74\
-\x2c\0\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x69\x66\x20\x28\x28\x65\
-\x78\x74\x5f\x72\x74\x2e\x74\x79\x70\x65\x20\x3d\x3d\x20\x49\x50\x56\x36\x5f\
-\x53\x52\x43\x52\x54\x5f\x54\x59\x50\x45\x5f\x32\x29\x20\x26\x26\0\x20\x20\x20\
-\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x2a\x6c\
-\x34\x5f\x6f\x66\x66\x73\x65\x74\x20\x2b\x20\x6f\x66\x66\x73\x65\x74\x6f\x66\
-\x28\x73\x74\x72\x75\x63\x74\x20\x72\x74\x32\x5f\x68\x64\x72\x2c\x20\x61\x64\
-\x64\x72\x29\x2c\0\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\
-\x20\x65\x72\x72\x20\x3d\x20\x62\x70\x66\x5f\x73\x6b\x62\x5f\x6c\x6f\x61\x64\
-\x5f\x62\x79\x74\x65\x73\x5f\x72\x65\x6c\x61\x74\x69\x76\x65\x28\x73\x6b\x62\
-\x2c\0\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x69\x66\
-\x20\x28\x65\x72\x72\x29\x20\x7b\0\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\
-\x20\x20\x20\x20\x20\x69\x6e\x66\x6f\x2d\x3e\x69\x73\x5f\x69\x70\x76\x36\x5f\
-\x65\x78\x74\x5f\x64\x73\x74\x20\x3d\x20\x31\x3b\0\x20\x20\x20\x20\x20\x20\x20\
-\x20\x20\x20\x20\x20\x7d\x20\x5f\x5f\x61\x74\x74\x72\x69\x62\x75\x74\x65\x5f\
-\x5f\x28\x28\x70\x61\x63\x6b\x65\x64\x29\x29\x20\x6f\x70\x74\x20\x3d\x20\x7b\
-\x7d\x3b\0\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x6f\
-\x70\x74\x5f\x6f\x66\x66\x73\x65\x74\x20\x2b\x3d\x20\x28\x6f\x70\x74\x2e\x74\
-\x79\x70\x65\x20\x3d\x3d\x20\x49\x50\x56\x36\x5f\x54\x4c\x56\x5f\x50\x41\x44\
-\x31\x29\x20\x3f\0\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\
-\x20\x69\x66\x20\x28\x6f\x70\x74\x5f\x6f\x66\x66\x73\x65\x74\x20\x2b\x20\x31\
-\x20\x3e\x3d\x20\x65\x78\x74\x5f\x68\x64\x72\x2e\x68\x64\x72\x6c\x65\x6e\x20\
-\x2a\x20\x38\x29\x20\x7b\0\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\
-\x20\x20\x20\x65\x72\x72\x20\x3d\x20\x62\x70\x66\x5f\x73\x6b\x62\x5f\x6c\x6f\
-\x61\x64\x5f\x62\x79\x74\x65\x73\x5f\x72\x65\x6c\x61\x74\x69\x76\x65\x28\x73\
-\x6b\x62\x2c\x20\x2a\x6c\x34\x5f\x6f\x66\x66\x73\x65\x74\x20\x2b\x20\x6f\x70\
-\x74\x5f\x6f\x66\x66\x73\x65\x74\x2c\0\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\
-\x20\x20\x20\x20\x20\x20\x69\x66\x20\x28\x6f\x70\x74\x2e\x74\x79\x70\x65\x20\
-\x3d\x3d\x20\x49\x50\x56\x36\x5f\x54\x4c\x56\x5f\x48\x41\x4f\x29\x20\x7b\0\x20\
+\x74\x2c\x20\x26\x65\x78\x74\x5f\x72\x74\x2c\0\x20\x20\x20\x20\x20\x20\x20\x20\
+\x20\x20\x20\x20\x69\x66\x20\x28\x28\x65\x78\x74\x5f\x72\x74\x2e\x74\x79\x70\
+\x65\x20\x3d\x3d\x20\x49\x50\x56\x36\x5f\x53\x52\x43\x52\x54\x5f\x54\x59\x50\
+\x45\x5f\x32\x29\x20\x26\x26\0\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\
+\x20\x20\x20\x20\x20\x20\x20\x20\x2a\x6c\x34\x5f\x6f\x66\x66\x73\x65\x74\x20\
+\x2b\x20\x6f\x66\x66\x73\x65\x74\x6f\x66\x28\x73\x74\x72\x75\x63\x74\x20\x72\
+\x74\x32\x5f\x68\x64\x72\x2c\x20\x61\x64\x64\x72\x29\x2c\0\x20\x20\x20\x20\x20\
+\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x65\x72\x72\x20\x3d\x20\x62\x70\
+\x66\x5f\x73\x6b\x62\x5f\x6c\x6f\x61\x64\x5f\x62\x79\x74\x65\x73\x5f\x72\x65\
+\x6c\x61\x74\x69\x76\x65\x28\x73\x6b\x62\x2c\0\x20\x20\x20\x20\x20\x20\x20\x20\
+\x20\x20\x20\x20\x20\x20\x20\x20\x69\x66\x20\x28\x65\x72\x72\x29\x20\x7b\0\x20\
+\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x69\x6e\x66\x6f\
+\x2d\x3e\x69\x73\x5f\x69\x70\x76\x36\x5f\x65\x78\x74\x5f\x64\x73\x74\x20\x3d\
+\x20\x31\x3b\0\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x7d\x20\x5f\x5f\
+\x61\x74\x74\x72\x69\x62\x75\x74\x65\x5f\x5f\x28\x28\x70\x61\x63\x6b\x65\x64\
+\x29\x29\x20\x6f\x70\x74\x20\x3d\x20\x7b\x7d\x3b\0\x20\x20\x20\x20\x20\x20\x20\
+\x20\x20\x20\x20\x20\x20\x20\x20\x20\x6f\x70\x74\x5f\x6f\x66\x66\x73\x65\x74\
+\x20\x2b\x3d\x20\x28\x6f\x70\x74\x2e\x74\x79\x70\x65\x20\x3d\x3d\x20\x49\x50\
+\x56\x36\x5f\x54\x4c\x56\x5f\x50\x41\x44\x31\x29\x20\x3f\0\x20\x20\x20\x20\x20\
+\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x69\x66\x20\x28\x6f\x70\x74\x5f\
+\x6f\x66\x66\x73\x65\x74\x20\x2b\x20\x31\x20\x3e\x3d\x20\x65\x78\x74\x5f\x68\
+\x64\x72\x2e\x68\x64\x72\x6c\x65\x6e\x20\x2a\x20\x38\x29\x20\x7b\0\x20\x20\x20\
+\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x65\x72\x72\x20\x3d\x20\
+\x62\x70\x66\x5f\x73\x6b\x62\x5f\x6c\x6f\x61\x64\x5f\x62\x79\x74\x65\x73\x5f\
+\x72\x65\x6c\x61\x74\x69\x76\x65\x28\x73\x6b\x62\x2c\x20\x2a\x6c\x34\x5f\x6f\
+\x66\x66\x73\x65\x74\x20\x2b\x20\x6f\x70\x74\x5f\x6f\x66\x66\x73\x65\x74\x2c\0\
+\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x69\x66\x20\
+\x28\x6f\x70\x74\x2e\x74\x79\x70\x65\x20\x3d\x3d\x20\x49\x50\x56\x36\x5f\x54\
+\x4c\x56\x5f\x48\x41\x4f\x29\x20\x7b\0\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\
+\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x2a\x6c\x34\x5f\x6f\
+\x66\x66\x73\x65\x74\x20\x2b\x20\x6f\x70\x74\x5f\x6f\x66\x66\x73\x65\x74\0\x20\
 \x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\
-\x20\x20\x20\x20\x2a\x6c\x34\x5f\x6f\x66\x66\x73\x65\x74\x20\x2b\x20\x6f\x70\
-\x74\x5f\x6f\x66\x66\x73\x65\x74\0\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\
-\x20\x20\x20\x20\x20\x20\x20\x20\x20\x65\x72\x72\x20\x3d\x20\x62\x70\x66\x5f\
-\x73\x6b\x62\x5f\x6c\x6f\x61\x64\x5f\x62\x79\x74\x65\x73\x5f\x72\x65\x6c\x61\
-\x74\x69\x76\x65\x28\x73\x6b\x62\x2c\0\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\
-\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x69\x66\x20\x28\x65\x72\x72\x29\x20\
-\x7b\0\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\
-\x20\x20\x69\x6e\x66\x6f\x2d\x3e\x69\x73\x5f\x69\x70\x76\x36\x5f\x65\x78\x74\
-\x5f\x73\x72\x63\x20\x3d\x20\x31\x3b\0\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\
-\x20\x20\x69\x6e\x66\x6f\x2d\x3e\x69\x73\x5f\x66\x72\x61\x67\x6d\x65\x6e\x74\
-\x65\x64\x20\x3d\x20\x74\x72\x75\x65\x3b\0\x20\x20\x20\x20\x20\x20\x20\x20\x2a\
-\x6c\x34\x5f\x6f\x66\x66\x73\x65\x74\x20\x2b\x3d\x20\x28\x65\x78\x74\x5f\x68\
-\x64\x72\x2e\x68\x64\x72\x6c\x65\x6e\x20\x2b\x20\x31\x29\x20\x2a\x20\x38\x3b\0\
-\x20\x20\x20\x20\x20\x20\x20\x20\x2a\x6c\x34\x5f\x70\x72\x6f\x74\x6f\x63\x6f\
-\x6c\x20\x3d\x20\x65\x78\x74\x5f\x68\x64\x72\x2e\x6e\x65\x78\x74\x68\x64\x72\
-\x3b\0\x20\x20\x20\x20\x66\x6f\x72\x20\x28\x75\x6e\x73\x69\x67\x6e\x65\x64\x20\
-\x69\x6e\x74\x20\x69\x20\x3d\x20\x30\x3b\x20\x69\x20\x3c\x20\x49\x50\x36\x5f\
-\x45\x58\x54\x45\x4e\x53\x49\x4f\x4e\x53\x5f\x43\x4f\x55\x4e\x54\x3b\x20\x2b\
-\x2b\x69\x29\x20\x7b\0\x20\x20\x20\x20\x7d\x20\x65\x6c\x73\x65\x20\x69\x66\x20\
-\x28\x70\x61\x63\x6b\x65\x74\x5f\x69\x6e\x66\x6f\x2e\x69\x73\x5f\x69\x70\x76\
-\x36\x29\x20\x7b\0\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x69\x66\x20\
-\x28\x70\x61\x63\x6b\x65\x74\x5f\x69\x6e\x66\x6f\x2e\x69\x73\x5f\x69\x70\x76\
-\x36\x5f\x65\x78\x74\x5f\x64\x73\x74\x20\x26\x26\0\x20\x20\x20\x20\x20\x20\x20\
-\x20\x20\x20\x20\x20\x69\x66\x20\x28\x70\x61\x63\x6b\x65\x74\x5f\x69\x6e\x66\
-\x6f\x2e\x69\x73\x5f\x69\x70\x76\x36\x5f\x65\x78\x74\x5f\x73\x72\x63\x20\x26\
-\x26\0\x20\x20\x20\x20\x20\x20\x20\x20\x7d\x20\x65\x6c\x73\x65\x20\x69\x66\x20\
-\x28\x70\x61\x63\x6b\x65\x74\x5f\x69\x6e\x66\x6f\x2e\x69\x73\x5f\x75\x64\x70\
-\x20\x26\x26\0\x20\x20\x20\x20\x20\x20\x20\x20\x7d\x20\x65\x6c\x73\x65\x20\x69\
-\x66\x20\x28\x63\x6f\x6e\x66\x69\x67\x2d\x3e\x68\x61\x73\x68\x5f\x74\x79\x70\
-\x65\x73\x20\x26\x20\x56\x49\x52\x54\x49\x4f\x5f\x4e\x45\x54\x5f\x52\x53\x53\
-\x5f\x48\x41\x53\x48\x5f\x54\x59\x50\x45\x5f\x49\x50\x76\x34\x29\x20\x7b\0\x20\
-\x20\x20\x20\x5f\x5f\x62\x75\x69\x6c\x74\x69\x6e\x5f\x6d\x65\x6d\x63\x70\x79\
-\x28\x26\x72\x73\x73\x5f\x69\x6e\x70\x75\x74\x5b\x2a\x62\x79\x74\x65\x73\x5f\
-\x77\x72\x69\x74\x74\x65\x6e\x5d\x2c\x20\x70\x74\x72\x2c\x20\x73\x69\x7a\x65\
-\x29\x3b\0\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x69\x6e\x66\x6f\x2d\
-\x3e\x69\x73\x5f\x75\x64\x70\x20\x3d\x20\x31\x3b\0\x20\x20\x20\x20\x20\x20\x20\
-\x20\x20\x20\x20\x20\x73\x74\x72\x75\x63\x74\x20\x75\x64\x70\x68\x64\x72\x20\
-\x75\x64\x70\x20\x3d\x20\x7b\x7d\x3b\0\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\
-\x20\x20\x65\x72\x72\x20\x3d\x20\x62\x70\x66\x5f\x73\x6b\x62\x5f\x6c\x6f\x61\
-\x64\x5f\x62\x79\x74\x65\x73\x5f\x72\x65\x6c\x61\x74\x69\x76\x65\x28\x73\x6b\
-\x62\x2c\x20\x6c\x34\x5f\x6f\x66\x66\x73\x65\x74\x2c\x20\x26\x75\x64\x70\x2c\
-\x20\x73\x69\x7a\x65\x6f\x66\x28\x75\x64\x70\x29\x2c\0\x20\x20\x20\x20\x20\x20\
+\x65\x72\x72\x20\x3d\x20\x62\x70\x66\x5f\x73\x6b\x62\x5f\x6c\x6f\x61\x64\x5f\
+\x62\x79\x74\x65\x73\x5f\x72\x65\x6c\x61\x74\x69\x76\x65\x28\x73\x6b\x62\x2c\0\
+\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\
+\x20\x69\x66\x20\x28\x65\x72\x72\x29\x20\x7b\0\x20\x20\x20\x20\x20\x20\x20\x20\
+\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x69\x6e\x66\x6f\x2d\x3e\x69\
+\x73\x5f\x69\x70\x76\x36\x5f\x65\x78\x74\x5f\x73\x72\x63\x20\x3d\x20\x31\x3b\0\
+\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x69\x6e\x66\x6f\x2d\x3e\x69\
+\x73\x5f\x66\x72\x61\x67\x6d\x65\x6e\x74\x65\x64\x20\x3d\x20\x74\x72\x75\x65\
+\x3b\0\x20\x20\x20\x20\x20\x20\x20\x20\x2a\x6c\x34\x5f\x6f\x66\x66\x73\x65\x74\
+\x20\x2b\x3d\x20\x28\x65\x78\x74\x5f\x68\x64\x72\x2e\x68\x64\x72\x6c\x65\x6e\
+\x20\x2b\x20\x31\x29\x20\x2a\x20\x38\x3b\0\x20\x20\x20\x20\x20\x20\x20\x20\x2a\
+\x6c\x34\x5f\x70\x72\x6f\x74\x6f\x63\x6f\x6c\x20\x3d\x20\x65\x78\x74\x5f\x68\
+\x64\x72\x2e\x6e\x65\x78\x74\x68\x64\x72\x3b\0\x20\x20\x20\x20\x66\x6f\x72\x20\
+\x28\x75\x6e\x73\x69\x67\x6e\x65\x64\x20\x69\x6e\x74\x20\x69\x20\x3d\x20\x30\
+\x3b\x20\x69\x20\x3c\x20\x49\x50\x36\x5f\x45\x58\x54\x45\x4e\x53\x49\x4f\x4e\
+\x53\x5f\x43\x4f\x55\x4e\x54\x3b\x20\x2b\x2b\x69\x29\x20\x7b\0\x20\x20\x20\x20\
+\x7d\x20\x65\x6c\x73\x65\x20\x69\x66\x20\x28\x70\x61\x63\x6b\x65\x74\x5f\x69\
+\x6e\x66\x6f\x2e\x69\x73\x5f\x69\x70\x76\x36\x29\x20\x7b\0\x20\x20\x20\x20\x20\
+\x20\x20\x20\x20\x20\x20\x20\x69\x66\x20\x28\x70\x61\x63\x6b\x65\x74\x5f\x69\
+\x6e\x66\x6f\x2e\x69\x73\x5f\x69\x70\x76\x36\x5f\x65\x78\x74\x5f\x64\x73\x74\
+\x20\x26\x26\0\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x69\x66\x20\x28\
+\x70\x61\x63\x6b\x65\x74\x5f\x69\x6e\x66\x6f\x2e\x69\x73\x5f\x69\x70\x76\x36\
+\x5f\x65\x78\x74\x5f\x73\x72\x63\x20\x26\x26\0\x20\x20\x20\x20\x20\x20\x20\x20\
+\x7d\x20\x65\x6c\x73\x65\x20\x69\x66\x20\x28\x70\x61\x63\x6b\x65\x74\x5f\x69\
+\x6e\x66\x6f\x2e\x69\x73\x5f\x75\x64\x70\x20\x26\x26\0\x20\x20\x20\x20\x20\x20\
 \x20\x20\x7d\x20\x65\x6c\x73\x65\x20\x69\x66\x20\x28\x63\x6f\x6e\x66\x69\x67\
 \x2d\x3e\x68\x61\x73\x68\x5f\x74\x79\x70\x65\x73\x20\x26\x20\x56\x49\x52\x54\
 \x49\x4f\x5f\x4e\x45\x54\x5f\x52\x53\x53\x5f\x48\x41\x53\x48\x5f\x54\x59\x50\
-\x45\x5f\x49\x50\x76\x36\x29\x20\x7b\0\x20\x20\x20\x20\x66\x6f\x72\x20\x28\x62\
-\x79\x74\x65\x20\x3d\x20\x30\x3b\x20\x62\x79\x74\x65\x20\x3c\x20\x48\x41\x53\
-\x48\x5f\x43\x41\x4c\x43\x55\x4c\x41\x54\x49\x4f\x4e\x5f\x42\x55\x46\x46\x45\
-\x52\x5f\x53\x49\x5a\x45\x3b\x20\x62\x79\x74\x65\x2b\x2b\x29\x20\x7b\0\x20\x20\
-\x20\x20\x5f\x5f\x75\x33\x32\x20\x6c\x65\x66\x74\x6d\x6f\x73\x74\x5f\x33\x32\
-\x5f\x62\x69\x74\x73\x20\x3d\x20\x6b\x65\x79\x2d\x3e\x6c\x65\x66\x74\x6d\x6f\
-\x73\x74\x5f\x33\x32\x5f\x62\x69\x74\x73\x3b\0\x20\x20\x20\x20\x20\x20\x20\x20\
-\x5f\x5f\x75\x38\x20\x69\x6e\x70\x75\x74\x5f\x62\x79\x74\x65\x20\x3d\x20\x69\
-\x6e\x70\x75\x74\x5b\x62\x79\x74\x65\x5d\x3b\0\x20\x20\x20\x20\x20\x20\x20\x20\
-\x20\x20\x20\x20\x69\x66\x20\x28\x69\x6e\x70\x75\x74\x5f\x62\x79\x74\x65\x20\
-\x26\x20\x28\x31\x20\x3c\x3c\x20\x37\x29\x29\x20\x7b\0\x20\x20\x20\x20\x20\x20\
-\x20\x20\x5f\x5f\x75\x38\x20\x6b\x65\x79\x5f\x62\x79\x74\x65\x20\x3d\x20\x6b\
-\x65\x79\x2d\x3e\x6e\x65\x78\x74\x5f\x62\x79\x74\x65\x5b\x62\x79\x74\x65\x5d\
-\x3b\0\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\
-\x20\x20\x28\x6c\x65\x66\x74\x6d\x6f\x73\x74\x5f\x33\x32\x5f\x62\x69\x74\x73\
-\x20\x3c\x3c\x20\x31\x29\x20\x7c\x20\x28\x28\x6b\x65\x79\x5f\x62\x79\x74\x65\
-\x20\x26\x20\x28\x31\x20\x3c\x3c\x20\x37\x29\x29\x20\x3e\x3e\x20\x37\x29\x3b\0\
-\x20\x20\x20\x20\x20\x20\x20\x20\x69\x66\x20\x28\x68\x61\x73\x68\x29\x20\x7b\0\
-\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x5f\x5f\x75\x33\x32\x20\x74\
-\x61\x62\x6c\x65\x5f\x69\x64\x78\x20\x3d\x20\x68\x61\x73\x68\x20\x25\x20\x63\
-\x6f\x6e\x66\x69\x67\x2d\x3e\x69\x6e\x64\x69\x72\x65\x63\x74\x69\x6f\x6e\x73\
-\x5f\x6c\x65\x6e\x3b\0\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x71\x75\
-\x65\x75\x65\x20\x3d\x20\x62\x70\x66\x5f\x6d\x61\x70\x5f\x6c\x6f\x6f\x6b\x75\
-\x70\x5f\x65\x6c\x65\x6d\x28\x26\x74\x61\x70\x5f\x72\x73\x73\x5f\x6d\x61\x70\
-\x5f\x69\x6e\x64\x69\x72\x65\x63\x74\x69\x6f\x6e\x5f\x74\x61\x62\x6c\x65\x2c\0\
-\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x69\x66\x20\x28\x71\x75\x65\
-\x75\x65\x29\x20\x7b\0\x7d\0\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\
-\x20\x20\x20\x20\x72\x65\x74\x75\x72\x6e\x20\x2a\x71\x75\x65\x75\x65\x3b\0\x63\
-\x68\x61\x72\0\x5f\x6c\x69\x63\x65\x6e\x73\x65\0\x2e\x6d\x61\x70\x73\0\x6c\x69\
-\x63\x65\x6e\x73\x65\0\x62\x70\x66\x5f\x66\x6c\x6f\x77\x5f\x6b\x65\x79\x73\0\
-\x62\x70\x66\x5f\x73\x6f\x63\x6b\0\0\0\0\x9f\xeb\x01\0\x20\0\0\0\0\0\0\0\x14\0\
-\0\0\x14\0\0\0\x6c\x0c\0\0\x80\x0c\0\0\0\0\0\0\x08\0\0\0\x26\x02\0\0\x01\0\0\0\
-\0\0\0\0\x24\0\0\0\x10\0\0\0\x26\x02\0\0\xc6\0\0\0\0\0\0\0\x37\x02\0\0\x61\x02\
-\0\0\0\x50\x08\0\x10\0\0\0\x37\x02\0\0\x92\x02\0\0\x0b\x68\x08\0\x20\0\0\0\x37\
-\x02\0\0\0\0\0\0\0\0\0\0\x28\0\0\0\x37\x02\0\0\xa5\x02\0\0\x0e\x74\x08\0\x50\0\
-\0\0\x37\x02\0\0\xea\x02\0\0\x0b\x78\x08\0\x88\0\0\0\x37\x02\0\0\x2a\x03\0\0\
-\x10\x80\x08\0\x90\0\0\0\x37\x02\0\0\0\0\0\0\0\0\0\0\x98\0\0\0\x37\x02\0\0\x2a\
-\x03\0\0\x10\x80\x08\0\xa0\0\0\0\x37\x02\0\0\x43\x03\0\0\x16\x84\x08\0\xa8\0\0\
-\0\x37\x02\0\0\x43\x03\0\0\x0d\x84\x08\0\xc0\0\0\0\x37\x02\0\0\x64\x03\0\0\x0a\
-\xfc\x05\0\xe8\0\0\0\x37\x02\0\0\x9b\x03\0\0\x1f\x0c\x06\0\x38\x01\0\0\x37\x02\
-\0\0\xcb\x03\0\0\x0f\xa0\x04\0\x40\x01\0\0\x37\x02\0\0\xe4\x03\0\0\x0c\x20\x04\
-\0\x50\x01\0\0\x37\x02\0\0\0\0\0\0\0\0\0\0\x58\x01\0\0\x37\x02\0\0\xf8\x03\0\0\
-\x0b\x2c\x04\0\x90\x01\0\0\x37\x02\0\0\x3e\x04\0\0\x09\x34\x04\0\xa0\x01\0\0\
-\x37\x02\0\0\x4d\x04\0\0\x0d\x44\x04\0\xb8\x01\0\0\x37\x02\0\0\x4d\x04\0\0\x05\
-\x44\x04\0\xd8\x01\0\0\x37\x02\0\0\0\0\0\0\0\0\0\0\xe0\x01\0\0\x37\x02\0\0\x6b\
-\x04\0\0\x0f\x58\x04\0\x10\x02\0\0\x37\x02\0\0\x3e\x04\0\0\x09\x70\x04\0\x18\
-\x02\0\0\x37\x02\0\0\xb5\x04\0\0\x0c\x80\x04\0\x20\x02\0\0\x37\x02\0\0\xc5\x04\
-\0\0\x09\xbc\x04\0\x50\x02\0\0\x37\x02\0\0\xe1\x04\0\0\x17\xd4\x04\0\x60\x02\0\
-\0\x37\x02\0\0\xfc\x04\0\0\x16\xdc\x04\0\x80\x02\0\0\x37\x02\0\0\xe1\x04\0\0\
-\x17\xd4\x04\0\x88\x02\0\0\x37\x02\0\0\x1a\x05\0\0\x0f\xe0\x04\0\xc0\x02\0\0\
-\x37\x02\0\0\x5d\x05\0\0\x0d\xe8\x04\0\xc8\x02\0\0\x37\x02\0\0\x70\x05\0\0\x24\
-\0\x05\0\xd0\x02\0\0\x37\x02\0\0\x70\x05\0\0\x20\0\x05\0\xe0\x02\0\0\x37\x02\0\
-\0\x9d\x05\0\0\x1b\xf8\x04\0\xe8\x02\0\0\x37\x02\0\0\x9d\x05\0\0\x16\xf8\x04\0\
-\xf0\x02\0\0\x37\x02\0\0\xbe\x05\0\0\x1b\xfc\x04\0\xf8\x02\0\0\x37\x02\0\0\xbe\
-\x05\0\0\x16\xfc\x04\0\0\x03\0\0\x37\x02\0\0\xdf\x05\0\0\x1a\x08\x05\0\x08\x03\
-\0\0\x37\x02\0\0\x70\x05\0\0\x1d\0\x05\0\x10\x03\0\0\x37\x02\0\0\x02\x06\0\0\
-\x18\x0c\x05\0\x18\x03\0\0\x37\x02\0\0\x02\x06\0\0\x1c\x0c\x05\0\x30\x03\0\0\
-\x37\x02\0\0\x22\x06\0\0\x15\x68\x05\0\x40\x03\0\0\x37\x02\0\0\x22\x06\0\0\x1a\
-\x68\x05\0\x58\x03\0\0\x37\x02\0\0\x56\x06\0\0\x0d\x6c\x05\0\x78\x03\0\0\x37\
-\x02\0\0\x80\x06\0\0\x1a\x70\x05\0\x88\x03\0\0\x37\x02\0\0\x9e\x06\0\0\x1b\x78\
-\x05\0\xa8\x03\0\0\x37\x02\0\0\x80\x06\0\0\x1a\x70\x05\0\xb0\x03\0\0\x37\x02\0\
-\0\xc2\x06\0\0\x13\x7c\x05\0\xe8\x03\0\0\x37\x02\0\0\x13\x07\0\0\x11\x84\x05\0\
-\xf0\x03\0\0\x37\x02\0\0\0\0\0\0\0\0\0\0\x10\x04\0\0\x37\x02\0\0\x2a\x07\0\0\
-\x15\x28\x06\0\x18\x04\0\0\x37\x02\0\0\x2a\x07\0\0\x09\x28\x06\0\x20\x04\0\0\
-\x37\x02\0\0\0\0\0\0\0\0\0\0\x70\x04\0\0\x37\x02\0\0\x49\x07\0\0\x19\x2c\x06\0\
-\x80\x04\0\0\x37\x02\0\0\x49\x07\0\0\x20\x2c\x06\0\xa0\x04\0\0\x37\x02\0\0\0\0\
-\0\0\0\0\0\0\xf0\x04\0\0\x37\x02\0\0\x6b\x07\0\0\x17\x14\x05\0\0\x05\0\0\x37\
-\x02\0\0\x86\x07\0\0\x18\x1c\x05\0\x30\x05\0\0\x37\x02\0\0\x6b\x07\0\0\x17\x14\
-\x05\0\x48\x05\0\0\x37\x02\0\0\xa7\x07\0\0\x0f\x20\x05\0\x80\x05\0\0\x37\x02\0\
-\0\x5d\x05\0\0\x0d\x28\x05\0\x88\x05\0\0\x37\x02\0\0\xec\x07\0\0\x1d\x38\x05\0\
-\xc8\x05\0\0\x37\x02\0\0\x0f\x08\0\0\x1d\x3c\x05\0\x08\x06\0\0\x37\x02\0\0\x32\
-\x08\0\0\x1b\x44\x05\0\x10\x06\0\0\x37\x02\0\0\x55\x08\0\0\x05\x30\x02\0\x58\
-\x06\0\0\x37\x02\0\0\x6d\x08\0\0\x19\xb8\x02\0\xd0\x06\0\0\x37\x02\0\0\0\0\0\0\
-\0\0\0\0\xd8\x06\0\0\x37\x02\0\0\x93\x08\0\0\x0f\xc8\x02\0\x10\x07\0\0\x37\x02\
-\0\0\x5d\x05\0\0\x0d\xd0\x02\0\x20\x07\0\0\x37\x02\0\0\xd8\x08\0\0\x0d\xe0\x02\
-\0\x40\x07\0\0\x37\x02\0\0\x07\x09\0\0\x20\xe4\x02\0\x68\x07\0\0\x37\x02\0\0\
-\x33\x09\0\0\x13\xec\x02\0\xa8\x07\0\0\x37\x02\0\0\x13\x07\0\0\x11\xf4\x02\0\
-\xb0\x07\0\0\x37\x02\0\0\x7b\x09\0\0\x19\x04\x03\0\xb8\x07\0\0\x37\x02\0\0\x7b\
-\x09\0\0\x34\x04\x03\0\xe0\x07\0\0\x37\x02\0\0\xb1\x09\0\0\x15\x18\x03\0\xf0\
-\x07\0\0\x37\x02\0\0\xf2\x09\0\0\x17\x14\x03\0\x30\x08\0\0\x37\x02\0\0\x29\x0a\
-\0\0\x15\x24\x03\0\x38\x08\0\0\x37\x02\0\0\x44\x0a\0\0\x27\x34\x03\0\x70\x08\0\
-\0\x37\x02\0\0\x6f\x0a\0\0\x27\x50\x03\0\x80\x08\0\0\x37\x02\0\0\x9f\x0a\0\0\
-\x1c\xb4\x03\0\x88\x08\0\0\x37\x02\0\0\xdb\x0a\0\0\x20\xc0\x03\0\x98\x08\0\0\
-\x37\x02\0\0\xdb\x0a\0\0\x2f\xc0\x03\0\xa0\x08\0\0\x37\x02\0\0\xdb\x0a\0\0\x36\
-\xc0\x03\0\xa8\x08\0\0\x37\x02\0\0\xdb\x0a\0\0\x15\xc0\x03\0\x18\x09\0\0\x37\
-\x02\0\0\x17\x0b\0\0\x43\x64\x03\0\x38\x09\0\0\x37\x02\0\0\0\0\0\0\0\0\0\0\x40\
-\x09\0\0\x37\x02\0\0\x17\x0b\0\0\x17\x64\x03\0\x80\x09\0\0\x37\x02\0\0\x29\x0a\
-\0\0\x15\x6c\x03\0\x88\x09\0\0\x37\x02\0\0\x67\x0b\0\0\x19\x7c\x03\0\x90\x09\0\
-\0\x37\x02\0\0\x67\x0b\0\0\x15\x7c\x03\0\x98\x09\0\0\x37\x02\0\0\x97\x0b\0\0\
-\x19\x84\x03\0\xa0\x09\0\0\x37\x02\0\0\xc7\x0b\0\0\x1b\x80\x03\0\xe8\x09\0\0\
-\x37\x02\0\0\x02\x0c\0\0\x19\x94\x03\0\xf0\x09\0\0\x37\x02\0\0\x21\x0c\0\0\x2b\
-\xa4\x03\0\x10\x0a\0\0\x37\x02\0\0\x9f\x0a\0\0\x1f\xb4\x03\0\x30\x0a\0\0\x37\
-\x02\0\0\x50\x0c\0\0\x21\xd4\x03\0\x40\x0a\0\0\x37\x02\0\0\x78\x0c\0\0\x20\xe4\
-\x03\0\x48\x0a\0\0\x37\x02\0\0\x78\x0c\0\0\x2c\xe4\x03\0\x60\x0a\0\0\x37\x02\0\
-\0\x78\x0c\0\0\x14\xe4\x03\0\x70\x0a\0\0\x37\x02\0\0\xa8\x0c\0\0\x20\xe0\x03\0\
-\x80\x0a\0\0\x37\x02\0\0\x55\x08\0\0\x05\x30\x02\0\xb0\x0a\0\0\x37\x02\0\0\xd0\
-\x0c\0\0\x38\xc0\x02\0\xd0\x0a\0\0\x37\x02\0\0\xd0\x0c\0\0\x05\xc0\x02\0\xe8\
-\x0a\0\0\x37\x02\0\0\x55\x08\0\0\x05\x30\x02\0\xf8\x0a\0\0\x37\x02\0\0\x0e\x0d\
-\0\0\x1c\xc4\x06\0\x08\x0b\0\0\x37\x02\0\0\x0e\x0d\0\0\x10\xc4\x06\0\x10\x0b\0\
-\0\x37\x02\0\0\0\0\0\0\0\0\0\0\x60\x0b\0\0\x37\x02\0\0\x49\x07\0\0\x19\xc8\x06\
-\0\x68\x0b\0\0\x37\x02\0\0\x49\x07\0\0\x20\xc8\x06\0\xa0\x0b\0\0\x37\x02\0\0\
-\x34\x0d\0\0\x2d\0\x07\0\xb0\x0b\0\0\x37\x02\0\0\x34\x0d\0\0\x1d\0\x07\0\xb8\
-\x0b\0\0\x37\x02\0\0\x34\x0d\0\0\x2d\0\x07\0\xc8\x0b\0\0\x37\x02\0\0\x63\x0d\0\
-\0\x2d\xd4\x06\0\xf8\x0b\0\0\x37\x02\0\0\x63\x0d\0\0\x1d\xd4\x06\0\x08\x0c\0\0\
-\x37\x02\0\0\x63\x0d\0\0\x2d\xd4\x06\0\x18\x0c\0\0\x37\x02\0\0\0\0\0\0\0\0\0\0\
-\xe8\x0c\0\0\x37\x02\0\0\x92\x0d\0\0\x20\x68\x06\0\xf0\x0c\0\0\x37\x02\0\0\x92\
-\x0d\0\0\x27\x68\x06\0\x18\x0d\0\0\x37\x02\0\0\xbb\x0d\0\0\x27\xa4\x06\0\x20\
-\x0d\0\0\x37\x02\0\0\xbb\x0d\0\0\x14\xa4\x06\0\x28\x0d\0\0\x37\x02\0\0\x04\x0e\
-\0\0\x05\x98\x01\0\x38\x0d\0\0\x37\x02\0\0\x04\x0e\0\0\x05\x98\x01\0\x60\x0d\0\
-\0\x37\x02\0\0\0\0\0\0\0\0\0\0\x70\x0d\0\0\x37\x02\0\0\0\0\0\0\0\0\0\0\x80\x0d\
-\0\0\x37\x02\0\0\x92\x0d\0\0\x20\x44\x07\0\x88\x0d\0\0\x37\x02\0\0\x92\x0d\0\0\
-\x27\x44\x07\0\xc0\x0d\0\0\x37\x02\0\0\x34\x0d\0\0\x2d\x7c\x07\0\xd0\x0d\0\0\
-\x37\x02\0\0\x34\x0d\0\0\x1d\x7c\x07\0\xd8\x0d\0\0\x37\x02\0\0\x34\x0d\0\0\x2d\
-\x7c\x07\0\xe8\x0d\0\0\x37\x02\0\0\x63\x0d\0\0\x2d\x50\x07\0\x18\x0e\0\0\x37\
-\x02\0\0\x63\x0d\0\0\x1d\x50\x07\0\x28\x0e\0\0\x37\x02\0\0\x63\x0d\0\0\x2d\x50\
-\x07\0\x40\x0e\0\0\x37\x02\0\0\x41\x0e\0\0\x1a\xa0\x05\0\x50\x0e\0\0\x37\x02\0\
-\0\x5f\x0e\0\0\x1b\xa8\x05\0\x60\x0e\0\0\x37\x02\0\0\x41\x0e\0\0\x1a\xa0\x05\0\
-\x68\x0e\0\0\x37\x02\0\0\x83\x0e\0\0\x13\xac\x05\0\xa0\x0e\0\0\x37\x02\0\0\x13\
-\x07\0\0\x11\xb4\x05\0\xb0\x0e\0\0\x37\x02\0\0\x55\x08\0\0\x05\x30\x02\0\xc0\
-\x0e\0\0\x37\x02\0\0\xd4\x0e\0\0\x27\xc8\x07\0\xd0\x0e\0\0\x37\x02\0\0\xd4\x0e\
-\0\0\x14\xc8\x07\0\xf0\x0e\0\0\x37\x02\0\0\x63\x0d\0\0\x2d\xcc\x07\0\0\x0f\0\0\
-\x37\x02\0\0\x63\x0d\0\0\x1d\xcc\x07\0\x08\x0f\0\0\x37\x02\0\0\x63\x0d\0\0\x2d\
-\xcc\x07\0\x30\x0f\0\0\x37\x02\0\0\0\0\0\0\0\0\0\0\x80\x0f\0\0\x37\x02\0\0\x34\
-\x0d\0\0\x1d\xf8\x07\0\x88\x0f\0\0\x37\x02\0\0\x34\x0d\0\0\x2d\xf8\x07\0\x98\
-\x0f\0\0\x37\x02\0\0\x04\x0e\0\0\x05\x98\x01\0\xf0\x0f\0\0\x37\x02\0\0\x04\x0e\
-\0\0\x05\x98\x01\0\x30\x10\0\0\x37\x02\0\0\0\0\0\0\0\0\0\0\x48\x10\0\0\x37\x02\
-\0\0\x1d\x0f\0\0\x05\xd0\x01\0\x50\x10\0\0\x37\x02\0\0\x5f\x0f\0\0\x23\xc4\x01\
-\0\x68\x10\0\0\x37\x02\0\0\0\0\0\0\0\0\0\0\x70\x10\0\0\x37\x02\0\0\x93\x0f\0\0\
-\x1b\xd4\x01\0\x90\x10\0\0\x37\x02\0\0\xba\x0f\0\0\x11\xe8\x01\0\xa8\x10\0\0\
-\x37\x02\0\0\xe3\x0f\0\0\x19\xd8\x01\0\xc0\x10\0\0\x37\x02\0\0\x11\x10\0\0\x27\
-\xfc\x01\0\xc8\x10\0\0\x37\x02\0\0\x11\x10\0\0\x46\xfc\x01\0\xd8\x10\0\0\x37\
-\x02\0\0\x11\x10\0\0\x2d\xfc\x01\0\xe0\x10\0\0\x37\x02\0\0\xba\x0f\0\0\x11\xe8\
-\x01\0\x08\x11\0\0\x37\x02\0\0\x11\x10\0\0\x46\xfc\x01\0\x20\x11\0\0\x37\x02\0\
-\0\x11\x10\0\0\x27\xfc\x01\0\x28\x11\0\0\x37\x02\0\0\x11\x10\0\0\x2d\xfc\x01\0\
-\x30\x11\0\0\x37\x02\0\0\xba\x0f\0\0\x11\xe8\x01\0\x58\x11\0\0\x37\x02\0\0\x11\
-\x10\0\0\x27\xfc\x01\0\x60\x11\0\0\x37\x02\0\0\x11\x10\0\0\x46\xfc\x01\0\x78\
-\x11\0\0\x37\x02\0\0\x11\x10\0\0\x2d\xfc\x01\0\x80\x11\0\0\x37\x02\0\0\xba\x0f\
-\0\0\x11\xe8\x01\0\xa8\x11\0\0\x37\x02\0\0\x11\x10\0\0\x27\xfc\x01\0\xb0\x11\0\
-\0\x37\x02\0\0\x11\x10\0\0\x46\xfc\x01\0\xc8\x11\0\0\x37\x02\0\0\x11\x10\0\0\
-\x2d\xfc\x01\0\xd0\x11\0\0\x37\x02\0\0\xba\x0f\0\0\x11\xe8\x01\0\xf8\x11\0\0\
-\x37\x02\0\0\x11\x10\0\0\x46\xfc\x01\0\x10\x12\0\0\x37\x02\0\0\x11\x10\0\0\x27\
-\xfc\x01\0\x18\x12\0\0\x37\x02\0\0\x11\x10\0\0\x2d\xfc\x01\0\x20\x12\0\0\x37\
-\x02\0\0\xba\x0f\0\0\x11\xe8\x01\0\x48\x12\0\0\x37\x02\0\0\x11\x10\0\0\x46\xfc\
-\x01\0\x60\x12\0\0\x37\x02\0\0\x11\x10\0\0\x27\xfc\x01\0\x68\x12\0\0\x37\x02\0\
-\0\x11\x10\0\0\x2d\xfc\x01\0\x70\x12\0\0\x37\x02\0\0\xba\x0f\0\0\x11\xe8\x01\0\
-\x98\x12\0\0\x37\x02\0\0\x11\x10\0\0\x46\xfc\x01\0\xb0\x12\0\0\x37\x02\0\0\x11\
-\x10\0\0\x27\xfc\x01\0\xb8\x12\0\0\x37\x02\0\0\x11\x10\0\0\x2d\xfc\x01\0\xc0\
-\x12\0\0\x37\x02\0\0\xba\x0f\0\0\x11\xe8\x01\0\xe0\x12\0\0\x37\x02\0\0\x11\x10\
-\0\0\x46\xfc\x01\0\xe8\x12\0\0\x37\x02\0\0\x11\x10\0\0\x27\xfc\x01\0\xf0\x12\0\
-\0\x37\x02\0\0\x11\x10\0\0\x2d\xfc\x01\0\xf8\x12\0\0\x37\x02\0\0\x1d\x0f\0\0\
-\x3d\xd0\x01\0\x08\x13\0\0\x37\x02\0\0\x1d\x0f\0\0\x05\xd0\x01\0\x18\x13\0\0\
-\x37\x02\0\0\x5d\x10\0\0\x0d\x98\x08\0\x30\x13\0\0\x37\x02\0\0\x5d\x10\0\0\x0d\
-\x98\x08\0\x38\x13\0\0\x37\x02\0\0\x71\x10\0\0\x2e\x9c\x08\0\x58\x13\0\0\x37\
-\x02\0\0\x71\x10\0\0\x24\x9c\x08\0\x70\x13\0\0\x37\x02\0\0\x71\x10\0\0\x13\x9c\
-\x08\0\x80\x13\0\0\x37\x02\0\0\x71\x10\0\0\x2e\x9c\x08\0\x88\x13\0\0\x37\x02\0\
-\0\xb0\x10\0\0\x15\xa8\x08\0\xa0\x13\0\0\x37\x02\0\0\xf8\x10\0\0\x11\xb4\x08\0\
-\xa8\x13\0\0\x37\x02\0\0\0\0\0\0\0\0\0\0\xc8\x13\0\0\x37\x02\0\0\x11\x11\0\0\
-\x01\xd8\x08\0\xd0\x13\0\0\x37\x02\0\0\x13\x11\0\0\x18\xb8\x08\0\0\0\0\0\0\0\0\
-\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x03\0\x03\0\0\0\0\0\0\0\0\0\0\0\0\0\
-\0\0\0\0\xde\0\0\0\0\0\x03\0\xc8\x13\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x7a\x01\0\0\0\
-\0\x03\0\xb8\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xff\0\0\0\0\0\x03\0\xa8\x13\0\0\0\0\
-\0\0\0\0\0\0\0\0\0\0\xc7\0\0\0\0\0\x03\0\xd0\x01\0\0\0\0\0\0\0\0\0\0\0\0\0\0\
-\x2c\x02\0\0\0\0\x03\0\x20\x02\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xf7\0\0\0\0\0\x03\0\
-\xe8\x04\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x1c\x02\0\0\0\0\x03\0\x10\x04\0\0\0\0\0\0\
-\0\0\0\0\0\0\0\0\x28\x01\0\0\0\0\x03\0\xe0\x02\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xf3\
-\x01\0\0\0\0\x03\0\x30\x03\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xeb\x01\0\0\0\0\x03\0\
-\x38\x0e\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x44\x02\0\0\0\0\x03\0\xf0\x03\0\0\0\0\0\0\
-\0\0\0\0\0\0\0\0\xe3\x01\0\0\0\0\x03\0\xf8\x0a\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x18\
-\x01\0\0\0\0\x03\0\xe8\x0c\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x30\x01\0\0\0\0\x03\0\
-\xa0\x04\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xa9\x01\0\0\0\0\x03\0\x40\x10\0\0\0\0\0\0\
-\0\0\0\0\0\0\0\0\x51\x01\0\0\0\0\x03\0\x78\x0d\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x5c\
-\x02\0\0\0\0\x03\0\xb0\x0e\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x03\x02\0\0\0\0\x03\0\
-\x50\x06\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xc2\x01\0\0\0\0\x03\0\xc0\x06\0\0\0\0\0\0\
-\0\0\0\0\0\0\0\0\x69\x01\0\0\0\0\x03\0\x20\x07\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x61\
-\x01\0\0\0\0\x03\0\x60\x08\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x38\x01\0\0\0\0\x03\0\
-\x30\x0a\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x20\x01\0\0\0\0\x03\0\x40\x0a\0\0\0\0\0\0\
-\0\0\0\0\0\0\0\0\xba\x01\0\0\0\0\x03\0\xe0\x0f\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xa1\
-\x01\0\0\0\0\x03\0\x48\x08\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x40\x01\0\0\0\0\x03\0\
-\x18\x09\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xfb\x01\0\0\0\0\x03\0\x80\x08\0\0\0\0\0\0\
-\0\0\0\0\0\0\0\0\x99\x01\0\0\0\0\x03\0\xf8\x08\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x59\
-\x01\0\0\0\0\x03\0\x50\x0d\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x54\x02\0\0\0\0\x03\0\
-\x08\x0a\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xef\0\0\0\0\0\x03\0\xe8\x0a\0\0\0\0\0\0\0\
-\0\0\0\0\0\0\0\x4c\x02\0\0\0\0\x03\0\xb0\x0a\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x24\
-\x02\0\0\0\0\x03\0\xd8\x0a\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x89\x01\0\0\0\0\x03\0\
-\x80\x0d\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x10\x01\0\0\0\0\x03\0\xb0\x0b\0\0\0\0\0\0\
-\0\0\0\0\0\0\0\0\xd6\0\0\0\0\0\x03\0\xc8\x0b\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x14\
-\x02\0\0\0\0\x03\0\xf8\x0b\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xb2\x01\0\0\0\0\x03\0\
-\x18\x0c\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xdb\x01\0\0\0\0\x03\0\x10\x0c\0\0\0\0\0\0\
-\0\0\0\0\0\0\0\0\x3c\x02\0\0\0\0\x03\0\x18\x0d\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x91\
-\x01\0\0\0\0\x03\0\x60\x0d\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x81\x01\0\0\0\0\x03\0\
-\xc0\x0e\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xe7\0\0\0\0\0\x03\0\xd0\x0d\0\0\0\0\0\0\0\
-\0\0\0\0\0\0\0\x34\x02\0\0\0\0\x03\0\xe8\x0d\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xd3\
-\x01\0\0\0\0\x03\0\x18\x0e\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x08\x01\0\0\0\0\x03\0\0\
-\x0f\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xce\0\0\0\0\0\x03\0\x18\x0f\0\0\0\0\0\0\0\0\0\
-\0\0\0\0\0\x0b\x02\0\0\0\0\x03\0\xf0\x0f\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xca\x01\0\
-\0\0\0\x03\0\x30\x10\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x71\x01\0\0\0\0\x03\0\x60\x10\
-\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x48\x01\0\0\0\0\x03\0\x18\x13\0\0\0\0\0\0\0\0\0\0\
-\0\0\0\0\x64\x02\0\0\0\0\x03\0\xd0\x13\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x4e\0\0\0\
-\x12\0\x03\0\0\0\0\0\0\0\0\0\xe0\x13\0\0\0\0\0\0\x33\0\0\0\x11\0\x05\0\0\0\0\0\
-\0\0\0\0\x20\0\0\0\0\0\0\0\x01\0\0\0\x11\0\x05\0\x20\0\0\0\0\0\0\0\x20\0\0\0\0\
-\0\0\0\x90\0\0\0\x11\0\x05\0\x40\0\0\0\0\0\0\0\x20\0\0\0\0\0\0\0\x87\0\0\0\x11\
-\0\x06\0\0\0\0\0\0\0\0\0\x07\0\0\0\0\0\0\0\x28\0\0\0\0\0\0\0\x01\0\0\0\x37\0\0\
-\0\x50\0\0\0\0\0\0\0\x01\0\0\0\x38\0\0\0\x88\x13\0\0\0\0\0\0\x01\0\0\0\x39\0\0\
-\0\xd8\x04\0\0\0\0\0\0\x04\0\0\0\x37\0\0\0\xe4\x04\0\0\0\0\0\0\x04\0\0\0\x38\0\
-\0\0\xf0\x04\0\0\0\0\0\0\x04\0\0\0\x39\0\0\0\x08\x05\0\0\0\0\0\0\x04\0\0\0\x3a\
-\0\0\0\x2c\0\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x40\0\0\0\0\0\0\0\x04\0\0\0\x01\0\
-\0\0\x50\0\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x60\0\0\0\0\0\0\0\x04\0\0\0\x01\0\0\
-\0\x70\0\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x80\0\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\
-\x90\0\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xa0\0\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\
-\xb0\0\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xc0\0\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\
-\xd0\0\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xe0\0\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\
-\xf0\0\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\0\x01\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\
-\x10\x01\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x20\x01\0\0\0\0\0\0\x04\0\0\0\x01\0\0\
-\0\x30\x01\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x40\x01\0\0\0\0\0\0\x04\0\0\0\x01\0\
-\0\0\x50\x01\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x60\x01\0\0\0\0\0\0\x04\0\0\0\x01\
-\0\0\0\x70\x01\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x80\x01\0\0\0\0\0\0\x04\0\0\0\
-\x01\0\0\0\x90\x01\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xa0\x01\0\0\0\0\0\0\x04\0\0\
-\0\x01\0\0\0\xb0\x01\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xc0\x01\0\0\0\0\0\0\x04\0\
-\0\0\x01\0\0\0\xd0\x01\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xe0\x01\0\0\0\0\0\0\x04\
-\0\0\0\x01\0\0\0\xf0\x01\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\0\x02\0\0\0\0\0\0\x04\
-\0\0\0\x01\0\0\0\x10\x02\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x20\x02\0\0\0\0\0\0\
-\x04\0\0\0\x01\0\0\0\x30\x02\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x40\x02\0\0\0\0\0\
-\0\x04\0\0\0\x01\0\0\0\x50\x02\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x60\x02\0\0\0\0\
-\0\0\x04\0\0\0\x01\0\0\0\x70\x02\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x80\x02\0\0\0\
-\0\0\0\x04\0\0\0\x01\0\0\0\x90\x02\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xa0\x02\0\0\
-\0\0\0\0\x04\0\0\0\x01\0\0\0\xb0\x02\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xc0\x02\0\
-\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xd0\x02\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xe0\x02\
-\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xf0\x02\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\0\x03\
-\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x10\x03\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x20\
-\x03\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x30\x03\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\
-\x40\x03\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x50\x03\0\0\0\0\0\0\x04\0\0\0\x01\0\0\
-\0\x60\x03\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x70\x03\0\0\0\0\0\0\x04\0\0\0\x01\0\
-\0\0\x80\x03\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x90\x03\0\0\0\0\0\0\x04\0\0\0\x01\
-\0\0\0\xa0\x03\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xb0\x03\0\0\0\0\0\0\x04\0\0\0\
-\x01\0\0\0\xc0\x03\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xd0\x03\0\0\0\0\0\0\x04\0\0\
-\0\x01\0\0\0\xe0\x03\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xf0\x03\0\0\0\0\0\0\x04\0\
-\0\0\x01\0\0\0\0\x04\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x10\x04\0\0\0\0\0\0\x04\0\
-\0\0\x01\0\0\0\x20\x04\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x30\x04\0\0\0\0\0\0\x04\
-\0\0\0\x01\0\0\0\x40\x04\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x50\x04\0\0\0\0\0\0\
-\x04\0\0\0\x01\0\0\0\x60\x04\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x70\x04\0\0\0\0\0\
-\0\x04\0\0\0\x01\0\0\0\x80\x04\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x90\x04\0\0\0\0\
-\0\0\x04\0\0\0\x01\0\0\0\xa0\x04\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xb0\x04\0\0\0\
-\0\0\0\x04\0\0\0\x01\0\0\0\xc0\x04\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xd0\x04\0\0\
-\0\0\0\0\x04\0\0\0\x01\0\0\0\xe0\x04\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xf0\x04\0\
-\0\0\0\0\0\x04\0\0\0\x01\0\0\0\0\x05\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x10\x05\0\
-\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x20\x05\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x30\x05\
-\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x40\x05\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x50\
-\x05\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x60\x05\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\
-\x70\x05\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x80\x05\0\0\0\0\0\0\x04\0\0\0\x01\0\0\
-\0\x90\x05\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xa0\x05\0\0\0\0\0\0\x04\0\0\0\x01\0\
-\0\0\xb0\x05\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xc0\x05\0\0\0\0\0\0\x04\0\0\0\x01\
-\0\0\0\xd0\x05\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xe0\x05\0\0\0\0\0\0\x04\0\0\0\
-\x01\0\0\0\xf0\x05\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\0\x06\0\0\0\0\0\0\x04\0\0\0\
-\x01\0\0\0\x10\x06\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x20\x06\0\0\0\0\0\0\x04\0\0\
-\0\x01\0\0\0\x30\x06\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x40\x06\0\0\0\0\0\0\x04\0\
-\0\0\x01\0\0\0\x50\x06\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x60\x06\0\0\0\0\0\0\x04\
-\0\0\0\x01\0\0\0\x70\x06\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x80\x06\0\0\0\0\0\0\
-\x04\0\0\0\x01\0\0\0\x90\x06\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xa0\x06\0\0\0\0\0\
-\0\x04\0\0\0\x01\0\0\0\xb0\x06\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xc0\x06\0\0\0\0\
-\0\0\x04\0\0\0\x01\0\0\0\xd0\x06\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xe0\x06\0\0\0\
-\0\0\0\x04\0\0\0\x01\0\0\0\xf0\x06\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\0\x07\0\0\0\
-\0\0\0\x04\0\0\0\x01\0\0\0\x10\x07\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x20\x07\0\0\
-\0\0\0\0\x04\0\0\0\x01\0\0\0\x30\x07\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x40\x07\0\
-\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x50\x07\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x60\x07\
-\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x70\x07\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x80\
-\x07\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x90\x07\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\
-\xa0\x07\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xb0\x07\0\0\0\0\0\0\x04\0\0\0\x01\0\0\
-\0\xc0\x07\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xd0\x07\0\0\0\0\0\0\x04\0\0\0\x01\0\
-\0\0\xe0\x07\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xf0\x07\0\0\0\0\0\0\x04\0\0\0\x01\
-\0\0\0\0\x08\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x10\x08\0\0\0\0\0\0\x04\0\0\0\x01\
-\0\0\0\x20\x08\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x30\x08\0\0\0\0\0\0\x04\0\0\0\
-\x01\0\0\0\x40\x08\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x50\x08\0\0\0\0\0\0\x04\0\0\
-\0\x01\0\0\0\x60\x08\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x70\x08\0\0\0\0\0\0\x04\0\
-\0\0\x01\0\0\0\x80\x08\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x90\x08\0\0\0\0\0\0\x04\
-\0\0\0\x01\0\0\0\xa0\x08\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xb0\x08\0\0\0\0\0\0\
-\x04\0\0\0\x01\0\0\0\xc0\x08\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xd0\x08\0\0\0\0\0\
-\0\x04\0\0\0\x01\0\0\0\xe0\x08\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xf0\x08\0\0\0\0\
-\0\0\x04\0\0\0\x01\0\0\0\0\x09\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x10\x09\0\0\0\0\
-\0\0\x04\0\0\0\x01\0\0\0\x20\x09\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x30\x09\0\0\0\
-\0\0\0\x04\0\0\0\x01\0\0\0\x40\x09\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x50\x09\0\0\
-\0\0\0\0\x04\0\0\0\x01\0\0\0\x60\x09\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x70\x09\0\
-\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x80\x09\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x90\x09\
-\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xa0\x09\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xb0\
-\x09\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xc0\x09\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\
-\xd0\x09\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xe0\x09\0\0\0\0\0\0\x04\0\0\0\x01\0\0\
-\0\xf0\x09\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\0\x0a\0\0\0\0\0\0\x04\0\0\0\x01\0\0\
-\0\x10\x0a\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x20\x0a\0\0\0\0\0\0\x04\0\0\0\x01\0\
-\0\0\x30\x0a\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x40\x0a\0\0\0\0\0\0\x04\0\0\0\x01\
-\0\0\0\x50\x0a\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x60\x0a\0\0\0\0\0\0\x04\0\0\0\
-\x01\0\0\0\x70\x0a\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x80\x0a\0\0\0\0\0\0\x04\0\0\
-\0\x01\0\0\0\x90\x0a\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xa0\x0a\0\0\0\0\0\0\x04\0\
-\0\0\x01\0\0\0\xb0\x0a\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xc0\x0a\0\0\0\0\0\0\x04\
-\0\0\0\x01\0\0\0\xd0\x0a\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xe0\x0a\0\0\0\0\0\0\
-\x04\0\0\0\x01\0\0\0\xf0\x0a\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\0\x0b\0\0\0\0\0\0\
-\x04\0\0\0\x01\0\0\0\x10\x0b\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x20\x0b\0\0\0\0\0\
-\0\x04\0\0\0\x01\0\0\0\x30\x0b\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x40\x0b\0\0\0\0\
-\0\0\x04\0\0\0\x01\0\0\0\x50\x0b\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x60\x0b\0\0\0\
-\0\0\0\x04\0\0\0\x01\0\0\0\x70\x0b\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x80\x0b\0\0\
-\0\0\0\0\x04\0\0\0\x01\0\0\0\x90\x0b\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xa0\x0b\0\
-\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xb0\x0b\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xc0\x0b\
-\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xd0\x0b\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xe0\
-\x0b\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xf0\x0b\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\0\
-\x0c\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x10\x0c\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\
-\x20\x0c\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x30\x0c\0\0\0\0\0\0\x04\0\0\0\x01\0\0\
-\0\x40\x0c\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x50\x0c\0\0\0\0\0\0\x04\0\0\0\x01\0\
-\0\0\x60\x0c\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x70\x0c\0\0\0\0\0\0\x04\0\0\0\x01\
-\0\0\0\x80\x0c\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x90\x0c\0\0\0\0\0\0\x04\0\0\0\
-\x01\0\0\0\x40\x41\x42\x43\x44\0\x74\x61\x70\x5f\x72\x73\x73\x5f\x6d\x61\x70\
-\x5f\x74\x6f\x65\x70\x6c\x69\x74\x7a\x5f\x6b\x65\x79\0\x2e\x74\x65\x78\x74\0\
-\x2e\x72\x65\x6c\x2e\x42\x54\x46\x2e\x65\x78\x74\0\x2e\x6d\x61\x70\x73\0\x74\
-\x61\x70\x5f\x72\x73\x73\x5f\x6d\x61\x70\x5f\x63\x6f\x6e\x66\x69\x67\x75\x72\
-\x61\x74\x69\x6f\x6e\x73\0\x74\x75\x6e\x5f\x72\x73\x73\x5f\x73\x74\x65\x65\x72\
-\x69\x6e\x67\x5f\x70\x72\x6f\x67\0\x2e\x72\x65\x6c\x74\x75\x6e\x5f\x72\x73\x73\
-\x5f\x73\x74\x65\x65\x72\x69\x6e\x67\0\x2e\x6c\x6c\x76\x6d\x5f\x61\x64\x64\x72\
-\x73\x69\x67\0\x5f\x6c\x69\x63\x65\x6e\x73\x65\0\x74\x61\x70\x5f\x72\x73\x73\
-\x5f\x6d\x61\x70\x5f\x69\x6e\x64\x69\x72\x65\x63\x74\x69\x6f\x6e\x5f\x74\x61\
-\x62\x6c\x65\0\x2e\x73\x74\x72\x74\x61\x62\0\x2e\x73\x79\x6d\x74\x61\x62\0\x2e\
-\x72\x65\x6c\x2e\x42\x54\x46\0\x4c\x42\x42\x30\x5f\x39\0\x4c\x42\x42\x30\x5f\
-\x39\x39\0\x4c\x42\x42\x30\x5f\x37\x39\0\x4c\x42\x42\x30\x5f\x31\x30\x39\0\x4c\
-\x42\x42\x30\x5f\x38\x38\0\x4c\x42\x42\x30\x5f\x34\x38\0\x4c\x42\x42\x30\x5f\
-\x31\x38\0\x4c\x42\x42\x30\x5f\x31\x30\x38\0\x4c\x42\x42\x30\x5f\x39\x37\0\x4c\
-\x42\x42\x30\x5f\x37\x37\0\x4c\x42\x42\x30\x5f\x36\x37\0\x4c\x42\x42\x30\x5f\
-\x34\x37\0\x4c\x42\x42\x30\x5f\x31\x37\0\x4c\x42\x42\x30\x5f\x36\x36\0\x4c\x42\
-\x42\x30\x5f\x34\x36\0\x4c\x42\x42\x30\x5f\x33\x36\0\x4c\x42\x42\x30\x5f\x31\
-\x30\x36\0\x4c\x42\x42\x30\x5f\x35\x35\0\x4c\x42\x42\x30\x5f\x34\x35\0\x4c\x42\
-\x42\x30\x5f\x33\x35\0\x4c\x42\x42\x30\x5f\x32\x35\0\x4c\x42\x42\x30\x5f\x31\
-\x30\x35\0\x4c\x42\x42\x30\x5f\x34\0\x4c\x42\x42\x30\x5f\x39\x34\0\x4c\x42\x42\
-\x30\x5f\x38\x34\0\x4c\x42\x42\x30\x5f\x35\x34\0\x4c\x42\x42\x30\x5f\x34\x34\0\
+\x45\x5f\x49\x50\x76\x34\x29\x20\x7b\0\x20\x20\x20\x20\x5f\x5f\x62\x75\x69\x6c\
+\x74\x69\x6e\x5f\x6d\x65\x6d\x63\x70\x79\x28\x26\x72\x73\x73\x5f\x69\x6e\x70\
+\x75\x74\x5b\x2a\x62\x79\x74\x65\x73\x5f\x77\x72\x69\x74\x74\x65\x6e\x5d\x2c\
+\x20\x70\x74\x72\x2c\x20\x73\x69\x7a\x65\x29\x3b\0\x20\x20\x20\x20\x20\x20\x20\
+\x20\x20\x20\x20\x20\x69\x6e\x66\x6f\x2d\x3e\x69\x73\x5f\x75\x64\x70\x20\x3d\
+\x20\x31\x3b\0\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x73\x74\x72\x75\
+\x63\x74\x20\x75\x64\x70\x68\x64\x72\x20\x75\x64\x70\x20\x3d\x20\x7b\x7d\x3b\0\
+\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x65\x72\x72\x20\x3d\x20\x62\
+\x70\x66\x5f\x73\x6b\x62\x5f\x6c\x6f\x61\x64\x5f\x62\x79\x74\x65\x73\x5f\x72\
+\x65\x6c\x61\x74\x69\x76\x65\x28\x73\x6b\x62\x2c\x20\x6c\x34\x5f\x6f\x66\x66\
+\x73\x65\x74\x2c\x20\x26\x75\x64\x70\x2c\x20\x73\x69\x7a\x65\x6f\x66\x28\x75\
+\x64\x70\x29\x2c\0\x20\x20\x20\x20\x20\x20\x20\x20\x7d\x20\x65\x6c\x73\x65\x20\
+\x69\x66\x20\x28\x63\x6f\x6e\x66\x69\x67\x2d\x3e\x68\x61\x73\x68\x5f\x74\x79\
+\x70\x65\x73\x20\x26\x20\x56\x49\x52\x54\x49\x4f\x5f\x4e\x45\x54\x5f\x52\x53\
+\x53\x5f\x48\x41\x53\x48\x5f\x54\x59\x50\x45\x5f\x49\x50\x76\x36\x29\x20\x7b\0\
+\x20\x20\x20\x20\x66\x6f\x72\x20\x28\x62\x79\x74\x65\x20\x3d\x20\x30\x3b\x20\
+\x62\x79\x74\x65\x20\x3c\x20\x48\x41\x53\x48\x5f\x43\x41\x4c\x43\x55\x4c\x41\
+\x54\x49\x4f\x4e\x5f\x42\x55\x46\x46\x45\x52\x5f\x53\x49\x5a\x45\x3b\x20\x62\
+\x79\x74\x65\x2b\x2b\x29\x20\x7b\0\x20\x20\x20\x20\x5f\x5f\x75\x33\x32\x20\x6c\
+\x65\x66\x74\x6d\x6f\x73\x74\x5f\x33\x32\x5f\x62\x69\x74\x73\x20\x3d\x20\x6b\
+\x65\x79\x2d\x3e\x6c\x65\x66\x74\x6d\x6f\x73\x74\x5f\x33\x32\x5f\x62\x69\x74\
+\x73\x3b\0\x20\x20\x20\x20\x20\x20\x20\x20\x5f\x5f\x75\x38\x20\x69\x6e\x70\x75\
+\x74\x5f\x62\x79\x74\x65\x20\x3d\x20\x69\x6e\x70\x75\x74\x5b\x62\x79\x74\x65\
+\x5d\x3b\0\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x69\x66\x20\x28\x69\
+\x6e\x70\x75\x74\x5f\x62\x79\x74\x65\x20\x26\x20\x28\x31\x20\x3c\x3c\x20\x37\
+\x29\x29\x20\x7b\0\x20\x20\x20\x20\x20\x20\x20\x20\x5f\x5f\x75\x38\x20\x6b\x65\
+\x79\x5f\x62\x79\x74\x65\x20\x3d\x20\x6b\x65\x79\x2d\x3e\x6e\x65\x78\x74\x5f\
+\x62\x79\x74\x65\x5b\x62\x79\x74\x65\x5d\x3b\0\x20\x20\x20\x20\x20\x20\x20\x20\
+\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x28\x6c\x65\x66\x74\x6d\x6f\
+\x73\x74\x5f\x33\x32\x5f\x62\x69\x74\x73\x20\x3c\x3c\x20\x31\x29\x20\x7c\x20\
+\x28\x28\x6b\x65\x79\x5f\x62\x79\x74\x65\x20\x26\x20\x28\x31\x20\x3c\x3c\x20\
+\x37\x29\x29\x20\x3e\x3e\x20\x37\x29\x3b\0\x20\x20\x20\x20\x20\x20\x20\x20\x69\
+\x66\x20\x28\x68\x61\x73\x68\x29\x20\x7b\0\x20\x20\x20\x20\x20\x20\x20\x20\x20\
+\x20\x20\x20\x5f\x5f\x75\x33\x32\x20\x74\x61\x62\x6c\x65\x5f\x69\x64\x78\x20\
+\x3d\x20\x68\x61\x73\x68\x20\x25\x20\x63\x6f\x6e\x66\x69\x67\x2d\x3e\x69\x6e\
+\x64\x69\x72\x65\x63\x74\x69\x6f\x6e\x73\x5f\x6c\x65\x6e\x3b\0\x20\x20\x20\x20\
+\x20\x20\x20\x20\x20\x20\x20\x20\x71\x75\x65\x75\x65\x20\x3d\x20\x62\x70\x66\
+\x5f\x6d\x61\x70\x5f\x6c\x6f\x6f\x6b\x75\x70\x5f\x65\x6c\x65\x6d\x28\x26\x74\
+\x61\x70\x5f\x72\x73\x73\x5f\x6d\x61\x70\x5f\x69\x6e\x64\x69\x72\x65\x63\x74\
+\x69\x6f\x6e\x5f\x74\x61\x62\x6c\x65\x2c\0\x20\x20\x20\x20\x20\x20\x20\x20\x20\
+\x20\x20\x20\x69\x66\x20\x28\x71\x75\x65\x75\x65\x29\x20\x7b\0\x7d\0\x20\x20\
+\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x72\x65\x74\x75\x72\
+\x6e\x20\x2a\x71\x75\x65\x75\x65\x3b\0\x63\x68\x61\x72\0\x5f\x6c\x69\x63\x65\
+\x6e\x73\x65\0\x2e\x6d\x61\x70\x73\0\x6c\x69\x63\x65\x6e\x73\x65\0\x62\x70\x66\
+\x5f\x66\x6c\x6f\x77\x5f\x6b\x65\x79\x73\0\x62\x70\x66\x5f\x73\x6f\x63\x6b\0\0\
+\x9f\xeb\x01\0\x20\0\0\0\0\0\0\0\x14\0\0\0\x14\0\0\0\x1c\x0d\0\0\x30\x0d\0\0\0\
+\0\0\0\x08\0\0\0\x30\x02\0\0\x01\0\0\0\0\0\0\0\x26\0\0\0\x10\0\0\0\x30\x02\0\0\
+\xd1\0\0\0\0\0\0\0\x37\x02\0\0\x67\x02\0\0\0\x5c\x08\0\x10\0\0\0\x37\x02\0\0\
+\x98\x02\0\0\x0b\x74\x08\0\x20\0\0\0\x37\x02\0\0\0\0\0\0\0\0\0\0\x28\0\0\0\x37\
+\x02\0\0\xab\x02\0\0\x0e\x80\x08\0\x50\0\0\0\x37\x02\0\0\xf0\x02\0\0\x0b\x84\
+\x08\0\x88\0\0\0\x37\x02\0\0\x30\x03\0\0\x10\x8c\x08\0\x90\0\0\0\x37\x02\0\0\0\
+\0\0\0\0\0\0\0\x98\0\0\0\x37\x02\0\0\x30\x03\0\0\x10\x8c\x08\0\xa0\0\0\0\x37\
+\x02\0\0\x49\x03\0\0\x16\x90\x08\0\xa8\0\0\0\x37\x02\0\0\x49\x03\0\0\x0d\x90\
+\x08\0\xc0\0\0\0\x37\x02\0\0\x6a\x03\0\0\x0a\x08\x06\0\xe8\0\0\0\x37\x02\0\0\
+\xa1\x03\0\0\x1f\x18\x06\0\x38\x01\0\0\x37\x02\0\0\xd1\x03\0\0\x0f\xac\x04\0\
+\x40\x01\0\0\x37\x02\0\0\xea\x03\0\0\x0c\x2c\x04\0\x50\x01\0\0\x37\x02\0\0\0\0\
+\0\0\0\0\0\0\x58\x01\0\0\x37\x02\0\0\xfe\x03\0\0\x0b\x38\x04\0\x80\x01\0\0\x37\
+\x02\0\0\x44\x04\0\0\x09\x40\x04\0\x90\x01\0\0\x37\x02\0\0\x44\x04\0\0\x09\x40\
+\x04\0\xa0\x01\0\0\x37\x02\0\0\x53\x04\0\0\x0d\x50\x04\0\xb8\x01\0\0\x37\x02\0\
+\0\x53\x04\0\0\x05\x50\x04\0\xd8\x01\0\0\x37\x02\0\0\0\0\0\0\0\0\0\0\xe0\x01\0\
+\0\x37\x02\0\0\x71\x04\0\0\x0f\x64\x04\0\0\x02\0\0\x37\x02\0\0\x71\x04\0\0\x09\
+\x64\x04\0\x10\x02\0\0\x37\x02\0\0\x44\x04\0\0\x09\x7c\x04\0\x18\x02\0\0\x37\
+\x02\0\0\xbb\x04\0\0\x0c\x8c\x04\0\x20\x02\0\0\x37\x02\0\0\xcb\x04\0\0\x09\xc8\
+\x04\0\x50\x02\0\0\x37\x02\0\0\xe7\x04\0\0\x17\xe0\x04\0\x60\x02\0\0\x37\x02\0\
+\0\x02\x05\0\0\x16\xe8\x04\0\x80\x02\0\0\x37\x02\0\0\xe7\x04\0\0\x17\xe0\x04\0\
+\x88\x02\0\0\x37\x02\0\0\x20\x05\0\0\x0f\xec\x04\0\xb0\x02\0\0\x37\x02\0\0\x63\
+\x05\0\0\x0d\xf4\x04\0\xc0\x02\0\0\x37\x02\0\0\x63\x05\0\0\x0d\xf4\x04\0\xc8\
+\x02\0\0\x37\x02\0\0\x76\x05\0\0\x24\x0c\x05\0\xd0\x02\0\0\x37\x02\0\0\x76\x05\
+\0\0\x20\x0c\x05\0\xe0\x02\0\0\x37\x02\0\0\xa3\x05\0\0\x1b\x04\x05\0\xe8\x02\0\
+\0\x37\x02\0\0\xa3\x05\0\0\x16\x04\x05\0\xf0\x02\0\0\x37\x02\0\0\xc4\x05\0\0\
+\x1b\x08\x05\0\xf8\x02\0\0\x37\x02\0\0\xc4\x05\0\0\x16\x08\x05\0\0\x03\0\0\x37\
+\x02\0\0\xe5\x05\0\0\x1a\x14\x05\0\x08\x03\0\0\x37\x02\0\0\x76\x05\0\0\x1d\x0c\
+\x05\0\x10\x03\0\0\x37\x02\0\0\x08\x06\0\0\x18\x18\x05\0\x18\x03\0\0\x37\x02\0\
+\0\x08\x06\0\0\x1c\x18\x05\0\x30\x03\0\0\x37\x02\0\0\x28\x06\0\0\x15\x74\x05\0\
+\x40\x03\0\0\x37\x02\0\0\x28\x06\0\0\x1a\x74\x05\0\x58\x03\0\0\x37\x02\0\0\x5c\
+\x06\0\0\x0d\x78\x05\0\x78\x03\0\0\x37\x02\0\0\x86\x06\0\0\x1a\x7c\x05\0\x88\
+\x03\0\0\x37\x02\0\0\xa4\x06\0\0\x1b\x84\x05\0\xa8\x03\0\0\x37\x02\0\0\x86\x06\
+\0\0\x1a\x7c\x05\0\xb0\x03\0\0\x37\x02\0\0\xc8\x06\0\0\x13\x88\x05\0\xd8\x03\0\
+\0\x37\x02\0\0\x19\x07\0\0\x11\x90\x05\0\xe8\x03\0\0\x37\x02\0\0\x19\x07\0\0\
+\x11\x90\x05\0\xf0\x03\0\0\x37\x02\0\0\0\0\0\0\0\0\0\0\x10\x04\0\0\x37\x02\0\0\
+\x30\x07\0\0\x15\x34\x06\0\x18\x04\0\0\x37\x02\0\0\x30\x07\0\0\x09\x34\x06\0\
+\x20\x04\0\0\x37\x02\0\0\0\0\0\0\0\0\0\0\x70\x04\0\0\x37\x02\0\0\x4f\x07\0\0\
+\x19\x38\x06\0\x80\x04\0\0\x37\x02\0\0\x4f\x07\0\0\x20\x38\x06\0\xa0\x04\0\0\
+\x37\x02\0\0\0\0\0\0\0\0\0\0\xf0\x04\0\0\x37\x02\0\0\x71\x07\0\0\x17\x20\x05\0\
+\0\x05\0\0\x37\x02\0\0\x8c\x07\0\0\x18\x28\x05\0\x30\x05\0\0\x37\x02\0\0\x71\
+\x07\0\0\x17\x20\x05\0\x48\x05\0\0\x37\x02\0\0\xad\x07\0\0\x0f\x2c\x05\0\x70\
+\x05\0\0\x37\x02\0\0\x63\x05\0\0\x0d\x34\x05\0\x80\x05\0\0\x37\x02\0\0\x63\x05\
+\0\0\x0d\x34\x05\0\x88\x05\0\0\x37\x02\0\0\xf2\x07\0\0\x1d\x44\x05\0\xc8\x05\0\
+\0\x37\x02\0\0\x15\x08\0\0\x1d\x48\x05\0\x08\x06\0\0\x37\x02\0\0\x38\x08\0\0\
+\x1b\x50\x05\0\x10\x06\0\0\x37\x02\0\0\x5b\x08\0\0\x05\x3c\x02\0\x58\x06\0\0\
+\x37\x02\0\0\x73\x08\0\0\x19\xc4\x02\0\xd0\x06\0\0\x37\x02\0\0\0\0\0\0\0\0\0\0\
+\xd8\x06\0\0\x37\x02\0\0\x99\x08\0\0\x0f\xd4\x02\0\0\x07\0\0\x37\x02\0\0\x63\
+\x05\0\0\x0d\xdc\x02\0\x18\x07\0\0\x37\x02\0\0\x63\x05\0\0\x0d\xdc\x02\0\x20\
+\x07\0\0\x37\x02\0\0\xde\x08\0\0\x0d\xec\x02\0\x40\x07\0\0\x37\x02\0\0\x0d\x09\
+\0\0\x20\xf0\x02\0\x68\x07\0\0\x37\x02\0\0\x39\x09\0\0\x13\xf8\x02\0\x90\x07\0\
+\0\x37\x02\0\0\x19\x07\0\0\x11\0\x03\0\xa8\x07\0\0\x37\x02\0\0\x19\x07\0\0\x11\
+\0\x03\0\xb0\x07\0\0\x37\x02\0\0\x81\x09\0\0\x19\x10\x03\0\xb8\x07\0\0\x37\x02\
+\0\0\x81\x09\0\0\x34\x10\x03\0\xe0\x07\0\0\x37\x02\0\0\xb7\x09\0\0\x15\x24\x03\
+\0\xf0\x07\0\0\x37\x02\0\0\xf8\x09\0\0\x17\x20\x03\0\x18\x08\0\0\x37\x02\0\0\
+\x2f\x0a\0\0\x15\x30\x03\0\x30\x08\0\0\x37\x02\0\0\x2f\x0a\0\0\x15\x30\x03\0\
+\x38\x08\0\0\x37\x02\0\0\x4a\x0a\0\0\x27\x40\x03\0\x70\x08\0\0\x37\x02\0\0\x75\
+\x0a\0\0\x27\x5c\x03\0\x80\x08\0\0\x37\x02\0\0\xa5\x0a\0\0\x1c\xc0\x03\0\x88\
+\x08\0\0\x37\x02\0\0\xe1\x0a\0\0\x20\xcc\x03\0\x98\x08\0\0\x37\x02\0\0\xe1\x0a\
+\0\0\x2f\xcc\x03\0\xa0\x08\0\0\x37\x02\0\0\xe1\x0a\0\0\x36\xcc\x03\0\xa8\x08\0\
+\0\x37\x02\0\0\xe1\x0a\0\0\x15\xcc\x03\0\x18\x09\0\0\x37\x02\0\0\x1d\x0b\0\0\
+\x43\x70\x03\0\x38\x09\0\0\x37\x02\0\0\0\0\0\0\0\0\0\0\x40\x09\0\0\x37\x02\0\0\
+\x1d\x0b\0\0\x17\x70\x03\0\x68\x09\0\0\x37\x02\0\0\x2f\x0a\0\0\x15\x78\x03\0\
+\x80\x09\0\0\x37\x02\0\0\x2f\x0a\0\0\x15\x78\x03\0\x88\x09\0\0\x37\x02\0\0\x6d\
+\x0b\0\0\x19\x88\x03\0\x90\x09\0\0\x37\x02\0\0\x6d\x0b\0\0\x15\x88\x03\0\x98\
+\x09\0\0\x37\x02\0\0\x9d\x0b\0\0\x19\x90\x03\0\xa0\x09\0\0\x37\x02\0\0\xcd\x0b\
+\0\0\x1b\x8c\x03\0\xd0\x09\0\0\x37\x02\0\0\x08\x0c\0\0\x19\xa0\x03\0\xe8\x09\0\
+\0\x37\x02\0\0\x08\x0c\0\0\x19\xa0\x03\0\xf0\x09\0\0\x37\x02\0\0\x27\x0c\0\0\
+\x2b\xb0\x03\0\x10\x0a\0\0\x37\x02\0\0\xa5\x0a\0\0\x1f\xc0\x03\0\x30\x0a\0\0\
+\x37\x02\0\0\x56\x0c\0\0\x21\xe0\x03\0\x40\x0a\0\0\x37\x02\0\0\x7e\x0c\0\0\x20\
+\xf0\x03\0\x48\x0a\0\0\x37\x02\0\0\x7e\x0c\0\0\x2c\xf0\x03\0\x60\x0a\0\0\x37\
+\x02\0\0\x7e\x0c\0\0\x14\xf0\x03\0\x70\x0a\0\0\x37\x02\0\0\xae\x0c\0\0\x20\xec\
+\x03\0\x80\x0a\0\0\x37\x02\0\0\x5b\x08\0\0\x05\x3c\x02\0\xb0\x0a\0\0\x37\x02\0\
+\0\xd6\x0c\0\0\x38\xcc\x02\0\xd0\x0a\0\0\x37\x02\0\0\xd6\x0c\0\0\x05\xcc\x02\0\
+\xe8\x0a\0\0\x37\x02\0\0\x5b\x08\0\0\x05\x3c\x02\0\xf8\x0a\0\0\x37\x02\0\0\x14\
+\x0d\0\0\x1c\xd0\x06\0\x08\x0b\0\0\x37\x02\0\0\x14\x0d\0\0\x10\xd0\x06\0\x10\
+\x0b\0\0\x37\x02\0\0\0\0\0\0\0\0\0\0\x60\x0b\0\0\x37\x02\0\0\x4f\x07\0\0\x19\
+\xd4\x06\0\x68\x0b\0\0\x37\x02\0\0\x4f\x07\0\0\x20\xd4\x06\0\xa0\x0b\0\0\x37\
+\x02\0\0\x3a\x0d\0\0\x2d\x0c\x07\0\xb0\x0b\0\0\x37\x02\0\0\x3a\x0d\0\0\x1d\x0c\
+\x07\0\xb8\x0b\0\0\x37\x02\0\0\x3a\x0d\0\0\x2d\x0c\x07\0\xc8\x0b\0\0\x37\x02\0\
+\0\x69\x0d\0\0\x2d\xe0\x06\0\xf8\x0b\0\0\x37\x02\0\0\x69\x0d\0\0\x1d\xe0\x06\0\
+\x08\x0c\0\0\x37\x02\0\0\x69\x0d\0\0\x2d\xe0\x06\0\x18\x0c\0\0\x37\x02\0\0\0\0\
+\0\0\0\0\0\0\xe8\x0c\0\0\x37\x02\0\0\x98\x0d\0\0\x20\x74\x06\0\xf0\x0c\0\0\x37\
+\x02\0\0\x98\x0d\0\0\x27\x74\x06\0\x18\x0d\0\0\x37\x02\0\0\xc1\x0d\0\0\x27\xb0\
+\x06\0\x20\x0d\0\0\x37\x02\0\0\xc1\x0d\0\0\x14\xb0\x06\0\x28\x0d\0\0\x37\x02\0\
+\0\x0a\x0e\0\0\x05\xa4\x01\0\x38\x0d\0\0\x37\x02\0\0\x0a\x0e\0\0\x05\xa4\x01\0\
+\x60\x0d\0\0\x37\x02\0\0\x63\x05\0\0\x0d\x60\x05\0\x70\x0d\0\0\x37\x02\0\0\0\0\
+\0\0\0\0\0\0\x80\x0d\0\0\x37\x02\0\0\x98\x0d\0\0\x20\x50\x07\0\x88\x0d\0\0\x37\
+\x02\0\0\x98\x0d\0\0\x27\x50\x07\0\xc0\x0d\0\0\x37\x02\0\0\x3a\x0d\0\0\x2d\x88\
+\x07\0\xd0\x0d\0\0\x37\x02\0\0\x3a\x0d\0\0\x1d\x88\x07\0\xd8\x0d\0\0\x37\x02\0\
+\0\x3a\x0d\0\0\x2d\x88\x07\0\xe8\x0d\0\0\x37\x02\0\0\x69\x0d\0\0\x2d\x5c\x07\0\
+\x18\x0e\0\0\x37\x02\0\0\x69\x0d\0\0\x1d\x5c\x07\0\x28\x0e\0\0\x37\x02\0\0\x69\
+\x0d\0\0\x2d\x5c\x07\0\x40\x0e\0\0\x37\x02\0\0\x47\x0e\0\0\x1a\xac\x05\0\x50\
+\x0e\0\0\x37\x02\0\0\x65\x0e\0\0\x1b\xb4\x05\0\x60\x0e\0\0\x37\x02\0\0\x47\x0e\
+\0\0\x1a\xac\x05\0\x68\x0e\0\0\x37\x02\0\0\x89\x0e\0\0\x13\xb8\x05\0\x90\x0e\0\
+\0\x37\x02\0\0\x19\x07\0\0\x11\xc0\x05\0\xa0\x0e\0\0\x37\x02\0\0\x19\x07\0\0\
+\x11\xc0\x05\0\xb0\x0e\0\0\x37\x02\0\0\x5b\x08\0\0\x05\x3c\x02\0\xc0\x0e\0\0\
+\x37\x02\0\0\xda\x0e\0\0\x27\xd4\x07\0\xd0\x0e\0\0\x37\x02\0\0\xda\x0e\0\0\x14\
+\xd4\x07\0\xf0\x0e\0\0\x37\x02\0\0\x69\x0d\0\0\x2d\xd8\x07\0\0\x0f\0\0\x37\x02\
+\0\0\x69\x0d\0\0\x1d\xd8\x07\0\x08\x0f\0\0\x37\x02\0\0\x69\x0d\0\0\x2d\xd8\x07\
+\0\x30\x0f\0\0\x37\x02\0\0\0\0\0\0\0\0\0\0\x80\x0f\0\0\x37\x02\0\0\x3a\x0d\0\0\
+\x1d\x04\x08\0\x88\x0f\0\0\x37\x02\0\0\x3a\x0d\0\0\x2d\x04\x08\0\x98\x0f\0\0\
+\x37\x02\0\0\x0a\x0e\0\0\x05\xa4\x01\0\xf0\x0f\0\0\x37\x02\0\0\x0a\x0e\0\0\x05\
+\xa4\x01\0\x30\x10\0\0\x37\x02\0\0\0\0\0\0\0\0\0\0\x48\x10\0\0\x37\x02\0\0\x23\
+\x0f\0\0\x05\xdc\x01\0\x50\x10\0\0\x37\x02\0\0\x65\x0f\0\0\x23\xd0\x01\0\x68\
+\x10\0\0\x37\x02\0\0\0\0\0\0\0\0\0\0\x70\x10\0\0\x37\x02\0\0\x99\x0f\0\0\x1b\
+\xe0\x01\0\x90\x10\0\0\x37\x02\0\0\xc0\x0f\0\0\x11\xf4\x01\0\xa8\x10\0\0\x37\
+\x02\0\0\xe9\x0f\0\0\x19\xe4\x01\0\xc0\x10\0\0\x37\x02\0\0\x17\x10\0\0\x27\x08\
+\x02\0\xc8\x10\0\0\x37\x02\0\0\x17\x10\0\0\x46\x08\x02\0\xd8\x10\0\0\x37\x02\0\
+\0\x17\x10\0\0\x2d\x08\x02\0\xe0\x10\0\0\x37\x02\0\0\xc0\x0f\0\0\x11\xf4\x01\0\
+\x08\x11\0\0\x37\x02\0\0\x17\x10\0\0\x46\x08\x02\0\x20\x11\0\0\x37\x02\0\0\x17\
+\x10\0\0\x27\x08\x02\0\x28\x11\0\0\x37\x02\0\0\x17\x10\0\0\x2d\x08\x02\0\x30\
+\x11\0\0\x37\x02\0\0\xc0\x0f\0\0\x11\xf4\x01\0\x58\x11\0\0\x37\x02\0\0\x17\x10\
+\0\0\x27\x08\x02\0\x60\x11\0\0\x37\x02\0\0\x17\x10\0\0\x46\x08\x02\0\x78\x11\0\
+\0\x37\x02\0\0\x17\x10\0\0\x2d\x08\x02\0\x80\x11\0\0\x37\x02\0\0\xc0\x0f\0\0\
+\x11\xf4\x01\0\xa8\x11\0\0\x37\x02\0\0\x17\x10\0\0\x27\x08\x02\0\xb0\x11\0\0\
+\x37\x02\0\0\x17\x10\0\0\x46\x08\x02\0\xc8\x11\0\0\x37\x02\0\0\x17\x10\0\0\x2d\
+\x08\x02\0\xd0\x11\0\0\x37\x02\0\0\xc0\x0f\0\0\x11\xf4\x01\0\xf8\x11\0\0\x37\
+\x02\0\0\x17\x10\0\0\x46\x08\x02\0\x10\x12\0\0\x37\x02\0\0\x17\x10\0\0\x27\x08\
+\x02\0\x18\x12\0\0\x37\x02\0\0\x17\x10\0\0\x2d\x08\x02\0\x20\x12\0\0\x37\x02\0\
+\0\xc0\x0f\0\0\x11\xf4\x01\0\x48\x12\0\0\x37\x02\0\0\x17\x10\0\0\x46\x08\x02\0\
+\x60\x12\0\0\x37\x02\0\0\x17\x10\0\0\x27\x08\x02\0\x68\x12\0\0\x37\x02\0\0\x17\
+\x10\0\0\x2d\x08\x02\0\x70\x12\0\0\x37\x02\0\0\xc0\x0f\0\0\x11\xf4\x01\0\x98\
+\x12\0\0\x37\x02\0\0\x17\x10\0\0\x46\x08\x02\0\xb0\x12\0\0\x37\x02\0\0\x17\x10\
+\0\0\x27\x08\x02\0\xb8\x12\0\0\x37\x02\0\0\x17\x10\0\0\x2d\x08\x02\0\xc0\x12\0\
+\0\x37\x02\0\0\xc0\x0f\0\0\x11\xf4\x01\0\xe0\x12\0\0\x37\x02\0\0\x17\x10\0\0\
+\x46\x08\x02\0\xe8\x12\0\0\x37\x02\0\0\x17\x10\0\0\x27\x08\x02\0\xf0\x12\0\0\
+\x37\x02\0\0\x17\x10\0\0\x2d\x08\x02\0\xf8\x12\0\0\x37\x02\0\0\x23\x0f\0\0\x3d\
+\xdc\x01\0\x08\x13\0\0\x37\x02\0\0\x23\x0f\0\0\x05\xdc\x01\0\x18\x13\0\0\x37\
+\x02\0\0\x63\x10\0\0\x0d\xa4\x08\0\x28\x13\0\0\x37\x02\0\0\x63\x10\0\0\x0d\xa4\
+\x08\0\x30\x13\0\0\x37\x02\0\0\x77\x10\0\0\x2e\xa8\x08\0\x50\x13\0\0\x37\x02\0\
+\0\x77\x10\0\0\x24\xa8\x08\0\x58\x13\0\0\x37\x02\0\0\x77\x10\0\0\x13\xa8\x08\0\
+\x68\x13\0\0\x37\x02\0\0\x77\x10\0\0\x2e\xa8\x08\0\x70\x13\0\0\x37\x02\0\0\xb6\
+\x10\0\0\x15\xb4\x08\0\x88\x13\0\0\x37\x02\0\0\xfe\x10\0\0\x11\xc0\x08\0\x90\
+\x13\0\0\x37\x02\0\0\0\0\0\0\0\0\0\0\xb0\x13\0\0\x37\x02\0\0\x17\x11\0\0\x01\
+\xe4\x08\0\xb8\x13\0\0\x37\x02\0\0\x19\x11\0\0\x18\xc4\x08\0\0\0\0\0\0\0\0\0\0\
+\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x03\0\x03\0\0\0\0\0\0\0\0\0\0\0\
+\0\0\0\0\0\0\x0d\x01\0\0\0\0\x03\0\xb0\x13\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x78\x01\
+\0\0\0\0\x03\0\xb8\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x2e\x01\0\0\0\0\x03\0\x90\x13\
+\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xbd\0\0\0\0\0\x03\0\xd0\x01\0\0\0\0\0\0\0\0\0\0\0\
+\0\0\0\x2a\x02\0\0\0\0\x03\0\x20\x02\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x05\x01\0\0\0\
+\0\x03\0\xe8\x04\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x43\x02\0\0\0\0\x03\0\x10\x04\0\0\
+\0\0\0\0\0\0\0\0\0\0\0\0\x26\x01\0\0\0\0\x03\0\xe0\x02\0\0\0\0\0\0\0\0\0\0\0\0\
+\0\0\x1a\x02\0\0\0\0\x03\0\x30\x03\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x12\x02\0\0\0\0\
+\x03\0\x38\x0e\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xd4\0\0\0\0\0\x03\0\xf0\x03\0\0\0\0\
+\0\0\0\0\0\0\0\0\0\0\x0a\x02\0\0\0\0\x03\0\xf8\x0a\0\0\0\0\0\0\0\0\0\0\0\0\0\0\
+\x47\x01\0\0\0\0\x03\0\xe8\x0c\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x57\x01\0\0\0\0\x03\
+\0\xa0\x04\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xd0\x01\0\0\0\0\x03\0\x40\x10\0\0\0\0\0\
+\0\0\0\0\0\0\0\0\0\x7f\x01\0\0\0\0\x03\0\x78\x0d\0\0\0\0\0\0\0\0\0\0\0\0\0\0\
+\x53\x02\0\0\0\0\x03\0\xb0\x0e\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xe9\x01\0\0\0\0\x03\
+\0\x50\x06\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xc8\x01\0\0\0\0\x03\0\xc0\x06\0\0\0\0\0\
+\0\0\0\0\0\0\0\0\0\xb0\x01\0\0\0\0\x03\0\x60\x0d\0\0\0\0\0\0\0\0\0\0\0\0\0\0\
+\x8f\x01\0\0\0\0\x03\0\x60\x08\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x5f\x01\0\0\0\0\x03\
+\0\x30\x0a\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x4f\x01\0\0\0\0\x03\0\x40\x0a\0\0\0\0\0\
+\0\0\0\0\0\0\0\0\0\xe1\x01\0\0\0\0\x03\0\xe0\x0f\0\0\0\0\0\0\0\0\0\0\0\0\0\0\
+\xc0\x01\0\0\0\0\x03\0\x48\x08\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x67\x01\0\0\0\0\x03\
+\0\x18\x09\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x22\x02\0\0\0\0\x03\0\x80\x08\0\0\0\0\0\
+\0\0\0\0\0\0\0\0\0\xb8\x01\0\0\0\0\x03\0\xf8\x08\0\0\0\0\0\0\0\0\0\0\0\0\0\0\
+\x87\x01\0\0\0\0\x03\0\x50\x0d\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xe4\0\0\0\0\0\x03\0\
+\x08\x0a\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x1e\x01\0\0\0\0\x03\0\xe8\x0a\0\0\0\0\0\0\
+\0\0\0\0\0\0\0\0\xdc\0\0\0\0\0\x03\0\xb0\x0a\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x4b\
+\x02\0\0\0\0\x03\0\xd8\x0a\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xa8\x01\0\0\0\0\x03\0\
+\x80\x0d\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x3f\x01\0\0\0\0\x03\0\xb0\x0b\0\0\0\0\0\0\
+\0\0\0\0\0\0\0\0\xfd\0\0\0\0\0\x03\0\xc8\x0b\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x3b\
+\x02\0\0\0\0\x03\0\xf8\x0b\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xd9\x01\0\0\0\0\x03\0\
+\x18\x0c\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x02\x02\0\0\0\0\x03\0\x10\x0c\0\0\0\0\0\0\
+\0\0\0\0\0\0\0\0\xcc\0\0\0\0\0\x03\0\x18\x0d\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xa0\
+\x01\0\0\0\0\x03\0\xc0\x0e\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x16\x01\0\0\0\0\x03\0\
+\xd0\x0d\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xc4\0\0\0\0\0\x03\0\xe8\x0d\0\0\0\0\0\0\0\
+\0\0\0\0\0\0\0\xfa\x01\0\0\0\0\x03\0\x18\x0e\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x37\
+\x01\0\0\0\0\x03\0\0\x0f\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xf5\0\0\0\0\0\x03\0\x18\
+\x0f\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x32\x02\0\0\0\0\x03\0\xf0\x0f\0\0\0\0\0\0\0\0\
+\0\0\0\0\0\0\xf1\x01\0\0\0\0\x03\0\x30\x10\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x97\x01\
+\0\0\0\0\x03\0\x60\x10\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x6f\x01\0\0\0\0\x03\0\x18\
+\x13\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xec\0\0\0\0\0\x03\0\xb8\x13\0\0\0\0\0\0\0\0\0\
+\0\0\0\0\0\x59\0\0\0\x12\0\x03\0\0\0\0\0\0\0\0\0\xc8\x13\0\0\0\0\0\0\x3e\0\0\0\
+\x11\0\x05\0\0\0\0\0\0\0\0\0\x28\0\0\0\0\0\0\0\x01\0\0\0\x11\0\x05\0\x28\0\0\0\
+\0\0\0\0\x28\0\0\0\0\0\0\0\x86\0\0\0\x11\0\x05\0\x50\0\0\0\0\0\0\0\x28\0\0\0\0\
+\0\0\0\x7d\0\0\0\x11\0\x06\0\0\0\0\0\0\0\0\0\x07\0\0\0\0\0\0\0\x28\0\0\0\0\0\0\
+\0\x01\0\0\0\x36\0\0\0\x50\0\0\0\0\0\0\0\x01\0\0\0\x37\0\0\0\x70\x13\0\0\0\0\0\
+\0\x01\0\0\0\x38\0\0\0\x20\x05\0\0\0\0\0\0\x04\0\0\0\x36\0\0\0\x2c\x05\0\0\0\0\
+\0\0\x04\0\0\0\x37\0\0\0\x38\x05\0\0\0\0\0\0\x04\0\0\0\x38\0\0\0\x50\x05\0\0\0\
+\0\0\0\x04\0\0\0\x39\0\0\0\x2c\0\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x40\0\0\0\0\0\
+\0\0\x04\0\0\0\x01\0\0\0\x50\0\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x60\0\0\0\0\0\0\
+\0\x04\0\0\0\x01\0\0\0\x70\0\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x80\0\0\0\0\0\0\0\
+\x04\0\0\0\x01\0\0\0\x90\0\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xa0\0\0\0\0\0\0\0\
+\x04\0\0\0\x01\0\0\0\xb0\0\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xc0\0\0\0\0\0\0\0\
+\x04\0\0\0\x01\0\0\0\xd0\0\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xe0\0\0\0\0\0\0\0\
+\x04\0\0\0\x01\0\0\0\xf0\0\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\0\x01\0\0\0\0\0\0\
+\x04\0\0\0\x01\0\0\0\x10\x01\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x20\x01\0\0\0\0\0\
+\0\x04\0\0\0\x01\0\0\0\x30\x01\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x40\x01\0\0\0\0\
+\0\0\x04\0\0\0\x01\0\0\0\x50\x01\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x60\x01\0\0\0\
+\0\0\0\x04\0\0\0\x01\0\0\0\x70\x01\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x80\x01\0\0\
+\0\0\0\0\x04\0\0\0\x01\0\0\0\x90\x01\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xa0\x01\0\
+\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xb0\x01\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xc0\x01\
+\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xd0\x01\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xe0\
+\x01\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xf0\x01\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\0\
+\x02\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x10\x02\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\
+\x20\x02\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x30\x02\0\0\0\0\0\0\x04\0\0\0\x01\0\0\
+\0\x40\x02\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x50\x02\0\0\0\0\0\0\x04\0\0\0\x01\0\
+\0\0\x60\x02\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x70\x02\0\0\0\0\0\0\x04\0\0\0\x01\
+\0\0\0\x80\x02\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x90\x02\0\0\0\0\0\0\x04\0\0\0\
+\x01\0\0\0\xa0\x02\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xb0\x02\0\0\0\0\0\0\x04\0\0\
+\0\x01\0\0\0\xc0\x02\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xd0\x02\0\0\0\0\0\0\x04\0\
+\0\0\x01\0\0\0\xe0\x02\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xf0\x02\0\0\0\0\0\0\x04\
+\0\0\0\x01\0\0\0\0\x03\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x10\x03\0\0\0\0\0\0\x04\
+\0\0\0\x01\0\0\0\x20\x03\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x30\x03\0\0\0\0\0\0\
+\x04\0\0\0\x01\0\0\0\x40\x03\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x50\x03\0\0\0\0\0\
+\0\x04\0\0\0\x01\0\0\0\x60\x03\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x70\x03\0\0\0\0\
+\0\0\x04\0\0\0\x01\0\0\0\x80\x03\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x90\x03\0\0\0\
+\0\0\0\x04\0\0\0\x01\0\0\0\xa0\x03\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xb0\x03\0\0\
+\0\0\0\0\x04\0\0\0\x01\0\0\0\xc0\x03\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xd0\x03\0\
+\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xe0\x03\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xf0\x03\
+\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\0\x04\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x10\x04\
+\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x20\x04\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x30\
+\x04\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x40\x04\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\
+\x50\x04\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x60\x04\0\0\0\0\0\0\x04\0\0\0\x01\0\0\
+\0\x70\x04\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x80\x04\0\0\0\0\0\0\x04\0\0\0\x01\0\
+\0\0\x90\x04\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xa0\x04\0\0\0\0\0\0\x04\0\0\0\x01\
+\0\0\0\xb0\x04\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xc0\x04\0\0\0\0\0\0\x04\0\0\0\
+\x01\0\0\0\xd0\x04\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xe0\x04\0\0\0\0\0\0\x04\0\0\
+\0\x01\0\0\0\xf0\x04\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\0\x05\0\0\0\0\0\0\x04\0\0\
+\0\x01\0\0\0\x10\x05\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x20\x05\0\0\0\0\0\0\x04\0\
+\0\0\x01\0\0\0\x30\x05\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x40\x05\0\0\0\0\0\0\x04\
+\0\0\0\x01\0\0\0\x50\x05\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x60\x05\0\0\0\0\0\0\
+\x04\0\0\0\x01\0\0\0\x70\x05\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x80\x05\0\0\0\0\0\
+\0\x04\0\0\0\x01\0\0\0\x90\x05\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xa0\x05\0\0\0\0\
+\0\0\x04\0\0\0\x01\0\0\0\xb0\x05\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xc0\x05\0\0\0\
+\0\0\0\x04\0\0\0\x01\0\0\0\xd0\x05\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xe0\x05\0\0\
+\0\0\0\0\x04\0\0\0\x01\0\0\0\xf0\x05\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\0\x06\0\0\
+\0\0\0\0\x04\0\0\0\x01\0\0\0\x10\x06\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x20\x06\0\
+\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x30\x06\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x40\x06\
+\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x50\x06\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x60\
+\x06\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x70\x06\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\
+\x80\x06\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x90\x06\0\0\0\0\0\0\x04\0\0\0\x01\0\0\
+\0\xa0\x06\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xb0\x06\0\0\0\0\0\0\x04\0\0\0\x01\0\
+\0\0\xc0\x06\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xd0\x06\0\0\0\0\0\0\x04\0\0\0\x01\
+\0\0\0\xe0\x06\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xf0\x06\0\0\0\0\0\0\x04\0\0\0\
+\x01\0\0\0\0\x07\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x10\x07\0\0\0\0\0\0\x04\0\0\0\
+\x01\0\0\0\x20\x07\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x30\x07\0\0\0\0\0\0\x04\0\0\
+\0\x01\0\0\0\x40\x07\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x50\x07\0\0\0\0\0\0\x04\0\
+\0\0\x01\0\0\0\x60\x07\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x70\x07\0\0\0\0\0\0\x04\
+\0\0\0\x01\0\0\0\x80\x07\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x90\x07\0\0\0\0\0\0\
+\x04\0\0\0\x01\0\0\0\xa0\x07\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xb0\x07\0\0\0\0\0\
+\0\x04\0\0\0\x01\0\0\0\xc0\x07\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xd0\x07\0\0\0\0\
+\0\0\x04\0\0\0\x01\0\0\0\xe0\x07\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xf0\x07\0\0\0\
+\0\0\0\x04\0\0\0\x01\0\0\0\0\x08\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x10\x08\0\0\0\
+\0\0\0\x04\0\0\0\x01\0\0\0\x20\x08\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x30\x08\0\0\
+\0\0\0\0\x04\0\0\0\x01\0\0\0\x40\x08\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x50\x08\0\
+\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x60\x08\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x70\x08\
+\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x80\x08\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x90\
+\x08\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xa0\x08\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\
+\xb0\x08\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xc0\x08\0\0\0\0\0\0\x04\0\0\0\x01\0\0\
+\0\xd0\x08\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xe0\x08\0\0\0\0\0\0\x04\0\0\0\x01\0\
+\0\0\xf0\x08\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\0\x09\0\0\0\0\0\0\x04\0\0\0\x01\0\
+\0\0\x10\x09\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x20\x09\0\0\0\0\0\0\x04\0\0\0\x01\
+\0\0\0\x30\x09\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x40\x09\0\0\0\0\0\0\x04\0\0\0\
+\x01\0\0\0\x50\x09\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x60\x09\0\0\0\0\0\0\x04\0\0\
+\0\x01\0\0\0\x70\x09\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x80\x09\0\0\0\0\0\0\x04\0\
+\0\0\x01\0\0\0\x90\x09\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xa0\x09\0\0\0\0\0\0\x04\
+\0\0\0\x01\0\0\0\xb0\x09\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xc0\x09\0\0\0\0\0\0\
+\x04\0\0\0\x01\0\0\0\xd0\x09\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xe0\x09\0\0\0\0\0\
+\0\x04\0\0\0\x01\0\0\0\xf0\x09\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\0\x0a\0\0\0\0\0\
+\0\x04\0\0\0\x01\0\0\0\x10\x0a\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x20\x0a\0\0\0\0\
+\0\0\x04\0\0\0\x01\0\0\0\x30\x0a\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x40\x0a\0\0\0\
+\0\0\0\x04\0\0\0\x01\0\0\0\x50\x0a\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x60\x0a\0\0\
+\0\0\0\0\x04\0\0\0\x01\0\0\0\x70\x0a\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x80\x0a\0\
+\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x90\x0a\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xa0\x0a\
+\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xb0\x0a\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xc0\
+\x0a\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xd0\x0a\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\
+\xe0\x0a\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xf0\x0a\0\0\0\0\0\0\x04\0\0\0\x01\0\0\
+\0\0\x0b\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x10\x0b\0\0\0\0\0\0\x04\0\0\0\x01\0\0\
+\0\x20\x0b\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x30\x0b\0\0\0\0\0\0\x04\0\0\0\x01\0\
+\0\0\x40\x0b\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x50\x0b\0\0\0\0\0\0\x04\0\0\0\x01\
+\0\0\0\x60\x0b\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x70\x0b\0\0\0\0\0\0\x04\0\0\0\
+\x01\0\0\0\x80\x0b\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x90\x0b\0\0\0\0\0\0\x04\0\0\
+\0\x01\0\0\0\xa0\x0b\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xb0\x0b\0\0\0\0\0\0\x04\0\
+\0\0\x01\0\0\0\xc0\x0b\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xd0\x0b\0\0\0\0\0\0\x04\
+\0\0\0\x01\0\0\0\xe0\x0b\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xf0\x0b\0\0\0\0\0\0\
+\x04\0\0\0\x01\0\0\0\0\x0c\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x10\x0c\0\0\0\0\0\0\
+\x04\0\0\0\x01\0\0\0\x20\x0c\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x30\x0c\0\0\0\0\0\
+\0\x04\0\0\0\x01\0\0\0\x40\x0c\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x50\x0c\0\0\0\0\
+\0\0\x04\0\0\0\x01\0\0\0\x60\x0c\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x70\x0c\0\0\0\
+\0\0\0\x04\0\0\0\x01\0\0\0\x80\x0c\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x90\x0c\0\0\
+\0\0\0\0\x04\0\0\0\x01\0\0\0\xa0\x0c\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xb0\x0c\0\
+\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xc0\x0c\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xd0\x0c\
+\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xe0\x0c\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\xf0\
+\x0c\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\0\x0d\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x10\
+\x0d\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x20\x0d\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\
+\x30\x0d\0\0\0\0\0\0\x04\0\0\0\x01\0\0\0\x40\x0d\0\0\0\0\0\0\x04\0\0\0\x01\0\0\
+\0\x3f\x40\x41\x42\x43\0\x74\x61\x70\x5f\x72\x73\x73\x5f\x6d\x61\x70\x5f\x74\
+\x6f\x65\x70\x6c\x69\x74\x7a\x5f\x6b\x65\x79\0\x2e\x74\x65\x78\x74\0\x2e\x72\
+\x65\x6c\x2e\x42\x54\x46\x2e\x65\x78\x74\0\x2e\x72\x65\x6c\x73\x6f\x63\x6b\x65\
+\x74\0\x2e\x6d\x61\x70\x73\0\x74\x61\x70\x5f\x72\x73\x73\x5f\x6d\x61\x70\x5f\
+\x63\x6f\x6e\x66\x69\x67\x75\x72\x61\x74\x69\x6f\x6e\x73\0\x74\x75\x6e\x5f\x72\
+\x73\x73\x5f\x73\x74\x65\x65\x72\x69\x6e\x67\x5f\x70\x72\x6f\x67\0\x2e\x6c\x6c\
+\x76\x6d\x5f\x61\x64\x64\x72\x73\x69\x67\0\x5f\x6c\x69\x63\x65\x6e\x73\x65\0\
+\x74\x61\x70\x5f\x72\x73\x73\x5f\x6d\x61\x70\x5f\x69\x6e\x64\x69\x72\x65\x63\
+\x74\x69\x6f\x6e\x5f\x74\x61\x62\x6c\x65\0\x2e\x73\x74\x72\x74\x61\x62\0\x2e\
+\x73\x79\x6d\x74\x61\x62\0\x2e\x72\x65\x6c\x2e\x42\x54\x46\0\x4c\x42\x42\x30\
+\x5f\x39\0\x4c\x42\x42\x30\x5f\x38\x39\0\x4c\x42\x42\x30\x5f\x36\x39\0\x4c\x42\
+\x42\x30\x5f\x35\x39\0\x4c\x42\x42\x30\x5f\x34\x39\0\x4c\x42\x42\x30\x5f\x33\
+\x39\0\x4c\x42\x42\x30\x5f\x31\x30\x39\0\x4c\x42\x42\x30\x5f\x39\x38\0\x4c\x42\
+\x42\x30\x5f\x37\x38\0\x4c\x42\x42\x30\x5f\x31\x38\0\x4c\x42\x42\x30\x5f\x31\
+\x30\x38\0\x4c\x42\x42\x30\x5f\x38\x37\0\x4c\x42\x42\x30\x5f\x34\x37\0\x4c\x42\
+\x42\x30\x5f\x31\x37\0\x4c\x42\x42\x30\x5f\x31\x30\x37\0\x4c\x42\x42\x30\x5f\
+\x39\x36\0\x4c\x42\x42\x30\x5f\x37\x36\0\x4c\x42\x42\x30\x5f\x36\x36\0\x4c\x42\
+\x42\x30\x5f\x34\x36\0\x4c\x42\x42\x30\x5f\x36\x35\0\x4c\x42\x42\x30\x5f\x34\
+\x35\0\x4c\x42\x42\x30\x5f\x33\x35\0\x4c\x42\x42\x30\x5f\x31\x30\x35\0\x4c\x42\
+\x42\x30\x5f\x34\0\x4c\x42\x42\x30\x5f\x35\x34\0\x4c\x42\x42\x30\x5f\x34\x34\0\
 \x4c\x42\x42\x30\x5f\x33\x34\0\x4c\x42\x42\x30\x5f\x31\x30\x34\0\x4c\x42\x42\
-\x30\x5f\x38\x33\0\x4c\x42\x42\x30\x5f\x35\x33\0\x4c\x42\x42\x30\x5f\x32\x33\0\
-\x4c\x42\x42\x30\x5f\x31\x30\x33\0\x4c\x42\x42\x30\x5f\x39\x32\0\x4c\x42\x42\
-\x30\x5f\x38\x32\0\x4c\x42\x42\x30\x5f\x37\x32\0\x4c\x42\x42\x30\x5f\x36\x32\0\
-\x4c\x42\x42\x30\x5f\x35\x32\0\x4c\x42\x42\x30\x5f\x34\x32\0\x4c\x42\x42\x30\
-\x5f\x32\x32\0\x4c\x42\x42\x30\x5f\x31\x30\x32\0\x4c\x42\x42\x30\x5f\x38\x31\0\
-\x4c\x42\x42\x30\x5f\x36\x31\0\x4c\x42\x42\x30\x5f\x35\x31\0\x4c\x42\x42\x30\
-\x5f\x31\x31\0\x4c\x42\x42\x30\x5f\x39\x30\0\x4c\x42\x42\x30\x5f\x37\x30\0\x4c\
-\x42\x42\x30\x5f\x36\x30\0\x4c\x42\x42\x30\x5f\x35\x30\0\x4c\x42\x42\x30\x5f\
-\x34\x30\0\x4c\x42\x42\x30\x5f\x32\x30\0\x4c\x42\x42\x30\x5f\x31\x31\x30\0\0\0\
+\x30\x5f\x39\x33\0\x4c\x42\x42\x30\x5f\x38\x33\0\x4c\x42\x42\x30\x5f\x35\x33\0\
+\x4c\x42\x42\x30\x5f\x34\x33\0\x4c\x42\x42\x30\x5f\x33\x33\0\x4c\x42\x42\x30\
+\x5f\x32\x33\0\x4c\x42\x42\x30\x5f\x31\x30\x33\0\x4c\x42\x42\x30\x5f\x38\x32\0\
+\x4c\x42\x42\x30\x5f\x35\x32\0\x4c\x42\x42\x30\x5f\x32\x32\0\x4c\x42\x42\x30\
+\x5f\x31\x30\x32\0\x4c\x42\x42\x30\x5f\x39\x31\0\x4c\x42\x42\x30\x5f\x38\x31\0\
+\x4c\x42\x42\x30\x5f\x37\x31\0\x4c\x42\x42\x30\x5f\x36\x31\0\x4c\x42\x42\x30\
+\x5f\x35\x31\0\x4c\x42\x42\x30\x5f\x34\x31\0\x4c\x42\x42\x30\x5f\x31\x31\0\x4c\
+\x42\x42\x30\x5f\x31\x30\x31\0\x4c\x42\x42\x30\x5f\x38\x30\0\x4c\x42\x42\x30\
+\x5f\x36\x30\0\x4c\x42\x42\x30\x5f\x35\x30\0\x4c\x42\x42\x30\x5f\x32\x30\0\0\0\
 \0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\
-\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xae\0\0\0\x03\0\0\0\
-\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x25\x4a\0\0\0\0\0\0\x6d\x02\0\0\0\0\0\0\0\0\0\
-\0\0\0\0\0\x01\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x1a\0\0\0\x01\0\0\0\x06\0\0\0\0\0\
-\0\0\0\0\0\0\0\0\0\0\x40\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x04\0\0\
-\0\0\0\0\0\0\0\0\0\0\0\0\0\x68\0\0\0\x01\0\0\0\x06\0\0\0\0\0\0\0\0\0\0\0\0\0\0\
-\0\x40\0\0\0\0\0\0\0\xe0\x13\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x08\0\0\0\0\0\0\0\0\0\
-\0\0\0\0\0\0\x64\0\0\0\x09\0\0\0\x40\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x40\x3d\0\0\
-\0\0\0\0\x30\0\0\0\0\0\0\0\x0c\0\0\0\x03\0\0\0\x08\0\0\0\0\0\0\0\x10\0\0\0\0\0\
-\0\0\x2d\0\0\0\x01\0\0\0\x03\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x20\x14\0\0\0\0\0\0\
-\x60\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x08\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x88\0\0\0\
-\x01\0\0\0\x03\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x80\x14\0\0\0\0\0\0\x07\0\0\0\0\0\
-\0\0\0\0\0\0\0\0\0\0\x01\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xc2\0\0\0\x01\0\0\0\0\0\
-\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x88\x14\0\0\0\0\0\0\x8d\x16\0\0\0\0\0\0\0\0\0\0\0\
-\0\0\0\x04\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xbe\0\0\0\x09\0\0\0\x40\0\0\0\0\0\0\0\
-\0\0\0\0\0\0\0\0\x70\x3d\0\0\0\0\0\0\x40\0\0\0\0\0\0\0\x0c\0\0\0\x07\0\0\0\x08\
-\0\0\0\0\0\0\0\x10\0\0\0\0\0\0\0\x24\0\0\0\x01\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\
-\0\0\0\x18\x2b\0\0\0\0\0\0\xa0\x0c\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x04\0\0\0\0\0\0\
-\0\0\0\0\0\0\0\0\0\x20\0\0\0\x09\0\0\0\x40\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xb0\
-\x3d\0\0\0\0\0\0\x70\x0c\0\0\0\0\0\0\x0c\0\0\0\x09\0\0\0\x08\0\0\0\0\0\0\0\x10\
-\0\0\0\0\0\0\0\x79\0\0\0\x03\x4c\xff\x6f\0\0\0\x80\0\0\0\0\0\0\0\0\0\0\0\0\x20\
-\x4a\0\0\0\0\0\0\x05\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x01\0\0\0\0\0\0\0\0\0\0\0\0\
-\0\0\0\xb6\0\0\0\x02\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xb8\x37\0\0\0\0\0\0\
-\x88\x05\0\0\0\0\0\0\x01\0\0\0\x36\0\0\0\x08\0\0\0\0\0\0\0\x18\0\0\0\0\0\0\0";
+\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xa4\0\0\0\x03\0\0\0\0\0\0\0\0\0\
+\0\0\0\0\0\0\0\0\0\0\xbd\x4b\0\0\0\0\0\0\x5b\x02\0\0\0\0\0\0\0\0\0\0\0\0\0\0\
+\x01\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x1a\0\0\0\x01\0\0\0\x06\0\0\0\0\0\0\0\0\0\0\
+\0\0\0\0\0\x40\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x04\0\0\0\0\0\0\0\
+\0\0\0\0\0\0\0\0\x31\0\0\0\x01\0\0\0\x06\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x40\0\0\
+\0\0\0\0\0\xc8\x13\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x08\0\0\0\0\0\0\0\0\0\0\0\0\0\0\
+\0\x2d\0\0\0\x09\0\0\0\x40\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x28\x3e\0\0\0\0\0\0\
+\x30\0\0\0\0\0\0\0\x0c\0\0\0\x03\0\0\0\x08\0\0\0\0\0\0\0\x10\0\0\0\0\0\0\0\x38\
+\0\0\0\x01\0\0\0\x03\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x08\x14\0\0\0\0\0\0\x78\0\0\
+\0\0\0\0\0\0\0\0\0\0\0\0\0\x08\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x7e\0\0\0\x01\0\0\
+\0\x03\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x80\x14\0\0\0\0\0\0\x07\0\0\0\0\0\0\0\0\0\
+\0\0\0\0\0\0\x01\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xb8\0\0\0\x01\0\0\0\0\0\0\0\0\0\
+\0\0\0\0\0\0\0\0\0\0\x88\x14\0\0\0\0\0\0\xdb\x16\0\0\0\0\0\0\0\0\0\0\0\0\0\0\
+\x04\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xb4\0\0\0\x09\0\0\0\x40\0\0\0\0\0\0\0\0\0\0\
+\0\0\0\0\0\x58\x3e\0\0\0\0\0\0\x40\0\0\0\0\0\0\0\x0c\0\0\0\x07\0\0\0\x08\0\0\0\
+\0\0\0\0\x10\0\0\0\0\0\0\0\x24\0\0\0\x01\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\
+\x64\x2b\0\0\0\0\0\0\x50\x0d\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x04\0\0\0\0\0\0\0\0\0\
+\0\0\0\0\0\0\x20\0\0\0\x09\0\0\0\x40\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x98\x3e\0\0\
+\0\0\0\0\x20\x0d\0\0\0\0\0\0\x0c\0\0\0\x09\0\0\0\x08\0\0\0\0\0\0\0\x10\0\0\0\0\
+\0\0\0\x6f\0\0\0\x03\x4c\xff\x6f\0\0\0\x80\0\0\0\0\0\0\0\0\0\0\0\0\xb8\x4b\0\0\
+\0\0\0\0\x05\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x01\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\
+\xac\0\0\0\x02\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xb8\x38\0\0\0\0\0\0\x70\
+\x05\0\0\0\0\0\0\x01\0\0\0\x35\0\0\0\x08\0\0\0\0\0\0\0\x18\0\0\0\0\0\0\0";
 }
 
 #ifdef __cplusplus
diff --git a/meson.build b/meson.build
index XXXXXXX..XXXXXXX 100644
--- a/meson.build
+++ b/meson.build
@@ -XXX,XX +XXX,XX @@ elif get_option('vduse_blk_export').disabled()
 endif
 
 # libbpf
-libbpf = dependency('libbpf', required: get_option('bpf'), method: 'pkg-config')
+bpf_version = '1.1.0'
+libbpf = dependency('libbpf', version: '>=' + bpf_version, required: get_option('bpf'), method: 'pkg-config')
 if libbpf.found() and not cc.links('''
    #include <bpf/libbpf.h>
+   #include <linux/bpf.h>
    int main(void)
    {
+     // check flag availability
+     int flag = BPF_F_MMAPABLE;
      bpf_object__destroy_skeleton(NULL);
      return 0;
    }''', dependencies: libbpf)
   libbpf = not_found
   if get_option('bpf').enabled()
-    error('libbpf skeleton test failed')
+    error('libbpf skeleton/mmaping test failed')
   else
-    warning('libbpf skeleton test failed, disabling')
+    warning('libbpf skeleton/mmaping test failed, disabling')
   endif
 endif
 
diff --git a/tools/ebpf/rss.bpf.c b/tools/ebpf/rss.bpf.c
index XXXXXXX..XXXXXXX 100644
--- a/tools/ebpf/rss.bpf.c
+++ b/tools/ebpf/rss.bpf.c
@@ -XXX,XX +XXX,XX @@ struct {
     __uint(key_size, sizeof(__u32));
     __uint(value_size, sizeof(struct rss_config_t));
     __uint(max_entries, 1);
+    __uint(map_flags, BPF_F_MMAPABLE);
 } tap_rss_map_configurations SEC(".maps");
 
 struct {
@@ -XXX,XX +XXX,XX @@ struct {
     __uint(key_size, sizeof(__u32));
     __uint(value_size, sizeof(struct toeplitz_key_data_t));
     __uint(max_entries, 1);
+    __uint(map_flags, BPF_F_MMAPABLE);
 } tap_rss_map_toeplitz_key SEC(".maps");
 
 struct {
@@ -XXX,XX +XXX,XX @@ struct {
     __uint(key_size, sizeof(__u32));
     __uint(value_size, sizeof(__u16));
     __uint(max_entries, INDIRECTION_TABLE_SIZE);
+    __uint(map_flags, BPF_F_MMAPABLE);
 } tap_rss_map_indirection_table SEC(".maps");
 
 static inline void net_rx_rss_add_chunk(__u8 *rss_input, size_t *bytes_written,
@@ -XXX,XX +XXX,XX @@ static inline __u32 calculate_rss_hash(struct __sk_buff *skb,
     return result;
 }
 
-SEC("tun_rss_steering")
+SEC("socket")
 int tun_rss_steering_prog(struct __sk_buff *skb)
 {
 
-- 
2.7.4

The following changes since commit 352998df1c53b366413690d95b35f76d0721ebed:

Merge tag 'i2c-20220314' of https://github.com/philmd/qemu into staging (2022-03-14 14:39:33 +0000)

are available in the git repository at:

https://github.com/jasowang/qemu.git tags/net-pull-request

for you to fetch changes up to 12a195fa343aae2ead1301ce04727bd0ae25eb15:

vdpa: Expose VHOST_F_LOG_ALL on SVQ (2022-03-15 13:57:44 +0800)

----------------------------------------------------------------

Changes since V2:
- fix 32bit build errros

----------------------------------------------------------------
Eugenio Pérez (14):
      vhost: Add VhostShadowVirtqueue
      vhost: Add Shadow VirtQueue kick forwarding capabilities
      vhost: Add Shadow VirtQueue call forwarding capabilities
      vhost: Add vhost_svq_valid_features to shadow vq
      virtio: Add vhost_svq_get_vring_addr
      vdpa: adapt vhost_ops callbacks to svq
      vhost: Shadow virtqueue buffers forwarding
      util: Add iova_tree_alloc_map
      util: add iova_tree_find_iova
      vhost: Add VhostIOVATree
      vdpa: Add custom IOTLB translations to SVQ
      vdpa: Adapt vhost_vdpa_get_vring_base to SVQ
      vdpa: Never set log_base addr if SVQ is enabled
      vdpa: Expose VHOST_F_LOG_ALL on SVQ

Jason Wang (1):
      virtio-net: fix map leaking on error during receive

hw/net/virtio-net.c                |   1 +
 hw/virtio/meson.build              |   2 +-
 hw/virtio/vhost-iova-tree.c        | 110 +++++++
 hw/virtio/vhost-iova-tree.h        |  27 ++
 hw/virtio/vhost-shadow-virtqueue.c | 636 +++++++++++++++++++++++++++++++++++++
 hw/virtio/vhost-shadow-virtqueue.h |  87 +++++
 hw/virtio/vhost-vdpa.c             | 522 +++++++++++++++++++++++++++++-
 include/hw/virtio/vhost-vdpa.h     |   8 +
 include/qemu/iova-tree.h           |  38 ++-
 util/iova-tree.c                   | 170 ++++++++++
 10 files changed, 1584 insertions(+), 17 deletions(-)
 create mode 100644 hw/virtio/vhost-iova-tree.c
 create mode 100644 hw/virtio/vhost-iova-tree.h
 create mode 100644 hw/virtio/vhost-shadow-virtqueue.c
 create mode 100644 hw/virtio/vhost-shadow-virtqueue.h

Commit bedd7e93d0196 ("virtio-net: fix use after unmap/free for sg")
tries to fix the use after free of the sg by caching the virtqueue
elements in an array and unmap them at once after receiving the
packets, But it forgot to unmap the cached elements on error which
will lead to leaking of mapping and other unexpected results.

Fixing this by detaching the cached elements on error. This addresses
CVE-2022-26353.

Reported-by: Victor Tom <vv474172261@gmail.com>
Cc: qemu-stable@nongnu.org
Fixes: CVE-2022-26353
Fixes: bedd7e93d0196 ("virtio-net: fix use after unmap/free for sg")
Reviewed-by: Michael S. Tsirkin <mst@redhat.com>
Signed-off-by: Jason Wang <jasowang@redhat.com>
---
 hw/net/virtio-net.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/hw/net/virtio-net.c b/hw/net/virtio-net.c
index XXXXXXX..XXXXXXX 100644
--- a/hw/net/virtio-net.c
+++ b/hw/net/virtio-net.c
@@ -XXX,XX +XXX,XX @@ static ssize_t virtio_net_receive_rcu(NetClientState *nc, const uint8_t *buf,
 
 err:
     for (j = 0; j < i; j++) {
+        virtqueue_detach_element(q->rx_vq, elems[j], lens[j]);
         g_free(elems[j]);
     }
 
-- 
2.7.4

From: Eugenio Pérez <eperezma@redhat.com>

Vhost shadow virtqueue (SVQ) is an intermediate jump for virtqueue
notifications and buffers, allowing qemu to track them. While qemu is
forwarding the buffers and virtqueue changes, it is able to commit the
memory it's being dirtied, the same way regular qemu's VirtIO devices
do.

This commit only exposes basic SVQ allocation and free. Next patches of
the series add functionality like notifications and buffers forwarding.

Signed-off-by: Eugenio Pérez <eperezma@redhat.com>
Acked-by: Michael S. Tsirkin <mst@redhat.com>
Signed-off-by: Jason Wang <jasowang@redhat.com>
---
 hw/virtio/meson.build              |  2 +-
 hw/virtio/vhost-shadow-virtqueue.c | 62 ++++++++++++++++++++++++++++++++++++++
 hw/virtio/vhost-shadow-virtqueue.h | 28 +++++++++++++++++
 3 files changed, 91 insertions(+), 1 deletion(-)
 create mode 100644 hw/virtio/vhost-shadow-virtqueue.c
 create mode 100644 hw/virtio/vhost-shadow-virtqueue.h

diff --git a/hw/virtio/meson.build b/hw/virtio/meson.build
index XXXXXXX..XXXXXXX 100644
--- a/hw/virtio/meson.build
+++ b/hw/virtio/meson.build
@@ -XXX,XX +XXX,XX @@ softmmu_ss.add(when: 'CONFIG_ALL', if_true: files('vhost-stub.c'))
 
 virtio_ss = ss.source_set()
 virtio_ss.add(files('virtio.c'))
-virtio_ss.add(when: 'CONFIG_VHOST', if_true: files('vhost.c', 'vhost-backend.c'))
+virtio_ss.add(when: 'CONFIG_VHOST', if_true: files('vhost.c', 'vhost-backend.c', 'vhost-shadow-virtqueue.c'))
 virtio_ss.add(when: 'CONFIG_VHOST_USER', if_true: files('vhost-user.c'))
 virtio_ss.add(when: 'CONFIG_VHOST_VDPA', if_true: files('vhost-vdpa.c'))
 virtio_ss.add(when: 'CONFIG_VIRTIO_BALLOON', if_true: files('virtio-balloon.c'))
diff --git a/hw/virtio/vhost-shadow-virtqueue.c b/hw/virtio/vhost-shadow-virtqueue.c
new file mode 100644
index XXXXXXX..XXXXXXX
--- /dev/null
+++ b/hw/virtio/vhost-shadow-virtqueue.c
@@ -XXX,XX +XXX,XX @@
+/*
+ * vhost shadow virtqueue
+ *
+ * SPDX-FileCopyrightText: Red Hat, Inc. 2021
+ * SPDX-FileContributor: Author: Eugenio Pérez <eperezma@redhat.com>
+ *
+ * SPDX-License-Identifier: GPL-2.0-or-later
+ */
+
+#include "qemu/osdep.h"
+#include "hw/virtio/vhost-shadow-virtqueue.h"
+
+#include "qemu/error-report.h"
+
+/**
+ * Creates vhost shadow virtqueue, and instructs the vhost device to use the
+ * shadow methods and file descriptors.
+ *
+ * Returns the new virtqueue or NULL.
+ *
+ * In case of error, reason is reported through error_report.
+ */
+VhostShadowVirtqueue *vhost_svq_new(void)
+{
+    g_autofree VhostShadowVirtqueue *svq = g_new0(VhostShadowVirtqueue, 1);
+    int r;
+
+    r = event_notifier_init(&svq->hdev_kick, 0);
+    if (r != 0) {
+        error_report("Couldn't create kick event notifier: %s (%d)",
+                     g_strerror(errno), errno);
+        goto err_init_hdev_kick;
+    }
+
+    r = event_notifier_init(&svq->hdev_call, 0);
+    if (r != 0) {
+        error_report("Couldn't create call event notifier: %s (%d)",
+                     g_strerror(errno), errno);
+        goto err_init_hdev_call;
+    }
+
+    return g_steal_pointer(&svq);
+
+err_init_hdev_call:
+    event_notifier_cleanup(&svq->hdev_kick);
+
+err_init_hdev_kick:
+    return NULL;
+}
+
+/**
+ * Free the resources of the shadow virtqueue.
+ *
+ * @pvq: gpointer to SVQ so it can be used by autofree functions.
+ */
+void vhost_svq_free(gpointer pvq)
+{
+    VhostShadowVirtqueue *vq = pvq;
+    event_notifier_cleanup(&vq->hdev_kick);
+    event_notifier_cleanup(&vq->hdev_call);
+    g_free(vq);
+}
diff --git a/hw/virtio/vhost-shadow-virtqueue.h b/hw/virtio/vhost-shadow-virtqueue.h
new file mode 100644
index XXXXXXX..XXXXXXX
--- /dev/null
+++ b/hw/virtio/vhost-shadow-virtqueue.h
@@ -XXX,XX +XXX,XX @@
+/*
+ * vhost shadow virtqueue
+ *
+ * SPDX-FileCopyrightText: Red Hat, Inc. 2021
+ * SPDX-FileContributor: Author: Eugenio Pérez <eperezma@redhat.com>
+ *
+ * SPDX-License-Identifier: GPL-2.0-or-later
+ */
+
+#ifndef VHOST_SHADOW_VIRTQUEUE_H
+#define VHOST_SHADOW_VIRTQUEUE_H
+
+#include "qemu/event_notifier.h"
+
+/* Shadow virtqueue to relay notifications */
+typedef struct VhostShadowVirtqueue {
+    /* Shadow kick notifier, sent to vhost */
+    EventNotifier hdev_kick;
+    /* Shadow call notifier, sent to vhost */
+    EventNotifier hdev_call;
+} VhostShadowVirtqueue;
+
+VhostShadowVirtqueue *vhost_svq_new(void);
+
+void vhost_svq_free(gpointer vq);
+G_DEFINE_AUTOPTR_CLEANUP_FUNC(VhostShadowVirtqueue, vhost_svq_free);
+
+#endif
-- 
2.7.4

From: Eugenio Pérez <eperezma@redhat.com>

At this mode no buffer forwarding will be performed in SVQ mode: Qemu
will just forward the guest's kicks to the device.

Host memory notifiers regions are left out for simplicity, and they will
not be addressed in this series.

Signed-off-by: Eugenio Pérez <eperezma@redhat.com>
Acked-by: Michael S. Tsirkin <mst@redhat.com>
Signed-off-by: Jason Wang <jasowang@redhat.com>
---
 hw/virtio/vhost-shadow-virtqueue.c |  55 ++++++++++++++
 hw/virtio/vhost-shadow-virtqueue.h |  14 ++++
 hw/virtio/vhost-vdpa.c             | 144 ++++++++++++++++++++++++++++++++++++-
 include/hw/virtio/vhost-vdpa.h     |   4 ++
 4 files changed, 215 insertions(+), 2 deletions(-)

From: Eugenio Pérez <eperezma@redhat.com>

This will make qemu aware of the device used buffers, allowing it to
write the guest memory with its contents if needed.

Signed-off-by: Eugenio Pérez <eperezma@redhat.com>
Acked-by: Michael S. Tsirkin <mst@redhat.com>
Signed-off-by: Jason Wang <jasowang@redhat.com>
---
 hw/virtio/vhost-shadow-virtqueue.c | 38 ++++++++++++++++++++++++++++++++++++++
 hw/virtio/vhost-shadow-virtqueue.h |  4 ++++
 hw/virtio/vhost-vdpa.c             | 31 +++++++++++++++++++++++++++++--
 3 files changed, 71 insertions(+), 2 deletions(-)

diff --git a/hw/virtio/vhost-shadow-virtqueue.c b/hw/virtio/vhost-shadow-virtqueue.c
index XXXXXXX..XXXXXXX 100644
--- a/hw/virtio/vhost-shadow-virtqueue.c
+++ b/hw/virtio/vhost-shadow-virtqueue.c
@@ -XXX,XX +XXX,XX @@ static void vhost_handle_guest_kick(EventNotifier *n)
 }
 
 /**
+ * Forward vhost notifications
+ *
+ * @n: hdev call event notifier, the one that device set to notify svq.
+ */
+static void vhost_svq_handle_call(EventNotifier *n)
+{
+    VhostShadowVirtqueue *svq = container_of(n, VhostShadowVirtqueue,
+                                             hdev_call);
+    event_notifier_test_and_clear(n);
+    event_notifier_set(&svq->svq_call);
+}
+
+/**
+ * Set the call notifier for the SVQ to call the guest
+ *
+ * @svq: Shadow virtqueue
+ * @call_fd: call notifier
+ *
+ * Called on BQL context.
+ */
+void vhost_svq_set_svq_call_fd(VhostShadowVirtqueue *svq, int call_fd)
+{
+    if (call_fd == VHOST_FILE_UNBIND) {
+        /*
+         * Fail event_notifier_set if called handling device call.
+         *
+         * SVQ still needs device notifications, since it needs to keep
+         * forwarding used buffers even with the unbind.
+         */
+        memset(&svq->svq_call, 0, sizeof(svq->svq_call));
+    } else {
+        event_notifier_init_fd(&svq->svq_call, call_fd);
+    }
+}
+
+/**
  * Set a new file descriptor for the guest to kick the SVQ and notify for avail
  *
  * @svq: The svq
@@ -XXX,XX +XXX,XX @@ VhostShadowVirtqueue *vhost_svq_new(void)
     }
 
     event_notifier_init_fd(&svq->svq_kick, VHOST_FILE_UNBIND);
+    event_notifier_set_handler(&svq->hdev_call, vhost_svq_handle_call);
     return g_steal_pointer(&svq);
 
 err_init_hdev_call:
@@ -XXX,XX +XXX,XX @@ void vhost_svq_free(gpointer pvq)
     VhostShadowVirtqueue *vq = pvq;
     vhost_svq_stop(vq);
     event_notifier_cleanup(&vq->hdev_kick);
+    event_notifier_set_handler(&vq->hdev_call, NULL);
     event_notifier_cleanup(&vq->hdev_call);
     g_free(vq);
 }
diff --git a/hw/virtio/vhost-shadow-virtqueue.h b/hw/virtio/vhost-shadow-virtqueue.h
index XXXXXXX..XXXXXXX 100644
--- a/hw/virtio/vhost-shadow-virtqueue.h
+++ b/hw/virtio/vhost-shadow-virtqueue.h
@@ -XXX,XX +XXX,XX @@ typedef struct VhostShadowVirtqueue {
      * So shadow virtqueue must not clean it, or we would lose VirtQueue one.
      */
     EventNotifier svq_kick;
+
+    /* Guest's call notifier, where the SVQ calls guest. */
+    EventNotifier svq_call;
 } VhostShadowVirtqueue;
 
 void vhost_svq_set_svq_kick_fd(VhostShadowVirtqueue *svq, int svq_kick_fd);
+void vhost_svq_set_svq_call_fd(VhostShadowVirtqueue *svq, int call_fd);
 
 void vhost_svq_stop(VhostShadowVirtqueue *svq);
 
diff --git a/hw/virtio/vhost-vdpa.c b/hw/virtio/vhost-vdpa.c
index XXXXXXX..XXXXXXX 100644
--- a/hw/virtio/vhost-vdpa.c
+++ b/hw/virtio/vhost-vdpa.c
@@ -XXX,XX +XXX,XX @@ static int vhost_vdpa_set_vring_dev_kick(struct vhost_dev *dev,
     return vhost_vdpa_call(dev, VHOST_SET_VRING_KICK, file);
 }
 
+static int vhost_vdpa_set_vring_dev_call(struct vhost_dev *dev,
+                                         struct vhost_vring_file *file)
+{
+    trace_vhost_vdpa_set_vring_call(dev, file->index, file->fd);
+    return vhost_vdpa_call(dev, VHOST_SET_VRING_CALL, file);
+}
+
 /**
  * Set the shadow virtqueue descriptors to the device
  *
@@ -XXX,XX +XXX,XX @@ static int vhost_vdpa_set_vring_dev_kick(struct vhost_dev *dev,
  * @svq: The shadow virtqueue
  * @idx: The index of the virtqueue in the vhost device
  * @errp: Error
+ *
+ * Note that this function does not rewind kick file descriptor if cannot set
+ * call one.
  */
 static bool vhost_vdpa_svq_setup(struct vhost_dev *dev,
                                  VhostShadowVirtqueue *svq, unsigned idx,
@@ -XXX,XX +XXX,XX @@ static bool vhost_vdpa_svq_setup(struct vhost_dev *dev,
     r = vhost_vdpa_set_vring_dev_kick(dev, &file);
     if (unlikely(r != 0)) {
         error_setg_errno(errp, -r, "Can't set device kick fd");
+        return false;
+    }
+
+    event_notifier = &svq->hdev_call;
+    file.fd = event_notifier_get_fd(event_notifier);
+    r = vhost_vdpa_set_vring_dev_call(dev, &file);
+    if (unlikely(r != 0)) {
+        error_setg_errno(errp, -r, "Can't set device call fd");
     }
 
     return r == 0;
@@ -XXX,XX +XXX,XX @@ static int vhost_vdpa_set_vring_kick(struct vhost_dev *dev,
 static int vhost_vdpa_set_vring_call(struct vhost_dev *dev,
                                        struct vhost_vring_file *file)
 {
-    trace_vhost_vdpa_set_vring_call(dev, file->index, file->fd);
-    return vhost_vdpa_call(dev, VHOST_SET_VRING_CALL, file);
+    struct vhost_vdpa *v = dev->opaque;
+
+    if (v->shadow_vqs_enabled) {
+        int vdpa_idx = file->index - dev->vq_index;
+        VhostShadowVirtqueue *svq = g_ptr_array_index(v->shadow_vqs, vdpa_idx);
+
+        vhost_svq_set_svq_call_fd(svq, file->fd);
+        return 0;
+    } else {
+        return vhost_vdpa_set_vring_dev_call(dev, file);
+    }
 }
 
 static int vhost_vdpa_get_features(struct vhost_dev *dev,
-- 
2.7.4

From: Eugenio Pérez <eperezma@redhat.com>

This allows SVQ to negotiate features with the guest and the device. For
the device, SVQ is a driver. While this function bypasses all
non-transport features, it needs to disable the features that SVQ does
not support when forwarding buffers. This includes packed vq layout,
indirect descriptors or event idx.

Future changes can add support to offer more features to the guest,
since the use of VirtQueue gives this for free. This is left out at the
moment for simplicity.

Signed-off-by: Eugenio Pérez <eperezma@redhat.com>
Acked-by: Michael S. Tsirkin <mst@redhat.com>
Signed-off-by: Jason Wang <jasowang@redhat.com>
---
 hw/virtio/vhost-shadow-virtqueue.c | 44 ++++++++++++++++++++++++++++++++++++++
 hw/virtio/vhost-shadow-virtqueue.h |  2 ++
 hw/virtio/vhost-vdpa.c             | 15 +++++++++++++
 3 files changed, 61 insertions(+)

From: Eugenio Pérez <eperezma@redhat.com>

It reports the shadow virtqueue address from qemu virtual address space.

Since this will be different from the guest's vaddr, but the device can
access it, SVQ takes special care about its alignment & lack of garbage
data. It assumes that IOMMU will work in host_page_size ranges for that.

diff --git a/hw/virtio/vhost-shadow-virtqueue.c b/hw/virtio/vhost-shadow-virtqueue.c
index XXXXXXX..XXXXXXX 100644
--- a/hw/virtio/vhost-shadow-virtqueue.c
+++ b/hw/virtio/vhost-shadow-virtqueue.c
@@ -XXX,XX +XXX,XX @@ void vhost_svq_set_svq_call_fd(VhostShadowVirtqueue *svq, int call_fd)
 }
 
 /**
+ * Get the shadow vq vring address.
+ * @svq: Shadow virtqueue
+ * @addr: Destination to store address
+ */
+void vhost_svq_get_vring_addr(const VhostShadowVirtqueue *svq,
+                              struct vhost_vring_addr *addr)
+{
+    addr->desc_user_addr = (uint64_t)(intptr_t)svq->vring.desc;
+    addr->avail_user_addr = (uint64_t)(intptr_t)svq->vring.avail;
+    addr->used_user_addr = (uint64_t)(intptr_t)svq->vring.used;
+}
+
+size_t vhost_svq_driver_area_size(const VhostShadowVirtqueue *svq)
+{
+    size_t desc_size = sizeof(vring_desc_t) * svq->vring.num;
+    size_t avail_size = offsetof(vring_avail_t, ring) +
+                                             sizeof(uint16_t) * svq->vring.num;
+
+    return ROUND_UP(desc_size + avail_size, qemu_real_host_page_size);
+}
+
+size_t vhost_svq_device_area_size(const VhostShadowVirtqueue *svq)
+{
+    size_t used_size = offsetof(vring_used_t, ring) +
+                                    sizeof(vring_used_elem_t) * svq->vring.num;
+    return ROUND_UP(used_size, qemu_real_host_page_size);
+}
+
+/**
  * Set a new file descriptor for the guest to kick the SVQ and notify for avail
  *
  * @svq: The svq
diff --git a/hw/virtio/vhost-shadow-virtqueue.h b/hw/virtio/vhost-shadow-virtqueue.h
index XXXXXXX..XXXXXXX 100644
--- a/hw/virtio/vhost-shadow-virtqueue.h
+++ b/hw/virtio/vhost-shadow-virtqueue.h
@@ -XXX,XX +XXX,XX @@
 #define VHOST_SHADOW_VIRTQUEUE_H
 
 #include "qemu/event_notifier.h"
+#include "hw/virtio/virtio.h"
+#include "standard-headers/linux/vhost_types.h"
 
 /* Shadow virtqueue to relay notifications */
 typedef struct VhostShadowVirtqueue {
+    /* Shadow vring */
+    struct vring vring;
+
     /* Shadow kick notifier, sent to vhost */
     EventNotifier hdev_kick;
     /* Shadow call notifier, sent to vhost */
@@ -XXX,XX +XXX,XX @@ bool vhost_svq_valid_features(uint64_t features, Error **errp);
 
 void vhost_svq_set_svq_kick_fd(VhostShadowVirtqueue *svq, int svq_kick_fd);
 void vhost_svq_set_svq_call_fd(VhostShadowVirtqueue *svq, int call_fd);
+void vhost_svq_get_vring_addr(const VhostShadowVirtqueue *svq,
+                              struct vhost_vring_addr *addr);
+size_t vhost_svq_driver_area_size(const VhostShadowVirtqueue *svq);
+size_t vhost_svq_device_area_size(const VhostShadowVirtqueue *svq);
 
 void vhost_svq_stop(VhostShadowVirtqueue *svq);
 
-- 
2.7.4

From: Eugenio Pérez <eperezma@redhat.com>

First half of the buffers forwarding part, preparing vhost-vdpa
callbacks to SVQ to offer it. QEMU cannot enable it at this moment, so
this is effectively dead code at the moment, but it helps to reduce
patch size.

diff --git a/hw/virtio/vhost-vdpa.c b/hw/virtio/vhost-vdpa.c
index XXXXXXX..XXXXXXX 100644
--- a/hw/virtio/vhost-vdpa.c
+++ b/hw/virtio/vhost-vdpa.c
@@ -XXX,XX +XXX,XX @@ static int vhost_vdpa_get_config(struct vhost_dev *dev, uint8_t *config,
     return ret;
  }
 
+static int vhost_vdpa_set_dev_vring_base(struct vhost_dev *dev,
+                                         struct vhost_vring_state *ring)
+{
+    trace_vhost_vdpa_set_vring_base(dev, ring->index, ring->num);
+    return vhost_vdpa_call(dev, VHOST_SET_VRING_BASE, ring);
+}
+
 static int vhost_vdpa_set_vring_dev_kick(struct vhost_dev *dev,
                                          struct vhost_vring_file *file)
 {
@@ -XXX,XX +XXX,XX @@ static int vhost_vdpa_set_vring_dev_call(struct vhost_dev *dev,
     return vhost_vdpa_call(dev, VHOST_SET_VRING_CALL, file);
 }
 
+static int vhost_vdpa_set_vring_dev_addr(struct vhost_dev *dev,
+                                         struct vhost_vring_addr *addr)
+{
+    trace_vhost_vdpa_set_vring_addr(dev, addr->index, addr->flags,
+                                addr->desc_user_addr, addr->used_user_addr,
+                                addr->avail_user_addr,
+                                addr->log_guest_addr);
+
+    return vhost_vdpa_call(dev, VHOST_SET_VRING_ADDR, addr);
+
+}
+
 /**
  * Set the shadow virtqueue descriptors to the device
  *
@@ -XXX,XX +XXX,XX @@ static int vhost_vdpa_set_log_base(struct vhost_dev *dev, uint64_t base,
 static int vhost_vdpa_set_vring_addr(struct vhost_dev *dev,
                                        struct vhost_vring_addr *addr)
 {
-    trace_vhost_vdpa_set_vring_addr(dev, addr->index, addr->flags,
-                                    addr->desc_user_addr, addr->used_user_addr,
-                                    addr->avail_user_addr,
-                                    addr->log_guest_addr);
-    return vhost_vdpa_call(dev, VHOST_SET_VRING_ADDR, addr);
+    struct vhost_vdpa *v = dev->opaque;
+
+    if (v->shadow_vqs_enabled) {
+        /*
+         * Device vring addr was set at device start. SVQ base is handled by
+         * VirtQueue code.
+         */
+        return 0;
+    }
+
+    return vhost_vdpa_set_vring_dev_addr(dev, addr);
 }
 
 static int vhost_vdpa_set_vring_num(struct vhost_dev *dev,
@@ -XXX,XX +XXX,XX @@ static int vhost_vdpa_set_vring_num(struct vhost_dev *dev,
 static int vhost_vdpa_set_vring_base(struct vhost_dev *dev,
                                        struct vhost_vring_state *ring)
 {
-    trace_vhost_vdpa_set_vring_base(dev, ring->index, ring->num);
-    return vhost_vdpa_call(dev, VHOST_SET_VRING_BASE, ring);
+    struct vhost_vdpa *v = dev->opaque;
+
+    if (v->shadow_vqs_enabled) {
+        /*
+         * Device vring base was set at device start. SVQ base is handled by
+         * VirtQueue code.
+         */
+        return 0;
+    }
+
+    return vhost_vdpa_set_dev_vring_base(dev, ring);
 }
 
 static int vhost_vdpa_get_vring_base(struct vhost_dev *dev,
-- 
2.7.4

From: Eugenio Pérez <eperezma@redhat.com>

Initial version of shadow virtqueue that actually forward buffers. There
is no iommu support at the moment, and that will be addressed in future
patches of this series. Since all vhost-vdpa devices use forced IOMMU,
this means that SVQ is not usable at this point of the series on any
device.

For simplicity it only supports modern devices, that expects vring
in little endian, with split ring and no event idx or indirect
descriptors. Support for them will not be added in this series.

It reuses the VirtQueue code for the device part. The driver part is
based on Linux's virtio_ring driver, but with stripped functionality
and optimizations so it's easier to review.

However, forwarding buffers have some particular pieces: One of the most
unexpected ones is that a guest's buffer can expand through more than
one descriptor in SVQ. While this is handled gracefully by qemu's
emulated virtio devices, it may cause unexpected SVQ queue full. This
patch also solves it by checking for this condition at both guest's
kicks and device's calls. The code may be more elegant in the future if
SVQ code runs in its own iocontext.

Signed-off-by: Eugenio Pérez <eperezma@redhat.com>
Acked-by: Michael S. Tsirkin <mst@redhat.com>
Signed-off-by: Jason Wang <jasowang@redhat.com>
---
 hw/virtio/vhost-shadow-virtqueue.c | 352 ++++++++++++++++++++++++++++++++++++-
 hw/virtio/vhost-shadow-virtqueue.h |  26 +++
 hw/virtio/vhost-vdpa.c             | 155 +++++++++++++++-
 3 files changed, 522 insertions(+), 11 deletions(-)

diff --git a/hw/virtio/vhost-shadow-virtqueue.c b/hw/virtio/vhost-shadow-virtqueue.c
index XXXXXXX..XXXXXXX 100644
--- a/hw/virtio/vhost-shadow-virtqueue.c
+++ b/hw/virtio/vhost-shadow-virtqueue.c
@@ -XXX,XX +XXX,XX @@
 #include "qemu/error-report.h"
 #include "qapi/error.h"
 #include "qemu/main-loop.h"
+#include "qemu/log.h"
+#include "qemu/memalign.h"
 #include "linux-headers/linux/vhost.h"
 
 /**
@@ -XXX,XX +XXX,XX @@ bool vhost_svq_valid_features(uint64_t features, Error **errp)
 }
 
 /**
- * Forward guest notifications.
+ * Number of descriptors that the SVQ can make available from the guest.
+ *
+ * @svq: The svq
+ */
+static uint16_t vhost_svq_available_slots(const VhostShadowVirtqueue *svq)
+{
+    return svq->vring.num - (svq->shadow_avail_idx - svq->shadow_used_idx);
+}
+
+static void vhost_vring_write_descs(VhostShadowVirtqueue *svq,
+                                    const struct iovec *iovec, size_t num,
+                                    bool more_descs, bool write)
+{
+    uint16_t i = svq->free_head, last = svq->free_head;
+    unsigned n;
+    uint16_t flags = write ? cpu_to_le16(VRING_DESC_F_WRITE) : 0;
+    vring_desc_t *descs = svq->vring.desc;
+
+    if (num == 0) {
+        return;
+    }
+
+    for (n = 0; n < num; n++) {
+        if (more_descs || (n + 1 < num)) {
+            descs[i].flags = flags | cpu_to_le16(VRING_DESC_F_NEXT);
+        } else {
+            descs[i].flags = flags;
+        }
+        descs[i].addr = cpu_to_le64((hwaddr)(intptr_t)iovec[n].iov_base);
+        descs[i].len = cpu_to_le32(iovec[n].iov_len);
+
+        last = i;
+        i = cpu_to_le16(descs[i].next);
+    }
+
+    svq->free_head = le16_to_cpu(descs[last].next);
+}
+
+static bool vhost_svq_add_split(VhostShadowVirtqueue *svq,
+                                VirtQueueElement *elem, unsigned *head)
+{
+    unsigned avail_idx;
+    vring_avail_t *avail = svq->vring.avail;
+
+    *head = svq->free_head;
+
+    /* We need some descriptors here */
+    if (unlikely(!elem->out_num && !elem->in_num)) {
+        qemu_log_mask(LOG_GUEST_ERROR,
+                      "Guest provided element with no descriptors");
+        return false;
+    }
+
+    vhost_vring_write_descs(svq, elem->out_sg, elem->out_num, elem->in_num > 0,
+                            false);
+    vhost_vring_write_descs(svq, elem->in_sg, elem->in_num, false, true);
+
+    /*
+     * Put the entry in the available array (but don't update avail->idx until
+     * they do sync).
+     */
+    avail_idx = svq->shadow_avail_idx & (svq->vring.num - 1);
+    avail->ring[avail_idx] = cpu_to_le16(*head);
+    svq->shadow_avail_idx++;
+
+    /* Update the avail index after write the descriptor */
+    smp_wmb();
+    avail->idx = cpu_to_le16(svq->shadow_avail_idx);
+
+    return true;
+}
+
+static bool vhost_svq_add(VhostShadowVirtqueue *svq, VirtQueueElement *elem)
+{
+    unsigned qemu_head;
+    bool ok = vhost_svq_add_split(svq, elem, &qemu_head);
+    if (unlikely(!ok)) {
+        return false;
+    }
+
+    svq->ring_id_maps[qemu_head] = elem;
+    return true;
+}
+
+static void vhost_svq_kick(VhostShadowVirtqueue *svq)
+{
+    /*
+     * We need to expose the available array entries before checking the used
+     * flags
+     */
+    smp_mb();
+    if (svq->vring.used->flags & VRING_USED_F_NO_NOTIFY) {
+        return;
+    }
+
+    event_notifier_set(&svq->hdev_kick);
+}
+
+/**
+ * Forward available buffers.
+ *
+ * @svq: Shadow VirtQueue
+ *
+ * Note that this function does not guarantee that all guest's available
+ * buffers are available to the device in SVQ avail ring. The guest may have
+ * exposed a GPA / GIOVA contiguous buffer, but it may not be contiguous in
+ * qemu vaddr.
+ *
+ * If that happens, guest's kick notifications will be disabled until the
+ * device uses some buffers.
+ */
+static void vhost_handle_guest_kick(VhostShadowVirtqueue *svq)
+{
+    /* Clear event notifier */
+    event_notifier_test_and_clear(&svq->svq_kick);
+
+    /* Forward to the device as many available buffers as possible */
+    do {
+        virtio_queue_set_notification(svq->vq, false);
+
+        while (true) {
+            VirtQueueElement *elem;
+            bool ok;
+
+            if (svq->next_guest_avail_elem) {
+                elem = g_steal_pointer(&svq->next_guest_avail_elem);
+            } else {
+                elem = virtqueue_pop(svq->vq, sizeof(*elem));
+            }
+
+            if (!elem) {
+                break;
+            }
+
+            if (elem->out_num + elem->in_num > vhost_svq_available_slots(svq)) {
+                /*
+                 * This condition is possible since a contiguous buffer in GPA
+                 * does not imply a contiguous buffer in qemu's VA
+                 * scatter-gather segments. If that happens, the buffer exposed
+                 * to the device needs to be a chain of descriptors at this
+                 * moment.
+                 *
+                 * SVQ cannot hold more available buffers if we are here:
+                 * queue the current guest descriptor and ignore further kicks
+                 * until some elements are used.
+                 */
+                svq->next_guest_avail_elem = elem;
+                return;
+            }
+
+            ok = vhost_svq_add(svq, elem);
+            if (unlikely(!ok)) {
+                /* VQ is broken, just return and ignore any other kicks */
+                return;
+            }
+            vhost_svq_kick(svq);
+        }
+
+        virtio_queue_set_notification(svq->vq, true);
+    } while (!virtio_queue_empty(svq->vq));
+}
+
+/**
+ * Handle guest's kick.
  *
  * @n: guest kick event notifier, the one that guest set to notify svq.
  */
-static void vhost_handle_guest_kick(EventNotifier *n)
+static void vhost_handle_guest_kick_notifier(EventNotifier *n)
 {
     VhostShadowVirtqueue *svq = container_of(n, VhostShadowVirtqueue, svq_kick);
     event_notifier_test_and_clear(n);
-    event_notifier_set(&svq->hdev_kick);
+    vhost_handle_guest_kick(svq);
+}
+
+static bool vhost_svq_more_used(VhostShadowVirtqueue *svq)
+{
+    if (svq->last_used_idx != svq->shadow_used_idx) {
+        return true;
+    }
+
+    svq->shadow_used_idx = cpu_to_le16(svq->vring.used->idx);
+
+    return svq->last_used_idx != svq->shadow_used_idx;
 }
 
 /**
- * Forward vhost notifications
+ * Enable vhost device calls after disable them.
+ *
+ * @svq: The svq
+ *
+ * It returns false if there are pending used buffers from the vhost device,
+ * avoiding the possible races between SVQ checking for more work and enabling
+ * callbacks. True if SVQ used vring has no more pending buffers.
+ */
+static bool vhost_svq_enable_notification(VhostShadowVirtqueue *svq)
+{
+    svq->vring.avail->flags &= ~cpu_to_le16(VRING_AVAIL_F_NO_INTERRUPT);
+    /* Make sure the flag is written before the read of used_idx */
+    smp_mb();
+    return !vhost_svq_more_used(svq);
+}
+
+static void vhost_svq_disable_notification(VhostShadowVirtqueue *svq)
+{
+    svq->vring.avail->flags |= cpu_to_le16(VRING_AVAIL_F_NO_INTERRUPT);
+}
+
+static VirtQueueElement *vhost_svq_get_buf(VhostShadowVirtqueue *svq,
+                                           uint32_t *len)
+{
+    vring_desc_t *descs = svq->vring.desc;
+    const vring_used_t *used = svq->vring.used;
+    vring_used_elem_t used_elem;
+    uint16_t last_used;
+
+    if (!vhost_svq_more_used(svq)) {
+        return NULL;
+    }
+
+    /* Only get used array entries after they have been exposed by dev */
+    smp_rmb();
+    last_used = svq->last_used_idx & (svq->vring.num - 1);
+    used_elem.id = le32_to_cpu(used->ring[last_used].id);
+    used_elem.len = le32_to_cpu(used->ring[last_used].len);
+
+    svq->last_used_idx++;
+    if (unlikely(used_elem.id >= svq->vring.num)) {
+        qemu_log_mask(LOG_GUEST_ERROR, "Device %s says index %u is used",
+                      svq->vdev->name, used_elem.id);
+        return NULL;
+    }
+
+    if (unlikely(!svq->ring_id_maps[used_elem.id])) {
+        qemu_log_mask(LOG_GUEST_ERROR,
+            "Device %s says index %u is used, but it was not available",
+            svq->vdev->name, used_elem.id);
+        return NULL;
+    }
+
+    descs[used_elem.id].next = svq->free_head;
+    svq->free_head = used_elem.id;
+
+    *len = used_elem.len;
+    return g_steal_pointer(&svq->ring_id_maps[used_elem.id]);
+}
+
+static void vhost_svq_flush(VhostShadowVirtqueue *svq,
+                            bool check_for_avail_queue)
+{
+    VirtQueue *vq = svq->vq;
+
+    /* Forward as many used buffers as possible. */
+    do {
+        unsigned i = 0;
+
+        vhost_svq_disable_notification(svq);
+        while (true) {
+            uint32_t len;
+            g_autofree VirtQueueElement *elem = vhost_svq_get_buf(svq, &len);
+            if (!elem) {
+                break;
+            }
+
+            if (unlikely(i >= svq->vring.num)) {
+                qemu_log_mask(LOG_GUEST_ERROR,
+                         "More than %u used buffers obtained in a %u size SVQ",
+                         i, svq->vring.num);
+                virtqueue_fill(vq, elem, len, i);
+                virtqueue_flush(vq, i);
+                return;
+            }
+            virtqueue_fill(vq, elem, len, i++);
+        }
+
+        virtqueue_flush(vq, i);
+        event_notifier_set(&svq->svq_call);
+
+        if (check_for_avail_queue && svq->next_guest_avail_elem) {
+            /*
+             * Avail ring was full when vhost_svq_flush was called, so it's a
+             * good moment to make more descriptors available if possible.
+             */
+            vhost_handle_guest_kick(svq);
+        }
+    } while (!vhost_svq_enable_notification(svq));
+}
+
+/**
+ * Forward used buffers.
  *
  * @n: hdev call event notifier, the one that device set to notify svq.
+ *
+ * Note that we are not making any buffers available in the loop, there is no
+ * way that it runs more than virtqueue size times.
  */
 static void vhost_svq_handle_call(EventNotifier *n)
 {
     VhostShadowVirtqueue *svq = container_of(n, VhostShadowVirtqueue,
                                              hdev_call);
     event_notifier_test_and_clear(n);
-    event_notifier_set(&svq->svq_call);
+    vhost_svq_flush(svq, true);
 }
 
 /**
@@ -XXX,XX +XXX,XX @@ void vhost_svq_set_svq_kick_fd(VhostShadowVirtqueue *svq, int svq_kick_fd)
     if (poll_start) {
         event_notifier_init_fd(svq_kick, svq_kick_fd);
         event_notifier_set(svq_kick);
-        event_notifier_set_handler(svq_kick, vhost_handle_guest_kick);
+        event_notifier_set_handler(svq_kick, vhost_handle_guest_kick_notifier);
+    }
+}
+
+/**
+ * Start the shadow virtqueue operation.
+ *
+ * @svq: Shadow Virtqueue
+ * @vdev: VirtIO device
+ * @vq: Virtqueue to shadow
+ */
+void vhost_svq_start(VhostShadowVirtqueue *svq, VirtIODevice *vdev,
+                     VirtQueue *vq)
+{
+    size_t desc_size, driver_size, device_size;
+
+    svq->next_guest_avail_elem = NULL;
+    svq->shadow_avail_idx = 0;
+    svq->shadow_used_idx = 0;
+    svq->last_used_idx = 0;
+    svq->vdev = vdev;
+    svq->vq = vq;
+
+    svq->vring.num = virtio_queue_get_num(vdev, virtio_get_queue_index(vq));
+    driver_size = vhost_svq_driver_area_size(svq);
+    device_size = vhost_svq_device_area_size(svq);
+    svq->vring.desc = qemu_memalign(qemu_real_host_page_size, driver_size);
+    desc_size = sizeof(vring_desc_t) * svq->vring.num;
+    svq->vring.avail = (void *)((char *)svq->vring.desc + desc_size);
+    memset(svq->vring.desc, 0, driver_size);
+    svq->vring.used = qemu_memalign(qemu_real_host_page_size, device_size);
+    memset(svq->vring.used, 0, device_size);
+    svq->ring_id_maps = g_new0(VirtQueueElement *, svq->vring.num);
+    for (unsigned i = 0; i < svq->vring.num - 1; i++) {
+        svq->vring.desc[i].next = cpu_to_le16(i + 1);
     }
 }
 
@@ -XXX,XX +XXX,XX @@ void vhost_svq_set_svq_kick_fd(VhostShadowVirtqueue *svq, int svq_kick_fd)
 void vhost_svq_stop(VhostShadowVirtqueue *svq)
 {
     event_notifier_set_handler(&svq->svq_kick, NULL);
+    g_autofree VirtQueueElement *next_avail_elem = NULL;
+
+    if (!svq->vq) {
+        return;
+    }
+
+    /* Send all pending used descriptors to guest */
+    vhost_svq_flush(svq, false);
+
+    for (unsigned i = 0; i < svq->vring.num; ++i) {
+        g_autofree VirtQueueElement *elem = NULL;
+        elem = g_steal_pointer(&svq->ring_id_maps[i]);
+        if (elem) {
+            virtqueue_detach_element(svq->vq, elem, 0);
+        }
+    }
+
+    next_avail_elem = g_steal_pointer(&svq->next_guest_avail_elem);
+    if (next_avail_elem) {
+        virtqueue_detach_element(svq->vq, next_avail_elem, 0);
+    }
+    svq->vq = NULL;
+    g_free(svq->ring_id_maps);
+    qemu_vfree(svq->vring.desc);
+    qemu_vfree(svq->vring.used);
 }
 
 /**
diff --git a/hw/virtio/vhost-shadow-virtqueue.h b/hw/virtio/vhost-shadow-virtqueue.h
index XXXXXXX..XXXXXXX 100644
--- a/hw/virtio/vhost-shadow-virtqueue.h
+++ b/hw/virtio/vhost-shadow-virtqueue.h
@@ -XXX,XX +XXX,XX @@ typedef struct VhostShadowVirtqueue {
 
     /* Guest's call notifier, where the SVQ calls guest. */
     EventNotifier svq_call;
+
+    /* Virtio queue shadowing */
+    VirtQueue *vq;
+
+    /* Virtio device */
+    VirtIODevice *vdev;
+
+    /* Map for use the guest's descriptors */
+    VirtQueueElement **ring_id_maps;
+
+    /* Next VirtQueue element that guest made available */
+    VirtQueueElement *next_guest_avail_elem;
+
+    /* Next head to expose to the device */
+    uint16_t shadow_avail_idx;
+
+    /* Next free descriptor */
+    uint16_t free_head;
+
+    /* Last seen used idx */
+    uint16_t shadow_used_idx;
+
+    /* Next head to consume from the device */
+    uint16_t last_used_idx;
 } VhostShadowVirtqueue;
 
 bool vhost_svq_valid_features(uint64_t features, Error **errp);
@@ -XXX,XX +XXX,XX @@ void vhost_svq_get_vring_addr(const VhostShadowVirtqueue *svq,
 size_t vhost_svq_driver_area_size(const VhostShadowVirtqueue *svq);
 size_t vhost_svq_device_area_size(const VhostShadowVirtqueue *svq);
 
+void vhost_svq_start(VhostShadowVirtqueue *svq, VirtIODevice *vdev,
+                     VirtQueue *vq);
 void vhost_svq_stop(VhostShadowVirtqueue *svq);
 
 VhostShadowVirtqueue *vhost_svq_new(void);
diff --git a/hw/virtio/vhost-vdpa.c b/hw/virtio/vhost-vdpa.c
index XXXXXXX..XXXXXXX 100644
--- a/hw/virtio/vhost-vdpa.c
+++ b/hw/virtio/vhost-vdpa.c
@@ -XXX,XX +XXX,XX @@ static int vhost_vdpa_set_vring_dev_addr(struct vhost_dev *dev,
  * Note that this function does not rewind kick file descriptor if cannot set
  * call one.
  */
-static bool vhost_vdpa_svq_setup(struct vhost_dev *dev,
-                                 VhostShadowVirtqueue *svq, unsigned idx,
-                                 Error **errp)
+static int vhost_vdpa_svq_set_fds(struct vhost_dev *dev,
+                                  VhostShadowVirtqueue *svq, unsigned idx,
+                                  Error **errp)
 {
     struct vhost_vring_file file = {
         .index = dev->vq_index + idx,
@@ -XXX,XX +XXX,XX @@ static bool vhost_vdpa_svq_setup(struct vhost_dev *dev,
     r = vhost_vdpa_set_vring_dev_kick(dev, &file);
     if (unlikely(r != 0)) {
         error_setg_errno(errp, -r, "Can't set device kick fd");
-        return false;
+        return r;
     }
 
     event_notifier = &svq->hdev_call;
@@ -XXX,XX +XXX,XX @@ static bool vhost_vdpa_svq_setup(struct vhost_dev *dev,
         error_setg_errno(errp, -r, "Can't set device call fd");
     }
 
+    return r;
+}
+
+/**
+ * Unmap a SVQ area in the device
+ */
+static bool vhost_vdpa_svq_unmap_ring(struct vhost_vdpa *v, hwaddr iova,
+                                      hwaddr size)
+{
+    int r;
+
+    size = ROUND_UP(size, qemu_real_host_page_size);
+    r = vhost_vdpa_dma_unmap(v, iova, size);
+    return r == 0;
+}
+
+static bool vhost_vdpa_svq_unmap_rings(struct vhost_dev *dev,
+                                       const VhostShadowVirtqueue *svq)
+{
+    struct vhost_vdpa *v = dev->opaque;
+    struct vhost_vring_addr svq_addr;
+    size_t device_size = vhost_svq_device_area_size(svq);
+    size_t driver_size = vhost_svq_driver_area_size(svq);
+    bool ok;
+
+    vhost_svq_get_vring_addr(svq, &svq_addr);
+
+    ok = vhost_vdpa_svq_unmap_ring(v, svq_addr.desc_user_addr, driver_size);
+    if (unlikely(!ok)) {
+        return false;
+    }
+
+    return vhost_vdpa_svq_unmap_ring(v, svq_addr.used_user_addr, device_size);
+}
+
+/**
+ * Map the shadow virtqueue rings in the device
+ *
+ * @dev: The vhost device
+ * @svq: The shadow virtqueue
+ * @addr: Assigned IOVA addresses
+ * @errp: Error pointer
+ */
+static bool vhost_vdpa_svq_map_rings(struct vhost_dev *dev,
+                                     const VhostShadowVirtqueue *svq,
+                                     struct vhost_vring_addr *addr,
+                                     Error **errp)
+{
+    struct vhost_vdpa *v = dev->opaque;
+    size_t device_size = vhost_svq_device_area_size(svq);
+    size_t driver_size = vhost_svq_driver_area_size(svq);
+    int r;
+
+    ERRP_GUARD();
+    vhost_svq_get_vring_addr(svq, addr);
+
+    r = vhost_vdpa_dma_map(v, addr->desc_user_addr, driver_size,
+                           (void *)(uintptr_t)addr->desc_user_addr, true);
+    if (unlikely(r != 0)) {
+        error_setg_errno(errp, -r, "Cannot create vq driver region: ");
+        return false;
+    }
+
+    r = vhost_vdpa_dma_map(v, addr->used_user_addr, device_size,
+                           (void *)(intptr_t)addr->used_user_addr, false);
+    if (unlikely(r != 0)) {
+        error_setg_errno(errp, -r, "Cannot create vq device region: ");
+    }
+
+    return r == 0;
+}
+
+static bool vhost_vdpa_svq_setup(struct vhost_dev *dev,
+                                 VhostShadowVirtqueue *svq, unsigned idx,
+                                 Error **errp)
+{
+    uint16_t vq_index = dev->vq_index + idx;
+    struct vhost_vring_state s = {
+        .index = vq_index,
+    };
+    int r;
+
+    r = vhost_vdpa_set_dev_vring_base(dev, &s);
+    if (unlikely(r)) {
+        error_setg_errno(errp, -r, "Cannot set vring base");
+        return false;
+    }
+
+    r = vhost_vdpa_svq_set_fds(dev, svq, idx, errp);
     return r == 0;
 }
 
@@ -XXX,XX +XXX,XX @@ static bool vhost_vdpa_svqs_start(struct vhost_dev *dev)
     }
 
     for (i = 0; i < v->shadow_vqs->len; ++i) {
+        VirtQueue *vq = virtio_get_queue(dev->vdev, dev->vq_index + i);
         VhostShadowVirtqueue *svq = g_ptr_array_index(v->shadow_vqs, i);
+        struct vhost_vring_addr addr = {
+            .index = i,
+        };
+        int r;
         bool ok = vhost_vdpa_svq_setup(dev, svq, i, &err);
         if (unlikely(!ok)) {
-            error_reportf_err(err, "Cannot setup SVQ %u: ", i);
+            goto err;
+        }
+
+        vhost_svq_start(svq, dev->vdev, vq);
+        ok = vhost_vdpa_svq_map_rings(dev, svq, &addr, &err);
+        if (unlikely(!ok)) {
+            goto err_map;
+        }
+
+        /* Override vring GPA set by vhost subsystem */
+        r = vhost_vdpa_set_vring_dev_addr(dev, &addr);
+        if (unlikely(r != 0)) {
+            error_setg_errno(&err, -r, "Cannot set device address");
+            goto err_set_addr;
+        }
+    }
+
+    return true;
+
+err_set_addr:
+    vhost_vdpa_svq_unmap_rings(dev, g_ptr_array_index(v->shadow_vqs, i));
+
+err_map:
+    vhost_svq_stop(g_ptr_array_index(v->shadow_vqs, i));
+
+err:
+    error_reportf_err(err, "Cannot setup SVQ %u: ", i);
+    for (unsigned j = 0; j < i; ++j) {
+        VhostShadowVirtqueue *svq = g_ptr_array_index(v->shadow_vqs, j);
+        vhost_vdpa_svq_unmap_rings(dev, svq);
+        vhost_svq_stop(svq);
+    }
+
+    return false;
+}
+
+static bool vhost_vdpa_svqs_stop(struct vhost_dev *dev)
+{
+    struct vhost_vdpa *v = dev->opaque;
+
+    if (!v->shadow_vqs) {
+        return true;
+    }
+
+    for (unsigned i = 0; i < v->shadow_vqs->len; ++i) {
+        VhostShadowVirtqueue *svq = g_ptr_array_index(v->shadow_vqs, i);
+        bool ok = vhost_vdpa_svq_unmap_rings(dev, svq);
+        if (unlikely(!ok)) {
             return false;
         }
     }
@@ -XXX,XX +XXX,XX @@ static int vhost_vdpa_dev_start(struct vhost_dev *dev, bool started)
         }
         vhost_vdpa_set_vring_ready(dev);
     } else {
+        ok = vhost_vdpa_svqs_stop(dev);
+        if (unlikely(!ok)) {
+            return -1;
+        }
         vhost_vdpa_host_notifiers_uninit(dev, dev->nvqs);
     }
 
-- 
2.7.4

From: Eugenio Pérez <eperezma@redhat.com>

This iova tree function allows it to look for a hole in allocated
regions and return a totally new translation for a given translated
address.

It's usage is mainly to allow devices to access qemu address space,
remapping guest's one into a new iova space where qemu can add chunks of
addresses.

Signed-off-by: Eugenio Pérez <eperezma@redhat.com>
Reviewed-by: Peter Xu <peterx@redhat.com>
Acked-by: Michael S. Tsirkin <mst@redhat.com>
Signed-off-by: Jason Wang <jasowang@redhat.com>
---
 include/qemu/iova-tree.h |  18 +++++++
 util/iova-tree.c         | 136 +++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 154 insertions(+)

diff --git a/include/qemu/iova-tree.h b/include/qemu/iova-tree.h
index XXXXXXX..XXXXXXX 100644
--- a/include/qemu/iova-tree.h
+++ b/include/qemu/iova-tree.h
@@ -XXX,XX +XXX,XX @@
 #define  IOVA_OK           (0)
 #define  IOVA_ERR_INVALID  (-1) /* Invalid parameters */
 #define  IOVA_ERR_OVERLAP  (-2) /* IOVA range overlapped */
+#define  IOVA_ERR_NOMEM    (-3) /* Cannot allocate */
 
 typedef struct IOVATree IOVATree;
 typedef struct DMAMap {
@@ -XXX,XX +XXX,XX @@ const DMAMap *iova_tree_find_address(const IOVATree *tree, hwaddr iova);
 void iova_tree_foreach(IOVATree *tree, iova_tree_iterator iterator);
 
 /**
+ * iova_tree_alloc_map:
+ *
+ * @tree: the iova tree to allocate from
+ * @map: the new map (as translated addr & size) to allocate in the iova region
+ * @iova_begin: the minimum address of the allocation
+ * @iova_end: the maximum addressable direction of the allocation
+ *
+ * Allocates a new region of a given size, between iova_min and iova_max.
+ *
+ * Return: Same as iova_tree_insert, but cannot overlap and can return error if
+ * iova tree is out of free contiguous range. The caller gets the assigned iova
+ * in map->iova.
+ */
+int iova_tree_alloc_map(IOVATree *tree, DMAMap *map, hwaddr iova_begin,
+                        hwaddr iova_end);
+
+/**
  * iova_tree_destroy:
  *
  * @tree: the iova tree to destroy
diff --git a/util/iova-tree.c b/util/iova-tree.c
index XXXXXXX..XXXXXXX 100644
--- a/util/iova-tree.c
+++ b/util/iova-tree.c
@@ -XXX,XX +XXX,XX @@ struct IOVATree {
     GTree *tree;
 };
 
+/* Args to pass to iova_tree_alloc foreach function. */
+struct IOVATreeAllocArgs {
+    /* Size of the desired allocation */
+    size_t new_size;
+
+    /* The minimum address allowed in the allocation */
+    hwaddr iova_begin;
+
+    /* Map at the left of the hole, can be NULL if "this" is first one */
+    const DMAMap *prev;
+
+    /* Map at the right of the hole, can be NULL if "prev" is the last one */
+    const DMAMap *this;
+
+    /* If found, we fill in the IOVA here */
+    hwaddr iova_result;
+
+    /* Whether have we found a valid IOVA */
+    bool iova_found;
+};
+
+/**
+ * Iterate args to the next hole
+ *
+ * @args: The alloc arguments
+ * @next: The next mapping in the tree. Can be NULL to signal the last one
+ */
+static void iova_tree_alloc_args_iterate(struct IOVATreeAllocArgs *args,
+                                         const DMAMap *next)
+{
+    args->prev = args->this;
+    args->this = next;
+}
+
 static int iova_tree_compare(gconstpointer a, gconstpointer b, gpointer data)
 {
     const DMAMap *m1 = a, *m2 = b;
@@ -XXX,XX +XXX,XX @@ int iova_tree_remove(IOVATree *tree, const DMAMap *map)
     return IOVA_OK;
 }
 
+/**
+ * Try to find an unallocated IOVA range between prev and this elements.
+ *
+ * @args: Arguments to allocation
+ *
+ * Cases:
+ *
+ * (1) !prev, !this: No entries allocated, always succeed
+ *
+ * (2) !prev, this: We're iterating at the 1st element.
+ *
+ * (3) prev, !this: We're iterating at the last element.
+ *
+ * (4) prev, this: this is the most common case, we'll try to find a hole
+ * between "prev" and "this" mapping.
+ *
+ * Note that this function assumes the last valid iova is HWADDR_MAX, but it
+ * searches linearly so it's easy to discard the result if it's not the case.
+ */
+static void iova_tree_alloc_map_in_hole(struct IOVATreeAllocArgs *args)
+{
+    const DMAMap *prev = args->prev, *this = args->this;
+    uint64_t hole_start, hole_last;
+
+    if (this && this->iova + this->size < args->iova_begin) {
+        return;
+    }
+
+    hole_start = MAX(prev ? prev->iova + prev->size + 1 : 0, args->iova_begin);
+    hole_last = this ? this->iova : HWADDR_MAX;
+
+    if (hole_last - hole_start > args->new_size) {
+        args->iova_result = hole_start;
+        args->iova_found = true;
+    }
+}
+
+/**
+ * Foreach dma node in the tree, compare if there is a hole with its previous
+ * node (or minimum iova address allowed) and the node.
+ *
+ * @key: Node iterating
+ * @value: Node iterating
+ * @pargs: Struct to communicate with the outside world
+ *
+ * Return: false to keep iterating, true if needs break.
+ */
+static gboolean iova_tree_alloc_traverse(gpointer key, gpointer value,
+                                         gpointer pargs)
+{
+    struct IOVATreeAllocArgs *args = pargs;
+    DMAMap *node = value;
+
+    assert(key == value);
+
+    iova_tree_alloc_args_iterate(args, node);
+    iova_tree_alloc_map_in_hole(args);
+    return args->iova_found;
+}
+
+int iova_tree_alloc_map(IOVATree *tree, DMAMap *map, hwaddr iova_begin,
+                        hwaddr iova_last)
+{
+    struct IOVATreeAllocArgs args = {
+        .new_size = map->size,
+        .iova_begin = iova_begin,
+    };
+
+    if (unlikely(iova_last < iova_begin)) {
+        return IOVA_ERR_INVALID;
+    }
+
+    /*
+     * Find a valid hole for the mapping
+     *
+     * Assuming low iova_begin, so no need to do a binary search to
+     * locate the first node.
+     *
+     * TODO: Replace all this with g_tree_node_first/next/last when available
+     * (from glib since 2.68). To do it with g_tree_foreach complicates the
+     * code a lot.
+     *
+     */
+    g_tree_foreach(tree->tree, iova_tree_alloc_traverse, &args);
+    if (!args.iova_found) {
+        /*
+         * Either tree is empty or the last hole is still not checked.
+         * g_tree_foreach does not compare (last, iova_last] range, so we check
+         * it here.
+         */
+        iova_tree_alloc_args_iterate(&args, NULL);
+        iova_tree_alloc_map_in_hole(&args);
+    }
+
+    if (!args.iova_found || args.iova_result + map->size > iova_last) {
+        return IOVA_ERR_NOMEM;
+    }
+
+    map->iova = args.iova_result;
+    return iova_tree_insert(tree, map);
+}
+
 void iova_tree_destroy(IOVATree *tree)
 {
     g_tree_destroy(tree->tree);
-- 
2.7.4

From: Eugenio Pérez <eperezma@redhat.com>

This function does the reverse operation of iova_tree_find: To look for
a mapping that match a translated address so we can do the reverse.

This have linear complexity instead of logarithmic, but it supports
overlapping HVA. Future developments could reduce it.

Signed-off-by: Eugenio Pérez <eperezma@redhat.com>
Acked-by: Michael S. Tsirkin <mst@redhat.com>
Signed-off-by: Jason Wang <jasowang@redhat.com>
---
 include/qemu/iova-tree.h | 20 +++++++++++++++++++-
 util/iova-tree.c         | 34 ++++++++++++++++++++++++++++++++++
 2 files changed, 53 insertions(+), 1 deletion(-)

diff --git a/include/qemu/iova-tree.h b/include/qemu/iova-tree.h
index XXXXXXX..XXXXXXX 100644
--- a/include/qemu/iova-tree.h
+++ b/include/qemu/iova-tree.h
@@ -XXX,XX +XXX,XX @@ int iova_tree_remove(IOVATree *tree, const DMAMap *map);
  * @tree: the iova tree to search from
  * @map: the mapping to search
  *
- * Search for a mapping in the iova tree that overlaps with the
+ * Search for a mapping in the iova tree that iova overlaps with the
  * mapping range specified.  Only the first found mapping will be
  * returned.
  *
@@ -XXX,XX +XXX,XX @@ int iova_tree_remove(IOVATree *tree, const DMAMap *map);
 const DMAMap *iova_tree_find(const IOVATree *tree, const DMAMap *map);
 
 /**
+ * iova_tree_find_iova:
+ *
+ * @tree: the iova tree to search from
+ * @map: the mapping to search
+ *
+ * Search for a mapping in the iova tree that translated_addr overlaps with the
+ * mapping range specified.  Only the first found mapping will be
+ * returned.
+ *
+ * Return: DMAMap pointer if found, or NULL if not found.  Note that
+ * the returned DMAMap pointer is maintained internally.  User should
+ * only read the content but never modify or free the content.  Also,
+ * user is responsible to make sure the pointer is valid (say, no
+ * concurrent deletion in progress).
+ */
+const DMAMap *iova_tree_find_iova(const IOVATree *tree, const DMAMap *map);
+
+/**
  * iova_tree_find_address:
  *
  * @tree: the iova tree to search from
diff --git a/util/iova-tree.c b/util/iova-tree.c
index XXXXXXX..XXXXXXX 100644
--- a/util/iova-tree.c
+++ b/util/iova-tree.c
@@ -XXX,XX +XXX,XX @@ struct IOVATreeAllocArgs {
     bool iova_found;
 };
 
+typedef struct IOVATreeFindIOVAArgs {
+    const DMAMap *needle;
+    const DMAMap *result;
+} IOVATreeFindIOVAArgs;
+
 /**
  * Iterate args to the next hole
  *
@@ -XXX,XX +XXX,XX @@ const DMAMap *iova_tree_find(const IOVATree *tree, const DMAMap *map)
     return g_tree_lookup(tree->tree, map);
 }
 
+static gboolean iova_tree_find_address_iterator(gpointer key, gpointer value,
+                                                gpointer data)
+{
+    const DMAMap *map = key;
+    IOVATreeFindIOVAArgs *args = data;
+    const DMAMap *needle;
+
+    g_assert(key == value);
+
+    needle = args->needle;
+    if (map->translated_addr + map->size < needle->translated_addr ||
+        needle->translated_addr + needle->size < map->translated_addr) {
+        return false;
+    }
+
+    args->result = map;
+    return true;
+}
+
+const DMAMap *iova_tree_find_iova(const IOVATree *tree, const DMAMap *map)
+{
+    IOVATreeFindIOVAArgs args = {
+        .needle = map,
+    };
+
+    g_tree_foreach(tree->tree, iova_tree_find_address_iterator, &args);
+    return args.result;
+}
+
 const DMAMap *iova_tree_find_address(const IOVATree *tree, hwaddr iova)
 {
     const DMAMap map = { .iova = iova, .size = 0 };
-- 
2.7.4

From: Eugenio Pérez <eperezma@redhat.com>

This tree is able to look for a translated address from an IOVA address.

At first glance it is similar to util/iova-tree. However, SVQ working on
devices with limited IOVA space need more capabilities, like allocating
IOVA chunks or performing reverse translations (qemu addresses to iova).

The allocation capability, as "assign a free IOVA address to this chunk
of memory in qemu's address space" allows shadow virtqueue to create a
new address space that is not restricted by guest's addressable one, so
we can allocate shadow vqs vrings outside of it.

It duplicates the tree so it can search efficiently in both directions,
and it will signal overlap if iova or the translated address is present
in any tree.

Signed-off-by: Eugenio Pérez <eperezma@redhat.com>
Acked-by: Michael S. Tsirkin <mst@redhat.com>
Signed-off-by: Jason Wang <jasowang@redhat.com>
---
 hw/virtio/meson.build       |   2 +-
 hw/virtio/vhost-iova-tree.c | 110 ++++++++++++++++++++++++++++++++++++++++++++
 hw/virtio/vhost-iova-tree.h |  27 +++++++++++
 3 files changed, 138 insertions(+), 1 deletion(-)
 create mode 100644 hw/virtio/vhost-iova-tree.c
 create mode 100644 hw/virtio/vhost-iova-tree.h

diff --git a/hw/virtio/meson.build b/hw/virtio/meson.build
index XXXXXXX..XXXXXXX 100644
--- a/hw/virtio/meson.build
+++ b/hw/virtio/meson.build
@@ -XXX,XX +XXX,XX @@ softmmu_ss.add(when: 'CONFIG_ALL', if_true: files('vhost-stub.c'))
 
 virtio_ss = ss.source_set()
 virtio_ss.add(files('virtio.c'))
-virtio_ss.add(when: 'CONFIG_VHOST', if_true: files('vhost.c', 'vhost-backend.c', 'vhost-shadow-virtqueue.c'))
+virtio_ss.add(when: 'CONFIG_VHOST', if_true: files('vhost.c', 'vhost-backend.c', 'vhost-shadow-virtqueue.c', 'vhost-iova-tree.c'))
 virtio_ss.add(when: 'CONFIG_VHOST_USER', if_true: files('vhost-user.c'))
 virtio_ss.add(when: 'CONFIG_VHOST_VDPA', if_true: files('vhost-vdpa.c'))
 virtio_ss.add(when: 'CONFIG_VIRTIO_BALLOON', if_true: files('virtio-balloon.c'))
diff --git a/hw/virtio/vhost-iova-tree.c b/hw/virtio/vhost-iova-tree.c
new file mode 100644
index XXXXXXX..XXXXXXX
--- /dev/null
+++ b/hw/virtio/vhost-iova-tree.c
@@ -XXX,XX +XXX,XX @@
+/*
+ * vhost software live migration iova tree
+ *
+ * SPDX-FileCopyrightText: Red Hat, Inc. 2021
+ * SPDX-FileContributor: Author: Eugenio Pérez <eperezma@redhat.com>
+ *
+ * SPDX-License-Identifier: GPL-2.0-or-later
+ */
+
+#include "qemu/osdep.h"
+#include "qemu/iova-tree.h"
+#include "vhost-iova-tree.h"
+
+#define iova_min_addr qemu_real_host_page_size
+
+/**
+ * VhostIOVATree, able to:
+ * - Translate iova address
+ * - Reverse translate iova address (from translated to iova)
+ * - Allocate IOVA regions for translated range (linear operation)
+ */
+struct VhostIOVATree {
+    /* First addressable iova address in the device */
+    uint64_t iova_first;
+
+    /* Last addressable iova address in the device */
+    uint64_t iova_last;
+
+    /* IOVA address to qemu memory maps. */
+    IOVATree *iova_taddr_map;
+};
+
+/**
+ * Create a new IOVA tree
+ *
+ * Returns the new IOVA tree
+ */
+VhostIOVATree *vhost_iova_tree_new(hwaddr iova_first, hwaddr iova_last)
+{
+    VhostIOVATree *tree = g_new(VhostIOVATree, 1);
+
+    /* Some devices do not like 0 addresses */
+    tree->iova_first = MAX(iova_first, iova_min_addr);
+    tree->iova_last = iova_last;
+
+    tree->iova_taddr_map = iova_tree_new();
+    return tree;
+}
+
+/**
+ * Delete an iova tree
+ */
+void vhost_iova_tree_delete(VhostIOVATree *iova_tree)
+{
+    iova_tree_destroy(iova_tree->iova_taddr_map);
+    g_free(iova_tree);
+}
+
+/**
+ * Find the IOVA address stored from a memory address
+ *
+ * @tree: The iova tree
+ * @map: The map with the memory address
+ *
+ * Return the stored mapping, or NULL if not found.
+ */
+const DMAMap *vhost_iova_tree_find_iova(const VhostIOVATree *tree,
+                                        const DMAMap *map)
+{
+    return iova_tree_find_iova(tree->iova_taddr_map, map);
+}
+
+/**
+ * Allocate a new mapping
+ *
+ * @tree: The iova tree
+ * @map: The iova map
+ *
+ * Returns:
+ * - IOVA_OK if the map fits in the container
+ * - IOVA_ERR_INVALID if the map does not make sense (like size overflow)
+ * - IOVA_ERR_NOMEM if tree cannot allocate more space.
+ *
+ * It returns assignated iova in map->iova if return value is VHOST_DMA_MAP_OK.
+ */
+int vhost_iova_tree_map_alloc(VhostIOVATree *tree, DMAMap *map)
+{
+    /* Some vhost devices do not like addr 0. Skip first page */
+    hwaddr iova_first = tree->iova_first ?: qemu_real_host_page_size;
+
+    if (map->translated_addr + map->size < map->translated_addr ||
+        map->perm == IOMMU_NONE) {
+        return IOVA_ERR_INVALID;
+    }
+
+    /* Allocate a node in IOVA address */
+    return iova_tree_alloc_map(tree->iova_taddr_map, map, iova_first,
+                               tree->iova_last);
+}
+
+/**
+ * Remove existing mappings from iova tree
+ *
+ * @iova_tree: The vhost iova tree
+ * @map: The map to remove
+ */
+void vhost_iova_tree_remove(VhostIOVATree *iova_tree, const DMAMap *map)
+{
+    iova_tree_remove(iova_tree->iova_taddr_map, map);
+}
diff --git a/hw/virtio/vhost-iova-tree.h b/hw/virtio/vhost-iova-tree.h
new file mode 100644
index XXXXXXX..XXXXXXX
--- /dev/null
+++ b/hw/virtio/vhost-iova-tree.h
@@ -XXX,XX +XXX,XX @@
+/*
+ * vhost software live migration iova tree
+ *
+ * SPDX-FileCopyrightText: Red Hat, Inc. 2021
+ * SPDX-FileContributor: Author: Eugenio Pérez <eperezma@redhat.com>
+ *
+ * SPDX-License-Identifier: GPL-2.0-or-later
+ */
+
+#ifndef HW_VIRTIO_VHOST_IOVA_TREE_H
+#define HW_VIRTIO_VHOST_IOVA_TREE_H
+
+#include "qemu/iova-tree.h"
+#include "exec/memory.h"
+
+typedef struct VhostIOVATree VhostIOVATree;
+
+VhostIOVATree *vhost_iova_tree_new(uint64_t iova_first, uint64_t iova_last);
+void vhost_iova_tree_delete(VhostIOVATree *iova_tree);
+G_DEFINE_AUTOPTR_CLEANUP_FUNC(VhostIOVATree, vhost_iova_tree_delete);
+
+const DMAMap *vhost_iova_tree_find_iova(const VhostIOVATree *iova_tree,
+                                        const DMAMap *map);
+int vhost_iova_tree_map_alloc(VhostIOVATree *iova_tree, DMAMap *map);
+void vhost_iova_tree_remove(VhostIOVATree *iova_tree, const DMAMap *map);
+
+#endif
-- 
2.7.4

From: Eugenio Pérez <eperezma@redhat.com>

Use translations added in VhostIOVATree in SVQ.

Only introduce usage here, not allocation and deallocation. As with
previous patches, we use the dead code paths of shadow_vqs_enabled to
avoid commiting too many changes at once. These are impossible to take
at the moment.

Signed-off-by: Eugenio Pérez <eperezma@redhat.com>
Acked-by: Michael S. Tsirkin <mst@redhat.com>
Signed-off-by: Jason Wang <jasowang@redhat.com>
---
 hw/virtio/vhost-shadow-virtqueue.c |  86 +++++++++++++++++++++++---
 hw/virtio/vhost-shadow-virtqueue.h |   6 +-
 hw/virtio/vhost-vdpa.c             | 122 +++++++++++++++++++++++++++++++------
 include/hw/virtio/vhost-vdpa.h     |   3 +
 4 files changed, 187 insertions(+), 30 deletions(-)

diff --git a/hw/virtio/vhost-shadow-virtqueue.c b/hw/virtio/vhost-shadow-virtqueue.c
index XXXXXXX..XXXXXXX 100644
--- a/hw/virtio/vhost-shadow-virtqueue.c
+++ b/hw/virtio/vhost-shadow-virtqueue.c
@@ -XXX,XX +XXX,XX @@ static uint16_t vhost_svq_available_slots(const VhostShadowVirtqueue *svq)
     return svq->vring.num - (svq->shadow_avail_idx - svq->shadow_used_idx);
 }
 
-static void vhost_vring_write_descs(VhostShadowVirtqueue *svq,
+/**
+ * Translate addresses between the qemu's virtual address and the SVQ IOVA
+ *
+ * @svq: Shadow VirtQueue
+ * @vaddr: Translated IOVA addresses
+ * @iovec: Source qemu's VA addresses
+ * @num: Length of iovec and minimum length of vaddr
+ */
+static bool vhost_svq_translate_addr(const VhostShadowVirtqueue *svq,
+                                     hwaddr *addrs, const struct iovec *iovec,
+                                     size_t num)
+{
+    if (num == 0) {
+        return true;
+    }
+
+    for (size_t i = 0; i < num; ++i) {
+        DMAMap needle = {
+            .translated_addr = (hwaddr)(uintptr_t)iovec[i].iov_base,
+            .size = iovec[i].iov_len,
+        };
+        Int128 needle_last, map_last;
+        size_t off;
+
+        const DMAMap *map = vhost_iova_tree_find_iova(svq->iova_tree, &needle);
+        /*
+         * Map cannot be NULL since iova map contains all guest space and
+         * qemu already has a physical address mapped
+         */
+        if (unlikely(!map)) {
+            qemu_log_mask(LOG_GUEST_ERROR,
+                          "Invalid address 0x%"HWADDR_PRIx" given by guest",
+                          needle.translated_addr);
+            return false;
+        }
+
+        off = needle.translated_addr - map->translated_addr;
+        addrs[i] = map->iova + off;
+
+        needle_last = int128_add(int128_make64(needle.translated_addr),
+                                 int128_make64(iovec[i].iov_len));
+        map_last = int128_make64(map->translated_addr + map->size);
+        if (unlikely(int128_gt(needle_last, map_last))) {
+            qemu_log_mask(LOG_GUEST_ERROR,
+                          "Guest buffer expands over iova range");
+            return false;
+        }
+    }
+
+    return true;
+}
+
+static void vhost_vring_write_descs(VhostShadowVirtqueue *svq, hwaddr *sg,
                                     const struct iovec *iovec, size_t num,
                                     bool more_descs, bool write)
 {
@@ -XXX,XX +XXX,XX @@ static void vhost_vring_write_descs(VhostShadowVirtqueue *svq,
         } else {
             descs[i].flags = flags;
         }
-        descs[i].addr = cpu_to_le64((hwaddr)(intptr_t)iovec[n].iov_base);
+        descs[i].addr = cpu_to_le64(sg[n]);
         descs[i].len = cpu_to_le32(iovec[n].iov_len);
 
         last = i;
@@ -XXX,XX +XXX,XX @@ static bool vhost_svq_add_split(VhostShadowVirtqueue *svq,
 {
     unsigned avail_idx;
     vring_avail_t *avail = svq->vring.avail;
+    bool ok;
+    g_autofree hwaddr *sgs = g_new(hwaddr, MAX(elem->out_num, elem->in_num));
 
     *head = svq->free_head;
 
@@ -XXX,XX +XXX,XX @@ static bool vhost_svq_add_split(VhostShadowVirtqueue *svq,
         return false;
     }
 
-    vhost_vring_write_descs(svq, elem->out_sg, elem->out_num, elem->in_num > 0,
-                            false);
-    vhost_vring_write_descs(svq, elem->in_sg, elem->in_num, false, true);
+    ok = vhost_svq_translate_addr(svq, sgs, elem->out_sg, elem->out_num);
+    if (unlikely(!ok)) {
+        return false;
+    }
+    vhost_vring_write_descs(svq, sgs, elem->out_sg, elem->out_num,
+                            elem->in_num > 0, false);
+
+
+    ok = vhost_svq_translate_addr(svq, sgs, elem->in_sg, elem->in_num);
+    if (unlikely(!ok)) {
+        return false;
+    }
+
+    vhost_vring_write_descs(svq, sgs, elem->in_sg, elem->in_num, false, true);
 
     /*
      * Put the entry in the available array (but don't update avail->idx until
@@ -XXX,XX +XXX,XX @@ void vhost_svq_set_svq_call_fd(VhostShadowVirtqueue *svq, int call_fd)
 void vhost_svq_get_vring_addr(const VhostShadowVirtqueue *svq,
                               struct vhost_vring_addr *addr)
 {
-    addr->desc_user_addr = (uint64_t)(intptr_t)svq->vring.desc;
-    addr->avail_user_addr = (uint64_t)(intptr_t)svq->vring.avail;
-    addr->used_user_addr = (uint64_t)(intptr_t)svq->vring.used;
+    addr->desc_user_addr = (uint64_t)(uintptr_t)svq->vring.desc;
+    addr->avail_user_addr = (uint64_t)(uintptr_t)svq->vring.avail;
+    addr->used_user_addr = (uint64_t)(uintptr_t)svq->vring.used;
 }
 
 size_t vhost_svq_driver_area_size(const VhostShadowVirtqueue *svq)
@@ -XXX,XX +XXX,XX @@ void vhost_svq_stop(VhostShadowVirtqueue *svq)
  * Creates vhost shadow virtqueue, and instructs the vhost device to use the
  * shadow methods and file descriptors.
  *
+ * @iova_tree: Tree to perform descriptors translations
+ *
  * Returns the new virtqueue or NULL.
  *
  * In case of error, reason is reported through error_report.
  */
-VhostShadowVirtqueue *vhost_svq_new(void)
+VhostShadowVirtqueue *vhost_svq_new(VhostIOVATree *iova_tree)
 {
     g_autofree VhostShadowVirtqueue *svq = g_new0(VhostShadowVirtqueue, 1);
     int r;
@@ -XXX,XX +XXX,XX @@ VhostShadowVirtqueue *vhost_svq_new(void)
 
     event_notifier_init_fd(&svq->svq_kick, VHOST_FILE_UNBIND);
     event_notifier_set_handler(&svq->hdev_call, vhost_svq_handle_call);
+    svq->iova_tree = iova_tree;
     return g_steal_pointer(&svq);
 
 err_init_hdev_call:
diff --git a/hw/virtio/vhost-shadow-virtqueue.h b/hw/virtio/vhost-shadow-virtqueue.h
index XXXXXXX..XXXXXXX 100644
--- a/hw/virtio/vhost-shadow-virtqueue.h
+++ b/hw/virtio/vhost-shadow-virtqueue.h
@@ -XXX,XX +XXX,XX @@
 #include "qemu/event_notifier.h"
 #include "hw/virtio/virtio.h"
 #include "standard-headers/linux/vhost_types.h"
+#include "hw/virtio/vhost-iova-tree.h"
 
 /* Shadow virtqueue to relay notifications */
 typedef struct VhostShadowVirtqueue {
@@ -XXX,XX +XXX,XX @@ typedef struct VhostShadowVirtqueue {
     /* Virtio device */
     VirtIODevice *vdev;
 
+    /* IOVA mapping */
+    VhostIOVATree *iova_tree;
+
     /* Map for use the guest's descriptors */
     VirtQueueElement **ring_id_maps;
 
@@ -XXX,XX +XXX,XX @@ void vhost_svq_start(VhostShadowVirtqueue *svq, VirtIODevice *vdev,
                      VirtQueue *vq);
 void vhost_svq_stop(VhostShadowVirtqueue *svq);
 
-VhostShadowVirtqueue *vhost_svq_new(void);
+VhostShadowVirtqueue *vhost_svq_new(VhostIOVATree *iova_tree);
 
 void vhost_svq_free(gpointer vq);
 G_DEFINE_AUTOPTR_CLEANUP_FUNC(VhostShadowVirtqueue, vhost_svq_free);
diff --git a/hw/virtio/vhost-vdpa.c b/hw/virtio/vhost-vdpa.c
index XXXXXXX..XXXXXXX 100644
--- a/hw/virtio/vhost-vdpa.c
+++ b/hw/virtio/vhost-vdpa.c
@@ -XXX,XX +XXX,XX @@ static void vhost_vdpa_listener_region_add(MemoryListener *listener,
                                          vaddr, section->readonly);
 
     llsize = int128_sub(llend, int128_make64(iova));
+    if (v->shadow_vqs_enabled) {
+        DMAMap mem_region = {
+            .translated_addr = (hwaddr)(uintptr_t)vaddr,
+            .size = int128_get64(llsize) - 1,
+            .perm = IOMMU_ACCESS_FLAG(true, section->readonly),
+        };
+
+        int r = vhost_iova_tree_map_alloc(v->iova_tree, &mem_region);
+        if (unlikely(r != IOVA_OK)) {
+            error_report("Can't allocate a mapping (%d)", r);
+            goto fail;
+        }
+
+        iova = mem_region.iova;
+    }
 
     vhost_vdpa_iotlb_batch_begin_once(v);
     ret = vhost_vdpa_dma_map(v, iova, int128_get64(llsize),
@@ -XXX,XX +XXX,XX @@ static void vhost_vdpa_listener_region_del(MemoryListener *listener,
 
     llsize = int128_sub(llend, int128_make64(iova));
 
+    if (v->shadow_vqs_enabled) {
+        const DMAMap *result;
+        const void *vaddr = memory_region_get_ram_ptr(section->mr) +
+            section->offset_within_region +
+            (iova - section->offset_within_address_space);
+        DMAMap mem_region = {
+            .translated_addr = (hwaddr)(uintptr_t)vaddr,
+            .size = int128_get64(llsize) - 1,
+        };
+
+        result = vhost_iova_tree_find_iova(v->iova_tree, &mem_region);
+        iova = result->iova;
+        vhost_iova_tree_remove(v->iova_tree, &mem_region);
+    }
     vhost_vdpa_iotlb_batch_begin_once(v);
     ret = vhost_vdpa_dma_unmap(v, iova, int128_get64(llsize));
     if (ret) {
@@ -XXX,XX +XXX,XX @@ static int vhost_vdpa_init_svq(struct vhost_dev *hdev, struct vhost_vdpa *v,
 
     shadow_vqs = g_ptr_array_new_full(hdev->nvqs, vhost_svq_free);
     for (unsigned n = 0; n < hdev->nvqs; ++n) {
-        g_autoptr(VhostShadowVirtqueue) svq = vhost_svq_new();
+        g_autoptr(VhostShadowVirtqueue) svq = vhost_svq_new(v->iova_tree);
 
         if (unlikely(!svq)) {
             error_setg(errp, "Cannot create svq %u", n);
@@ -XXX,XX +XXX,XX @@ static int vhost_vdpa_svq_set_fds(struct vhost_dev *dev,
 /**
  * Unmap a SVQ area in the device
  */
-static bool vhost_vdpa_svq_unmap_ring(struct vhost_vdpa *v, hwaddr iova,
-                                      hwaddr size)
+static bool vhost_vdpa_svq_unmap_ring(struct vhost_vdpa *v,
+                                      const DMAMap *needle)
 {
+    const DMAMap *result = vhost_iova_tree_find_iova(v->iova_tree, needle);
+    hwaddr size;
     int r;
 
-    size = ROUND_UP(size, qemu_real_host_page_size);
-    r = vhost_vdpa_dma_unmap(v, iova, size);
+    if (unlikely(!result)) {
+        error_report("Unable to find SVQ address to unmap");
+        return false;
+    }
+
+    size = ROUND_UP(result->size, qemu_real_host_page_size);
+    r = vhost_vdpa_dma_unmap(v, result->iova, size);
     return r == 0;
 }
 
 static bool vhost_vdpa_svq_unmap_rings(struct vhost_dev *dev,
                                        const VhostShadowVirtqueue *svq)
 {
+    DMAMap needle = {};
     struct vhost_vdpa *v = dev->opaque;
     struct vhost_vring_addr svq_addr;
-    size_t device_size = vhost_svq_device_area_size(svq);
-    size_t driver_size = vhost_svq_driver_area_size(svq);
     bool ok;
 
     vhost_svq_get_vring_addr(svq, &svq_addr);
 
-    ok = vhost_vdpa_svq_unmap_ring(v, svq_addr.desc_user_addr, driver_size);
+    needle.translated_addr = svq_addr.desc_user_addr;
+    ok = vhost_vdpa_svq_unmap_ring(v, &needle);
     if (unlikely(!ok)) {
         return false;
     }
 
-    return vhost_vdpa_svq_unmap_ring(v, svq_addr.used_user_addr, device_size);
+    needle.translated_addr = svq_addr.used_user_addr;
+    return vhost_vdpa_svq_unmap_ring(v, &needle);
+}
+
+/**
+ * Map the SVQ area in the device
+ *
+ * @v: Vhost-vdpa device
+ * @needle: The area to search iova
+ * @errorp: Error pointer
+ */
+static bool vhost_vdpa_svq_map_ring(struct vhost_vdpa *v, DMAMap *needle,
+                                    Error **errp)
+{
+    int r;
+
+    r = vhost_iova_tree_map_alloc(v->iova_tree, needle);
+    if (unlikely(r != IOVA_OK)) {
+        error_setg(errp, "Cannot allocate iova (%d)", r);
+        return false;
+    }
+
+    r = vhost_vdpa_dma_map(v, needle->iova, needle->size + 1,
+                           (void *)(uintptr_t)needle->translated_addr,
+                           needle->perm == IOMMU_RO);
+    if (unlikely(r != 0)) {
+        error_setg_errno(errp, -r, "Cannot map region to device");
+        vhost_iova_tree_remove(v->iova_tree, needle);
+    }
+
+    return r == 0;
 }
 
 /**
@@ -XXX,XX +XXX,XX @@ static bool vhost_vdpa_svq_map_rings(struct vhost_dev *dev,
                                      struct vhost_vring_addr *addr,
                                      Error **errp)
 {
+    DMAMap device_region, driver_region;
+    struct vhost_vring_addr svq_addr;
     struct vhost_vdpa *v = dev->opaque;
     size_t device_size = vhost_svq_device_area_size(svq);
     size_t driver_size = vhost_svq_driver_area_size(svq);
-    int r;
+    size_t avail_offset;
+    bool ok;
 
     ERRP_GUARD();
-    vhost_svq_get_vring_addr(svq, addr);
+    vhost_svq_get_vring_addr(svq, &svq_addr);
 
-    r = vhost_vdpa_dma_map(v, addr->desc_user_addr, driver_size,
-                           (void *)(uintptr_t)addr->desc_user_addr, true);
-    if (unlikely(r != 0)) {
-        error_setg_errno(errp, -r, "Cannot create vq driver region: ");
+    driver_region = (DMAMap) {
+        .translated_addr = svq_addr.desc_user_addr,
+        .size = driver_size - 1,
+        .perm = IOMMU_RO,
+    };
+    ok = vhost_vdpa_svq_map_ring(v, &driver_region, errp);
+    if (unlikely(!ok)) {
+        error_prepend(errp, "Cannot create vq driver region: ");
         return false;
     }
+    addr->desc_user_addr = driver_region.iova;
+    avail_offset = svq_addr.avail_user_addr - svq_addr.desc_user_addr;
+    addr->avail_user_addr = driver_region.iova + avail_offset;
 
-    r = vhost_vdpa_dma_map(v, addr->used_user_addr, device_size,
-                           (void *)(intptr_t)addr->used_user_addr, false);
-    if (unlikely(r != 0)) {
-        error_setg_errno(errp, -r, "Cannot create vq device region: ");
+    device_region = (DMAMap) {
+        .translated_addr = svq_addr.used_user_addr,
+        .size = device_size - 1,
+        .perm = IOMMU_RW,
+    };
+    ok = vhost_vdpa_svq_map_ring(v, &device_region, errp);
+    if (unlikely(!ok)) {
+        error_prepend(errp, "Cannot create vq device region: ");
+        vhost_vdpa_svq_unmap_ring(v, &driver_region);
     }
+    addr->used_user_addr = device_region.iova;
 
-    return r == 0;
+    return ok;
 }
 
 static bool vhost_vdpa_svq_setup(struct vhost_dev *dev,
diff --git a/include/hw/virtio/vhost-vdpa.h b/include/hw/virtio/vhost-vdpa.h
index XXXXXXX..XXXXXXX 100644
--- a/include/hw/virtio/vhost-vdpa.h
+++ b/include/hw/virtio/vhost-vdpa.h
@@ -XXX,XX +XXX,XX @@
 
 #include <gmodule.h>
 
+#include "hw/virtio/vhost-iova-tree.h"
 #include "hw/virtio/virtio.h"
 #include "standard-headers/linux/vhost_types.h"
 
@@ -XXX,XX +XXX,XX @@ typedef struct vhost_vdpa {
     MemoryListener listener;
     struct vhost_vdpa_iova_range iova_range;
     bool shadow_vqs_enabled;
+    /* IOVA mapping used by the Shadow Virtqueue */
+    VhostIOVATree *iova_tree;
     GPtrArray *shadow_vqs;
     struct vhost_dev *dev;
     VhostVDPAHostNotifier notifier[VIRTIO_QUEUE_MAX];
-- 
2.7.4

From: Eugenio Pérez <eperezma@redhat.com>

This is needed to achieve migration, so the destination can restore its
index.

Setting base as last used idx, so destination will see as available all
the entries that the device did not use, including the in-flight
processing ones.

This is ok for networking, but other kinds of devices might have
problems with these retransmissions.

diff --git a/hw/virtio/vhost-vdpa.c b/hw/virtio/vhost-vdpa.c
index XXXXXXX..XXXXXXX 100644
--- a/hw/virtio/vhost-vdpa.c
+++ b/hw/virtio/vhost-vdpa.c
@@ -XXX,XX +XXX,XX @@ static int vhost_vdpa_set_vring_base(struct vhost_dev *dev,
 static int vhost_vdpa_get_vring_base(struct vhost_dev *dev,
                                        struct vhost_vring_state *ring)
 {
+    struct vhost_vdpa *v = dev->opaque;
     int ret;
 
+    if (v->shadow_vqs_enabled) {
+        VhostShadowVirtqueue *svq = g_ptr_array_index(v->shadow_vqs,
+                                                      ring->index);
+
+        /*
+         * Setting base as last used idx, so destination will see as available
+         * all the entries that the device did not use, including the in-flight
+         * processing ones.
+         *
+         * TODO: This is ok for networking, but other kinds of devices might
+         * have problems with these retransmissions.
+         */
+        ring->num = svq->last_used_idx;
+        return 0;
+    }
+
     ret = vhost_vdpa_call(dev, VHOST_GET_VRING_BASE, ring);
     trace_vhost_vdpa_get_vring_base(dev, ring->index, ring->num);
     return ret;
-- 
2.7.4

From: Eugenio Pérez <eperezma@redhat.com>

SVQ is able to log the dirty bits by itself, so let's use it to not
block migration.

Also, ignore set and clear of VHOST_F_LOG_ALL on set_features if SVQ is
enabled. Even if the device supports it, the reports would be nonsense
because SVQ memory is in the qemu region.

The log region is still allocated. Future changes might skip that, but
this series is already long enough.

Signed-off-by: Eugenio Pérez <eperezma@redhat.com>
Acked-by: Michael S. Tsirkin <mst@redhat.com>
Signed-off-by: Jason Wang <jasowang@redhat.com>
---
 hw/virtio/vhost-vdpa.c         | 39 +++++++++++++++++++++++++++++++++++----
 include/hw/virtio/vhost-vdpa.h |  1 +
 2 files changed, 36 insertions(+), 4 deletions(-)

diff --git a/hw/virtio/vhost-vdpa.c b/hw/virtio/vhost-vdpa.c
index XXXXXXX..XXXXXXX 100644
--- a/hw/virtio/vhost-vdpa.c
+++ b/hw/virtio/vhost-vdpa.c
@@ -XXX,XX +XXX,XX @@ static bool vhost_vdpa_one_time_request(struct vhost_dev *dev)
     return v->index != 0;
 }
 
+static int vhost_vdpa_get_dev_features(struct vhost_dev *dev,
+                                       uint64_t *features)
+{
+    int ret;
+
+    ret = vhost_vdpa_call(dev, VHOST_GET_FEATURES, features);
+    trace_vhost_vdpa_get_features(dev, *features);
+    return ret;
+}
+
 static int vhost_vdpa_init_svq(struct vhost_dev *hdev, struct vhost_vdpa *v,
                                Error **errp)
 {
@@ -XXX,XX +XXX,XX @@ static int vhost_vdpa_init_svq(struct vhost_dev *hdev, struct vhost_vdpa *v,
         return 0;
     }
 
-    r = hdev->vhost_ops->vhost_get_features(hdev, &dev_features);
+    r = vhost_vdpa_get_dev_features(hdev, &dev_features);
     if (r != 0) {
         error_setg_errno(errp, -r, "Can't get vdpa device features");
         return r;
@@ -XXX,XX +XXX,XX @@ static int vhost_vdpa_set_mem_table(struct vhost_dev *dev,
 static int vhost_vdpa_set_features(struct vhost_dev *dev,
                                    uint64_t features)
 {
+    struct vhost_vdpa *v = dev->opaque;
     int ret;
 
     if (vhost_vdpa_one_time_request(dev)) {
         return 0;
     }
 
+    if (v->shadow_vqs_enabled) {
+        if ((v->acked_features ^ features) == BIT_ULL(VHOST_F_LOG_ALL)) {
+            /*
+             * QEMU is just trying to enable or disable logging. SVQ handles
+             * this sepparately, so no need to forward this.
+             */
+            v->acked_features = features;
+            return 0;
+        }
+
+        v->acked_features = features;
+
+        /* We must not ack _F_LOG if SVQ is enabled */
+        features &= ~BIT_ULL(VHOST_F_LOG_ALL);
+    }
+
     trace_vhost_vdpa_set_features(dev, features);
     ret = vhost_vdpa_call(dev, VHOST_SET_FEATURES, &features);
     if (ret) {
@@ -XXX,XX +XXX,XX @@ static int vhost_vdpa_set_vring_call(struct vhost_dev *dev,
 static int vhost_vdpa_get_features(struct vhost_dev *dev,
                                      uint64_t *features)
 {
-    int ret;
+    struct vhost_vdpa *v = dev->opaque;
+    int ret = vhost_vdpa_get_dev_features(dev, features);
+
+    if (ret == 0 && v->shadow_vqs_enabled) {
+        /* Add SVQ logging capabilities */
+        *features |= BIT_ULL(VHOST_F_LOG_ALL);
+    }
 
-    ret = vhost_vdpa_call(dev, VHOST_GET_FEATURES, features);
-    trace_vhost_vdpa_get_features(dev, *features);
     return ret;
 }
 
diff --git a/include/hw/virtio/vhost-vdpa.h b/include/hw/virtio/vhost-vdpa.h
index XXXXXXX..XXXXXXX 100644
--- a/include/hw/virtio/vhost-vdpa.h
+++ b/include/hw/virtio/vhost-vdpa.h
@@ -XXX,XX +XXX,XX @@ typedef struct vhost_vdpa {
     bool iotlb_batch_begin_sent;
     MemoryListener listener;
     struct vhost_vdpa_iova_range iova_range;
+    uint64_t acked_features;
     bool shadow_vqs_enabled;
     /* IOVA mapping used by the Shadow Virtqueue */
     VhostIOVATree *iova_tree;
-- 
2.7.4