Bhyve supports vcpu pinning using the `-p vcpu:hostcpu`
argument. This argument can be specified multiple times for the same
vcpu to pin it to multiple hostcpu's.

Bhyve currently does not allow to change vcpu pinning configuration for
the VM that is already running.

Use this to support domain's vcpupin configuration such as:

  <cputune>
    <vcpupin vcpu="0" cpuset="1,2,3"/>
  </cputune>

Signed-off-by: Roman Bogorodskiy <bogorodskiy@gmail.com>
---
 docs/drvbhyve.rst                             | 20 ++++++++++++++
 src/bhyve/bhyve_command.c                     | 15 +++++++++++
 .../x86_64/bhyvexml2argv-vcpupin.args         | 15 +++++++++++
 .../x86_64/bhyvexml2argv-vcpupin.ldargs       |  4 +++
 .../x86_64/bhyvexml2argv-vcpupin.xml          | 27 +++++++++++++++++++
 tests/bhyvexml2argvtest.c                     |  1 +
 6 files changed, 82 insertions(+)
 create mode 100644 tests/bhyvexml2argvdata/x86_64/bhyvexml2argv-vcpupin.args
 create mode 100644 tests/bhyvexml2argvdata/x86_64/bhyvexml2argv-vcpupin.ldargs
 create mode 100644 tests/bhyvexml2argvdata/x86_64/bhyvexml2argv-vcpupin.xml

diff --git a/docs/drvbhyve.rst b/docs/drvbhyve.rst
index 5a45e9056f..f4aec9ffad 100644
--- a/docs/drvbhyve.rst
+++ b/docs/drvbhyve.rst
@@ -737,6 +737,26 @@ Example:
 Please refer to ``cam(4)``, ``ctl(4)``, and ``ctld(8)`` manual pages
 for more details on CAM and CTL.
 
+vCPU pinning
+~~~~~~~~~~~~
+
+:since:`Since 12.1.0`, it is possible to pin domain vCPUs
+to the specific host CPUs.
+
+Example:
+
+::
+
+  <domain type='bhyve'>
+    ...
+    <vcpu>2</vcpu>
+    <cputune>
+      <vcpupin vcpu="0" cpuset="1-4,^2"/>
+      <vcpupin vcpu="1" cpuset="0,4"/>
+    </cputune>
+    ...
+  </domain>
+
 Guest-specific considerations
 -----------------------------
 
diff --git a/src/bhyve/bhyve_command.c b/src/bhyve/bhyve_command.c
index ff079da9ef..7a92966d39 100644
--- a/src/bhyve/bhyve_command.c
+++ b/src/bhyve/bhyve_command.c
@@ -940,6 +940,21 @@ virBhyveProcessBuildBhyveCmd(struct _bhyveConn *driver, virDomainDef *def,
         virCommandAddArgFormat(cmd, "%d", nvcpus);
     }
 
+    /* CPU tuning */
+    for (i = 0; i < virDomainDefGetVcpusMax(def); i++) {
+        virDomainVcpuDef *vcpu = virDomainDefGetVcpu(def, i);
+
+        if (vcpu->cpumask) {
+            ssize_t j = -1;
+
+            while ((j = virBitmapNextSetBit(vcpu->cpumask, j)) >= 0) {
+                virCommandAddArg(cmd, "-p");
+                virCommandAddArgFormat(cmd, "%zu:%zu", i, j);
+            }
+
+        }
+    }
+
     /* Memory */
     virCommandAddArg(cmd, "-m");
     virCommandAddArgFormat(cmd, "%llu",
diff --git a/tests/bhyvexml2argvdata/x86_64/bhyvexml2argv-vcpupin.args b/tests/bhyvexml2argvdata/x86_64/bhyvexml2argv-vcpupin.args
new file mode 100644
index 0000000000..c873766220
--- /dev/null
+++ b/tests/bhyvexml2argvdata/x86_64/bhyvexml2argv-vcpupin.args
@@ -0,0 +1,15 @@
+bhyve \
+-c 2 \
+-p 0:1 \
+-p 0:3 \
+-p 0:4 \
+-p 1:0 \
+-p 1:4 \
+-m 214 \
+-u \
+-H \
+-P \
+-s 0:0,hostbridge \
+-s 2:0,ahci,hd:/tmp/freebsd.img \
+-s 3:0,virtio-net,faketapdev,mac=52:54:00:b9:94:02 \
+bhyve
diff --git a/tests/bhyvexml2argvdata/x86_64/bhyvexml2argv-vcpupin.ldargs b/tests/bhyvexml2argvdata/x86_64/bhyvexml2argv-vcpupin.ldargs
new file mode 100644
index 0000000000..5905f4b3e6
--- /dev/null
+++ b/tests/bhyvexml2argvdata/x86_64/bhyvexml2argv-vcpupin.ldargs
@@ -0,0 +1,4 @@
+bhyveload \
+-m 214 \
+-d /tmp/freebsd.img \
+bhyve
diff --git a/tests/bhyvexml2argvdata/x86_64/bhyvexml2argv-vcpupin.xml b/tests/bhyvexml2argvdata/x86_64/bhyvexml2argv-vcpupin.xml
new file mode 100644
index 0000000000..bfa13c7cab
--- /dev/null
+++ b/tests/bhyvexml2argvdata/x86_64/bhyvexml2argv-vcpupin.xml
@@ -0,0 +1,27 @@
+<domain type='bhyve'>
+  <name>bhyve</name>
+  <uuid>df3be7e7-a104-11e3-aeb0-50e5492bd3dc</uuid>
+  <memory>219136</memory>
+  <vcpu>2</vcpu>
+  <cputune>
+    <vcpupin vcpu="0" cpuset="1-4,^2"/>
+    <vcpupin vcpu="1" cpuset="0,4"/>
+  </cputune>
+  <os>
+    <type>hvm</type>
+  </os>
+  <devices>
+    <disk type='file'>
+      <driver name='file' type='raw'/>
+      <source file='/tmp/freebsd.img'/>
+      <target dev='hda' bus='sata'/>
+      <address type='drive' controller='0' bus='0' target='2' unit='0'/>
+    </disk>
+    <interface type='bridge'>
+      <mac address='52:54:00:b9:94:02'/>
+      <model type='virtio'/>
+      <source bridge="virbr0"/>
+      <address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x0'/>
+    </interface>
+  </devices>
+</domain>
diff --git a/tests/bhyvexml2argvtest.c b/tests/bhyvexml2argvtest.c
index ae43445cde..423edaa428 100644
--- a/tests/bhyvexml2argvtest.c
+++ b/tests/bhyvexml2argvtest.c
@@ -278,6 +278,7 @@ mymain(void)
     DO_TEST("slirp-mac-addr");
     DO_TEST_FAILURE("slirp-ip");
     DO_TEST("virtio-scsi");
+    DO_TEST("vcpupin");
 
     /* Address allocation tests */
     DO_TEST("addr-single-sata-disk");
-- 
2.52.0