FreeBSD supports resource limiting with the rctl(4) framework.
It supports various resource types, including I/O resources.
It allows to limit resources for users, processes, login classes,
and jails.

To apply blkiotune limits set limits for the bhyve process.

I/O related resources supported by rctl(4) are:

  readbps            filesystem reads, in bytes per second
  writebps           filesystem writes, in bytes per second
  readiops           filesystem reads, in operations per second
  writeiops          filesystem writes, in operations per second

Thus, the actual commands look like:

rctl -a process:$bhyvepid:writebps:throttle=10000000
rctl -a process:$bhyvepid:readbps:throttle=10000000
rctl -a process:$bhyvepid:writeiops:throttle=20000
rctl -a process:$bhyvepid:readiops:throttle=20000

This is different from the current blkiotune modeling in libvirt as
it requires specific device to apply limits to. To adapt this model
to per-domain I/O limits, update domain schema to specify "*" as a
device name.

The rctl(8) may be not available or not enabled, so add a capability
check for that.

Per process rules get removed when the process disappears, so no special
clean up is necessary.

Signed-off-by: Roman Bogorodskiy <bogorodskiy@gmail.com>
---
 src/bhyve/bhyve_capabilities.c                | 25 +++++++-
 src/bhyve/bhyve_capabilities.h                |  1 +
 src/bhyve/bhyve_domain.c                      |  6 ++
 src/bhyve/bhyve_process.c                     | 59 +++++++++++++++++++
 src/conf/schemas/domaincommon.rng             |  5 +-
 ...yvexml2argv-blkiotune-multiple-devices.xml | 39 ++++++++++++
 .../x86_64/bhyvexml2argv-blkiotune.args       | 10 ++++
 .../x86_64/bhyvexml2argv-blkiotune.ldargs     |  4 ++
 .../x86_64/bhyvexml2argv-blkiotune.xml        | 32 ++++++++++
 tests/bhyvexml2argvtest.c                     |  2 +
 .../x86_64/bhyvexml2xmlout-blkiotune.xml      | 42 +++++++++++++
 tests/bhyvexml2xmltest.c                      |  2 +
 12 files changed, 225 insertions(+), 2 deletions(-)
 create mode 100644 tests/bhyvexml2argvdata/x86_64/bhyvexml2argv-blkiotune-multiple-devices.xml
 create mode 100644 tests/bhyvexml2argvdata/x86_64/bhyvexml2argv-blkiotune.args
 create mode 100644 tests/bhyvexml2argvdata/x86_64/bhyvexml2argv-blkiotune.ldargs
 create mode 100644 tests/bhyvexml2argvdata/x86_64/bhyvexml2argv-blkiotune.xml
 create mode 100644 tests/bhyvexml2xmloutdata/x86_64/bhyvexml2xmlout-blkiotune.xml

diff --git a/src/bhyve/bhyve_capabilities.c b/src/bhyve/bhyve_capabilities.c
index c3fb88fe9f..1088d75407 100644
--- a/src/bhyve/bhyve_capabilities.c
+++ b/src/bhyve/bhyve_capabilities.c
@@ -24,6 +24,7 @@
 #include <config.h>
 #include <sys/utsname.h>
 #include <dirent.h>
+#include <sys/sysctl.h>
 #include <sys/types.h>
 
 #include "viralloc.h"
@@ -40,7 +41,6 @@
 
 VIR_LOG_INIT("bhyve.bhyve_capabilities");
 
-
 virCaps *
 virBhyveCapsBuild(void)
 {
@@ -334,6 +334,27 @@ bhyveProbeCapsVNCPassword(unsigned int *caps, char *binary)
 }
 
 
+static int
+bhyveProbeCapsRctl(unsigned int *caps)
+{
+    bool racct_enable;
+    size_t racct_enable_len;
+    g_autofree char *rctl = NULL;
+
+    if (!(rctl = virFindFileInPath("rctl")))
+        return 0;
+
+    racct_enable_len = sizeof(racct_enable);
+    if (sysctlbyname("kern.racct.enable", &racct_enable,
+                     &racct_enable_len, NULL, 0) < 0)
+        return 0;
+
+    if (racct_enable)
+        *caps |= BHYVE_CAP_RCTL;
+
+    return 0;
+}
+
 int
 virBhyveProbeCaps(unsigned int *caps)
 {
@@ -356,6 +377,8 @@ virBhyveProbeCaps(unsigned int *caps)
     if ((ret = bhyveProbeCapsVNCPassword(caps, binary)))
         goto out;
 
+    if ((ret = bhyveProbeCapsRctl(caps)))
+        goto out;
 
  out:
     VIR_FREE(binary);
diff --git a/src/bhyve/bhyve_capabilities.h b/src/bhyve/bhyve_capabilities.h
index 31fd9ab86a..0302b68e22 100644
--- a/src/bhyve/bhyve_capabilities.h
+++ b/src/bhyve/bhyve_capabilities.h
@@ -57,6 +57,7 @@ typedef enum {
     BHYVE_CAP_NVME = 1 << 11,
     BHYVE_CAP_ACPI = 1 << 12,
     BHYVE_CAP_NUMA = 1 << 13,
+    BHYVE_CAP_RCTL = 1 << 14,
 } virBhyveCapsFlags;
 
 int virBhyveProbeGrubCaps(virBhyveGrubCapsFlags *caps);
diff --git a/src/bhyve/bhyve_domain.c b/src/bhyve/bhyve_domain.c
index 4594d7673f..48e1451d8b 100644
--- a/src/bhyve/bhyve_domain.c
+++ b/src/bhyve/bhyve_domain.c
@@ -464,6 +464,12 @@ bhyveDomainDefValidate(const virDomainDef *def,
         }
     }
 
+    if (def->blkio.ndevices > 1) {
+        virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
+                       _("Per device I/O tuning is not supported"));
+        return -1;
+    }
+
     if (!def->os.loader)
         return 0;
 
diff --git a/src/bhyve/bhyve_process.c b/src/bhyve/bhyve_process.c
index 1d436da609..b5ebe0aa61 100644
--- a/src/bhyve/bhyve_process.c
+++ b/src/bhyve/bhyve_process.c
@@ -34,6 +34,7 @@
 
 #include "bhyve_device.h"
 #include "bhyve_driver.h"
+#include "bhyve_capabilities.h"
 #include "bhyve_command.h"
 #include "bhyve_firmware.h"
 #include "bhyve_monitor.h"
@@ -132,6 +133,61 @@ bhyveProcessStopHook(struct _bhyveConn *driver,
                 VIR_HOOK_SUBOP_END, NULL, xml, NULL);
 }
 
+static int
+bhyveSetResourceLimits(struct _bhyveConn *driver, virDomainObj *vm)
+{
+    virBlkioDevice *device;
+    char **rules = NULL;
+    size_t i, nrules = 0;
+    int ret = -1;
+
+    if (!vm->def->blkio.ndevices)
+        return 0;
+
+    if ((bhyveDriverGetBhyveCaps(driver) & BHYVE_CAP_RCTL) == 0) {
+        virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
+                       _("Cannot set resource limits: RACCT/RCTL is either not supported or not enabled"));
+        return ret;
+    }
+
+    device = &vm->def->blkio.devices[0];
+
+#define BHYVE_APPEND_RCTL_RULE(field, type, format) \
+    do { \
+        if ((field)) { \
+            virBuffer buf = VIR_BUFFER_INITIALIZER; \
+            char *rule; \
+            virBufferAsprintf(&buf, "process:%d:" type ":throttle=" format, \
+                              vm->pid, (field)); \
+            rule = virBufferContentAndReset(&buf); \
+            VIR_APPEND_ELEMENT(rules, nrules, rule); \
+         } \
+    } while (0)
+
+    BHYVE_APPEND_RCTL_RULE(device->riops, "readiops", "%u");
+    BHYVE_APPEND_RCTL_RULE(device->wiops, "writeiops", "%u");
+    BHYVE_APPEND_RCTL_RULE(device->rbps, "readbps", "%llu");
+    BHYVE_APPEND_RCTL_RULE(device->wbps, "writebps", "%llu");
+
+#undef BHYVE_APPEND_RCTL_RULE
+
+    for (i = 0; i < nrules; i++) {
+        g_autoptr(virCommand) cmd = virCommandNew("rctl");
+        virCommandAddArgList(cmd, "-a", rules[i], NULL);
+
+        if (virCommandRun(cmd, NULL) < 0)
+            goto cleanup;
+    }
+
+    ret = 0;
+
+ cleanup:
+    for (i = 0; i < nrules; i++)
+        VIR_FREE(rules[i]);
+    VIR_FREE(rules);
+    return ret;
+}
+
 static int
 virBhyveProcessStartImpl(struct _bhyveConn *driver,
                          virDomainObj *vm,
@@ -258,6 +314,9 @@ virBhyveProcessStartImpl(struct _bhyveConn *driver,
                          BHYVE_STATE_DIR) < 0)
         goto cleanup;
 
+    if (bhyveSetResourceLimits(driver, vm) < 0)
+        goto cleanup;
+
     if (bhyveProcessStartHook(driver, vm, VIR_HOOK_BHYVE_OP_STARTED) < 0)
         goto cleanup;
 
diff --git a/src/conf/schemas/domaincommon.rng b/src/conf/schemas/domaincommon.rng
index db1dcd3bb7..c7f442a4c1 100644
--- a/src/conf/schemas/domaincommon.rng
+++ b/src/conf/schemas/domaincommon.rng
@@ -1049,7 +1049,10 @@
           <element name="device">
             <interleave>
               <element name="path">
-                <ref name="absFilePath"/>
+                <choice>
+                  <ref name="absFilePath"/>
+                  <value>*</value>
+                </choice>
               </element>
               <optional>
                 <element name="weight">
diff --git a/tests/bhyvexml2argvdata/x86_64/bhyvexml2argv-blkiotune-multiple-devices.xml b/tests/bhyvexml2argvdata/x86_64/bhyvexml2argv-blkiotune-multiple-devices.xml
new file mode 100644
index 0000000000..a6b689ad9a
--- /dev/null
+++ b/tests/bhyvexml2argvdata/x86_64/bhyvexml2argv-blkiotune-multiple-devices.xml
@@ -0,0 +1,39 @@
+<domain type='bhyve'>
+  <name>bhyve</name>
+  <uuid>df3be7e7-a104-11e3-aeb0-50e5492bd3dc</uuid>
+  <memory>219136</memory>
+  <vcpu>1</vcpu>
+  <os>
+    <type>hvm</type>
+  </os>
+  <blkiotune>
+    <device>
+      <path>*</path>
+      <read_bytes_sec>10000</read_bytes_sec>
+      <write_bytes_sec>10000</write_bytes_sec>
+      <read_iops_sec>20000</read_iops_sec>
+      <write_iops_sec>20000</write_iops_sec>
+    </device>
+    <device>
+      <path>/dev/hda</path>
+      <read_bytes_sec>10000</read_bytes_sec>
+      <write_bytes_sec>10000</write_bytes_sec>
+      <read_iops_sec>20000</read_iops_sec>
+      <write_iops_sec>20000</write_iops_sec>
+    </device>
+  </blkiotune>
+  <devices>
+    <disk type='file'>
+      <driver name='file' type='raw'/>
+      <source file='/tmp/freebsd.img'/>
+      <target dev='hda' bus='sata'/>
+      <address type='drive' controller='0' bus='0' target='2' unit='0'/>
+    </disk>
+    <interface type='bridge'>
+      <mac address='52:54:00:b9:94:02'/>
+      <model type='virtio'/>
+      <source bridge="virbr0"/>
+      <address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x0'/>
+    </interface>
+  </devices>
+</domain>
diff --git a/tests/bhyvexml2argvdata/x86_64/bhyvexml2argv-blkiotune.args b/tests/bhyvexml2argvdata/x86_64/bhyvexml2argv-blkiotune.args
new file mode 100644
index 0000000000..507e0be668
--- /dev/null
+++ b/tests/bhyvexml2argvdata/x86_64/bhyvexml2argv-blkiotune.args
@@ -0,0 +1,10 @@
+bhyve \
+-c 1 \
+-m 214 \
+-u \
+-H \
+-P \
+-s 0:0,hostbridge \
+-s 2:0,ahci,hd:/tmp/freebsd.img \
+-s 3:0,virtio-net,faketapdev,mac=52:54:00:b9:94:02 \
+bhyve
diff --git a/tests/bhyvexml2argvdata/x86_64/bhyvexml2argv-blkiotune.ldargs b/tests/bhyvexml2argvdata/x86_64/bhyvexml2argv-blkiotune.ldargs
new file mode 100644
index 0000000000..5905f4b3e6
--- /dev/null
+++ b/tests/bhyvexml2argvdata/x86_64/bhyvexml2argv-blkiotune.ldargs
@@ -0,0 +1,4 @@
+bhyveload \
+-m 214 \
+-d /tmp/freebsd.img \
+bhyve
diff --git a/tests/bhyvexml2argvdata/x86_64/bhyvexml2argv-blkiotune.xml b/tests/bhyvexml2argvdata/x86_64/bhyvexml2argv-blkiotune.xml
new file mode 100644
index 0000000000..956d96cf18
--- /dev/null
+++ b/tests/bhyvexml2argvdata/x86_64/bhyvexml2argv-blkiotune.xml
@@ -0,0 +1,32 @@
+<domain type='bhyve'>
+  <name>bhyve</name>
+  <uuid>df3be7e7-a104-11e3-aeb0-50e5492bd3dc</uuid>
+  <memory>219136</memory>
+  <vcpu>1</vcpu>
+  <os>
+    <type>hvm</type>
+  </os>
+  <blkiotune>
+    <device>
+      <path>*</path>
+      <read_bytes_sec>10000</read_bytes_sec>
+      <write_bytes_sec>10000</write_bytes_sec>
+      <read_iops_sec>20000</read_iops_sec>
+      <write_iops_sec>20000</write_iops_sec>
+    </device>
+  </blkiotune>
+  <devices>
+    <disk type='file'>
+      <driver name='file' type='raw'/>
+      <source file='/tmp/freebsd.img'/>
+      <target dev='hda' bus='sata'/>
+      <address type='drive' controller='0' bus='0' target='2' unit='0'/>
+    </disk>
+    <interface type='bridge'>
+      <mac address='52:54:00:b9:94:02'/>
+      <model type='virtio'/>
+      <source bridge="virbr0"/>
+      <address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x0'/>
+    </interface>
+  </devices>
+</domain>
diff --git a/tests/bhyvexml2argvtest.c b/tests/bhyvexml2argvtest.c
index b7749fec6f..c098149ad5 100644
--- a/tests/bhyvexml2argvtest.c
+++ b/tests/bhyvexml2argvtest.c
@@ -289,6 +289,8 @@ mymain(void)
     DO_TEST_FAILURE("slirp-ip");
     DO_TEST("virtio-scsi");
     DO_TEST("vcpupin");
+    DO_TEST("blkiotune");
+    DO_TEST_FAILURE("blkiotune-multiple-devices");
 
     /* Address allocation tests */
     DO_TEST("addr-single-sata-disk");
diff --git a/tests/bhyvexml2xmloutdata/x86_64/bhyvexml2xmlout-blkiotune.xml b/tests/bhyvexml2xmloutdata/x86_64/bhyvexml2xmlout-blkiotune.xml
new file mode 100644
index 0000000000..4170303a6e
--- /dev/null
+++ b/tests/bhyvexml2xmloutdata/x86_64/bhyvexml2xmlout-blkiotune.xml
@@ -0,0 +1,42 @@
+<domain type='bhyve'>
+  <name>bhyve</name>
+  <uuid>df3be7e7-a104-11e3-aeb0-50e5492bd3dc</uuid>
+  <memory unit='KiB'>219136</memory>
+  <currentMemory unit='KiB'>219136</currentMemory>
+  <blkiotune>
+    <device>
+      <path>*</path>
+      <read_iops_sec>20000</read_iops_sec>
+      <write_iops_sec>20000</write_iops_sec>
+      <read_bytes_sec>10000</read_bytes_sec>
+      <write_bytes_sec>10000</write_bytes_sec>
+    </device>
+  </blkiotune>
+  <vcpu placement='static'>1</vcpu>
+  <os>
+    <type arch='x86_64'>hvm</type>
+    <boot dev='hd'/>
+  </os>
+  <clock offset='utc'/>
+  <on_poweroff>destroy</on_poweroff>
+  <on_reboot>restart</on_reboot>
+  <on_crash>destroy</on_crash>
+  <devices>
+    <disk type='file' device='disk'>
+      <driver name='file' type='raw'/>
+      <source file='/tmp/freebsd.img'/>
+      <target dev='hda' bus='sata'/>
+      <address type='drive' controller='0' bus='0' target='2' unit='0'/>
+    </disk>
+    <controller type='pci' index='0' model='pci-root'/>
+    <controller type='sata' index='0'>
+      <address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x0'/>
+    </controller>
+    <interface type='bridge'>
+      <mac address='52:54:00:b9:94:02'/>
+      <source bridge='virbr0'/>
+      <model type='virtio'/>
+      <address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x0'/>
+    </interface>
+  </devices>
+</domain>
diff --git a/tests/bhyvexml2xmltest.c b/tests/bhyvexml2xmltest.c
index 950aaea672..ae12b37064 100644
--- a/tests/bhyvexml2xmltest.c
+++ b/tests/bhyvexml2xmltest.c
@@ -133,6 +133,8 @@ mymain(void)
     DO_TEST_DIFFERENT("slirp");
     DO_TEST_DIFFERENT("virtio-scsi");
     DO_TEST_DIFFERENT("numa");
+    DO_TEST_DIFFERENT("blkiotune");
+    DO_TEST_FAILURE("blkiotune-multiple-devices");
 
     /* Address allocation tests */
     DO_TEST_DIFFERENT("addr-single-sata-disk");
-- 
2.52.0