docs/formatdomain.rst | 13 ++++-- src/bhyve/bhyve_capabilities.c | 23 ++++++++++ src/bhyve/bhyve_capabilities.h | 1 + src/bhyve/bhyve_domain.c | 20 +++++++++ src/bhyve/bhyve_process.c | 45 +++++++++++++++++++ src/conf/schemas/domaincommon.rng | 5 ++- ...bhyvexml2argv-blkiotune-invalid-device.xml | 32 +++++++++++++ ...yvexml2argv-blkiotune-multiple-devices.xml | 39 ++++++++++++++++ .../x86_64/bhyvexml2argv-blkiotune-weight.xml | 33 ++++++++++++++ .../x86_64/bhyvexml2argv-blkiotune.args | 10 +++++ .../x86_64/bhyvexml2argv-blkiotune.ldargs | 4 ++ .../x86_64/bhyvexml2argv-blkiotune.xml | 32 +++++++++++++ tests/bhyvexml2argvtest.c | 4 ++ .../x86_64/bhyvexml2xmlout-blkiotune.xml | 42 +++++++++++++++++ tests/bhyvexml2xmltest.c | 4 ++ 15 files changed, 302 insertions(+), 5 deletions(-) create mode 100644 tests/bhyvexml2argvdata/x86_64/bhyvexml2argv-blkiotune-invalid-device.xml create mode 100644 tests/bhyvexml2argvdata/x86_64/bhyvexml2argv-blkiotune-multiple-devices.xml create mode 100644 tests/bhyvexml2argvdata/x86_64/bhyvexml2argv-blkiotune-weight.xml create mode 100644 tests/bhyvexml2argvdata/x86_64/bhyvexml2argv-blkiotune.args create mode 100644 tests/bhyvexml2argvdata/x86_64/bhyvexml2argv-blkiotune.ldargs create mode 100644 tests/bhyvexml2argvdata/x86_64/bhyvexml2argv-blkiotune.xml create mode 100644 tests/bhyvexml2xmloutdata/x86_64/bhyvexml2xmlout-blkiotune.xml
FreeBSD supports resource limiting with the rctl(4) framework.
It supports various resource types, including I/O resources.
It allows to limit resources for users, processes, login classes,
and jails.
To apply blkiotune limits set limits for the bhyve process.
I/O related resources supported by rctl(4) are:
readbps filesystem reads, in bytes per second
writebps filesystem writes, in bytes per second
readiops filesystem reads, in operations per second
writeiops filesystem writes, in operations per second
Thus, the actual commands look like:
rctl -a process:$bhyvepid:writebps:throttle=10000000
rctl -a process:$bhyvepid:readbps:throttle=10000000
rctl -a process:$bhyvepid:writeiops:throttle=20000
rctl -a process:$bhyvepid:readiops:throttle=20000
This is different from the current blkiotune modeling in libvirt as
it requires specific device to apply limits to. To adapt this model
to per-domain I/O limits, update domain schema to specify "*" as a
device name.
The rctl(8) may be not available or not enabled, so add a capability
check for that.
Per process rules get removed when the process disappears, so no special
clean up is necessary.
Signed-off-by: Roman Bogorodskiy <bogorodskiy@gmail.com>
---
Changes since v1:
- Documented the "*" device name value in formatdomain.rst
- Extended bhyve driver validation to allow only "*" devices
and do no allow specifying device weight, added tests
for these cases
- Updated the RCTL macro to execute the rctl(8) command right
away instead of building an array of rules and looping
through it once again
- Minor formatting fixes
docs/formatdomain.rst | 13 ++++--
src/bhyve/bhyve_capabilities.c | 23 ++++++++++
src/bhyve/bhyve_capabilities.h | 1 +
src/bhyve/bhyve_domain.c | 20 +++++++++
src/bhyve/bhyve_process.c | 45 +++++++++++++++++++
src/conf/schemas/domaincommon.rng | 5 ++-
...bhyvexml2argv-blkiotune-invalid-device.xml | 32 +++++++++++++
...yvexml2argv-blkiotune-multiple-devices.xml | 39 ++++++++++++++++
.../x86_64/bhyvexml2argv-blkiotune-weight.xml | 33 ++++++++++++++
.../x86_64/bhyvexml2argv-blkiotune.args | 10 +++++
.../x86_64/bhyvexml2argv-blkiotune.ldargs | 4 ++
.../x86_64/bhyvexml2argv-blkiotune.xml | 32 +++++++++++++
tests/bhyvexml2argvtest.c | 4 ++
.../x86_64/bhyvexml2xmlout-blkiotune.xml | 42 +++++++++++++++++
tests/bhyvexml2xmltest.c | 4 ++
15 files changed, 302 insertions(+), 5 deletions(-)
create mode 100644 tests/bhyvexml2argvdata/x86_64/bhyvexml2argv-blkiotune-invalid-device.xml
create mode 100644 tests/bhyvexml2argvdata/x86_64/bhyvexml2argv-blkiotune-multiple-devices.xml
create mode 100644 tests/bhyvexml2argvdata/x86_64/bhyvexml2argv-blkiotune-weight.xml
create mode 100644 tests/bhyvexml2argvdata/x86_64/bhyvexml2argv-blkiotune.args
create mode 100644 tests/bhyvexml2argvdata/x86_64/bhyvexml2argv-blkiotune.ldargs
create mode 100644 tests/bhyvexml2argvdata/x86_64/bhyvexml2argv-blkiotune.xml
create mode 100644 tests/bhyvexml2xmloutdata/x86_64/bhyvexml2xmlout-blkiotune.xml
diff --git a/docs/formatdomain.rst b/docs/formatdomain.rst
index 1a4bd4c6e9..6c4e067072 100644
--- a/docs/formatdomain.rst
+++ b/docs/formatdomain.rst
@@ -1367,10 +1367,15 @@ Block I/O Tuning
associated with each guest disk device (contrast this to the <iotune>
element of a disk definition (See `Hard drives, floppy disks, CDROMs`_)
which can applies to an individual disk). Each ``device`` element has
- two mandatory sub-elements, ``path`` describing the absolute path of the
- device, and ``weight`` giving the relative weight of that device, in the
- range [100, 1000]. After kernel 2.6.39, the value could be in the range [10,
- 1000]. :since:`Since 0.9.8`
+ a mandatory ``path`` sub-element describing the absolute path of the
+ device.
+
+ A special value ``*`` can be used to throttle all domain
+ devices. :since:`Since 12.3.0, bhyve`
+
+ An optional ``weight`` sub-element specifies the relative
+ weight of the device, in the range [100, 1000]. After kernel 2.6.39,
+ the value could be in the range [10, 1000]. :since:`Since 0.9.8`
Additionally, the following optional sub-elements can be used:
``read_bytes_sec``
diff --git a/src/bhyve/bhyve_capabilities.c b/src/bhyve/bhyve_capabilities.c
index c3fb88fe9f..1fe0a3ad77 100644
--- a/src/bhyve/bhyve_capabilities.c
+++ b/src/bhyve/bhyve_capabilities.c
@@ -24,6 +24,7 @@
#include <config.h>
#include <sys/utsname.h>
#include <dirent.h>
+#include <sys/sysctl.h>
#include <sys/types.h>
#include "viralloc.h"
@@ -334,6 +335,27 @@ bhyveProbeCapsVNCPassword(unsigned int *caps, char *binary)
}
+static void
+bhyveProbeCapsRctl(unsigned int *caps)
+{
+ bool racct_enable;
+ size_t racct_enable_len;
+ g_autofree char *rctl = NULL;
+
+ if (!(rctl = virFindFileInPath("rctl")))
+ return;
+
+ racct_enable_len = sizeof(racct_enable);
+ if (sysctlbyname("kern.racct.enable", &racct_enable,
+ &racct_enable_len, NULL, 0) < 0)
+ return;
+
+ if (racct_enable)
+ *caps |= BHYVE_CAP_RCTL;
+
+ return;
+}
+
int
virBhyveProbeCaps(unsigned int *caps)
{
@@ -356,6 +378,7 @@ virBhyveProbeCaps(unsigned int *caps)
if ((ret = bhyveProbeCapsVNCPassword(caps, binary)))
goto out;
+ bhyveProbeCapsRctl(caps);
out:
VIR_FREE(binary);
diff --git a/src/bhyve/bhyve_capabilities.h b/src/bhyve/bhyve_capabilities.h
index 31fd9ab86a..0302b68e22 100644
--- a/src/bhyve/bhyve_capabilities.h
+++ b/src/bhyve/bhyve_capabilities.h
@@ -57,6 +57,7 @@ typedef enum {
BHYVE_CAP_NVME = 1 << 11,
BHYVE_CAP_ACPI = 1 << 12,
BHYVE_CAP_NUMA = 1 << 13,
+ BHYVE_CAP_RCTL = 1 << 14,
} virBhyveCapsFlags;
int virBhyveProbeGrubCaps(virBhyveGrubCapsFlags *caps);
diff --git a/src/bhyve/bhyve_domain.c b/src/bhyve/bhyve_domain.c
index 4594d7673f..7bce6a4bc0 100644
--- a/src/bhyve/bhyve_domain.c
+++ b/src/bhyve/bhyve_domain.c
@@ -464,6 +464,26 @@ bhyveDomainDefValidate(const virDomainDef *def,
}
}
+ if (def->blkio.ndevices > 1) {
+ virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
+ _("Per device I/O tuning is not supported"));
+ return -1;
+ } else if (def->blkio.ndevices == 1) {
+ virBlkioDevice *device = &def->blkio.devices[0];
+
+ if (STRNEQ(device->path, "*")) {
+ virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
+ _("Per device I/O tuning is not supported"));
+ return -1;
+ }
+
+ if (device->weight != 0) {
+ virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
+ _("Only \"*\" is supported as a device path for I/O tuning"));
+ return -1;
+ }
+ }
+
if (!def->os.loader)
return 0;
diff --git a/src/bhyve/bhyve_process.c b/src/bhyve/bhyve_process.c
index 1d436da609..3eb807b9fe 100644
--- a/src/bhyve/bhyve_process.c
+++ b/src/bhyve/bhyve_process.c
@@ -34,6 +34,7 @@
#include "bhyve_device.h"
#include "bhyve_driver.h"
+#include "bhyve_capabilities.h"
#include "bhyve_command.h"
#include "bhyve_firmware.h"
#include "bhyve_monitor.h"
@@ -132,6 +133,47 @@ bhyveProcessStopHook(struct _bhyveConn *driver,
VIR_HOOK_SUBOP_END, NULL, xml, NULL);
}
+static int
+bhyveSetResourceLimits(struct _bhyveConn *driver, virDomainObj *vm)
+{
+ virBlkioDevice *device;
+
+ if (vm->def->blkio.ndevices != 1)
+ return 0;
+
+ if ((bhyveDriverGetBhyveCaps(driver) & BHYVE_CAP_RCTL) == 0) {
+ virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
+ _("Cannot set resource limits: RACCT/RCTL is either not supported or not enabled"));
+ return -1;
+ }
+
+ device = &vm->def->blkio.devices[0];
+
+#define BHYVE_APPLY_RCTL_RULE(field, type, format) \
+ do { \
+ if ((field)) { \
+ g_auto(virBuffer) buf = VIR_BUFFER_INITIALIZER; \
+ g_autofree char *rule = NULL; \
+ g_autoptr(virCommand) cmd = virCommandNew("rctl"); \
+ virBufferAsprintf(&buf, "process:%d:" type ":throttle=" format, \
+ vm->pid, (field)); \
+ rule = virBufferContentAndReset(&buf); \
+ virCommandAddArgList(cmd, "-a", rule, NULL); \
+ if (virCommandRun(cmd, NULL) < 0) \
+ return -1; \
+ } \
+ } while (0)
+
+ BHYVE_APPLY_RCTL_RULE(device->riops, "readiops", "%u");
+ BHYVE_APPLY_RCTL_RULE(device->wiops, "writeiops", "%u");
+ BHYVE_APPLY_RCTL_RULE(device->rbps, "readbps", "%llu");
+ BHYVE_APPLY_RCTL_RULE(device->wbps, "writebps", "%llu");
+
+#undef BHYVE_APPLY_RCTL_RULE
+
+ return 0;
+}
+
static int
virBhyveProcessStartImpl(struct _bhyveConn *driver,
virDomainObj *vm,
@@ -258,6 +300,9 @@ virBhyveProcessStartImpl(struct _bhyveConn *driver,
BHYVE_STATE_DIR) < 0)
goto cleanup;
+ if (bhyveSetResourceLimits(driver, vm) < 0)
+ goto cleanup;
+
if (bhyveProcessStartHook(driver, vm, VIR_HOOK_BHYVE_OP_STARTED) < 0)
goto cleanup;
diff --git a/src/conf/schemas/domaincommon.rng b/src/conf/schemas/domaincommon.rng
index db1dcd3bb7..c7f442a4c1 100644
--- a/src/conf/schemas/domaincommon.rng
+++ b/src/conf/schemas/domaincommon.rng
@@ -1049,7 +1049,10 @@
<element name="device">
<interleave>
<element name="path">
- <ref name="absFilePath"/>
+ <choice>
+ <ref name="absFilePath"/>
+ <value>*</value>
+ </choice>
</element>
<optional>
<element name="weight">
diff --git a/tests/bhyvexml2argvdata/x86_64/bhyvexml2argv-blkiotune-invalid-device.xml b/tests/bhyvexml2argvdata/x86_64/bhyvexml2argv-blkiotune-invalid-device.xml
new file mode 100644
index 0000000000..68d6871e0c
--- /dev/null
+++ b/tests/bhyvexml2argvdata/x86_64/bhyvexml2argv-blkiotune-invalid-device.xml
@@ -0,0 +1,32 @@
+<domain type='bhyve'>
+ <name>bhyve</name>
+ <uuid>df3be7e7-a104-11e3-aeb0-50e5492bd3dc</uuid>
+ <memory>219136</memory>
+ <vcpu>1</vcpu>
+ <os>
+ <type>hvm</type>
+ </os>
+ <blkiotune>
+ <device>
+ <path>/dev/sda</path>
+ <read_bytes_sec>10000</read_bytes_sec>
+ <write_bytes_sec>10000</write_bytes_sec>
+ <read_iops_sec>20000</read_iops_sec>
+ <write_iops_sec>20000</write_iops_sec>
+ </device>
+ </blkiotune>
+ <devices>
+ <disk type='file'>
+ <driver name='file' type='raw'/>
+ <source file='/tmp/freebsd.img'/>
+ <target dev='hda' bus='sata'/>
+ <address type='drive' controller='0' bus='0' target='2' unit='0'/>
+ </disk>
+ <interface type='bridge'>
+ <mac address='52:54:00:b9:94:02'/>
+ <model type='virtio'/>
+ <source bridge="virbr0"/>
+ <address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x0'/>
+ </interface>
+ </devices>
+</domain>
diff --git a/tests/bhyvexml2argvdata/x86_64/bhyvexml2argv-blkiotune-multiple-devices.xml b/tests/bhyvexml2argvdata/x86_64/bhyvexml2argv-blkiotune-multiple-devices.xml
new file mode 100644
index 0000000000..a6b689ad9a
--- /dev/null
+++ b/tests/bhyvexml2argvdata/x86_64/bhyvexml2argv-blkiotune-multiple-devices.xml
@@ -0,0 +1,39 @@
+<domain type='bhyve'>
+ <name>bhyve</name>
+ <uuid>df3be7e7-a104-11e3-aeb0-50e5492bd3dc</uuid>
+ <memory>219136</memory>
+ <vcpu>1</vcpu>
+ <os>
+ <type>hvm</type>
+ </os>
+ <blkiotune>
+ <device>
+ <path>*</path>
+ <read_bytes_sec>10000</read_bytes_sec>
+ <write_bytes_sec>10000</write_bytes_sec>
+ <read_iops_sec>20000</read_iops_sec>
+ <write_iops_sec>20000</write_iops_sec>
+ </device>
+ <device>
+ <path>/dev/hda</path>
+ <read_bytes_sec>10000</read_bytes_sec>
+ <write_bytes_sec>10000</write_bytes_sec>
+ <read_iops_sec>20000</read_iops_sec>
+ <write_iops_sec>20000</write_iops_sec>
+ </device>
+ </blkiotune>
+ <devices>
+ <disk type='file'>
+ <driver name='file' type='raw'/>
+ <source file='/tmp/freebsd.img'/>
+ <target dev='hda' bus='sata'/>
+ <address type='drive' controller='0' bus='0' target='2' unit='0'/>
+ </disk>
+ <interface type='bridge'>
+ <mac address='52:54:00:b9:94:02'/>
+ <model type='virtio'/>
+ <source bridge="virbr0"/>
+ <address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x0'/>
+ </interface>
+ </devices>
+</domain>
diff --git a/tests/bhyvexml2argvdata/x86_64/bhyvexml2argv-blkiotune-weight.xml b/tests/bhyvexml2argvdata/x86_64/bhyvexml2argv-blkiotune-weight.xml
new file mode 100644
index 0000000000..f7603828e0
--- /dev/null
+++ b/tests/bhyvexml2argvdata/x86_64/bhyvexml2argv-blkiotune-weight.xml
@@ -0,0 +1,33 @@
+<domain type='bhyve'>
+ <name>bhyve</name>
+ <uuid>df3be7e7-a104-11e3-aeb0-50e5492bd3dc</uuid>
+ <memory>219136</memory>
+ <vcpu>1</vcpu>
+ <os>
+ <type>hvm</type>
+ </os>
+ <blkiotune>
+ <device>
+ <path>*</path>
+ <weight>100</weight>
+ <read_bytes_sec>10000</read_bytes_sec>
+ <write_bytes_sec>10000</write_bytes_sec>
+ <read_iops_sec>20000</read_iops_sec>
+ <write_iops_sec>20000</write_iops_sec>
+ </device>
+ </blkiotune>
+ <devices>
+ <disk type='file'>
+ <driver name='file' type='raw'/>
+ <source file='/tmp/freebsd.img'/>
+ <target dev='hda' bus='sata'/>
+ <address type='drive' controller='0' bus='0' target='2' unit='0'/>
+ </disk>
+ <interface type='bridge'>
+ <mac address='52:54:00:b9:94:02'/>
+ <model type='virtio'/>
+ <source bridge="virbr0"/>
+ <address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x0'/>
+ </interface>
+ </devices>
+</domain>
diff --git a/tests/bhyvexml2argvdata/x86_64/bhyvexml2argv-blkiotune.args b/tests/bhyvexml2argvdata/x86_64/bhyvexml2argv-blkiotune.args
new file mode 100644
index 0000000000..507e0be668
--- /dev/null
+++ b/tests/bhyvexml2argvdata/x86_64/bhyvexml2argv-blkiotune.args
@@ -0,0 +1,10 @@
+bhyve \
+-c 1 \
+-m 214 \
+-u \
+-H \
+-P \
+-s 0:0,hostbridge \
+-s 2:0,ahci,hd:/tmp/freebsd.img \
+-s 3:0,virtio-net,faketapdev,mac=52:54:00:b9:94:02 \
+bhyve
diff --git a/tests/bhyvexml2argvdata/x86_64/bhyvexml2argv-blkiotune.ldargs b/tests/bhyvexml2argvdata/x86_64/bhyvexml2argv-blkiotune.ldargs
new file mode 100644
index 0000000000..5905f4b3e6
--- /dev/null
+++ b/tests/bhyvexml2argvdata/x86_64/bhyvexml2argv-blkiotune.ldargs
@@ -0,0 +1,4 @@
+bhyveload \
+-m 214 \
+-d /tmp/freebsd.img \
+bhyve
diff --git a/tests/bhyvexml2argvdata/x86_64/bhyvexml2argv-blkiotune.xml b/tests/bhyvexml2argvdata/x86_64/bhyvexml2argv-blkiotune.xml
new file mode 100644
index 0000000000..956d96cf18
--- /dev/null
+++ b/tests/bhyvexml2argvdata/x86_64/bhyvexml2argv-blkiotune.xml
@@ -0,0 +1,32 @@
+<domain type='bhyve'>
+ <name>bhyve</name>
+ <uuid>df3be7e7-a104-11e3-aeb0-50e5492bd3dc</uuid>
+ <memory>219136</memory>
+ <vcpu>1</vcpu>
+ <os>
+ <type>hvm</type>
+ </os>
+ <blkiotune>
+ <device>
+ <path>*</path>
+ <read_bytes_sec>10000</read_bytes_sec>
+ <write_bytes_sec>10000</write_bytes_sec>
+ <read_iops_sec>20000</read_iops_sec>
+ <write_iops_sec>20000</write_iops_sec>
+ </device>
+ </blkiotune>
+ <devices>
+ <disk type='file'>
+ <driver name='file' type='raw'/>
+ <source file='/tmp/freebsd.img'/>
+ <target dev='hda' bus='sata'/>
+ <address type='drive' controller='0' bus='0' target='2' unit='0'/>
+ </disk>
+ <interface type='bridge'>
+ <mac address='52:54:00:b9:94:02'/>
+ <model type='virtio'/>
+ <source bridge="virbr0"/>
+ <address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x0'/>
+ </interface>
+ </devices>
+</domain>
diff --git a/tests/bhyvexml2argvtest.c b/tests/bhyvexml2argvtest.c
index b7749fec6f..71c67ba2a2 100644
--- a/tests/bhyvexml2argvtest.c
+++ b/tests/bhyvexml2argvtest.c
@@ -289,6 +289,10 @@ mymain(void)
DO_TEST_FAILURE("slirp-ip");
DO_TEST("virtio-scsi");
DO_TEST("vcpupin");
+ DO_TEST("blkiotune");
+ DO_TEST_FAILURE("blkiotune-invalid-device");
+ DO_TEST_FAILURE("blkiotune-multiple-devices");
+ DO_TEST_FAILURE("blkiotune-weight");
/* Address allocation tests */
DO_TEST("addr-single-sata-disk");
diff --git a/tests/bhyvexml2xmloutdata/x86_64/bhyvexml2xmlout-blkiotune.xml b/tests/bhyvexml2xmloutdata/x86_64/bhyvexml2xmlout-blkiotune.xml
new file mode 100644
index 0000000000..4170303a6e
--- /dev/null
+++ b/tests/bhyvexml2xmloutdata/x86_64/bhyvexml2xmlout-blkiotune.xml
@@ -0,0 +1,42 @@
+<domain type='bhyve'>
+ <name>bhyve</name>
+ <uuid>df3be7e7-a104-11e3-aeb0-50e5492bd3dc</uuid>
+ <memory unit='KiB'>219136</memory>
+ <currentMemory unit='KiB'>219136</currentMemory>
+ <blkiotune>
+ <device>
+ <path>*</path>
+ <read_iops_sec>20000</read_iops_sec>
+ <write_iops_sec>20000</write_iops_sec>
+ <read_bytes_sec>10000</read_bytes_sec>
+ <write_bytes_sec>10000</write_bytes_sec>
+ </device>
+ </blkiotune>
+ <vcpu placement='static'>1</vcpu>
+ <os>
+ <type arch='x86_64'>hvm</type>
+ <boot dev='hd'/>
+ </os>
+ <clock offset='utc'/>
+ <on_poweroff>destroy</on_poweroff>
+ <on_reboot>restart</on_reboot>
+ <on_crash>destroy</on_crash>
+ <devices>
+ <disk type='file' device='disk'>
+ <driver name='file' type='raw'/>
+ <source file='/tmp/freebsd.img'/>
+ <target dev='hda' bus='sata'/>
+ <address type='drive' controller='0' bus='0' target='2' unit='0'/>
+ </disk>
+ <controller type='pci' index='0' model='pci-root'/>
+ <controller type='sata' index='0'>
+ <address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x0'/>
+ </controller>
+ <interface type='bridge'>
+ <mac address='52:54:00:b9:94:02'/>
+ <source bridge='virbr0'/>
+ <model type='virtio'/>
+ <address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x0'/>
+ </interface>
+ </devices>
+</domain>
diff --git a/tests/bhyvexml2xmltest.c b/tests/bhyvexml2xmltest.c
index 950aaea672..9fa8a9765d 100644
--- a/tests/bhyvexml2xmltest.c
+++ b/tests/bhyvexml2xmltest.c
@@ -133,6 +133,10 @@ mymain(void)
DO_TEST_DIFFERENT("slirp");
DO_TEST_DIFFERENT("virtio-scsi");
DO_TEST_DIFFERENT("numa");
+ DO_TEST_DIFFERENT("blkiotune");
+ DO_TEST_FAILURE("blkiotune-invalid-device");
+ DO_TEST_FAILURE("blkiotune-weight");
+ DO_TEST_FAILURE("blkiotune-multiple-devices");
/* Address allocation tests */
DO_TEST_DIFFERENT("addr-single-sata-disk");
--
2.52.0On Wed, Apr 15, 2026 at 19:28:15 +0200, Roman Bogorodskiy wrote:
> FreeBSD supports resource limiting with the rctl(4) framework.
> It supports various resource types, including I/O resources.
> It allows to limit resources for users, processes, login classes,
> and jails.
>
> To apply blkiotune limits set limits for the bhyve process.
>
> I/O related resources supported by rctl(4) are:
>
> readbps filesystem reads, in bytes per second
> writebps filesystem writes, in bytes per second
> readiops filesystem reads, in operations per second
> writeiops filesystem writes, in operations per second
>
> Thus, the actual commands look like:
>
> rctl -a process:$bhyvepid:writebps:throttle=10000000
> rctl -a process:$bhyvepid:readbps:throttle=10000000
> rctl -a process:$bhyvepid:writeiops:throttle=20000
> rctl -a process:$bhyvepid:readiops:throttle=20000
>
> This is different from the current blkiotune modeling in libvirt as
> it requires specific device to apply limits to. To adapt this model
> to per-domain I/O limits, update domain schema to specify "*" as a
> device name.
>
> The rctl(8) may be not available or not enabled, so add a capability
> check for that.
>
> Per process rules get removed when the process disappears, so no special
> clean up is necessary.
>
> Signed-off-by: Roman Bogorodskiy <bogorodskiy@gmail.com>
> ---
> Changes since v1:
>
> - Documented the "*" device name value in formatdomain.rst
> - Extended bhyve driver validation to allow only "*" devices
> and do no allow specifying device weight, added tests
> for these cases
> - Updated the RCTL macro to execute the rctl(8) command right
> away instead of building an array of rules and looping
> through it once again
> - Minor formatting fixes
>
> docs/formatdomain.rst | 13 ++++--
> src/bhyve/bhyve_capabilities.c | 23 ++++++++++
> src/bhyve/bhyve_capabilities.h | 1 +
> src/bhyve/bhyve_domain.c | 20 +++++++++
> src/bhyve/bhyve_process.c | 45 +++++++++++++++++++
> src/conf/schemas/domaincommon.rng | 5 ++-
> ...bhyvexml2argv-blkiotune-invalid-device.xml | 32 +++++++++++++
> ...yvexml2argv-blkiotune-multiple-devices.xml | 39 ++++++++++++++++
> .../x86_64/bhyvexml2argv-blkiotune-weight.xml | 33 ++++++++++++++
> .../x86_64/bhyvexml2argv-blkiotune.args | 10 +++++
> .../x86_64/bhyvexml2argv-blkiotune.ldargs | 4 ++
> .../x86_64/bhyvexml2argv-blkiotune.xml | 32 +++++++++++++
> tests/bhyvexml2argvtest.c | 4 ++
> .../x86_64/bhyvexml2xmlout-blkiotune.xml | 42 +++++++++++++++++
> tests/bhyvexml2xmltest.c | 4 ++
> 15 files changed, 302 insertions(+), 5 deletions(-)
> create mode 100644 tests/bhyvexml2argvdata/x86_64/bhyvexml2argv-blkiotune-invalid-device.xml
> create mode 100644 tests/bhyvexml2argvdata/x86_64/bhyvexml2argv-blkiotune-multiple-devices.xml
> create mode 100644 tests/bhyvexml2argvdata/x86_64/bhyvexml2argv-blkiotune-weight.xml
> create mode 100644 tests/bhyvexml2argvdata/x86_64/bhyvexml2argv-blkiotune.args
> create mode 100644 tests/bhyvexml2argvdata/x86_64/bhyvexml2argv-blkiotune.ldargs
> create mode 100644 tests/bhyvexml2argvdata/x86_64/bhyvexml2argv-blkiotune.xml
> create mode 100644 tests/bhyvexml2xmloutdata/x86_64/bhyvexml2xmlout-blkiotune.xml
>
> diff --git a/docs/formatdomain.rst b/docs/formatdomain.rst
> index 1a4bd4c6e9..6c4e067072 100644
> --- a/docs/formatdomain.rst
> +++ b/docs/formatdomain.rst
> @@ -1367,10 +1367,15 @@ Block I/O Tuning
> associated with each guest disk device (contrast this to the <iotune>
> element of a disk definition (See `Hard drives, floppy disks, CDROMs`_)
> which can applies to an individual disk). Each ``device`` element has
> - two mandatory sub-elements, ``path`` describing the absolute path of the
> - device, and ``weight`` giving the relative weight of that device, in the
> - range [100, 1000]. After kernel 2.6.39, the value could be in the range [10,
> - 1000]. :since:`Since 0.9.8`
> + a mandatory ``path`` sub-element describing the absolute path of the
> + device.
> +
> + A special value ``*`` can be used to throttle all domain
> + devices. :since:`Since 12.3.0, bhyve`
> +
> + An optional ``weight`` sub-element specifies the relative
> + weight of the device, in the range [100, 1000]. After kernel 2.6.39,
> + the value could be in the range [10, 1000]. :since:`Since 0.9.8`
> Additionally, the following optional sub-elements can be used:
>
> ``read_bytes_sec``
[...]
> diff --git a/src/bhyve/bhyve_domain.c b/src/bhyve/bhyve_domain.c
> index 4594d7673f..7bce6a4bc0 100644
> --- a/src/bhyve/bhyve_domain.c
> +++ b/src/bhyve/bhyve_domain.c
> @@ -464,6 +464,26 @@ bhyveDomainDefValidate(const virDomainDef *def,
> }
> }
>
> + if (def->blkio.ndevices > 1) {
> + virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
> + _("Per device I/O tuning is not supported"));
> + return -1;
> + } else if (def->blkio.ndevices == 1) {
> + virBlkioDevice *device = &def->blkio.devices[0];
> +
> + if (STRNEQ(device->path, "*")) {
> + virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
> + _("Per device I/O tuning is not supported"));
> + return -1;
> + }
> +
> + if (device->weight != 0) {
> + virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
> + _("Only \"*\" is supported as a device path for I/O tuning"));
The message doesn't correspond with what the check rejects.
I'd go with something along:
_("The 'weight' I/O tuning setting doesn't make sense with '*')
> + return -1;
> + }
> + }
> +
> if (!def->os.loader)
> return 0;
>
> diff --git a/src/bhyve/bhyve_process.c b/src/bhyve/bhyve_process.c
> index 1d436da609..3eb807b9fe 100644
> --- a/src/bhyve/bhyve_process.c
> +++ b/src/bhyve/bhyve_process.c
> @@ -34,6 +34,7 @@
>
> #include "bhyve_device.h"
> #include "bhyve_driver.h"
> +#include "bhyve_capabilities.h"
> #include "bhyve_command.h"
> #include "bhyve_firmware.h"
> #include "bhyve_monitor.h"
> @@ -132,6 +133,47 @@ bhyveProcessStopHook(struct _bhyveConn *driver,
> VIR_HOOK_SUBOP_END, NULL, xml, NULL);
> }
>
> +static int
> +bhyveSetResourceLimits(struct _bhyveConn *driver, virDomainObj *vm)
> +{
> + virBlkioDevice *device;
> +
> + if (vm->def->blkio.ndevices != 1)
> + return 0;
> +
> + if ((bhyveDriverGetBhyveCaps(driver) & BHYVE_CAP_RCTL) == 0) {
> + virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
> + _("Cannot set resource limits: RACCT/RCTL is either not supported or not enabled"));
> + return -1;
> + }
> +
> + device = &vm->def->blkio.devices[0];
> +
> +#define BHYVE_APPLY_RCTL_RULE(field, type, format) \
> + do { \
> + if ((field)) { \
> + g_auto(virBuffer) buf = VIR_BUFFER_INITIALIZER; \
> + g_autofree char *rule = NULL; \
> + g_autoptr(virCommand) cmd = virCommandNew("rctl"); \
> + virBufferAsprintf(&buf, "process:%d:" type ":throttle=" format, \
> + vm->pid, (field)); \
> + rule = virBufferContentAndReset(&buf); \
> + virCommandAddArgList(cmd, "-a", rule, NULL); \
You can simplify formatting of the command so that the extra virBuffer
and char* are not needed:
g_autoptr(virCommand) cmd = virCommandNewList("rctl", "-a", NULL);
virCommandAddArgFormat(cmd, "process:%d:" type ":throttle=" format,
vm->pid, (field));
> + if (virCommandRun(cmd, NULL) < 0) \
> + return -1; \
> + } \
> + } while (0)
> +
> + BHYVE_APPLY_RCTL_RULE(device->riops, "readiops", "%u");
> + BHYVE_APPLY_RCTL_RULE(device->wiops, "writeiops", "%u");
> + BHYVE_APPLY_RCTL_RULE(device->rbps, "readbps", "%llu");
> + BHYVE_APPLY_RCTL_RULE(device->wbps, "writebps", "%llu");
> +
> +#undef BHYVE_APPLY_RCTL_RULE
> +
> + return 0;
> +}
> +
> static int
> virBhyveProcessStartImpl(struct _bhyveConn *driver,
> virDomainObj *vm,
Reviewed-by: Peter Krempa <pkrempa@redhat.com>
© 2016 - 2026 Red Hat, Inc.