From nobody Mon Feb 9 23:00:19 2026 Delivered-To: importer@patchew.org Received-SPF: pass (zohomail.com: domain of redhat.com designates 216.205.24.124 as permitted sender) client-ip=216.205.24.124; envelope-from=libvir-list-bounces@redhat.com; helo=us-smtp-delivery-124.mimecast.com; Authentication-Results: mx.zohomail.com; dkim=pass; spf=pass (zohomail.com: domain of redhat.com designates 216.205.24.124 as permitted sender) smtp.mailfrom=libvir-list-bounces@redhat.com; dmarc=pass(p=none dis=none) header.from=redhat.com ARC-Seal: i=1; a=rsa-sha256; t=1622450213; cv=none; d=zohomail.com; s=zohoarc; b=R3BMkQ0At1sMZkoyERYVcBqViQygHNmFiTbs4Phi+MEk8J1Uv1PXvN34MlEm4VGeTL+VoNIZvSuuMFHvFYcfWbRl9amtS7VgUW3yg3UXqwdLkj9AMSzIwJ1F60B2MLGnqiVT46IQhYkhdkaCo+3UhVhbY/ADGGdpZXLGmK33eOg= ARC-Message-Signature: i=1; a=rsa-sha256; c=relaxed/relaxed; d=zohomail.com; s=zohoarc; t=1622450213; h=Content-Type:Content-Transfer-Encoding:Date:From:In-Reply-To:List-Subscribe:List-Post:List-Id:List-Archive:List-Help:List-Unsubscribe:MIME-Version:Message-ID:References:Sender:Subject:To; bh=Hpx+0Ngl1k7oYC3WQ3cdyNEeYp5RG9q0Xd0AxdG6IMk=; b=EEzlvAk+H3NhYc8kUioNiC4ua2r2E9t0R7+qhjd30GWXRahzZOKyWOLcwQHTzqPOWg7G/mCGzSEbg9ofLqktB+GC4mkqaVt9LdoN34+E7p/u16W8uON+qJFB4dtRiKSkfO3KddnN0QeUfSpQrCtm4Cg0m5E4j/ITVT0F1A2DdC4= ARC-Authentication-Results: i=1; mx.zohomail.com; dkim=pass; spf=pass (zohomail.com: domain of redhat.com designates 216.205.24.124 as permitted sender) smtp.mailfrom=libvir-list-bounces@redhat.com; dmarc=pass header.from= (p=none dis=none) header.from= Return-Path: Received: from us-smtp-delivery-124.mimecast.com (us-smtp-delivery-124.mimecast.com [216.205.24.124]) by mx.zohomail.com with SMTPS id 1622450213332218.75453455733725; Mon, 31 May 2021 01:36:53 -0700 (PDT) Received: from mimecast-mx01.redhat.com (mimecast-mx01.redhat.com [209.132.183.4]) (Using TLS) by relay.mimecast.com with ESMTP id us-mta-125-Mlc6CGyKOZasa1MY3T_4qw-1; Mon, 31 May 2021 04:36:49 -0400 Received: from smtp.corp.redhat.com (int-mx01.intmail.prod.int.phx2.redhat.com [10.5.11.11]) (using TLSv1.2 with cipher AECDH-AES256-SHA (256/256 bits)) (No client certificate requested) by mimecast-mx01.redhat.com (Postfix) with ESMTPS id 74BD380EDAF; Mon, 31 May 2021 08:36:42 +0000 (UTC) Received: from colo-mx.corp.redhat.com (colo-mx02.intmail.prod.int.phx2.redhat.com [10.5.11.21]) by smtp.corp.redhat.com (Postfix) with ESMTPS id 5780069FA4; Mon, 31 May 2021 08:36:42 +0000 (UTC) Received: from lists01.pubmisc.prod.ext.phx2.redhat.com (lists01.pubmisc.prod.ext.phx2.redhat.com [10.5.19.33]) by colo-mx.corp.redhat.com (Postfix) with ESMTP id 2143B59CD1; Mon, 31 May 2021 08:36:42 +0000 (UTC) Received: from smtp.corp.redhat.com (int-mx03.intmail.prod.int.phx2.redhat.com [10.5.11.13]) by lists01.pubmisc.prod.ext.phx2.redhat.com (8.13.8/8.13.8) with ESMTP id 14V8aSfQ007497 for ; Mon, 31 May 2021 04:36:28 -0400 Received: by smtp.corp.redhat.com (Postfix) id 18158687EE; Mon, 31 May 2021 08:36:28 +0000 (UTC) Received: from localhost.localdomain (unknown [10.40.192.219]) by smtp.corp.redhat.com (Postfix) with ESMTP id 7E4067770F for ; Mon, 31 May 2021 08:36:27 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=redhat.com; s=mimecast20190719; t=1622450212; h=from:from:sender:sender:reply-to:subject:subject:date:date: message-id:message-id:to:to:cc:mime-version:mime-version: content-type:content-type: content-transfer-encoding:content-transfer-encoding: in-reply-to:in-reply-to:references:references:list-id:list-help: list-unsubscribe:list-subscribe:list-post; bh=Hpx+0Ngl1k7oYC3WQ3cdyNEeYp5RG9q0Xd0AxdG6IMk=; b=fjvDWe97W3zON59J7tq1hN8ytnxpbsXLIKgK48GKzlEYtYLUPiw0DLtxhyk3VvRoats3rf gXkOesS5fojK8CzMuFMLO4MpEHbrBC3K2trzAcYw99WwExiJYZiiDb3rfEUEqoD177bSw4 wxC9MxMtDwGONYHtGd1rcfAre1MGM3g= X-MC-Unique: Mlc6CGyKOZasa1MY3T_4qw-1 From: Michal Privoznik To: libvir-list@redhat.com Subject: [PATCH v1 06/10] capabilities: Expose NUMA memory side cache Date: Mon, 31 May 2021 10:36:07 +0200 Message-Id: In-Reply-To: References: MIME-Version: 1.0 X-Scanned-By: MIMEDefang 2.79 on 10.5.11.13 X-loop: libvir-list@redhat.com X-BeenThere: libvir-list@redhat.com X-Mailman-Version: 2.1.12 Precedence: junk List-Id: Development discussions about the libvirt library & tools List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Sender: libvir-list-bounces@redhat.com Errors-To: libvir-list-bounces@redhat.com X-Scanned-By: MIMEDefang 2.79 on 10.5.11.11 Authentication-Results: relay.mimecast.com; auth=pass smtp.auth=CUSA124A263 smtp.mailfrom=libvir-list-bounces@redhat.com X-Mimecast-Spam-Score: 0 X-Mimecast-Originator: redhat.com Content-Transfer-Encoding: quoted-printable X-ZohoMail-DKIM: pass (identity @redhat.com) Content-Type: text/plain; charset="utf-8" Memory on a NUMA node can have a side caches. Configuring these for a domain was implemented in v6.6.0-rc1~249 and friends. However, up until now mgmt applications did not really know what values to pass because we were not exposing caches of the host. With recent enough kernel these are exposed under sysfs and with a bit of parsing we can extend our capabilities XML. The sysfs structure is documented in kernel's Documentation/admin-guide/mm/numaperf.rst and basically maps in 1:1 fashion to our virNumaCache structure. Signed-off-by: Michal Privoznik --- docs/schemas/capability.rng | 4 + src/conf/capabilities.c | 149 ++++++++++++++++++++++++++++++++- src/conf/capabilities.h | 4 +- src/libxl/libxl_capabilities.c | 3 +- src/test/test_driver.c | 3 +- tests/testutils.c | 3 +- 6 files changed, 158 insertions(+), 8 deletions(-) diff --git a/docs/schemas/capability.rng b/docs/schemas/capability.rng index fcc262ae1a..5c1fb3607c 100644 --- a/docs/schemas/capability.rng +++ b/docs/schemas/capability.rng @@ -163,6 +163,10 @@ =20 + + + + diff --git a/src/conf/capabilities.c b/src/conf/capabilities.c index 9332daf6a6..7b60676070 100644 --- a/src/conf/capabilities.c +++ b/src/conf/capabilities.c @@ -121,6 +121,8 @@ virCapabilitiesFreeHostNUMACell(virCapsHostNUMACell *ce= ll) g_free(cell->cpus); g_free(cell->distances); g_free(cell->pageinfo); + if (cell->caches) + g_array_unref(cell->caches); g_free(cell); } =20 @@ -335,6 +337,7 @@ virCapabilitiesSetNetPrefix(virCaps *caps, * @distances: NUMA distances to other nodes * @npageinfo: number of pages at node @num * @pageinfo: info on each single memory page + * @caches: info on memory side caches * * Registers a new NUMA cell for a host, passing in a array of * CPU IDs belonging to the cell, distances to other NUMA nodes @@ -351,7 +354,8 @@ virCapabilitiesHostNUMAAddCell(virCapsHostNUMA *caps, int ndistances, virNumaDistance **distances, int npageinfo, - virCapsHostNUMACellPageInfo **pageinfo) + virCapsHostNUMACellPageInfo **pageinfo, + GArray **caches) { virCapsHostNUMACell *cell =3D g_new0(virCapsHostNUMACell, 1); =20 @@ -369,6 +373,9 @@ virCapabilitiesHostNUMAAddCell(virCapsHostNUMA *caps, cell->npageinfo =3D npageinfo; cell->pageinfo =3D g_steal_pointer(pageinfo); } + if (caches) { + cell->caches =3D g_steal_pointer(caches); + } =20 g_ptr_array_add(caps->cells, cell); } @@ -870,6 +877,11 @@ virCapabilitiesHostNUMAFormat(virBuffer *buf, =20 virNumaDistanceFormat(buf, cell->distances, cell->ndistances); =20 + if (cell->caches) { + virNumaCache *caches =3D &g_array_index(cell->caches, virNumaC= ache, 0); + virNumaCacheFormat(buf, caches, cell->caches->len); + } + if (virCapsHostNUMACellCPUFormat(buf, cell->cpus, cell->ncpus) < 0) return -1; =20 @@ -1535,6 +1547,129 @@ virCapabilitiesGetNUMAPagesInfo(int node, } =20 =20 +static int +virCapabilitiesGetNodeCacheReadFile(const char *prefix, + const char *dir, + const char *file, + unsigned int *value) +{ + g_autofree char *path =3D g_strdup_printf("%s/%s/%s", prefix, dir, fil= e); + int rv =3D virFileReadValueUint(value, "%s", path); + + if (rv < 0) { + if (rv =3D=3D -2) { + virReportError(VIR_ERR_INTERNAL_ERROR, + _("File '%s' does not exist"), + path); + } + return -1; + } + + return 0; +} + + +static int +virCapsHostNUMACellCacheComparator(const void *a, + const void *b) +{ + const virNumaCache *aa =3D a; + const virNumaCache *bb =3D b; + + return aa->level - bb->level; +} + + +static int +virCapabilitiesGetNodeCache(int node, + GArray **cachesRet) +{ + g_autoptr(DIR) dir =3D NULL; + int direrr =3D 0; + struct dirent *entry; + g_autofree char *path =3D NULL; + g_autoptr(GArray) caches =3D g_array_new(FALSE, FALSE, sizeof(virNumaC= ache)); + + path =3D g_strdup_printf(SYSFS_SYSTEM_PATH "/node/node%d/memory_side_c= ache", node); + + if (virDirOpenIfExists(&dir, path) < 0) + return -1; + + while (dir && (direrr =3D virDirRead(dir, &entry, path)) > 0) { + const char *dname =3D STRSKIP(entry->d_name, "index"); + virNumaCache cache =3D { 0 }; + unsigned int indexing; + unsigned int write_policy; + + if (!dname) + continue; + + if (virStrToLong_ui(dname, NULL, 10, &cache.level) < 0) { + virReportError(VIR_ERR_INTERNAL_ERROR, + _("unable to parse %s"), + entry->d_name); + return -1; + } + + if (virCapabilitiesGetNodeCacheReadFile(path, entry->d_name, + "size", &cache.size) < 0) + return -1; + + cache.size >>=3D 10; /* read in bytes but stored in kibibytes */ + + if (virCapabilitiesGetNodeCacheReadFile(path, entry->d_name, + "line_size", &cache.line) = < 0) + return -1; + + if (virCapabilitiesGetNodeCacheReadFile(path, entry->d_name, + "indexing", &indexing) < 0) + return -1; + + /* see enum cache_indexing in kernel */ + switch (indexing) { + case 0: cache.associativity =3D VIR_NUMA_CACHE_ASSOCIATIVITY_DIREC= T; break; + case 1: cache.associativity =3D VIR_NUMA_CACHE_ASSOCIATIVITY_FULL;= break; + case 2: cache.associativity =3D VIR_NUMA_CACHE_ASSOCIATIVITY_NONE;= break; + default: + virReportError(VIR_ERR_INTERNAL_ERROR, + _("unknown indexing value '%u'"), + indexing); + return -1; + } + + if (virCapabilitiesGetNodeCacheReadFile(path, entry->d_name, + "write_policy", &write_pol= icy) < 0) + return -1; + + /* see enum cache_write_policy in kernel */ + switch (write_policy) { + case 0: cache.policy =3D VIR_NUMA_CACHE_POLICY_WRITEBACK; break; + case 1: cache.policy =3D VIR_NUMA_CACHE_POLICY_WRITETHROUGH; break; + case 2: cache.policy =3D VIR_NUMA_CACHE_POLICY_NONE; break; + default: + virReportError(VIR_ERR_INTERNAL_ERROR, + _("unknown write_policy value '%u'"), + write_policy); + return -1; + } + + g_array_append_val(caches, cache); + } + + if (direrr < 0) + return -1; + + if (caches->len > 0) { + g_array_sort(caches, virCapsHostNUMACellCacheComparator); + *cachesRet =3D g_steal_pointer(&caches); + } else { + *cachesRet =3D NULL; + } + + return 0; +} + + static int virCapabilitiesHostNUMAInitFake(virCapsHostNUMA *caps) { @@ -1586,7 +1721,8 @@ virCapabilitiesHostNUMAInitFake(virCapsHostNUMA *caps) nodeinfo.memory, cid, &cpus, 0, NULL, - 0, NULL); + 0, NULL, + NULL); } =20 return 0; @@ -1616,8 +1752,9 @@ virCapabilitiesHostNUMAInitReal(virCapsHostNUMA *caps) g_autofree virNumaDistance *distances =3D NULL; int ndistances =3D 0; g_autofree virCapsHostNUMACellPageInfo *pageinfo =3D NULL; - int npageinfo; + int npageinfo =3D 0; unsigned long long memory; + g_autoptr(GArray) caches =3D NULL; int cpu; size_t i; =20 @@ -1644,6 +1781,9 @@ virCapabilitiesHostNUMAInitReal(virCapsHostNUMA *caps) if (virCapabilitiesGetNUMAPagesInfo(n, &pageinfo, &npageinfo) < 0) goto cleanup; =20 + if (virCapabilitiesGetNodeCache(n, &caches) < 0) + goto cleanup; + /* Detect the amount of memory in the numa cell in KiB */ virNumaGetNodeMemory(n, &memory, NULL); memory >>=3D 10; @@ -1651,7 +1791,8 @@ virCapabilitiesHostNUMAInitReal(virCapsHostNUMA *caps) virCapabilitiesHostNUMAAddCell(caps, n, memory, ncpus, &cpus, ndistances, &distances, - npageinfo, &pageinfo); + npageinfo, &pageinfo, + &caches); } =20 ret =3D 0; diff --git a/src/conf/capabilities.h b/src/conf/capabilities.h index 4d4ac476ea..334b361e7a 100644 --- a/src/conf/capabilities.h +++ b/src/conf/capabilities.h @@ -108,6 +108,7 @@ struct _virCapsHostNUMACell { virNumaDistance *distances; int npageinfo; virCapsHostNUMACellPageInfo *pageinfo; + GArray *caches; /* virNumaCache */ }; =20 struct _virCapsHostNUMA { @@ -253,7 +254,8 @@ virCapabilitiesHostNUMAAddCell(virCapsHostNUMA *caps, int ndistances, virNumaDistance **distances, int npageinfo, - virCapsHostNUMACellPageInfo **pageinfo); + virCapsHostNUMACellPageInfo **pageinfo, + GArray **caches); =20 virCapsGuestMachine ** virCapabilitiesAllocMachines(const char *const *names, diff --git a/src/libxl/libxl_capabilities.c b/src/libxl/libxl_capabilities.c index b8600ca479..7385ad0d38 100644 --- a/src/libxl/libxl_capabilities.c +++ b/src/libxl/libxl_capabilities.c @@ -332,7 +332,8 @@ libxlCapsInitNuma(libxl_ctx *ctx, virCaps *caps) numa_info[i].size / 1024, nr_cpus_node[i], &cpus[i], nr_distances, &distances, - 0, NULL); + 0, NULL, + NULL); =20 /* This is safe, as the CPU list is now stored in the NUMA cell */ cpus[i] =3D NULL; diff --git a/src/test/test_driver.c b/src/test/test_driver.c index ea5a5005e7..1b36c67eb2 100644 --- a/src/test/test_driver.c +++ b/src/test/test_driver.c @@ -331,7 +331,8 @@ testBuildCapabilities(virConnectPtr conn) i, privconn->cells[i].mem, privconn->cells[i].numCpus, &cpu_ce= lls, 0, NULL, - nPages, &pages); + nPages, &pages, + NULL); } =20 for (i =3D 0; i < G_N_ELEMENTS(guest_types); i++) { diff --git a/tests/testutils.c b/tests/testutils.c index eb3bd48b6a..7d87e30a5c 100644 --- a/tests/testutils.c +++ b/tests/testutils.c @@ -947,7 +947,8 @@ virTestCapsBuildNUMATopology(int seq) MAX_MEM_IN_CELL, MAX_CPUS_IN_CELL, &cell_cpus, 0, NULL, - 0, NULL); + 0, NULL, + NULL); =20 cell_cpus =3D NULL; } --=20 2.31.1