From 5f6723e71e3765d1d43bfa9ba1c66e0e05e11a48 Mon Sep 17 00:00:00 2001

Message-Id: <5f6723e71e3765d1d43bfa9ba1c66e0e05e11a48@dist-git>

From: Michal Privoznik <mprivozn@redhat.com>

Date: Mon, 9 Nov 2020 17:22:32 +0100

Subject: [PATCH] Allow NUMA nodes without vCPUs

MIME-Version: 1.0

Content-Type: text/plain; charset=UTF-8

Content-Transfer-Encoding: 8bit

QEMU allows creating NUMA nodes that have memory only.

These are somehow important for HMAT.

With check done in qemuValidateDomainDef() for QEMU 2.7 or newer

(checked via QEMU_CAPS_NUMA), we can be sure that the vCPUs are

fully assigned to NUMA nodes in domain XML.

Signed-off-by: Michal Privoznik <mprivozn@redhat.com>

Reviewed-by: Daniel Henrique Barboza <danielhb413@gmail.com>

(cherry picked from commit a26f61ee0cffa421b87ef568002b684dd8025432)

Resolves: https://bugzilla.redhat.com/show_bug.cgi?id=1749518

Conflicts:

- src/qemu/qemu_validate.c: This file doesn't exist in downstream

yet, so I've moved the change that original patch would do to

qemu_domain.c where the validator lives.

Signed-off-by: Michal Privoznik <mprivozn@redhat.com>

Message-Id: <365508c75e579e9037ad555d6c372068ccd50c95.1604938867.git.mprivozn@redhat.com>

Reviewed-by: Ján Tomko <jtomko@redhat.com>

---

 docs/formatdomain.html.in                     |  2 +

 docs/schemas/cputypes.rng                     |  8 ++-

 src/conf/numa_conf.c                          | 59 ++++++++++---------

 src/libxl/xen_xl.c                            | 10 ++--

 src/qemu/qemu_command.c                       | 26 ++++----

 src/qemu/qemu_domain.c                        | 22 +++----

 tests/qemuxml2argvdata/numatune-no-vcpu.args  | 33 +++++++++++

 tests/qemuxml2argvdata/numatune-no-vcpu.xml   | 42 +++++++++++++

 tests/qemuxml2argvtest.c                      |  1 +

 tests/qemuxml2xmloutdata/numatune-no-vcpu.xml |  1 +

 tests/qemuxml2xmltest.c                       |  1 +

 11 files changed, 149 insertions(+), 56 deletions(-)

 create mode 100644 tests/qemuxml2argvdata/numatune-no-vcpu.args

 create mode 100644 tests/qemuxml2argvdata/numatune-no-vcpu.xml

 create mode 120000 tests/qemuxml2xmloutdata/numatune-no-vcpu.xml

diff --git a/docs/formatdomain.html.in b/docs/formatdomain.html.in

index 76799f5ffc..4b8d312596 100644

--- a/docs/formatdomain.html.in

+++ b/docs/formatdomain.html.in

@@ -1783,6 +1783,8 @@

       cpus specifies the CPU or range of CPUs that are

       part of the node. memory specifies the node memory

       in kibibytes (i.e. blocks of 1024 bytes).

+      Since 6.6.0 the cpus attribute

+      is optional and if omitted a CPU-less NUMA node is created.

       Since 1.2.11 one can use an additional 

           href="#elementsMemoryAllocation">unit attribute to

       define units in which memory is specified.

diff --git a/docs/schemas/cputypes.rng b/docs/schemas/cputypes.rng

index e2744acad3..a1682a1003 100644

--- a/docs/schemas/cputypes.rng

+++ b/docs/schemas/cputypes.rng

@@ -115,9 +115,11 @@

           <ref name="unsignedInt"/>

         </attribute>

       </optional>

-      <attribute name="cpus">

-        <ref name="cpuset"/>

-      </attribute>

+      <optional>

+        <attribute name="cpus">

+          <ref name="cpuset"/>

+        </attribute>

+      </optional>

       <attribute name="memory">

         <ref name="memoryKB"/>

       </attribute>

diff --git a/src/conf/numa_conf.c b/src/conf/numa_conf.c

index c9cc8ac22e..a805336d16 100644

--- a/src/conf/numa_conf.c

+++ b/src/conf/numa_conf.c

@@ -889,32 +889,28 @@ virDomainNumaDefParseXML(virDomainNumaPtr def,

         }

         VIR_FREE(tmp);

-        if (def->mem_nodes[cur_cell].cpumask) {

+        if (def->mem_nodes[cur_cell].mem) {

             virReportError(VIR_ERR_XML_ERROR,

                            _("Duplicate NUMA cell info for cell id '%u'"),

                            cur_cell);

             goto cleanup;

         }

-        if (!(tmp = virXMLPropString(nodes[i], "cpus"))) {

-            virReportError(VIR_ERR_XML_ERROR, "%s",

-                           _("Missing 'cpus' attribute in NUMA cell"));

-            goto cleanup;

-        }

+        if ((tmp = virXMLPropString(nodes[i], "cpus"))) {

+            g_autoptr(virBitmap) cpumask = NULL;

-        if (virBitmapParse(tmp, &def->mem_nodes[cur_cell].cpumask,

-                           VIR_DOMAIN_CPUMASK_LEN) < 0)

-            goto cleanup;

+            if (virBitmapParse(tmp, &cpumask, VIR_DOMAIN_CPUMASK_LEN) < 0)

+                goto cleanup;

-        if (virBitmapIsAllClear(def->mem_nodes[cur_cell].cpumask)) {

-            virReportError(VIR_ERR_CONFIG_UNSUPPORTED,

-                          _("NUMA cell %d has no vCPUs assigned"), cur_cell);

-            goto cleanup;

+            if (!virBitmapIsAllClear(cpumask))

+                def->mem_nodes[cur_cell].cpumask = g_steal_pointer(&cpumask);

+            VIR_FREE(tmp);

         }

-        VIR_FREE(tmp);

         for (j = 0; j < n; j++) {

-            if (j == cur_cell || !def->mem_nodes[j].cpumask)

+            if (j == cur_cell ||

+                !def->mem_nodes[j].cpumask ||

+                !def->mem_nodes[cur_cell].cpumask)

                 continue;

             if (virBitmapOverlaps(def->mem_nodes[j].cpumask,

@@ -976,7 +972,6 @@ virDomainNumaDefFormatXML(virBufferPtr buf,

 {

     virDomainMemoryAccess memAccess;

     virTristateBool discard;

-    char *cpustr;

     size_t ncells = virDomainNumaGetNodeCount(def);

     size_t i;

@@ -986,17 +981,22 @@ virDomainNumaDefFormatXML(virBufferPtr buf,

     virBufferAddLit(buf, "<numa>\n");

     virBufferAdjustIndent(buf, 2);

     for (i = 0; i < ncells; i++) {

+        virBitmapPtr cpumask = virDomainNumaGetNodeCpumask(def, i);

         int ndistances;

         memAccess = virDomainNumaGetNodeMemoryAccessMode(def, i);

         discard = virDomainNumaGetNodeDiscard(def, i);

-        if (!(cpustr = virBitmapFormat(virDomainNumaGetNodeCpumask(def, i))))

-            return -1;

-

         virBufferAddLit(buf, "

         virBufferAsprintf(buf, " id='%zu'", i);

-        virBufferAsprintf(buf, " cpus='%s'", cpustr);

+

+        if (cpumask) {

+            g_autofree char *cpustr = virBitmapFormat(cpumask);

+

+            if (!cpustr)

+                return -1;

+            virBufferAsprintf(buf, " cpus='%s'", cpustr);

+        }

         virBufferAsprintf(buf, " memory='%llu'",

                           virDomainNumaGetNodeMemorySize(def, i));

         virBufferAddLit(buf, " unit='KiB'");

@@ -1032,8 +1032,6 @@ virDomainNumaDefFormatXML(virBufferPtr buf,

             virBufferAdjustIndent(buf, -2);

             virBufferAddLit(buf, "</cell>\n");

         }

-

-        VIR_FREE(cpustr);

     }

     virBufferAdjustIndent(buf, -2);

     virBufferAddLit(buf, "</numa>\n");

@@ -1048,8 +1046,12 @@ virDomainNumaGetCPUCountTotal(virDomainNumaPtr numa)

     size_t i;

     unsigned int ret = 0;

-    for (i = 0; i < numa->nmem_nodes; i++)

-        ret += virBitmapCountBits(virDomainNumaGetNodeCpumask(numa, i));

+    for (i = 0; i < numa->nmem_nodes; i++) {

+        virBitmapPtr cpumask = virDomainNumaGetNodeCpumask(numa, i);

+

+        if (cpumask)

+            ret += virBitmapCountBits(cpumask);

+    }

     return ret;

 }

@@ -1061,11 +1063,14 @@ virDomainNumaGetMaxCPUID(virDomainNumaPtr numa)

     unsigned int ret = 0;

     for (i = 0; i < numa->nmem_nodes; i++) {

+        virBitmapPtr cpumask = virDomainNumaGetNodeCpumask(numa, i);

         int bit;

-        bit = virBitmapLastSetBit(virDomainNumaGetNodeCpumask(numa, i));

-        if (bit > ret)

-            ret = bit;

+        if (cpumask) {

+            bit = virBitmapLastSetBit(cpumask);

+            if (bit > ret)

+                ret = bit;

+        }

     }

     return ret;

diff --git a/src/libxl/xen_xl.c b/src/libxl/xen_xl.c

index edea30a86a..752fa925ec 100644

--- a/src/libxl/xen_xl.c

+++ b/src/libxl/xen_xl.c

@@ -1443,19 +1443,21 @@ xenFormatXLVnuma(virConfValuePtr list,

 {

     int ret = -1;

     size_t i;

-

     virBuffer buf = VIR_BUFFER_INITIALIZER;

     virConfValuePtr numaVnode, tmp;

-

+    virBitmapPtr cpumask = virDomainNumaGetNodeCpumask(numa, node);

     size_t nodeSize = virDomainNumaGetNodeMemorySize(numa, node) / 1024;

-    char *nodeVcpus = virBitmapFormat(virDomainNumaGetNodeCpumask(numa, node));

+    g_autofree char *nodeVcpus = NULL;

-    if (VIR_ALLOC(numaVnode) < 0)

+    if (!cpumask ||

+        VIR_ALLOC(numaVnode) < 0)

         goto cleanup;

     numaVnode->type = VIR_CONF_LIST;

     numaVnode->list = NULL;

+    nodeVcpus = virBitmapFormat(cpumask);

+

     /* pnode */

     virBufferAsprintf(&buf, "pnode=%zu", node);

     xenFormatXLVnode(numaVnode, &buf;;

diff --git a/src/qemu/qemu_command.c b/src/qemu/qemu_command.c

index 1a573c2817..ac63d18a42 100644

--- a/src/qemu/qemu_command.c

+++ b/src/qemu/qemu_command.c

@@ -7364,8 +7364,6 @@ qemuBuildNumaCommandLine(virQEMUDriverConfigPtr cfg,

     size_t i, j;

     virQEMUCapsPtr qemuCaps = priv->qemuCaps;

     g_auto(virBuffer) buf = VIR_BUFFER_INITIALIZER;

-    char *cpumask = NULL;

-    char *tmpmask = NULL;

     char *next = NULL;

     virBufferPtr nodeBackends = NULL;

     bool needBackend = false;

@@ -7400,9 +7398,7 @@ qemuBuildNumaCommandLine(virQEMUDriverConfigPtr cfg,

         goto cleanup;

     for (i = 0; i < ncells; i++) {

-        VIR_FREE(cpumask);

-        if (!(cpumask = virBitmapFormat(virDomainNumaGetNodeCpumask(def->numa, i))))

-            goto cleanup;

+        virBitmapPtr cpumask = virDomainNumaGetNodeCpumask(def->numa, i);

         if (needBackend) {

             virCommandAddArg(cmd, "-object");

@@ -7412,11 +7408,19 @@ qemuBuildNumaCommandLine(virQEMUDriverConfigPtr cfg,

         virCommandAddArg(cmd, "-numa");

         virBufferAsprintf(&buf, "node,nodeid=%zu", i);

-        for (tmpmask = cpumask; tmpmask; tmpmask = next) {

-            if ((next = strchr(tmpmask, ',')))

-                *(next++) = '\0';

-            virBufferAddLit(&buf, ",cpus=");

-            virBufferAdd(&buf, tmpmask, -1);

+        if (cpumask) {

+            g_autofree char *cpumaskStr = NULL;

+            char *tmpmask;

+

+            if (!(cpumaskStr = virBitmapFormat(cpumask)))

+                goto cleanup;

+

+            for (tmpmask = cpumaskStr; tmpmask; tmpmask = next) {

+                if ((next = strchr(tmpmask, ',')))

+                    *(next++) = '\0';

+                virBufferAddLit(&buf, ",cpus=");

+                virBufferAdd(&buf, tmpmask, -1);

+            }

         }

         if (needBackend)

@@ -7447,8 +7451,6 @@ qemuBuildNumaCommandLine(virQEMUDriverConfigPtr cfg,

     ret = 0;

  cleanup:

-    VIR_FREE(cpumask);

-

     if (nodeBackends) {

         for (i = 0; i < ncells; i++)

             virBufferFreeAndReset(&nodeBackends[i]);

diff --git a/src/qemu/qemu_domain.c b/src/qemu/qemu_domain.c

index 35b536868a..be25790f12 100644

--- a/src/qemu/qemu_domain.c

+++ b/src/qemu/qemu_domain.c

@@ -5373,7 +5373,7 @@ qemuDomainDefValidateNuma(const virDomainDef *def,

     }

     for (i = 0; i < ncells; i++) {

-        g_autofree char * cpumask = NULL;

+        virBitmapPtr cpumask = virDomainNumaGetNodeCpumask(def->numa, i);

         if (!hasMemoryCap &&

             virDomainNumaGetNodeMemoryAccessMode(def->numa, i)) {

@@ -5383,17 +5383,19 @@ qemuDomainDefValidateNuma(const virDomainDef *def,

             return -1;

         }

-        if (!(cpumask = virBitmapFormat(virDomainNumaGetNodeCpumask(def->numa, i))))

-            return -1;

+        if (cpumask) {

+            g_autofree char * cpumaskStr = NULL;

+            if (!(cpumaskStr = virBitmapFormat(cpumask)))

+                return -1;

-        if (strchr(cpumask, ',') &&

-            !virQEMUCapsGet(qemuCaps, QEMU_CAPS_NUMA)) {

-            virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",

-                           _("disjoint NUMA cpu ranges are not supported "

-                             "with this QEMU"));

-            return -1;

+            if (strchr(cpumaskStr, ',') &&

+                !virQEMUCapsGet(qemuCaps, QEMU_CAPS_NUMA)) {

+                virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",

+                               _("disjoint NUMA cpu ranges are not supported "

+                                 "with this QEMU"));

+                return -1;

+            }

         }

-

     }

     if (virDomainNumaNodesDistancesAreBeingSet(def->numa) &&

diff --git a/tests/qemuxml2argvdata/numatune-no-vcpu.args b/tests/qemuxml2argvdata/numatune-no-vcpu.args

new file mode 100644

index 0000000000..a1f1ee044e

--- /dev/null

+++ b/tests/qemuxml2argvdata/numatune-no-vcpu.args

@@ -0,0 +1,33 @@

+LC_ALL=C \

+PATH=/bin \

+HOME=/tmp/lib/domain--1-QEMUGuest \

+USER=test \

+LOGNAME=test \

+XDG_DATA_HOME=/tmp/lib/domain--1-QEMUGuest/.local/share \

+XDG_CACHE_HOME=/tmp/lib/domain--1-QEMUGuest/.cache \

+XDG_CONFIG_HOME=/tmp/lib/domain--1-QEMUGuest/.config \

+QEMU_AUDIO_DRV=none \

+/usr/bin/qemu-system-x86_64 \

+-name QEMUGuest \

+-S \

+-machine pc,accel=tcg,usb=off,dump-guest-core=off \

+-m 12288 \

+-realtime mlock=off \

+-smp 12,sockets=12,cores=1,threads=1 \

+-numa node,nodeid=0,cpus=0-3,mem=2048 \

+-numa node,nodeid=1,cpus=4-7,mem=2048 \

+-numa node,nodeid=2,cpus=8-11,mem=2048 \

+-numa node,nodeid=3,mem=2048 \

+-numa node,nodeid=4,mem=2048 \

+-numa node,nodeid=5,mem=2048 \

+-uuid c7a5fdb2-cdaf-9455-926a-d65c16db1809 \

+-display none \

+-no-user-config \

+-nodefaults \

+-chardev socket,id=charmonitor,path=/tmp/lib/domain--1-QEMUGuest/monitor.sock,\

+server,nowait \

+-mon chardev=charmonitor,id=monitor,mode=control \

+-rtc base=utc \

+-no-shutdown \

+-usb \

+-device virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3

diff --git a/tests/qemuxml2argvdata/numatune-no-vcpu.xml b/tests/qemuxml2argvdata/numatune-no-vcpu.xml

new file mode 100644

index 0000000000..f25a07d7ed

--- /dev/null

+++ b/tests/qemuxml2argvdata/numatune-no-vcpu.xml

@@ -0,0 +1,42 @@

+<domain type='qemu'>

+  <name>QEMUGuest</name>

+  <uuid>c7a5fdb2-cdaf-9455-926a-d65c16db1809</uuid>

+  <memory unit='KiB'>12582912</memory>

+  <currentMemory unit='KiB'>12582912</currentMemory>

+  <vcpu placement='static'>12</vcpu>

+  <os>

+    <type arch='x86_64' machine='pc'>hvm</type>

+    <boot dev='hd'/>

+  </os>

+  <features>

+    <acpi/>

+    <apic/>

+    <pae/>

+  </features>

+  <cpu>

+    <numa>

+      <cell id='0' cpus='0-3' memory='2097152' unit='KiB'/>

+      <cell id='1' cpus='4-7' memory='2097152' unit='KiB'/>

+      <cell id='2' cpus='8-11' memory='2097152' unit='KiB'/>

+      <cell id='3' memory='2097152' unit='KiB'/>

+      <cell id='4' memory='2097152' unit='KiB'/>

+      <cell id='5' memory='2097152' unit='KiB'/>

+    </numa>

+  </cpu>

+  <clock offset='utc'/>

+  <on_poweroff>destroy</on_poweroff>

+  <on_reboot>restart</on_reboot>

+  <on_crash>restart</on_crash>

+  <devices>

+    <emulator>/usr/bin/qemu-system-x86_64</emulator>

+    <controller type='usb' index='0'>

+      <address type='pci' domain='0x0000' bus='0x00' slot='0x01' function='0x2'/>

+    </controller>

+    <controller type='pci' index='0' model='pci-root'/>

+    <input type='mouse' bus='ps2'/>

+    <input type='keyboard' bus='ps2'/>

+    <memballoon model='virtio'>

+      <address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x0'/>

+    </memballoon>

+  </devices>

+</domain>

diff --git a/tests/qemuxml2argvtest.c b/tests/qemuxml2argvtest.c

index ff92af606d..49699e495d 100644

--- a/tests/qemuxml2argvtest.c

+++ b/tests/qemuxml2argvtest.c

@@ -1812,6 +1812,7 @@ mymain(void)

     DO_TEST_PARSE_ERROR("numatune-memnode-no-memory", NONE);

     DO_TEST("numatune-distances", QEMU_CAPS_NUMA, QEMU_CAPS_NUMA_DIST);

+    DO_TEST("numatune-no-vcpu", NONE);

     DO_TEST("numatune-auto-nodeset-invalid", NONE);

     DO_TEST("numatune-auto-prefer", QEMU_CAPS_OBJECT_MEMORY_RAM,

diff --git a/tests/qemuxml2xmloutdata/numatune-no-vcpu.xml b/tests/qemuxml2xmloutdata/numatune-no-vcpu.xml

new file mode 120000

index 0000000000..f213032685

--- /dev/null

+++ b/tests/qemuxml2xmloutdata/numatune-no-vcpu.xml

@@ -0,0 +1 @@

+../qemuxml2argvdata/numatune-no-vcpu.xml

\ No newline at end of file

diff --git a/tests/qemuxml2xmltest.c b/tests/qemuxml2xmltest.c

index 6c3f5c4a9e..1ddeba30f0 100644

--- a/tests/qemuxml2xmltest.c

+++ b/tests/qemuxml2xmltest.c

@@ -1105,6 +1105,7 @@ mymain(void)

     DO_TEST("numatune-memnode", QEMU_CAPS_NUMA, QEMU_CAPS_OBJECT_MEMORY_FILE);

     DO_TEST("numatune-memnode-no-memory", QEMU_CAPS_OBJECT_MEMORY_FILE);

     DO_TEST("numatune-distances", QEMU_CAPS_NUMA, QEMU_CAPS_NUMA_DIST);

+    DO_TEST("numatune-no-vcpu", QEMU_CAPS_NUMA);

     DO_TEST("bios-nvram", NONE);

     DO_TEST("bios-nvram-os-interleave", NONE);

-- 

2.29.2