From c9fb2f1ef7a756909c705df0c6a323af826be139 Mon Sep 17 00:00:00 2001

Message-Id: <c9fb2f1ef7a756909c705df0c6a323af826be139@dist-git>

From: Peter Krempa <pkrempa@redhat.com>

Date: Wed, 24 Aug 2016 16:11:33 -0400

Subject: [PATCH] conf: Add XML for individual vCPU hotplug

https://bugzilla.redhat.com/show_bug.cgi?id=1097930

https://bugzilla.redhat.com/show_bug.cgi?id=1224341

Individual vCPU hotplug requires us to track the state of any vCPU. To

allow this add the following XML:

<domain>

  ...

  <vcpu current='2'>3</vcpu>

  <vcpus>

    <vcpu id='0' enabled='yes' hotpluggable='no' order='1'/>

    <vcpu id='1' enabled='yes' hotpluggable='yes' order='2'/>

    <vcpu id='1' enabled='no' hotpluggable='yes'/>

  </vcpus>

  ...

The 'enabled' attribute allows to control the state of the vcpu.

'hotpluggable' controls whether given vcpu can be hotplugged and 'order'

allows to specify the order to add the vcpus.

(cherry picked from commit 5847bc5c64f205a2f00315a2434e8a7bd48c5508)

---

 docs/formatdomain.html.in                          |  40 ++++++

 docs/schemas/domaincommon.rng                      |  25 ++++

 src/conf/domain_conf.c                             | 154 ++++++++++++++++++++-

 src/conf/domain_conf.h                             |   6 +

 src/qemu/qemu_domain.c                             |  11 +-

 .../generic-vcpus-individual.xml                   |  23 +++

 tests/genericxml2xmltest.c                         |   2 +

 tests/testutils.c                                  |   4 +-

 8 files changed, 261 insertions(+), 4 deletions(-)

 create mode 100644 tests/genericxml2xmlindata/generic-vcpus-individual.xml

diff --git a/docs/formatdomain.html.in b/docs/formatdomain.html.in

index b74057f..252f91c 100644

--- a/docs/formatdomain.html.in

+++ b/docs/formatdomain.html.in

@@ -489,6 +489,10 @@

 <domain>

   ...

   <vcpu placement='static' cpuset="1-4,^3,6" current="1">2</vcpu>

+  <vcpus>

+    <vcpu id='0' enabled='yes' hotpluggable='no' order='1'/>

+    <vcpu id='1' enabled='no' hotpluggable='yes'/>

+  </vcpus>

   ...

 </domain>

@@ -542,6 +546,42 @@

+      
vcpus

+      

+        The vcpus element allows to control state of individual vcpus.

+

+        The id attribute specifies the vCPU id as used by libvirt

+        in other places such as vcpu pinning, scheduler information and NUMA

+        assignment. Note that the vcpu ID as seen in the guest may differ from

+        libvirt ID in certain cases. Valid IDs are from 0 to the maximum vcpu

+        count as set by the vcpu element minus 1.

+

+        The enabled attribute allows to control the state of the

+        vcpu. Valid values are yes and no.

+

+        hotpluggable controls whether given vcpu can be hotplugged

+        and hotunplugged in cases when the cpu is enabled at boot. Note that

+        all disabled vcpus must be hotpluggable. Valid values are

+        yes and no.

+

+        order allows to specify the order to add the vcpus. For

+        hypervisors/platforms that require to insert multiple vcpus at once

+        the order may be be duplicated accross all vcpus that need to be

+        enabled at once. Specifying order is not necessary, vcpus are then

+        added in an arbitrary order.

+

+        Note that hypervisors may create hotpluggable vcpus differently from

+        boot vcpus thus special initialization may be necessary.

+

+        Hypervisors may require that vcpus enabled on boot which are not

+        hotpluggable are clustered at the beginning starting with ID 0. It may

+        be also required that vcpu 0 is always present and non-hotpluggable.

+

+        Note that providing state for individual cpus may be necessary to enable

+        support of addressable vCPU hotplug and this feature may not be

+        supported by all hypervisors.

+        Since 2.2.0 (QEMU only)

+      

     
IOThreads Allocation

diff --git a/docs/schemas/domaincommon.rng b/docs/schemas/domaincommon.rng

index 497597c..bf4d795 100644

--- a/docs/schemas/domaincommon.rng

+++ b/docs/schemas/domaincommon.rng

@@ -583,6 +583,31 @@

       </optional>

       <optional>

+        <element name="vcpus">

+          <zeroOrMore>

+            <element name="vcpu">

+              <attribute name="id">

+                <ref name="unsignedInt"/>

+              </attribute>

+              <attribute name="enabled">

+                <ref name="virYesNo"/>

+              </attribute>

+              <optional>

+                <attribute name="hotpluggable">

+                  <ref name="virYesNo"/>

+                </attribute>

+              </optional>

+              <optional>

+                <attribute name="order">

+                  <ref name="unsignedInt"/>

+                </attribute>

+              </optional>

+            </element>

+          </zeroOrMore>

+        </element>

+      </optional>

+

+      <optional>

         <element name="iothreads">

           <ref name="unsignedInt"/>

         </element>

diff --git a/src/conf/domain_conf.c b/src/conf/domain_conf.c

index 4e703d9..d3a0400 100644

--- a/src/conf/domain_conf.c

+++ b/src/conf/domain_conf.c

@@ -4425,6 +4425,13 @@ virDomainDefPostParseCheckFeatures(virDomainDefPtr def,

         }

     }

+    if (UNSUPPORTED(VIR_DOMAIN_DEF_FEATURE_INDIVIDUAL_VCPUS) &&

+        def->individualvcpus) {

+        virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",

+                       _("individual CPU state configuration is not supported"));

+        return -1;

+    }

+

     return 0;

 }

@@ -4499,6 +4506,43 @@ virDomainDefPostParseDeviceIterator(virDomainDefPtr def,

 static int

+virDomainVcpuDefPostParse(virDomainDefPtr def)

+{

+    virDomainVcpuDefPtr vcpu;

+    size_t maxvcpus = virDomainDefGetVcpusMax(def);

+    size_t i;

+

+    for (i = 0; i < maxvcpus; i++) {

+        vcpu = virDomainDefGetVcpu(def, i);

+

+        switch (vcpu->hotpluggable) {

+        case VIR_TRISTATE_BOOL_ABSENT:

+            if (vcpu->online)

+                vcpu->hotpluggable = VIR_TRISTATE_BOOL_NO;

+            else

+                vcpu->hotpluggable = VIR_TRISTATE_BOOL_YES;

+            break;

+

+        case VIR_TRISTATE_BOOL_NO:

+            if (!vcpu->online) {

+                virReportError(VIR_ERR_CONFIG_UNSUPPORTED,

+                               _("vcpu '%zu' is both offline and not "

+                                 "hotpluggable"), i);

+                return -1;

+            }

+            break;

+

+        case VIR_TRISTATE_BOOL_YES:

+        case VIR_TRISTATE_BOOL_LAST:

+            break;

+        }

+    }

+

+    return 0;

+}

+

+

+static int

 virDomainDefPostParseInternal(virDomainDefPtr def,

                               struct virDomainDefPostParseDeviceIteratorData *data)

 {

@@ -4509,6 +4553,9 @@ virDomainDefPostParseInternal(virDomainDefPtr def,

         return -1;

     }

+    if (virDomainVcpuDefPostParse(def) < 0)

+        return -1;

+

     if (virDomainDefPostParseMemory(def, data->parseFlags) < 0)

         return -1;

@@ -15590,6 +15637,8 @@ virDomainVcpuParse(virDomainDefPtr def,

                    virDomainXMLOptionPtr xmlopt)

 {

     int n;

+    xmlNodePtr *nodes = NULL;

+    size_t i;

     char *tmp = NULL;

     unsigned int maxvcpus;

     unsigned int vcpus;

@@ -15618,8 +15667,6 @@ virDomainVcpuParse(virDomainDefPtr def,

         vcpus = maxvcpus;

     }

-    if (virDomainDefSetVcpus(def, vcpus) < 0)

-        goto cleanup;

     tmp = virXPathString("string(./vcpu[1]/@placement)", ctxt);

     if (tmp) {

@@ -15651,9 +15698,82 @@ virDomainVcpuParse(virDomainDefPtr def,

         }

     }

+    if ((n = virXPathNodeSet("./vcpus/vcpu", ctxt, &nodes)) < 0)

+        goto cleanup;

+

+    if (n) {

+        /* if individual vcpu states are provided take them as master */

+        def->individualvcpus = true;

+

+        for (i = 0; i < n; i++) {

+            virDomainVcpuDefPtr vcpu;

+            int state;

+            unsigned int id;

+            unsigned int order;

+

+            if (!(tmp = virXMLPropString(nodes[i], "id")) ||

+                virStrToLong_uip(tmp, NULL, 10, &id) < 0) {

+                virReportError(VIR_ERR_XML_ERROR, "%s",

+                               _("missing or invalid vcpu id"));

+                goto cleanup;

+            }

+

+            VIR_FREE(tmp);

+

+            if (id >= def->maxvcpus) {

+                virReportError(VIR_ERR_XML_ERROR,

+                               _("vcpu id '%u' is out of range of maximum "

+                                 "vcpu count"), id);

+                goto cleanup;

+            }

+

+            vcpu = virDomainDefGetVcpu(def, id);

+

+            if (!(tmp = virXMLPropString(nodes[i], "enabled"))) {

+                virReportError(VIR_ERR_XML_ERROR, "%s",

+                               _("missing vcpu enabled state"));

+                goto cleanup;

+            }

+

+            if ((state = virTristateBoolTypeFromString(tmp)) < 0) {

+                virReportError(VIR_ERR_XML_ERROR,

+                               _("invalid vcpu 'enabled' value '%s'"), tmp);

+                goto cleanup;

+            }

+            VIR_FREE(tmp);

+

+            vcpu->online = state == VIR_TRISTATE_BOOL_YES;

+

+            if ((tmp = virXMLPropString(nodes[i], "hotpluggable"))) {

+                int hotpluggable;

+                if ((hotpluggable = virTristateBoolTypeFromString(tmp)) < 0) {

+                    virReportError(VIR_ERR_XML_ERROR,

+                                   _("invalid vcpu 'hotpluggable' value '%s'"), tmp);

+                    goto cleanup;

+                }

+                vcpu->hotpluggable = hotpluggable;

+                VIR_FREE(tmp);

+            }

+

+            if ((tmp = virXMLPropString(nodes[i], "order"))) {

+                if (virStrToLong_uip(tmp, NULL, 10, &order) < 0) {

+                    virReportError(VIR_ERR_XML_ERROR, "%s",

+                                   _("invalid vcpu order"));

+                    goto cleanup;

+                }

+                vcpu->order = order;

+                VIR_FREE(tmp);

+            }

+        }

+    } else {

+        if (virDomainDefSetVcpus(def, vcpus) < 0)

+            goto cleanup;

+    }

+

     ret = 0;

  cleanup:

+    VIR_FREE(nodes);

     VIR_FREE(tmp);

     return ret;

@@ -18679,6 +18799,13 @@ virDomainDefVcpuCheckAbiStability(virDomainDefPtr src,

                              "destination definitions"), i);

             return false;

         }

+

+        if (svcpu->order != dvcpu->order) {

+            virReportError(VIR_ERR_CONFIG_UNSUPPORTED,

+                           _("vcpu enable order of vCPU '%zu' differs between "

+                             "source and destination definitions"), i);

+            return false;

+        }

     }

     return true;

@@ -22921,6 +23048,8 @@ static int

 virDomainCpuDefFormat(virBufferPtr buf,

                       const virDomainDef *def)

 {

+    virDomainVcpuDefPtr vcpu;

+    size_t i;

     char *cpumask = NULL;

     int ret = -1;

@@ -22937,6 +23066,27 @@ virDomainCpuDefFormat(virBufferPtr buf,

         virBufferAsprintf(buf, " current='%u'", virDomainDefGetVcpus(def));

     virBufferAsprintf(buf, ">%u</vcpu>\n", virDomainDefGetVcpusMax(def));

+    if (def->individualvcpus) {

+        virBufferAddLit(buf, "<vcpus>\n");

+        virBufferAdjustIndent(buf, 2);

+        for (i = 0; i < def->maxvcpus; i++) {

+            vcpu = def->vcpus[i];

+

+            virBufferAsprintf(buf, "

+                              i, vcpu->online ? "yes" : "no");

+            if (vcpu->hotpluggable)

+                virBufferAsprintf(buf, " hotpluggable='%s'",

+                                  virTristateBoolTypeToString(vcpu->hotpluggable));

+

+            if (vcpu->order != 0)

+                virBufferAsprintf(buf, " order='%d'", vcpu->order);

+

+            virBufferAddLit(buf, "/>\n");

+        }

+        virBufferAdjustIndent(buf, -2);

+        virBufferAddLit(buf, "</vcpus>\n");

+    }

+

     ret = 0;

  cleanup:

diff --git a/src/conf/domain_conf.h b/src/conf/domain_conf.h

index e7ff401..34da54c 100644

--- a/src/conf/domain_conf.h

+++ b/src/conf/domain_conf.h

@@ -2040,6 +2040,9 @@ typedef virDomainVcpuDef *virDomainVcpuDefPtr;

 struct _virDomainVcpuDef {

     bool online;

+    virTristateBool hotpluggable;

+    unsigned int order;

+

     virBitmapPtr cpumask;

     virDomainThreadSchedParam sched;

@@ -2136,6 +2139,8 @@ struct _virDomainDef {

     virDomainVcpuDefPtr *vcpus;

     size_t maxvcpus;

+    /* set if the vcpu definition was specified individually */

+    bool individualvcpus;

     int placement_mode;

     virBitmapPtr cpumask;

@@ -2338,6 +2343,7 @@ typedef enum {

     VIR_DOMAIN_DEF_FEATURE_MEMORY_HOTPLUG = (1 << 1),

     VIR_DOMAIN_DEF_FEATURE_OFFLINE_VCPUPIN = (1 << 2),

     VIR_DOMAIN_DEF_FEATURE_NAME_SLASH = (1 << 3),

+    VIR_DOMAIN_DEF_FEATURE_INDIVIDUAL_VCPUS = (1 << 4),

 } virDomainDefFeatures;

diff --git a/src/qemu/qemu_domain.c b/src/qemu/qemu_domain.c

index add8379..7a71bfb 100644

--- a/src/qemu/qemu_domain.c

+++ b/src/qemu/qemu_domain.c

@@ -5796,8 +5796,17 @@ qemuDomainRefreshVcpuInfo(virQEMUDriverPtr driver,

         VIR_STEAL_PTR(vcpupriv->alias, info[i].alias);

         vcpupriv->enable_id = info[i].id;

-        if (hotplug && state)

+        if (hotplug && state) {

             vcpu->online = !!info[i].qom_path;

+

+            /* mark cpus that don't have an alias as non-hotpluggable */

+            if (vcpu->online) {

+                if (vcpupriv->alias)

+                    vcpu->hotpluggable = VIR_TRISTATE_BOOL_YES;

+                else

+                    vcpu->hotpluggable = VIR_TRISTATE_BOOL_NO;

+            }

+        }

     }

     ret = 0;

diff --git a/tests/genericxml2xmlindata/generic-vcpus-individual.xml b/tests/genericxml2xmlindata/generic-vcpus-individual.xml

new file mode 100644

index 0000000..cbcf8fd

--- /dev/null

+++ b/tests/genericxml2xmlindata/generic-vcpus-individual.xml

@@ -0,0 +1,23 @@

+<domain type='qemu'>

+  <name>foobar</name>

+  <uuid>c7a5fdbd-edaf-9455-926a-d65c16db1809</uuid>

+  <memory unit='KiB'>219136</memory>

+  <currentMemory unit='KiB'>219136</currentMemory>

+  <vcpu placement='static' current='2'>4</vcpu>

+  <vcpus>

+    <vcpu id='0' enabled='no' hotpluggable='yes' order='1'/>

+    <vcpu id='1' enabled='yes' hotpluggable='no'/>

+    <vcpu id='2' enabled='no' hotpluggable='yes' order='2'/>

+    <vcpu id='3' enabled='yes' hotpluggable='no'/>

+  </vcpus>

+  <os>

+    <type arch='i686' machine='pc'>hvm</type>

+    <boot dev='hd'/>

+  </os>

+  <clock offset='utc'/>

+  <on_poweroff>destroy</on_poweroff>

+  <on_reboot>restart</on_reboot>

+  <on_crash>destroy</on_crash>

+  <devices>

+  </devices>

+</domain>

diff --git a/tests/genericxml2xmltest.c b/tests/genericxml2xmltest.c

index a487727..2ea2396 100644

--- a/tests/genericxml2xmltest.c

+++ b/tests/genericxml2xmltest.c

@@ -97,6 +97,8 @@ mymain(void)

     DO_TEST("perf");

+    DO_TEST("vcpus-individual");

+

     virObjectUnref(caps);

     virObjectUnref(xmlopt);

diff --git a/tests/testutils.c b/tests/testutils.c

index be61e4d..2daad43 100644

--- a/tests/testutils.c

+++ b/tests/testutils.c

@@ -1075,7 +1075,9 @@ virCapsPtr virTestGenericCapsInit(void)

     return NULL;

 }

-static virDomainDefParserConfig virTestGenericDomainDefParserConfig;

+static virDomainDefParserConfig virTestGenericDomainDefParserConfig = {

+    .features = VIR_DOMAIN_DEF_FEATURE_INDIVIDUAL_VCPUS,

+};

 static virDomainXMLPrivateDataCallbacks virTestGenericPrivateDataCallbacks;

 virDomainXMLOptionPtr virTestGenericDomainXMLConfInit(void)

-- 

2.10.0