From 9211c21a4a98c63aafcc08981eb8773a25df92c1 Mon Sep 17 00:00:00 2001
Message-Id: <9211c21a4a98c63aafcc08981eb8773a25df92c1.1389183249.git.jdenemar@redhat.com>
From: Shivaprasad G Bhat <shivaprasadbhat@gmail.com>
Date: Mon, 6 Jan 2014 17:02:26 +0100
Subject: [PATCH] qemu: fix live pinning to memory node on NUMA system

https://bugzilla.redhat.com/show_bug.cgi?id=1047234

Ever since the subcpusets(vcpu,emulator) were introduced, the parent
cpuset cannot be modified to remove the nodes that are in use by the
subcpusets.
The fix is to break the memory node modification into three steps:
 1. assign new nodes into the parent,
 2. change the nodes in the child nodes,
 3. remove the old nodes on the parent node.

Resolves: https://bugzilla.redhat.com/show_bug.cgi?id=1009880

Signed-off-by: Shivaprasad G Bhat <sbhat@linux.vnet.ibm.com>
Signed-off-by: Martin Kletzander <mkletzan@redhat.com>
(cherry picked from commit 81fae6b95cfe72d0f5a987b6b5cd4bf86e32798c)
Signed-off-by: Jiri Denemark <jdenemar@redhat.com>
---
 src/qemu/qemu_driver.c | 102 ++++++++++++++++++++++++++++++++++++++-----------
 1 file changed, 79 insertions(+), 23 deletions(-)

diff --git a/src/qemu/qemu_driver.c b/src/qemu/qemu_driver.c
index 5216e44..a3d4725 100644
--- a/src/qemu/qemu_driver.c
+++ b/src/qemu/qemu_driver.c
@@ -8413,6 +8413,84 @@ cleanup:
 }
 
 static int
+qemuDomainSetNumaParamsLive(virDomainObjPtr vm,
+                            virCapsPtr caps,
+                            virBitmapPtr nodeset)
+{
+    virCgroupPtr cgroup_temp = NULL;
+    virBitmapPtr temp_nodeset = NULL;
+    qemuDomainObjPrivatePtr priv = vm->privateData;
+    char *nodeset_str = NULL;
+    size_t i = 0;
+    int ret = -1;
+
+    if (vm->def->numatune.memory.mode != VIR_DOMAIN_NUMATUNE_MEM_STRICT) {
+        virReportError(VIR_ERR_OPERATION_INVALID, "%s",
+                       _("change of nodeset for running domain "
+                         "requires strict numa mode"));
+        goto cleanup;
+    }
+
+    /*Get Exisitng nodeset values */
+    if (virCgroupGetCpusetMems(priv->cgroup, &nodeset_str) < 0 ||
+        virBitmapParse(nodeset_str, 0, &temp_nodeset,
+                       VIR_DOMAIN_CPUMASK_LEN) < 0)
+        goto cleanup;
+    VIR_FREE(nodeset_str);
+
+    for (i = 0; i < caps->host.nnumaCell; i++) {
+        bool result;
+        if (virBitmapGetBit(nodeset, i, &result) < 0) {
+            virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
+                           _("Failed to get cpuset bit values"));
+            goto cleanup;
+        }
+        if (result && (virBitmapSetBit(temp_nodeset, i) < 0)) {
+            virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
+                           _("Failed to set temporary cpuset bit values"));
+            goto cleanup;
+        }
+    }
+
+    if (!(nodeset_str = virBitmapFormat(temp_nodeset))) {
+        virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
+                       _("Failed to format nodeset"));
+        goto cleanup;
+    }
+
+    if (virCgroupSetCpusetMems(priv->cgroup, nodeset_str) < 0)
+        goto cleanup;
+    VIR_FREE(nodeset_str);
+
+    /* Ensure the cpuset string is formated before passing to cgroup */
+    if (!(nodeset_str = virBitmapFormat(nodeset))) {
+        virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
+                       _("Failed to format nodeset"));
+        goto cleanup;
+    }
+
+    for (i = 0; i < priv->nvcpupids; i++) {
+        if (virCgroupNewVcpu(priv->cgroup, i, false, &cgroup_temp) < 0 ||
+            virCgroupSetCpusetMems(cgroup_temp, nodeset_str) < 0)
+            goto cleanup;
+        virCgroupFree(&cgroup_temp);
+    }
+
+    if (virCgroupNewEmulator(priv->cgroup, false, &cgroup_temp) < 0 ||
+        virCgroupSetCpusetMems(cgroup_temp, nodeset_str) < 0 ||
+        virCgroupSetCpusetMems(priv->cgroup, nodeset_str) < 0)
+        goto cleanup;
+
+    ret = 0;
+ cleanup:
+    VIR_FREE(nodeset_str);
+    virBitmapFree(temp_nodeset);
+    virCgroupFree(&cgroup_temp);
+
+    return ret;
+}
+
+static int
 qemuDomainSetNumaParameters(virDomainPtr dom,
                             virTypedParameterPtr params,
                             int nparams,
@@ -8479,7 +8557,6 @@ qemuDomainSetNumaParameters(virDomainPtr dom,
             }
         } else if (STREQ(param->field, VIR_DOMAIN_NUMA_NODESET)) {
             virBitmapPtr nodeset = NULL;
-            char *nodeset_str = NULL;
 
             if (virBitmapParse(params[i].value.s,
                                0, &nodeset,
@@ -8489,32 +8566,11 @@ qemuDomainSetNumaParameters(virDomainPtr dom,
             }
 
             if (flags & VIR_DOMAIN_AFFECT_LIVE) {
-                if (vm->def->numatune.memory.mode !=
-                    VIR_DOMAIN_NUMATUNE_MEM_STRICT) {
-                    virReportError(VIR_ERR_OPERATION_INVALID, "%s",
-                                   _("change of nodeset for running domain "
-                                     "requires strict numa mode"));
-                    virBitmapFree(nodeset);
-                    ret = -1;
-                    continue;
-                }
-
-                /* Ensure the cpuset string is formated before passing to cgroup */
-                if (!(nodeset_str = virBitmapFormat(nodeset))) {
-                    virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
-                                   _("Failed to format nodeset"));
-                    virBitmapFree(nodeset);
-                    ret = -1;
-                    continue;
-                }
-
-                if (virCgroupSetCpusetMems(priv->cgroup, nodeset_str) < 0) {
+                if (qemuDomainSetNumaParamsLive(vm, caps, nodeset) < 0) {
                     virBitmapFree(nodeset);
-                    VIR_FREE(nodeset_str);
                     ret = -1;
                     continue;
                 }
-                VIR_FREE(nodeset_str);
 
                 /* update vm->def here so that dumpxml can read the new
                  * values from vm->def. */
-- 
1.8.5.2