From 2427e21de274cf7b56ef79e4a7ba78a08def7a58 Mon Sep 17 00:00:00 2001

From: Paolo Bonzini <pbonzini@redhat.com>

Date: Mon, 22 Jul 2019 18:22:18 +0100

Subject: [PATCH 37/39] target/i386: kvm: Demand nested migration kernel

 capabilities only when vCPU may have enabled VMX

RH-Author: Paolo Bonzini <pbonzini@redhat.com>

Message-id: <20190722182220.19374-17-pbonzini@redhat.com>

Patchwork-id: 89634

O-Subject: [RHEL-8.1.0 PATCH qemu-kvm v3 16/18] target/i386: kvm: Demand nested migration kernel capabilities only when vCPU may have enabled VMX

Bugzilla: 1689269

RH-Acked-by: Peter Xu <zhexu@redhat.com>

RH-Acked-by: Laurent Vivier <lvivier@redhat.com>

RH-Acked-by: Dr. David Alan Gilbert <dgilbert@redhat.com>

From: Liran Alon <liran.alon@oracle.com>

Previous to this change, a vCPU exposed with VMX running on a kernel

without KVM_CAP_NESTED_STATE or KVM_CAP_EXCEPTION_PAYLOAD resulted in

adding a migration blocker. This was because when the code was written

it was thought there is no way to reliably know if a vCPU is utilising

VMX or not at runtime. However, it turns out that this can be known to

some extent:

In order for a vCPU to enter VMX operation it must have CR4.VMXE set.

Since it was set, CR4.VMXE must remain set as long as the vCPU is in

VMX operation. This is because CR4.VMXE is one of the bits set

in MSR_IA32_VMX_CR4_FIXED1.

There is one exception to the above statement when vCPU enters SMM mode.

When a vCPU enters SMM mode, it temporarily exits VMX operation and

may also reset CR4.VMXE during execution in SMM mode.

When the vCPU exits SMM mode, vCPU state is restored to be in VMX operation

and CR4.VMXE is restored to its original state of being set.

Therefore, when the vCPU is not in SMM mode, we can infer whether

VMX is being used by examining CR4.VMXE. Otherwise, we cannot

know for certain but assume the worse that vCPU may utilise VMX.

Summaring all the above, a vCPU may have enabled VMX in case

CR4.VMXE is set or vCPU is in SMM mode.

Therefore, remove migration blocker and check before migration

(cpu_pre_save()) if the vCPU may have enabled VMX. If true, only then

require relevant kernel capabilities.

While at it, demand KVM_CAP_EXCEPTION_PAYLOAD only when the vCPU is in

guest-mode and there is a pending/injected exception. Otherwise, this

kernel capability is not required for proper migration.

Reviewed-by: Joao Martins <joao.m.martins@oracle.com>

Signed-off-by: Liran Alon <liran.alon@oracle.com>

Reviewed-by: Maran Wilson <maran.wilson@oracle.com>

Tested-by: Maran Wilson <maran.wilson@oracle.com>

Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>

(cherry picked from commit 79a197ab180e75838523c58973b1221ad7bf51eb)

Signed-off-by: Danilo C. L. de Paula <ddepaula@redhat.com>

---

 target/i386/cpu.h      | 22 ++++++++++++++++++++++

 target/i386/kvm.c      | 26 ++++++--------------------

 target/i386/kvm_i386.h |  1 +

 target/i386/machine.c  | 24 ++++++++++++++++++++----

 4 files changed, 49 insertions(+), 24 deletions(-)

diff --git a/target/i386/cpu.h b/target/i386/cpu.h

index d120f62..273c90b 100644

--- a/target/i386/cpu.h

+++ b/target/i386/cpu.h

@@ -1848,6 +1848,28 @@ static inline bool cpu_has_vmx(CPUX86State *env)

     return env->features[FEAT_1_ECX] & CPUID_EXT_VMX;

 }

+/*

+ * In order for a vCPU to enter VMX operation it must have CR4.VMXE set.

+ * Since it was set, CR4.VMXE must remain set as long as vCPU is in

+ * VMX operation. This is because CR4.VMXE is one of the bits set

+ * in MSR_IA32_VMX_CR4_FIXED1.

+ *

+ * There is one exception to above statement when vCPU enters SMM mode.

+ * When a vCPU enters SMM mode, it temporarily exit VMX operation and

+ * may also reset CR4.VMXE during execution in SMM mode.

+ * When vCPU exits SMM mode, vCPU state is restored to be in VMX operation

+ * and CR4.VMXE is restored to it's original value of being set.

+ *

+ * Therefore, when vCPU is not in SMM mode, we can infer whether

+ * VMX is being used by examining CR4.VMXE. Otherwise, we cannot

+ * know for certain.

+ */

+static inline bool cpu_vmx_maybe_enabled(CPUX86State *env)

+{

+    return cpu_has_vmx(env) &&

+           ((env->cr[4] & CR4_VMXE_MASK) || (env->hflags & HF_SMM_MASK));

+}

+

 /* fpu_helper.c */

 void update_fp_status(CPUX86State *env);

 void update_mxcsr_status(CPUX86State *env);

diff --git a/target/i386/kvm.c b/target/i386/kvm.c

index 0619aba..0bd286e 100644

--- a/target/i386/kvm.c

+++ b/target/i386/kvm.c

@@ -127,6 +127,11 @@ bool kvm_has_adjust_clock_stable(void)

     return (ret == KVM_CLOCK_TSC_STABLE);

 }

+bool kvm_has_exception_payload(void)

+{

+    return has_exception_payload;

+}

+

 bool kvm_allows_irq0_override(void)

 {

     return !kvm_irqchip_in_kernel() || kvm_has_gsi_routing();

@@ -814,7 +819,6 @@ static int hyperv_handle_properties(CPUState *cs)

 }

 static Error *invtsc_mig_blocker;

-static Error *nested_virt_mig_blocker;

 #define KVM_MAX_CPUID_ENTRIES  100

@@ -1159,22 +1163,6 @@ int kvm_arch_init_vcpu(CPUState *cs)

                                   !!(c->ecx & CPUID_EXT_SMX);

     }

-    if (cpu_has_vmx(env) && !nested_virt_mig_blocker &&

-        ((kvm_max_nested_state_length() <= 0) || !has_exception_payload)) {

-        error_setg(&nested_virt_mig_blocker,

-                   "Kernel do not provide required capabilities for "

-                   "nested virtualization migration. "

-                   "(CAP_NESTED_STATE=%d, CAP_EXCEPTION_PAYLOAD=%d)",

-                   kvm_max_nested_state_length() > 0,

-                   has_exception_payload);

-        r = migrate_add_blocker(nested_virt_mig_blocker, &local_err);

-        if (local_err) {

-            error_report_err(local_err);

-            error_free(nested_virt_mig_blocker);

-            return r;

-        }

-    }

-

     if (env->mcg_cap & MCG_LMCE_P) {

         has_msr_mcg_ext_ctl = has_msr_feature_control = true;

     }

@@ -1190,7 +1178,7 @@ int kvm_arch_init_vcpu(CPUState *cs)

             if (local_err) {

                 error_report_err(local_err);

                 error_free(invtsc_mig_blocker);

-                goto fail2;

+                return r;

             }

             /* for savevm */

             vmstate_x86_cpu.unmigratable = 1;

@@ -1256,8 +1244,6 @@ int kvm_arch_init_vcpu(CPUState *cs)

  fail:

     migrate_del_blocker(invtsc_mig_blocker);

- fail2:

-    migrate_del_blocker(nested_virt_mig_blocker);

     return r;

 }

diff --git a/target/i386/kvm_i386.h b/target/i386/kvm_i386.h

index 1de9876..df9bbf3 100644

--- a/target/i386/kvm_i386.h

+++ b/target/i386/kvm_i386.h

@@ -41,6 +41,7 @@

 bool kvm_allows_irq0_override(void);

 bool kvm_has_smm(void);

 bool kvm_has_adjust_clock_stable(void);

+bool kvm_has_exception_payload(void);

 void kvm_synchronize_all_tsc(void);

 void kvm_arch_reset_vcpu(X86CPU *cs);

 void kvm_arch_do_init_vcpu(X86CPU *cs);

diff --git a/target/i386/machine.c b/target/i386/machine.c

index 5ffee8f..8d90d98 100644

--- a/target/i386/machine.c

+++ b/target/i386/machine.c

@@ -7,6 +7,7 @@

 #include "hw/i386/pc.h"

 #include "hw/isa/isa.h"

 #include "migration/cpu.h"

+#include "kvm_i386.h"

 #include "sysemu/kvm.h"

@@ -231,10 +232,25 @@ static int cpu_pre_save(void *opaque)

     }

 #ifdef CONFIG_KVM

-    /* Verify we have nested virtualization state from kernel if required */

-    if (kvm_enabled() && cpu_has_vmx(env) && !env->nested_state) {

-        error_report("Guest enabled nested virtualization but kernel "

-                "does not support saving of nested state");

+    /*

+     * In case vCPU may have enabled VMX, we need to make sure kernel have

+     * required capabilities in order to perform migration correctly:

+     *

+     * 1) We must be able to extract vCPU nested-state from KVM.

+     *

+     * 2) In case vCPU is running in guest-mode and it has a pending exception,

+     * we must be able to determine if it's in a pending or injected state.

+     * Note that in case KVM don't have required capability to do so,

+     * a pending/injected exception will always appear as an

+     * injected exception.

+     */

+    if (kvm_enabled() && cpu_vmx_maybe_enabled(env) &&

+        (!env->nested_state ||

+         (!kvm_has_exception_payload() && (env->hflags & HF_GUEST_MASK) &&

+          env->exception_injected))) {

+        error_report("Guest maybe enabled nested virtualization but kernel "

+                "does not support required capabilities to save vCPU "

+                "nested state");

         return -EINVAL;

     }

 #endif

-- 

1.8.3.1