arch/x86/kvm/svm.c

   1 /*
   2  * Kernel-based Virtual Machine driver for Linux
   3  *
   4  * AMD SVM support
   5  *
   6  * Copyright (C) 2006 Qumranet, Inc.
   7  * Copyright 2010 Red Hat, Inc. and/or its affiliates.
   8  *
   9  * Authors:
  10  *   Yaniv Kamay  <yaniv@qumranet.com>
  11  *   Avi Kivity   <avi@qumranet.com>
  12  *
  13  * This work is licensed under the terms of the GNU GPL, version 2.  See
  14  * the COPYING file in the top-level directory.
  15  *
  16  */
  17 #include <linux/kvm_host.h>
  18
  19 #include "irq.h"
  20 #include "mmu.h"
  21 #include "kvm_cache_regs.h"
  22 #include "x86.h"
  23 #include "cpuid.h"
  24 #include "pmu.h"
  25
  26 #include <linux/module.h>
  27 #include <linux/mod_devicetable.h>
  28 #include <linux/kernel.h>
  29 #include <linux/vmalloc.h>
  30 #include <linux/highmem.h>
  31 #include <linux/sched.h>
  32 #include <linux/trace_events.h>
  33 #include <linux/slab.h>
  34
  35 #include <asm/perf_event.h>
  36 #include <asm/tlbflush.h>
  37 #include <asm/desc.h>
  38 #include <asm/debugreg.h>
  39 #include <asm/kvm_para.h>
  40
  41 #include <asm/virtext.h>
  42 #include "trace.h"
  43
  44 #define __ex(x) __kvm_handle_fault_on_reboot(x)
  45
  46 MODULE_AUTHOR("Qumranet");
  47 MODULE_LICENSE("GPL");
  48
  49 static const struct x86_cpu_id svm_cpu_id[] = {
  50         X86_FEATURE_MATCH(X86_FEATURE_SVM),
  51         {}
  52 };
  53 MODULE_DEVICE_TABLE(x86cpu, svm_cpu_id);
  54
  55 #define IOPM_ALLOC_ORDER 2
  56 #define MSRPM_ALLOC_ORDER 1
  57
  58 #define SEG_TYPE_LDT 2
  59 #define SEG_TYPE_BUSY_TSS16 3
  60
  61 #define SVM_FEATURE_NPT            (1 <<  0)
  62 #define SVM_FEATURE_LBRV           (1 <<  1)
  63 #define SVM_FEATURE_SVML           (1 <<  2)
  64 #define SVM_FEATURE_NRIP           (1 <<  3)
  65 #define SVM_FEATURE_TSC_RATE       (1 <<  4)
  66 #define SVM_FEATURE_VMCB_CLEAN     (1 <<  5)
  67 #define SVM_FEATURE_FLUSH_ASID     (1 <<  6)
  68 #define SVM_FEATURE_DECODE_ASSIST  (1 <<  7)
  69 #define SVM_FEATURE_PAUSE_FILTER   (1 << 10)
  70
  71 #define NESTED_EXIT_HOST        0       /* Exit handled on host level */
  72 #define NESTED_EXIT_DONE        1       /* Exit caused nested vmexit  */
  73 #define NESTED_EXIT_CONTINUE    2       /* Further checks needed      */
  74
  75 #define DEBUGCTL_RESERVED_BITS (~(0x3fULL))
  76
  77 #define TSC_RATIO_RSVD          0xffffff0000000000ULL
  78 #define TSC_RATIO_MIN           0x0000000000000001ULL
  79 #define TSC_RATIO_MAX           0x000000ffffffffffULL
  80
  81 static bool erratum_383_found __read_mostly;
  82
  83 static const u32 host_save_user_msrs[] = {
  84 #ifdef CONFIG_X86_64
  85         MSR_STAR, MSR_LSTAR, MSR_CSTAR, MSR_SYSCALL_MASK, MSR_KERNEL_GS_BASE,
  86         MSR_FS_BASE,
  87 #endif
  88         MSR_IA32_SYSENTER_CS, MSR_IA32_SYSENTER_ESP, MSR_IA32_SYSENTER_EIP,
  89 };
  90
  91 #define NR_HOST_SAVE_USER_MSRS ARRAY_SIZE(host_save_user_msrs)
  92
  93 struct kvm_vcpu;
  94
  95 struct nested_state {
  96         struct vmcb *hsave;
  97         u64 hsave_msr;
  98         u64 vm_cr_msr;
  99         u64 vmcb;
 100
 101         /* These are the merged vectors */
 102         u32 *msrpm;
 103
 104         /* gpa pointers to the real vectors */
 105         u64 vmcb_msrpm;
 106         u64 vmcb_iopm;
 107
 108         /* A VMEXIT is required but not yet emulated */
 109         bool exit_required;
 110
 111         /* cache for intercepts of the guest */
 112         u32 intercept_cr;
 113         u32 intercept_dr;
 114         u32 intercept_exceptions;
 115         u64 intercept;
 116
 117         /* Nested Paging related state */
 118         u64 nested_cr3;
 119 };
 120
 121 #define MSRPM_OFFSETS   16
 122 static u32 msrpm_offsets[MSRPM_OFFSETS] __read_mostly;
 123
 124 /*
 125  * Set osvw_len to higher value when updated Revision Guides
 126  * are published and we know what the new status bits are
 127  */
 128 static uint64_t osvw_len = 4, osvw_status;
 129
 130 struct vcpu_svm {
 131         struct kvm_vcpu vcpu;
 132         struct vmcb *vmcb;
 133         unsigned long vmcb_pa;
 134         struct svm_cpu_data *svm_data;
 135         uint64_t asid_generation;
 136         uint64_t sysenter_esp;
 137         uint64_t sysenter_eip;
 138
 139         u64 next_rip;
 140
 141         u64 host_user_msrs[NR_HOST_SAVE_USER_MSRS];
 142         struct {
 143                 u16 fs;
 144                 u16 gs;
 145                 u16 ldt;
 146                 u64 gs_base;
 147         } host;
 148
 149         u32 *msrpm;
 150
 151         ulong nmi_iret_rip;
 152
 153         struct nested_state nested;
 154
 155         bool nmi_singlestep;
 156
 157         unsigned int3_injected;
 158         unsigned long int3_rip;
 159         u32 apf_reason;
 160
 161         u64  tsc_ratio;
 162 };
 163
 164 static DEFINE_PER_CPU(u64, current_tsc_ratio);
 165 #define TSC_RATIO_DEFAULT       0x0100000000ULL
 166
 167 #define MSR_INVALID                     0xffffffffU
 168
 169 static const struct svm_direct_access_msrs {
 170         u32 index;   /* Index of the MSR */
 171         bool always; /* True if intercept is always on */
 172 } direct_access_msrs[] = {
 173         { .index = MSR_STAR,                            .always = true  },
 174         { .index = MSR_IA32_SYSENTER_CS,                .always = true  },
 175 #ifdef CONFIG_X86_64
 176         { .index = MSR_GS_BASE,                         .always = true  },
 177         { .index = MSR_FS_BASE,                         .always = true  },
 178         { .index = MSR_KERNEL_GS_BASE,                  .always = true  },
 179         { .index = MSR_LSTAR,                           .always = true  },
 180         { .index = MSR_CSTAR,                           .always = true  },
 181         { .index = MSR_SYSCALL_MASK,                    .always = true  },
 182 #endif
 183         { .index = MSR_IA32_LASTBRANCHFROMIP,           .always = false },
 184         { .index = MSR_IA32_LASTBRANCHTOIP,             .always = false },
 185         { .index = MSR_IA32_LASTINTFROMIP,              .always = false },
 186         { .index = MSR_IA32_LASTINTTOIP,                .always = false },
 187         { .index = MSR_INVALID,                         .always = false },
 188 };
 189
 190 /* enable NPT for AMD64 and X86 with PAE */
 191 #if defined(CONFIG_X86_64) || defined(CONFIG_X86_PAE)
 192 static bool npt_enabled = true;
 193 #else
 194 static bool npt_enabled;
 195 #endif
 196
 197 /* allow nested paging (virtualized MMU) for all guests */
 198 static int npt = true;
 199 module_param(npt, int, S_IRUGO);
 200
 201 /* allow nested virtualization in KVM/SVM */
 202 static int nested = true;
 203 module_param(nested, int, S_IRUGO);
 204
 205 static void svm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0);
 206 static void svm_flush_tlb(struct kvm_vcpu *vcpu);
 207 static void svm_complete_interrupts(struct vcpu_svm *svm);
 208
 209 static int nested_svm_exit_handled(struct vcpu_svm *svm);
 210 static int nested_svm_intercept(struct vcpu_svm *svm);
 211 static int nested_svm_vmexit(struct vcpu_svm *svm);
 212 static int nested_svm_check_exception(struct vcpu_svm *svm, unsigned nr,
 213                                       bool has_error_code, u32 error_code);
 214 static u64 __scale_tsc(u64 ratio, u64 tsc);
 215
 216 enum {
 217         VMCB_INTERCEPTS, /* Intercept vectors, TSC offset,
 218                             pause filter count */
 219         VMCB_PERM_MAP,   /* IOPM Base and MSRPM Base */
 220         VMCB_ASID,       /* ASID */
 221         VMCB_INTR,       /* int_ctl, int_vector */
 222         VMCB_NPT,        /* npt_en, nCR3, gPAT */
 223         VMCB_CR,         /* CR0, CR3, CR4, EFER */
 224         VMCB_DR,         /* DR6, DR7 */
 225         VMCB_DT,         /* GDT, IDT */
 226         VMCB_SEG,        /* CS, DS, SS, ES, CPL */
 227         VMCB_CR2,        /* CR2 only */
 228         VMCB_LBR,        /* DBGCTL, BR_FROM, BR_TO, LAST_EX_FROM, LAST_EX_TO */
 229         VMCB_DIRTY_MAX,
 230 };
 231
 232 /* TPR and CR2 are always written before VMRUN */
 233 #define VMCB_ALWAYS_DIRTY_MASK  ((1U << VMCB_INTR) | (1U << VMCB_CR2))
 234
 235 static inline void mark_all_dirty(struct vmcb *vmcb)
 236 {
 237         vmcb->control.clean = 0;
 238 }
 239
 240 static inline void mark_all_clean(struct vmcb *vmcb)
 241 {
 242         vmcb->control.clean = ((1 << VMCB_DIRTY_MAX) - 1)
 243                                & ~VMCB_ALWAYS_DIRTY_MASK;
 244 }
 245
 246 static inline void mark_dirty(struct vmcb *vmcb, int bit)
 247 {
 248         vmcb->control.clean &= ~(1 << bit);
 249 }
 250
 251 static inline struct vcpu_svm *to_svm(struct kvm_vcpu *vcpu)
 252 {
 253         return container_of(vcpu, struct vcpu_svm, vcpu);
 254 }
 255
 256 static void recalc_intercepts(struct vcpu_svm *svm)
 257 {
 258         struct vmcb_control_area *c, *h;
 259         struct nested_state *g;
 260
 261         mark_dirty(svm->vmcb, VMCB_INTERCEPTS);
 262
 263         if (!is_guest_mode(&svm->vcpu))
 264                 return;
 265
 266         c = &svm->vmcb->control;
 267         h = &svm->nested.hsave->control;
 268         g = &svm->nested;
 269
 270         c->intercept_cr = h->intercept_cr | g->intercept_cr;
 271         c->intercept_dr = h->intercept_dr | g->intercept_dr;
 272         c->intercept_exceptions = h->intercept_exceptions | g->intercept_exceptions;
 273         c->intercept = h->intercept | g->intercept;
 274 }
 275
 276 static inline struct vmcb *get_host_vmcb(struct vcpu_svm *svm)
 277 {
 278         if (is_guest_mode(&svm->vcpu))
 279                 return svm->nested.hsave;
 280         else
 281                 return svm->vmcb;
 282 }
 283
 284 static inline void set_cr_intercept(struct vcpu_svm *svm, int bit)
 285 {
 286         struct vmcb *vmcb = get_host_vmcb(svm);
 287
 288         vmcb->control.intercept_cr |= (1U << bit);
 289
 290         recalc_intercepts(svm);
 291 }
 292
 293 static inline void clr_cr_intercept(struct vcpu_svm *svm, int bit)
 294 {
 295         struct vmcb *vmcb = get_host_vmcb(svm);
 296
 297         vmcb->control.intercept_cr &= ~(1U << bit);
 298
 299         recalc_intercepts(svm);
 300 }
 301
 302 static inline bool is_cr_intercept(struct vcpu_svm *svm, int bit)
 303 {
 304         struct vmcb *vmcb = get_host_vmcb(svm);
 305
 306         return vmcb->control.intercept_cr & (1U << bit);
 307 }
 308
 309 static inline void set_dr_intercepts(struct vcpu_svm *svm)
 310 {
 311         struct vmcb *vmcb = get_host_vmcb(svm);
 312
 313         vmcb->control.intercept_dr = (1 << INTERCEPT_DR0_READ)
 314                 | (1 << INTERCEPT_DR1_READ)
 315                 | (1 << INTERCEPT_DR2_READ)
 316                 | (1 << INTERCEPT_DR3_READ)
 317                 | (1 << INTERCEPT_DR4_READ)
 318                 | (1 << INTERCEPT_DR5_READ)
 319                 | (1 << INTERCEPT_DR6_READ)
 320                 | (1 << INTERCEPT_DR7_READ)
 321                 | (1 << INTERCEPT_DR0_WRITE)
 322                 | (1 << INTERCEPT_DR1_WRITE)
 323                 | (1 << INTERCEPT_DR2_WRITE)
 324                 | (1 << INTERCEPT_DR3_WRITE)
 325                 | (1 << INTERCEPT_DR4_WRITE)
 326                 | (1 << INTERCEPT_DR5_WRITE)
 327                 | (1 << INTERCEPT_DR6_WRITE)
 328                 | (1 << INTERCEPT_DR7_WRITE);
 329
 330         recalc_intercepts(svm);
 331 }
 332
 333 static inline void clr_dr_intercepts(struct vcpu_svm *svm)
 334 {
 335         struct vmcb *vmcb = get_host_vmcb(svm);
 336
 337         vmcb->control.intercept_dr = 0;
 338
 339         recalc_intercepts(svm);
 340 }
 341
 342 static inline void set_exception_intercept(struct vcpu_svm *svm, int bit)
 343 {
 344         struct vmcb *vmcb = get_host_vmcb(svm);
 345
 346         vmcb->control.intercept_exceptions |= (1U << bit);
 347
 348         recalc_intercepts(svm);
 349 }
 350
 351 static inline void clr_exception_intercept(struct vcpu_svm *svm, int bit)
 352 {
 353         struct vmcb *vmcb = get_host_vmcb(svm);
 354
 355         vmcb->control.intercept_exceptions &= ~(1U << bit);
 356
 357         recalc_intercepts(svm);
 358 }
 359
 360 static inline void set_intercept(struct vcpu_svm *svm, int bit)
 361 {
 362         struct vmcb *vmcb = get_host_vmcb(svm);
 363
 364         vmcb->control.intercept |= (1ULL << bit);
 365
 366         recalc_intercepts(svm);
 367 }
 368
 369 static inline void clr_intercept(struct vcpu_svm *svm, int bit)
 370 {
 371         struct vmcb *vmcb = get_host_vmcb(svm);
 372
 373         vmcb->control.intercept &= ~(1ULL << bit);
 374
 375         recalc_intercepts(svm);
 376 }
 377
 378 static inline void enable_gif(struct vcpu_svm *svm)
 379 {
 380         svm->vcpu.arch.hflags |= HF_GIF_MASK;
 381 }
 382
 383 static inline void disable_gif(struct vcpu_svm *svm)
 384 {
 385         svm->vcpu.arch.hflags &= ~HF_GIF_MASK;
 386 }
 387
 388 static inline bool gif_set(struct vcpu_svm *svm)
 389 {
 390         return !!(svm->vcpu.arch.hflags & HF_GIF_MASK);
 391 }
 392
 393 static unsigned long iopm_base;
 394
 395 struct kvm_ldttss_desc {
 396         u16 limit0;
 397         u16 base0;
 398         unsigned base1:8, type:5, dpl:2, p:1;
 399         unsigned limit1:4, zero0:3, g:1, base2:8;
 400         u32 base3;
 401         u32 zero1;
 402 } __attribute__((packed));
 403
 404 struct svm_cpu_data {
 405         int cpu;
 406
 407         u64 asid_generation;
 408         u32 max_asid;
 409         u32 next_asid;
 410         struct kvm_ldttss_desc *tss_desc;
 411
 412         struct page *save_area;
 413 };
 414
 415 static DEFINE_PER_CPU(struct svm_cpu_data *, svm_data);
 416
 417 struct svm_init_data {
 418         int cpu;
 419         int r;
 420 };
 421
 422 static const u32 msrpm_ranges[] = {0, 0xc0000000, 0xc0010000};
 423
 424 #define NUM_MSR_MAPS ARRAY_SIZE(msrpm_ranges)
 425 #define MSRS_RANGE_SIZE 2048
 426 #define MSRS_IN_RANGE (MSRS_RANGE_SIZE * 8 / 2)
 427
 428 static u32 svm_msrpm_offset(u32 msr)
 429 {
 430         u32 offset;
 431         int i;
 432
 433         for (i = 0; i < NUM_MSR_MAPS; i++) {
 434                 if (msr < msrpm_ranges[i] ||
 435                     msr >= msrpm_ranges[i] + MSRS_IN_RANGE)
 436                         continue;
 437
 438                 offset  = (msr - msrpm_ranges[i]) / 4; /* 4 msrs per u8 */
 439                 offset += (i * MSRS_RANGE_SIZE);       /* add range offset */
 440
 441                 /* Now we have the u8 offset - but need the u32 offset */
 442                 return offset / 4;
 443         }
 444
 445         /* MSR not in any range */
 446         return MSR_INVALID;
 447 }
 448
 449 #define MAX_INST_SIZE 15
 450
 451 static inline void clgi(void)
 452 {
 453         asm volatile (__ex(SVM_CLGI));
 454 }
 455
 456 static inline void stgi(void)
 457 {
 458         asm volatile (__ex(SVM_STGI));
 459 }
 460
 461 static inline void invlpga(unsigned long addr, u32 asid)
 462 {
 463         asm volatile (__ex(SVM_INVLPGA) : : "a"(addr), "c"(asid));
 464 }
 465
 466 static int get_npt_level(void)
 467 {
 468 #ifdef CONFIG_X86_64
 469         return PT64_ROOT_LEVEL;
 470 #else
 471         return PT32E_ROOT_LEVEL;
 472 #endif
 473 }
 474
 475 static void svm_set_efer(struct kvm_vcpu *vcpu, u64 efer)
 476 {
 477         vcpu->arch.efer = efer;
 478         if (!npt_enabled && !(efer & EFER_LMA))
 479                 efer &= ~EFER_LME;
 480
 481         to_svm(vcpu)->vmcb->save.efer = efer | EFER_SVME;
 482         mark_dirty(to_svm(vcpu)->vmcb, VMCB_CR);
 483 }
 484
 485 static int is_external_interrupt(u32 info)
 486 {
 487         info &= SVM_EVTINJ_TYPE_MASK | SVM_EVTINJ_VALID;
 488         return info == (SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_INTR);
 489 }
 490
 491 static u32 svm_get_interrupt_shadow(struct kvm_vcpu *vcpu)
 492 {
 493         struct vcpu_svm *svm = to_svm(vcpu);
 494         u32 ret = 0;
 495
 496         if (svm->vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK)
 497                 ret = KVM_X86_SHADOW_INT_STI | KVM_X86_SHADOW_INT_MOV_SS;
 498         return ret;
 499 }
 500
 501 static void svm_set_interrupt_shadow(struct kvm_vcpu *vcpu, int mask)
 502 {
 503         struct vcpu_svm *svm = to_svm(vcpu);
 504
 505         if (mask == 0)
 506                 svm->vmcb->control.int_state &= ~SVM_INTERRUPT_SHADOW_MASK;
 507         else
 508                 svm->vmcb->control.int_state |= SVM_INTERRUPT_SHADOW_MASK;
 509
 510 }
 511
 512 static void skip_emulated_instruction(struct kvm_vcpu *vcpu)
 513 {
 514         struct vcpu_svm *svm = to_svm(vcpu);
 515
 516         if (svm->vmcb->control.next_rip != 0) {
 517                 WARN_ON(!static_cpu_has(X86_FEATURE_NRIPS));
 518                 svm->next_rip = svm->vmcb->control.next_rip;
 519         }
 520
 521         if (!svm->next_rip) {
 522                 if (emulate_instruction(vcpu, EMULTYPE_SKIP) !=
 523                                 EMULATE_DONE)
 524                         printk(KERN_DEBUG "%s: NOP\n", __func__);
 525                 return;
 526         }
 527         if (svm->next_rip - kvm_rip_read(vcpu) > MAX_INST_SIZE)
 528                 printk(KERN_ERR "%s: ip 0x%lx next 0x%llx\n",
 529                        __func__, kvm_rip_read(vcpu), svm->next_rip);
 530
 531         kvm_rip_write(vcpu, svm->next_rip);
 532         svm_set_interrupt_shadow(vcpu, 0);
 533 }
 534
 535 static void svm_queue_exception(struct kvm_vcpu *vcpu, unsigned nr,
 536                                 bool has_error_code, u32 error_code,
 537                                 bool reinject)
 538 {
 539         struct vcpu_svm *svm = to_svm(vcpu);
 540
 541         /*
 542          * If we are within a nested VM we'd better #VMEXIT and let the guest
 543          * handle the exception
 544          */
 545         if (!reinject &&
 546             nested_svm_check_exception(svm, nr, has_error_code, error_code))
 547                 return;
 548
 549         if (nr == BP_VECTOR && !static_cpu_has(X86_FEATURE_NRIPS)) {
 550                 unsigned long rip, old_rip = kvm_rip_read(&svm->vcpu);
 551
 552                 /*
 553                  * For guest debugging where we have to reinject #BP if some
 554                  * INT3 is guest-owned:
 555                  * Emulate nRIP by moving RIP forward. Will fail if injection
 556                  * raises a fault that is not intercepted. Still better than
 557                  * failing in all cases.
 558                  */
 559                 skip_emulated_instruction(&svm->vcpu);
 560                 rip = kvm_rip_read(&svm->vcpu);
 561                 svm->int3_rip = rip + svm->vmcb->save.cs.base;
 562                 svm->int3_injected = rip - old_rip;
 563         }
 564
 565         svm->vmcb->control.event_inj = nr
 566                 | SVM_EVTINJ_VALID
 567                 | (has_error_code ? SVM_EVTINJ_VALID_ERR : 0)
 568                 | SVM_EVTINJ_TYPE_EXEPT;
 569         svm->vmcb->control.event_inj_err = error_code;
 570 }
 571
 572 static void svm_init_erratum_383(void)
 573 {
 574         u32 low, high;
 575         int err;
 576         u64 val;
 577
 578         if (!static_cpu_has_bug(X86_BUG_AMD_TLB_MMATCH))
 579                 return;
 580
 581         /* Use _safe variants to not break nested virtualization */
 582         val = native_read_msr_safe(MSR_AMD64_DC_CFG, &err);
 583         if (err)
 584                 return;
 585
 586         val |= (1ULL << 47);
 587
 588         low  = lower_32_bits(val);
 589         high = upper_32_bits(val);
 590
 591         native_write_msr_safe(MSR_AMD64_DC_CFG, low, high);
 592
 593         erratum_383_found = true;
 594 }
 595
 596 static void svm_init_osvw(struct kvm_vcpu *vcpu)
 597 {
 598         /*
 599          * Guests should see errata 400 and 415 as fixed (assuming that
 600          * HLT and IO instructions are intercepted).
 601          */
 602         vcpu->arch.osvw.length = (osvw_len >= 3) ? (osvw_len) : 3;
 603         vcpu->arch.osvw.status = osvw_status & ~(6ULL);
 604
 605         /*
 606          * By increasing VCPU's osvw.length to 3 we are telling the guest that
 607          * all osvw.status bits inside that length, including bit 0 (which is
 608          * reserved for erratum 298), are valid. However, if host processor's
 609          * osvw_len is 0 then osvw_status[0] carries no information. We need to
 610          * be conservative here and therefore we tell the guest that erratum 298
 611          * is present (because we really don't know).
 612          */
 613         if (osvw_len == 0 && boot_cpu_data.x86 == 0x10)
 614                 vcpu->arch.osvw.status |= 1;
 615 }
 616
 617 static int has_svm(void)
 618 {
 619         const char *msg;
 620
 621         if (!cpu_has_svm(&msg)) {
 622                 printk(KERN_INFO "has_svm: %s\n", msg);
 623                 return 0;
 624         }
 625
 626         return 1;
 627 }
 628
 629 static void svm_hardware_disable(void)
 630 {
 631         /* Make sure we clean up behind us */
 632         if (static_cpu_has(X86_FEATURE_TSCRATEMSR))
 633                 wrmsrl(MSR_AMD64_TSC_RATIO, TSC_RATIO_DEFAULT);
 634
 635         cpu_svm_disable();
 636
 637         amd_pmu_disable_virt();
 638 }
 639
 640 static int svm_hardware_enable(void)
 641 {
 642
 643         struct svm_cpu_data *sd;
 644         uint64_t efer;
 645         struct desc_ptr gdt_descr;
 646         struct desc_struct *gdt;
 647         int me = raw_smp_processor_id();
 648
 649         rdmsrl(MSR_EFER, efer);
 650         if (efer & EFER_SVME)
 651                 return -EBUSY;
 652
 653         if (!has_svm()) {
 654                 pr_err("%s: err EOPNOTSUPP on %d\n", __func__, me);
 655                 return -EINVAL;
 656         }
 657         sd = per_cpu(svm_data, me);
 658         if (!sd) {
 659                 pr_err("%s: svm_data is NULL on %d\n", __func__, me);
 660                 return -EINVAL;
 661         }
 662
 663         sd->asid_generation = 1;
 664         sd->max_asid = cpuid_ebx(SVM_CPUID_FUNC) - 1;
 665         sd->next_asid = sd->max_asid + 1;
 666
 667         native_store_gdt(&gdt_descr);
 668         gdt = (struct desc_struct *)gdt_descr.address;
 669         sd->tss_desc = (struct kvm_ldttss_desc *)(gdt + GDT_ENTRY_TSS);
 670
 671         wrmsrl(MSR_EFER, efer | EFER_SVME);
 672
 673         wrmsrl(MSR_VM_HSAVE_PA, page_to_pfn(sd->save_area) << PAGE_SHIFT);
 674
 675         if (static_cpu_has(X86_FEATURE_TSCRATEMSR)) {
 676                 wrmsrl(MSR_AMD64_TSC_RATIO, TSC_RATIO_DEFAULT);
 677                 __this_cpu_write(current_tsc_ratio, TSC_RATIO_DEFAULT);
 678         }
 679
 680
 681         /*
 682          * Get OSVW bits.
 683          *
 684          * Note that it is possible to have a system with mixed processor
 685          * revisions and therefore different OSVW bits. If bits are not the same
 686          * on different processors then choose the worst case (i.e. if erratum
 687          * is present on one processor and not on another then assume that the
 688          * erratum is present everywhere).
 689          */
 690         if (cpu_has(&boot_cpu_data, X86_FEATURE_OSVW)) {
 691                 uint64_t len, status = 0;
 692                 int err;
 693
 694                 len = native_read_msr_safe(MSR_AMD64_OSVW_ID_LENGTH, &err);
 695                 if (!err)
 696                         status = native_read_msr_safe(MSR_AMD64_OSVW_STATUS,
 697                                                       &err);
 698
 699                 if (err)
 700                         osvw_status = osvw_len = 0;
 701                 else {
 702                         if (len < osvw_len)
 703                                 osvw_len = len;
 704                         osvw_status |= status;
 705                         osvw_status &= (1ULL << osvw_len) - 1;
 706                 }
 707         } else
 708                 osvw_status = osvw_len = 0;
 709
 710         svm_init_erratum_383();
 711
 712         amd_pmu_enable_virt();
 713
 714         return 0;
 715 }
 716
 717 static void svm_cpu_uninit(int cpu)
 718 {
 719         struct svm_cpu_data *sd = per_cpu(svm_data, raw_smp_processor_id());
 720
 721         if (!sd)
 722                 return;
 723
 724         per_cpu(svm_data, raw_smp_processor_id()) = NULL;
 725         __free_page(sd->save_area);
 726         kfree(sd);
 727 }
 728
 729 static int svm_cpu_init(int cpu)
 730 {
 731         struct svm_cpu_data *sd;
 732         int r;
 733
 734         sd = kzalloc(sizeof(struct svm_cpu_data), GFP_KERNEL);
 735         if (!sd)
 736                 return -ENOMEM;
 737         sd->cpu = cpu;
 738         sd->save_area = alloc_page(GFP_KERNEL);
 739         r = -ENOMEM;
 740         if (!sd->save_area)
 741                 goto err_1;
 742
 743         per_cpu(svm_data, cpu) = sd;
 744
 745         return 0;
 746
 747 err_1:
 748         kfree(sd);
 749         return r;
 750
 751 }
 752
 753 static bool valid_msr_intercept(u32 index)
 754 {
 755         int i;
 756
 757         for (i = 0; direct_access_msrs[i].index != MSR_INVALID; i++)
 758                 if (direct_access_msrs[i].index == index)
 759                         return true;
 760
 761         return false;
 762 }
 763
 764 static void set_msr_interception(u32 *msrpm, unsigned msr,
 765                                  int read, int write)
 766 {
 767         u8 bit_read, bit_write;
 768         unsigned long tmp;
 769         u32 offset;
 770
 771         /*
 772          * If this warning triggers extend the direct_access_msrs list at the
 773          * beginning of the file
 774          */
 775         WARN_ON(!valid_msr_intercept(msr));
 776
 777         offset    = svm_msrpm_offset(msr);
 778         bit_read  = 2 * (msr & 0x0f);
 779         bit_write = 2 * (msr & 0x0f) + 1;
 780         tmp       = msrpm[offset];
 781
 782         BUG_ON(offset == MSR_INVALID);
 783
 784         read  ? clear_bit(bit_read,  &tmp) : set_bit(bit_read,  &tmp);
 785         write ? clear_bit(bit_write, &tmp) : set_bit(bit_write, &tmp);
 786
 787         msrpm[offset] = tmp;
 788 }
 789
 790 static void svm_vcpu_init_msrpm(u32 *msrpm)
 791 {
 792         int i;
 793
 794         memset(msrpm, 0xff, PAGE_SIZE * (1 << MSRPM_ALLOC_ORDER));
 795
 796         for (i = 0; direct_access_msrs[i].index != MSR_INVALID; i++) {
 797                 if (!direct_access_msrs[i].always)
 798                         continue;
 799
 800                 set_msr_interception(msrpm, direct_access_msrs[i].index, 1, 1);
 801         }
 802 }
 803
 804 static void add_msr_offset(u32 offset)
 805 {
 806         int i;
 807
 808         for (i = 0; i < MSRPM_OFFSETS; ++i) {
 809
 810                 /* Offset already in list? */
 811                 if (msrpm_offsets[i] == offset)
 812                         return;
 813
 814                 /* Slot used by another offset? */
 815                 if (msrpm_offsets[i] != MSR_INVALID)
 816                         continue;
 817
 818                 /* Add offset to list */
 819                 msrpm_offsets[i] = offset;
 820
 821                 return;
 822         }
 823
 824         /*
 825          * If this BUG triggers the msrpm_offsets table has an overflow. Just
 826          * increase MSRPM_OFFSETS in this case.
 827          */
 828         BUG();
 829 }
 830
 831 static void init_msrpm_offsets(void)
 832 {
 833         int i;
 834
 835         memset(msrpm_offsets, 0xff, sizeof(msrpm_offsets));
 836
 837         for (i = 0; direct_access_msrs[i].index != MSR_INVALID; i++) {
 838                 u32 offset;
 839
 840                 offset = svm_msrpm_offset(direct_access_msrs[i].index);
 841                 BUG_ON(offset == MSR_INVALID);
 842
 843                 add_msr_offset(offset);
 844         }
 845 }
 846
 847 static void svm_enable_lbrv(struct vcpu_svm *svm)
 848 {
 849         u32 *msrpm = svm->msrpm;
 850
 851         svm->vmcb->control.lbr_ctl = 1;
 852         set_msr_interception(msrpm, MSR_IA32_LASTBRANCHFROMIP, 1, 1);
 853         set_msr_interception(msrpm, MSR_IA32_LASTBRANCHTOIP, 1, 1);
 854         set_msr_interception(msrpm, MSR_IA32_LASTINTFROMIP, 1, 1);
 855         set_msr_interception(msrpm, MSR_IA32_LASTINTTOIP, 1, 1);
 856 }
 857
 858 static void svm_disable_lbrv(struct vcpu_svm *svm)
 859 {
 860         u32 *msrpm = svm->msrpm;
 861
 862         svm->vmcb->control.lbr_ctl = 0;
 863         set_msr_interception(msrpm, MSR_IA32_LASTBRANCHFROMIP, 0, 0);
 864         set_msr_interception(msrpm, MSR_IA32_LASTBRANCHTOIP, 0, 0);
 865         set_msr_interception(msrpm, MSR_IA32_LASTINTFROMIP, 0, 0);
 866         set_msr_interception(msrpm, MSR_IA32_LASTINTTOIP, 0, 0);
 867 }
 868
 869 #define MTRR_TYPE_UC_MINUS      7
 870 #define MTRR2PROTVAL_INVALID 0xff
 871
 872 static u8 mtrr2protval[8];
 873
 874 static u8 fallback_mtrr_type(int mtrr)
 875 {
 876         /*
 877          * WT and WP aren't always available in the host PAT.  Treat
 878          * them as UC and UC- respectively.  Everything else should be
 879          * there.
 880          */
 881         switch (mtrr)
 882         {
 883         case MTRR_TYPE_WRTHROUGH:
 884                 return MTRR_TYPE_UNCACHABLE;
 885         case MTRR_TYPE_WRPROT:
 886                 return MTRR_TYPE_UC_MINUS;
 887         default:
 888                 BUG();
 889         }
 890 }
 891
 892 static void build_mtrr2protval(void)
 893 {
 894         int i;
 895         u64 pat;
 896
 897         for (i = 0; i < 8; i++)
 898                 mtrr2protval[i] = MTRR2PROTVAL_INVALID;
 899
 900         /* Ignore the invalid MTRR types.  */
 901         mtrr2protval[2] = 0;
 902         mtrr2protval[3] = 0;
 903
 904         /*
 905          * Use host PAT value to figure out the mapping from guest MTRR
 906          * values to nested page table PAT/PCD/PWT values.  We do not
 907          * want to change the host PAT value every time we enter the
 908          * guest.
 909          */
 910         rdmsrl(MSR_IA32_CR_PAT, pat);
 911         for (i = 0; i < 8; i++) {
 912                 u8 mtrr = pat >> (8 * i);
 913
 914                 if (mtrr2protval[mtrr] == MTRR2PROTVAL_INVALID)
 915                         mtrr2protval[mtrr] = __cm_idx2pte(i);
 916         }
 917
 918         for (i = 0; i < 8; i++) {
 919                 if (mtrr2protval[i] == MTRR2PROTVAL_INVALID) {
 920                         u8 fallback = fallback_mtrr_type(i);
 921                         mtrr2protval[i] = mtrr2protval[fallback];
 922                         BUG_ON(mtrr2protval[i] == MTRR2PROTVAL_INVALID);
 923                 }
 924         }
 925 }
 926
 927 static __init int svm_hardware_setup(void)
 928 {
 929         int cpu;
 930         struct page *iopm_pages;
 931         void *iopm_va;
 932         int r;
 933
 934         iopm_pages = alloc_pages(GFP_KERNEL, IOPM_ALLOC_ORDER);
 935
 936         if (!iopm_pages)
 937                 return -ENOMEM;
 938
 939         iopm_va = page_address(iopm_pages);
 940         memset(iopm_va, 0xff, PAGE_SIZE * (1 << IOPM_ALLOC_ORDER));
 941         iopm_base = page_to_pfn(iopm_pages) << PAGE_SHIFT;
 942
 943         init_msrpm_offsets();
 944
 945         if (boot_cpu_has(X86_FEATURE_NX))
 946                 kvm_enable_efer_bits(EFER_NX);
 947
 948         if (boot_cpu_has(X86_FEATURE_FXSR_OPT))
 949                 kvm_enable_efer_bits(EFER_FFXSR);
 950
 951         if (boot_cpu_has(X86_FEATURE_TSCRATEMSR)) {
 952                 u64 max;
 953
 954                 kvm_has_tsc_control = true;
 955
 956                 /*
 957                  * Make sure the user can only configure tsc_khz values that
 958                  * fit into a signed integer.
 959                  * A min value is not calculated needed because it will always
 960                  * be 1 on all machines and a value of 0 is used to disable
 961                  * tsc-scaling for the vcpu.
 962                  */
 963                 max = min(0x7fffffffULL, __scale_tsc(tsc_khz, TSC_RATIO_MAX));
 964
 965                 kvm_max_guest_tsc_khz = max;
 966         }
 967
 968         if (nested) {
 969                 printk(KERN_INFO "kvm: Nested Virtualization enabled\n");
 970                 kvm_enable_efer_bits(EFER_SVME | EFER_LMSLE);
 971         }
 972
 973         for_each_possible_cpu(cpu) {
 974                 r = svm_cpu_init(cpu);
 975                 if (r)
 976                         goto err;
 977         }
 978
 979         if (!boot_cpu_has(X86_FEATURE_NPT))
 980                 npt_enabled = false;
 981
 982         if (npt_enabled && !npt) {
 983                 printk(KERN_INFO "kvm: Nested Paging disabled\n");
 984                 npt_enabled = false;
 985         }
 986
 987         if (npt_enabled) {
 988                 printk(KERN_INFO "kvm: Nested Paging enabled\n");
 989                 kvm_enable_tdp();
 990         } else
 991                 kvm_disable_tdp();
 992
 993         build_mtrr2protval();
 994         return 0;
 995
 996 err:
 997         __free_pages(iopm_pages, IOPM_ALLOC_ORDER);
 998         iopm_base = 0;
 999         return r;
1000 }
1001
1002 static __exit void svm_hardware_unsetup(void)
1003 {
1004         int cpu;
1005
1006         for_each_possible_cpu(cpu)
1007                 svm_cpu_uninit(cpu);
1008
1009         __free_pages(pfn_to_page(iopm_base >> PAGE_SHIFT), IOPM_ALLOC_ORDER);
1010         iopm_base = 0;
1011 }
1012
1013 static void init_seg(struct vmcb_seg *seg)
1014 {
1015         seg->selector = 0;
1016         seg->attrib = SVM_SELECTOR_P_MASK | SVM_SELECTOR_S_MASK |
1017                       SVM_SELECTOR_WRITE_MASK; /* Read/Write Data Segment */
1018         seg->limit = 0xffff;
1019         seg->base = 0;
1020 }
1021
1022 static void init_sys_seg(struct vmcb_seg *seg, uint32_t type)
1023 {
1024         seg->selector = 0;
1025         seg->attrib = SVM_SELECTOR_P_MASK | type;
1026         seg->limit = 0xffff;
1027         seg->base = 0;
1028 }
1029
1030 static u64 __scale_tsc(u64 ratio, u64 tsc)
1031 {
1032         u64 mult, frac, _tsc;
1033
1034         mult  = ratio >> 32;
1035         frac  = ratio & ((1ULL << 32) - 1);
1036
1037         _tsc  = tsc;
1038         _tsc *= mult;
1039         _tsc += (tsc >> 32) * frac;
1040         _tsc += ((tsc & ((1ULL << 32) - 1)) * frac) >> 32;
1041
1042         return _tsc;
1043 }
1044
1045 static u64 svm_scale_tsc(struct kvm_vcpu *vcpu, u64 tsc)
1046 {
1047         struct vcpu_svm *svm = to_svm(vcpu);
1048         u64 _tsc = tsc;
1049
1050         if (svm->tsc_ratio != TSC_RATIO_DEFAULT)
1051                 _tsc = __scale_tsc(svm->tsc_ratio, tsc);
1052
1053         return _tsc;
1054 }
1055
1056 static void svm_set_tsc_khz(struct kvm_vcpu *vcpu, u32 user_tsc_khz, bool scale)
1057 {
1058         struct vcpu_svm *svm = to_svm(vcpu);
1059         u64 ratio;
1060         u64 khz;
1061
1062         /* Guest TSC same frequency as host TSC? */
1063         if (!scale) {
1064                 svm->tsc_ratio = TSC_RATIO_DEFAULT;
1065                 return;
1066         }
1067
1068         /* TSC scaling supported? */
1069         if (!boot_cpu_has(X86_FEATURE_TSCRATEMSR)) {
1070                 if (user_tsc_khz > tsc_khz) {
1071                         vcpu->arch.tsc_catchup = 1;
1072                         vcpu->arch.tsc_always_catchup = 1;
1073                 } else
1074                         WARN(1, "user requested TSC rate below hardware speed\n");
1075                 return;
1076         }
1077
1078         khz = user_tsc_khz;
1079
1080         /* TSC scaling required  - calculate ratio */
1081         ratio = khz << 32;
1082         do_div(ratio, tsc_khz);
1083
1084         if (ratio == 0 || ratio & TSC_RATIO_RSVD) {
1085                 WARN_ONCE(1, "Invalid TSC ratio - virtual-tsc-khz=%u\n",
1086                                 user_tsc_khz);
1087                 return;
1088         }
1089         svm->tsc_ratio             = ratio;
1090 }
1091
1092 static u64 svm_read_tsc_offset(struct kvm_vcpu *vcpu)
1093 {
1094         struct vcpu_svm *svm = to_svm(vcpu);
1095
1096         return svm->vmcb->control.tsc_offset;
1097 }
1098
1099 static void svm_write_tsc_offset(struct kvm_vcpu *vcpu, u64 offset)
1100 {
1101         struct vcpu_svm *svm = to_svm(vcpu);
1102         u64 g_tsc_offset = 0;
1103
1104         if (is_guest_mode(vcpu)) {
1105                 g_tsc_offset = svm->vmcb->control.tsc_offset -
1106                                svm->nested.hsave->control.tsc_offset;
1107                 svm->nested.hsave->control.tsc_offset = offset;
1108         } else
1109                 trace_kvm_write_tsc_offset(vcpu->vcpu_id,
1110                                            svm->vmcb->control.tsc_offset,
1111                                            offset);
1112
1113         svm->vmcb->control.tsc_offset = offset + g_tsc_offset;
1114
1115         mark_dirty(svm->vmcb, VMCB_INTERCEPTS);
1116 }
1117
1118 static void svm_adjust_tsc_offset(struct kvm_vcpu *vcpu, s64 adjustment, bool host)
1119 {
1120         struct vcpu_svm *svm = to_svm(vcpu);
1121
1122         if (host) {
1123                 if (svm->tsc_ratio != TSC_RATIO_DEFAULT)
1124                         WARN_ON(adjustment < 0);
1125                 adjustment = svm_scale_tsc(vcpu, (u64)adjustment);
1126         }
1127
1128         svm->vmcb->control.tsc_offset += adjustment;
1129         if (is_guest_mode(vcpu))
1130                 svm->nested.hsave->control.tsc_offset += adjustment;
1131         else
1132                 trace_kvm_write_tsc_offset(vcpu->vcpu_id,
1133                                      svm->vmcb->control.tsc_offset - adjustment,
1134                                      svm->vmcb->control.tsc_offset);
1135
1136         mark_dirty(svm->vmcb, VMCB_INTERCEPTS);
1137 }
1138
1139 static u64 svm_compute_tsc_offset(struct kvm_vcpu *vcpu, u64 target_tsc)
1140 {
1141         u64 tsc;
1142
1143         tsc = svm_scale_tsc(vcpu, rdtsc());
1144
1145         return target_tsc - tsc;
1146 }
1147
1148 static void svm_set_guest_pat(struct vcpu_svm *svm, u64 *g_pat)
1149 {
1150         struct kvm_vcpu *vcpu = &svm->vcpu;
1151
1152         /* Unlike Intel, AMD takes the guest's CR0.CD into account.
1153          *
1154          * AMD does not have IPAT.  To emulate it for the case of guests
1155          * with no assigned devices, just set everything to WB.  If guests
1156          * have assigned devices, however, we cannot force WB for RAM
1157          * pages only, so use the guest PAT directly.
1158          */
1159         if (!kvm_arch_has_assigned_device(vcpu->kvm))
1160                 *g_pat = 0x0606060606060606;
1161         else
1162                 *g_pat = vcpu->arch.pat;
1163 }
1164
1165 static u64 svm_get_mt_mask(struct kvm_vcpu *vcpu, gfn_t gfn, bool is_mmio)
1166 {
1167         u8 mtrr;
1168
1169         /*
1170          * 1. MMIO: always map as UC
1171          * 2. No passthrough: always map as WB, and force guest PAT to WB as well
1172          * 3. Passthrough: can't guarantee the result, try to trust guest.
1173          */
1174         if (is_mmio)
1175                 return _PAGE_NOCACHE;
1176
1177         if (!kvm_arch_has_assigned_device(vcpu->kvm))
1178                 return 0;
1179
1180         if (!kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_CD_NW_CLEARED) &&
1181             kvm_read_cr0(vcpu) & X86_CR0_CD)
1182                 return _PAGE_NOCACHE;
1183
1184         mtrr = kvm_mtrr_get_guest_memory_type(vcpu, gfn);
1185         return mtrr2protval[mtrr];
1186 }
1187
1188 static void init_vmcb(struct vcpu_svm *svm, bool init_event)
1189 {
1190         struct vmcb_control_area *control = &svm->vmcb->control;
1191         struct vmcb_save_area *save = &svm->vmcb->save;
1192
1193         svm->vcpu.fpu_active = 1;
1194         svm->vcpu.arch.hflags = 0;
1195
1196         set_cr_intercept(svm, INTERCEPT_CR0_READ);
1197         set_cr_intercept(svm, INTERCEPT_CR3_READ);
1198         set_cr_intercept(svm, INTERCEPT_CR4_READ);
1199         set_cr_intercept(svm, INTERCEPT_CR0_WRITE);
1200         set_cr_intercept(svm, INTERCEPT_CR3_WRITE);
1201         set_cr_intercept(svm, INTERCEPT_CR4_WRITE);
1202         set_cr_intercept(svm, INTERCEPT_CR8_WRITE);
1203
1204         set_dr_intercepts(svm);
1205
1206         set_exception_intercept(svm, PF_VECTOR);
1207         set_exception_intercept(svm, UD_VECTOR);
1208         set_exception_intercept(svm, MC_VECTOR);
1209
1210         set_intercept(svm, INTERCEPT_INTR);
1211         set_intercept(svm, INTERCEPT_NMI);
1212         set_intercept(svm, INTERCEPT_SMI);
1213         set_intercept(svm, INTERCEPT_SELECTIVE_CR0);
1214         set_intercept(svm, INTERCEPT_RDPMC);
1215         set_intercept(svm, INTERCEPT_CPUID);
1216         set_intercept(svm, INTERCEPT_INVD);
1217         set_intercept(svm, INTERCEPT_HLT);
1218         set_intercept(svm, INTERCEPT_INVLPG);
1219         set_intercept(svm, INTERCEPT_INVLPGA);
1220         set_intercept(svm, INTERCEPT_IOIO_PROT);
1221         set_intercept(svm, INTERCEPT_MSR_PROT);
1222         set_intercept(svm, INTERCEPT_TASK_SWITCH);
1223         set_intercept(svm, INTERCEPT_SHUTDOWN);
1224         set_intercept(svm, INTERCEPT_VMRUN);
1225         set_intercept(svm, INTERCEPT_VMMCALL);
1226         set_intercept(svm, INTERCEPT_VMLOAD);
1227         set_intercept(svm, INTERCEPT_VMSAVE);
1228         set_intercept(svm, INTERCEPT_STGI);
1229         set_intercept(svm, INTERCEPT_CLGI);
1230         set_intercept(svm, INTERCEPT_SKINIT);
1231         set_intercept(svm, INTERCEPT_WBINVD);
1232         set_intercept(svm, INTERCEPT_MONITOR);
1233         set_intercept(svm, INTERCEPT_MWAIT);
1234         set_intercept(svm, INTERCEPT_XSETBV);
1235
1236         control->iopm_base_pa = iopm_base;
1237         control->msrpm_base_pa = __pa(svm->msrpm);
1238         control->int_ctl = V_INTR_MASKING_MASK;
1239
1240         init_seg(&save->es);
1241         init_seg(&save->ss);
1242         init_seg(&save->ds);
1243         init_seg(&save->fs);
1244         init_seg(&save->gs);
1245
1246         save->cs.selector = 0xf000;
1247         save->cs.base = 0xffff0000;
1248         /* Executable/Readable Code Segment */
1249         save->cs.attrib = SVM_SELECTOR_READ_MASK | SVM_SELECTOR_P_MASK |
1250                 SVM_SELECTOR_S_MASK | SVM_SELECTOR_CODE_MASK;
1251         save->cs.limit = 0xffff;
1252
1253         save->gdtr.limit = 0xffff;
1254         save->idtr.limit = 0xffff;
1255
1256         init_sys_seg(&save->ldtr, SEG_TYPE_LDT);
1257         init_sys_seg(&save->tr, SEG_TYPE_BUSY_TSS16);
1258
1259         if (!init_event)
1260                 svm_set_efer(&svm->vcpu, 0);
1261         save->dr6 = 0xffff0ff0;
1262         kvm_set_rflags(&svm->vcpu, 2);
1263         save->rip = 0x0000fff0;
1264         svm->vcpu.arch.regs[VCPU_REGS_RIP] = save->rip;
1265
1266         /*
1267          * svm_set_cr0() sets PG and WP and clears NW and CD on save->cr0.
1268          * It also updates the guest-visible cr0 value.
1269          */
1270         svm_set_cr0(&svm->vcpu, X86_CR0_NW | X86_CR0_CD | X86_CR0_ET);
1271         kvm_mmu_reset_context(&svm->vcpu);
1272
1273         save->cr4 = X86_CR4_PAE;
1274         /* rdx = ?? */
1275
1276         if (npt_enabled) {
1277                 /* Setup VMCB for Nested Paging */
1278                 control->nested_ctl = 1;
1279                 clr_intercept(svm, INTERCEPT_INVLPG);
1280                 clr_exception_intercept(svm, PF_VECTOR);
1281                 clr_cr_intercept(svm, INTERCEPT_CR3_READ);
1282                 clr_cr_intercept(svm, INTERCEPT_CR3_WRITE);
1283                 save->g_pat = svm->vcpu.arch.pat;
1284                 svm_set_guest_pat(svm, &save->g_pat);
1285                 save->cr3 = 0;
1286                 save->cr4 = 0;
1287         }
1288         svm->asid_generation = 0;
1289
1290         svm->nested.vmcb = 0;
1291         svm->vcpu.arch.hflags = 0;
1292
1293         if (boot_cpu_has(X86_FEATURE_PAUSEFILTER)) {
1294                 control->pause_filter_count = 3000;
1295                 set_intercept(svm, INTERCEPT_PAUSE);
1296         }
1297
1298         mark_all_dirty(svm->vmcb);
1299
1300         enable_gif(svm);
1301 }
1302
1303 static void svm_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event)
1304 {
1305         struct vcpu_svm *svm = to_svm(vcpu);
1306         u32 dummy;
1307         u32 eax = 1;
1308
1309         if (!init_event) {
1310                 svm->vcpu.arch.apic_base = APIC_DEFAULT_PHYS_BASE |
1311                                            MSR_IA32_APICBASE_ENABLE;
1312                 if (kvm_vcpu_is_reset_bsp(&svm->vcpu))
1313                         svm->vcpu.arch.apic_base |= MSR_IA32_APICBASE_BSP;
1314         }
1315         init_vmcb(svm, init_event);
1316
1317         kvm_cpuid(vcpu, &eax, &dummy, &dummy, &dummy);
1318         kvm_register_write(vcpu, VCPU_REGS_RDX, eax);
1319 }
1320
1321 static struct kvm_vcpu *svm_create_vcpu(struct kvm *kvm, unsigned int id)
1322 {
1323         struct vcpu_svm *svm;
1324         struct page *page;
1325         struct page *msrpm_pages;
1326         struct page *hsave_page;
1327         struct page *nested_msrpm_pages;
1328         int err;
1329
1330         svm = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL);
1331         if (!svm) {
1332                 err = -ENOMEM;
1333                 goto out;
1334         }
1335
1336         svm->tsc_ratio = TSC_RATIO_DEFAULT;
1337
1338         err = kvm_vcpu_init(&svm->vcpu, kvm, id);
1339         if (err)
1340                 goto free_svm;
1341
1342         err = -ENOMEM;
1343         page = alloc_page(GFP_KERNEL);
1344         if (!page)
1345                 goto uninit;
1346
1347         msrpm_pages = alloc_pages(GFP_KERNEL, MSRPM_ALLOC_ORDER);
1348         if (!msrpm_pages)
1349                 goto free_page1;
1350
1351         nested_msrpm_pages = alloc_pages(GFP_KERNEL, MSRPM_ALLOC_ORDER);
1352         if (!nested_msrpm_pages)
1353                 goto free_page2;
1354
1355         hsave_page = alloc_page(GFP_KERNEL);
1356         if (!hsave_page)
1357                 goto free_page3;
1358
1359         svm->nested.hsave = page_address(hsave_page);
1360
1361         svm->msrpm = page_address(msrpm_pages);
1362         svm_vcpu_init_msrpm(svm->msrpm);
1363
1364         svm->nested.msrpm = page_address(nested_msrpm_pages);
1365         svm_vcpu_init_msrpm(svm->nested.msrpm);
1366
1367         svm->vmcb = page_address(page);
1368         clear_page(svm->vmcb);
1369         svm->vmcb_pa = page_to_pfn(page) << PAGE_SHIFT;
1370         svm->asid_generation = 0;
1371         init_vmcb(svm, false);
1372
1373         svm_init_osvw(&svm->vcpu);
1374
1375         return &svm->vcpu;
1376
1377 free_page3:
1378         __free_pages(nested_msrpm_pages, MSRPM_ALLOC_ORDER);
1379 free_page2:
1380         __free_pages(msrpm_pages, MSRPM_ALLOC_ORDER);
1381 free_page1:
1382         __free_page(page);
1383 uninit:
1384         kvm_vcpu_uninit(&svm->vcpu);
1385 free_svm:
1386         kmem_cache_free(kvm_vcpu_cache, svm);
1387 out:
1388         return ERR_PTR(err);
1389 }
1390
1391 static void svm_free_vcpu(struct kvm_vcpu *vcpu)
1392 {
1393         struct vcpu_svm *svm = to_svm(vcpu);
1394
1395         __free_page(pfn_to_page(svm->vmcb_pa >> PAGE_SHIFT));
1396         __free_pages(virt_to_page(svm->msrpm), MSRPM_ALLOC_ORDER);
1397         __free_page(virt_to_page(svm->nested.hsave));
1398         __free_pages(virt_to_page(svm->nested.msrpm), MSRPM_ALLOC_ORDER);
1399         kvm_vcpu_uninit(vcpu);
1400         kmem_cache_free(kvm_vcpu_cache, svm);
1401 }
1402
1403 static void svm_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
1404 {
1405         struct vcpu_svm *svm = to_svm(vcpu);
1406         int i;
1407
1408         if (unlikely(cpu != vcpu->cpu)) {
1409                 svm->asid_generation = 0;
1410                 mark_all_dirty(svm->vmcb);
1411         }
1412
1413 #ifdef CONFIG_X86_64
1414         rdmsrl(MSR_GS_BASE, to_svm(vcpu)->host.gs_base);
1415 #endif
1416         savesegment(fs, svm->host.fs);
1417         savesegment(gs, svm->host.gs);
1418         svm->host.ldt = kvm_read_ldt();
1419
1420         for (i = 0; i < NR_HOST_SAVE_USER_MSRS; i++)
1421                 rdmsrl(host_save_user_msrs[i], svm->host_user_msrs[i]);
1422
1423         if (static_cpu_has(X86_FEATURE_TSCRATEMSR) &&
1424             svm->tsc_ratio != __this_cpu_read(current_tsc_ratio)) {
1425                 __this_cpu_write(current_tsc_ratio, svm->tsc_ratio);
1426                 wrmsrl(MSR_AMD64_TSC_RATIO, svm->tsc_ratio);
1427         }
1428 }
1429
1430 static void svm_vcpu_put(struct kvm_vcpu *vcpu)
1431 {
1432         struct vcpu_svm *svm = to_svm(vcpu);
1433         int i;
1434
1435         ++vcpu->stat.host_state_reload;
1436         kvm_load_ldt(svm->host.ldt);
1437 #ifdef CONFIG_X86_64
1438         loadsegment(fs, svm->host.fs);
1439         wrmsrl(MSR_KERNEL_GS_BASE, current->thread.gs);
1440         load_gs_index(svm->host.gs);
1441 #else
1442 #ifdef CONFIG_X86_32_LAZY_GS
1443         loadsegment(gs, svm->host.gs);
1444 #endif
1445 #endif
1446         for (i = 0; i < NR_HOST_SAVE_USER_MSRS; i++)
1447                 wrmsrl(host_save_user_msrs[i], svm->host_user_msrs[i]);
1448 }
1449
1450 static unsigned long svm_get_rflags(struct kvm_vcpu *vcpu)
1451 {
1452         return to_svm(vcpu)->vmcb->save.rflags;
1453 }
1454
1455 static void svm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags)
1456 {
1457        /*
1458         * Any change of EFLAGS.VM is accompained by a reload of SS
1459         * (caused by either a task switch or an inter-privilege IRET),
1460         * so we do not need to update the CPL here.
1461         */
1462         to_svm(vcpu)->vmcb->save.rflags = rflags;
1463 }
1464
1465 static void svm_cache_reg(struct kvm_vcpu *vcpu, enum kvm_reg reg)
1466 {
1467         switch (reg) {
1468         case VCPU_EXREG_PDPTR:
1469                 BUG_ON(!npt_enabled);
1470                 load_pdptrs(vcpu, vcpu->arch.walk_mmu, kvm_read_cr3(vcpu));
1471                 break;
1472         default:
1473                 BUG();
1474         }
1475 }
1476
1477 static void svm_set_vintr(struct vcpu_svm *svm)
1478 {
1479         set_intercept(svm, INTERCEPT_VINTR);
1480 }
1481
1482 static void svm_clear_vintr(struct vcpu_svm *svm)
1483 {
1484         clr_intercept(svm, INTERCEPT_VINTR);
1485 }
1486
1487 static struct vmcb_seg *svm_seg(struct kvm_vcpu *vcpu, int seg)
1488 {
1489         struct vmcb_save_area *save = &to_svm(vcpu)->vmcb->save;
1490
1491         switch (seg) {
1492         case VCPU_SREG_CS: return &save->cs;
1493         case VCPU_SREG_DS: return &save->ds;
1494         case VCPU_SREG_ES: return &save->es;
1495         case VCPU_SREG_FS: return &save->fs;
1496         case VCPU_SREG_GS: return &save->gs;
1497         case VCPU_SREG_SS: return &save->ss;
1498         case VCPU_SREG_TR: return &save->tr;
1499         case VCPU_SREG_LDTR: return &save->ldtr;
1500         }
1501         BUG();
1502         return NULL;
1503 }
1504
1505 static u64 svm_get_segment_base(struct kvm_vcpu *vcpu, int seg)
1506 {
1507         struct vmcb_seg *s = svm_seg(vcpu, seg);
1508
1509         return s->base;
1510 }
1511
1512 static void svm_get_segment(struct kvm_vcpu *vcpu,
1513                             struct kvm_segment *var, int seg)
1514 {
1515         struct vmcb_seg *s = svm_seg(vcpu, seg);
1516
1517         var->base = s->base;
1518         var->limit = s->limit;
1519         var->selector = s->selector;
1520         var->type = s->attrib & SVM_SELECTOR_TYPE_MASK;
1521         var->s = (s->attrib >> SVM_SELECTOR_S_SHIFT) & 1;
1522         var->dpl = (s->attrib >> SVM_SELECTOR_DPL_SHIFT) & 3;
1523         var->present = (s->attrib >> SVM_SELECTOR_P_SHIFT) & 1;
1524         var->avl = (s->attrib >> SVM_SELECTOR_AVL_SHIFT) & 1;
1525         var->l = (s->attrib >> SVM_SELECTOR_L_SHIFT) & 1;
1526         var->db = (s->attrib >> SVM_SELECTOR_DB_SHIFT) & 1;
1527
1528         /*
1529          * AMD CPUs circa 2014 track the G bit for all segments except CS.
1530          * However, the SVM spec states that the G bit is not observed by the
1531          * CPU, and some VMware virtual CPUs drop the G bit for all segments.
1532          * So let's synthesize a legal G bit for all segments, this helps
1533          * running KVM nested. It also helps cross-vendor migration, because
1534          * Intel's vmentry has a check on the 'G' bit.
1535          */
1536         var->g = s->limit > 0xfffff;
1537
1538         /*
1539          * AMD's VMCB does not have an explicit unusable field, so emulate it
1540          * for cross vendor migration purposes by "not present"
1541          */
1542         var->unusable = !var->present || (var->type == 0);
1543
1544         switch (seg) {
1545         case VCPU_SREG_TR:
1546                 /*
1547                  * Work around a bug where the busy flag in the tr selector
1548                  * isn't exposed
1549                  */
1550                 var->type |= 0x2;
1551                 break;
1552         case VCPU_SREG_DS:
1553         case VCPU_SREG_ES:
1554         case VCPU_SREG_FS:
1555         case VCPU_SREG_GS:
1556                 /*
1557                  * The accessed bit must always be set in the segment
1558                  * descriptor cache, although it can be cleared in the
1559                  * descriptor, the cached bit always remains at 1. Since
1560                  * Intel has a check on this, set it here to support
1561                  * cross-vendor migration.
1562                  */
1563                 if (!var->unusable)
1564                         var->type |= 0x1;
1565                 break;
1566         case VCPU_SREG_SS:
1567                 /*
1568                  * On AMD CPUs sometimes the DB bit in the segment
1569                  * descriptor is left as 1, although the whole segment has
1570                  * been made unusable. Clear it here to pass an Intel VMX
1571                  * entry check when cross vendor migrating.
1572                  */
1573                 if (var->unusable)
1574                         var->db = 0;
1575                 var->dpl = to_svm(vcpu)->vmcb->save.cpl;
1576                 break;
1577         }
1578 }
1579
1580 static int svm_get_cpl(struct kvm_vcpu *vcpu)
1581 {
1582         struct vmcb_save_area *save = &to_svm(vcpu)->vmcb->save;
1583
1584         return save->cpl;
1585 }
1586
1587 static void svm_get_idt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
1588 {
1589         struct vcpu_svm *svm = to_svm(vcpu);
1590
1591         dt->size = svm->vmcb->save.idtr.limit;
1592         dt->address = svm->vmcb->save.idtr.base;
1593 }
1594
1595 static void svm_set_idt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
1596 {
1597         struct vcpu_svm *svm = to_svm(vcpu);
1598
1599         svm->vmcb->save.idtr.limit = dt->size;
1600         svm->vmcb->save.idtr.base = dt->address ;
1601         mark_dirty(svm->vmcb, VMCB_DT);
1602 }
1603
1604 static void svm_get_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
1605 {
1606         struct vcpu_svm *svm = to_svm(vcpu);
1607
1608         dt->size = svm->vmcb->save.gdtr.limit;
1609         dt->address = svm->vmcb->save.gdtr.base;
1610 }
1611
1612 static void svm_set_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
1613 {
1614         struct vcpu_svm *svm = to_svm(vcpu);
1615
1616         svm->vmcb->save.gdtr.limit = dt->size;
1617         svm->vmcb->save.gdtr.base = dt->address ;
1618         mark_dirty(svm->vmcb, VMCB_DT);
1619 }
1620
1621 static void svm_decache_cr0_guest_bits(struct kvm_vcpu *vcpu)
1622 {
1623 }
1624
1625 static void svm_decache_cr3(struct kvm_vcpu *vcpu)
1626 {
1627 }
1628
1629 static void svm_decache_cr4_guest_bits(struct kvm_vcpu *vcpu)
1630 {
1631 }
1632
1633 static void update_cr0_intercept(struct vcpu_svm *svm)
1634 {
1635         ulong gcr0 = svm->vcpu.arch.cr0;
1636         u64 *hcr0 = &svm->vmcb->save.cr0;
1637
1638         if (!svm->vcpu.fpu_active)
1639                 *hcr0 |= SVM_CR0_SELECTIVE_MASK;
1640         else
1641                 *hcr0 = (*hcr0 & ~SVM_CR0_SELECTIVE_MASK)
1642                         | (gcr0 & SVM_CR0_SELECTIVE_MASK);
1643
1644         mark_dirty(svm->vmcb, VMCB_CR);
1645
1646         if (gcr0 == *hcr0 && svm->vcpu.fpu_active) {
1647                 clr_cr_intercept(svm, INTERCEPT_CR0_READ);
1648                 clr_cr_intercept(svm, INTERCEPT_CR0_WRITE);
1649         } else {
1650                 set_cr_intercept(svm, INTERCEPT_CR0_READ);
1651                 set_cr_intercept(svm, INTERCEPT_CR0_WRITE);
1652         }
1653 }
1654
1655 static void svm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
1656 {
1657         struct vcpu_svm *svm = to_svm(vcpu);
1658
1659 #ifdef CONFIG_X86_64
1660         if (vcpu->arch.efer & EFER_LME) {
1661                 if (!is_paging(vcpu) && (cr0 & X86_CR0_PG)) {
1662                         vcpu->arch.efer |= EFER_LMA;
1663                         svm->vmcb->save.efer |= EFER_LMA | EFER_LME;
1664                 }
1665
1666                 if (is_paging(vcpu) && !(cr0 & X86_CR0_PG)) {
1667                         vcpu->arch.efer &= ~EFER_LMA;
1668                         svm->vmcb->save.efer &= ~(EFER_LMA | EFER_LME);
1669                 }
1670         }
1671 #endif
1672         vcpu->arch.cr0 = cr0;
1673
1674         if (!npt_enabled)
1675                 cr0 |= X86_CR0_PG | X86_CR0_WP;
1676
1677         if (!vcpu->fpu_active)
1678                 cr0 |= X86_CR0_TS;
1679
1680         /* These are emulated via page tables.  */
1681         cr0 &= ~(X86_CR0_CD | X86_CR0_NW);
1682
1683         svm->vmcb->save.cr0 = cr0;
1684         mark_dirty(svm->vmcb, VMCB_CR);
1685         update_cr0_intercept(svm);
1686 }
1687
1688 static int svm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
1689 {
1690         unsigned long host_cr4_mce = cr4_read_shadow() & X86_CR4_MCE;
1691         unsigned long old_cr4 = to_svm(vcpu)->vmcb->save.cr4;
1692
1693         if (cr4 & X86_CR4_VMXE)
1694                 return 1;
1695
1696         if (npt_enabled && ((old_cr4 ^ cr4) & X86_CR4_PGE))
1697                 svm_flush_tlb(vcpu);
1698
1699         vcpu->arch.cr4 = cr4;
1700         if (!npt_enabled)
1701                 cr4 |= X86_CR4_PAE;
1702         cr4 |= host_cr4_mce;
1703         to_svm(vcpu)->vmcb->save.cr4 = cr4;
1704         mark_dirty(to_svm(vcpu)->vmcb, VMCB_CR);
1705         return 0;
1706 }
1707
1708 static void svm_set_segment(struct kvm_vcpu *vcpu,
1709                             struct kvm_segment *var, int seg)
1710 {
1711         struct vcpu_svm *svm = to_svm(vcpu);
1712         struct vmcb_seg *s = svm_seg(vcpu, seg);
1713
1714         s->base = var->base;
1715         s->limit = var->limit;
1716         s->selector = var->selector;
1717         if (var->unusable)
1718                 s->attrib = 0;
1719         else {
1720                 s->attrib = (var->type & SVM_SELECTOR_TYPE_MASK);
1721                 s->attrib |= (var->s & 1) << SVM_SELECTOR_S_SHIFT;
1722                 s->attrib |= (var->dpl & 3) << SVM_SELECTOR_DPL_SHIFT;
1723                 s->attrib |= (var->present & 1) << SVM_SELECTOR_P_SHIFT;
1724                 s->attrib |= (var->avl & 1) << SVM_SELECTOR_AVL_SHIFT;
1725                 s->attrib |= (var->l & 1) << SVM_SELECTOR_L_SHIFT;
1726                 s->attrib |= (var->db & 1) << SVM_SELECTOR_DB_SHIFT;
1727                 s->attrib |= (var->g & 1) << SVM_SELECTOR_G_SHIFT;
1728         }
1729
1730         /*
1731          * This is always accurate, except if SYSRET returned to a segment
1732          * with SS.DPL != 3.  Intel does not have this quirk, and always
1733          * forces SS.DPL to 3 on sysret, so we ignore that case; fixing it
1734          * would entail passing the CPL to userspace and back.
1735          */
1736         if (seg == VCPU_SREG_SS)
1737                 svm->vmcb->save.cpl = (s->attrib >> SVM_SELECTOR_DPL_SHIFT) & 3;
1738
1739         mark_dirty(svm->vmcb, VMCB_SEG);
1740 }
1741
1742 static void update_db_bp_intercept(struct kvm_vcpu *vcpu)
1743 {
1744         struct vcpu_svm *svm = to_svm(vcpu);
1745
1746         clr_exception_intercept(svm, DB_VECTOR);
1747         clr_exception_intercept(svm, BP_VECTOR);
1748
1749         if (svm->nmi_singlestep)
1750                 set_exception_intercept(svm, DB_VECTOR);
1751
1752         if (vcpu->guest_debug & KVM_GUESTDBG_ENABLE) {
1753                 if (vcpu->guest_debug &
1754                     (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP))
1755                         set_exception_intercept(svm, DB_VECTOR);
1756                 if (vcpu->guest_debug & KVM_GUESTDBG_USE_SW_BP)
1757                         set_exception_intercept(svm, BP_VECTOR);
1758         } else
1759                 vcpu->guest_debug = 0;
1760 }
1761
1762 static void new_asid(struct vcpu_svm *svm, struct svm_cpu_data *sd)
1763 {
1764         if (sd->next_asid > sd->max_asid) {
1765                 ++sd->asid_generation;
1766                 sd->next_asid = 1;
1767                 svm->vmcb->control.tlb_ctl = TLB_CONTROL_FLUSH_ALL_ASID;
1768         }
1769
1770         svm->asid_generation = sd->asid_generation;
1771         svm->vmcb->control.asid = sd->next_asid++;
1772
1773         mark_dirty(svm->vmcb, VMCB_ASID);
1774 }
1775
1776 static u64 svm_get_dr6(struct kvm_vcpu *vcpu)
1777 {
1778         return to_svm(vcpu)->vmcb->save.dr6;
1779 }
1780
1781 static void svm_set_dr6(struct kvm_vcpu *vcpu, unsigned long value)
1782 {
1783         struct vcpu_svm *svm = to_svm(vcpu);
1784
1785         svm->vmcb->save.dr6 = value;
1786         mark_dirty(svm->vmcb, VMCB_DR);
1787 }
1788
1789 static void svm_sync_dirty_debug_regs(struct kvm_vcpu *vcpu)
1790 {
1791         struct vcpu_svm *svm = to_svm(vcpu);
1792
1793         get_debugreg(vcpu->arch.db[0], 0);
1794         get_debugreg(vcpu->arch.db[1], 1);
1795         get_debugreg(vcpu->arch.db[2], 2);
1796         get_debugreg(vcpu->arch.db[3], 3);
1797         vcpu->arch.dr6 = svm_get_dr6(vcpu);
1798         vcpu->arch.dr7 = svm->vmcb->save.dr7;
1799
1800         vcpu->arch.switch_db_regs &= ~KVM_DEBUGREG_WONT_EXIT;
1801         set_dr_intercepts(svm);
1802 }
1803
1804 static void svm_set_dr7(struct kvm_vcpu *vcpu, unsigned long value)
1805 {
1806         struct vcpu_svm *svm = to_svm(vcpu);
1807
1808         svm->vmcb->save.dr7 = value;
1809         mark_dirty(svm->vmcb, VMCB_DR);
1810 }
1811
1812 static int pf_interception(struct vcpu_svm *svm)
1813 {
1814         u64 fault_address = svm->vmcb->control.exit_info_2;
1815         u32 error_code;
1816         int r = 1;
1817
1818         switch (svm->apf_reason) {
1819         default:
1820                 error_code = svm->vmcb->control.exit_info_1;
1821
1822                 trace_kvm_page_fault(fault_address, error_code);
1823                 if (!npt_enabled && kvm_event_needs_reinjection(&svm->vcpu))
1824                         kvm_mmu_unprotect_page_virt(&svm->vcpu, fault_address);
1825                 r = kvm_mmu_page_fault(&svm->vcpu, fault_address, error_code,
1826                         svm->vmcb->control.insn_bytes,
1827                         svm->vmcb->control.insn_len);
1828                 break;
1829         case KVM_PV_REASON_PAGE_NOT_PRESENT:
1830                 svm->apf_reason = 0;
1831                 local_irq_disable();
1832                 kvm_async_pf_task_wait(fault_address);
1833                 local_irq_enable();
1834                 break;
1835         case KVM_PV_REASON_PAGE_READY:
1836                 svm->apf_reason = 0;
1837                 local_irq_disable();
1838                 kvm_async_pf_task_wake(fault_address);
1839                 local_irq_enable();
1840                 break;
1841         }
1842         return r;
1843 }
1844
1845 static int db_interception(struct vcpu_svm *svm)
1846 {
1847         struct kvm_run *kvm_run = svm->vcpu.run;
1848
1849         if (!(svm->vcpu.guest_debug &
1850               (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP)) &&
1851                 !svm->nmi_singlestep) {
1852                 kvm_queue_exception(&svm->vcpu, DB_VECTOR);
1853                 return 1;
1854         }
1855
1856         if (svm->nmi_singlestep) {
1857                 svm->nmi_singlestep = false;
1858                 if (!(svm->vcpu.guest_debug & KVM_GUESTDBG_SINGLESTEP))
1859                         svm->vmcb->save.rflags &=
1860                                 ~(X86_EFLAGS_TF | X86_EFLAGS_RF);
1861                 update_db_bp_intercept(&svm->vcpu);
1862         }
1863
1864         if (svm->vcpu.guest_debug &
1865             (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP)) {
1866                 kvm_run->exit_reason = KVM_EXIT_DEBUG;
1867                 kvm_run->debug.arch.pc =
1868                         svm->vmcb->save.cs.base + svm->vmcb->save.rip;
1869                 kvm_run->debug.arch.exception = DB_VECTOR;
1870                 return 0;
1871         }
1872
1873         return 1;
1874 }
1875
1876 static int bp_interception(struct vcpu_svm *svm)
1877 {
1878         struct kvm_run *kvm_run = svm->vcpu.run;
1879
1880         kvm_run->exit_reason = KVM_EXIT_DEBUG;
1881         kvm_run->debug.arch.pc = svm->vmcb->save.cs.base + svm->vmcb->save.rip;
1882         kvm_run->debug.arch.exception = BP_VECTOR;
1883         return 0;
1884 }
1885
1886 static int ud_interception(struct vcpu_svm *svm)
1887 {
1888         int er;
1889
1890         er = emulate_instruction(&svm->vcpu, EMULTYPE_TRAP_UD);
1891         if (er != EMULATE_DONE)
1892                 kvm_queue_exception(&svm->vcpu, UD_VECTOR);
1893         return 1;
1894 }
1895
1896 static void svm_fpu_activate(struct kvm_vcpu *vcpu)
1897 {
1898         struct vcpu_svm *svm = to_svm(vcpu);
1899
1900         clr_exception_intercept(svm, NM_VECTOR);
1901
1902         svm->vcpu.fpu_active = 1;
1903         update_cr0_intercept(svm);
1904 }
1905
1906 static int nm_interception(struct vcpu_svm *svm)
1907 {
1908         svm_fpu_activate(&svm->vcpu);
1909         return 1;
1910 }
1911
1912 static bool is_erratum_383(void)
1913 {
1914         int err, i;
1915         u64 value;
1916
1917         if (!erratum_383_found)
1918                 return false;
1919
1920         value = native_read_msr_safe(MSR_IA32_MC0_STATUS, &err);
1921         if (err)
1922                 return false;
1923
1924         /* Bit 62 may or may not be set for this mce */
1925         value &= ~(1ULL << 62);
1926
1927         if (value != 0xb600000000010015ULL)
1928                 return false;
1929
1930         /* Clear MCi_STATUS registers */
1931         for (i = 0; i < 6; ++i)
1932                 native_write_msr_safe(MSR_IA32_MCx_STATUS(i), 0, 0);
1933
1934         value = native_read_msr_safe(MSR_IA32_MCG_STATUS, &err);
1935         if (!err) {
1936                 u32 low, high;
1937
1938                 value &= ~(1ULL << 2);
1939                 low    = lower_32_bits(value);
1940                 high   = upper_32_bits(value);
1941
1942                 native_write_msr_safe(MSR_IA32_MCG_STATUS, low, high);
1943         }
1944
1945         /* Flush tlb to evict multi-match entries */
1946         __flush_tlb_all();
1947
1948         return true;
1949 }
1950
1951 static void svm_handle_mce(struct vcpu_svm *svm)
1952 {
1953         if (is_erratum_383()) {
1954                 /*
1955                  * Erratum 383 triggered. Guest state is corrupt so kill the
1956                  * guest.
1957                  */
1958                 pr_err("KVM: Guest triggered AMD Erratum 383\n");
1959
1960                 kvm_make_request(KVM_REQ_TRIPLE_FAULT, &svm->vcpu);
1961
1962                 return;
1963         }
1964
1965         /*
1966          * On an #MC intercept the MCE handler is not called automatically in
1967          * the host. So do it by hand here.
1968          */
1969         asm volatile (
1970                 "int $0x12\n");
1971         /* not sure if we ever come back to this point */
1972
1973         return;
1974 }
1975
1976 static int mc_interception(struct vcpu_svm *svm)
1977 {
1978         return 1;
1979 }
1980
1981 static int shutdown_interception(struct vcpu_svm *svm)
1982 {
1983         struct kvm_run *kvm_run = svm->vcpu.run;
1984
1985         /*
1986          * VMCB is undefined after a SHUTDOWN intercept
1987          * so reinitialize it.
1988          */
1989         clear_page(svm->vmcb);
1990         init_vmcb(svm, false);
1991
1992         kvm_run->exit_reason = KVM_EXIT_SHUTDOWN;
1993         return 0;
1994 }
1995
1996 static int io_interception(struct vcpu_svm *svm)
1997 {
1998         struct kvm_vcpu *vcpu = &svm->vcpu;
1999         u32 io_info = svm->vmcb->control.exit_info_1; /* address size bug? */
2000         int size, in, string;
2001         unsigned port;
2002
2003         ++svm->vcpu.stat.io_exits;
2004         string = (io_info & SVM_IOIO_STR_MASK) != 0;
2005         in = (io_info & SVM_IOIO_TYPE_MASK) != 0;
2006         if (string || in)
2007                 return emulate_instruction(vcpu, 0) == EMULATE_DONE;
2008
2009         port = io_info >> 16;
2010         size = (io_info & SVM_IOIO_SIZE_MASK) >> SVM_IOIO_SIZE_SHIFT;
2011         svm->next_rip = svm->vmcb->control.exit_info_2;
2012         skip_emulated_instruction(&svm->vcpu);
2013
2014         return kvm_fast_pio_out(vcpu, size, port);
2015 }
2016
2017 static int nmi_interception(struct vcpu_svm *svm)
2018 {
2019         return 1;
2020 }
2021
2022 static int intr_interception(struct vcpu_svm *svm)
2023 {
2024         ++svm->vcpu.stat.irq_exits;
2025         return 1;
2026 }
2027
2028 static int nop_on_interception(struct vcpu_svm *svm)
2029 {
2030         return 1;
2031 }
2032
2033 static int halt_interception(struct vcpu_svm *svm)
2034 {
2035         svm->next_rip = kvm_rip_read(&svm->vcpu) + 1;
2036         return kvm_emulate_halt(&svm->vcpu);
2037 }
2038
2039 static int vmmcall_interception(struct vcpu_svm *svm)
2040 {
2041         svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
2042         kvm_emulate_hypercall(&svm->vcpu);
2043         return 1;
2044 }
2045
2046 static unsigned long nested_svm_get_tdp_cr3(struct kvm_vcpu *vcpu)
2047 {
2048         struct vcpu_svm *svm = to_svm(vcpu);
2049
2050         return svm->nested.nested_cr3;
2051 }
2052
2053 static u64 nested_svm_get_tdp_pdptr(struct kvm_vcpu *vcpu, int index)
2054 {
2055         struct vcpu_svm *svm = to_svm(vcpu);
2056         u64 cr3 = svm->nested.nested_cr3;
2057         u64 pdpte;
2058         int ret;
2059
2060         ret = kvm_vcpu_read_guest_page(vcpu, gpa_to_gfn(cr3), &pdpte,
2061                                        offset_in_page(cr3) + index * 8, 8);
2062         if (ret)
2063                 return 0;
2064         return pdpte;
2065 }
2066
2067 static void nested_svm_set_tdp_cr3(struct kvm_vcpu *vcpu,
2068                                    unsigned long root)
2069 {
2070         struct vcpu_svm *svm = to_svm(vcpu);
2071
2072         svm->vmcb->control.nested_cr3 = root;
2073         mark_dirty(svm->vmcb, VMCB_NPT);
2074         svm_flush_tlb(vcpu);
2075 }
2076
2077 static void nested_svm_inject_npf_exit(struct kvm_vcpu *vcpu,
2078                                        struct x86_exception *fault)
2079 {
2080         struct vcpu_svm *svm = to_svm(vcpu);
2081
2082         if (svm->vmcb->control.exit_code != SVM_EXIT_NPF) {
2083                 /*
2084                  * TODO: track the cause of the nested page fault, and
2085                  * correctly fill in the high bits of exit_info_1.
2086                  */
2087                 svm->vmcb->control.exit_code = SVM_EXIT_NPF;
2088                 svm->vmcb->control.exit_code_hi = 0;
2089                 svm->vmcb->control.exit_info_1 = (1ULL << 32);
2090                 svm->vmcb->control.exit_info_2 = fault->address;
2091         }
2092
2093         svm->vmcb->control.exit_info_1 &= ~0xffffffffULL;
2094         svm->vmcb->control.exit_info_1 |= fault->error_code;
2095
2096         /*
2097          * The present bit is always zero for page structure faults on real
2098          * hardware.
2099          */
2100         if (svm->vmcb->control.exit_info_1 & (2ULL << 32))
2101                 svm->vmcb->control.exit_info_1 &= ~1;
2102
2103         nested_svm_vmexit(svm);
2104 }
2105
2106 static void nested_svm_init_mmu_context(struct kvm_vcpu *vcpu)
2107 {
2108         WARN_ON(mmu_is_nested(vcpu));
2109         kvm_init_shadow_mmu(vcpu);
2110         vcpu->arch.mmu.set_cr3           = nested_svm_set_tdp_cr3;
2111         vcpu->arch.mmu.get_cr3           = nested_svm_get_tdp_cr3;
2112         vcpu->arch.mmu.get_pdptr         = nested_svm_get_tdp_pdptr;
2113         vcpu->arch.mmu.inject_page_fault = nested_svm_inject_npf_exit;
2114         vcpu->arch.mmu.shadow_root_level = get_npt_level();
2115         reset_shadow_zero_bits_mask(vcpu, &vcpu->arch.mmu);
2116         vcpu->arch.walk_mmu              = &vcpu->arch.nested_mmu;
2117 }
2118
2119 static void nested_svm_uninit_mmu_context(struct kvm_vcpu *vcpu)
2120 {
2121         vcpu->arch.walk_mmu = &vcpu->arch.mmu;
2122 }
2123
2124 static int nested_svm_check_permissions(struct vcpu_svm *svm)
2125 {
2126         if (!(svm->vcpu.arch.efer & EFER_SVME)
2127             || !is_paging(&svm->vcpu)) {
2128                 kvm_queue_exception(&svm->vcpu, UD_VECTOR);
2129                 return 1;
2130         }
2131
2132         if (svm->vmcb->save.cpl) {
2133                 kvm_inject_gp(&svm->vcpu, 0);
2134                 return 1;
2135         }
2136
2137        return 0;
2138 }
2139
2140 static int nested_svm_check_exception(struct vcpu_svm *svm, unsigned nr,
2141                                       bool has_error_code, u32 error_code)
2142 {
2143         int vmexit;
2144
2145         if (!is_guest_mode(&svm->vcpu))
2146                 return 0;
2147
2148         svm->vmcb->control.exit_code = SVM_EXIT_EXCP_BASE + nr;
2149         svm->vmcb->control.exit_code_hi = 0;
2150         svm->vmcb->control.exit_info_1 = error_code;
2151         svm->vmcb->control.exit_info_2 = svm->vcpu.arch.cr2;
2152
2153         vmexit = nested_svm_intercept(svm);
2154         if (vmexit == NESTED_EXIT_DONE)
2155                 svm->nested.exit_required = true;
2156
2157         return vmexit;
2158 }
2159
2160 /* This function returns true if it is save to enable the irq window */
2161 static inline bool nested_svm_intr(struct vcpu_svm *svm)
2162 {
2163         if (!is_guest_mode(&svm->vcpu))
2164                 return true;
2165
2166         if (!(svm->vcpu.arch.hflags & HF_VINTR_MASK))
2167                 return true;
2168
2169         if (!(svm->vcpu.arch.hflags & HF_HIF_MASK))
2170                 return false;
2171
2172         /*
2173          * if vmexit was already requested (by intercepted exception
2174          * for instance) do not overwrite it with "external interrupt"
2175          * vmexit.
2176          */
2177         if (svm->nested.exit_required)
2178                 return false;
2179
2180         svm->vmcb->control.exit_code   = SVM_EXIT_INTR;
2181         svm->vmcb->control.exit_info_1 = 0;
2182         svm->vmcb->control.exit_info_2 = 0;
2183
2184         if (svm->nested.intercept & 1ULL) {
2185                 /*
2186                  * The #vmexit can't be emulated here directly because this
2187                  * code path runs with irqs and preemption disabled. A
2188                  * #vmexit emulation might sleep. Only signal request for
2189                  * the #vmexit here.
2190                  */
2191                 svm->nested.exit_required = true;
2192                 trace_kvm_nested_intr_vmexit(svm->vmcb->save.rip);
2193                 return false;
2194         }
2195
2196         return true;
2197 }
2198
2199 /* This function returns true if it is save to enable the nmi window */
2200 static inline bool nested_svm_nmi(struct vcpu_svm *svm)
2201 {
2202         if (!is_guest_mode(&svm->vcpu))
2203                 return true;
2204
2205         if (!(svm->nested.intercept & (1ULL << INTERCEPT_NMI)))
2206                 return true;
2207
2208         svm->vmcb->control.exit_code = SVM_EXIT_NMI;
2209         svm->nested.exit_required = true;
2210
2211         return false;
2212 }
2213
2214 static void *nested_svm_map(struct vcpu_svm *svm, u64 gpa, struct page **_page)
2215 {
2216         struct page *page;
2217
2218         might_sleep();
2219
2220         page = kvm_vcpu_gfn_to_page(&svm->vcpu, gpa >> PAGE_SHIFT);
2221         if (is_error_page(page))
2222                 goto error;
2223
2224         *_page = page;
2225
2226         return kmap(page);
2227
2228 error:
2229         kvm_inject_gp(&svm->vcpu, 0);
2230
2231         return NULL;
2232 }
2233
2234 static void nested_svm_unmap(struct page *page)
2235 {
2236         kunmap(page);
2237         kvm_release_page_dirty(page);
2238 }
2239
2240 static int nested_svm_intercept_ioio(struct vcpu_svm *svm)
2241 {
2242         unsigned port, size, iopm_len;
2243         u16 val, mask;
2244         u8 start_bit;
2245         u64 gpa;
2246
2247         if (!(svm->nested.intercept & (1ULL << INTERCEPT_IOIO_PROT)))
2248                 return NESTED_EXIT_HOST;
2249
2250         port = svm->vmcb->control.exit_info_1 >> 16;
2251         size = (svm->vmcb->control.exit_info_1 & SVM_IOIO_SIZE_MASK) >>
2252                 SVM_IOIO_SIZE_SHIFT;
2253         gpa  = svm->nested.vmcb_iopm + (port / 8);
2254         start_bit = port % 8;
2255         iopm_len = (start_bit + size > 8) ? 2 : 1;
2256         mask = (0xf >> (4 - size)) << start_bit;
2257         val = 0;
2258
2259         if (kvm_vcpu_read_guest(&svm->vcpu, gpa, &val, iopm_len))
2260                 return NESTED_EXIT_DONE;
2261
2262         return (val & mask) ? NESTED_EXIT_DONE : NESTED_EXIT_HOST;
2263 }
2264
2265 static int nested_svm_exit_handled_msr(struct vcpu_svm *svm)
2266 {
2267         u32 offset, msr, value;
2268         int write, mask;
2269
2270         if (!(svm->nested.intercept & (1ULL << INTERCEPT_MSR_PROT)))
2271                 return NESTED_EXIT_HOST;
2272
2273         msr    = svm->vcpu.arch.regs[VCPU_REGS_RCX];
2274         offset = svm_msrpm_offset(msr);
2275         write  = svm->vmcb->control.exit_info_1 & 1;
2276         mask   = 1 << ((2 * (msr & 0xf)) + write);
2277
2278         if (offset == MSR_INVALID)
2279                 return NESTED_EXIT_DONE;
2280
2281         /* Offset is in 32 bit units but need in 8 bit units */
2282         offset *= 4;
2283
2284         if (kvm_vcpu_read_guest(&svm->vcpu, svm->nested.vmcb_msrpm + offset, &value, 4))
2285                 return NESTED_EXIT_DONE;
2286
2287         return (value & mask) ? NESTED_EXIT_DONE : NESTED_EXIT_HOST;
2288 }
2289
2290 static int nested_svm_exit_special(struct vcpu_svm *svm)
2291 {
2292         u32 exit_code = svm->vmcb->control.exit_code;
2293
2294         switch (exit_code) {
2295         case SVM_EXIT_INTR:
2296         case SVM_EXIT_NMI:
2297         case SVM_EXIT_EXCP_BASE + MC_VECTOR:
2298                 return NESTED_EXIT_HOST;
2299         case SVM_EXIT_NPF:
2300                 /* For now we are always handling NPFs when using them */
2301                 if (npt_enabled)
2302                         return NESTED_EXIT_HOST;
2303                 break;
2304         case SVM_EXIT_EXCP_BASE + PF_VECTOR:
2305                 /* When we're shadowing, trap PFs, but not async PF */
2306                 if (!npt_enabled && svm->apf_reason == 0)
2307                         return NESTED_EXIT_HOST;
2308                 break;
2309         case SVM_EXIT_EXCP_BASE + NM_VECTOR:
2310                 nm_interception(svm);
2311                 break;
2312         default:
2313                 break;
2314         }
2315
2316         return NESTED_EXIT_CONTINUE;
2317 }
2318
2319 /*
2320  * If this function returns true, this #vmexit was already handled
2321  */
2322 static int nested_svm_intercept(struct vcpu_svm *svm)
2323 {
2324         u32 exit_code = svm->vmcb->control.exit_code;
2325         int vmexit = NESTED_EXIT_HOST;
2326
2327         switch (exit_code) {
2328         case SVM_EXIT_MSR:
2329                 vmexit = nested_svm_exit_handled_msr(svm);
2330                 break;
2331         case SVM_EXIT_IOIO:
2332                 vmexit = nested_svm_intercept_ioio(svm);
2333                 break;
2334         case SVM_EXIT_READ_CR0 ... SVM_EXIT_WRITE_CR8: {
2335                 u32 bit = 1U << (exit_code - SVM_EXIT_READ_CR0);
2336                 if (svm->nested.intercept_cr & bit)
2337                         vmexit = NESTED_EXIT_DONE;
2338                 break;
2339         }
2340         case SVM_EXIT_READ_DR0 ... SVM_EXIT_WRITE_DR7: {
2341                 u32 bit = 1U << (exit_code - SVM_EXIT_READ_DR0);
2342                 if (svm->nested.intercept_dr & bit)
2343                         vmexit = NESTED_EXIT_DONE;
2344                 break;
2345         }
2346         case SVM_EXIT_EXCP_BASE ... SVM_EXIT_EXCP_BASE + 0x1f: {
2347                 u32 excp_bits = 1 << (exit_code - SVM_EXIT_EXCP_BASE);
2348                 if (svm->nested.intercept_exceptions & excp_bits)
2349                         vmexit = NESTED_EXIT_DONE;
2350                 /* async page fault always cause vmexit */
2351                 else if ((exit_code == SVM_EXIT_EXCP_BASE + PF_VECTOR) &&
2352                          svm->apf_reason != 0)
2353                         vmexit = NESTED_EXIT_DONE;
2354                 break;
2355         }
2356         case SVM_EXIT_ERR: {
2357                 vmexit = NESTED_EXIT_DONE;
2358                 break;
2359         }
2360         default: {
2361                 u64 exit_bits = 1ULL << (exit_code - SVM_EXIT_INTR);
2362                 if (svm->nested.intercept & exit_bits)
2363                         vmexit = NESTED_EXIT_DONE;
2364         }
2365         }
2366
2367         return vmexit;
2368 }
2369
2370 static int nested_svm_exit_handled(struct vcpu_svm *svm)
2371 {
2372         int vmexit;
2373
2374         vmexit = nested_svm_intercept(svm);
2375
2376         if (vmexit == NESTED_EXIT_DONE)
2377                 nested_svm_vmexit(svm);
2378
2379         return vmexit;
2380 }
2381
2382 static inline void copy_vmcb_control_area(struct vmcb *dst_vmcb, struct vmcb *from_vmcb)
2383 {
2384         struct vmcb_control_area *dst  = &dst_vmcb->control;
2385         struct vmcb_control_area *from = &from_vmcb->control;
2386
2387         dst->intercept_cr         = from->intercept_cr;
2388         dst->intercept_dr         = from->intercept_dr;
2389         dst->intercept_exceptions = from->intercept_exceptions;
2390         dst->intercept            = from->intercept;
2391         dst->iopm_base_pa         = from->iopm_base_pa;
2392         dst->msrpm_base_pa        = from->msrpm_base_pa;
2393         dst->tsc_offset           = from->tsc_offset;
2394         dst->asid                 = from->asid;
2395         dst->tlb_ctl              = from->tlb_ctl;
2396         dst->int_ctl              = from->int_ctl;
2397         dst->int_vector           = from->int_vector;
2398         dst->int_state            = from->int_state;
2399         dst->exit_code            = from->exit_code;
2400         dst->exit_code_hi         = from->exit_code_hi;
2401         dst->exit_info_1          = from->exit_info_1;
2402         dst->exit_info_2          = from->exit_info_2;
2403         dst->exit_int_info        = from->exit_int_info;
2404         dst->exit_int_info_err    = from->exit_int_info_err;
2405         dst->nested_ctl           = from->nested_ctl;
2406         dst->event_inj            = from->event_inj;
2407         dst->event_inj_err        = from->event_inj_err;
2408         dst->nested_cr3           = from->nested_cr3;
2409         dst->lbr_ctl              = from->lbr_ctl;
2410 }
2411
2412 static int nested_svm_vmexit(struct vcpu_svm *svm)
2413 {
2414         struct vmcb *nested_vmcb;
2415         struct vmcb *hsave = svm->nested.hsave;
2416         struct vmcb *vmcb = svm->vmcb;
2417         struct page *page;
2418
2419         trace_kvm_nested_vmexit_inject(vmcb->control.exit_code,
2420                                        vmcb->control.exit_info_1,
2421                                        vmcb->control.exit_info_2,
2422                                        vmcb->control.exit_int_info,
2423                                        vmcb->control.exit_int_info_err,
2424                                        KVM_ISA_SVM);
2425
2426         nested_vmcb = nested_svm_map(svm, svm->nested.vmcb, &page);
2427         if (!nested_vmcb)
2428                 return 1;
2429
2430         /* Exit Guest-Mode */
2431         leave_guest_mode(&svm->vcpu);
2432         svm->nested.vmcb = 0;
2433
2434         /* Give the current vmcb to the guest */
2435         disable_gif(svm);
2436
2437         nested_vmcb->save.es     = vmcb->save.es;
2438         nested_vmcb->save.cs     = vmcb->save.cs;
2439         nested_vmcb->save.ss     = vmcb->save.ss;
2440         nested_vmcb->save.ds     = vmcb->save.ds;
2441         nested_vmcb->save.gdtr   = vmcb->save.gdtr;
2442         nested_vmcb->save.idtr   = vmcb->save.idtr;
2443         nested_vmcb->save.efer   = svm->vcpu.arch.efer;
2444         nested_vmcb->save.cr0    = kvm_read_cr0(&svm->vcpu);
2445         nested_vmcb->save.cr3    = kvm_read_cr3(&svm->vcpu);
2446         nested_vmcb->save.cr2    = vmcb->save.cr2;
2447         nested_vmcb->save.cr4    = svm->vcpu.arch.cr4;
2448         nested_vmcb->save.rflags = kvm_get_rflags(&svm->vcpu);
2449         nested_vmcb->save.rip    = vmcb->save.rip;
2450         nested_vmcb->save.rsp    = vmcb->save.rsp;
2451         nested_vmcb->save.rax    = vmcb->save.rax;
2452         nested_vmcb->save.dr7    = vmcb->save.dr7;
2453         nested_vmcb->save.dr6    = vmcb->save.dr6;
2454         nested_vmcb->save.cpl    = vmcb->save.cpl;
2455
2456         nested_vmcb->control.int_ctl           = vmcb->control.int_ctl;
2457         nested_vmcb->control.int_vector        = vmcb->control.int_vector;
2458         nested_vmcb->control.int_state         = vmcb->control.int_state;
2459         nested_vmcb->control.exit_code         = vmcb->control.exit_code;
2460         nested_vmcb->control.exit_code_hi      = vmcb->control.exit_code_hi;
2461         nested_vmcb->control.exit_info_1       = vmcb->control.exit_info_1;
2462         nested_vmcb->control.exit_info_2       = vmcb->control.exit_info_2;
2463         nested_vmcb->control.exit_int_info     = vmcb->control.exit_int_info;
2464         nested_vmcb->control.exit_int_info_err = vmcb->control.exit_int_info_err;
2465         nested_vmcb->control.next_rip          = vmcb->control.next_rip;
2466
2467         /*
2468          * If we emulate a VMRUN/#VMEXIT in the same host #vmexit cycle we have
2469          * to make sure that we do not lose injected events. So check event_inj
2470          * here and copy it to exit_int_info if it is valid.
2471          * Exit_int_info and event_inj can't be both valid because the case
2472          * below only happens on a VMRUN instruction intercept which has
2473          * no valid exit_int_info set.
2474          */
2475         if (vmcb->control.event_inj & SVM_EVTINJ_VALID) {
2476                 struct vmcb_control_area *nc = &nested_vmcb->control;
2477
2478                 nc->exit_int_info     = vmcb->control.event_inj;
2479                 nc->exit_int_info_err = vmcb->control.event_inj_err;
2480         }
2481
2482         nested_vmcb->control.tlb_ctl           = 0;
2483         nested_vmcb->control.event_inj         = 0;
2484         nested_vmcb->control.event_inj_err     = 0;
2485
2486         /* We always set V_INTR_MASKING and remember the old value in hflags */
2487         if (!(svm->vcpu.arch.hflags & HF_VINTR_MASK))
2488                 nested_vmcb->control.int_ctl &= ~V_INTR_MASKING_MASK;
2489
2490         /* Restore the original control entries */
2491         copy_vmcb_control_area(vmcb, hsave);
2492
2493         kvm_clear_exception_queue(&svm->vcpu);
2494         kvm_clear_interrupt_queue(&svm->vcpu);
2495
2496         svm->nested.nested_cr3 = 0;
2497
2498         /* Restore selected save entries */
2499         svm->vmcb->save.es = hsave->save.es;
2500         svm->vmcb->save.cs = hsave->save.cs;
2501         svm->vmcb->save.ss = hsave->save.ss;
2502         svm->vmcb->save.ds = hsave->save.ds;
2503         svm->vmcb->save.gdtr = hsave->save.gdtr;
2504         svm->vmcb->save.idtr = hsave->save.idtr;
2505         kvm_set_rflags(&svm->vcpu, hsave->save.rflags);
2506         svm_set_efer(&svm->vcpu, hsave->save.efer);
2507         svm_set_cr0(&svm->vcpu, hsave->save.cr0 | X86_CR0_PE);
2508         svm_set_cr4(&svm->vcpu, hsave->save.cr4);
2509         if (npt_enabled) {
2510                 svm->vmcb->save.cr3 = hsave->save.cr3;
2511                 svm->vcpu.arch.cr3 = hsave->save.cr3;
2512         } else {
2513                 (void)kvm_set_cr3(&svm->vcpu, hsave->save.cr3);
2514         }
2515         kvm_register_write(&svm->vcpu, VCPU_REGS_RAX, hsave->save.rax);
2516         kvm_register_write(&svm->vcpu, VCPU_REGS_RSP, hsave->save.rsp);
2517         kvm_register_write(&svm->vcpu, VCPU_REGS_RIP, hsave->save.rip);
2518         svm->vmcb->save.dr7 = 0;
2519         svm->vmcb->save.cpl = 0;
2520         svm->vmcb->control.exit_int_info = 0;
2521
2522         mark_all_dirty(svm->vmcb);
2523
2524         nested_svm_unmap(page);
2525
2526         nested_svm_uninit_mmu_context(&svm->vcpu);
2527         kvm_mmu_reset_context(&svm->vcpu);
2528         kvm_mmu_load(&svm->vcpu);
2529
2530         return 0;
2531 }
2532
2533 static bool nested_svm_vmrun_msrpm(struct vcpu_svm *svm)
2534 {
2535         /*
2536          * This function merges the msr permission bitmaps of kvm and the
2537          * nested vmcb. It is optimized in that it only merges the parts where
2538          * the kvm msr permission bitmap may contain zero bits
2539          */
2540         int i;
2541
2542         if (!(svm->nested.intercept & (1ULL << INTERCEPT_MSR_PROT)))
2543                 return true;
2544
2545         for (i = 0; i < MSRPM_OFFSETS; i++) {
2546                 u32 value, p;
2547                 u64 offset;
2548
2549                 if (msrpm_offsets[i] == 0xffffffff)
2550                         break;
2551
2552                 p      = msrpm_offsets[i];
2553                 offset = svm->nested.vmcb_msrpm + (p * 4);
2554
2555                 if (kvm_vcpu_read_guest(&svm->vcpu, offset, &value, 4))
2556                         return false;
2557
2558                 svm->nested.msrpm[p] = svm->msrpm[p] | value;
2559         }
2560
2561         svm->vmcb->control.msrpm_base_pa = __pa(svm->nested.msrpm);
2562
2563         return true;
2564 }
2565
2566 static bool nested_vmcb_checks(struct vmcb *vmcb)
2567 {
2568         if ((vmcb->control.intercept & (1ULL << INTERCEPT_VMRUN)) == 0)
2569                 return false;
2570
2571         if (vmcb->control.asid == 0)
2572                 return false;
2573
2574         if (vmcb->control.nested_ctl && !npt_enabled)
2575                 return false;
2576
2577         return true;
2578 }
2579
2580 static bool nested_svm_vmrun(struct vcpu_svm *svm)
2581 {
2582         struct vmcb *nested_vmcb;
2583         struct vmcb *hsave = svm->nested.hsave;
2584         struct vmcb *vmcb = svm->vmcb;
2585         struct page *page;
2586         u64 vmcb_gpa;
2587
2588         vmcb_gpa = svm->vmcb->save.rax;
2589
2590         nested_vmcb = nested_svm_map(svm, svm->vmcb->save.rax, &page);
2591         if (!nested_vmcb)
2592                 return false;
2593
2594         if (!nested_vmcb_checks(nested_vmcb)) {
2595                 nested_vmcb->control.exit_code    = SVM_EXIT_ERR;
2596                 nested_vmcb->control.exit_code_hi = 0;
2597                 nested_vmcb->control.exit_info_1  = 0;
2598                 nested_vmcb->control.exit_info_2  = 0;
2599
2600                 nested_svm_unmap(page);
2601
2602                 return false;
2603         }
2604
2605         trace_kvm_nested_vmrun(svm->vmcb->save.rip, vmcb_gpa,
2606                                nested_vmcb->save.rip,
2607                                nested_vmcb->control.int_ctl,
2608                                nested_vmcb->control.event_inj,
2609                                nested_vmcb->control.nested_ctl);
2610
2611         trace_kvm_nested_intercepts(nested_vmcb->control.intercept_cr & 0xffff,
2612                                     nested_vmcb->control.intercept_cr >> 16,
2613                                     nested_vmcb->control.intercept_exceptions,
2614                                     nested_vmcb->control.intercept);
2615
2616         /* Clear internal status */
2617         kvm_clear_exception_queue(&svm->vcpu);
2618         kvm_clear_interrupt_queue(&svm->vcpu);
2619
2620         /*
2621          * Save the old vmcb, so we don't need to pick what we save, but can
2622          * restore everything when a VMEXIT occurs
2623          */
2624         hsave->save.es     = vmcb->save.es;
2625         hsave->save.cs     = vmcb->save.cs;
2626         hsave->save.ss     = vmcb->save.ss;
2627         hsave->save.ds     = vmcb->save.ds;
2628         hsave->save.gdtr   = vmcb->save.gdtr;
2629         hsave->save.idtr   = vmcb->save.idtr;
2630         hsave->save.efer   = svm->vcpu.arch.efer;
2631         hsave->save.cr0    = kvm_read_cr0(&svm->vcpu);
2632         hsave->save.cr4    = svm->vcpu.arch.cr4;
2633         hsave->save.rflags = kvm_get_rflags(&svm->vcpu);
2634         hsave->save.rip    = kvm_rip_read(&svm->vcpu);
2635         hsave->save.rsp    = vmcb->save.rsp;
2636         hsave->save.rax    = vmcb->save.rax;
2637         if (npt_enabled)
2638                 hsave->save.cr3    = vmcb->save.cr3;
2639         else
2640                 hsave->save.cr3    = kvm_read_cr3(&svm->vcpu);
2641
2642         copy_vmcb_control_area(hsave, vmcb);
2643
2644         if (kvm_get_rflags(&svm->vcpu) & X86_EFLAGS_IF)
2645                 svm->vcpu.arch.hflags |= HF_HIF_MASK;
2646         else
2647                 svm->vcpu.arch.hflags &= ~HF_HIF_MASK;
2648
2649         if (nested_vmcb->control.nested_ctl) {
2650                 kvm_mmu_unload(&svm->vcpu);
2651                 svm->nested.nested_cr3 = nested_vmcb->control.nested_cr3;
2652                 nested_svm_init_mmu_context(&svm->vcpu);
2653         }
2654
2655         /* Load the nested guest state */
2656         svm->vmcb->save.es = nested_vmcb->save.es;
2657         svm->vmcb->save.cs = nested_vmcb->save.cs;
2658         svm->vmcb->save.ss = nested_vmcb->save.ss;
2659         svm->vmcb->save.ds = nested_vmcb->save.ds;
2660         svm->vmcb->save.gdtr = nested_vmcb->save.gdtr;
2661         svm->vmcb->save.idtr = nested_vmcb->save.idtr;
2662         kvm_set_rflags(&svm->vcpu, nested_vmcb->save.rflags);
2663         svm_set_efer(&svm->vcpu, nested_vmcb->save.efer);
2664         svm_set_cr0(&svm->vcpu, nested_vmcb->save.cr0);
2665         svm_set_cr4(&svm->vcpu, nested_vmcb->save.cr4);
2666         if (npt_enabled) {
2667                 svm->vmcb->save.cr3 = nested_vmcb->save.cr3;
2668                 svm->vcpu.arch.cr3 = nested_vmcb->save.cr3;
2669         } else
2670                 (void)kvm_set_cr3(&svm->vcpu, nested_vmcb->save.cr3);
2671
2672         /* Guest paging mode is active - reset mmu */
2673         kvm_mmu_reset_context(&svm->vcpu);
2674
2675         svm->vmcb->save.cr2 = svm->vcpu.arch.cr2 = nested_vmcb->save.cr2;
2676         kvm_register_write(&svm->vcpu, VCPU_REGS_RAX, nested_vmcb->save.rax);
2677         kvm_register_write(&svm->vcpu, VCPU_REGS_RSP, nested_vmcb->save.rsp);
2678         kvm_register_write(&svm->vcpu, VCPU_REGS_RIP, nested_vmcb->save.rip);
2679
2680         /* In case we don't even reach vcpu_run, the fields are not updated */
2681         svm->vmcb->save.rax = nested_vmcb->save.rax;
2682         svm->vmcb->save.rsp = nested_vmcb->save.rsp;
2683         svm->vmcb->save.rip = nested_vmcb->save.rip;
2684         svm->vmcb->save.dr7 = nested_vmcb->save.dr7;
2685         svm->vmcb->save.dr6 = nested_vmcb->save.dr6;
2686         svm->vmcb->save.cpl = nested_vmcb->save.cpl;
2687
2688         svm->nested.vmcb_msrpm = nested_vmcb->control.msrpm_base_pa & ~0x0fffULL;
2689         svm->nested.vmcb_iopm  = nested_vmcb->control.iopm_base_pa  & ~0x0fffULL;
2690
2691         /* cache intercepts */
2692         svm->nested.intercept_cr         = nested_vmcb->control.intercept_cr;
2693         svm->nested.intercept_dr         = nested_vmcb->control.intercept_dr;
2694         svm->nested.intercept_exceptions = nested_vmcb->control.intercept_exceptions;
2695         svm->nested.intercept            = nested_vmcb->control.intercept;
2696
2697         svm_flush_tlb(&svm->vcpu);
2698         svm->vmcb->control.int_ctl = nested_vmcb->control.int_ctl | V_INTR_MASKING_MASK;
2699         if (nested_vmcb->control.int_ctl & V_INTR_MASKING_MASK)
2700                 svm->vcpu.arch.hflags |= HF_VINTR_MASK;
2701         else
2702                 svm->vcpu.arch.hflags &= ~HF_VINTR_MASK;
2703
2704         if (svm->vcpu.arch.hflags & HF_VINTR_MASK) {
2705                 /* We only want the cr8 intercept bits of the guest */
2706                 clr_cr_intercept(svm, INTERCEPT_CR8_READ);
2707                 clr_cr_intercept(svm, INTERCEPT_CR8_WRITE);
2708         }
2709
2710         /* We don't want to see VMMCALLs from a nested guest */
2711         clr_intercept(svm, INTERCEPT_VMMCALL);
2712
2713         svm->vmcb->control.lbr_ctl = nested_vmcb->control.lbr_ctl;
2714         svm->vmcb->control.int_vector = nested_vmcb->control.int_vector;
2715         svm->vmcb->control.int_state = nested_vmcb->control.int_state;
2716         svm->vmcb->control.tsc_offset += nested_vmcb->control.tsc_offset;
2717         svm->vmcb->control.event_inj = nested_vmcb->control.event_inj;
2718         svm->vmcb->control.event_inj_err = nested_vmcb->control.event_inj_err;
2719
2720         nested_svm_unmap(page);
2721
2722         /* Enter Guest-Mode */
2723         enter_guest_mode(&svm->vcpu);
2724
2725         /*
2726          * Merge guest and host intercepts - must be called  with vcpu in
2727          * guest-mode to take affect here
2728          */
2729         recalc_intercepts(svm);
2730
2731         svm->nested.vmcb = vmcb_gpa;
2732
2733         enable_gif(svm);
2734
2735         mark_all_dirty(svm->vmcb);
2736
2737         return true;
2738 }
2739
2740 static void nested_svm_vmloadsave(struct vmcb *from_vmcb, struct vmcb *to_vmcb)
2741 {
2742         to_vmcb->save.fs = from_vmcb->save.fs;
2743         to_vmcb->save.gs = from_vmcb->save.gs;
2744         to_vmcb->save.tr = from_vmcb->save.tr;
2745         to_vmcb->save.ldtr = from_vmcb->save.ldtr;
2746         to_vmcb->save.kernel_gs_base = from_vmcb->save.kernel_gs_base;
2747         to_vmcb->save.star = from_vmcb->save.star;
2748         to_vmcb->save.lstar = from_vmcb->save.lstar;
2749         to_vmcb->save.cstar = from_vmcb->save.cstar;
2750         to_vmcb->save.sfmask = from_vmcb->save.sfmask;
2751         to_vmcb->save.sysenter_cs = from_vmcb->save.sysenter_cs;
2752         to_vmcb->save.sysenter_esp = from_vmcb->save.sysenter_esp;
2753         to_vmcb->save.sysenter_eip = from_vmcb->save.sysenter_eip;
2754 }
2755
2756 static int vmload_interception(struct vcpu_svm *svm)
2757 {
2758         struct vmcb *nested_vmcb;
2759         struct page *page;
2760
2761         if (nested_svm_check_permissions(svm))
2762                 return 1;
2763
2764         nested_vmcb = nested_svm_map(svm, svm->vmcb->save.rax, &page);
2765         if (!nested_vmcb)
2766                 return 1;
2767
2768         svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
2769         skip_emulated_instruction(&svm->vcpu);
2770
2771         nested_svm_vmloadsave(nested_vmcb, svm->vmcb);
2772         nested_svm_unmap(page);
2773
2774         return 1;
2775 }
2776
2777 static int vmsave_interception(struct vcpu_svm *svm)
2778 {
2779         struct vmcb *nested_vmcb;
2780         struct page *page;
2781
2782         if (nested_svm_check_permissions(svm))
2783                 return 1;
2784
2785         nested_vmcb = nested_svm_map(svm, svm->vmcb->save.rax, &page);
2786         if (!nested_vmcb)
2787                 return 1;
2788
2789         svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
2790         skip_emulated_instruction(&svm->vcpu);
2791
2792         nested_svm_vmloadsave(svm->vmcb, nested_vmcb);
2793         nested_svm_unmap(page);
2794
2795         return 1;
2796 }
2797
2798 static int vmrun_interception(struct vcpu_svm *svm)
2799 {
2800         if (nested_svm_check_permissions(svm))
2801                 return 1;
2802
2803         /* Save rip after vmrun instruction */
2804         kvm_rip_write(&svm->vcpu, kvm_rip_read(&svm->vcpu) + 3);
2805
2806         if (!nested_svm_vmrun(svm))
2807                 return 1;
2808
2809         if (!nested_svm_vmrun_msrpm(svm))
2810                 goto failed;
2811
2812         return 1;
2813
2814 failed:
2815
2816         svm->vmcb->control.exit_code    = SVM_EXIT_ERR;
2817         svm->vmcb->control.exit_code_hi = 0;
2818         svm->vmcb->control.exit_info_1  = 0;
2819         svm->vmcb->control.exit_info_2  = 0;
2820
2821         nested_svm_vmexit(svm);
2822
2823         return 1;
2824 }
2825
2826 static int stgi_interception(struct vcpu_svm *svm)
2827 {
2828         if (nested_svm_check_permissions(svm))
2829                 return 1;
2830
2831         svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
2832         skip_emulated_instruction(&svm->vcpu);
2833         kvm_make_request(KVM_REQ_EVENT, &svm->vcpu);
2834
2835         enable_gif(svm);
2836
2837         return 1;
2838 }
2839
2840 static int clgi_interception(struct vcpu_svm *svm)
2841 {
2842         if (nested_svm_check_permissions(svm))
2843                 return 1;
2844
2845         svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
2846         skip_emulated_instruction(&svm->vcpu);
2847
2848         disable_gif(svm);
2849
2850         /* After a CLGI no interrupts should come */
2851         svm_clear_vintr(svm);
2852         svm->vmcb->control.int_ctl &= ~V_IRQ_MASK;
2853
2854         mark_dirty(svm->vmcb, VMCB_INTR);
2855
2856         return 1;
2857 }
2858
2859 static int invlpga_interception(struct vcpu_svm *svm)
2860 {
2861         struct kvm_vcpu *vcpu = &svm->vcpu;
2862
2863         trace_kvm_invlpga(svm->vmcb->save.rip, kvm_register_read(&svm->vcpu, VCPU_REGS_RCX),
2864                           kvm_register_read(&svm->vcpu, VCPU_REGS_RAX));
2865
2866         /* Let's treat INVLPGA the same as INVLPG (can be optimized!) */
2867         kvm_mmu_invlpg(vcpu, kvm_register_read(&svm->vcpu, VCPU_REGS_RAX));
2868
2869         svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
2870         skip_emulated_instruction(&svm->vcpu);
2871         return 1;
2872 }
2873
2874 static int skinit_interception(struct vcpu_svm *svm)
2875 {
2876         trace_kvm_skinit(svm->vmcb->save.rip, kvm_register_read(&svm->vcpu, VCPU_REGS_RAX));
2877
2878         kvm_queue_exception(&svm->vcpu, UD_VECTOR);
2879         return 1;
2880 }
2881
2882 static int wbinvd_interception(struct vcpu_svm *svm)
2883 {
2884         kvm_emulate_wbinvd(&svm->vcpu);
2885         return 1;
2886 }
2887
2888 static int xsetbv_interception(struct vcpu_svm *svm)
2889 {
2890         u64 new_bv = kvm_read_edx_eax(&svm->vcpu);
2891         u32 index = kvm_register_read(&svm->vcpu, VCPU_REGS_RCX);
2892
2893         if (kvm_set_xcr(&svm->vcpu, index, new_bv) == 0) {
2894                 svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
2895                 skip_emulated_instruction(&svm->vcpu);
2896         }
2897
2898         return 1;
2899 }
2900
2901 static int task_switch_interception(struct vcpu_svm *svm)
2902 {
2903         u16 tss_selector;
2904         int reason;
2905         int int_type = svm->vmcb->control.exit_int_info &
2906                 SVM_EXITINTINFO_TYPE_MASK;
2907         int int_vec = svm->vmcb->control.exit_int_info & SVM_EVTINJ_VEC_MASK;
2908         uint32_t type =
2909                 svm->vmcb->control.exit_int_info & SVM_EXITINTINFO_TYPE_MASK;
2910         uint32_t idt_v =
2911                 svm->vmcb->control.exit_int_info & SVM_EXITINTINFO_VALID;
2912         bool has_error_code = false;
2913         u32 error_code = 0;
2914
2915         tss_selector = (u16)svm->vmcb->control.exit_info_1;
2916
2917         if (svm->vmcb->control.exit_info_2 &
2918             (1ULL << SVM_EXITINFOSHIFT_TS_REASON_IRET))
2919                 reason = TASK_SWITCH_IRET;
2920         else if (svm->vmcb->control.exit_info_2 &
2921                  (1ULL << SVM_EXITINFOSHIFT_TS_REASON_JMP))
2922                 reason = TASK_SWITCH_JMP;
2923         else if (idt_v)
2924                 reason = TASK_SWITCH_GATE;
2925         else
2926                 reason = TASK_SWITCH_CALL;
2927
2928         if (reason == TASK_SWITCH_GATE) {
2929                 switch (type) {
2930                 case SVM_EXITINTINFO_TYPE_NMI:
2931                         svm->vcpu.arch.nmi_injected = false;
2932                         break;
2933                 case SVM_EXITINTINFO_TYPE_EXEPT:
2934                         if (svm->vmcb->control.exit_info_2 &
2935                             (1ULL << SVM_EXITINFOSHIFT_TS_HAS_ERROR_CODE)) {
2936                                 has_error_code = true;
2937                                 error_code =
2938                                         (u32)svm->vmcb->control.exit_info_2;
2939                         }
2940                         kvm_clear_exception_queue(&svm->vcpu);
2941                         break;
2942                 case SVM_EXITINTINFO_TYPE_INTR:
2943                         kvm_clear_interrupt_queue(&svm->vcpu);
2944                         break;
2945                 default:
2946                         break;
2947                 }
2948         }
2949
2950         if (reason != TASK_SWITCH_GATE ||
2951             int_type == SVM_EXITINTINFO_TYPE_SOFT ||
2952             (int_type == SVM_EXITINTINFO_TYPE_EXEPT &&
2953              (int_vec == OF_VECTOR || int_vec == BP_VECTOR)))
2954                 skip_emulated_instruction(&svm->vcpu);
2955
2956         if (int_type != SVM_EXITINTINFO_TYPE_SOFT)
2957                 int_vec = -1;
2958
2959         if (kvm_task_switch(&svm->vcpu, tss_selector, int_vec, reason,
2960                                 has_error_code, error_code) == EMULATE_FAIL) {
2961                 svm->vcpu.run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
2962                 svm->vcpu.run->internal.suberror = KVM_INTERNAL_ERROR_EMULATION;
2963                 svm->vcpu.run->internal.ndata = 0;
2964                 return 0;
2965         }
2966         return 1;
2967 }
2968
2969 static int cpuid_interception(struct vcpu_svm *svm)
2970 {
2971         svm->next_rip = kvm_rip_read(&svm->vcpu) + 2;
2972         kvm_emulate_cpuid(&svm->vcpu);
2973         return 1;
2974 }
2975
2976 static int iret_interception(struct vcpu_svm *svm)
2977 {
2978         ++svm->vcpu.stat.nmi_window_exits;
2979         clr_intercept(svm, INTERCEPT_IRET);
2980         svm->vcpu.arch.hflags |= HF_IRET_MASK;
2981         svm->nmi_iret_rip = kvm_rip_read(&svm->vcpu);
2982         kvm_make_request(KVM_REQ_EVENT, &svm->vcpu);
2983         return 1;
2984 }
2985
2986 static int invlpg_interception(struct vcpu_svm *svm)
2987 {
2988         if (!static_cpu_has(X86_FEATURE_DECODEASSISTS))
2989                 return emulate_instruction(&svm->vcpu, 0) == EMULATE_DONE;
2990
2991         kvm_mmu_invlpg(&svm->vcpu, svm->vmcb->control.exit_info_1);
2992         skip_emulated_instruction(&svm->vcpu);
2993         return 1;
2994 }
2995
2996 static int emulate_on_interception(struct vcpu_svm *svm)
2997 {
2998         return emulate_instruction(&svm->vcpu, 0) == EMULATE_DONE;
2999 }
3000
3001 static int rdpmc_interception(struct vcpu_svm *svm)
3002 {
3003         int err;
3004
3005         if (!static_cpu_has(X86_FEATURE_NRIPS))
3006                 return emulate_on_interception(svm);
3007
3008         err = kvm_rdpmc(&svm->vcpu);
3009         kvm_complete_insn_gp(&svm->vcpu, err);
3010
3011         return 1;
3012 }
3013
3014 static bool check_selective_cr0_intercepted(struct vcpu_svm *svm,
3015                                             unsigned long val)
3016 {
3017         unsigned long cr0 = svm->vcpu.arch.cr0;
3018         bool ret = false;
3019         u64 intercept;
3020
3021         intercept = svm->nested.intercept;
3022
3023         if (!is_guest_mode(&svm->vcpu) ||
3024             (!(intercept & (1ULL << INTERCEPT_SELECTIVE_CR0))))
3025                 return false;
3026
3027         cr0 &= ~SVM_CR0_SELECTIVE_MASK;
3028         val &= ~SVM_CR0_SELECTIVE_MASK;
3029
3030         if (cr0 ^ val) {
3031                 svm->vmcb->control.exit_code = SVM_EXIT_CR0_SEL_WRITE;
3032                 ret = (nested_svm_exit_handled(svm) == NESTED_EXIT_DONE);
3033         }
3034
3035         return ret;
3036 }
3037
3038 #define CR_VALID (1ULL << 63)
3039
3040 static int cr_interception(struct vcpu_svm *svm)
3041 {
3042         int reg, cr;
3043         unsigned long val;
3044         int err;
3045
3046         if (!static_cpu_has(X86_FEATURE_DECODEASSISTS))
3047                 return emulate_on_interception(svm);
3048
3049         if (unlikely((svm->vmcb->control.exit_info_1 & CR_VALID) == 0))
3050                 return emulate_on_interception(svm);
3051
3052         reg = svm->vmcb->control.exit_info_1 & SVM_EXITINFO_REG_MASK;
3053         if (svm->vmcb->control.exit_code == SVM_EXIT_CR0_SEL_WRITE)
3054                 cr = SVM_EXIT_WRITE_CR0 - SVM_EXIT_READ_CR0;
3055         else
3056                 cr = svm->vmcb->control.exit_code - SVM_EXIT_READ_CR0;
3057
3058         err = 0;
3059         if (cr >= 16) { /* mov to cr */
3060                 cr -= 16;
3061                 val = kvm_register_read(&svm->vcpu, reg);
3062                 switch (cr) {
3063                 case 0:
3064                         if (!check_selective_cr0_intercepted(svm, val))
3065                                 err = kvm_set_cr0(&svm->vcpu, val);
3066                         else
3067                                 return 1;
3068
3069                         break;
3070                 case 3:
3071                         err = kvm_set_cr3(&svm->vcpu, val);
3072                         break;
3073                 case 4:
3074                         err = kvm_set_cr4(&svm->vcpu, val);
3075                         break;
3076                 case 8:
3077                         err = kvm_set_cr8(&svm->vcpu, val);
3078                         break;
3079                 default:
3080                         WARN(1, "unhandled write to CR%d", cr);
3081                         kvm_queue_exception(&svm->vcpu, UD_VECTOR);
3082                         return 1;
3083                 }
3084         } else { /* mov from cr */
3085                 switch (cr) {
3086                 case 0:
3087                         val = kvm_read_cr0(&svm->vcpu);
3088                         break;
3089                 case 2:
3090                         val = svm->vcpu.arch.cr2;
3091                         break;
3092                 case 3:
3093                         val = kvm_read_cr3(&svm->vcpu);
3094                         break;
3095                 case 4:
3096                         val = kvm_read_cr4(&svm->vcpu);
3097                         break;
3098                 case 8:
3099                         val = kvm_get_cr8(&svm->vcpu);
3100                         break;
3101                 default:
3102                         WARN(1, "unhandled read from CR%d", cr);
3103                         kvm_queue_exception(&svm->vcpu, UD_VECTOR);
3104                         return 1;
3105                 }
3106                 kvm_register_write(&svm->vcpu, reg, val);
3107         }
3108         kvm_complete_insn_gp(&svm->vcpu, err);
3109
3110         return 1;
3111 }
3112
3113 static int dr_interception(struct vcpu_svm *svm)
3114 {
3115         int reg, dr;
3116         unsigned long val;
3117
3118         if (svm->vcpu.guest_debug == 0) {
3119                 /*
3120                  * No more DR vmexits; force a reload of the debug registers
3121                  * and reenter on this instruction.  The next vmexit will
3122                  * retrieve the full state of the debug registers.
3123                  */
3124                 clr_dr_intercepts(svm);
3125                 svm->vcpu.arch.switch_db_regs |= KVM_DEBUGREG_WONT_EXIT;
3126                 return 1;
3127         }
3128
3129         if (!boot_cpu_has(X86_FEATURE_DECODEASSISTS))
3130                 return emulate_on_interception(svm);
3131
3132         reg = svm->vmcb->control.exit_info_1 & SVM_EXITINFO_REG_MASK;
3133         dr = svm->vmcb->control.exit_code - SVM_EXIT_READ_DR0;
3134
3135         if (dr >= 16) { /* mov to DRn */
3136                 if (!kvm_require_dr(&svm->vcpu, dr - 16))
3137                         return 1;
3138                 val = kvm_register_read(&svm->vcpu, reg);
3139                 kvm_set_dr(&svm->vcpu, dr - 16, val);
3140         } else {
3141                 if (!kvm_require_dr(&svm->vcpu, dr))
3142                         return 1;
3143                 kvm_get_dr(&svm->vcpu, dr, &val);
3144                 kvm_register_write(&svm->vcpu, reg, val);
3145         }
3146
3147         skip_emulated_instruction(&svm->vcpu);
3148
3149         return 1;
3150 }
3151
3152 static int cr8_write_interception(struct vcpu_svm *svm)
3153 {
3154         struct kvm_run *kvm_run = svm->vcpu.run;
3155         int r;
3156
3157         u8 cr8_prev = kvm_get_cr8(&svm->vcpu);
3158         /* instruction emulation calls kvm_set_cr8() */
3159         r = cr_interception(svm);
3160         if (irqchip_in_kernel(svm->vcpu.kvm))
3161                 return r;
3162         if (cr8_prev <= kvm_get_cr8(&svm->vcpu))
3163                 return r;
3164         kvm_run->exit_reason = KVM_EXIT_SET_TPR;
3165         return 0;
3166 }
3167
3168 static u64 svm_read_l1_tsc(struct kvm_vcpu *vcpu, u64 host_tsc)
3169 {
3170         struct vmcb *vmcb = get_host_vmcb(to_svm(vcpu));
3171         return vmcb->control.tsc_offset +
3172                 svm_scale_tsc(vcpu, host_tsc);
3173 }
3174
3175 static int svm_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
3176 {
3177         struct vcpu_svm *svm = to_svm(vcpu);
3178
3179         switch (msr_info->index) {
3180         case MSR_IA32_TSC: {
3181                 msr_info->data = svm->vmcb->control.tsc_offset +
3182                         svm_scale_tsc(vcpu, rdtsc());
3183
3184                 break;
3185         }
3186         case MSR_STAR:
3187                 msr_info->data = svm->vmcb->save.star;
3188                 break;
3189 #ifdef CONFIG_X86_64
3190         case MSR_LSTAR:
3191                 msr_info->data = svm->vmcb->save.lstar;
3192                 break;
3193         case MSR_CSTAR:
3194                 msr_info->data = svm->vmcb->save.cstar;
3195                 break;
3196         case MSR_KERNEL_GS_BASE:
3197                 msr_info->data = svm->vmcb->save.kernel_gs_base;
3198                 break;
3199         case MSR_SYSCALL_MASK:
3200                 msr_info->data = svm->vmcb->save.sfmask;
3201                 break;
3202 #endif
3203         case MSR_IA32_SYSENTER_CS:
3204                 msr_info->data = svm->vmcb->save.sysenter_cs;
3205                 break;
3206         case MSR_IA32_SYSENTER_EIP:
3207                 msr_info->data = svm->sysenter_eip;
3208                 break;
3209         case MSR_IA32_SYSENTER_ESP:
3210                 msr_info->data = svm->sysenter_esp;
3211                 break;
3212         /*
3213          * Nobody will change the following 5 values in the VMCB so we can
3214          * safely return them on rdmsr. They will always be 0 until LBRV is
3215          * implemented.
3216          */
3217         case MSR_IA32_DEBUGCTLMSR:
3218                 msr_info->data = svm->vmcb->save.dbgctl;
3219                 break;
3220         case MSR_IA32_LASTBRANCHFROMIP:
3221                 msr_info->data = svm->vmcb->save.br_from;
3222                 break;
3223         case MSR_IA32_LASTBRANCHTOIP:
3224                 msr_info->data = svm->vmcb->save.br_to;
3225                 break;
3226         case MSR_IA32_LASTINTFROMIP:
3227                 msr_info->data = svm->vmcb->save.last_excp_from;
3228                 break;
3229         case MSR_IA32_LASTINTTOIP:
3230                 msr_info->data = svm->vmcb->save.last_excp_to;
3231                 break;
3232         case MSR_VM_HSAVE_PA:
3233                 msr_info->data = svm->nested.hsave_msr;
3234                 break;
3235         case MSR_VM_CR:
3236                 msr_info->data = svm->nested.vm_cr_msr;
3237                 break;
3238         case MSR_IA32_UCODE_REV:
3239                 msr_info->data = 0x01000065;
3240                 break;
3241         default:
3242                 return kvm_get_msr_common(vcpu, msr_info);
3243         }
3244         return 0;
3245 }
3246
3247 static int rdmsr_interception(struct vcpu_svm *svm)
3248 {
3249         u32 ecx = kvm_register_read(&svm->vcpu, VCPU_REGS_RCX);
3250         struct msr_data msr_info;
3251
3252         msr_info.index = ecx;
3253         msr_info.host_initiated = false;
3254         if (svm_get_msr(&svm->vcpu, &msr_info)) {
3255                 trace_kvm_msr_read_ex(ecx);
3256                 kvm_inject_gp(&svm->vcpu, 0);
3257         } else {
3258                 trace_kvm_msr_read(ecx, msr_info.data);
3259
3260                 kvm_register_write(&svm->vcpu, VCPU_REGS_RAX,
3261                                    msr_info.data & 0xffffffff);
3262                 kvm_register_write(&svm->vcpu, VCPU_REGS_RDX,
3263                                    msr_info.data >> 32);
3264                 svm->next_rip = kvm_rip_read(&svm->vcpu) + 2;
3265                 skip_emulated_instruction(&svm->vcpu);
3266         }
3267         return 1;
3268 }
3269
3270 static int svm_set_vm_cr(struct kvm_vcpu *vcpu, u64 data)
3271 {
3272         struct vcpu_svm *svm = to_svm(vcpu);
3273         int svm_dis, chg_mask;
3274
3275         if (data & ~SVM_VM_CR_VALID_MASK)
3276                 return 1;
3277
3278         chg_mask = SVM_VM_CR_VALID_MASK;
3279
3280         if (svm->nested.vm_cr_msr & SVM_VM_CR_SVM_DIS_MASK)
3281                 chg_mask &= ~(SVM_VM_CR_SVM_LOCK_MASK | SVM_VM_CR_SVM_DIS_MASK);
3282
3283         svm->nested.vm_cr_msr &= ~chg_mask;
3284         svm->nested.vm_cr_msr |= (data & chg_mask);
3285
3286         svm_dis = svm->nested.vm_cr_msr & SVM_VM_CR_SVM_DIS_MASK;
3287
3288         /* check for svm_disable while efer.svme is set */
3289         if (svm_dis && (vcpu->arch.efer & EFER_SVME))
3290                 return 1;
3291
3292         return 0;
3293 }
3294
3295 static int svm_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr)
3296 {
3297         struct vcpu_svm *svm = to_svm(vcpu);
3298
3299         u32 ecx = msr->index;
3300         u64 data = msr->data;
3301         switch (ecx) {
3302         case MSR_IA32_TSC:
3303                 kvm_write_tsc(vcpu, msr);
3304                 break;
3305         case MSR_STAR:
3306                 svm->vmcb->save.star = data;
3307                 break;
3308 #ifdef CONFIG_X86_64
3309         case MSR_LSTAR:
3310                 svm->vmcb->save.lstar = data;
3311                 break;
3312         case MSR_CSTAR:
3313                 svm->vmcb->save.cstar = data;
3314                 break;
3315         case MSR_KERNEL_GS_BASE:
3316                 svm->vmcb->save.kernel_gs_base = data;
3317                 break;
3318         case MSR_SYSCALL_MASK:
3319                 svm->vmcb->save.sfmask = data;
3320                 break;
3321 #endif
3322         case MSR_IA32_SYSENTER_CS:
3323                 svm->vmcb->save.sysenter_cs = data;
3324                 break;
3325         case MSR_IA32_SYSENTER_EIP:
3326                 svm->sysenter_eip = data;
3327                 svm->vmcb->save.sysenter_eip = data;
3328                 break;
3329         case MSR_IA32_SYSENTER_ESP:
3330                 svm->sysenter_esp = data;
3331                 svm->vmcb->save.sysenter_esp = data;
3332                 break;
3333         case MSR_IA32_DEBUGCTLMSR:
3334                 if (!boot_cpu_has(X86_FEATURE_LBRV)) {
3335                         vcpu_unimpl(vcpu, "%s: MSR_IA32_DEBUGCTL 0x%llx, nop\n",
3336                                     __func__, data);
3337                         break;
3338                 }
3339                 if (data & DEBUGCTL_RESERVED_BITS)
3340                         return 1;
3341
3342                 svm->vmcb->save.dbgctl = data;
3343                 mark_dirty(svm->vmcb, VMCB_LBR);
3344                 if (data & (1ULL<<0))
3345                         svm_enable_lbrv(svm);
3346                 else
3347                         svm_disable_lbrv(svm);
3348                 break;
3349         case MSR_VM_HSAVE_PA:
3350                 svm->nested.hsave_msr = data;
3351                 break;
3352         case MSR_VM_CR:
3353                 return svm_set_vm_cr(vcpu, data);
3354         case MSR_VM_IGNNE:
3355                 vcpu_unimpl(vcpu, "unimplemented wrmsr: 0x%x data 0x%llx\n", ecx, data);
3356                 break;
3357         case MSR_IA32_CR_PAT:
3358                 if (npt_enabled) {
3359                         if (!kvm_mtrr_valid(vcpu, MSR_IA32_CR_PAT, data))
3360                                 return 1;
3361                         vcpu->arch.pat = data;
3362                         svm_set_guest_pat(svm, &svm->vmcb->save.g_pat);
3363                         mark_dirty(svm->vmcb, VMCB_NPT);
3364                         break;
3365                 }
3366                 /* fall through */
3367         default:
3368                 return kvm_set_msr_common(vcpu, msr);
3369         }
3370         return 0;
3371 }
3372
3373 static int wrmsr_interception(struct vcpu_svm *svm)
3374 {
3375         struct msr_data msr;
3376         u32 ecx = kvm_register_read(&svm->vcpu, VCPU_REGS_RCX);
3377         u64 data = kvm_read_edx_eax(&svm->vcpu);
3378
3379         msr.data = data;
3380         msr.index = ecx;
3381         msr.host_initiated = false;
3382
3383         svm->next_rip = kvm_rip_read(&svm->vcpu) + 2;
3384         if (kvm_set_msr(&svm->vcpu, &msr)) {
3385                 trace_kvm_msr_write_ex(ecx, data);
3386                 kvm_inject_gp(&svm->vcpu, 0);
3387         } else {
3388                 trace_kvm_msr_write(ecx, data);
3389                 skip_emulated_instruction(&svm->vcpu);
3390         }
3391         return 1;
3392 }
3393
3394 static int msr_interception(struct vcpu_svm *svm)
3395 {
3396         if (svm->vmcb->control.exit_info_1)
3397                 return wrmsr_interception(svm);
3398         else
3399                 return rdmsr_interception(svm);
3400 }
3401
3402 static int interrupt_window_interception(struct vcpu_svm *svm)
3403 {
3404         struct kvm_run *kvm_run = svm->vcpu.run;
3405
3406         kvm_make_request(KVM_REQ_EVENT, &svm->vcpu);
3407         svm_clear_vintr(svm);
3408         svm->vmcb->control.int_ctl &= ~V_IRQ_MASK;
3409         mark_dirty(svm->vmcb, VMCB_INTR);
3410         ++svm->vcpu.stat.irq_window_exits;
3411         /*
3412          * If the user space waits to inject interrupts, exit as soon as
3413          * possible
3414          */
3415         if (!irqchip_in_kernel(svm->vcpu.kvm) &&
3416             kvm_run->request_interrupt_window &&
3417             !kvm_cpu_has_interrupt(&svm->vcpu)) {
3418                 kvm_run->exit_reason = KVM_EXIT_IRQ_WINDOW_OPEN;
3419                 return 0;
3420         }
3421
3422         return 1;
3423 }
3424
3425 static int pause_interception(struct vcpu_svm *svm)
3426 {
3427         kvm_vcpu_on_spin(&(svm->vcpu));
3428         return 1;
3429 }
3430
3431 static int nop_interception(struct vcpu_svm *svm)
3432 {
3433         skip_emulated_instruction(&(svm->vcpu));
3434         return 1;
3435 }
3436
3437 static int monitor_interception(struct vcpu_svm *svm)
3438 {
3439         printk_once(KERN_WARNING "kvm: MONITOR instruction emulated as NOP!\n");
3440         return nop_interception(svm);
3441 }
3442
3443 static int mwait_interception(struct vcpu_svm *svm)
3444 {
3445         printk_once(KERN_WARNING "kvm: MWAIT instruction emulated as NOP!\n");
3446         return nop_interception(svm);
3447 }
3448
3449 static int (*const svm_exit_handlers[])(struct vcpu_svm *svm) = {
3450         [SVM_EXIT_READ_CR0]                     = cr_interception,
3451         [SVM_EXIT_READ_CR3]                     = cr_interception,
3452         [SVM_EXIT_READ_CR4]                     = cr_interception,
3453         [SVM_EXIT_READ_CR8]                     = cr_interception,
3454         [SVM_EXIT_CR0_SEL_WRITE]                = cr_interception,
3455         [SVM_EXIT_WRITE_CR0]                    = cr_interception,
3456         [SVM_EXIT_WRITE_CR3]                    = cr_interception,
3457         [SVM_EXIT_WRITE_CR4]                    = cr_interception,
3458         [SVM_EXIT_WRITE_CR8]                    = cr8_write_interception,
3459         [SVM_EXIT_READ_DR0]                     = dr_interception,
3460         [SVM_EXIT_READ_DR1]                     = dr_interception,
3461         [SVM_EXIT_READ_DR2]                     = dr_interception,
3462         [SVM_EXIT_READ_DR3]                     = dr_interception,
3463         [SVM_EXIT_READ_DR4]                     = dr_interception,
3464         [SVM_EXIT_READ_DR5]                     = dr_interception,
3465         [SVM_EXIT_READ_DR6]                     = dr_interception,
3466         [SVM_EXIT_READ_DR7]                     = dr_interception,
3467         [SVM_EXIT_WRITE_DR0]                    = dr_interception,
3468         [SVM_EXIT_WRITE_DR1]                    = dr_interception,
3469         [SVM_EXIT_WRITE_DR2]                    = dr_interception,
3470         [SVM_EXIT_WRITE_DR3]                    = dr_interception,
3471         [SVM_EXIT_WRITE_DR4]                    = dr_interception,
3472         [SVM_EXIT_WRITE_DR5]                    = dr_interception,
3473         [SVM_EXIT_WRITE_DR6]                    = dr_interception,
3474         [SVM_EXIT_WRITE_DR7]                    = dr_interception,
3475         [SVM_EXIT_EXCP_BASE + DB_VECTOR]        = db_interception,
3476         [SVM_EXIT_EXCP_BASE + BP_VECTOR]        = bp_interception,
3477         [SVM_EXIT_EXCP_BASE + UD_VECTOR]        = ud_interception,
3478         [SVM_EXIT_EXCP_BASE + PF_VECTOR]        = pf_interception,
3479         [SVM_EXIT_EXCP_BASE + NM_VECTOR]        = nm_interception,
3480         [SVM_EXIT_EXCP_BASE + MC_VECTOR]        = mc_interception,
3481         [SVM_EXIT_INTR]                         = intr_interception,
3482         [SVM_EXIT_NMI]                          = nmi_interception,
3483         [SVM_EXIT_SMI]                          = nop_on_interception,
3484         [SVM_EXIT_INIT]                         = nop_on_interception,
3485         [SVM_EXIT_VINTR]                        = interrupt_window_interception,
3486         [SVM_EXIT_RDPMC]                        = rdpmc_interception,
3487         [SVM_EXIT_CPUID]                        = cpuid_interception,
3488         [SVM_EXIT_IRET]                         = iret_interception,
3489         [SVM_EXIT_INVD]                         = emulate_on_interception,
3490         [SVM_EXIT_PAUSE]                        = pause_interception,
3491         [SVM_EXIT_HLT]                          = halt_interception,
3492         [SVM_EXIT_INVLPG]                       = invlpg_interception,
3493         [SVM_EXIT_INVLPGA]                      = invlpga_interception,
3494         [SVM_EXIT_IOIO]                         = io_interception,
3495         [SVM_EXIT_MSR]                          = msr_interception,
3496         [SVM_EXIT_TASK_SWITCH]                  = task_switch_interception,
3497         [SVM_EXIT_SHUTDOWN]                     = shutdown_interception,
3498         [SVM_EXIT_VMRUN]                        = vmrun_interception,
3499         [SVM_EXIT_VMMCALL]                      = vmmcall_interception,
3500         [SVM_EXIT_VMLOAD]                       = vmload_interception,
3501         [SVM_EXIT_VMSAVE]                       = vmsave_interception,
3502         [SVM_EXIT_STGI]                         = stgi_interception,
3503         [SVM_EXIT_CLGI]                         = clgi_interception,
3504         [SVM_EXIT_SKINIT]                       = skinit_interception,
3505         [SVM_EXIT_WBINVD]                       = wbinvd_interception,
3506         [SVM_EXIT_MONITOR]                      = monitor_interception,
3507         [SVM_EXIT_MWAIT]                        = mwait_interception,
3508         [SVM_EXIT_XSETBV]                       = xsetbv_interception,
3509         [SVM_EXIT_NPF]                          = pf_interception,
3510         [SVM_EXIT_RSM]                          = emulate_on_interception,
3511 };
3512
3513 static void dump_vmcb(struct kvm_vcpu *vcpu)
3514 {
3515         struct vcpu_svm *svm = to_svm(vcpu);
3516         struct vmcb_control_area *control = &svm->vmcb->control;
3517         struct vmcb_save_area *save = &svm->vmcb->save;
3518
3519         pr_err("VMCB Control Area:\n");
3520         pr_err("%-20s%04x\n", "cr_read:", control->intercept_cr & 0xffff);
3521         pr_err("%-20s%04x\n", "cr_write:", control->intercept_cr >> 16);
3522         pr_err("%-20s%04x\n", "dr_read:", control->intercept_dr & 0xffff);
3523         pr_err("%-20s%04x\n", "dr_write:", control->intercept_dr >> 16);
3524         pr_err("%-20s%08x\n", "exceptions:", control->intercept_exceptions);
3525         pr_err("%-20s%016llx\n", "intercepts:", control->intercept);
3526         pr_err("%-20s%d\n", "pause filter count:", control->pause_filter_count);
3527         pr_err("%-20s%016llx\n", "iopm_base_pa:", control->iopm_base_pa);
3528         pr_err("%-20s%016llx\n", "msrpm_base_pa:", control->msrpm_base_pa);
3529         pr_err("%-20s%016llx\n", "tsc_offset:", control->tsc_offset);
3530         pr_err("%-20s%d\n", "asid:", control->asid);
3531         pr_err("%-20s%d\n", "tlb_ctl:", control->tlb_ctl);
3532         pr_err("%-20s%08x\n", "int_ctl:", control->int_ctl);
3533         pr_err("%-20s%08x\n", "int_vector:", control->int_vector);
3534         pr_err("%-20s%08x\n", "int_state:", control->int_state);
3535         pr_err("%-20s%08x\n", "exit_code:", control->exit_code);
3536         pr_err("%-20s%016llx\n", "exit_info1:", control->exit_info_1);
3537         pr_err("%-20s%016llx\n", "exit_info2:", control->exit_info_2);
3538         pr_err("%-20s%08x\n", "exit_int_info:", control->exit_int_info);
3539         pr_err("%-20s%08x\n", "exit_int_info_err:", control->exit_int_info_err);
3540         pr_err("%-20s%lld\n", "nested_ctl:", control->nested_ctl);
3541         pr_err("%-20s%016llx\n", "nested_cr3:", control->nested_cr3);
3542         pr_err("%-20s%08x\n", "event_inj:", control->event_inj);
3543         pr_err("%-20s%08x\n", "event_inj_err:", control->event_inj_err);
3544         pr_err("%-20s%lld\n", "lbr_ctl:", control->lbr_ctl);
3545         pr_err("%-20s%016llx\n", "next_rip:", control->next_rip);
3546         pr_err("VMCB State Save Area:\n");
3547         pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3548                "es:",
3549                save->es.selector, save->es.attrib,
3550                save->es.limit, save->es.base);
3551         pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3552                "cs:",
3553                save->cs.selector, save->cs.attrib,
3554                save->cs.limit, save->cs.base);
3555         pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3556                "ss:",
3557                save->ss.selector, save->ss.attrib,
3558                save->ss.limit, save->ss.base);
3559         pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3560                "ds:",
3561                save->ds.selector, save->ds.attrib,
3562                save->ds.limit, save->ds.base);
3563         pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3564                "fs:",
3565                save->fs.selector, save->fs.attrib,
3566                save->fs.limit, save->fs.base);
3567         pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3568                "gs:",
3569                save->gs.selector, save->gs.attrib,
3570                save->gs.limit, save->gs.base);
3571         pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3572                "gdtr:",
3573                save->gdtr.selector, save->gdtr.attrib,
3574                save->gdtr.limit, save->gdtr.base);
3575         pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3576                "ldtr:",
3577                save->ldtr.selector, save->ldtr.attrib,
3578                save->ldtr.limit, save->ldtr.base);
3579         pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3580                "idtr:",
3581                save->idtr.selector, save->idtr.attrib,
3582                save->idtr.limit, save->idtr.base);
3583         pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3584                "tr:",
3585                save->tr.selector, save->tr.attrib,
3586                save->tr.limit, save->tr.base);
3587         pr_err("cpl:            %d                efer:         %016llx\n",
3588                 save->cpl, save->efer);
3589         pr_err("%-15s %016llx %-13s %016llx\n",
3590                "cr0:", save->cr0, "cr2:", save->cr2);
3591         pr_err("%-15s %016llx %-13s %016llx\n",
3592                "cr3:", save->cr3, "cr4:", save->cr4);
3593         pr_err("%-15s %016llx %-13s %016llx\n",
3594                "dr6:", save->dr6, "dr7:", save->dr7);
3595         pr_err("%-15s %016llx %-13s %016llx\n",
3596                "rip:", save->rip, "rflags:", save->rflags);
3597         pr_err("%-15s %016llx %-13s %016llx\n",
3598                "rsp:", save->rsp, "rax:", save->rax);
3599         pr_err("%-15s %016llx %-13s %016llx\n",
3600                "star:", save->star, "lstar:", save->lstar);
3601         pr_err("%-15s %016llx %-13s %016llx\n",
3602                "cstar:", save->cstar, "sfmask:", save->sfmask);
3603         pr_err("%-15s %016llx %-13s %016llx\n",
3604                "kernel_gs_base:", save->kernel_gs_base,
3605                "sysenter_cs:", save->sysenter_cs);
3606         pr_err("%-15s %016llx %-13s %016llx\n",
3607                "sysenter_esp:", save->sysenter_esp,
3608                "sysenter_eip:", save->sysenter_eip);
3609         pr_err("%-15s %016llx %-13s %016llx\n",
3610                "gpat:", save->g_pat, "dbgctl:", save->dbgctl);
3611         pr_err("%-15s %016llx %-13s %016llx\n",
3612                "br_from:", save->br_from, "br_to:", save->br_to);
3613         pr_err("%-15s %016llx %-13s %016llx\n",
3614                "excp_from:", save->last_excp_from,
3615                "excp_to:", save->last_excp_to);
3616 }
3617
3618 static void svm_get_exit_info(struct kvm_vcpu *vcpu, u64 *info1, u64 *info2)
3619 {
3620         struct vmcb_control_area *control = &to_svm(vcpu)->vmcb->control;
3621
3622         *info1 = control->exit_info_1;
3623         *info2 = control->exit_info_2;
3624 }
3625
3626 static int handle_exit(struct kvm_vcpu *vcpu)
3627 {
3628         struct vcpu_svm *svm = to_svm(vcpu);
3629         struct kvm_run *kvm_run = vcpu->run;
3630         u32 exit_code = svm->vmcb->control.exit_code;
3631
3632         if (!is_cr_intercept(svm, INTERCEPT_CR0_WRITE))
3633                 vcpu->arch.cr0 = svm->vmcb->save.cr0;
3634         if (npt_enabled)
3635                 vcpu->arch.cr3 = svm->vmcb->save.cr3;
3636
3637         if (unlikely(svm->nested.exit_required)) {
3638                 nested_svm_vmexit(svm);
3639                 svm->nested.exit_required = false;
3640
3641                 return 1;
3642         }
3643
3644         if (is_guest_mode(vcpu)) {
3645                 int vmexit;
3646
3647                 trace_kvm_nested_vmexit(svm->vmcb->save.rip, exit_code,
3648                                         svm->vmcb->control.exit_info_1,
3649                                         svm->vmcb->control.exit_info_2,
3650                                         svm->vmcb->control.exit_int_info,
3651                                         svm->vmcb->control.exit_int_info_err,
3652                                         KVM_ISA_SVM);
3653
3654                 vmexit = nested_svm_exit_special(svm);
3655
3656                 if (vmexit == NESTED_EXIT_CONTINUE)
3657                         vmexit = nested_svm_exit_handled(svm);
3658
3659                 if (vmexit == NESTED_EXIT_DONE)
3660                         return 1;
3661         }
3662
3663         svm_complete_interrupts(svm);
3664
3665         if (svm->vmcb->control.exit_code == SVM_EXIT_ERR) {
3666                 kvm_run->exit_reason = KVM_EXIT_FAIL_ENTRY;
3667                 kvm_run->fail_entry.hardware_entry_failure_reason
3668                         = svm->vmcb->control.exit_code;
3669                 pr_err("KVM: FAILED VMRUN WITH VMCB:\n");
3670                 dump_vmcb(vcpu);
3671                 return 0;
3672         }
3673
3674         if (is_external_interrupt(svm->vmcb->control.exit_int_info) &&
3675             exit_code != SVM_EXIT_EXCP_BASE + PF_VECTOR &&
3676             exit_code != SVM_EXIT_NPF && exit_code != SVM_EXIT_TASK_SWITCH &&
3677             exit_code != SVM_EXIT_INTR && exit_code != SVM_EXIT_NMI)
3678                 printk(KERN_ERR "%s: unexpected exit_int_info 0x%x "
3679                        "exit_code 0x%x\n",
3680                        __func__, svm->vmcb->control.exit_int_info,
3681                        exit_code);
3682
3683         if (exit_code >= ARRAY_SIZE(svm_exit_handlers)
3684             || !svm_exit_handlers[exit_code]) {
3685                 WARN_ONCE(1, "svm: unexpected exit reason 0x%x\n", exit_code);
3686                 kvm_queue_exception(vcpu, UD_VECTOR);
3687                 return 1;
3688         }
3689
3690         return svm_exit_handlers[exit_code](svm);
3691 }
3692
3693 static void reload_tss(struct kvm_vcpu *vcpu)
3694 {
3695         int cpu = raw_smp_processor_id();
3696
3697         struct svm_cpu_data *sd = per_cpu(svm_data, cpu);
3698         sd->tss_desc->type = 9; /* available 32/64-bit TSS */
3699         load_TR_desc();
3700 }
3701
3702 static void pre_svm_run(struct vcpu_svm *svm)
3703 {
3704         int cpu = raw_smp_processor_id();
3705
3706         struct svm_cpu_data *sd = per_cpu(svm_data, cpu);
3707
3708         /* FIXME: handle wraparound of asid_generation */
3709         if (svm->asid_generation != sd->asid_generation)
3710                 new_asid(svm, sd);
3711 }
3712
3713 static void svm_inject_nmi(struct kvm_vcpu *vcpu)
3714 {
3715         struct vcpu_svm *svm = to_svm(vcpu);
3716
3717         svm->vmcb->control.event_inj = SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_NMI;
3718         vcpu->arch.hflags |= HF_NMI_MASK;
3719         set_intercept(svm, INTERCEPT_IRET);
3720         ++vcpu->stat.nmi_injections;
3721 }
3722
3723 static inline void svm_inject_irq(struct vcpu_svm *svm, int irq)
3724 {
3725         struct vmcb_control_area *control;
3726
3727         control = &svm->vmcb->control;
3728         control->int_vector = irq;
3729         control->int_ctl &= ~V_INTR_PRIO_MASK;
3730         control->int_ctl |= V_IRQ_MASK |
3731                 ((/*control->int_vector >> 4*/ 0xf) << V_INTR_PRIO_SHIFT);
3732         mark_dirty(svm->vmcb, VMCB_INTR);
3733 }
3734
3735 static void svm_set_irq(struct kvm_vcpu *vcpu)
3736 {
3737         struct vcpu_svm *svm = to_svm(vcpu);
3738
3739         BUG_ON(!(gif_set(svm)));
3740
3741         trace_kvm_inj_virq(vcpu->arch.interrupt.nr);
3742         ++vcpu->stat.irq_injections;
3743
3744         svm->vmcb->control.event_inj = vcpu->arch.interrupt.nr |
3745                 SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_INTR;
3746 }
3747
3748 static void update_cr8_intercept(struct kvm_vcpu *vcpu, int tpr, int irr)
3749 {
3750         struct vcpu_svm *svm = to_svm(vcpu);
3751
3752         if (is_guest_mode(vcpu) && (vcpu->arch.hflags & HF_VINTR_MASK))
3753                 return;
3754
3755         clr_cr_intercept(svm, INTERCEPT_CR8_WRITE);
3756
3757         if (irr == -1)
3758                 return;
3759
3760         if (tpr >= irr)
3761                 set_cr_intercept(svm, INTERCEPT_CR8_WRITE);
3762 }
3763
3764 static void svm_set_virtual_x2apic_mode(struct kvm_vcpu *vcpu, bool set)
3765 {
3766         return;
3767 }
3768
3769 static int svm_vm_has_apicv(struct kvm *kvm)
3770 {
3771         return 0;
3772 }
3773
3774 static void svm_load_eoi_exitmap(struct kvm_vcpu *vcpu, u64 *eoi_exit_bitmap)
3775 {
3776         return;
3777 }
3778
3779 static void svm_sync_pir_to_irr(struct kvm_vcpu *vcpu)
3780 {
3781         return;
3782 }
3783
3784 static int svm_nmi_allowed(struct kvm_vcpu *vcpu)
3785 {
3786         struct vcpu_svm *svm = to_svm(vcpu);
3787         struct vmcb *vmcb = svm->vmcb;
3788         int ret;
3789         ret = !(vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK) &&
3790               !(svm->vcpu.arch.hflags & HF_NMI_MASK);
3791         ret = ret && gif_set(svm) && nested_svm_nmi(svm);
3792
3793         return ret;
3794 }
3795
3796 static bool svm_get_nmi_mask(struct kvm_vcpu *vcpu)
3797 {
3798         struct vcpu_svm *svm = to_svm(vcpu);
3799
3800         return !!(svm->vcpu.arch.hflags & HF_NMI_MASK);
3801 }
3802
3803 static void svm_set_nmi_mask(struct kvm_vcpu *vcpu, bool masked)
3804 {
3805         struct vcpu_svm *svm = to_svm(vcpu);
3806
3807         if (masked) {
3808                 svm->vcpu.arch.hflags |= HF_NMI_MASK;
3809                 set_intercept(svm, INTERCEPT_IRET);
3810         } else {
3811                 svm->vcpu.arch.hflags &= ~HF_NMI_MASK;
3812                 clr_intercept(svm, INTERCEPT_IRET);
3813         }
3814 }
3815
3816 static int svm_interrupt_allowed(struct kvm_vcpu *vcpu)
3817 {
3818         struct vcpu_svm *svm = to_svm(vcpu);
3819         struct vmcb *vmcb = svm->vmcb;
3820         int ret;
3821
3822         if (!gif_set(svm) ||
3823              (vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK))
3824                 return 0;
3825
3826         ret = !!(kvm_get_rflags(vcpu) & X86_EFLAGS_IF);
3827
3828         if (is_guest_mode(vcpu))
3829                 return ret && !(svm->vcpu.arch.hflags & HF_VINTR_MASK);
3830
3831         return ret;
3832 }
3833
3834 static void enable_irq_window(struct kvm_vcpu *vcpu)
3835 {
3836         struct vcpu_svm *svm = to_svm(vcpu);
3837
3838         /*
3839          * In case GIF=0 we can't rely on the CPU to tell us when GIF becomes
3840          * 1, because that's a separate STGI/VMRUN intercept.  The next time we
3841          * get that intercept, this function will be called again though and
3842          * we'll get the vintr intercept.
3843          */
3844         if (gif_set(svm) && nested_svm_intr(svm)) {
3845                 svm_set_vintr(svm);
3846                 svm_inject_irq(svm, 0x0);
3847         }
3848 }
3849
3850 static void enable_nmi_window(struct kvm_vcpu *vcpu)
3851 {
3852         struct vcpu_svm *svm = to_svm(vcpu);
3853
3854         if ((svm->vcpu.arch.hflags & (HF_NMI_MASK | HF_IRET_MASK))
3855             == HF_NMI_MASK)
3856                 return; /* IRET will cause a vm exit */
3857
3858         /*
3859          * Something prevents NMI from been injected. Single step over possible
3860          * problem (IRET or exception injection or interrupt shadow)
3861          */
3862         svm->nmi_singlestep = true;
3863         svm->vmcb->save.rflags |= (X86_EFLAGS_TF | X86_EFLAGS_RF);
3864         update_db_bp_intercept(vcpu);
3865 }
3866
3867 static int svm_set_tss_addr(struct kvm *kvm, unsigned int addr)
3868 {
3869         return 0;
3870 }
3871
3872 static void svm_flush_tlb(struct kvm_vcpu *vcpu)
3873 {
3874         struct vcpu_svm *svm = to_svm(vcpu);
3875
3876         if (static_cpu_has(X86_FEATURE_FLUSHBYASID))
3877                 svm->vmcb->control.tlb_ctl = TLB_CONTROL_FLUSH_ASID;
3878         else
3879                 svm->asid_generation--;
3880 }
3881
3882 static void svm_prepare_guest_switch(struct kvm_vcpu *vcpu)
3883 {
3884 }
3885
3886 static inline void sync_cr8_to_lapic(struct kvm_vcpu *vcpu)
3887 {
3888         struct vcpu_svm *svm = to_svm(vcpu);
3889
3890         if (is_guest_mode(vcpu) && (vcpu->arch.hflags & HF_VINTR_MASK))
3891                 return;
3892
3893         if (!is_cr_intercept(svm, INTERCEPT_CR8_WRITE)) {
3894                 int cr8 = svm->vmcb->control.int_ctl & V_TPR_MASK;
3895                 kvm_set_cr8(vcpu, cr8);
3896         }
3897 }
3898
3899 static inline void sync_lapic_to_cr8(struct kvm_vcpu *vcpu)
3900 {
3901         struct vcpu_svm *svm = to_svm(vcpu);
3902         u64 cr8;
3903
3904         if (is_guest_mode(vcpu) && (vcpu->arch.hflags & HF_VINTR_MASK))
3905                 return;
3906
3907         cr8 = kvm_get_cr8(vcpu);
3908         svm->vmcb->control.int_ctl &= ~V_TPR_MASK;
3909         svm->vmcb->control.int_ctl |= cr8 & V_TPR_MASK;
3910 }
3911
3912 static void svm_complete_interrupts(struct vcpu_svm *svm)
3913 {
3914         u8 vector;
3915         int type;
3916         u32 exitintinfo = svm->vmcb->control.exit_int_info;
3917         unsigned int3_injected = svm->int3_injected;
3918
3919         svm->int3_injected = 0;
3920
3921         /*
3922          * If we've made progress since setting HF_IRET_MASK, we've
3923          * executed an IRET and can allow NMI injection.
3924          */
3925         if ((svm->vcpu.arch.hflags & HF_IRET_MASK)
3926             && kvm_rip_read(&svm->vcpu) != svm->nmi_iret_rip) {
3927                 svm->vcpu.arch.hflags &= ~(HF_NMI_MASK | HF_IRET_MASK);
3928                 kvm_make_request(KVM_REQ_EVENT, &svm->vcpu);
3929         }
3930
3931         svm->vcpu.arch.nmi_injected = false;
3932         kvm_clear_exception_queue(&svm->vcpu);
3933         kvm_clear_interrupt_queue(&svm->vcpu);
3934
3935         if (!(exitintinfo & SVM_EXITINTINFO_VALID))
3936                 return;
3937
3938         kvm_make_request(KVM_REQ_EVENT, &svm->vcpu);
3939
3940         vector = exitintinfo & SVM_EXITINTINFO_VEC_MASK;
3941         type = exitintinfo & SVM_EXITINTINFO_TYPE_MASK;
3942
3943         switch (type) {
3944         case SVM_EXITINTINFO_TYPE_NMI:
3945                 svm->vcpu.arch.nmi_injected = true;
3946                 break;
3947         case SVM_EXITINTINFO_TYPE_EXEPT:
3948                 /*
3949                  * In case of software exceptions, do not reinject the vector,
3950                  * but re-execute the instruction instead. Rewind RIP first
3951                  * if we emulated INT3 before.
3952                  */
3953                 if (kvm_exception_is_soft(vector)) {
3954                         if (vector == BP_VECTOR && int3_injected &&
3955                             kvm_is_linear_rip(&svm->vcpu, svm->int3_rip))
3956                                 kvm_rip_write(&svm->vcpu,
3957                                               kvm_rip_read(&svm->vcpu) -
3958                                               int3_injected);
3959                         break;
3960                 }
3961                 if (exitintinfo & SVM_EXITINTINFO_VALID_ERR) {
3962                         u32 err = svm->vmcb->control.exit_int_info_err;
3963                         kvm_requeue_exception_e(&svm->vcpu, vector, err);
3964
3965                 } else
3966                         kvm_requeue_exception(&svm->vcpu, vector);
3967                 break;
3968         case SVM_EXITINTINFO_TYPE_INTR:
3969                 kvm_queue_interrupt(&svm->vcpu, vector, false);
3970                 break;
3971         default:
3972                 break;
3973         }
3974 }
3975
3976 static void svm_cancel_injection(struct kvm_vcpu *vcpu)
3977 {
3978         struct vcpu_svm *svm = to_svm(vcpu);
3979         struct vmcb_control_area *control = &svm->vmcb->control;
3980
3981         control->exit_int_info = control->event_inj;
3982         control->exit_int_info_err = control->event_inj_err;
3983         control->event_inj = 0;
3984         svm_complete_interrupts(svm);
3985 }
3986
3987 static void svm_vcpu_run(struct kvm_vcpu *vcpu)
3988 {
3989         struct vcpu_svm *svm = to_svm(vcpu);
3990
3991         svm->vmcb->save.rax = vcpu->arch.regs[VCPU_REGS_RAX];
3992         svm->vmcb->save.rsp = vcpu->arch.regs[VCPU_REGS_RSP];
3993         svm->vmcb->save.rip = vcpu->arch.regs[VCPU_REGS_RIP];
3994
3995         /*
3996          * A vmexit emulation is required before the vcpu can be executed
3997          * again.
3998          */
3999         if (unlikely(svm->nested.exit_required))
4000                 return;
4001
4002         pre_svm_run(svm);
4003
4004         sync_lapic_to_cr8(vcpu);
4005
4006         svm->vmcb->save.cr2 = vcpu->arch.cr2;
4007
4008         clgi();
4009
4010         local_irq_enable();
4011
4012         asm volatile (
4013                 "push %%" _ASM_BP "; \n\t"
4014                 "mov %c[rbx](%[svm]), %%" _ASM_BX " \n\t"
4015                 "mov %c[rcx](%[svm]), %%" _ASM_CX " \n\t"
4016                 "mov %c[rdx](%[svm]), %%" _ASM_DX " \n\t"
4017                 "mov %c[rsi](%[svm]), %%" _ASM_SI " \n\t"
4018                 "mov %c[rdi](%[svm]), %%" _ASM_DI " \n\t"
4019                 "mov %c[rbp](%[svm]), %%" _ASM_BP " \n\t"
4020 #ifdef CONFIG_X86_64
4021                 "mov %c[r8](%[svm]),  %%r8  \n\t"
4022                 "mov %c[r9](%[svm]),  %%r9  \n\t"
4023                 "mov %c[r10](%[svm]), %%r10 \n\t"
4024                 "mov %c[r11](%[svm]), %%r11 \n\t"
4025                 "mov %c[r12](%[svm]), %%r12 \n\t"
4026                 "mov %c[r13](%[svm]), %%r13 \n\t"
4027                 "mov %c[r14](%[svm]), %%r14 \n\t"
4028                 "mov %c[r15](%[svm]), %%r15 \n\t"
4029 #endif
4030
4031                 /* Enter guest mode */
4032                 "push %%" _ASM_AX " \n\t"
4033                 "mov %c[vmcb](%[svm]), %%" _ASM_AX " \n\t"
4034                 __ex(SVM_VMLOAD) "\n\t"
4035                 __ex(SVM_VMRUN) "\n\t"
4036                 __ex(SVM_VMSAVE) "\n\t"
4037                 "pop %%" _ASM_AX " \n\t"
4038
4039                 /* Save guest registers, load host registers */
4040                 "mov %%" _ASM_BX ", %c[rbx](%[svm]) \n\t"
4041                 "mov %%" _ASM_CX ", %c[rcx](%[svm]) \n\t"
4042                 "mov %%" _ASM_DX ", %c[rdx](%[svm]) \n\t"
4043                 "mov %%" _ASM_SI ", %c[rsi](%[svm]) \n\t"
4044                 "mov %%" _ASM_DI ", %c[rdi](%[svm]) \n\t"
4045                 "mov %%" _ASM_BP ", %c[rbp](%[svm]) \n\t"
4046 #ifdef CONFIG_X86_64
4047                 "mov %%r8,  %c[r8](%[svm]) \n\t"
4048                 "mov %%r9,  %c[r9](%[svm]) \n\t"
4049                 "mov %%r10, %c[r10](%[svm]) \n\t"
4050                 "mov %%r11, %c[r11](%[svm]) \n\t"
4051                 "mov %%r12, %c[r12](%[svm]) \n\t"
4052                 "mov %%r13, %c[r13](%[svm]) \n\t"
4053                 "mov %%r14, %c[r14](%[svm]) \n\t"
4054                 "mov %%r15, %c[r15](%[svm]) \n\t"
4055 #endif
4056                 "pop %%" _ASM_BP
4057                 :
4058                 : [svm]"a"(svm),
4059                   [vmcb]"i"(offsetof(struct vcpu_svm, vmcb_pa)),
4060                   [rbx]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RBX])),
4061                   [rcx]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RCX])),
4062                   [rdx]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RDX])),
4063                   [rsi]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RSI])),
4064                   [rdi]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RDI])),
4065                   [rbp]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RBP]))
4066 #ifdef CONFIG_X86_64
4067                   , [r8]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R8])),
4068                   [r9]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R9])),
4069                   [r10]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R10])),
4070                   [r11]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R11])),
4071                   [r12]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R12])),
4072                   [r13]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R13])),
4073                   [r14]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R14])),
4074                   [r15]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R15]))
4075 #endif
4076                 : "cc", "memory"
4077 #ifdef CONFIG_X86_64
4078                 , "rbx", "rcx", "rdx", "rsi", "rdi"
4079                 , "r8", "r9", "r10", "r11" , "r12", "r13", "r14", "r15"
4080 #else
4081                 , "ebx", "ecx", "edx", "esi", "edi"
4082 #endif
4083                 );
4084
4085 #ifdef CONFIG_X86_64
4086         wrmsrl(MSR_GS_BASE, svm->host.gs_base);
4087 #else
4088         loadsegment(fs, svm->host.fs);
4089 #ifndef CONFIG_X86_32_LAZY_GS
4090         loadsegment(gs, svm->host.gs);
4091 #endif
4092 #endif
4093
4094         reload_tss(vcpu);
4095
4096         local_irq_disable();
4097
4098         vcpu->arch.cr2 = svm->vmcb->save.cr2;
4099         vcpu->arch.regs[VCPU_REGS_RAX] = svm->vmcb->save.rax;
4100         vcpu->arch.regs[VCPU_REGS_RSP] = svm->vmcb->save.rsp;
4101         vcpu->arch.regs[VCPU_REGS_RIP] = svm->vmcb->save.rip;
4102
4103         trace_kvm_exit(svm->vmcb->control.exit_code, vcpu, KVM_ISA_SVM);
4104
4105         if (unlikely(svm->vmcb->control.exit_code == SVM_EXIT_NMI))
4106                 kvm_before_handle_nmi(&svm->vcpu);
4107
4108         stgi();
4109
4110         /* Any pending NMI will happen here */
4111
4112         if (unlikely(svm->vmcb->control.exit_code == SVM_EXIT_NMI))
4113                 kvm_after_handle_nmi(&svm->vcpu);
4114
4115         sync_cr8_to_lapic(vcpu);
4116
4117         svm->next_rip = 0;
4118
4119         svm->vmcb->control.tlb_ctl = TLB_CONTROL_DO_NOTHING;
4120
4121         /* if exit due to PF check for async PF */
4122         if (svm->vmcb->control.exit_code == SVM_EXIT_EXCP_BASE + PF_VECTOR)
4123                 svm->apf_reason = kvm_read_and_reset_pf_reason();
4124
4125         if (npt_enabled) {
4126                 vcpu->arch.regs_avail &= ~(1 << VCPU_EXREG_PDPTR);
4127                 vcpu->arch.regs_dirty &= ~(1 << VCPU_EXREG_PDPTR);
4128         }
4129
4130         /*
4131          * We need to handle MC intercepts here before the vcpu has a chance to
4132          * change the physical cpu
4133          */
4134         if (unlikely(svm->vmcb->control.exit_code ==
4135                      SVM_EXIT_EXCP_BASE + MC_VECTOR))
4136                 svm_handle_mce(svm);
4137
4138         mark_all_clean(svm->vmcb);
4139 }
4140
4141 static void svm_set_cr3(struct kvm_vcpu *vcpu, unsigned long root)
4142 {
4143         struct vcpu_svm *svm = to_svm(vcpu);
4144
4145         svm->vmcb->save.cr3 = root;
4146         mark_dirty(svm->vmcb, VMCB_CR);
4147         svm_flush_tlb(vcpu);
4148 }
4149
4150 static void set_tdp_cr3(struct kvm_vcpu *vcpu, unsigned long root)
4151 {
4152         struct vcpu_svm *svm = to_svm(vcpu);
4153
4154         svm->vmcb->control.nested_cr3 = root;
4155         mark_dirty(svm->vmcb, VMCB_NPT);
4156
4157         /* Also sync guest cr3 here in case we live migrate */
4158         svm->vmcb->save.cr3 = kvm_read_cr3(vcpu);
4159         mark_dirty(svm->vmcb, VMCB_CR);
4160
4161         svm_flush_tlb(vcpu);
4162 }
4163
4164 static int is_disabled(void)
4165 {
4166         u64 vm_cr;
4167
4168         rdmsrl(MSR_VM_CR, vm_cr);
4169         if (vm_cr & (1 << SVM_VM_CR_SVM_DISABLE))
4170                 return 1;
4171
4172         return 0;
4173 }
4174
4175 static void
4176 svm_patch_hypercall(struct kvm_vcpu *vcpu, unsigned char *hypercall)
4177 {
4178         /*
4179          * Patch in the VMMCALL instruction:
4180          */
4181         hypercall[0] = 0x0f;
4182         hypercall[1] = 0x01;
4183         hypercall[2] = 0xd9;
4184 }
4185
4186 static void svm_check_processor_compat(void *rtn)
4187 {
4188         *(int *)rtn = 0;
4189 }
4190
4191 static bool svm_cpu_has_accelerated_tpr(void)
4192 {
4193         return false;
4194 }
4195
4196 static bool svm_has_high_real_mode_segbase(void)
4197 {
4198         return true;
4199 }
4200
4201 static void svm_cpuid_update(struct kvm_vcpu *vcpu)
4202 {
4203 }
4204
4205 static void svm_set_supported_cpuid(u32 func, struct kvm_cpuid_entry2 *entry)
4206 {
4207         switch (func) {
4208         case 0x80000001:
4209                 if (nested)
4210                         entry->ecx |= (1 << 2); /* Set SVM bit */
4211                 break;
4212         case 0x8000000A:
4213                 entry->eax = 1; /* SVM revision 1 */
4214                 entry->ebx = 8; /* Lets support 8 ASIDs in case we add proper
4215                                    ASID emulation to nested SVM */
4216                 entry->ecx = 0; /* Reserved */
4217                 entry->edx = 0; /* Per default do not support any
4218                                    additional features */
4219
4220                 /* Support next_rip if host supports it */
4221                 if (boot_cpu_has(X86_FEATURE_NRIPS))
4222                         entry->edx |= SVM_FEATURE_NRIP;
4223
4224                 /* Support NPT for the guest if enabled */
4225                 if (npt_enabled)
4226                         entry->edx |= SVM_FEATURE_NPT;
4227
4228                 break;
4229         }
4230 }
4231
4232 static int svm_get_lpage_level(void)
4233 {
4234         return PT_PDPE_LEVEL;
4235 }
4236
4237 static bool svm_rdtscp_supported(void)
4238 {
4239         return false;
4240 }
4241
4242 static bool svm_invpcid_supported(void)
4243 {
4244         return false;
4245 }
4246
4247 static bool svm_mpx_supported(void)
4248 {
4249         return false;
4250 }
4251
4252 static bool svm_xsaves_supported(void)
4253 {
4254         return false;
4255 }
4256
4257 static bool svm_has_wbinvd_exit(void)
4258 {
4259         return true;
4260 }
4261
4262 static void svm_fpu_deactivate(struct kvm_vcpu *vcpu)
4263 {
4264         struct vcpu_svm *svm = to_svm(vcpu);
4265
4266         set_exception_intercept(svm, NM_VECTOR);
4267         update_cr0_intercept(svm);
4268 }
4269
4270 #define PRE_EX(exit)  { .exit_code = (exit), \
4271                         .stage = X86_ICPT_PRE_EXCEPT, }
4272 #define POST_EX(exit) { .exit_code = (exit), \
4273                         .stage = X86_ICPT_POST_EXCEPT, }
4274 #define POST_MEM(exit) { .exit_code = (exit), \
4275                         .stage = X86_ICPT_POST_MEMACCESS, }
4276
4277 static const struct __x86_intercept {
4278         u32 exit_code;
4279         enum x86_intercept_stage stage;
4280 } x86_intercept_map[] = {
4281         [x86_intercept_cr_read]         = POST_EX(SVM_EXIT_READ_CR0),
4282         [x86_intercept_cr_write]        = POST_EX(SVM_EXIT_WRITE_CR0),
4283         [x86_intercept_clts]            = POST_EX(SVM_EXIT_WRITE_CR0),
4284         [x86_intercept_lmsw]            = POST_EX(SVM_EXIT_WRITE_CR0),
4285         [x86_intercept_smsw]            = POST_EX(SVM_EXIT_READ_CR0),
4286         [x86_intercept_dr_read]         = POST_EX(SVM_EXIT_READ_DR0),
4287         [x86_intercept_dr_write]        = POST_EX(SVM_EXIT_WRITE_DR0),
4288         [x86_intercept_sldt]            = POST_EX(SVM_EXIT_LDTR_READ),
4289         [x86_intercept_str]             = POST_EX(SVM_EXIT_TR_READ),
4290         [x86_intercept_lldt]            = POST_EX(SVM_EXIT_LDTR_WRITE),
4291         [x86_intercept_ltr]             = POST_EX(SVM_EXIT_TR_WRITE),
4292         [x86_intercept_sgdt]            = POST_EX(SVM_EXIT_GDTR_READ),
4293         [x86_intercept_sidt]            = POST_EX(SVM_EXIT_IDTR_READ),
4294         [x86_intercept_lgdt]            = POST_EX(SVM_EXIT_GDTR_WRITE),
4295         [x86_intercept_lidt]            = POST_EX(SVM_EXIT_IDTR_WRITE),
4296         [x86_intercept_vmrun]           = POST_EX(SVM_EXIT_VMRUN),
4297         [x86_intercept_vmmcall]         = POST_EX(SVM_EXIT_VMMCALL),
4298         [x86_intercept_vmload]          = POST_EX(SVM_EXIT_VMLOAD),
4299         [x86_intercept_vmsave]          = POST_EX(SVM_EXIT_VMSAVE),
4300         [x86_intercept_stgi]            = POST_EX(SVM_EXIT_STGI),
4301         [x86_intercept_clgi]            = POST_EX(SVM_EXIT_CLGI),
4302         [x86_intercept_skinit]          = POST_EX(SVM_EXIT_SKINIT),
4303         [x86_intercept_invlpga]         = POST_EX(SVM_EXIT_INVLPGA),
4304         [x86_intercept_rdtscp]          = POST_EX(SVM_EXIT_RDTSCP),
4305         [x86_intercept_monitor]         = POST_MEM(SVM_EXIT_MONITOR),
4306         [x86_intercept_mwait]           = POST_EX(SVM_EXIT_MWAIT),
4307         [x86_intercept_invlpg]          = POST_EX(SVM_EXIT_INVLPG),
4308         [x86_intercept_invd]            = POST_EX(SVM_EXIT_INVD),
4309         [x86_intercept_wbinvd]          = POST_EX(SVM_EXIT_WBINVD),
4310         [x86_intercept_wrmsr]           = POST_EX(SVM_EXIT_MSR),
4311         [x86_intercept_rdtsc]           = POST_EX(SVM_EXIT_RDTSC),
4312         [x86_intercept_rdmsr]           = POST_EX(SVM_EXIT_MSR),
4313         [x86_intercept_rdpmc]           = POST_EX(SVM_EXIT_RDPMC),
4314         [x86_intercept_cpuid]           = PRE_EX(SVM_EXIT_CPUID),
4315         [x86_intercept_rsm]             = PRE_EX(SVM_EXIT_RSM),
4316         [x86_intercept_pause]           = PRE_EX(SVM_EXIT_PAUSE),
4317         [x86_intercept_pushf]           = PRE_EX(SVM_EXIT_PUSHF),
4318         [x86_intercept_popf]            = PRE_EX(SVM_EXIT_POPF),
4319         [x86_intercept_intn]            = PRE_EX(SVM_EXIT_SWINT),
4320         [x86_intercept_iret]            = PRE_EX(SVM_EXIT_IRET),
4321         [x86_intercept_icebp]           = PRE_EX(SVM_EXIT_ICEBP),
4322         [x86_intercept_hlt]             = POST_EX(SVM_EXIT_HLT),
4323         [x86_intercept_in]              = POST_EX(SVM_EXIT_IOIO),
4324         [x86_intercept_ins]             = POST_EX(SVM_EXIT_IOIO),
4325         [x86_intercept_out]             = POST_EX(SVM_EXIT_IOIO),
4326         [x86_intercept_outs]            = POST_EX(SVM_EXIT_IOIO),
4327 };
4328
4329 #undef PRE_EX
4330 #undef POST_EX
4331 #undef POST_MEM
4332
4333 static int svm_check_intercept(struct kvm_vcpu *vcpu,
4334                                struct x86_instruction_info *info,
4335                                enum x86_intercept_stage stage)
4336 {
4337         struct vcpu_svm *svm = to_svm(vcpu);
4338         int vmexit, ret = X86EMUL_CONTINUE;
4339         struct __x86_intercept icpt_info;
4340         struct vmcb *vmcb = svm->vmcb;
4341
4342         if (info->intercept >= ARRAY_SIZE(x86_intercept_map))
4343                 goto out;
4344
4345         icpt_info = x86_intercept_map[info->intercept];
4346
4347         if (stage != icpt_info.stage)
4348                 goto out;
4349
4350         switch (icpt_info.exit_code) {
4351         case SVM_EXIT_READ_CR0:
4352                 if (info->intercept == x86_intercept_cr_read)
4353                         icpt_info.exit_code += info->modrm_reg;
4354                 break;
4355         case SVM_EXIT_WRITE_CR0: {
4356                 unsigned long cr0, val;
4357                 u64 intercept;
4358
4359                 if (info->intercept == x86_intercept_cr_write)
4360                         icpt_info.exit_code += info->modrm_reg;
4361
4362                 if (icpt_info.exit_code != SVM_EXIT_WRITE_CR0 ||
4363                     info->intercept == x86_intercept_clts)
4364                         break;
4365
4366                 intercept = svm->nested.intercept;
4367
4368                 if (!(intercept & (1ULL << INTERCEPT_SELECTIVE_CR0)))
4369                         break;
4370
4371                 cr0 = vcpu->arch.cr0 & ~SVM_CR0_SELECTIVE_MASK;
4372                 val = info->src_val  & ~SVM_CR0_SELECTIVE_MASK;
4373
4374                 if (info->intercept == x86_intercept_lmsw) {
4375                         cr0 &= 0xfUL;
4376                         val &= 0xfUL;
4377                         /* lmsw can't clear PE - catch this here */
4378                         if (cr0 & X86_CR0_PE)
4379                                 val |= X86_CR0_PE;
4380                 }
4381
4382                 if (cr0 ^ val)
4383                         icpt_info.exit_code = SVM_EXIT_CR0_SEL_WRITE;
4384
4385                 break;
4386         }
4387         case SVM_EXIT_READ_DR0:
4388         case SVM_EXIT_WRITE_DR0:
4389                 icpt_info.exit_code += info->modrm_reg;
4390                 break;
4391         case SVM_EXIT_MSR:
4392                 if (info->intercept == x86_intercept_wrmsr)
4393                         vmcb->control.exit_info_1 = 1;
4394                 else
4395                         vmcb->control.exit_info_1 = 0;
4396                 break;
4397         case SVM_EXIT_PAUSE:
4398                 /*
4399                  * We get this for NOP only, but pause
4400                  * is rep not, check this here
4401                  */
4402                 if (info->rep_prefix != REPE_PREFIX)
4403                         goto out;
4404         case SVM_EXIT_IOIO: {
4405                 u64 exit_info;
4406                 u32 bytes;
4407
4408                 if (info->intercept == x86_intercept_in ||
4409                     info->intercept == x86_intercept_ins) {
4410                         exit_info = ((info->src_val & 0xffff) << 16) |
4411                                 SVM_IOIO_TYPE_MASK;
4412                         bytes = info->dst_bytes;
4413                 } else {
4414                         exit_info = (info->dst_val & 0xffff) << 16;
4415                         bytes = info->src_bytes;
4416                 }
4417
4418                 if (info->intercept == x86_intercept_outs ||
4419                     info->intercept == x86_intercept_ins)
4420                         exit_info |= SVM_IOIO_STR_MASK;
4421
4422                 if (info->rep_prefix)
4423                         exit_info |= SVM_IOIO_REP_MASK;
4424
4425                 bytes = min(bytes, 4u);
4426
4427                 exit_info |= bytes << SVM_IOIO_SIZE_SHIFT;
4428
4429                 exit_info |= (u32)info->ad_bytes << (SVM_IOIO_ASIZE_SHIFT - 1);
4430
4431                 vmcb->control.exit_info_1 = exit_info;
4432                 vmcb->control.exit_info_2 = info->next_rip;
4433
4434                 break;
4435         }
4436         default:
4437                 break;
4438         }
4439
4440         /* TODO: Advertise NRIPS to guest hypervisor unconditionally */
4441         if (static_cpu_has(X86_FEATURE_NRIPS))
4442                 vmcb->control.next_rip  = info->next_rip;
4443         vmcb->control.exit_code = icpt_info.exit_code;
4444         vmexit = nested_svm_exit_handled(svm);
4445
4446         ret = (vmexit == NESTED_EXIT_DONE) ? X86EMUL_INTERCEPTED
4447                                            : X86EMUL_CONTINUE;
4448
4449 out:
4450         return ret;
4451 }
4452
4453 static void svm_handle_external_intr(struct kvm_vcpu *vcpu)
4454 {
4455         local_irq_enable();
4456 }
4457
4458 static void svm_sched_in(struct kvm_vcpu *vcpu, int cpu)
4459 {
4460 }
4461
4462 static struct kvm_x86_ops svm_x86_ops = {
4463         .cpu_has_kvm_support = has_svm,
4464         .disabled_by_bios = is_disabled,
4465         .hardware_setup = svm_hardware_setup,
4466         .hardware_unsetup = svm_hardware_unsetup,
4467         .check_processor_compatibility = svm_check_processor_compat,
4468         .hardware_enable = svm_hardware_enable,
4469         .hardware_disable = svm_hardware_disable,
4470         .cpu_has_accelerated_tpr = svm_cpu_has_accelerated_tpr,
4471         .cpu_has_high_real_mode_segbase = svm_has_high_real_mode_segbase,
4472
4473         .vcpu_create = svm_create_vcpu,
4474         .vcpu_free = svm_free_vcpu,
4475         .vcpu_reset = svm_vcpu_reset,
4476
4477         .prepare_guest_switch = svm_prepare_guest_switch,
4478         .vcpu_load = svm_vcpu_load,
4479         .vcpu_put = svm_vcpu_put,
4480
4481         .update_db_bp_intercept = update_db_bp_intercept,
4482         .get_msr = svm_get_msr,
4483         .set_msr = svm_set_msr,
4484         .get_segment_base = svm_get_segment_base,
4485         .get_segment = svm_get_segment,
4486         .set_segment = svm_set_segment,
4487         .get_cpl = svm_get_cpl,
4488         .get_cs_db_l_bits = kvm_get_cs_db_l_bits,
4489         .decache_cr0_guest_bits = svm_decache_cr0_guest_bits,
4490         .decache_cr3 = svm_decache_cr3,
4491         .decache_cr4_guest_bits = svm_decache_cr4_guest_bits,
4492         .set_cr0 = svm_set_cr0,
4493         .set_cr3 = svm_set_cr3,
4494         .set_cr4 = svm_set_cr4,
4495         .set_efer = svm_set_efer,
4496         .get_idt = svm_get_idt,
4497         .set_idt = svm_set_idt,
4498         .get_gdt = svm_get_gdt,
4499         .set_gdt = svm_set_gdt,
4500         .get_dr6 = svm_get_dr6,
4501         .set_dr6 = svm_set_dr6,
4502         .set_dr7 = svm_set_dr7,
4503         .sync_dirty_debug_regs = svm_sync_dirty_debug_regs,
4504         .cache_reg = svm_cache_reg,
4505         .get_rflags = svm_get_rflags,
4506         .set_rflags = svm_set_rflags,
4507         .fpu_activate = svm_fpu_activate,
4508         .fpu_deactivate = svm_fpu_deactivate,
4509
4510         .tlb_flush = svm_flush_tlb,
4511
4512         .run = svm_vcpu_run,
4513         .handle_exit = handle_exit,
4514         .skip_emulated_instruction = skip_emulated_instruction,
4515         .set_interrupt_shadow = svm_set_interrupt_shadow,
4516         .get_interrupt_shadow = svm_get_interrupt_shadow,
4517         .patch_hypercall = svm_patch_hypercall,
4518         .set_irq = svm_set_irq,
4519         .set_nmi = svm_inject_nmi,
4520         .queue_exception = svm_queue_exception,
4521         .cancel_injection = svm_cancel_injection,
4522         .interrupt_allowed = svm_interrupt_allowed,
4523         .nmi_allowed = svm_nmi_allowed,
4524         .get_nmi_mask = svm_get_nmi_mask,
4525         .set_nmi_mask = svm_set_nmi_mask,
4526         .enable_nmi_window = enable_nmi_window,
4527         .enable_irq_window = enable_irq_window,
4528         .update_cr8_intercept = update_cr8_intercept,
4529         .set_virtual_x2apic_mode = svm_set_virtual_x2apic_mode,
4530         .vm_has_apicv = svm_vm_has_apicv,
4531         .load_eoi_exitmap = svm_load_eoi_exitmap,
4532         .sync_pir_to_irr = svm_sync_pir_to_irr,
4533
4534         .set_tss_addr = svm_set_tss_addr,
4535         .get_tdp_level = get_npt_level,
4536         .get_mt_mask = svm_get_mt_mask,
4537
4538         .get_exit_info = svm_get_exit_info,
4539
4540         .get_lpage_level = svm_get_lpage_level,
4541
4542         .cpuid_update = svm_cpuid_update,
4543
4544         .rdtscp_supported = svm_rdtscp_supported,
4545         .invpcid_supported = svm_invpcid_supported,
4546         .mpx_supported = svm_mpx_supported,
4547         .xsaves_supported = svm_xsaves_supported,
4548
4549         .set_supported_cpuid = svm_set_supported_cpuid,
4550
4551         .has_wbinvd_exit = svm_has_wbinvd_exit,
4552
4553         .set_tsc_khz = svm_set_tsc_khz,
4554         .read_tsc_offset = svm_read_tsc_offset,
4555         .write_tsc_offset = svm_write_tsc_offset,
4556         .adjust_tsc_offset = svm_adjust_tsc_offset,
4557         .compute_tsc_offset = svm_compute_tsc_offset,
4558         .read_l1_tsc = svm_read_l1_tsc,
4559
4560         .set_tdp_cr3 = set_tdp_cr3,
4561
4562         .check_intercept = svm_check_intercept,
4563         .handle_external_intr = svm_handle_external_intr,
4564
4565         .sched_in = svm_sched_in,
4566
4567         .pmu_ops = &amd_pmu_ops,
4568 };
4569
4570 static int __init svm_init(void)
4571 {
4572         return kvm_init(&svm_x86_ops, sizeof(struct vcpu_svm),
4573                         __alignof__(struct vcpu_svm), THIS_MODULE);
4574 }
4575
4576 static void __exit svm_exit(void)
4577 {
4578         kvm_exit();
4579 }
4580
4581 module_init(svm_init)
4582 module_exit(svm_exit)