err.no Git - linux-2.6/blob - arch/x86/kvm/x86.c

   1 /*
   2  * Kernel-based Virtual Machine driver for Linux
   3  *
   4  * derived from drivers/kvm/kvm_main.c
   5  *
   6  * Copyright (C) 2006 Qumranet, Inc.
   7  *
   8  * Authors:
   9  *   Avi Kivity   <avi@qumranet.com>
  10  *   Yaniv Kamay  <yaniv@qumranet.com>
  11  *
  12  * This work is licensed under the terms of the GNU GPL, version 2.  See
  13  * the COPYING file in the top-level directory.
  14  *
  15  */
  16
  17 #include <linux/kvm_host.h>
  18 #include "segment_descriptor.h"
  19 #include "irq.h"
  20 #include "mmu.h"
  21
  22 #include <linux/kvm.h>
  23 #include <linux/fs.h>
  24 #include <linux/vmalloc.h>
  25 #include <linux/module.h>
  26 #include <linux/mman.h>
  27 #include <linux/highmem.h>
  28
  29 #include <asm/uaccess.h>
  30 #include <asm/msr.h>
  31
  32 #define MAX_IO_MSRS 256
  33 #define CR0_RESERVED_BITS                                               \
  34         (~(unsigned long)(X86_CR0_PE | X86_CR0_MP | X86_CR0_EM | X86_CR0_TS \
  35                           | X86_CR0_ET | X86_CR0_NE | X86_CR0_WP | X86_CR0_AM \
  36                           | X86_CR0_NW | X86_CR0_CD | X86_CR0_PG))
  37 #define CR4_RESERVED_BITS                                               \
  38         (~(unsigned long)(X86_CR4_VME | X86_CR4_PVI | X86_CR4_TSD | X86_CR4_DE\
  39                           | X86_CR4_PSE | X86_CR4_PAE | X86_CR4_MCE     \
  40                           | X86_CR4_PGE | X86_CR4_PCE | X86_CR4_OSFXSR  \
  41                           | X86_CR4_OSXMMEXCPT | X86_CR4_VMXE))
  42
  43 #define CR8_RESERVED_BITS (~(unsigned long)X86_CR8_TPR)
  44 /* EFER defaults:
  45  * - enable syscall per default because its emulated by KVM
  46  * - enable LME and LMA per default on 64 bit KVM
  47  */
  48 #ifdef CONFIG_X86_64
  49 static u64 __read_mostly efer_reserved_bits = 0xfffffffffffffafeULL;
  50 #else
  51 static u64 __read_mostly efer_reserved_bits = 0xfffffffffffffffeULL;
  52 #endif
  53
  54 #define VM_STAT(x) offsetof(struct kvm, stat.x), KVM_STAT_VM
  55 #define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU
  56
  57 static int kvm_dev_ioctl_get_supported_cpuid(struct kvm_cpuid2 *cpuid,
  58                                     struct kvm_cpuid_entry2 __user *entries);
  59
  60 struct kvm_x86_ops *kvm_x86_ops;
  61
  62 struct kvm_stats_debugfs_item debugfs_entries[] = {
  63         { "pf_fixed", VCPU_STAT(pf_fixed) },
  64         { "pf_guest", VCPU_STAT(pf_guest) },
  65         { "tlb_flush", VCPU_STAT(tlb_flush) },
  66         { "invlpg", VCPU_STAT(invlpg) },
  67         { "exits", VCPU_STAT(exits) },
  68         { "io_exits", VCPU_STAT(io_exits) },
  69         { "mmio_exits", VCPU_STAT(mmio_exits) },
  70         { "signal_exits", VCPU_STAT(signal_exits) },
  71         { "irq_window", VCPU_STAT(irq_window_exits) },
  72         { "halt_exits", VCPU_STAT(halt_exits) },
  73         { "halt_wakeup", VCPU_STAT(halt_wakeup) },
  74         { "request_irq", VCPU_STAT(request_irq_exits) },
  75         { "irq_exits", VCPU_STAT(irq_exits) },
  76         { "host_state_reload", VCPU_STAT(host_state_reload) },
  77         { "efer_reload", VCPU_STAT(efer_reload) },
  78         { "fpu_reload", VCPU_STAT(fpu_reload) },
  79         { "insn_emulation", VCPU_STAT(insn_emulation) },
  80         { "insn_emulation_fail", VCPU_STAT(insn_emulation_fail) },
  81         { "mmu_shadow_zapped", VM_STAT(mmu_shadow_zapped) },
  82         { "mmu_pte_write", VM_STAT(mmu_pte_write) },
  83         { "mmu_pte_updated", VM_STAT(mmu_pte_updated) },
  84         { "mmu_pde_zapped", VM_STAT(mmu_pde_zapped) },
  85         { "mmu_flooded", VM_STAT(mmu_flooded) },
  86         { "mmu_recycled", VM_STAT(mmu_recycled) },
  87         { "mmu_cache_miss", VM_STAT(mmu_cache_miss) },
  88         { "remote_tlb_flush", VM_STAT(remote_tlb_flush) },
  89         { NULL }
  90 };
  91
  92
  93 unsigned long segment_base(u16 selector)
  94 {
  95         struct descriptor_table gdt;
  96         struct segment_descriptor *d;
  97         unsigned long table_base;
  98         unsigned long v;
  99
 100         if (selector == 0)
 101                 return 0;
 102
 103         asm("sgdt %0" : "=m"(gdt));
 104         table_base = gdt.base;
 105
 106         if (selector & 4) {           /* from ldt */
 107                 u16 ldt_selector;
 108
 109                 asm("sldt %0" : "=g"(ldt_selector));
 110                 table_base = segment_base(ldt_selector);
 111         }
 112         d = (struct segment_descriptor *)(table_base + (selector & ~7));
 113         v = d->base_low | ((unsigned long)d->base_mid << 16) |
 114                 ((unsigned long)d->base_high << 24);
 115 #ifdef CONFIG_X86_64
 116         if (d->system == 0 && (d->type == 2 || d->type == 9 || d->type == 11))
 117                 v |= ((unsigned long) \
 118                       ((struct segment_descriptor_64 *)d)->base_higher) << 32;
 119 #endif
 120         return v;
 121 }
 122 EXPORT_SYMBOL_GPL(segment_base);
 123
 124 u64 kvm_get_apic_base(struct kvm_vcpu *vcpu)
 125 {
 126         if (irqchip_in_kernel(vcpu->kvm))
 127                 return vcpu->arch.apic_base;
 128         else
 129                 return vcpu->arch.apic_base;
 130 }
 131 EXPORT_SYMBOL_GPL(kvm_get_apic_base);
 132
 133 void kvm_set_apic_base(struct kvm_vcpu *vcpu, u64 data)
 134 {
 135         /* TODO: reserve bits check */
 136         if (irqchip_in_kernel(vcpu->kvm))
 137                 kvm_lapic_set_base(vcpu, data);
 138         else
 139                 vcpu->arch.apic_base = data;
 140 }
 141 EXPORT_SYMBOL_GPL(kvm_set_apic_base);
 142
 143 void kvm_queue_exception(struct kvm_vcpu *vcpu, unsigned nr)
 144 {
 145         WARN_ON(vcpu->arch.exception.pending);
 146         vcpu->arch.exception.pending = true;
 147         vcpu->arch.exception.has_error_code = false;
 148         vcpu->arch.exception.nr = nr;
 149 }
 150 EXPORT_SYMBOL_GPL(kvm_queue_exception);
 151
 152 void kvm_inject_page_fault(struct kvm_vcpu *vcpu, unsigned long addr,
 153                            u32 error_code)
 154 {
 155         ++vcpu->stat.pf_guest;
 156         if (vcpu->arch.exception.pending && vcpu->arch.exception.nr == PF_VECTOR) {
 157                 printk(KERN_DEBUG "kvm: inject_page_fault:"
 158                        " double fault 0x%lx\n", addr);
 159                 vcpu->arch.exception.nr = DF_VECTOR;
 160                 vcpu->arch.exception.error_code = 0;
 161                 return;
 162         }
 163         vcpu->arch.cr2 = addr;
 164         kvm_queue_exception_e(vcpu, PF_VECTOR, error_code);
 165 }
 166
 167 void kvm_queue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code)
 168 {
 169         WARN_ON(vcpu->arch.exception.pending);
 170         vcpu->arch.exception.pending = true;
 171         vcpu->arch.exception.has_error_code = true;
 172         vcpu->arch.exception.nr = nr;
 173         vcpu->arch.exception.error_code = error_code;
 174 }
 175 EXPORT_SYMBOL_GPL(kvm_queue_exception_e);
 176
 177 static void __queue_exception(struct kvm_vcpu *vcpu)
 178 {
 179         kvm_x86_ops->queue_exception(vcpu, vcpu->arch.exception.nr,
 180                                      vcpu->arch.exception.has_error_code,
 181                                      vcpu->arch.exception.error_code);
 182 }
 183
 184 /*
 185  * Load the pae pdptrs.  Return true is they are all valid.
 186  */
 187 int load_pdptrs(struct kvm_vcpu *vcpu, unsigned long cr3)
 188 {
 189         gfn_t pdpt_gfn = cr3 >> PAGE_SHIFT;
 190         unsigned offset = ((cr3 & (PAGE_SIZE-1)) >> 5) << 2;
 191         int i;
 192         int ret;
 193         u64 pdpte[ARRAY_SIZE(vcpu->arch.pdptrs)];
 194
 195         down_read(&vcpu->kvm->slots_lock);
 196         ret = kvm_read_guest_page(vcpu->kvm, pdpt_gfn, pdpte,
 197                                   offset * sizeof(u64), sizeof(pdpte));
 198         if (ret < 0) {
 199                 ret = 0;
 200                 goto out;
 201         }
 202         for (i = 0; i < ARRAY_SIZE(pdpte); ++i) {
 203                 if ((pdpte[i] & 1) && (pdpte[i] & 0xfffffff0000001e6ull)) {
 204                         ret = 0;
 205                         goto out;
 206                 }
 207         }
 208         ret = 1;
 209
 210         memcpy(vcpu->arch.pdptrs, pdpte, sizeof(vcpu->arch.pdptrs));
 211 out:
 212         up_read(&vcpu->kvm->slots_lock);
 213
 214         return ret;
 215 }
 216
 217 static bool pdptrs_changed(struct kvm_vcpu *vcpu)
 218 {
 219         u64 pdpte[ARRAY_SIZE(vcpu->arch.pdptrs)];
 220         bool changed = true;
 221         int r;
 222
 223         if (is_long_mode(vcpu) || !is_pae(vcpu))
 224                 return false;
 225
 226         down_read(&vcpu->kvm->slots_lock);
 227         r = kvm_read_guest(vcpu->kvm, vcpu->arch.cr3 & ~31u, pdpte, sizeof(pdpte));
 228         if (r < 0)
 229                 goto out;
 230         changed = memcmp(pdpte, vcpu->arch.pdptrs, sizeof(pdpte)) != 0;
 231 out:
 232         up_read(&vcpu->kvm->slots_lock);
 233
 234         return changed;
 235 }
 236
 237 void set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
 238 {
 239         if (cr0 & CR0_RESERVED_BITS) {
 240                 printk(KERN_DEBUG "set_cr0: 0x%lx #GP, reserved bits 0x%lx\n",
 241                        cr0, vcpu->arch.cr0);
 242                 kvm_inject_gp(vcpu, 0);
 243                 return;
 244         }
 245
 246         if ((cr0 & X86_CR0_NW) && !(cr0 & X86_CR0_CD)) {
 247                 printk(KERN_DEBUG "set_cr0: #GP, CD == 0 && NW == 1\n");
 248                 kvm_inject_gp(vcpu, 0);
 249                 return;
 250         }
 251
 252         if ((cr0 & X86_CR0_PG) && !(cr0 & X86_CR0_PE)) {
 253                 printk(KERN_DEBUG "set_cr0: #GP, set PG flag "
 254                        "and a clear PE flag\n");
 255                 kvm_inject_gp(vcpu, 0);
 256                 return;
 257         }
 258
 259         if (!is_paging(vcpu) && (cr0 & X86_CR0_PG)) {
 260 #ifdef CONFIG_X86_64
 261                 if ((vcpu->arch.shadow_efer & EFER_LME)) {
 262                         int cs_db, cs_l;
 263
 264                         if (!is_pae(vcpu)) {
 265                                 printk(KERN_DEBUG "set_cr0: #GP, start paging "
 266                                        "in long mode while PAE is disabled\n");
 267                                 kvm_inject_gp(vcpu, 0);
 268                                 return;
 269                         }
 270                         kvm_x86_ops->get_cs_db_l_bits(vcpu, &cs_db, &cs_l);
 271                         if (cs_l) {
 272                                 printk(KERN_DEBUG "set_cr0: #GP, start paging "
 273                                        "in long mode while CS.L == 1\n");
 274                                 kvm_inject_gp(vcpu, 0);
 275                                 return;
 276
 277                         }
 278                 } else
 279 #endif
 280                 if (is_pae(vcpu) && !load_pdptrs(vcpu, vcpu->arch.cr3)) {
 281                         printk(KERN_DEBUG "set_cr0: #GP, pdptrs "
 282                                "reserved bits\n");
 283                         kvm_inject_gp(vcpu, 0);
 284                         return;
 285                 }
 286
 287         }
 288
 289         kvm_x86_ops->set_cr0(vcpu, cr0);
 290         vcpu->arch.cr0 = cr0;
 291
 292         kvm_mmu_reset_context(vcpu);
 293         return;
 294 }
 295 EXPORT_SYMBOL_GPL(set_cr0);
 296
 297 void lmsw(struct kvm_vcpu *vcpu, unsigned long msw)
 298 {
 299         set_cr0(vcpu, (vcpu->arch.cr0 & ~0x0ful) | (msw & 0x0f));
 300 }
 301 EXPORT_SYMBOL_GPL(lmsw);
 302
 303 void set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
 304 {
 305         if (cr4 & CR4_RESERVED_BITS) {
 306                 printk(KERN_DEBUG "set_cr4: #GP, reserved bits\n");
 307                 kvm_inject_gp(vcpu, 0);
 308                 return;
 309         }
 310
 311         if (is_long_mode(vcpu)) {
 312                 if (!(cr4 & X86_CR4_PAE)) {
 313                         printk(KERN_DEBUG "set_cr4: #GP, clearing PAE while "
 314                                "in long mode\n");
 315                         kvm_inject_gp(vcpu, 0);
 316                         return;
 317                 }
 318         } else if (is_paging(vcpu) && !is_pae(vcpu) && (cr4 & X86_CR4_PAE)
 319                    && !load_pdptrs(vcpu, vcpu->arch.cr3)) {
 320                 printk(KERN_DEBUG "set_cr4: #GP, pdptrs reserved bits\n");
 321                 kvm_inject_gp(vcpu, 0);
 322                 return;
 323         }
 324
 325         if (cr4 & X86_CR4_VMXE) {
 326                 printk(KERN_DEBUG "set_cr4: #GP, setting VMXE\n");
 327                 kvm_inject_gp(vcpu, 0);
 328                 return;
 329         }
 330         kvm_x86_ops->set_cr4(vcpu, cr4);
 331         vcpu->arch.cr4 = cr4;
 332         kvm_mmu_reset_context(vcpu);
 333 }
 334 EXPORT_SYMBOL_GPL(set_cr4);
 335
 336 void set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3)
 337 {
 338         if (cr3 == vcpu->arch.cr3 && !pdptrs_changed(vcpu)) {
 339                 kvm_mmu_flush_tlb(vcpu);
 340                 return;
 341         }
 342
 343         if (is_long_mode(vcpu)) {
 344                 if (cr3 & CR3_L_MODE_RESERVED_BITS) {
 345                         printk(KERN_DEBUG "set_cr3: #GP, reserved bits\n");
 346                         kvm_inject_gp(vcpu, 0);
 347                         return;
 348                 }
 349         } else {
 350                 if (is_pae(vcpu)) {
 351                         if (cr3 & CR3_PAE_RESERVED_BITS) {
 352                                 printk(KERN_DEBUG
 353                                        "set_cr3: #GP, reserved bits\n");
 354                                 kvm_inject_gp(vcpu, 0);
 355                                 return;
 356                         }
 357                         if (is_paging(vcpu) && !load_pdptrs(vcpu, cr3)) {
 358                                 printk(KERN_DEBUG "set_cr3: #GP, pdptrs "
 359                                        "reserved bits\n");
 360                                 kvm_inject_gp(vcpu, 0);
 361                                 return;
 362                         }
 363                 }
 364                 /*
 365                  * We don't check reserved bits in nonpae mode, because
 366                  * this isn't enforced, and VMware depends on this.
 367                  */
 368         }
 369
 370         down_read(&vcpu->kvm->slots_lock);
 371         /*
 372          * Does the new cr3 value map to physical memory? (Note, we
 373          * catch an invalid cr3 even in real-mode, because it would
 374          * cause trouble later on when we turn on paging anyway.)
 375          *
 376          * A real CPU would silently accept an invalid cr3 and would
 377          * attempt to use it - with largely undefined (and often hard
 378          * to debug) behavior on the guest side.
 379          */
 380         if (unlikely(!gfn_to_memslot(vcpu->kvm, cr3 >> PAGE_SHIFT)))
 381                 kvm_inject_gp(vcpu, 0);
 382         else {
 383                 vcpu->arch.cr3 = cr3;
 384                 vcpu->arch.mmu.new_cr3(vcpu);
 385         }
 386         up_read(&vcpu->kvm->slots_lock);
 387 }
 388 EXPORT_SYMBOL_GPL(set_cr3);
 389
 390 void set_cr8(struct kvm_vcpu *vcpu, unsigned long cr8)
 391 {
 392         if (cr8 & CR8_RESERVED_BITS) {
 393                 printk(KERN_DEBUG "set_cr8: #GP, reserved bits 0x%lx\n", cr8);
 394                 kvm_inject_gp(vcpu, 0);
 395                 return;
 396         }
 397         if (irqchip_in_kernel(vcpu->kvm))
 398                 kvm_lapic_set_tpr(vcpu, cr8);
 399         else
 400                 vcpu->arch.cr8 = cr8;
 401 }
 402 EXPORT_SYMBOL_GPL(set_cr8);
 403
 404 unsigned long get_cr8(struct kvm_vcpu *vcpu)
 405 {
 406         if (irqchip_in_kernel(vcpu->kvm))
 407                 return kvm_lapic_get_cr8(vcpu);
 408         else
 409                 return vcpu->arch.cr8;
 410 }
 411 EXPORT_SYMBOL_GPL(get_cr8);
 412
 413 /*
 414  * List of msr numbers which we expose to userspace through KVM_GET_MSRS
 415  * and KVM_SET_MSRS, and KVM_GET_MSR_INDEX_LIST.
 416  *
 417  * This list is modified at module load time to reflect the
 418  * capabilities of the host cpu.
 419  */
 420 static u32 msrs_to_save[] = {
 421         MSR_IA32_SYSENTER_CS, MSR_IA32_SYSENTER_ESP, MSR_IA32_SYSENTER_EIP,
 422         MSR_K6_STAR,
 423 #ifdef CONFIG_X86_64
 424         MSR_CSTAR, MSR_KERNEL_GS_BASE, MSR_SYSCALL_MASK, MSR_LSTAR,
 425 #endif
 426         MSR_IA32_TIME_STAMP_COUNTER,
 427 };
 428
 429 static unsigned num_msrs_to_save;
 430
 431 static u32 emulated_msrs[] = {
 432         MSR_IA32_MISC_ENABLE,
 433 };
 434
 435 static void set_efer(struct kvm_vcpu *vcpu, u64 efer)
 436 {
 437         if (efer & efer_reserved_bits) {
 438                 printk(KERN_DEBUG "set_efer: 0x%llx #GP, reserved bits\n",
 439                        efer);
 440                 kvm_inject_gp(vcpu, 0);
 441                 return;
 442         }
 443
 444         if (is_paging(vcpu)
 445             && (vcpu->arch.shadow_efer & EFER_LME) != (efer & EFER_LME)) {
 446                 printk(KERN_DEBUG "set_efer: #GP, change LME while paging\n");
 447                 kvm_inject_gp(vcpu, 0);
 448                 return;
 449         }
 450
 451         kvm_x86_ops->set_efer(vcpu, efer);
 452
 453         efer &= ~EFER_LMA;
 454         efer |= vcpu->arch.shadow_efer & EFER_LMA;
 455
 456         vcpu->arch.shadow_efer = efer;
 457 }
 458
 459 void kvm_enable_efer_bits(u64 mask)
 460 {
 461        efer_reserved_bits &= ~mask;
 462 }
 463 EXPORT_SYMBOL_GPL(kvm_enable_efer_bits);
 464
 465
 466 /*
 467  * Writes msr value into into the appropriate "register".
 468  * Returns 0 on success, non-0 otherwise.
 469  * Assumes vcpu_load() was already called.
 470  */
 471 int kvm_set_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 data)
 472 {
 473         return kvm_x86_ops->set_msr(vcpu, msr_index, data);
 474 }
 475
 476 /*
 477  * Adapt set_msr() to msr_io()'s calling convention
 478  */
 479 static int do_set_msr(struct kvm_vcpu *vcpu, unsigned index, u64 *data)
 480 {
 481         return kvm_set_msr(vcpu, index, *data);
 482 }
 483
 484
 485 int kvm_set_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 data)
 486 {
 487         switch (msr) {
 488         case MSR_EFER:
 489                 set_efer(vcpu, data);
 490                 break;
 491         case MSR_IA32_MC0_STATUS:
 492                 pr_unimpl(vcpu, "%s: MSR_IA32_MC0_STATUS 0x%llx, nop\n",
 493                        __FUNCTION__, data);
 494                 break;
 495         case MSR_IA32_MCG_STATUS:
 496                 pr_unimpl(vcpu, "%s: MSR_IA32_MCG_STATUS 0x%llx, nop\n",
 497                         __FUNCTION__, data);
 498                 break;
 499         case MSR_IA32_MCG_CTL:
 500                 pr_unimpl(vcpu, "%s: MSR_IA32_MCG_CTL 0x%llx, nop\n",
 501                         __FUNCTION__, data);
 502                 break;
 503         case MSR_IA32_UCODE_REV:
 504         case MSR_IA32_UCODE_WRITE:
 505         case 0x200 ... 0x2ff: /* MTRRs */
 506                 break;
 507         case MSR_IA32_APICBASE:
 508                 kvm_set_apic_base(vcpu, data);
 509                 break;
 510         case MSR_IA32_MISC_ENABLE:
 511                 vcpu->arch.ia32_misc_enable_msr = data;
 512                 break;
 513         default:
 514                 pr_unimpl(vcpu, "unhandled wrmsr: 0x%x data %llx\n", msr, data);
 515                 return 1;
 516         }
 517         return 0;
 518 }
 519 EXPORT_SYMBOL_GPL(kvm_set_msr_common);
 520
 521
 522 /*
 523  * Reads an msr value (of 'msr_index') into 'pdata'.
 524  * Returns 0 on success, non-0 otherwise.
 525  * Assumes vcpu_load() was already called.
 526  */
 527 int kvm_get_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 *pdata)
 528 {
 529         return kvm_x86_ops->get_msr(vcpu, msr_index, pdata);
 530 }
 531
 532 int kvm_get_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata)
 533 {
 534         u64 data;
 535
 536         switch (msr) {
 537         case 0xc0010010: /* SYSCFG */
 538         case 0xc0010015: /* HWCR */
 539         case MSR_IA32_PLATFORM_ID:
 540         case MSR_IA32_P5_MC_ADDR:
 541         case MSR_IA32_P5_MC_TYPE:
 542         case MSR_IA32_MC0_CTL:
 543         case MSR_IA32_MCG_STATUS:
 544         case MSR_IA32_MCG_CAP:
 545         case MSR_IA32_MCG_CTL:
 546         case MSR_IA32_MC0_MISC:
 547         case MSR_IA32_MC0_MISC+4:
 548         case MSR_IA32_MC0_MISC+8:
 549         case MSR_IA32_MC0_MISC+12:
 550         case MSR_IA32_MC0_MISC+16:
 551         case MSR_IA32_UCODE_REV:
 552         case MSR_IA32_PERF_STATUS:
 553         case MSR_IA32_EBL_CR_POWERON:
 554                 /* MTRR registers */
 555         case 0xfe:
 556         case 0x200 ... 0x2ff:
 557                 data = 0;
 558                 break;
 559         case 0xcd: /* fsb frequency */
 560                 data = 3;
 561                 break;
 562         case MSR_IA32_APICBASE:
 563                 data = kvm_get_apic_base(vcpu);
 564                 break;
 565         case MSR_IA32_MISC_ENABLE:
 566                 data = vcpu->arch.ia32_misc_enable_msr;
 567                 break;
 568         case MSR_EFER:
 569                 data = vcpu->arch.shadow_efer;
 570                 break;
 571         default:
 572                 pr_unimpl(vcpu, "unhandled rdmsr: 0x%x\n", msr);
 573                 return 1;
 574         }
 575         *pdata = data;
 576         return 0;
 577 }
 578 EXPORT_SYMBOL_GPL(kvm_get_msr_common);
 579
 580 /*
 581  * Read or write a bunch of msrs. All parameters are kernel addresses.
 582  *
 583  * @return number of msrs set successfully.
 584  */
 585 static int __msr_io(struct kvm_vcpu *vcpu, struct kvm_msrs *msrs,
 586                     struct kvm_msr_entry *entries,
 587                     int (*do_msr)(struct kvm_vcpu *vcpu,
 588                                   unsigned index, u64 *data))
 589 {
 590         int i;
 591
 592         vcpu_load(vcpu);
 593
 594         for (i = 0; i < msrs->nmsrs; ++i)
 595                 if (do_msr(vcpu, entries[i].index, &entries[i].data))
 596                         break;
 597
 598         vcpu_put(vcpu);
 599
 600         return i;
 601 }
 602
 603 /*
 604  * Read or write a bunch of msrs. Parameters are user addresses.
 605  *
 606  * @return number of msrs set successfully.
 607  */
 608 static int msr_io(struct kvm_vcpu *vcpu, struct kvm_msrs __user *user_msrs,
 609                   int (*do_msr)(struct kvm_vcpu *vcpu,
 610                                 unsigned index, u64 *data),
 611                   int writeback)
 612 {
 613         struct kvm_msrs msrs;
 614         struct kvm_msr_entry *entries;
 615         int r, n;
 616         unsigned size;
 617
 618         r = -EFAULT;
 619         if (copy_from_user(&msrs, user_msrs, sizeof msrs))
 620                 goto out;
 621
 622         r = -E2BIG;
 623         if (msrs.nmsrs >= MAX_IO_MSRS)
 624                 goto out;
 625
 626         r = -ENOMEM;
 627         size = sizeof(struct kvm_msr_entry) * msrs.nmsrs;
 628         entries = vmalloc(size);
 629         if (!entries)
 630                 goto out;
 631
 632         r = -EFAULT;
 633         if (copy_from_user(entries, user_msrs->entries, size))
 634                 goto out_free;
 635
 636         r = n = __msr_io(vcpu, &msrs, entries, do_msr);
 637         if (r < 0)
 638                 goto out_free;
 639
 640         r = -EFAULT;
 641         if (writeback && copy_to_user(user_msrs->entries, entries, size))
 642                 goto out_free;
 643
 644         r = n;
 645
 646 out_free:
 647         vfree(entries);
 648 out:
 649         return r;
 650 }
 651
 652 /*
 653  * Make sure that a cpu that is being hot-unplugged does not have any vcpus
 654  * cached on it.
 655  */
 656 void decache_vcpus_on_cpu(int cpu)
 657 {
 658         struct kvm *vm;
 659         struct kvm_vcpu *vcpu;
 660         int i;
 661
 662         spin_lock(&kvm_lock);
 663         list_for_each_entry(vm, &vm_list, vm_list)
 664                 for (i = 0; i < KVM_MAX_VCPUS; ++i) {
 665                         vcpu = vm->vcpus[i];
 666                         if (!vcpu)
 667                                 continue;
 668                         /*
 669                          * If the vcpu is locked, then it is running on some
 670                          * other cpu and therefore it is not cached on the
 671                          * cpu in question.
 672                          *
 673                          * If it's not locked, check the last cpu it executed
 674                          * on.
 675                          */
 676                         if (mutex_trylock(&vcpu->mutex)) {
 677                                 if (vcpu->cpu == cpu) {
 678                                         kvm_x86_ops->vcpu_decache(vcpu);
 679                                         vcpu->cpu = -1;
 680                                 }
 681                                 mutex_unlock(&vcpu->mutex);
 682                         }
 683                 }
 684         spin_unlock(&kvm_lock);
 685 }
 686
 687 int kvm_dev_ioctl_check_extension(long ext)
 688 {
 689         int r;
 690
 691         switch (ext) {
 692         case KVM_CAP_IRQCHIP:
 693         case KVM_CAP_HLT:
 694         case KVM_CAP_MMU_SHADOW_CACHE_CONTROL:
 695         case KVM_CAP_USER_MEMORY:
 696         case KVM_CAP_SET_TSS_ADDR:
 697         case KVM_CAP_EXT_CPUID:
 698                 r = 1;
 699                 break;
 700         case KVM_CAP_VAPIC:
 701                 r = !kvm_x86_ops->cpu_has_accelerated_tpr();
 702                 break;
 703         default:
 704                 r = 0;
 705                 break;
 706         }
 707         return r;
 708
 709 }
 710
 711 long kvm_arch_dev_ioctl(struct file *filp,
 712                         unsigned int ioctl, unsigned long arg)
 713 {
 714         void __user *argp = (void __user *)arg;
 715         long r;
 716
 717         switch (ioctl) {
 718         case KVM_GET_MSR_INDEX_LIST: {
 719                 struct kvm_msr_list __user *user_msr_list = argp;
 720                 struct kvm_msr_list msr_list;
 721                 unsigned n;
 722
 723                 r = -EFAULT;
 724                 if (copy_from_user(&msr_list, user_msr_list, sizeof msr_list))
 725                         goto out;
 726                 n = msr_list.nmsrs;
 727                 msr_list.nmsrs = num_msrs_to_save + ARRAY_SIZE(emulated_msrs);
 728                 if (copy_to_user(user_msr_list, &msr_list, sizeof msr_list))
 729                         goto out;
 730                 r = -E2BIG;
 731                 if (n < num_msrs_to_save)
 732                         goto out;
 733                 r = -EFAULT;
 734                 if (copy_to_user(user_msr_list->indices, &msrs_to_save,
 735                                  num_msrs_to_save * sizeof(u32)))
 736                         goto out;
 737                 if (copy_to_user(user_msr_list->indices
 738                                  + num_msrs_to_save * sizeof(u32),
 739                                  &emulated_msrs,
 740                                  ARRAY_SIZE(emulated_msrs) * sizeof(u32)))
 741                         goto out;
 742                 r = 0;
 743                 break;
 744         }
 745         case KVM_GET_SUPPORTED_CPUID: {
 746                 struct kvm_cpuid2 __user *cpuid_arg = argp;
 747                 struct kvm_cpuid2 cpuid;
 748
 749                 r = -EFAULT;
 750                 if (copy_from_user(&cpuid, cpuid_arg, sizeof cpuid))
 751                         goto out;
 752                 r = kvm_dev_ioctl_get_supported_cpuid(&cpuid,
 753                         cpuid_arg->entries);
 754                 if (r)
 755                         goto out;
 756
 757                 r = -EFAULT;
 758                 if (copy_to_user(cpuid_arg, &cpuid, sizeof cpuid))
 759                         goto out;
 760                 r = 0;
 761                 break;
 762         }
 763         default:
 764                 r = -EINVAL;
 765         }
 766 out:
 767         return r;
 768 }
 769
 770 void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
 771 {
 772         kvm_x86_ops->vcpu_load(vcpu, cpu);
 773 }
 774
 775 void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
 776 {
 777         kvm_x86_ops->vcpu_put(vcpu);
 778         kvm_put_guest_fpu(vcpu);
 779 }
 780
 781 static int is_efer_nx(void)
 782 {
 783         u64 efer;
 784
 785         rdmsrl(MSR_EFER, efer);
 786         return efer & EFER_NX;
 787 }
 788
 789 static void cpuid_fix_nx_cap(struct kvm_vcpu *vcpu)
 790 {
 791         int i;
 792         struct kvm_cpuid_entry2 *e, *entry;
 793
 794         entry = NULL;
 795         for (i = 0; i < vcpu->arch.cpuid_nent; ++i) {
 796                 e = &vcpu->arch.cpuid_entries[i];
 797                 if (e->function == 0x80000001) {
 798                         entry = e;
 799                         break;
 800                 }
 801         }
 802         if (entry && (entry->edx & (1 << 20)) && !is_efer_nx()) {
 803                 entry->edx &= ~(1 << 20);
 804                 printk(KERN_INFO "kvm: guest NX capability removed\n");
 805         }
 806 }
 807
 808 /* when an old userspace process fills a new kernel module */
 809 static int kvm_vcpu_ioctl_set_cpuid(struct kvm_vcpu *vcpu,
 810                                     struct kvm_cpuid *cpuid,
 811                                     struct kvm_cpuid_entry __user *entries)
 812 {
 813         int r, i;
 814         struct kvm_cpuid_entry *cpuid_entries;
 815
 816         r = -E2BIG;
 817         if (cpuid->nent > KVM_MAX_CPUID_ENTRIES)
 818                 goto out;
 819         r = -ENOMEM;
 820         cpuid_entries = vmalloc(sizeof(struct kvm_cpuid_entry) * cpuid->nent);
 821         if (!cpuid_entries)
 822                 goto out;
 823         r = -EFAULT;
 824         if (copy_from_user(cpuid_entries, entries,
 825                            cpuid->nent * sizeof(struct kvm_cpuid_entry)))
 826                 goto out_free;
 827         for (i = 0; i < cpuid->nent; i++) {
 828                 vcpu->arch.cpuid_entries[i].function = cpuid_entries[i].function;
 829                 vcpu->arch.cpuid_entries[i].eax = cpuid_entries[i].eax;
 830                 vcpu->arch.cpuid_entries[i].ebx = cpuid_entries[i].ebx;
 831                 vcpu->arch.cpuid_entries[i].ecx = cpuid_entries[i].ecx;
 832                 vcpu->arch.cpuid_entries[i].edx = cpuid_entries[i].edx;
 833                 vcpu->arch.cpuid_entries[i].index = 0;
 834                 vcpu->arch.cpuid_entries[i].flags = 0;
 835                 vcpu->arch.cpuid_entries[i].padding[0] = 0;
 836                 vcpu->arch.cpuid_entries[i].padding[1] = 0;
 837                 vcpu->arch.cpuid_entries[i].padding[2] = 0;
 838         }
 839         vcpu->arch.cpuid_nent = cpuid->nent;
 840         cpuid_fix_nx_cap(vcpu);
 841         r = 0;
 842
 843 out_free:
 844         vfree(cpuid_entries);
 845 out:
 846         return r;
 847 }
 848
 849 static int kvm_vcpu_ioctl_set_cpuid2(struct kvm_vcpu *vcpu,
 850                                     struct kvm_cpuid2 *cpuid,
 851                                     struct kvm_cpuid_entry2 __user *entries)
 852 {
 853         int r;
 854
 855         r = -E2BIG;
 856         if (cpuid->nent > KVM_MAX_CPUID_ENTRIES)
 857                 goto out;
 858         r = -EFAULT;
 859         if (copy_from_user(&vcpu->arch.cpuid_entries, entries,
 860                            cpuid->nent * sizeof(struct kvm_cpuid_entry2)))
 861                 goto out;
 862         vcpu->arch.cpuid_nent = cpuid->nent;
 863         return 0;
 864
 865 out:
 866         return r;
 867 }
 868
 869 static int kvm_vcpu_ioctl_get_cpuid2(struct kvm_vcpu *vcpu,
 870                                     struct kvm_cpuid2 *cpuid,
 871                                     struct kvm_cpuid_entry2 __user *entries)
 872 {
 873         int r;
 874
 875         r = -E2BIG;
 876         if (cpuid->nent < vcpu->arch.cpuid_nent)
 877                 goto out;
 878         r = -EFAULT;
 879         if (copy_to_user(entries, &vcpu->arch.cpuid_entries,
 880                            vcpu->arch.cpuid_nent * sizeof(struct kvm_cpuid_entry2)))
 881                 goto out;
 882         return 0;
 883
 884 out:
 885         cpuid->nent = vcpu->arch.cpuid_nent;
 886         return r;
 887 }
 888
 889 static inline u32 bit(int bitno)
 890 {
 891         return 1 << (bitno & 31);
 892 }
 893
 894 static void do_cpuid_1_ent(struct kvm_cpuid_entry2 *entry, u32 function,
 895                           u32 index)
 896 {
 897         entry->function = function;
 898         entry->index = index;
 899         cpuid_count(entry->function, entry->index,
 900                 &entry->eax, &entry->ebx, &entry->ecx, &entry->edx);
 901         entry->flags = 0;
 902 }
 903
 904 static void do_cpuid_ent(struct kvm_cpuid_entry2 *entry, u32 function,
 905                          u32 index, int *nent, int maxnent)
 906 {
 907         const u32 kvm_supported_word0_x86_features = bit(X86_FEATURE_FPU) |
 908                 bit(X86_FEATURE_VME) | bit(X86_FEATURE_DE) |
 909                 bit(X86_FEATURE_PSE) | bit(X86_FEATURE_TSC) |
 910                 bit(X86_FEATURE_MSR) | bit(X86_FEATURE_PAE) |
 911                 bit(X86_FEATURE_CX8) | bit(X86_FEATURE_APIC) |
 912                 bit(X86_FEATURE_SEP) | bit(X86_FEATURE_PGE) |
 913                 bit(X86_FEATURE_CMOV) | bit(X86_FEATURE_PSE36) |
 914                 bit(X86_FEATURE_CLFLSH) | bit(X86_FEATURE_MMX) |
 915                 bit(X86_FEATURE_FXSR) | bit(X86_FEATURE_XMM) |
 916                 bit(X86_FEATURE_XMM2) | bit(X86_FEATURE_SELFSNOOP);
 917         const u32 kvm_supported_word1_x86_features = bit(X86_FEATURE_FPU) |
 918                 bit(X86_FEATURE_VME) | bit(X86_FEATURE_DE) |
 919                 bit(X86_FEATURE_PSE) | bit(X86_FEATURE_TSC) |
 920                 bit(X86_FEATURE_MSR) | bit(X86_FEATURE_PAE) |
 921                 bit(X86_FEATURE_CX8) | bit(X86_FEATURE_APIC) |
 922                 bit(X86_FEATURE_PGE) |
 923                 bit(X86_FEATURE_CMOV) | bit(X86_FEATURE_PSE36) |
 924                 bit(X86_FEATURE_MMX) | bit(X86_FEATURE_FXSR) |
 925                 bit(X86_FEATURE_SYSCALL) |
 926                 (bit(X86_FEATURE_NX) && is_efer_nx()) |
 927 #ifdef CONFIG_X86_64
 928                 bit(X86_FEATURE_LM) |
 929 #endif
 930                 bit(X86_FEATURE_MMXEXT) |
 931                 bit(X86_FEATURE_3DNOWEXT) |
 932                 bit(X86_FEATURE_3DNOW);
 933         const u32 kvm_supported_word3_x86_features =
 934                 bit(X86_FEATURE_XMM3) | bit(X86_FEATURE_CX16);
 935         const u32 kvm_supported_word6_x86_features =
 936                 bit(X86_FEATURE_LAHF_LM) | bit(X86_FEATURE_CMP_LEGACY);
 937
 938         /* all func 2 cpuid_count() should be called on the same cpu */
 939         get_cpu();
 940         do_cpuid_1_ent(entry, function, index);
 941         ++*nent;
 942
 943         switch (function) {
 944         case 0:
 945                 entry->eax = min(entry->eax, (u32)0xb);
 946                 break;
 947         case 1:
 948                 entry->edx &= kvm_supported_word0_x86_features;
 949                 entry->ecx &= kvm_supported_word3_x86_features;
 950                 break;
 951         /* function 2 entries are STATEFUL. That is, repeated cpuid commands
 952          * may return different values. This forces us to get_cpu() before
 953          * issuing the first command, and also to emulate this annoying behavior
 954          * in kvm_emulate_cpuid() using KVM_CPUID_FLAG_STATE_READ_NEXT */
 955         case 2: {
 956                 int t, times = entry->eax & 0xff;
 957
 958                 entry->flags |= KVM_CPUID_FLAG_STATEFUL_FUNC;
 959                 for (t = 1; t < times && *nent < maxnent; ++t) {
 960                         do_cpuid_1_ent(&entry[t], function, 0);
 961                         entry[t].flags |= KVM_CPUID_FLAG_STATEFUL_FUNC;
 962                         ++*nent;
 963                 }
 964                 break;
 965         }
 966         /* function 4 and 0xb have additional index. */
 967         case 4: {
 968                 int index, cache_type;
 969
 970                 entry->flags |= KVM_CPUID_FLAG_SIGNIFCANT_INDEX;
 971                 /* read more entries until cache_type is zero */
 972                 for (index = 1; *nent < maxnent; ++index) {
 973                         cache_type = entry[index - 1].eax & 0x1f;
 974                         if (!cache_type)
 975                                 break;
 976                         do_cpuid_1_ent(&entry[index], function, index);
 977                         entry[index].flags |=
 978                                KVM_CPUID_FLAG_SIGNIFCANT_INDEX;
 979                         ++*nent;
 980                 }
 981                 break;
 982         }
 983         case 0xb: {
 984                 int index, level_type;
 985
 986                 entry->flags |= KVM_CPUID_FLAG_SIGNIFCANT_INDEX;
 987                 /* read more entries until level_type is zero */
 988                 for (index = 1; *nent < maxnent; ++index) {
 989                         level_type = entry[index - 1].ecx & 0xff;
 990                         if (!level_type)
 991                                 break;
 992                         do_cpuid_1_ent(&entry[index], function, index);
 993                         entry[index].flags |=
 994                                KVM_CPUID_FLAG_SIGNIFCANT_INDEX;
 995                         ++*nent;
 996                 }
 997                 break;
 998         }
 999         case 0x80000000:
1000                 entry->eax = min(entry->eax, 0x8000001a);
1001                 break;
1002         case 0x80000001:
1003                 entry->edx &= kvm_supported_word1_x86_features;
1004                 entry->ecx &= kvm_supported_word6_x86_features;
1005                 break;
1006         }
1007         put_cpu();
1008 }
1009
1010 static int kvm_dev_ioctl_get_supported_cpuid(struct kvm_cpuid2 *cpuid,
1011                                     struct kvm_cpuid_entry2 __user *entries)
1012 {
1013         struct kvm_cpuid_entry2 *cpuid_entries;
1014         int limit, nent = 0, r = -E2BIG;
1015         u32 func;
1016
1017         if (cpuid->nent < 1)
1018                 goto out;
1019         r = -ENOMEM;
1020         cpuid_entries = vmalloc(sizeof(struct kvm_cpuid_entry2) * cpuid->nent);
1021         if (!cpuid_entries)
1022                 goto out;
1023
1024         do_cpuid_ent(&cpuid_entries[0], 0, 0, &nent, cpuid->nent);
1025         limit = cpuid_entries[0].eax;
1026         for (func = 1; func <= limit && nent < cpuid->nent; ++func)
1027                 do_cpuid_ent(&cpuid_entries[nent], func, 0,
1028                                 &nent, cpuid->nent);
1029         r = -E2BIG;
1030         if (nent >= cpuid->nent)
1031                 goto out_free;
1032
1033         do_cpuid_ent(&cpuid_entries[nent], 0x80000000, 0, &nent, cpuid->nent);
1034         limit = cpuid_entries[nent - 1].eax;
1035         for (func = 0x80000001; func <= limit && nent < cpuid->nent; ++func)
1036                 do_cpuid_ent(&cpuid_entries[nent], func, 0,
1037                                &nent, cpuid->nent);
1038         r = -EFAULT;
1039         if (copy_to_user(entries, cpuid_entries,
1040                         nent * sizeof(struct kvm_cpuid_entry2)))
1041                 goto out_free;
1042         cpuid->nent = nent;
1043         r = 0;
1044
1045 out_free:
1046         vfree(cpuid_entries);
1047 out:
1048         return r;
1049 }
1050
1051 static int kvm_vcpu_ioctl_get_lapic(struct kvm_vcpu *vcpu,
1052                                     struct kvm_lapic_state *s)
1053 {
1054         vcpu_load(vcpu);
1055         memcpy(s->regs, vcpu->arch.apic->regs, sizeof *s);
1056         vcpu_put(vcpu);
1057
1058         return 0;
1059 }
1060
1061 static int kvm_vcpu_ioctl_set_lapic(struct kvm_vcpu *vcpu,
1062                                     struct kvm_lapic_state *s)
1063 {
1064         vcpu_load(vcpu);
1065         memcpy(vcpu->arch.apic->regs, s->regs, sizeof *s);
1066         kvm_apic_post_state_restore(vcpu);
1067         vcpu_put(vcpu);
1068
1069         return 0;
1070 }
1071
1072 static int kvm_vcpu_ioctl_interrupt(struct kvm_vcpu *vcpu,
1073                                     struct kvm_interrupt *irq)
1074 {
1075         if (irq->irq < 0 || irq->irq >= 256)
1076                 return -EINVAL;
1077         if (irqchip_in_kernel(vcpu->kvm))
1078                 return -ENXIO;
1079         vcpu_load(vcpu);
1080
1081         set_bit(irq->irq, vcpu->arch.irq_pending);
1082         set_bit(irq->irq / BITS_PER_LONG, &vcpu->arch.irq_summary);
1083
1084         vcpu_put(vcpu);
1085
1086         return 0;
1087 }
1088
1089 static int vcpu_ioctl_tpr_access_reporting(struct kvm_vcpu *vcpu,
1090                                            struct kvm_tpr_access_ctl *tac)
1091 {
1092         if (tac->flags)
1093                 return -EINVAL;
1094         vcpu->arch.tpr_access_reporting = !!tac->enabled;
1095         return 0;
1096 }
1097
1098 long kvm_arch_vcpu_ioctl(struct file *filp,
1099                          unsigned int ioctl, unsigned long arg)
1100 {
1101         struct kvm_vcpu *vcpu = filp->private_data;
1102         void __user *argp = (void __user *)arg;
1103         int r;
1104
1105         switch (ioctl) {
1106         case KVM_GET_LAPIC: {
1107                 struct kvm_lapic_state lapic;
1108
1109                 memset(&lapic, 0, sizeof lapic);
1110                 r = kvm_vcpu_ioctl_get_lapic(vcpu, &lapic);
1111                 if (r)
1112                         goto out;
1113                 r = -EFAULT;
1114                 if (copy_to_user(argp, &lapic, sizeof lapic))
1115                         goto out;
1116                 r = 0;
1117                 break;
1118         }
1119         case KVM_SET_LAPIC: {
1120                 struct kvm_lapic_state lapic;
1121
1122                 r = -EFAULT;
1123                 if (copy_from_user(&lapic, argp, sizeof lapic))
1124                         goto out;
1125                 r = kvm_vcpu_ioctl_set_lapic(vcpu, &lapic);;
1126                 if (r)
1127                         goto out;
1128                 r = 0;
1129                 break;
1130         }
1131         case KVM_INTERRUPT: {
1132                 struct kvm_interrupt irq;
1133
1134                 r = -EFAULT;
1135                 if (copy_from_user(&irq, argp, sizeof irq))
1136                         goto out;
1137                 r = kvm_vcpu_ioctl_interrupt(vcpu, &irq);
1138                 if (r)
1139                         goto out;
1140                 r = 0;
1141                 break;
1142         }
1143         case KVM_SET_CPUID: {
1144                 struct kvm_cpuid __user *cpuid_arg = argp;
1145                 struct kvm_cpuid cpuid;
1146
1147                 r = -EFAULT;
1148                 if (copy_from_user(&cpuid, cpuid_arg, sizeof cpuid))
1149                         goto out;
1150                 r = kvm_vcpu_ioctl_set_cpuid(vcpu, &cpuid, cpuid_arg->entries);
1151                 if (r)
1152                         goto out;
1153                 break;
1154         }
1155         case KVM_SET_CPUID2: {
1156                 struct kvm_cpuid2 __user *cpuid_arg = argp;
1157                 struct kvm_cpuid2 cpuid;
1158
1159                 r = -EFAULT;
1160                 if (copy_from_user(&cpuid, cpuid_arg, sizeof cpuid))
1161                         goto out;
1162                 r = kvm_vcpu_ioctl_set_cpuid2(vcpu, &cpuid,
1163                                 cpuid_arg->entries);
1164                 if (r)
1165                         goto out;
1166                 break;
1167         }
1168         case KVM_GET_CPUID2: {
1169                 struct kvm_cpuid2 __user *cpuid_arg = argp;
1170                 struct kvm_cpuid2 cpuid;
1171
1172                 r = -EFAULT;
1173                 if (copy_from_user(&cpuid, cpuid_arg, sizeof cpuid))
1174                         goto out;
1175                 r = kvm_vcpu_ioctl_get_cpuid2(vcpu, &cpuid,
1176                                 cpuid_arg->entries);
1177                 if (r)
1178                         goto out;
1179                 r = -EFAULT;
1180                 if (copy_to_user(cpuid_arg, &cpuid, sizeof cpuid))
1181                         goto out;
1182                 r = 0;
1183                 break;
1184         }
1185         case KVM_GET_MSRS:
1186                 r = msr_io(vcpu, argp, kvm_get_msr, 1);
1187                 break;
1188         case KVM_SET_MSRS:
1189                 r = msr_io(vcpu, argp, do_set_msr, 0);
1190                 break;
1191         case KVM_TPR_ACCESS_REPORTING: {
1192                 struct kvm_tpr_access_ctl tac;
1193
1194                 r = -EFAULT;
1195                 if (copy_from_user(&tac, argp, sizeof tac))
1196                         goto out;
1197                 r = vcpu_ioctl_tpr_access_reporting(vcpu, &tac);
1198                 if (r)
1199                         goto out;
1200                 r = -EFAULT;
1201                 if (copy_to_user(argp, &tac, sizeof tac))
1202                         goto out;
1203                 r = 0;
1204                 break;
1205         };
1206         case KVM_SET_VAPIC_ADDR: {
1207                 struct kvm_vapic_addr va;
1208
1209                 r = -EINVAL;
1210                 if (!irqchip_in_kernel(vcpu->kvm))
1211                         goto out;
1212                 r = -EFAULT;
1213                 if (copy_from_user(&va, argp, sizeof va))
1214                         goto out;
1215                 r = 0;
1216                 kvm_lapic_set_vapic_addr(vcpu, va.vapic_addr);
1217                 break;
1218         }
1219         default:
1220                 r = -EINVAL;
1221         }
1222 out:
1223         return r;
1224 }
1225
1226 static int kvm_vm_ioctl_set_tss_addr(struct kvm *kvm, unsigned long addr)
1227 {
1228         int ret;
1229
1230         if (addr > (unsigned int)(-3 * PAGE_SIZE))
1231                 return -1;
1232         ret = kvm_x86_ops->set_tss_addr(kvm, addr);
1233         return ret;
1234 }
1235
1236 static int kvm_vm_ioctl_set_nr_mmu_pages(struct kvm *kvm,
1237                                           u32 kvm_nr_mmu_pages)
1238 {
1239         if (kvm_nr_mmu_pages < KVM_MIN_ALLOC_MMU_PAGES)
1240                 return -EINVAL;
1241
1242         down_write(&kvm->slots_lock);
1243
1244         kvm_mmu_change_mmu_pages(kvm, kvm_nr_mmu_pages);
1245         kvm->arch.n_requested_mmu_pages = kvm_nr_mmu_pages;
1246
1247         up_write(&kvm->slots_lock);
1248         return 0;
1249 }
1250
1251 static int kvm_vm_ioctl_get_nr_mmu_pages(struct kvm *kvm)
1252 {
1253         return kvm->arch.n_alloc_mmu_pages;
1254 }
1255
1256 gfn_t unalias_gfn(struct kvm *kvm, gfn_t gfn)
1257 {
1258         int i;
1259         struct kvm_mem_alias *alias;
1260
1261         for (i = 0; i < kvm->arch.naliases; ++i) {
1262                 alias = &kvm->arch.aliases[i];
1263                 if (gfn >= alias->base_gfn
1264                     && gfn < alias->base_gfn + alias->npages)
1265                         return alias->target_gfn + gfn - alias->base_gfn;
1266         }
1267         return gfn;
1268 }
1269
1270 /*
1271  * Set a new alias region.  Aliases map a portion of physical memory into
1272  * another portion.  This is useful for memory windows, for example the PC
1273  * VGA region.
1274  */
1275 static int kvm_vm_ioctl_set_memory_alias(struct kvm *kvm,
1276                                          struct kvm_memory_alias *alias)
1277 {
1278         int r, n;
1279         struct kvm_mem_alias *p;
1280
1281         r = -EINVAL;
1282         /* General sanity checks */
1283         if (alias->memory_size & (PAGE_SIZE - 1))
1284                 goto out;
1285         if (alias->guest_phys_addr & (PAGE_SIZE - 1))
1286                 goto out;
1287         if (alias->slot >= KVM_ALIAS_SLOTS)
1288                 goto out;
1289         if (alias->guest_phys_addr + alias->memory_size
1290             < alias->guest_phys_addr)
1291                 goto out;
1292         if (alias->target_phys_addr + alias->memory_size
1293             < alias->target_phys_addr)
1294                 goto out;
1295
1296         down_write(&kvm->slots_lock);
1297
1298         p = &kvm->arch.aliases[alias->slot];
1299         p->base_gfn = alias->guest_phys_addr >> PAGE_SHIFT;
1300         p->npages = alias->memory_size >> PAGE_SHIFT;
1301         p->target_gfn = alias->target_phys_addr >> PAGE_SHIFT;
1302
1303         for (n = KVM_ALIAS_SLOTS; n > 0; --n)
1304                 if (kvm->arch.aliases[n - 1].npages)
1305                         break;
1306         kvm->arch.naliases = n;
1307
1308         kvm_mmu_zap_all(kvm);
1309
1310         up_write(&kvm->slots_lock);
1311
1312         return 0;
1313
1314 out:
1315         return r;
1316 }
1317
1318 static int kvm_vm_ioctl_get_irqchip(struct kvm *kvm, struct kvm_irqchip *chip)
1319 {
1320         int r;
1321
1322         r = 0;
1323         switch (chip->chip_id) {
1324         case KVM_IRQCHIP_PIC_MASTER:
1325                 memcpy(&chip->chip.pic,
1326                         &pic_irqchip(kvm)->pics[0],
1327                         sizeof(struct kvm_pic_state));
1328                 break;
1329         case KVM_IRQCHIP_PIC_SLAVE:
1330                 memcpy(&chip->chip.pic,
1331                         &pic_irqchip(kvm)->pics[1],
1332                         sizeof(struct kvm_pic_state));
1333                 break;
1334         case KVM_IRQCHIP_IOAPIC:
1335                 memcpy(&chip->chip.ioapic,
1336                         ioapic_irqchip(kvm),
1337                         sizeof(struct kvm_ioapic_state));
1338                 break;
1339         default:
1340                 r = -EINVAL;
1341                 break;
1342         }
1343         return r;
1344 }
1345
1346 static int kvm_vm_ioctl_set_irqchip(struct kvm *kvm, struct kvm_irqchip *chip)
1347 {
1348         int r;
1349
1350         r = 0;
1351         switch (chip->chip_id) {
1352         case KVM_IRQCHIP_PIC_MASTER:
1353                 memcpy(&pic_irqchip(kvm)->pics[0],
1354                         &chip->chip.pic,
1355                         sizeof(struct kvm_pic_state));
1356                 break;
1357         case KVM_IRQCHIP_PIC_SLAVE:
1358                 memcpy(&pic_irqchip(kvm)->pics[1],
1359                         &chip->chip.pic,
1360                         sizeof(struct kvm_pic_state));
1361                 break;
1362         case KVM_IRQCHIP_IOAPIC:
1363                 memcpy(ioapic_irqchip(kvm),
1364                         &chip->chip.ioapic,
1365                         sizeof(struct kvm_ioapic_state));
1366                 break;
1367         default:
1368                 r = -EINVAL;
1369                 break;
1370         }
1371         kvm_pic_update_irq(pic_irqchip(kvm));
1372         return r;
1373 }
1374
1375 /*
1376  * Get (and clear) the dirty memory log for a memory slot.
1377  */
1378 int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
1379                                       struct kvm_dirty_log *log)
1380 {
1381         int r;
1382         int n;
1383         struct kvm_memory_slot *memslot;
1384         int is_dirty = 0;
1385
1386         down_write(&kvm->slots_lock);
1387
1388         r = kvm_get_dirty_log(kvm, log, &is_dirty);
1389         if (r)
1390                 goto out;
1391
1392         /* If nothing is dirty, don't bother messing with page tables. */
1393         if (is_dirty) {
1394                 kvm_mmu_slot_remove_write_access(kvm, log->slot);
1395                 kvm_flush_remote_tlbs(kvm);
1396                 memslot = &kvm->memslots[log->slot];
1397                 n = ALIGN(memslot->npages, BITS_PER_LONG) / 8;
1398                 memset(memslot->dirty_bitmap, 0, n);
1399         }
1400         r = 0;
1401 out:
1402         up_write(&kvm->slots_lock);
1403         return r;
1404 }
1405
1406 long kvm_arch_vm_ioctl(struct file *filp,
1407                        unsigned int ioctl, unsigned long arg)
1408 {
1409         struct kvm *kvm = filp->private_data;
1410         void __user *argp = (void __user *)arg;
1411         int r = -EINVAL;
1412
1413         switch (ioctl) {
1414         case KVM_SET_TSS_ADDR:
1415                 r = kvm_vm_ioctl_set_tss_addr(kvm, arg);
1416                 if (r < 0)
1417                         goto out;
1418                 break;
1419         case KVM_SET_MEMORY_REGION: {
1420                 struct kvm_memory_region kvm_mem;
1421                 struct kvm_userspace_memory_region kvm_userspace_mem;
1422
1423                 r = -EFAULT;
1424                 if (copy_from_user(&kvm_mem, argp, sizeof kvm_mem))
1425                         goto out;
1426                 kvm_userspace_mem.slot = kvm_mem.slot;
1427                 kvm_userspace_mem.flags = kvm_mem.flags;
1428                 kvm_userspace_mem.guest_phys_addr = kvm_mem.guest_phys_addr;
1429                 kvm_userspace_mem.memory_size = kvm_mem.memory_size;
1430                 r = kvm_vm_ioctl_set_memory_region(kvm, &kvm_userspace_mem, 0);
1431                 if (r)
1432                         goto out;
1433                 break;
1434         }
1435         case KVM_SET_NR_MMU_PAGES:
1436                 r = kvm_vm_ioctl_set_nr_mmu_pages(kvm, arg);
1437                 if (r)
1438                         goto out;
1439                 break;
1440         case KVM_GET_NR_MMU_PAGES:
1441                 r = kvm_vm_ioctl_get_nr_mmu_pages(kvm);
1442                 break;
1443         case KVM_SET_MEMORY_ALIAS: {
1444                 struct kvm_memory_alias alias;
1445
1446                 r = -EFAULT;
1447                 if (copy_from_user(&alias, argp, sizeof alias))
1448                         goto out;
1449                 r = kvm_vm_ioctl_set_memory_alias(kvm, &alias);
1450                 if (r)
1451                         goto out;
1452                 break;
1453         }
1454         case KVM_CREATE_IRQCHIP:
1455                 r = -ENOMEM;
1456                 kvm->arch.vpic = kvm_create_pic(kvm);
1457                 if (kvm->arch.vpic) {
1458                         r = kvm_ioapic_init(kvm);
1459                         if (r) {
1460                                 kfree(kvm->arch.vpic);
1461                                 kvm->arch.vpic = NULL;
1462                                 goto out;
1463                         }
1464                 } else
1465                         goto out;
1466                 break;
1467         case KVM_IRQ_LINE: {
1468                 struct kvm_irq_level irq_event;
1469
1470                 r = -EFAULT;
1471                 if (copy_from_user(&irq_event, argp, sizeof irq_event))
1472                         goto out;
1473                 if (irqchip_in_kernel(kvm)) {
1474                         mutex_lock(&kvm->lock);
1475                         if (irq_event.irq < 16)
1476                                 kvm_pic_set_irq(pic_irqchip(kvm),
1477                                         irq_event.irq,
1478                                         irq_event.level);
1479                         kvm_ioapic_set_irq(kvm->arch.vioapic,
1480                                         irq_event.irq,
1481                                         irq_event.level);
1482                         mutex_unlock(&kvm->lock);
1483                         r = 0;
1484                 }
1485                 break;
1486         }
1487         case KVM_GET_IRQCHIP: {
1488                 /* 0: PIC master, 1: PIC slave, 2: IOAPIC */
1489                 struct kvm_irqchip chip;
1490
1491                 r = -EFAULT;
1492                 if (copy_from_user(&chip, argp, sizeof chip))
1493                         goto out;
1494                 r = -ENXIO;
1495                 if (!irqchip_in_kernel(kvm))
1496                         goto out;
1497                 r = kvm_vm_ioctl_get_irqchip(kvm, &chip);
1498                 if (r)
1499                         goto out;
1500                 r = -EFAULT;
1501                 if (copy_to_user(argp, &chip, sizeof chip))
1502                         goto out;
1503                 r = 0;
1504                 break;
1505         }
1506         case KVM_SET_IRQCHIP: {
1507                 /* 0: PIC master, 1: PIC slave, 2: IOAPIC */
1508                 struct kvm_irqchip chip;
1509
1510                 r = -EFAULT;
1511                 if (copy_from_user(&chip, argp, sizeof chip))
1512                         goto out;
1513                 r = -ENXIO;
1514                 if (!irqchip_in_kernel(kvm))
1515                         goto out;
1516                 r = kvm_vm_ioctl_set_irqchip(kvm, &chip);
1517                 if (r)
1518                         goto out;
1519                 r = 0;
1520                 break;
1521         }
1522         default:
1523                 ;
1524         }
1525 out:
1526         return r;
1527 }
1528
1529 static void kvm_init_msr_list(void)
1530 {
1531         u32 dummy[2];
1532         unsigned i, j;
1533
1534         for (i = j = 0; i < ARRAY_SIZE(msrs_to_save); i++) {
1535                 if (rdmsr_safe(msrs_to_save[i], &dummy[0], &dummy[1]) < 0)
1536                         continue;
1537                 if (j < i)
1538                         msrs_to_save[j] = msrs_to_save[i];
1539                 j++;
1540         }
1541         num_msrs_to_save = j;
1542 }
1543
1544 /*
1545  * Only apic need an MMIO device hook, so shortcut now..
1546  */
1547 static struct kvm_io_device *vcpu_find_pervcpu_dev(struct kvm_vcpu *vcpu,
1548                                                 gpa_t addr)
1549 {
1550         struct kvm_io_device *dev;
1551
1552         if (vcpu->arch.apic) {
1553                 dev = &vcpu->arch.apic->dev;
1554                 if (dev->in_range(dev, addr))
1555                         return dev;
1556         }
1557         return NULL;
1558 }
1559
1560
1561 static struct kvm_io_device *vcpu_find_mmio_dev(struct kvm_vcpu *vcpu,
1562                                                 gpa_t addr)
1563 {
1564         struct kvm_io_device *dev;
1565
1566         dev = vcpu_find_pervcpu_dev(vcpu, addr);
1567         if (dev == NULL)
1568                 dev = kvm_io_bus_find_dev(&vcpu->kvm->mmio_bus, addr);
1569         return dev;
1570 }
1571
1572 int emulator_read_std(unsigned long addr,
1573                              void *val,
1574                              unsigned int bytes,
1575                              struct kvm_vcpu *vcpu)
1576 {
1577         void *data = val;
1578         int r = X86EMUL_CONTINUE;
1579
1580         down_read(&vcpu->kvm->slots_lock);
1581         while (bytes) {
1582                 gpa_t gpa = vcpu->arch.mmu.gva_to_gpa(vcpu, addr);
1583                 unsigned offset = addr & (PAGE_SIZE-1);
1584                 unsigned tocopy = min(bytes, (unsigned)PAGE_SIZE - offset);
1585                 int ret;
1586
1587                 if (gpa == UNMAPPED_GVA) {
1588                         r = X86EMUL_PROPAGATE_FAULT;
1589                         goto out;
1590                 }
1591                 ret = kvm_read_guest(vcpu->kvm, gpa, data, tocopy);
1592                 if (ret < 0) {
1593                         r = X86EMUL_UNHANDLEABLE;
1594                         goto out;
1595                 }
1596
1597                 bytes -= tocopy;
1598                 data += tocopy;
1599                 addr += tocopy;
1600         }
1601 out:
1602         up_read(&vcpu->kvm->slots_lock);
1603         return r;
1604 }
1605 EXPORT_SYMBOL_GPL(emulator_read_std);
1606
1607 static int emulator_read_emulated(unsigned long addr,
1608                                   void *val,
1609                                   unsigned int bytes,
1610                                   struct kvm_vcpu *vcpu)
1611 {
1612         struct kvm_io_device *mmio_dev;
1613         gpa_t                 gpa;
1614
1615         if (vcpu->mmio_read_completed) {
1616                 memcpy(val, vcpu->mmio_data, bytes);
1617                 vcpu->mmio_read_completed = 0;
1618                 return X86EMUL_CONTINUE;
1619         }
1620
1621         down_read(&vcpu->kvm->slots_lock);
1622         gpa = vcpu->arch.mmu.gva_to_gpa(vcpu, addr);
1623         up_read(&vcpu->kvm->slots_lock);
1624
1625         /* For APIC access vmexit */
1626         if ((gpa & PAGE_MASK) == APIC_DEFAULT_PHYS_BASE)
1627                 goto mmio;
1628
1629         if (emulator_read_std(addr, val, bytes, vcpu)
1630                         == X86EMUL_CONTINUE)
1631                 return X86EMUL_CONTINUE;
1632         if (gpa == UNMAPPED_GVA)
1633                 return X86EMUL_PROPAGATE_FAULT;
1634
1635 mmio:
1636         /*
1637          * Is this MMIO handled locally?
1638          */
1639         mutex_lock(&vcpu->kvm->lock);
1640         mmio_dev = vcpu_find_mmio_dev(vcpu, gpa);
1641         if (mmio_dev) {
1642                 kvm_iodevice_read(mmio_dev, gpa, bytes, val);
1643                 mutex_unlock(&vcpu->kvm->lock);
1644                 return X86EMUL_CONTINUE;
1645         }
1646         mutex_unlock(&vcpu->kvm->lock);
1647
1648         vcpu->mmio_needed = 1;
1649         vcpu->mmio_phys_addr = gpa;
1650         vcpu->mmio_size = bytes;
1651         vcpu->mmio_is_write = 0;
1652
1653         return X86EMUL_UNHANDLEABLE;
1654 }
1655
1656 static int emulator_write_phys(struct kvm_vcpu *vcpu, gpa_t gpa,
1657                                const void *val, int bytes)
1658 {
1659         int ret;
1660
1661         down_read(&vcpu->kvm->slots_lock);
1662         ret = kvm_write_guest(vcpu->kvm, gpa, val, bytes);
1663         if (ret < 0) {
1664                 up_read(&vcpu->kvm->slots_lock);
1665                 return 0;
1666         }
1667         kvm_mmu_pte_write(vcpu, gpa, val, bytes);
1668         up_read(&vcpu->kvm->slots_lock);
1669         return 1;
1670 }
1671
1672 static int emulator_write_emulated_onepage(unsigned long addr,
1673                                            const void *val,
1674                                            unsigned int bytes,
1675                                            struct kvm_vcpu *vcpu)
1676 {
1677         struct kvm_io_device *mmio_dev;
1678         gpa_t                 gpa;
1679
1680         down_read(&vcpu->kvm->slots_lock);
1681         gpa = vcpu->arch.mmu.gva_to_gpa(vcpu, addr);
1682         up_read(&vcpu->kvm->slots_lock);
1683
1684         if (gpa == UNMAPPED_GVA) {
1685                 kvm_inject_page_fault(vcpu, addr, 2);
1686                 return X86EMUL_PROPAGATE_FAULT;
1687         }
1688
1689         /* For APIC access vmexit */
1690         if ((gpa & PAGE_MASK) == APIC_DEFAULT_PHYS_BASE)
1691                 goto mmio;
1692
1693         if (emulator_write_phys(vcpu, gpa, val, bytes))
1694                 return X86EMUL_CONTINUE;
1695
1696 mmio:
1697         /*
1698          * Is this MMIO handled locally?
1699          */
1700         mutex_lock(&vcpu->kvm->lock);
1701         mmio_dev = vcpu_find_mmio_dev(vcpu, gpa);
1702         if (mmio_dev) {
1703                 kvm_iodevice_write(mmio_dev, gpa, bytes, val);
1704                 mutex_unlock(&vcpu->kvm->lock);
1705                 return X86EMUL_CONTINUE;
1706         }
1707         mutex_unlock(&vcpu->kvm->lock);
1708
1709         vcpu->mmio_needed = 1;
1710         vcpu->mmio_phys_addr = gpa;
1711         vcpu->mmio_size = bytes;
1712         vcpu->mmio_is_write = 1;
1713         memcpy(vcpu->mmio_data, val, bytes);
1714
1715         return X86EMUL_CONTINUE;
1716 }
1717
1718 int emulator_write_emulated(unsigned long addr,
1719                                    const void *val,
1720                                    unsigned int bytes,
1721                                    struct kvm_vcpu *vcpu)
1722 {
1723         /* Crossing a page boundary? */
1724         if (((addr + bytes - 1) ^ addr) & PAGE_MASK) {
1725                 int rc, now;
1726
1727                 now = -addr & ~PAGE_MASK;
1728                 rc = emulator_write_emulated_onepage(addr, val, now, vcpu);
1729                 if (rc != X86EMUL_CONTINUE)
1730                         return rc;
1731                 addr += now;
1732                 val += now;
1733                 bytes -= now;
1734         }
1735         return emulator_write_emulated_onepage(addr, val, bytes, vcpu);
1736 }
1737 EXPORT_SYMBOL_GPL(emulator_write_emulated);
1738
1739 static int emulator_cmpxchg_emulated(unsigned long addr,
1740                                      const void *old,
1741                                      const void *new,
1742                                      unsigned int bytes,
1743                                      struct kvm_vcpu *vcpu)
1744 {
1745         static int reported;
1746
1747         if (!reported) {
1748                 reported = 1;
1749                 printk(KERN_WARNING "kvm: emulating exchange as write\n");
1750         }
1751 #ifndef CONFIG_X86_64
1752         /* guests cmpxchg8b have to be emulated atomically */
1753         if (bytes == 8) {
1754                 gpa_t gpa;
1755                 struct page *page;
1756                 char *kaddr;
1757                 u64 val;
1758
1759                 down_read(&vcpu->kvm->slots_lock);
1760                 gpa = vcpu->arch.mmu.gva_to_gpa(vcpu, addr);
1761
1762                 if (gpa == UNMAPPED_GVA ||
1763                    (gpa & PAGE_MASK) == APIC_DEFAULT_PHYS_BASE)
1764                         goto emul_write;
1765
1766                 if (((gpa + bytes - 1) & PAGE_MASK) != (gpa & PAGE_MASK))
1767                         goto emul_write;
1768
1769                 val = *(u64 *)new;
1770
1771                 down_read(&current->mm->mmap_sem);
1772                 page = gfn_to_page(vcpu->kvm, gpa >> PAGE_SHIFT);
1773                 up_read(&current->mm->mmap_sem);
1774
1775                 kaddr = kmap_atomic(page, KM_USER0);
1776                 set_64bit((u64 *)(kaddr + offset_in_page(gpa)), val);
1777                 kunmap_atomic(kaddr, KM_USER0);
1778                 kvm_release_page_dirty(page);
1779         emul_write:
1780                 up_read(&vcpu->kvm->slots_lock);
1781         }
1782 #endif
1783
1784         return emulator_write_emulated(addr, new, bytes, vcpu);
1785 }
1786
1787 static unsigned long get_segment_base(struct kvm_vcpu *vcpu, int seg)
1788 {
1789         return kvm_x86_ops->get_segment_base(vcpu, seg);
1790 }
1791
1792 int emulate_invlpg(struct kvm_vcpu *vcpu, gva_t address)
1793 {
1794         return X86EMUL_CONTINUE;
1795 }
1796
1797 int emulate_clts(struct kvm_vcpu *vcpu)
1798 {
1799         kvm_x86_ops->set_cr0(vcpu, vcpu->arch.cr0 & ~X86_CR0_TS);
1800         return X86EMUL_CONTINUE;
1801 }
1802
1803 int emulator_get_dr(struct x86_emulate_ctxt *ctxt, int dr, unsigned long *dest)
1804 {
1805         struct kvm_vcpu *vcpu = ctxt->vcpu;
1806
1807         switch (dr) {
1808         case 0 ... 3:
1809                 *dest = kvm_x86_ops->get_dr(vcpu, dr);
1810                 return X86EMUL_CONTINUE;
1811         default:
1812                 pr_unimpl(vcpu, "%s: unexpected dr %u\n", __FUNCTION__, dr);
1813                 return X86EMUL_UNHANDLEABLE;
1814         }
1815 }
1816
1817 int emulator_set_dr(struct x86_emulate_ctxt *ctxt, int dr, unsigned long value)
1818 {
1819         unsigned long mask = (ctxt->mode == X86EMUL_MODE_PROT64) ? ~0ULL : ~0U;
1820         int exception;
1821
1822         kvm_x86_ops->set_dr(ctxt->vcpu, dr, value & mask, &exception);
1823         if (exception) {
1824                 /* FIXME: better handling */
1825                 return X86EMUL_UNHANDLEABLE;
1826         }
1827         return X86EMUL_CONTINUE;
1828 }
1829
1830 void kvm_report_emulation_failure(struct kvm_vcpu *vcpu, const char *context)
1831 {
1832         static int reported;
1833         u8 opcodes[4];
1834         unsigned long rip = vcpu->arch.rip;
1835         unsigned long rip_linear;
1836
1837         rip_linear = rip + get_segment_base(vcpu, VCPU_SREG_CS);
1838
1839         if (reported)
1840                 return;
1841
1842         emulator_read_std(rip_linear, (void *)opcodes, 4, vcpu);
1843
1844         printk(KERN_ERR "emulation failed (%s) rip %lx %02x %02x %02x %02x\n",
1845                context, rip, opcodes[0], opcodes[1], opcodes[2], opcodes[3]);
1846         reported = 1;
1847 }
1848 EXPORT_SYMBOL_GPL(kvm_report_emulation_failure);
1849
1850 struct x86_emulate_ops emulate_ops = {
1851         .read_std            = emulator_read_std,
1852         .read_emulated       = emulator_read_emulated,
1853         .write_emulated      = emulator_write_emulated,
1854         .cmpxchg_emulated    = emulator_cmpxchg_emulated,
1855 };
1856
1857 int emulate_instruction(struct kvm_vcpu *vcpu,
1858                         struct kvm_run *run,
1859                         unsigned long cr2,
1860                         u16 error_code,
1861                         int emulation_type)
1862 {
1863         int r;
1864         struct decode_cache *c;
1865
1866         vcpu->arch.mmio_fault_cr2 = cr2;
1867         kvm_x86_ops->cache_regs(vcpu);
1868
1869         vcpu->mmio_is_write = 0;
1870         vcpu->arch.pio.string = 0;
1871
1872         if (!(emulation_type & EMULTYPE_NO_DECODE)) {
1873                 int cs_db, cs_l;
1874                 kvm_x86_ops->get_cs_db_l_bits(vcpu, &cs_db, &cs_l);
1875
1876                 vcpu->arch.emulate_ctxt.vcpu = vcpu;
1877                 vcpu->arch.emulate_ctxt.eflags = kvm_x86_ops->get_rflags(vcpu);
1878                 vcpu->arch.emulate_ctxt.mode =
1879                         (vcpu->arch.emulate_ctxt.eflags & X86_EFLAGS_VM)
1880                         ? X86EMUL_MODE_REAL : cs_l
1881                         ? X86EMUL_MODE_PROT64 : cs_db
1882                         ? X86EMUL_MODE_PROT32 : X86EMUL_MODE_PROT16;
1883
1884                 if (vcpu->arch.emulate_ctxt.mode == X86EMUL_MODE_PROT64) {
1885                         vcpu->arch.emulate_ctxt.cs_base = 0;
1886                         vcpu->arch.emulate_ctxt.ds_base = 0;
1887                         vcpu->arch.emulate_ctxt.es_base = 0;
1888                         vcpu->arch.emulate_ctxt.ss_base = 0;
1889                 } else {
1890                         vcpu->arch.emulate_ctxt.cs_base =
1891                                         get_segment_base(vcpu, VCPU_SREG_CS);
1892                         vcpu->arch.emulate_ctxt.ds_base =
1893                                         get_segment_base(vcpu, VCPU_SREG_DS);
1894                         vcpu->arch.emulate_ctxt.es_base =
1895                                         get_segment_base(vcpu, VCPU_SREG_ES);
1896                         vcpu->arch.emulate_ctxt.ss_base =
1897                                         get_segment_base(vcpu, VCPU_SREG_SS);
1898                 }
1899
1900                 vcpu->arch.emulate_ctxt.gs_base =
1901                                         get_segment_base(vcpu, VCPU_SREG_GS);
1902                 vcpu->arch.emulate_ctxt.fs_base =
1903                                         get_segment_base(vcpu, VCPU_SREG_FS);
1904
1905                 r = x86_decode_insn(&vcpu->arch.emulate_ctxt, &emulate_ops);
1906
1907                 /* Reject the instructions other than VMCALL/VMMCALL when
1908                  * try to emulate invalid opcode */
1909                 c = &vcpu->arch.emulate_ctxt.decode;
1910                 if ((emulation_type & EMULTYPE_TRAP_UD) &&
1911                     (!(c->twobyte && c->b == 0x01 &&
1912                       (c->modrm_reg == 0 || c->modrm_reg == 3) &&
1913                        c->modrm_mod == 3 && c->modrm_rm == 1)))
1914                         return EMULATE_FAIL;
1915
1916                 ++vcpu->stat.insn_emulation;
1917                 if (r)  {
1918                         ++vcpu->stat.insn_emulation_fail;
1919                         if (kvm_mmu_unprotect_page_virt(vcpu, cr2))
1920                                 return EMULATE_DONE;
1921                         return EMULATE_FAIL;
1922                 }
1923         }
1924
1925         r = x86_emulate_insn(&vcpu->arch.emulate_ctxt, &emulate_ops);
1926
1927         if (vcpu->arch.pio.string)
1928                 return EMULATE_DO_MMIO;
1929
1930         if ((r || vcpu->mmio_is_write) && run) {
1931                 run->exit_reason = KVM_EXIT_MMIO;
1932                 run->mmio.phys_addr = vcpu->mmio_phys_addr;
1933                 memcpy(run->mmio.data, vcpu->mmio_data, 8);
1934                 run->mmio.len = vcpu->mmio_size;
1935                 run->mmio.is_write = vcpu->mmio_is_write;
1936         }
1937
1938         if (r) {
1939                 if (kvm_mmu_unprotect_page_virt(vcpu, cr2))
1940                         return EMULATE_DONE;
1941                 if (!vcpu->mmio_needed) {
1942                         kvm_report_emulation_failure(vcpu, "mmio");
1943                         return EMULATE_FAIL;
1944                 }
1945                 return EMULATE_DO_MMIO;
1946         }
1947
1948         kvm_x86_ops->decache_regs(vcpu);
1949         kvm_x86_ops->set_rflags(vcpu, vcpu->arch.emulate_ctxt.eflags);
1950
1951         if (vcpu->mmio_is_write) {
1952                 vcpu->mmio_needed = 0;
1953                 return EMULATE_DO_MMIO;
1954         }
1955
1956         return EMULATE_DONE;
1957 }
1958 EXPORT_SYMBOL_GPL(emulate_instruction);
1959
1960 static void free_pio_guest_pages(struct kvm_vcpu *vcpu)
1961 {
1962         int i;
1963
1964         for (i = 0; i < ARRAY_SIZE(vcpu->arch.pio.guest_pages); ++i)
1965                 if (vcpu->arch.pio.guest_pages[i]) {
1966                         kvm_release_page_dirty(vcpu->arch.pio.guest_pages[i]);
1967                         vcpu->arch.pio.guest_pages[i] = NULL;
1968                 }
1969 }
1970
1971 static int pio_copy_data(struct kvm_vcpu *vcpu)
1972 {
1973         void *p = vcpu->arch.pio_data;
1974         void *q;
1975         unsigned bytes;
1976         int nr_pages = vcpu->arch.pio.guest_pages[1] ? 2 : 1;
1977
1978         q = vmap(vcpu->arch.pio.guest_pages, nr_pages, VM_READ|VM_WRITE,
1979                  PAGE_KERNEL);
1980         if (!q) {
1981                 free_pio_guest_pages(vcpu);
1982                 return -ENOMEM;
1983         }
1984         q += vcpu->arch.pio.guest_page_offset;
1985         bytes = vcpu->arch.pio.size * vcpu->arch.pio.cur_count;
1986         if (vcpu->arch.pio.in)
1987                 memcpy(q, p, bytes);
1988         else
1989                 memcpy(p, q, bytes);
1990         q -= vcpu->arch.pio.guest_page_offset;
1991         vunmap(q);
1992         free_pio_guest_pages(vcpu);
1993         return 0;
1994 }
1995
1996 int complete_pio(struct kvm_vcpu *vcpu)
1997 {
1998         struct kvm_pio_request *io = &vcpu->arch.pio;
1999         long delta;
2000         int r;
2001
2002         kvm_x86_ops->cache_regs(vcpu);
2003
2004         if (!io->string) {
2005                 if (io->in)
2006                         memcpy(&vcpu->arch.regs[VCPU_REGS_RAX], vcpu->arch.pio_data,
2007                                io->size);
2008         } else {
2009                 if (io->in) {
2010                         r = pio_copy_data(vcpu);
2011                         if (r) {
2012                                 kvm_x86_ops->cache_regs(vcpu);
2013                                 return r;
2014                         }
2015                 }
2016
2017                 delta = 1;
2018                 if (io->rep) {
2019                         delta *= io->cur_count;
2020                         /*
2021                          * The size of the register should really depend on
2022                          * current address size.
2023                          */
2024                         vcpu->arch.regs[VCPU_REGS_RCX] -= delta;
2025                 }
2026                 if (io->down)
2027                         delta = -delta;
2028                 delta *= io->size;
2029                 if (io->in)
2030                         vcpu->arch.regs[VCPU_REGS_RDI] += delta;
2031                 else
2032                         vcpu->arch.regs[VCPU_REGS_RSI] += delta;
2033         }
2034
2035         kvm_x86_ops->decache_regs(vcpu);
2036
2037         io->count -= io->cur_count;
2038         io->cur_count = 0;
2039
2040         return 0;
2041 }
2042
2043 static void kernel_pio(struct kvm_io_device *pio_dev,
2044                        struct kvm_vcpu *vcpu,
2045                        void *pd)
2046 {
2047         /* TODO: String I/O for in kernel device */
2048
2049         mutex_lock(&vcpu->kvm->lock);
2050         if (vcpu->arch.pio.in)
2051                 kvm_iodevice_read(pio_dev, vcpu->arch.pio.port,
2052                                   vcpu->arch.pio.size,
2053                                   pd);
2054         else
2055                 kvm_iodevice_write(pio_dev, vcpu->arch.pio.port,
2056                                    vcpu->arch.pio.size,
2057                                    pd);
2058         mutex_unlock(&vcpu->kvm->lock);
2059 }
2060
2061 static void pio_string_write(struct kvm_io_device *pio_dev,
2062                              struct kvm_vcpu *vcpu)
2063 {
2064         struct kvm_pio_request *io = &vcpu->arch.pio;
2065         void *pd = vcpu->arch.pio_data;
2066         int i;
2067
2068         mutex_lock(&vcpu->kvm->lock);
2069         for (i = 0; i < io->cur_count; i++) {
2070                 kvm_iodevice_write(pio_dev, io->port,
2071                                    io->size,
2072                                    pd);
2073                 pd += io->size;
2074         }
2075         mutex_unlock(&vcpu->kvm->lock);
2076 }
2077
2078 static struct kvm_io_device *vcpu_find_pio_dev(struct kvm_vcpu *vcpu,
2079                                                gpa_t addr)
2080 {
2081         return kvm_io_bus_find_dev(&vcpu->kvm->pio_bus, addr);
2082 }
2083
2084 int kvm_emulate_pio(struct kvm_vcpu *vcpu, struct kvm_run *run, int in,
2085                   int size, unsigned port)
2086 {
2087         struct kvm_io_device *pio_dev;
2088
2089         vcpu->run->exit_reason = KVM_EXIT_IO;
2090         vcpu->run->io.direction = in ? KVM_EXIT_IO_IN : KVM_EXIT_IO_OUT;
2091         vcpu->run->io.size = vcpu->arch.pio.size = size;
2092         vcpu->run->io.data_offset = KVM_PIO_PAGE_OFFSET * PAGE_SIZE;
2093         vcpu->run->io.count = vcpu->arch.pio.count = vcpu->arch.pio.cur_count = 1;
2094         vcpu->run->io.port = vcpu->arch.pio.port = port;
2095         vcpu->arch.pio.in = in;
2096         vcpu->arch.pio.string = 0;
2097         vcpu->arch.pio.down = 0;
2098         vcpu->arch.pio.guest_page_offset = 0;
2099         vcpu->arch.pio.rep = 0;
2100
2101         kvm_x86_ops->cache_regs(vcpu);
2102         memcpy(vcpu->arch.pio_data, &vcpu->arch.regs[VCPU_REGS_RAX], 4);
2103         kvm_x86_ops->decache_regs(vcpu);
2104
2105         kvm_x86_ops->skip_emulated_instruction(vcpu);
2106
2107         pio_dev = vcpu_find_pio_dev(vcpu, port);
2108         if (pio_dev) {
2109                 kernel_pio(pio_dev, vcpu, vcpu->arch.pio_data);
2110                 complete_pio(vcpu);
2111                 return 1;
2112         }
2113         return 0;
2114 }
2115 EXPORT_SYMBOL_GPL(kvm_emulate_pio);
2116
2117 int kvm_emulate_pio_string(struct kvm_vcpu *vcpu, struct kvm_run *run, int in,
2118                   int size, unsigned long count, int down,
2119                   gva_t address, int rep, unsigned port)
2120 {
2121         unsigned now, in_page;
2122         int i, ret = 0;
2123         int nr_pages = 1;
2124         struct page *page;
2125         struct kvm_io_device *pio_dev;
2126
2127         vcpu->run->exit_reason = KVM_EXIT_IO;
2128         vcpu->run->io.direction = in ? KVM_EXIT_IO_IN : KVM_EXIT_IO_OUT;
2129         vcpu->run->io.size = vcpu->arch.pio.size = size;
2130         vcpu->run->io.data_offset = KVM_PIO_PAGE_OFFSET * PAGE_SIZE;
2131         vcpu->run->io.count = vcpu->arch.pio.count = vcpu->arch.pio.cur_count = count;
2132         vcpu->run->io.port = vcpu->arch.pio.port = port;
2133         vcpu->arch.pio.in = in;
2134         vcpu->arch.pio.string = 1;
2135         vcpu->arch.pio.down = down;
2136         vcpu->arch.pio.guest_page_offset = offset_in_page(address);
2137         vcpu->arch.pio.rep = rep;
2138
2139         if (!count) {
2140                 kvm_x86_ops->skip_emulated_instruction(vcpu);
2141                 return 1;
2142         }
2143
2144         if (!down)
2145                 in_page = PAGE_SIZE - offset_in_page(address);
2146         else
2147                 in_page = offset_in_page(address) + size;
2148         now = min(count, (unsigned long)in_page / size);
2149         if (!now) {
2150                 /*
2151                  * String I/O straddles page boundary.  Pin two guest pages
2152                  * so that we satisfy atomicity constraints.  Do just one
2153                  * transaction to avoid complexity.
2154                  */
2155                 nr_pages = 2;
2156                 now = 1;
2157         }
2158         if (down) {
2159                 /*
2160                  * String I/O in reverse.  Yuck.  Kill the guest, fix later.
2161                  */
2162                 pr_unimpl(vcpu, "guest string pio down\n");
2163                 kvm_inject_gp(vcpu, 0);
2164                 return 1;
2165         }
2166         vcpu->run->io.count = now;
2167         vcpu->arch.pio.cur_count = now;
2168
2169         if (vcpu->arch.pio.cur_count == vcpu->arch.pio.count)
2170                 kvm_x86_ops->skip_emulated_instruction(vcpu);
2171
2172         for (i = 0; i < nr_pages; ++i) {
2173                 down_read(&vcpu->kvm->slots_lock);
2174                 page = gva_to_page(vcpu, address + i * PAGE_SIZE);
2175                 vcpu->arch.pio.guest_pages[i] = page;
2176                 up_read(&vcpu->kvm->slots_lock);
2177                 if (!page) {
2178                         kvm_inject_gp(vcpu, 0);
2179                         free_pio_guest_pages(vcpu);
2180                         return 1;
2181                 }
2182         }
2183
2184         pio_dev = vcpu_find_pio_dev(vcpu, port);
2185         if (!vcpu->arch.pio.in) {
2186                 /* string PIO write */
2187                 ret = pio_copy_data(vcpu);
2188                 if (ret >= 0 && pio_dev) {
2189                         pio_string_write(pio_dev, vcpu);
2190                         complete_pio(vcpu);
2191                         if (vcpu->arch.pio.count == 0)
2192                                 ret = 1;
2193                 }
2194         } else if (pio_dev)
2195                 pr_unimpl(vcpu, "no string pio read support yet, "
2196                        "port %x size %d count %ld\n",
2197                         port, size, count);
2198
2199         return ret;
2200 }
2201 EXPORT_SYMBOL_GPL(kvm_emulate_pio_string);
2202
2203 int kvm_arch_init(void *opaque)
2204 {
2205         int r;
2206         struct kvm_x86_ops *ops = (struct kvm_x86_ops *)opaque;
2207
2208         if (kvm_x86_ops) {
2209                 printk(KERN_ERR "kvm: already loaded the other module\n");
2210                 r = -EEXIST;
2211                 goto out;
2212         }
2213
2214         if (!ops->cpu_has_kvm_support()) {
2215                 printk(KERN_ERR "kvm: no hardware support\n");
2216                 r = -EOPNOTSUPP;
2217                 goto out;
2218         }
2219         if (ops->disabled_by_bios()) {
2220                 printk(KERN_ERR "kvm: disabled by bios\n");
2221                 r = -EOPNOTSUPP;
2222                 goto out;
2223         }
2224
2225         r = kvm_mmu_module_init();
2226         if (r)
2227                 goto out;
2228
2229         kvm_init_msr_list();
2230
2231         kvm_x86_ops = ops;
2232         kvm_mmu_set_nonpresent_ptes(0ull, 0ull);
2233         return 0;
2234
2235 out:
2236         return r;
2237 }
2238
2239 void kvm_arch_exit(void)
2240 {
2241         kvm_x86_ops = NULL;
2242         kvm_mmu_module_exit();
2243 }
2244
2245 int kvm_emulate_halt(struct kvm_vcpu *vcpu)
2246 {
2247         ++vcpu->stat.halt_exits;
2248         if (irqchip_in_kernel(vcpu->kvm)) {
2249                 vcpu->arch.mp_state = VCPU_MP_STATE_HALTED;
2250                 kvm_vcpu_block(vcpu);
2251                 if (vcpu->arch.mp_state != VCPU_MP_STATE_RUNNABLE)
2252                         return -EINTR;
2253                 return 1;
2254         } else {
2255                 vcpu->run->exit_reason = KVM_EXIT_HLT;
2256                 return 0;
2257         }
2258 }
2259 EXPORT_SYMBOL_GPL(kvm_emulate_halt);
2260
2261 int kvm_emulate_hypercall(struct kvm_vcpu *vcpu)
2262 {
2263         unsigned long nr, a0, a1, a2, a3, ret;
2264
2265         kvm_x86_ops->cache_regs(vcpu);
2266
2267         nr = vcpu->arch.regs[VCPU_REGS_RAX];
2268         a0 = vcpu->arch.regs[VCPU_REGS_RBX];
2269         a1 = vcpu->arch.regs[VCPU_REGS_RCX];
2270         a2 = vcpu->arch.regs[VCPU_REGS_RDX];
2271         a3 = vcpu->arch.regs[VCPU_REGS_RSI];
2272
2273         if (!is_long_mode(vcpu)) {
2274                 nr &= 0xFFFFFFFF;
2275                 a0 &= 0xFFFFFFFF;
2276                 a1 &= 0xFFFFFFFF;
2277                 a2 &= 0xFFFFFFFF;
2278                 a3 &= 0xFFFFFFFF;
2279         }
2280
2281         switch (nr) {
2282         case KVM_HC_VAPIC_POLL_IRQ:
2283                 ret = 0;
2284                 break;
2285         default:
2286                 ret = -KVM_ENOSYS;
2287                 break;
2288         }
2289         vcpu->arch.regs[VCPU_REGS_RAX] = ret;
2290         kvm_x86_ops->decache_regs(vcpu);
2291         return 0;
2292 }
2293 EXPORT_SYMBOL_GPL(kvm_emulate_hypercall);
2294
2295 int kvm_fix_hypercall(struct kvm_vcpu *vcpu)
2296 {
2297         char instruction[3];
2298         int ret = 0;
2299
2300
2301         /*
2302          * Blow out the MMU to ensure that no other VCPU has an active mapping
2303          * to ensure that the updated hypercall appears atomically across all
2304          * VCPUs.
2305          */
2306         kvm_mmu_zap_all(vcpu->kvm);
2307
2308         kvm_x86_ops->cache_regs(vcpu);
2309         kvm_x86_ops->patch_hypercall(vcpu, instruction);
2310         if (emulator_write_emulated(vcpu->arch.rip, instruction, 3, vcpu)
2311             != X86EMUL_CONTINUE)
2312                 ret = -EFAULT;
2313
2314         return ret;
2315 }
2316
2317 static u64 mk_cr_64(u64 curr_cr, u32 new_val)
2318 {
2319         return (curr_cr & ~((1ULL << 32) - 1)) | new_val;
2320 }
2321
2322 void realmode_lgdt(struct kvm_vcpu *vcpu, u16 limit, unsigned long base)
2323 {
2324         struct descriptor_table dt = { limit, base };
2325
2326         kvm_x86_ops->set_gdt(vcpu, &dt);
2327 }
2328
2329 void realmode_lidt(struct kvm_vcpu *vcpu, u16 limit, unsigned long base)
2330 {
2331         struct descriptor_table dt = { limit, base };
2332
2333         kvm_x86_ops->set_idt(vcpu, &dt);
2334 }
2335
2336 void realmode_lmsw(struct kvm_vcpu *vcpu, unsigned long msw,
2337                    unsigned long *rflags)
2338 {
2339         lmsw(vcpu, msw);
2340         *rflags = kvm_x86_ops->get_rflags(vcpu);
2341 }
2342
2343 unsigned long realmode_get_cr(struct kvm_vcpu *vcpu, int cr)
2344 {
2345         kvm_x86_ops->decache_cr4_guest_bits(vcpu);
2346         switch (cr) {
2347         case 0:
2348                 return vcpu->arch.cr0;
2349         case 2:
2350                 return vcpu->arch.cr2;
2351         case 3:
2352                 return vcpu->arch.cr3;
2353         case 4:
2354                 return vcpu->arch.cr4;
2355         case 8:
2356                 return get_cr8(vcpu);
2357         default:
2358                 vcpu_printf(vcpu, "%s: unexpected cr %u\n", __FUNCTION__, cr);
2359                 return 0;
2360         }
2361 }
2362
2363 void realmode_set_cr(struct kvm_vcpu *vcpu, int cr, unsigned long val,
2364                      unsigned long *rflags)
2365 {
2366         switch (cr) {
2367         case 0:
2368                 set_cr0(vcpu, mk_cr_64(vcpu->arch.cr0, val));
2369                 *rflags = kvm_x86_ops->get_rflags(vcpu);
2370                 break;
2371         case 2:
2372                 vcpu->arch.cr2 = val;
2373                 break;
2374         case 3:
2375                 set_cr3(vcpu, val);
2376                 break;
2377         case 4:
2378                 set_cr4(vcpu, mk_cr_64(vcpu->arch.cr4, val));
2379                 break;
2380         case 8:
2381                 set_cr8(vcpu, val & 0xfUL);
2382                 break;
2383         default:
2384                 vcpu_printf(vcpu, "%s: unexpected cr %u\n", __FUNCTION__, cr);
2385         }
2386 }
2387
2388 static int move_to_next_stateful_cpuid_entry(struct kvm_vcpu *vcpu, int i)
2389 {
2390         struct kvm_cpuid_entry2 *e = &vcpu->arch.cpuid_entries[i];
2391         int j, nent = vcpu->arch.cpuid_nent;
2392
2393         e->flags &= ~KVM_CPUID_FLAG_STATE_READ_NEXT;
2394         /* when no next entry is found, the current entry[i] is reselected */
2395         for (j = i + 1; j == i; j = (j + 1) % nent) {
2396                 struct kvm_cpuid_entry2 *ej = &vcpu->arch.cpuid_entries[j];
2397                 if (ej->function == e->function) {
2398                         ej->flags |= KVM_CPUID_FLAG_STATE_READ_NEXT;
2399                         return j;
2400                 }
2401         }
2402         return 0; /* silence gcc, even though control never reaches here */
2403 }
2404
2405 /* find an entry with matching function, matching index (if needed), and that
2406  * should be read next (if it's stateful) */
2407 static int is_matching_cpuid_entry(struct kvm_cpuid_entry2 *e,
2408         u32 function, u32 index)
2409 {
2410         if (e->function != function)
2411                 return 0;
2412         if ((e->flags & KVM_CPUID_FLAG_SIGNIFCANT_INDEX) && e->index != index)
2413                 return 0;
2414         if ((e->flags & KVM_CPUID_FLAG_STATEFUL_FUNC) &&
2415                 !(e->flags & KVM_CPUID_FLAG_STATE_READ_NEXT))
2416                 return 0;
2417         return 1;
2418 }
2419
2420 void kvm_emulate_cpuid(struct kvm_vcpu *vcpu)
2421 {
2422         int i;
2423         u32 function, index;
2424         struct kvm_cpuid_entry2 *e, *best;
2425
2426         kvm_x86_ops->cache_regs(vcpu);
2427         function = vcpu->arch.regs[VCPU_REGS_RAX];
2428         index = vcpu->arch.regs[VCPU_REGS_RCX];
2429         vcpu->arch.regs[VCPU_REGS_RAX] = 0;
2430         vcpu->arch.regs[VCPU_REGS_RBX] = 0;
2431         vcpu->arch.regs[VCPU_REGS_RCX] = 0;
2432         vcpu->arch.regs[VCPU_REGS_RDX] = 0;
2433         best = NULL;
2434         for (i = 0; i < vcpu->arch.cpuid_nent; ++i) {
2435                 e = &vcpu->arch.cpuid_entries[i];
2436                 if (is_matching_cpuid_entry(e, function, index)) {
2437                         if (e->flags & KVM_CPUID_FLAG_STATEFUL_FUNC)
2438                                 move_to_next_stateful_cpuid_entry(vcpu, i);
2439                         best = e;
2440                         break;
2441                 }
2442                 /*
2443                  * Both basic or both extended?
2444                  */
2445                 if (((e->function ^ function) & 0x80000000) == 0)
2446                         if (!best || e->function > best->function)
2447                                 best = e;
2448         }
2449         if (best) {
2450                 vcpu->arch.regs[VCPU_REGS_RAX] = best->eax;
2451                 vcpu->arch.regs[VCPU_REGS_RBX] = best->ebx;
2452                 vcpu->arch.regs[VCPU_REGS_RCX] = best->ecx;
2453                 vcpu->arch.regs[VCPU_REGS_RDX] = best->edx;
2454         }
2455         kvm_x86_ops->decache_regs(vcpu);
2456         kvm_x86_ops->skip_emulated_instruction(vcpu);
2457 }
2458 EXPORT_SYMBOL_GPL(kvm_emulate_cpuid);
2459
2460 /*
2461  * Check if userspace requested an interrupt window, and that the
2462  * interrupt window is open.
2463  *
2464  * No need to exit to userspace if we already have an interrupt queued.
2465  */
2466 static int dm_request_for_irq_injection(struct kvm_vcpu *vcpu,
2467                                           struct kvm_run *kvm_run)
2468 {
2469         return (!vcpu->arch.irq_summary &&
2470                 kvm_run->request_interrupt_window &&
2471                 vcpu->arch.interrupt_window_open &&
2472                 (kvm_x86_ops->get_rflags(vcpu) & X86_EFLAGS_IF));
2473 }
2474
2475 static void post_kvm_run_save(struct kvm_vcpu *vcpu,
2476                               struct kvm_run *kvm_run)
2477 {
2478         kvm_run->if_flag = (kvm_x86_ops->get_rflags(vcpu) & X86_EFLAGS_IF) != 0;
2479         kvm_run->cr8 = get_cr8(vcpu);
2480         kvm_run->apic_base = kvm_get_apic_base(vcpu);
2481         if (irqchip_in_kernel(vcpu->kvm))
2482                 kvm_run->ready_for_interrupt_injection = 1;
2483         else
2484                 kvm_run->ready_for_interrupt_injection =
2485                                         (vcpu->arch.interrupt_window_open &&
2486                                          vcpu->arch.irq_summary == 0);
2487 }
2488
2489 static void vapic_enter(struct kvm_vcpu *vcpu)
2490 {
2491         struct kvm_lapic *apic = vcpu->arch.apic;
2492         struct page *page;
2493
2494         if (!apic || !apic->vapic_addr)
2495                 return;
2496
2497         down_read(&current->mm->mmap_sem);
2498         page = gfn_to_page(vcpu->kvm, apic->vapic_addr >> PAGE_SHIFT);
2499         up_read(&current->mm->mmap_sem);
2500
2501         vcpu->arch.apic->vapic_page = page;
2502 }
2503
2504 static void vapic_exit(struct kvm_vcpu *vcpu)
2505 {
2506         struct kvm_lapic *apic = vcpu->arch.apic;
2507
2508         if (!apic || !apic->vapic_addr)
2509                 return;
2510
2511         kvm_release_page_dirty(apic->vapic_page);
2512         mark_page_dirty(vcpu->kvm, apic->vapic_addr >> PAGE_SHIFT);
2513 }
2514
2515 static int __vcpu_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
2516 {
2517         int r;
2518
2519         if (unlikely(vcpu->arch.mp_state == VCPU_MP_STATE_SIPI_RECEIVED)) {
2520                 pr_debug("vcpu %d received sipi with vector # %x\n",
2521                        vcpu->vcpu_id, vcpu->arch.sipi_vector);
2522                 kvm_lapic_reset(vcpu);
2523                 r = kvm_x86_ops->vcpu_reset(vcpu);
2524                 if (r)
2525                         return r;
2526                 vcpu->arch.mp_state = VCPU_MP_STATE_RUNNABLE;
2527         }
2528
2529         vapic_enter(vcpu);
2530
2531 preempted:
2532         if (vcpu->guest_debug.enabled)
2533                 kvm_x86_ops->guest_debug_pre(vcpu);
2534
2535 again:
2536         r = kvm_mmu_reload(vcpu);
2537         if (unlikely(r))
2538                 goto out;
2539
2540         if (vcpu->requests) {
2541                 if (test_and_clear_bit(KVM_REQ_MIGRATE_TIMER, &vcpu->requests))
2542                         __kvm_migrate_apic_timer(vcpu);
2543                 if (test_and_clear_bit(KVM_REQ_REPORT_TPR_ACCESS,
2544                                        &vcpu->requests)) {
2545                         kvm_run->exit_reason = KVM_EXIT_TPR_ACCESS;
2546                         r = 0;
2547                         goto out;
2548                 }
2549         }
2550
2551         kvm_inject_pending_timer_irqs(vcpu);
2552
2553         preempt_disable();
2554
2555         kvm_x86_ops->prepare_guest_switch(vcpu);
2556         kvm_load_guest_fpu(vcpu);
2557
2558         local_irq_disable();
2559
2560         if (need_resched()) {
2561                 local_irq_enable();
2562                 preempt_enable();
2563                 r = 1;
2564                 goto out;
2565         }
2566
2567         if (signal_pending(current)) {
2568                 local_irq_enable();
2569                 preempt_enable();
2570                 r = -EINTR;
2571                 kvm_run->exit_reason = KVM_EXIT_INTR;
2572                 ++vcpu->stat.signal_exits;
2573                 goto out;
2574         }
2575
2576         if (vcpu->arch.exception.pending)
2577                 __queue_exception(vcpu);
2578         else if (irqchip_in_kernel(vcpu->kvm))
2579                 kvm_x86_ops->inject_pending_irq(vcpu);
2580         else
2581                 kvm_x86_ops->inject_pending_vectors(vcpu, kvm_run);
2582
2583         kvm_lapic_sync_to_vapic(vcpu);
2584
2585         vcpu->guest_mode = 1;
2586         kvm_guest_enter();
2587
2588         if (vcpu->requests)
2589                 if (test_and_clear_bit(KVM_REQ_TLB_FLUSH, &vcpu->requests))
2590                         kvm_x86_ops->tlb_flush(vcpu);
2591
2592         kvm_x86_ops->run(vcpu, kvm_run);
2593
2594         vcpu->guest_mode = 0;
2595         local_irq_enable();
2596
2597         ++vcpu->stat.exits;
2598
2599         /*
2600          * We must have an instruction between local_irq_enable() and
2601          * kvm_guest_exit(), so the timer interrupt isn't delayed by
2602          * the interrupt shadow.  The stat.exits increment will do nicely.
2603          * But we need to prevent reordering, hence this barrier():
2604          */
2605         barrier();
2606
2607         kvm_guest_exit();
2608
2609         preempt_enable();
2610
2611         /*
2612          * Profile KVM exit RIPs:
2613          */
2614         if (unlikely(prof_on == KVM_PROFILING)) {
2615                 kvm_x86_ops->cache_regs(vcpu);
2616                 profile_hit(KVM_PROFILING, (void *)vcpu->arch.rip);
2617         }
2618
2619         if (vcpu->arch.exception.pending && kvm_x86_ops->exception_injected(vcpu))
2620                 vcpu->arch.exception.pending = false;
2621
2622         kvm_lapic_sync_from_vapic(vcpu);
2623
2624         r = kvm_x86_ops->handle_exit(kvm_run, vcpu);
2625
2626         if (r > 0) {
2627                 if (dm_request_for_irq_injection(vcpu, kvm_run)) {
2628                         r = -EINTR;
2629                         kvm_run->exit_reason = KVM_EXIT_INTR;
2630                         ++vcpu->stat.request_irq_exits;
2631                         goto out;
2632                 }
2633                 if (!need_resched())
2634                         goto again;
2635         }
2636
2637 out:
2638         if (r > 0) {
2639                 kvm_resched(vcpu);
2640                 goto preempted;
2641         }
2642
2643         post_kvm_run_save(vcpu, kvm_run);
2644
2645         vapic_exit(vcpu);
2646
2647         return r;
2648 }
2649
2650 int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
2651 {
2652         int r;
2653         sigset_t sigsaved;
2654
2655         vcpu_load(vcpu);
2656
2657         if (unlikely(vcpu->arch.mp_state == VCPU_MP_STATE_UNINITIALIZED)) {
2658                 kvm_vcpu_block(vcpu);
2659                 vcpu_put(vcpu);
2660                 return -EAGAIN;
2661         }
2662
2663         if (vcpu->sigset_active)
2664                 sigprocmask(SIG_SETMASK, &vcpu->sigset, &sigsaved);
2665
2666         /* re-sync apic's tpr */
2667         if (!irqchip_in_kernel(vcpu->kvm))
2668                 set_cr8(vcpu, kvm_run->cr8);
2669
2670         if (vcpu->arch.pio.cur_count) {
2671                 r = complete_pio(vcpu);
2672                 if (r)
2673                         goto out;
2674         }
2675 #if CONFIG_HAS_IOMEM
2676         if (vcpu->mmio_needed) {
2677                 memcpy(vcpu->mmio_data, kvm_run->mmio.data, 8);
2678                 vcpu->mmio_read_completed = 1;
2679                 vcpu->mmio_needed = 0;
2680                 r = emulate_instruction(vcpu, kvm_run,
2681                                         vcpu->arch.mmio_fault_cr2, 0,
2682                                         EMULTYPE_NO_DECODE);
2683                 if (r == EMULATE_DO_MMIO) {
2684                         /*
2685                          * Read-modify-write.  Back to userspace.
2686                          */
2687                         r = 0;
2688                         goto out;
2689                 }
2690         }
2691 #endif
2692         if (kvm_run->exit_reason == KVM_EXIT_HYPERCALL) {
2693                 kvm_x86_ops->cache_regs(vcpu);
2694                 vcpu->arch.regs[VCPU_REGS_RAX] = kvm_run->hypercall.ret;
2695                 kvm_x86_ops->decache_regs(vcpu);
2696         }
2697
2698         r = __vcpu_run(vcpu, kvm_run);
2699
2700 out:
2701         if (vcpu->sigset_active)
2702                 sigprocmask(SIG_SETMASK, &sigsaved, NULL);
2703
2704         vcpu_put(vcpu);
2705         return r;
2706 }
2707
2708 int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
2709 {
2710         vcpu_load(vcpu);
2711
2712         kvm_x86_ops->cache_regs(vcpu);
2713
2714         regs->rax = vcpu->arch.regs[VCPU_REGS_RAX];
2715         regs->rbx = vcpu->arch.regs[VCPU_REGS_RBX];
2716         regs->rcx = vcpu->arch.regs[VCPU_REGS_RCX];
2717         regs->rdx = vcpu->arch.regs[VCPU_REGS_RDX];
2718         regs->rsi = vcpu->arch.regs[VCPU_REGS_RSI];
2719         regs->rdi = vcpu->arch.regs[VCPU_REGS_RDI];
2720         regs->rsp = vcpu->arch.regs[VCPU_REGS_RSP];
2721         regs->rbp = vcpu->arch.regs[VCPU_REGS_RBP];
2722 #ifdef CONFIG_X86_64
2723         regs->r8 = vcpu->arch.regs[VCPU_REGS_R8];
2724         regs->r9 = vcpu->arch.regs[VCPU_REGS_R9];
2725         regs->r10 = vcpu->arch.regs[VCPU_REGS_R10];
2726         regs->r11 = vcpu->arch.regs[VCPU_REGS_R11];
2727         regs->r12 = vcpu->arch.regs[VCPU_REGS_R12];
2728         regs->r13 = vcpu->arch.regs[VCPU_REGS_R13];
2729         regs->r14 = vcpu->arch.regs[VCPU_REGS_R14];
2730         regs->r15 = vcpu->arch.regs[VCPU_REGS_R15];
2731 #endif
2732
2733         regs->rip = vcpu->arch.rip;
2734         regs->rflags = kvm_x86_ops->get_rflags(vcpu);
2735
2736         /*
2737          * Don't leak debug flags in case they were set for guest debugging
2738          */
2739         if (vcpu->guest_debug.enabled && vcpu->guest_debug.singlestep)
2740                 regs->rflags &= ~(X86_EFLAGS_TF | X86_EFLAGS_RF);
2741
2742         vcpu_put(vcpu);
2743
2744         return 0;
2745 }
2746
2747 int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
2748 {
2749         vcpu_load(vcpu);
2750
2751         vcpu->arch.regs[VCPU_REGS_RAX] = regs->rax;
2752         vcpu->arch.regs[VCPU_REGS_RBX] = regs->rbx;
2753         vcpu->arch.regs[VCPU_REGS_RCX] = regs->rcx;
2754         vcpu->arch.regs[VCPU_REGS_RDX] = regs->rdx;
2755         vcpu->arch.regs[VCPU_REGS_RSI] = regs->rsi;
2756         vcpu->arch.regs[VCPU_REGS_RDI] = regs->rdi;
2757         vcpu->arch.regs[VCPU_REGS_RSP] = regs->rsp;
2758         vcpu->arch.regs[VCPU_REGS_RBP] = regs->rbp;
2759 #ifdef CONFIG_X86_64
2760         vcpu->arch.regs[VCPU_REGS_R8] = regs->r8;
2761         vcpu->arch.regs[VCPU_REGS_R9] = regs->r9;
2762         vcpu->arch.regs[VCPU_REGS_R10] = regs->r10;
2763         vcpu->arch.regs[VCPU_REGS_R11] = regs->r11;
2764         vcpu->arch.regs[VCPU_REGS_R12] = regs->r12;
2765         vcpu->arch.regs[VCPU_REGS_R13] = regs->r13;
2766         vcpu->arch.regs[VCPU_REGS_R14] = regs->r14;
2767         vcpu->arch.regs[VCPU_REGS_R15] = regs->r15;
2768 #endif
2769
2770         vcpu->arch.rip = regs->rip;
2771         kvm_x86_ops->set_rflags(vcpu, regs->rflags);
2772
2773         kvm_x86_ops->decache_regs(vcpu);
2774
2775         vcpu_put(vcpu);
2776
2777         return 0;
2778 }
2779
2780 static void get_segment(struct kvm_vcpu *vcpu,
2781                         struct kvm_segment *var, int seg)
2782 {
2783         return kvm_x86_ops->get_segment(vcpu, var, seg);
2784 }
2785
2786 void kvm_get_cs_db_l_bits(struct kvm_vcpu *vcpu, int *db, int *l)
2787 {
2788         struct kvm_segment cs;
2789
2790         get_segment(vcpu, &cs, VCPU_SREG_CS);
2791         *db = cs.db;
2792         *l = cs.l;
2793 }
2794 EXPORT_SYMBOL_GPL(kvm_get_cs_db_l_bits);
2795
2796 int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
2797                                   struct kvm_sregs *sregs)
2798 {
2799         struct descriptor_table dt;
2800         int pending_vec;
2801
2802         vcpu_load(vcpu);
2803
2804         get_segment(vcpu, &sregs->cs, VCPU_SREG_CS);
2805         get_segment(vcpu, &sregs->ds, VCPU_SREG_DS);
2806         get_segment(vcpu, &sregs->es, VCPU_SREG_ES);
2807         get_segment(vcpu, &sregs->fs, VCPU_SREG_FS);
2808         get_segment(vcpu, &sregs->gs, VCPU_SREG_GS);
2809         get_segment(vcpu, &sregs->ss, VCPU_SREG_SS);
2810
2811         get_segment(vcpu, &sregs->tr, VCPU_SREG_TR);
2812         get_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR);
2813
2814         kvm_x86_ops->get_idt(vcpu, &dt);
2815         sregs->idt.limit = dt.limit;
2816         sregs->idt.base = dt.base;
2817         kvm_x86_ops->get_gdt(vcpu, &dt);
2818         sregs->gdt.limit = dt.limit;
2819         sregs->gdt.base = dt.base;
2820
2821         kvm_x86_ops->decache_cr4_guest_bits(vcpu);
2822         sregs->cr0 = vcpu->arch.cr0;
2823         sregs->cr2 = vcpu->arch.cr2;
2824         sregs->cr3 = vcpu->arch.cr3;
2825         sregs->cr4 = vcpu->arch.cr4;
2826         sregs->cr8 = get_cr8(vcpu);
2827         sregs->efer = vcpu->arch.shadow_efer;
2828         sregs->apic_base = kvm_get_apic_base(vcpu);
2829
2830         if (irqchip_in_kernel(vcpu->kvm)) {
2831                 memset(sregs->interrupt_bitmap, 0,
2832                        sizeof sregs->interrupt_bitmap);
2833                 pending_vec = kvm_x86_ops->get_irq(vcpu);
2834                 if (pending_vec >= 0)
2835                         set_bit(pending_vec,
2836                                 (unsigned long *)sregs->interrupt_bitmap);
2837         } else
2838                 memcpy(sregs->interrupt_bitmap, vcpu->arch.irq_pending,
2839                        sizeof sregs->interrupt_bitmap);
2840
2841         vcpu_put(vcpu);
2842
2843         return 0;
2844 }
2845
2846 static void set_segment(struct kvm_vcpu *vcpu,
2847                         struct kvm_segment *var, int seg)
2848 {
2849         return kvm_x86_ops->set_segment(vcpu, var, seg);
2850 }
2851
2852 int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
2853                                   struct kvm_sregs *sregs)
2854 {
2855         int mmu_reset_needed = 0;
2856         int i, pending_vec, max_bits;
2857         struct descriptor_table dt;
2858
2859         vcpu_load(vcpu);
2860
2861         dt.limit = sregs->idt.limit;
2862         dt.base = sregs->idt.base;
2863         kvm_x86_ops->set_idt(vcpu, &dt);
2864         dt.limit = sregs->gdt.limit;
2865         dt.base = sregs->gdt.base;
2866         kvm_x86_ops->set_gdt(vcpu, &dt);
2867
2868         vcpu->arch.cr2 = sregs->cr2;
2869         mmu_reset_needed |= vcpu->arch.cr3 != sregs->cr3;
2870         vcpu->arch.cr3 = sregs->cr3;
2871
2872         set_cr8(vcpu, sregs->cr8);
2873
2874         mmu_reset_needed |= vcpu->arch.shadow_efer != sregs->efer;
2875         kvm_x86_ops->set_efer(vcpu, sregs->efer);
2876         kvm_set_apic_base(vcpu, sregs->apic_base);
2877
2878         kvm_x86_ops->decache_cr4_guest_bits(vcpu);
2879
2880         mmu_reset_needed |= vcpu->arch.cr0 != sregs->cr0;
2881         kvm_x86_ops->set_cr0(vcpu, sregs->cr0);
2882         vcpu->arch.cr0 = sregs->cr0;
2883
2884         mmu_reset_needed |= vcpu->arch.cr4 != sregs->cr4;
2885         kvm_x86_ops->set_cr4(vcpu, sregs->cr4);
2886         if (!is_long_mode(vcpu) && is_pae(vcpu))
2887                 load_pdptrs(vcpu, vcpu->arch.cr3);
2888
2889         if (mmu_reset_needed)
2890                 kvm_mmu_reset_context(vcpu);
2891
2892         if (!irqchip_in_kernel(vcpu->kvm)) {
2893                 memcpy(vcpu->arch.irq_pending, sregs->interrupt_bitmap,
2894                        sizeof vcpu->arch.irq_pending);
2895                 vcpu->arch.irq_summary = 0;
2896                 for (i = 0; i < ARRAY_SIZE(vcpu->arch.irq_pending); ++i)
2897                         if (vcpu->arch.irq_pending[i])
2898                                 __set_bit(i, &vcpu->arch.irq_summary);
2899         } else {
2900                 max_bits = (sizeof sregs->interrupt_bitmap) << 3;
2901                 pending_vec = find_first_bit(
2902                         (const unsigned long *)sregs->interrupt_bitmap,
2903                         max_bits);
2904                 /* Only pending external irq is handled here */
2905                 if (pending_vec < max_bits) {
2906                         kvm_x86_ops->set_irq(vcpu, pending_vec);
2907                         pr_debug("Set back pending irq %d\n",
2908                                  pending_vec);
2909                 }
2910         }
2911
2912         set_segment(vcpu, &sregs->cs, VCPU_SREG_CS);
2913         set_segment(vcpu, &sregs->ds, VCPU_SREG_DS);
2914         set_segment(vcpu, &sregs->es, VCPU_SREG_ES);
2915         set_segment(vcpu, &sregs->fs, VCPU_SREG_FS);
2916         set_segment(vcpu, &sregs->gs, VCPU_SREG_GS);
2917         set_segment(vcpu, &sregs->ss, VCPU_SREG_SS);
2918
2919         set_segment(vcpu, &sregs->tr, VCPU_SREG_TR);
2920         set_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR);
2921
2922         vcpu_put(vcpu);
2923
2924         return 0;
2925 }
2926
2927 int kvm_arch_vcpu_ioctl_debug_guest(struct kvm_vcpu *vcpu,
2928                                     struct kvm_debug_guest *dbg)
2929 {
2930         int r;
2931
2932         vcpu_load(vcpu);
2933
2934         r = kvm_x86_ops->set_guest_debug(vcpu, dbg);
2935
2936         vcpu_put(vcpu);
2937
2938         return r;
2939 }
2940
2941 /*
2942  * fxsave fpu state.  Taken from x86_64/processor.h.  To be killed when
2943  * we have asm/x86/processor.h
2944  */
2945 struct fxsave {
2946         u16     cwd;
2947         u16     swd;
2948         u16     twd;
2949         u16     fop;
2950         u64     rip;
2951         u64     rdp;
2952         u32     mxcsr;
2953         u32     mxcsr_mask;
2954         u32     st_space[32];   /* 8*16 bytes for each FP-reg = 128 bytes */
2955 #ifdef CONFIG_X86_64
2956         u32     xmm_space[64];  /* 16*16 bytes for each XMM-reg = 256 bytes */
2957 #else
2958         u32     xmm_space[32];  /* 8*16 bytes for each XMM-reg = 128 bytes */
2959 #endif
2960 };
2961
2962 /*
2963  * Translate a guest virtual address to a guest physical address.
2964  */
2965 int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
2966                                     struct kvm_translation *tr)
2967 {
2968         unsigned long vaddr = tr->linear_address;
2969         gpa_t gpa;
2970
2971         vcpu_load(vcpu);
2972         down_read(&vcpu->kvm->slots_lock);
2973         gpa = vcpu->arch.mmu.gva_to_gpa(vcpu, vaddr);
2974         up_read(&vcpu->kvm->slots_lock);
2975         tr->physical_address = gpa;
2976         tr->valid = gpa != UNMAPPED_GVA;
2977         tr->writeable = 1;
2978         tr->usermode = 0;
2979         vcpu_put(vcpu);
2980
2981         return 0;
2982 }
2983
2984 int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
2985 {
2986         struct fxsave *fxsave = (struct fxsave *)&vcpu->arch.guest_fx_image;
2987
2988         vcpu_load(vcpu);
2989
2990         memcpy(fpu->fpr, fxsave->st_space, 128);
2991         fpu->fcw = fxsave->cwd;
2992         fpu->fsw = fxsave->swd;
2993         fpu->ftwx = fxsave->twd;
2994         fpu->last_opcode = fxsave->fop;
2995         fpu->last_ip = fxsave->rip;
2996         fpu->last_dp = fxsave->rdp;
2997         memcpy(fpu->xmm, fxsave->xmm_space, sizeof fxsave->xmm_space);
2998
2999         vcpu_put(vcpu);
3000
3001         return 0;
3002 }
3003
3004 int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
3005 {
3006         struct fxsave *fxsave = (struct fxsave *)&vcpu->arch.guest_fx_image;
3007
3008         vcpu_load(vcpu);
3009
3010         memcpy(fxsave->st_space, fpu->fpr, 128);
3011         fxsave->cwd = fpu->fcw;
3012         fxsave->swd = fpu->fsw;
3013         fxsave->twd = fpu->ftwx;
3014         fxsave->fop = fpu->last_opcode;
3015         fxsave->rip = fpu->last_ip;
3016         fxsave->rdp = fpu->last_dp;
3017         memcpy(fxsave->xmm_space, fpu->xmm, sizeof fxsave->xmm_space);
3018
3019         vcpu_put(vcpu);
3020
3021         return 0;
3022 }
3023
3024 void fx_init(struct kvm_vcpu *vcpu)
3025 {
3026         unsigned after_mxcsr_mask;
3027
3028         /* Initialize guest FPU by resetting ours and saving into guest's */
3029         preempt_disable();
3030         fx_save(&vcpu->arch.host_fx_image);
3031         fpu_init();
3032         fx_save(&vcpu->arch.guest_fx_image);
3033         fx_restore(&vcpu->arch.host_fx_image);
3034         preempt_enable();
3035
3036         vcpu->arch.cr0 |= X86_CR0_ET;
3037         after_mxcsr_mask = offsetof(struct i387_fxsave_struct, st_space);
3038         vcpu->arch.guest_fx_image.mxcsr = 0x1f80;
3039         memset((void *)&vcpu->arch.guest_fx_image + after_mxcsr_mask,
3040                0, sizeof(struct i387_fxsave_struct) - after_mxcsr_mask);
3041 }
3042 EXPORT_SYMBOL_GPL(fx_init);
3043
3044 void kvm_load_guest_fpu(struct kvm_vcpu *vcpu)
3045 {
3046         if (!vcpu->fpu_active || vcpu->guest_fpu_loaded)
3047                 return;
3048
3049         vcpu->guest_fpu_loaded = 1;
3050         fx_save(&vcpu->arch.host_fx_image);
3051         fx_restore(&vcpu->arch.guest_fx_image);
3052 }
3053 EXPORT_SYMBOL_GPL(kvm_load_guest_fpu);
3054
3055 void kvm_put_guest_fpu(struct kvm_vcpu *vcpu)
3056 {
3057         if (!vcpu->guest_fpu_loaded)
3058                 return;
3059
3060         vcpu->guest_fpu_loaded = 0;
3061         fx_save(&vcpu->arch.guest_fx_image);
3062         fx_restore(&vcpu->arch.host_fx_image);
3063         ++vcpu->stat.fpu_reload;
3064 }
3065 EXPORT_SYMBOL_GPL(kvm_put_guest_fpu);
3066
3067 void kvm_arch_vcpu_free(struct kvm_vcpu *vcpu)
3068 {
3069         kvm_x86_ops->vcpu_free(vcpu);
3070 }
3071
3072 struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm,
3073                                                 unsigned int id)
3074 {
3075         return kvm_x86_ops->vcpu_create(kvm, id);
3076 }
3077
3078 int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu)
3079 {
3080         int r;
3081
3082         /* We do fxsave: this must be aligned. */
3083         BUG_ON((unsigned long)&vcpu->arch.host_fx_image & 0xF);
3084
3085         vcpu_load(vcpu);
3086         r = kvm_arch_vcpu_reset(vcpu);
3087         if (r == 0)
3088                 r = kvm_mmu_setup(vcpu);
3089         vcpu_put(vcpu);
3090         if (r < 0)
3091                 goto free_vcpu;
3092
3093         return 0;
3094 free_vcpu:
3095         kvm_x86_ops->vcpu_free(vcpu);
3096         return r;
3097 }
3098
3099 void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
3100 {
3101         vcpu_load(vcpu);
3102         kvm_mmu_unload(vcpu);
3103         vcpu_put(vcpu);
3104
3105         kvm_x86_ops->vcpu_free(vcpu);
3106 }
3107
3108 int kvm_arch_vcpu_reset(struct kvm_vcpu *vcpu)
3109 {
3110         return kvm_x86_ops->vcpu_reset(vcpu);
3111 }
3112
3113 void kvm_arch_hardware_enable(void *garbage)
3114 {
3115         kvm_x86_ops->hardware_enable(garbage);
3116 }
3117
3118 void kvm_arch_hardware_disable(void *garbage)
3119 {
3120         kvm_x86_ops->hardware_disable(garbage);
3121 }
3122
3123 int kvm_arch_hardware_setup(void)
3124 {
3125         return kvm_x86_ops->hardware_setup();
3126 }
3127
3128 void kvm_arch_hardware_unsetup(void)
3129 {
3130         kvm_x86_ops->hardware_unsetup();
3131 }
3132
3133 void kvm_arch_check_processor_compat(void *rtn)
3134 {
3135         kvm_x86_ops->check_processor_compatibility(rtn);
3136 }
3137
3138 int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
3139 {
3140         struct page *page;
3141         struct kvm *kvm;
3142         int r;
3143
3144         BUG_ON(vcpu->kvm == NULL);
3145         kvm = vcpu->kvm;
3146
3147         vcpu->arch.mmu.root_hpa = INVALID_PAGE;
3148         if (!irqchip_in_kernel(kvm) || vcpu->vcpu_id == 0)
3149                 vcpu->arch.mp_state = VCPU_MP_STATE_RUNNABLE;
3150         else
3151                 vcpu->arch.mp_state = VCPU_MP_STATE_UNINITIALIZED;
3152
3153         page = alloc_page(GFP_KERNEL | __GFP_ZERO);
3154         if (!page) {
3155                 r = -ENOMEM;
3156                 goto fail;
3157         }
3158         vcpu->arch.pio_data = page_address(page);
3159
3160         r = kvm_mmu_create(vcpu);
3161         if (r < 0)
3162                 goto fail_free_pio_data;
3163
3164         if (irqchip_in_kernel(kvm)) {
3165                 r = kvm_create_lapic(vcpu);
3166                 if (r < 0)
3167                         goto fail_mmu_destroy;
3168         }
3169
3170         return 0;
3171
3172 fail_mmu_destroy:
3173         kvm_mmu_destroy(vcpu);
3174 fail_free_pio_data:
3175         free_page((unsigned long)vcpu->arch.pio_data);
3176 fail:
3177         return r;
3178 }
3179
3180 void kvm_arch_vcpu_uninit(struct kvm_vcpu *vcpu)
3181 {
3182         kvm_free_lapic(vcpu);
3183         kvm_mmu_destroy(vcpu);
3184         free_page((unsigned long)vcpu->arch.pio_data);
3185 }
3186
3187 struct  kvm *kvm_arch_create_vm(void)
3188 {
3189         struct kvm *kvm = kzalloc(sizeof(struct kvm), GFP_KERNEL);
3190
3191         if (!kvm)
3192                 return ERR_PTR(-ENOMEM);
3193
3194         INIT_LIST_HEAD(&kvm->arch.active_mmu_pages);
3195
3196         return kvm;
3197 }
3198
3199 static void kvm_unload_vcpu_mmu(struct kvm_vcpu *vcpu)
3200 {
3201         vcpu_load(vcpu);
3202         kvm_mmu_unload(vcpu);
3203         vcpu_put(vcpu);
3204 }
3205
3206 static void kvm_free_vcpus(struct kvm *kvm)
3207 {
3208         unsigned int i;
3209
3210         /*
3211          * Unpin any mmu pages first.
3212          */
3213         for (i = 0; i < KVM_MAX_VCPUS; ++i)
3214                 if (kvm->vcpus[i])
3215                         kvm_unload_vcpu_mmu(kvm->vcpus[i]);
3216         for (i = 0; i < KVM_MAX_VCPUS; ++i) {
3217                 if (kvm->vcpus[i]) {
3218                         kvm_arch_vcpu_free(kvm->vcpus[i]);
3219                         kvm->vcpus[i] = NULL;
3220                 }
3221         }
3222
3223 }
3224
3225 void kvm_arch_destroy_vm(struct kvm *kvm)
3226 {
3227         kfree(kvm->arch.vpic);
3228         kfree(kvm->arch.vioapic);
3229         kvm_free_vcpus(kvm);
3230         kvm_free_physmem(kvm);
3231         kfree(kvm);
3232 }
3233
3234 int kvm_arch_set_memory_region(struct kvm *kvm,
3235                                 struct kvm_userspace_memory_region *mem,
3236                                 struct kvm_memory_slot old,
3237                                 int user_alloc)
3238 {
3239         int npages = mem->memory_size >> PAGE_SHIFT;
3240         struct kvm_memory_slot *memslot = &kvm->memslots[mem->slot];
3241
3242         /*To keep backward compatibility with older userspace,
3243          *x86 needs to hanlde !user_alloc case.
3244          */
3245         if (!user_alloc) {
3246                 if (npages && !old.rmap) {
3247                         down_write(&current->mm->mmap_sem);
3248                         memslot->userspace_addr = do_mmap(NULL, 0,
3249                                                      npages * PAGE_SIZE,
3250                                                      PROT_READ | PROT_WRITE,
3251                                                      MAP_SHARED | MAP_ANONYMOUS,
3252                                                      0);
3253                         up_write(&current->mm->mmap_sem);
3254
3255                         if (IS_ERR((void *)memslot->userspace_addr))
3256                                 return PTR_ERR((void *)memslot->userspace_addr);
3257                 } else {
3258                         if (!old.user_alloc && old.rmap) {
3259                                 int ret;
3260
3261                                 down_write(&current->mm->mmap_sem);
3262                                 ret = do_munmap(current->mm, old.userspace_addr,
3263                                                 old.npages * PAGE_SIZE);
3264                                 up_write(&current->mm->mmap_sem);
3265                                 if (ret < 0)
3266                                         printk(KERN_WARNING
3267                                        "kvm_vm_ioctl_set_memory_region: "
3268                                        "failed to munmap memory\n");
3269                         }
3270                 }
3271         }
3272
3273         if (!kvm->arch.n_requested_mmu_pages) {
3274                 unsigned int nr_mmu_pages = kvm_mmu_calculate_mmu_pages(kvm);
3275                 kvm_mmu_change_mmu_pages(kvm, nr_mmu_pages);
3276         }
3277
3278         kvm_mmu_slot_remove_write_access(kvm, mem->slot);
3279         kvm_flush_remote_tlbs(kvm);
3280
3281         return 0;
3282 }
3283
3284 int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
3285 {
3286         return vcpu->arch.mp_state == VCPU_MP_STATE_RUNNABLE
3287                || vcpu->arch.mp_state == VCPU_MP_STATE_SIPI_RECEIVED;
3288 }
3289
3290 static void vcpu_kick_intr(void *info)
3291 {
3292 #ifdef DEBUG
3293         struct kvm_vcpu *vcpu = (struct kvm_vcpu *)info;
3294         printk(KERN_DEBUG "vcpu_kick_intr %p \n", vcpu);
3295 #endif
3296 }
3297
3298 void kvm_vcpu_kick(struct kvm_vcpu *vcpu)
3299 {
3300         int ipi_pcpu = vcpu->cpu;
3301
3302         if (waitqueue_active(&vcpu->wq)) {
3303                 wake_up_interruptible(&vcpu->wq);
3304                 ++vcpu->stat.halt_wakeup;
3305         }
3306         if (vcpu->guest_mode)
3307                 smp_call_function_single(ipi_pcpu, vcpu_kick_intr, vcpu, 0, 0);
3308 }