err.no Git - linux-2.6/blob - drivers/kvm/kvm_main.c

   1 /*
   2  * Kernel-based Virtual Machine driver for Linux
   3  *
   4  * This module enables machines with Intel VT-x extensions to run virtual
   5  * machines without emulation or binary translation.
   6  *
   7  * Copyright (C) 2006 Qumranet, Inc.
   8  *
   9  * Authors:
  10  *   Avi Kivity   <avi@qumranet.com>
  11  *   Yaniv Kamay  <yaniv@qumranet.com>
  12  *
  13  * This work is licensed under the terms of the GNU GPL, version 2.  See
  14  * the COPYING file in the top-level directory.
  15  *
  16  */
  17
  18 #include "kvm.h"
  19 #include "x86_emulate.h"
  20 #include "segment_descriptor.h"
  21
  22 #include <linux/kvm.h>
  23 #include <linux/module.h>
  24 #include <linux/errno.h>
  25 #include <linux/percpu.h>
  26 #include <linux/gfp.h>
  27 #include <linux/mm.h>
  28 #include <linux/miscdevice.h>
  29 #include <linux/vmalloc.h>
  30 #include <linux/reboot.h>
  31 #include <linux/debugfs.h>
  32 #include <linux/highmem.h>
  33 #include <linux/file.h>
  34 #include <linux/sysdev.h>
  35 #include <linux/cpu.h>
  36 #include <linux/sched.h>
  37 #include <linux/cpumask.h>
  38 #include <linux/smp.h>
  39 #include <linux/anon_inodes.h>
  40
  41 #include <asm/processor.h>
  42 #include <asm/msr.h>
  43 #include <asm/io.h>
  44 #include <asm/uaccess.h>
  45 #include <asm/desc.h>
  46
  47 MODULE_AUTHOR("Qumranet");
  48 MODULE_LICENSE("GPL");
  49
  50 static DEFINE_SPINLOCK(kvm_lock);
  51 static LIST_HEAD(vm_list);
  52
  53 static cpumask_t cpus_hardware_enabled;
  54
  55 struct kvm_arch_ops *kvm_arch_ops;
  56
  57 static __read_mostly struct preempt_ops kvm_preempt_ops;
  58
  59 #define STAT_OFFSET(x) offsetof(struct kvm_vcpu, stat.x)
  60
  61 static struct kvm_stats_debugfs_item {
  62         const char *name;
  63         int offset;
  64         struct dentry *dentry;
  65 } debugfs_entries[] = {
  66         { "pf_fixed", STAT_OFFSET(pf_fixed) },
  67         { "pf_guest", STAT_OFFSET(pf_guest) },
  68         { "tlb_flush", STAT_OFFSET(tlb_flush) },
  69         { "invlpg", STAT_OFFSET(invlpg) },
  70         { "exits", STAT_OFFSET(exits) },
  71         { "io_exits", STAT_OFFSET(io_exits) },
  72         { "mmio_exits", STAT_OFFSET(mmio_exits) },
  73         { "signal_exits", STAT_OFFSET(signal_exits) },
  74         { "irq_window", STAT_OFFSET(irq_window_exits) },
  75         { "halt_exits", STAT_OFFSET(halt_exits) },
  76         { "request_irq", STAT_OFFSET(request_irq_exits) },
  77         { "irq_exits", STAT_OFFSET(irq_exits) },
  78         { "light_exits", STAT_OFFSET(light_exits) },
  79         { "efer_reload", STAT_OFFSET(efer_reload) },
  80         { NULL }
  81 };
  82
  83 static struct dentry *debugfs_dir;
  84
  85 #define MAX_IO_MSRS 256
  86
  87 #define CR0_RESERVED_BITS                                               \
  88         (~(unsigned long)(X86_CR0_PE | X86_CR0_MP | X86_CR0_EM | X86_CR0_TS \
  89                           | X86_CR0_ET | X86_CR0_NE | X86_CR0_WP | X86_CR0_AM \
  90                           | X86_CR0_NW | X86_CR0_CD | X86_CR0_PG))
  91 #define CR4_RESERVED_BITS                                               \
  92         (~(unsigned long)(X86_CR4_VME | X86_CR4_PVI | X86_CR4_TSD | X86_CR4_DE\
  93                           | X86_CR4_PSE | X86_CR4_PAE | X86_CR4_MCE     \
  94                           | X86_CR4_PGE | X86_CR4_PCE | X86_CR4_OSFXSR  \
  95                           | X86_CR4_OSXMMEXCPT | X86_CR4_VMXE))
  96
  97 #define CR8_RESERVED_BITS (~(unsigned long)X86_CR8_TPR)
  98 #define EFER_RESERVED_BITS 0xfffffffffffff2fe
  99
 100 #ifdef CONFIG_X86_64
 101 // LDT or TSS descriptor in the GDT. 16 bytes.
 102 struct segment_descriptor_64 {
 103         struct segment_descriptor s;
 104         u32 base_higher;
 105         u32 pad_zero;
 106 };
 107
 108 #endif
 109
 110 static long kvm_vcpu_ioctl(struct file *file, unsigned int ioctl,
 111                            unsigned long arg);
 112
 113 unsigned long segment_base(u16 selector)
 114 {
 115         struct descriptor_table gdt;
 116         struct segment_descriptor *d;
 117         unsigned long table_base;
 118         typedef unsigned long ul;
 119         unsigned long v;
 120
 121         if (selector == 0)
 122                 return 0;
 123
 124         asm ("sgdt %0" : "=m"(gdt));
 125         table_base = gdt.base;
 126
 127         if (selector & 4) {           /* from ldt */
 128                 u16 ldt_selector;
 129
 130                 asm ("sldt %0" : "=g"(ldt_selector));
 131                 table_base = segment_base(ldt_selector);
 132         }
 133         d = (struct segment_descriptor *)(table_base + (selector & ~7));
 134         v = d->base_low | ((ul)d->base_mid << 16) | ((ul)d->base_high << 24);
 135 #ifdef CONFIG_X86_64
 136         if (d->system == 0
 137             && (d->type == 2 || d->type == 9 || d->type == 11))
 138                 v |= ((ul)((struct segment_descriptor_64 *)d)->base_higher) << 32;
 139 #endif
 140         return v;
 141 }
 142 EXPORT_SYMBOL_GPL(segment_base);
 143
 144 static inline int valid_vcpu(int n)
 145 {
 146         return likely(n >= 0 && n < KVM_MAX_VCPUS);
 147 }
 148
 149 void kvm_load_guest_fpu(struct kvm_vcpu *vcpu)
 150 {
 151         if (!vcpu->fpu_active || vcpu->guest_fpu_loaded)
 152                 return;
 153
 154         vcpu->guest_fpu_loaded = 1;
 155         fx_save(vcpu->host_fx_image);
 156         fx_restore(vcpu->guest_fx_image);
 157 }
 158 EXPORT_SYMBOL_GPL(kvm_load_guest_fpu);
 159
 160 void kvm_put_guest_fpu(struct kvm_vcpu *vcpu)
 161 {
 162         if (!vcpu->guest_fpu_loaded)
 163                 return;
 164
 165         vcpu->guest_fpu_loaded = 0;
 166         fx_save(vcpu->guest_fx_image);
 167         fx_restore(vcpu->host_fx_image);
 168 }
 169 EXPORT_SYMBOL_GPL(kvm_put_guest_fpu);
 170
 171 /*
 172  * Switches to specified vcpu, until a matching vcpu_put()
 173  */
 174 static void vcpu_load(struct kvm_vcpu *vcpu)
 175 {
 176         int cpu;
 177
 178         mutex_lock(&vcpu->mutex);
 179         cpu = get_cpu();
 180         preempt_notifier_register(&vcpu->preempt_notifier);
 181         kvm_arch_ops->vcpu_load(vcpu, cpu);
 182         put_cpu();
 183 }
 184
 185 static void vcpu_put(struct kvm_vcpu *vcpu)
 186 {
 187         preempt_disable();
 188         kvm_arch_ops->vcpu_put(vcpu);
 189         preempt_notifier_unregister(&vcpu->preempt_notifier);
 190         preempt_enable();
 191         mutex_unlock(&vcpu->mutex);
 192 }
 193
 194 static void ack_flush(void *_completed)
 195 {
 196         atomic_t *completed = _completed;
 197
 198         atomic_inc(completed);
 199 }
 200
 201 void kvm_flush_remote_tlbs(struct kvm *kvm)
 202 {
 203         int i, cpu, needed;
 204         cpumask_t cpus;
 205         struct kvm_vcpu *vcpu;
 206         atomic_t completed;
 207
 208         atomic_set(&completed, 0);
 209         cpus_clear(cpus);
 210         needed = 0;
 211         for (i = 0; i < KVM_MAX_VCPUS; ++i) {
 212                 vcpu = kvm->vcpus[i];
 213                 if (!vcpu)
 214                         continue;
 215                 if (test_and_set_bit(KVM_TLB_FLUSH, &vcpu->requests))
 216                         continue;
 217                 cpu = vcpu->cpu;
 218                 if (cpu != -1 && cpu != raw_smp_processor_id())
 219                         if (!cpu_isset(cpu, cpus)) {
 220                                 cpu_set(cpu, cpus);
 221                                 ++needed;
 222                         }
 223         }
 224
 225         /*
 226          * We really want smp_call_function_mask() here.  But that's not
 227          * available, so ipi all cpus in parallel and wait for them
 228          * to complete.
 229          */
 230         for (cpu = first_cpu(cpus); cpu != NR_CPUS; cpu = next_cpu(cpu, cpus))
 231                 smp_call_function_single(cpu, ack_flush, &completed, 1, 0);
 232         while (atomic_read(&completed) != needed) {
 233                 cpu_relax();
 234                 barrier();
 235         }
 236 }
 237
 238 int kvm_vcpu_init(struct kvm_vcpu *vcpu, struct kvm *kvm, unsigned id)
 239 {
 240         struct page *page;
 241         int r;
 242
 243         mutex_init(&vcpu->mutex);
 244         vcpu->cpu = -1;
 245         vcpu->mmu.root_hpa = INVALID_PAGE;
 246         vcpu->kvm = kvm;
 247         vcpu->vcpu_id = id;
 248
 249         page = alloc_page(GFP_KERNEL | __GFP_ZERO);
 250         if (!page) {
 251                 r = -ENOMEM;
 252                 goto fail;
 253         }
 254         vcpu->run = page_address(page);
 255
 256         page = alloc_page(GFP_KERNEL | __GFP_ZERO);
 257         if (!page) {
 258                 r = -ENOMEM;
 259                 goto fail_free_run;
 260         }
 261         vcpu->pio_data = page_address(page);
 262
 263         vcpu->host_fx_image = (char*)ALIGN((hva_t)vcpu->fx_buf,
 264                                            FX_IMAGE_ALIGN);
 265         vcpu->guest_fx_image = vcpu->host_fx_image + FX_IMAGE_SIZE;
 266
 267         r = kvm_mmu_create(vcpu);
 268         if (r < 0)
 269                 goto fail_free_pio_data;
 270
 271         return 0;
 272
 273 fail_free_pio_data:
 274         free_page((unsigned long)vcpu->pio_data);
 275 fail_free_run:
 276         free_page((unsigned long)vcpu->run);
 277 fail:
 278         return -ENOMEM;
 279 }
 280 EXPORT_SYMBOL_GPL(kvm_vcpu_init);
 281
 282 void kvm_vcpu_uninit(struct kvm_vcpu *vcpu)
 283 {
 284         kvm_mmu_destroy(vcpu);
 285         free_page((unsigned long)vcpu->pio_data);
 286         free_page((unsigned long)vcpu->run);
 287 }
 288 EXPORT_SYMBOL_GPL(kvm_vcpu_uninit);
 289
 290 static struct kvm *kvm_create_vm(void)
 291 {
 292         struct kvm *kvm = kzalloc(sizeof(struct kvm), GFP_KERNEL);
 293
 294         if (!kvm)
 295                 return ERR_PTR(-ENOMEM);
 296
 297         kvm_io_bus_init(&kvm->pio_bus);
 298         mutex_init(&kvm->lock);
 299         INIT_LIST_HEAD(&kvm->active_mmu_pages);
 300         kvm_io_bus_init(&kvm->mmio_bus);
 301         spin_lock(&kvm_lock);
 302         list_add(&kvm->vm_list, &vm_list);
 303         spin_unlock(&kvm_lock);
 304         return kvm;
 305 }
 306
 307 static int kvm_dev_open(struct inode *inode, struct file *filp)
 308 {
 309         return 0;
 310 }
 311
 312 /*
 313  * Free any memory in @free but not in @dont.
 314  */
 315 static void kvm_free_physmem_slot(struct kvm_memory_slot *free,
 316                                   struct kvm_memory_slot *dont)
 317 {
 318         int i;
 319
 320         if (!dont || free->phys_mem != dont->phys_mem)
 321                 if (free->phys_mem) {
 322                         for (i = 0; i < free->npages; ++i)
 323                                 if (free->phys_mem[i])
 324                                         __free_page(free->phys_mem[i]);
 325                         vfree(free->phys_mem);
 326                 }
 327
 328         if (!dont || free->dirty_bitmap != dont->dirty_bitmap)
 329                 vfree(free->dirty_bitmap);
 330
 331         free->phys_mem = NULL;
 332         free->npages = 0;
 333         free->dirty_bitmap = NULL;
 334 }
 335
 336 static void kvm_free_physmem(struct kvm *kvm)
 337 {
 338         int i;
 339
 340         for (i = 0; i < kvm->nmemslots; ++i)
 341                 kvm_free_physmem_slot(&kvm->memslots[i], NULL);
 342 }
 343
 344 static void free_pio_guest_pages(struct kvm_vcpu *vcpu)
 345 {
 346         int i;
 347
 348         for (i = 0; i < ARRAY_SIZE(vcpu->pio.guest_pages); ++i)
 349                 if (vcpu->pio.guest_pages[i]) {
 350                         __free_page(vcpu->pio.guest_pages[i]);
 351                         vcpu->pio.guest_pages[i] = NULL;
 352                 }
 353 }
 354
 355 static void kvm_unload_vcpu_mmu(struct kvm_vcpu *vcpu)
 356 {
 357         vcpu_load(vcpu);
 358         kvm_mmu_unload(vcpu);
 359         vcpu_put(vcpu);
 360 }
 361
 362 static void kvm_free_vcpus(struct kvm *kvm)
 363 {
 364         unsigned int i;
 365
 366         /*
 367          * Unpin any mmu pages first.
 368          */
 369         for (i = 0; i < KVM_MAX_VCPUS; ++i)
 370                 if (kvm->vcpus[i])
 371                         kvm_unload_vcpu_mmu(kvm->vcpus[i]);
 372         for (i = 0; i < KVM_MAX_VCPUS; ++i) {
 373                 if (kvm->vcpus[i]) {
 374                         kvm_arch_ops->vcpu_free(kvm->vcpus[i]);
 375                         kvm->vcpus[i] = NULL;
 376                 }
 377         }
 378
 379 }
 380
 381 static int kvm_dev_release(struct inode *inode, struct file *filp)
 382 {
 383         return 0;
 384 }
 385
 386 static void kvm_destroy_vm(struct kvm *kvm)
 387 {
 388         spin_lock(&kvm_lock);
 389         list_del(&kvm->vm_list);
 390         spin_unlock(&kvm_lock);
 391         kvm_io_bus_destroy(&kvm->pio_bus);
 392         kvm_io_bus_destroy(&kvm->mmio_bus);
 393         kvm_free_vcpus(kvm);
 394         kvm_free_physmem(kvm);
 395         kfree(kvm);
 396 }
 397
 398 static int kvm_vm_release(struct inode *inode, struct file *filp)
 399 {
 400         struct kvm *kvm = filp->private_data;
 401
 402         kvm_destroy_vm(kvm);
 403         return 0;
 404 }
 405
 406 static void inject_gp(struct kvm_vcpu *vcpu)
 407 {
 408         kvm_arch_ops->inject_gp(vcpu, 0);
 409 }
 410
 411 /*
 412  * Load the pae pdptrs.  Return true is they are all valid.
 413  */
 414 static int load_pdptrs(struct kvm_vcpu *vcpu, unsigned long cr3)
 415 {
 416         gfn_t pdpt_gfn = cr3 >> PAGE_SHIFT;
 417         unsigned offset = ((cr3 & (PAGE_SIZE-1)) >> 5) << 2;
 418         int i;
 419         u64 *pdpt;
 420         int ret;
 421         struct page *page;
 422         u64 pdpte[ARRAY_SIZE(vcpu->pdptrs)];
 423
 424         mutex_lock(&vcpu->kvm->lock);
 425         page = gfn_to_page(vcpu->kvm, pdpt_gfn);
 426         if (!page) {
 427                 ret = 0;
 428                 goto out;
 429         }
 430
 431         pdpt = kmap_atomic(page, KM_USER0);
 432         memcpy(pdpte, pdpt+offset, sizeof(pdpte));
 433         kunmap_atomic(pdpt, KM_USER0);
 434
 435         for (i = 0; i < ARRAY_SIZE(pdpte); ++i) {
 436                 if ((pdpte[i] & 1) && (pdpte[i] & 0xfffffff0000001e6ull)) {
 437                         ret = 0;
 438                         goto out;
 439                 }
 440         }
 441         ret = 1;
 442
 443         memcpy(vcpu->pdptrs, pdpte, sizeof(vcpu->pdptrs));
 444 out:
 445         mutex_unlock(&vcpu->kvm->lock);
 446
 447         return ret;
 448 }
 449
 450 void set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
 451 {
 452         if (cr0 & CR0_RESERVED_BITS) {
 453                 printk(KERN_DEBUG "set_cr0: 0x%lx #GP, reserved bits 0x%lx\n",
 454                        cr0, vcpu->cr0);
 455                 inject_gp(vcpu);
 456                 return;
 457         }
 458
 459         if ((cr0 & X86_CR0_NW) && !(cr0 & X86_CR0_CD)) {
 460                 printk(KERN_DEBUG "set_cr0: #GP, CD == 0 && NW == 1\n");
 461                 inject_gp(vcpu);
 462                 return;
 463         }
 464
 465         if ((cr0 & X86_CR0_PG) && !(cr0 & X86_CR0_PE)) {
 466                 printk(KERN_DEBUG "set_cr0: #GP, set PG flag "
 467                        "and a clear PE flag\n");
 468                 inject_gp(vcpu);
 469                 return;
 470         }
 471
 472         if (!is_paging(vcpu) && (cr0 & X86_CR0_PG)) {
 473 #ifdef CONFIG_X86_64
 474                 if ((vcpu->shadow_efer & EFER_LME)) {
 475                         int cs_db, cs_l;
 476
 477                         if (!is_pae(vcpu)) {
 478                                 printk(KERN_DEBUG "set_cr0: #GP, start paging "
 479                                        "in long mode while PAE is disabled\n");
 480                                 inject_gp(vcpu);
 481                                 return;
 482                         }
 483                         kvm_arch_ops->get_cs_db_l_bits(vcpu, &cs_db, &cs_l);
 484                         if (cs_l) {
 485                                 printk(KERN_DEBUG "set_cr0: #GP, start paging "
 486                                        "in long mode while CS.L == 1\n");
 487                                 inject_gp(vcpu);
 488                                 return;
 489
 490                         }
 491                 } else
 492 #endif
 493                 if (is_pae(vcpu) && !load_pdptrs(vcpu, vcpu->cr3)) {
 494                         printk(KERN_DEBUG "set_cr0: #GP, pdptrs "
 495                                "reserved bits\n");
 496                         inject_gp(vcpu);
 497                         return;
 498                 }
 499
 500         }
 501
 502         kvm_arch_ops->set_cr0(vcpu, cr0);
 503         vcpu->cr0 = cr0;
 504
 505         mutex_lock(&vcpu->kvm->lock);
 506         kvm_mmu_reset_context(vcpu);
 507         mutex_unlock(&vcpu->kvm->lock);
 508         return;
 509 }
 510 EXPORT_SYMBOL_GPL(set_cr0);
 511
 512 void lmsw(struct kvm_vcpu *vcpu, unsigned long msw)
 513 {
 514         set_cr0(vcpu, (vcpu->cr0 & ~0x0ful) | (msw & 0x0f));
 515 }
 516 EXPORT_SYMBOL_GPL(lmsw);
 517
 518 void set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
 519 {
 520         if (cr4 & CR4_RESERVED_BITS) {
 521                 printk(KERN_DEBUG "set_cr4: #GP, reserved bits\n");
 522                 inject_gp(vcpu);
 523                 return;
 524         }
 525
 526         if (is_long_mode(vcpu)) {
 527                 if (!(cr4 & X86_CR4_PAE)) {
 528                         printk(KERN_DEBUG "set_cr4: #GP, clearing PAE while "
 529                                "in long mode\n");
 530                         inject_gp(vcpu);
 531                         return;
 532                 }
 533         } else if (is_paging(vcpu) && !is_pae(vcpu) && (cr4 & X86_CR4_PAE)
 534                    && !load_pdptrs(vcpu, vcpu->cr3)) {
 535                 printk(KERN_DEBUG "set_cr4: #GP, pdptrs reserved bits\n");
 536                 inject_gp(vcpu);
 537                 return;
 538         }
 539
 540         if (cr4 & X86_CR4_VMXE) {
 541                 printk(KERN_DEBUG "set_cr4: #GP, setting VMXE\n");
 542                 inject_gp(vcpu);
 543                 return;
 544         }
 545         kvm_arch_ops->set_cr4(vcpu, cr4);
 546         mutex_lock(&vcpu->kvm->lock);
 547         kvm_mmu_reset_context(vcpu);
 548         mutex_unlock(&vcpu->kvm->lock);
 549 }
 550 EXPORT_SYMBOL_GPL(set_cr4);
 551
 552 void set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3)
 553 {
 554         if (is_long_mode(vcpu)) {
 555                 if (cr3 & CR3_L_MODE_RESERVED_BITS) {
 556                         printk(KERN_DEBUG "set_cr3: #GP, reserved bits\n");
 557                         inject_gp(vcpu);
 558                         return;
 559                 }
 560         } else {
 561                 if (is_pae(vcpu)) {
 562                         if (cr3 & CR3_PAE_RESERVED_BITS) {
 563                                 printk(KERN_DEBUG
 564                                        "set_cr3: #GP, reserved bits\n");
 565                                 inject_gp(vcpu);
 566                                 return;
 567                         }
 568                         if (is_paging(vcpu) && !load_pdptrs(vcpu, cr3)) {
 569                                 printk(KERN_DEBUG "set_cr3: #GP, pdptrs "
 570                                        "reserved bits\n");
 571                                 inject_gp(vcpu);
 572                                 return;
 573                         }
 574                 } else {
 575                         if (cr3 & CR3_NONPAE_RESERVED_BITS) {
 576                                 printk(KERN_DEBUG
 577                                        "set_cr3: #GP, reserved bits\n");
 578                                 inject_gp(vcpu);
 579                                 return;
 580                         }
 581                 }
 582         }
 583
 584         vcpu->cr3 = cr3;
 585         mutex_lock(&vcpu->kvm->lock);
 586         /*
 587          * Does the new cr3 value map to physical memory? (Note, we
 588          * catch an invalid cr3 even in real-mode, because it would
 589          * cause trouble later on when we turn on paging anyway.)
 590          *
 591          * A real CPU would silently accept an invalid cr3 and would
 592          * attempt to use it - with largely undefined (and often hard
 593          * to debug) behavior on the guest side.
 594          */
 595         if (unlikely(!gfn_to_memslot(vcpu->kvm, cr3 >> PAGE_SHIFT)))
 596                 inject_gp(vcpu);
 597         else
 598                 vcpu->mmu.new_cr3(vcpu);
 599         mutex_unlock(&vcpu->kvm->lock);
 600 }
 601 EXPORT_SYMBOL_GPL(set_cr3);
 602
 603 void set_cr8(struct kvm_vcpu *vcpu, unsigned long cr8)
 604 {
 605         if (cr8 & CR8_RESERVED_BITS) {
 606                 printk(KERN_DEBUG "set_cr8: #GP, reserved bits 0x%lx\n", cr8);
 607                 inject_gp(vcpu);
 608                 return;
 609         }
 610         vcpu->cr8 = cr8;
 611 }
 612 EXPORT_SYMBOL_GPL(set_cr8);
 613
 614 void fx_init(struct kvm_vcpu *vcpu)
 615 {
 616         struct __attribute__ ((__packed__)) fx_image_s {
 617                 u16 control; //fcw
 618                 u16 status; //fsw
 619                 u16 tag; // ftw
 620                 u16 opcode; //fop
 621                 u64 ip; // fpu ip
 622                 u64 operand;// fpu dp
 623                 u32 mxcsr;
 624                 u32 mxcsr_mask;
 625
 626         } *fx_image;
 627
 628         /* Initialize guest FPU by resetting ours and saving into guest's */
 629         preempt_disable();
 630         fx_save(vcpu->host_fx_image);
 631         fpu_init();
 632         fx_save(vcpu->guest_fx_image);
 633         fx_restore(vcpu->host_fx_image);
 634         preempt_enable();
 635
 636         fx_image = (struct fx_image_s *)vcpu->guest_fx_image;
 637         fx_image->mxcsr = 0x1f80;
 638         memset(vcpu->guest_fx_image + sizeof(struct fx_image_s),
 639                0, FX_IMAGE_SIZE - sizeof(struct fx_image_s));
 640 }
 641 EXPORT_SYMBOL_GPL(fx_init);
 642
 643 /*
 644  * Allocate some memory and give it an address in the guest physical address
 645  * space.
 646  *
 647  * Discontiguous memory is allowed, mostly for framebuffers.
 648  */
 649 static int kvm_vm_ioctl_set_memory_region(struct kvm *kvm,
 650                                           struct kvm_memory_region *mem)
 651 {
 652         int r;
 653         gfn_t base_gfn;
 654         unsigned long npages;
 655         unsigned long i;
 656         struct kvm_memory_slot *memslot;
 657         struct kvm_memory_slot old, new;
 658         int memory_config_version;
 659
 660         r = -EINVAL;
 661         /* General sanity checks */
 662         if (mem->memory_size & (PAGE_SIZE - 1))
 663                 goto out;
 664         if (mem->guest_phys_addr & (PAGE_SIZE - 1))
 665                 goto out;
 666         if (mem->slot >= KVM_MEMORY_SLOTS)
 667                 goto out;
 668         if (mem->guest_phys_addr + mem->memory_size < mem->guest_phys_addr)
 669                 goto out;
 670
 671         memslot = &kvm->memslots[mem->slot];
 672         base_gfn = mem->guest_phys_addr >> PAGE_SHIFT;
 673         npages = mem->memory_size >> PAGE_SHIFT;
 674
 675         if (!npages)
 676                 mem->flags &= ~KVM_MEM_LOG_DIRTY_PAGES;
 677
 678 raced:
 679         mutex_lock(&kvm->lock);
 680
 681         memory_config_version = kvm->memory_config_version;
 682         new = old = *memslot;
 683
 684         new.base_gfn = base_gfn;
 685         new.npages = npages;
 686         new.flags = mem->flags;
 687
 688         /* Disallow changing a memory slot's size. */
 689         r = -EINVAL;
 690         if (npages && old.npages && npages != old.npages)
 691                 goto out_unlock;
 692
 693         /* Check for overlaps */
 694         r = -EEXIST;
 695         for (i = 0; i < KVM_MEMORY_SLOTS; ++i) {
 696                 struct kvm_memory_slot *s = &kvm->memslots[i];
 697
 698                 if (s == memslot)
 699                         continue;
 700                 if (!((base_gfn + npages <= s->base_gfn) ||
 701                       (base_gfn >= s->base_gfn + s->npages)))
 702                         goto out_unlock;
 703         }
 704         /*
 705          * Do memory allocations outside lock.  memory_config_version will
 706          * detect any races.
 707          */
 708         mutex_unlock(&kvm->lock);
 709
 710         /* Deallocate if slot is being removed */
 711         if (!npages)
 712                 new.phys_mem = NULL;
 713
 714         /* Free page dirty bitmap if unneeded */
 715         if (!(new.flags & KVM_MEM_LOG_DIRTY_PAGES))
 716                 new.dirty_bitmap = NULL;
 717
 718         r = -ENOMEM;
 719
 720         /* Allocate if a slot is being created */
 721         if (npages && !new.phys_mem) {
 722                 new.phys_mem = vmalloc(npages * sizeof(struct page *));
 723
 724                 if (!new.phys_mem)
 725                         goto out_free;
 726
 727                 memset(new.phys_mem, 0, npages * sizeof(struct page *));
 728                 for (i = 0; i < npages; ++i) {
 729                         new.phys_mem[i] = alloc_page(GFP_HIGHUSER
 730                                                      | __GFP_ZERO);
 731                         if (!new.phys_mem[i])
 732                                 goto out_free;
 733                         set_page_private(new.phys_mem[i],0);
 734                 }
 735         }
 736
 737         /* Allocate page dirty bitmap if needed */
 738         if ((new.flags & KVM_MEM_LOG_DIRTY_PAGES) && !new.dirty_bitmap) {
 739                 unsigned dirty_bytes = ALIGN(npages, BITS_PER_LONG) / 8;
 740
 741                 new.dirty_bitmap = vmalloc(dirty_bytes);
 742                 if (!new.dirty_bitmap)
 743                         goto out_free;
 744                 memset(new.dirty_bitmap, 0, dirty_bytes);
 745         }
 746
 747         mutex_lock(&kvm->lock);
 748
 749         if (memory_config_version != kvm->memory_config_version) {
 750                 mutex_unlock(&kvm->lock);
 751                 kvm_free_physmem_slot(&new, &old);
 752                 goto raced;
 753         }
 754
 755         r = -EAGAIN;
 756         if (kvm->busy)
 757                 goto out_unlock;
 758
 759         if (mem->slot >= kvm->nmemslots)
 760                 kvm->nmemslots = mem->slot + 1;
 761
 762         *memslot = new;
 763         ++kvm->memory_config_version;
 764
 765         kvm_mmu_slot_remove_write_access(kvm, mem->slot);
 766         kvm_flush_remote_tlbs(kvm);
 767
 768         mutex_unlock(&kvm->lock);
 769
 770         kvm_free_physmem_slot(&old, &new);
 771         return 0;
 772
 773 out_unlock:
 774         mutex_unlock(&kvm->lock);
 775 out_free:
 776         kvm_free_physmem_slot(&new, &old);
 777 out:
 778         return r;
 779 }
 780
 781 /*
 782  * Get (and clear) the dirty memory log for a memory slot.
 783  */
 784 static int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
 785                                       struct kvm_dirty_log *log)
 786 {
 787         struct kvm_memory_slot *memslot;
 788         int r, i;
 789         int n;
 790         unsigned long any = 0;
 791
 792         mutex_lock(&kvm->lock);
 793
 794         /*
 795          * Prevent changes to guest memory configuration even while the lock
 796          * is not taken.
 797          */
 798         ++kvm->busy;
 799         mutex_unlock(&kvm->lock);
 800         r = -EINVAL;
 801         if (log->slot >= KVM_MEMORY_SLOTS)
 802                 goto out;
 803
 804         memslot = &kvm->memslots[log->slot];
 805         r = -ENOENT;
 806         if (!memslot->dirty_bitmap)
 807                 goto out;
 808
 809         n = ALIGN(memslot->npages, BITS_PER_LONG) / 8;
 810
 811         for (i = 0; !any && i < n/sizeof(long); ++i)
 812                 any = memslot->dirty_bitmap[i];
 813
 814         r = -EFAULT;
 815         if (copy_to_user(log->dirty_bitmap, memslot->dirty_bitmap, n))
 816                 goto out;
 817
 818         mutex_lock(&kvm->lock);
 819         kvm_mmu_slot_remove_write_access(kvm, log->slot);
 820         kvm_flush_remote_tlbs(kvm);
 821         memset(memslot->dirty_bitmap, 0, n);
 822         mutex_unlock(&kvm->lock);
 823
 824         r = 0;
 825
 826 out:
 827         mutex_lock(&kvm->lock);
 828         --kvm->busy;
 829         mutex_unlock(&kvm->lock);
 830         return r;
 831 }
 832
 833 /*
 834  * Set a new alias region.  Aliases map a portion of physical memory into
 835  * another portion.  This is useful for memory windows, for example the PC
 836  * VGA region.
 837  */
 838 static int kvm_vm_ioctl_set_memory_alias(struct kvm *kvm,
 839                                          struct kvm_memory_alias *alias)
 840 {
 841         int r, n;
 842         struct kvm_mem_alias *p;
 843
 844         r = -EINVAL;
 845         /* General sanity checks */
 846         if (alias->memory_size & (PAGE_SIZE - 1))
 847                 goto out;
 848         if (alias->guest_phys_addr & (PAGE_SIZE - 1))
 849                 goto out;
 850         if (alias->slot >= KVM_ALIAS_SLOTS)
 851                 goto out;
 852         if (alias->guest_phys_addr + alias->memory_size
 853             < alias->guest_phys_addr)
 854                 goto out;
 855         if (alias->target_phys_addr + alias->memory_size
 856             < alias->target_phys_addr)
 857                 goto out;
 858
 859         mutex_lock(&kvm->lock);
 860
 861         p = &kvm->aliases[alias->slot];
 862         p->base_gfn = alias->guest_phys_addr >> PAGE_SHIFT;
 863         p->npages = alias->memory_size >> PAGE_SHIFT;
 864         p->target_gfn = alias->target_phys_addr >> PAGE_SHIFT;
 865
 866         for (n = KVM_ALIAS_SLOTS; n > 0; --n)
 867                 if (kvm->aliases[n - 1].npages)
 868                         break;
 869         kvm->naliases = n;
 870
 871         kvm_mmu_zap_all(kvm);
 872
 873         mutex_unlock(&kvm->lock);
 874
 875         return 0;
 876
 877 out:
 878         return r;
 879 }
 880
 881 static gfn_t unalias_gfn(struct kvm *kvm, gfn_t gfn)
 882 {
 883         int i;
 884         struct kvm_mem_alias *alias;
 885
 886         for (i = 0; i < kvm->naliases; ++i) {
 887                 alias = &kvm->aliases[i];
 888                 if (gfn >= alias->base_gfn
 889                     && gfn < alias->base_gfn + alias->npages)
 890                         return alias->target_gfn + gfn - alias->base_gfn;
 891         }
 892         return gfn;
 893 }
 894
 895 static struct kvm_memory_slot *__gfn_to_memslot(struct kvm *kvm, gfn_t gfn)
 896 {
 897         int i;
 898
 899         for (i = 0; i < kvm->nmemslots; ++i) {
 900                 struct kvm_memory_slot *memslot = &kvm->memslots[i];
 901
 902                 if (gfn >= memslot->base_gfn
 903                     && gfn < memslot->base_gfn + memslot->npages)
 904                         return memslot;
 905         }
 906         return NULL;
 907 }
 908
 909 struct kvm_memory_slot *gfn_to_memslot(struct kvm *kvm, gfn_t gfn)
 910 {
 911         gfn = unalias_gfn(kvm, gfn);
 912         return __gfn_to_memslot(kvm, gfn);
 913 }
 914
 915 struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn)
 916 {
 917         struct kvm_memory_slot *slot;
 918
 919         gfn = unalias_gfn(kvm, gfn);
 920         slot = __gfn_to_memslot(kvm, gfn);
 921         if (!slot)
 922                 return NULL;
 923         return slot->phys_mem[gfn - slot->base_gfn];
 924 }
 925 EXPORT_SYMBOL_GPL(gfn_to_page);
 926
 927 void mark_page_dirty(struct kvm *kvm, gfn_t gfn)
 928 {
 929         int i;
 930         struct kvm_memory_slot *memslot;
 931         unsigned long rel_gfn;
 932
 933         for (i = 0; i < kvm->nmemslots; ++i) {
 934                 memslot = &kvm->memslots[i];
 935
 936                 if (gfn >= memslot->base_gfn
 937                     && gfn < memslot->base_gfn + memslot->npages) {
 938
 939                         if (!memslot->dirty_bitmap)
 940                                 return;
 941
 942                         rel_gfn = gfn - memslot->base_gfn;
 943
 944                         /* avoid RMW */
 945                         if (!test_bit(rel_gfn, memslot->dirty_bitmap))
 946                                 set_bit(rel_gfn, memslot->dirty_bitmap);
 947                         return;
 948                 }
 949         }
 950 }
 951
 952 int emulator_read_std(unsigned long addr,
 953                              void *val,
 954                              unsigned int bytes,
 955                              struct kvm_vcpu *vcpu)
 956 {
 957         void *data = val;
 958
 959         while (bytes) {
 960                 gpa_t gpa = vcpu->mmu.gva_to_gpa(vcpu, addr);
 961                 unsigned offset = addr & (PAGE_SIZE-1);
 962                 unsigned tocopy = min(bytes, (unsigned)PAGE_SIZE - offset);
 963                 unsigned long pfn;
 964                 struct page *page;
 965                 void *page_virt;
 966
 967                 if (gpa == UNMAPPED_GVA)
 968                         return X86EMUL_PROPAGATE_FAULT;
 969                 pfn = gpa >> PAGE_SHIFT;
 970                 page = gfn_to_page(vcpu->kvm, pfn);
 971                 if (!page)
 972                         return X86EMUL_UNHANDLEABLE;
 973                 page_virt = kmap_atomic(page, KM_USER0);
 974
 975                 memcpy(data, page_virt + offset, tocopy);
 976
 977                 kunmap_atomic(page_virt, KM_USER0);
 978
 979                 bytes -= tocopy;
 980                 data += tocopy;
 981                 addr += tocopy;
 982         }
 983
 984         return X86EMUL_CONTINUE;
 985 }
 986 EXPORT_SYMBOL_GPL(emulator_read_std);
 987
 988 static int emulator_write_std(unsigned long addr,
 989                               const void *val,
 990                               unsigned int bytes,
 991                               struct kvm_vcpu *vcpu)
 992 {
 993         printk(KERN_ERR "emulator_write_std: addr %lx n %d\n",
 994                addr, bytes);
 995         return X86EMUL_UNHANDLEABLE;
 996 }
 997
 998 static struct kvm_io_device *vcpu_find_mmio_dev(struct kvm_vcpu *vcpu,
 999                                                 gpa_t addr)
1000 {
1001         /*
1002          * Note that its important to have this wrapper function because
1003          * in the very near future we will be checking for MMIOs against
1004          * the LAPIC as well as the general MMIO bus
1005          */
1006         return kvm_io_bus_find_dev(&vcpu->kvm->mmio_bus, addr);
1007 }
1008
1009 static struct kvm_io_device *vcpu_find_pio_dev(struct kvm_vcpu *vcpu,
1010                                                gpa_t addr)
1011 {
1012         return kvm_io_bus_find_dev(&vcpu->kvm->pio_bus, addr);
1013 }
1014
1015 static int emulator_read_emulated(unsigned long addr,
1016                                   void *val,
1017                                   unsigned int bytes,
1018                                   struct kvm_vcpu *vcpu)
1019 {
1020         struct kvm_io_device *mmio_dev;
1021         gpa_t                 gpa;
1022
1023         if (vcpu->mmio_read_completed) {
1024                 memcpy(val, vcpu->mmio_data, bytes);
1025                 vcpu->mmio_read_completed = 0;
1026                 return X86EMUL_CONTINUE;
1027         } else if (emulator_read_std(addr, val, bytes, vcpu)
1028                    == X86EMUL_CONTINUE)
1029                 return X86EMUL_CONTINUE;
1030
1031         gpa = vcpu->mmu.gva_to_gpa(vcpu, addr);
1032         if (gpa == UNMAPPED_GVA)
1033                 return X86EMUL_PROPAGATE_FAULT;
1034
1035         /*
1036          * Is this MMIO handled locally?
1037          */
1038         mmio_dev = vcpu_find_mmio_dev(vcpu, gpa);
1039         if (mmio_dev) {
1040                 kvm_iodevice_read(mmio_dev, gpa, bytes, val);
1041                 return X86EMUL_CONTINUE;
1042         }
1043
1044         vcpu->mmio_needed = 1;
1045         vcpu->mmio_phys_addr = gpa;
1046         vcpu->mmio_size = bytes;
1047         vcpu->mmio_is_write = 0;
1048
1049         return X86EMUL_UNHANDLEABLE;
1050 }
1051
1052 static int emulator_write_phys(struct kvm_vcpu *vcpu, gpa_t gpa,
1053                                const void *val, int bytes)
1054 {
1055         struct page *page;
1056         void *virt;
1057
1058         if (((gpa + bytes - 1) >> PAGE_SHIFT) != (gpa >> PAGE_SHIFT))
1059                 return 0;
1060         page = gfn_to_page(vcpu->kvm, gpa >> PAGE_SHIFT);
1061         if (!page)
1062                 return 0;
1063         mark_page_dirty(vcpu->kvm, gpa >> PAGE_SHIFT);
1064         virt = kmap_atomic(page, KM_USER0);
1065         kvm_mmu_pte_write(vcpu, gpa, val, bytes);
1066         memcpy(virt + offset_in_page(gpa), val, bytes);
1067         kunmap_atomic(virt, KM_USER0);
1068         return 1;
1069 }
1070
1071 static int emulator_write_emulated_onepage(unsigned long addr,
1072                                            const void *val,
1073                                            unsigned int bytes,
1074                                            struct kvm_vcpu *vcpu)
1075 {
1076         struct kvm_io_device *mmio_dev;
1077         gpa_t                 gpa = vcpu->mmu.gva_to_gpa(vcpu, addr);
1078
1079         if (gpa == UNMAPPED_GVA) {
1080                 kvm_arch_ops->inject_page_fault(vcpu, addr, 2);
1081                 return X86EMUL_PROPAGATE_FAULT;
1082         }
1083
1084         if (emulator_write_phys(vcpu, gpa, val, bytes))
1085                 return X86EMUL_CONTINUE;
1086
1087         /*
1088          * Is this MMIO handled locally?
1089          */
1090         mmio_dev = vcpu_find_mmio_dev(vcpu, gpa);
1091         if (mmio_dev) {
1092                 kvm_iodevice_write(mmio_dev, gpa, bytes, val);
1093                 return X86EMUL_CONTINUE;
1094         }
1095
1096         vcpu->mmio_needed = 1;
1097         vcpu->mmio_phys_addr = gpa;
1098         vcpu->mmio_size = bytes;
1099         vcpu->mmio_is_write = 1;
1100         memcpy(vcpu->mmio_data, val, bytes);
1101
1102         return X86EMUL_CONTINUE;
1103 }
1104
1105 int emulator_write_emulated(unsigned long addr,
1106                                    const void *val,
1107                                    unsigned int bytes,
1108                                    struct kvm_vcpu *vcpu)
1109 {
1110         /* Crossing a page boundary? */
1111         if (((addr + bytes - 1) ^ addr) & PAGE_MASK) {
1112                 int rc, now;
1113
1114                 now = -addr & ~PAGE_MASK;
1115                 rc = emulator_write_emulated_onepage(addr, val, now, vcpu);
1116                 if (rc != X86EMUL_CONTINUE)
1117                         return rc;
1118                 addr += now;
1119                 val += now;
1120                 bytes -= now;
1121         }
1122         return emulator_write_emulated_onepage(addr, val, bytes, vcpu);
1123 }
1124 EXPORT_SYMBOL_GPL(emulator_write_emulated);
1125
1126 static int emulator_cmpxchg_emulated(unsigned long addr,
1127                                      const void *old,
1128                                      const void *new,
1129                                      unsigned int bytes,
1130                                      struct kvm_vcpu *vcpu)
1131 {
1132         static int reported;
1133
1134         if (!reported) {
1135                 reported = 1;
1136                 printk(KERN_WARNING "kvm: emulating exchange as write\n");
1137         }
1138         return emulator_write_emulated(addr, new, bytes, vcpu);
1139 }
1140
1141 static unsigned long get_segment_base(struct kvm_vcpu *vcpu, int seg)
1142 {
1143         return kvm_arch_ops->get_segment_base(vcpu, seg);
1144 }
1145
1146 int emulate_invlpg(struct kvm_vcpu *vcpu, gva_t address)
1147 {
1148         return X86EMUL_CONTINUE;
1149 }
1150
1151 int emulate_clts(struct kvm_vcpu *vcpu)
1152 {
1153         unsigned long cr0;
1154
1155         cr0 = vcpu->cr0 & ~X86_CR0_TS;
1156         kvm_arch_ops->set_cr0(vcpu, cr0);
1157         return X86EMUL_CONTINUE;
1158 }
1159
1160 int emulator_get_dr(struct x86_emulate_ctxt* ctxt, int dr, unsigned long *dest)
1161 {
1162         struct kvm_vcpu *vcpu = ctxt->vcpu;
1163
1164         switch (dr) {
1165         case 0 ... 3:
1166                 *dest = kvm_arch_ops->get_dr(vcpu, dr);
1167                 return X86EMUL_CONTINUE;
1168         default:
1169                 printk(KERN_DEBUG "%s: unexpected dr %u\n",
1170                        __FUNCTION__, dr);
1171                 return X86EMUL_UNHANDLEABLE;
1172         }
1173 }
1174
1175 int emulator_set_dr(struct x86_emulate_ctxt *ctxt, int dr, unsigned long value)
1176 {
1177         unsigned long mask = (ctxt->mode == X86EMUL_MODE_PROT64) ? ~0ULL : ~0U;
1178         int exception;
1179
1180         kvm_arch_ops->set_dr(ctxt->vcpu, dr, value & mask, &exception);
1181         if (exception) {
1182                 /* FIXME: better handling */
1183                 return X86EMUL_UNHANDLEABLE;
1184         }
1185         return X86EMUL_CONTINUE;
1186 }
1187
1188 static void report_emulation_failure(struct x86_emulate_ctxt *ctxt)
1189 {
1190         static int reported;
1191         u8 opcodes[4];
1192         unsigned long rip = ctxt->vcpu->rip;
1193         unsigned long rip_linear;
1194
1195         rip_linear = rip + get_segment_base(ctxt->vcpu, VCPU_SREG_CS);
1196
1197         if (reported)
1198                 return;
1199
1200         emulator_read_std(rip_linear, (void *)opcodes, 4, ctxt->vcpu);
1201
1202         printk(KERN_ERR "emulation failed but !mmio_needed?"
1203                " rip %lx %02x %02x %02x %02x\n",
1204                rip, opcodes[0], opcodes[1], opcodes[2], opcodes[3]);
1205         reported = 1;
1206 }
1207
1208 struct x86_emulate_ops emulate_ops = {
1209         .read_std            = emulator_read_std,
1210         .write_std           = emulator_write_std,
1211         .read_emulated       = emulator_read_emulated,
1212         .write_emulated      = emulator_write_emulated,
1213         .cmpxchg_emulated    = emulator_cmpxchg_emulated,
1214 };
1215
1216 int emulate_instruction(struct kvm_vcpu *vcpu,
1217                         struct kvm_run *run,
1218                         unsigned long cr2,
1219                         u16 error_code)
1220 {
1221         struct x86_emulate_ctxt emulate_ctxt;
1222         int r;
1223         int cs_db, cs_l;
1224
1225         vcpu->mmio_fault_cr2 = cr2;
1226         kvm_arch_ops->cache_regs(vcpu);
1227
1228         kvm_arch_ops->get_cs_db_l_bits(vcpu, &cs_db, &cs_l);
1229
1230         emulate_ctxt.vcpu = vcpu;
1231         emulate_ctxt.eflags = kvm_arch_ops->get_rflags(vcpu);
1232         emulate_ctxt.cr2 = cr2;
1233         emulate_ctxt.mode = (emulate_ctxt.eflags & X86_EFLAGS_VM)
1234                 ? X86EMUL_MODE_REAL : cs_l
1235                 ? X86EMUL_MODE_PROT64 : cs_db
1236                 ? X86EMUL_MODE_PROT32 : X86EMUL_MODE_PROT16;
1237
1238         if (emulate_ctxt.mode == X86EMUL_MODE_PROT64) {
1239                 emulate_ctxt.cs_base = 0;
1240                 emulate_ctxt.ds_base = 0;
1241                 emulate_ctxt.es_base = 0;
1242                 emulate_ctxt.ss_base = 0;
1243         } else {
1244                 emulate_ctxt.cs_base = get_segment_base(vcpu, VCPU_SREG_CS);
1245                 emulate_ctxt.ds_base = get_segment_base(vcpu, VCPU_SREG_DS);
1246                 emulate_ctxt.es_base = get_segment_base(vcpu, VCPU_SREG_ES);
1247                 emulate_ctxt.ss_base = get_segment_base(vcpu, VCPU_SREG_SS);
1248         }
1249
1250         emulate_ctxt.gs_base = get_segment_base(vcpu, VCPU_SREG_GS);
1251         emulate_ctxt.fs_base = get_segment_base(vcpu, VCPU_SREG_FS);
1252
1253         vcpu->mmio_is_write = 0;
1254         r = x86_emulate_memop(&emulate_ctxt, &emulate_ops);
1255
1256         if ((r || vcpu->mmio_is_write) && run) {
1257                 run->exit_reason = KVM_EXIT_MMIO;
1258                 run->mmio.phys_addr = vcpu->mmio_phys_addr;
1259                 memcpy(run->mmio.data, vcpu->mmio_data, 8);
1260                 run->mmio.len = vcpu->mmio_size;
1261                 run->mmio.is_write = vcpu->mmio_is_write;
1262         }
1263
1264         if (r) {
1265                 if (kvm_mmu_unprotect_page_virt(vcpu, cr2))
1266                         return EMULATE_DONE;
1267                 if (!vcpu->mmio_needed) {
1268                         report_emulation_failure(&emulate_ctxt);
1269                         return EMULATE_FAIL;
1270                 }
1271                 return EMULATE_DO_MMIO;
1272         }
1273
1274         kvm_arch_ops->decache_regs(vcpu);
1275         kvm_arch_ops->set_rflags(vcpu, emulate_ctxt.eflags);
1276
1277         if (vcpu->mmio_is_write) {
1278                 vcpu->mmio_needed = 0;
1279                 return EMULATE_DO_MMIO;
1280         }
1281
1282         return EMULATE_DONE;
1283 }
1284 EXPORT_SYMBOL_GPL(emulate_instruction);
1285
1286 int kvm_emulate_halt(struct kvm_vcpu *vcpu)
1287 {
1288         if (vcpu->irq_summary)
1289                 return 1;
1290
1291         vcpu->run->exit_reason = KVM_EXIT_HLT;
1292         ++vcpu->stat.halt_exits;
1293         return 0;
1294 }
1295 EXPORT_SYMBOL_GPL(kvm_emulate_halt);
1296
1297 int kvm_hypercall(struct kvm_vcpu *vcpu, struct kvm_run *run)
1298 {
1299         unsigned long nr, a0, a1, a2, a3, a4, a5, ret;
1300
1301         kvm_arch_ops->cache_regs(vcpu);
1302         ret = -KVM_EINVAL;
1303 #ifdef CONFIG_X86_64
1304         if (is_long_mode(vcpu)) {
1305                 nr = vcpu->regs[VCPU_REGS_RAX];
1306                 a0 = vcpu->regs[VCPU_REGS_RDI];
1307                 a1 = vcpu->regs[VCPU_REGS_RSI];
1308                 a2 = vcpu->regs[VCPU_REGS_RDX];
1309                 a3 = vcpu->regs[VCPU_REGS_RCX];
1310                 a4 = vcpu->regs[VCPU_REGS_R8];
1311                 a5 = vcpu->regs[VCPU_REGS_R9];
1312         } else
1313 #endif
1314         {
1315                 nr = vcpu->regs[VCPU_REGS_RBX] & -1u;
1316                 a0 = vcpu->regs[VCPU_REGS_RAX] & -1u;
1317                 a1 = vcpu->regs[VCPU_REGS_RCX] & -1u;
1318                 a2 = vcpu->regs[VCPU_REGS_RDX] & -1u;
1319                 a3 = vcpu->regs[VCPU_REGS_RSI] & -1u;
1320                 a4 = vcpu->regs[VCPU_REGS_RDI] & -1u;
1321                 a5 = vcpu->regs[VCPU_REGS_RBP] & -1u;
1322         }
1323         switch (nr) {
1324         default:
1325                 run->hypercall.nr = nr;
1326                 run->hypercall.args[0] = a0;
1327                 run->hypercall.args[1] = a1;
1328                 run->hypercall.args[2] = a2;
1329                 run->hypercall.args[3] = a3;
1330                 run->hypercall.args[4] = a4;
1331                 run->hypercall.args[5] = a5;
1332                 run->hypercall.ret = ret;
1333                 run->hypercall.longmode = is_long_mode(vcpu);
1334                 kvm_arch_ops->decache_regs(vcpu);
1335                 return 0;
1336         }
1337         vcpu->regs[VCPU_REGS_RAX] = ret;
1338         kvm_arch_ops->decache_regs(vcpu);
1339         return 1;
1340 }
1341 EXPORT_SYMBOL_GPL(kvm_hypercall);
1342
1343 static u64 mk_cr_64(u64 curr_cr, u32 new_val)
1344 {
1345         return (curr_cr & ~((1ULL << 32) - 1)) | new_val;
1346 }
1347
1348 void realmode_lgdt(struct kvm_vcpu *vcpu, u16 limit, unsigned long base)
1349 {
1350         struct descriptor_table dt = { limit, base };
1351
1352         kvm_arch_ops->set_gdt(vcpu, &dt);
1353 }
1354
1355 void realmode_lidt(struct kvm_vcpu *vcpu, u16 limit, unsigned long base)
1356 {
1357         struct descriptor_table dt = { limit, base };
1358
1359         kvm_arch_ops->set_idt(vcpu, &dt);
1360 }
1361
1362 void realmode_lmsw(struct kvm_vcpu *vcpu, unsigned long msw,
1363                    unsigned long *rflags)
1364 {
1365         lmsw(vcpu, msw);
1366         *rflags = kvm_arch_ops->get_rflags(vcpu);
1367 }
1368
1369 unsigned long realmode_get_cr(struct kvm_vcpu *vcpu, int cr)
1370 {
1371         kvm_arch_ops->decache_cr4_guest_bits(vcpu);
1372         switch (cr) {
1373         case 0:
1374                 return vcpu->cr0;
1375         case 2:
1376                 return vcpu->cr2;
1377         case 3:
1378                 return vcpu->cr3;
1379         case 4:
1380                 return vcpu->cr4;
1381         default:
1382                 vcpu_printf(vcpu, "%s: unexpected cr %u\n", __FUNCTION__, cr);
1383                 return 0;
1384         }
1385 }
1386
1387 void realmode_set_cr(struct kvm_vcpu *vcpu, int cr, unsigned long val,
1388                      unsigned long *rflags)
1389 {
1390         switch (cr) {
1391         case 0:
1392                 set_cr0(vcpu, mk_cr_64(vcpu->cr0, val));
1393                 *rflags = kvm_arch_ops->get_rflags(vcpu);
1394                 break;
1395         case 2:
1396                 vcpu->cr2 = val;
1397                 break;
1398         case 3:
1399                 set_cr3(vcpu, val);
1400                 break;
1401         case 4:
1402                 set_cr4(vcpu, mk_cr_64(vcpu->cr4, val));
1403                 break;
1404         default:
1405                 vcpu_printf(vcpu, "%s: unexpected cr %u\n", __FUNCTION__, cr);
1406         }
1407 }
1408
1409 /*
1410  * Register the para guest with the host:
1411  */
1412 static int vcpu_register_para(struct kvm_vcpu *vcpu, gpa_t para_state_gpa)
1413 {
1414         struct kvm_vcpu_para_state *para_state;
1415         hpa_t para_state_hpa, hypercall_hpa;
1416         struct page *para_state_page;
1417         unsigned char *hypercall;
1418         gpa_t hypercall_gpa;
1419
1420         printk(KERN_DEBUG "kvm: guest trying to enter paravirtual mode\n");
1421         printk(KERN_DEBUG ".... para_state_gpa: %08Lx\n", para_state_gpa);
1422
1423         /*
1424          * Needs to be page aligned:
1425          */
1426         if (para_state_gpa != PAGE_ALIGN(para_state_gpa))
1427                 goto err_gp;
1428
1429         para_state_hpa = gpa_to_hpa(vcpu, para_state_gpa);
1430         printk(KERN_DEBUG ".... para_state_hpa: %08Lx\n", para_state_hpa);
1431         if (is_error_hpa(para_state_hpa))
1432                 goto err_gp;
1433
1434         mark_page_dirty(vcpu->kvm, para_state_gpa >> PAGE_SHIFT);
1435         para_state_page = pfn_to_page(para_state_hpa >> PAGE_SHIFT);
1436         para_state = kmap(para_state_page);
1437
1438         printk(KERN_DEBUG "....  guest version: %d\n", para_state->guest_version);
1439         printk(KERN_DEBUG "....           size: %d\n", para_state->size);
1440
1441         para_state->host_version = KVM_PARA_API_VERSION;
1442         /*
1443          * We cannot support guests that try to register themselves
1444          * with a newer API version than the host supports:
1445          */
1446         if (para_state->guest_version > KVM_PARA_API_VERSION) {
1447                 para_state->ret = -KVM_EINVAL;
1448                 goto err_kunmap_skip;
1449         }
1450
1451         hypercall_gpa = para_state->hypercall_gpa;
1452         hypercall_hpa = gpa_to_hpa(vcpu, hypercall_gpa);
1453         printk(KERN_DEBUG ".... hypercall_hpa: %08Lx\n", hypercall_hpa);
1454         if (is_error_hpa(hypercall_hpa)) {
1455                 para_state->ret = -KVM_EINVAL;
1456                 goto err_kunmap_skip;
1457         }
1458
1459         printk(KERN_DEBUG "kvm: para guest successfully registered.\n");
1460         vcpu->para_state_page = para_state_page;
1461         vcpu->para_state_gpa = para_state_gpa;
1462         vcpu->hypercall_gpa = hypercall_gpa;
1463
1464         mark_page_dirty(vcpu->kvm, hypercall_gpa >> PAGE_SHIFT);
1465         hypercall = kmap_atomic(pfn_to_page(hypercall_hpa >> PAGE_SHIFT),
1466                                 KM_USER1) + (hypercall_hpa & ~PAGE_MASK);
1467         kvm_arch_ops->patch_hypercall(vcpu, hypercall);
1468         kunmap_atomic(hypercall, KM_USER1);
1469
1470         para_state->ret = 0;
1471 err_kunmap_skip:
1472         kunmap(para_state_page);
1473         return 0;
1474 err_gp:
1475         return 1;
1476 }
1477
1478 int kvm_get_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata)
1479 {
1480         u64 data;
1481
1482         switch (msr) {
1483         case 0xc0010010: /* SYSCFG */
1484         case 0xc0010015: /* HWCR */
1485         case MSR_IA32_PLATFORM_ID:
1486         case MSR_IA32_P5_MC_ADDR:
1487         case MSR_IA32_P5_MC_TYPE:
1488         case MSR_IA32_MC0_CTL:
1489         case MSR_IA32_MCG_STATUS:
1490         case MSR_IA32_MCG_CAP:
1491         case MSR_IA32_MC0_MISC:
1492         case MSR_IA32_MC0_MISC+4:
1493         case MSR_IA32_MC0_MISC+8:
1494         case MSR_IA32_MC0_MISC+12:
1495         case MSR_IA32_MC0_MISC+16:
1496         case MSR_IA32_UCODE_REV:
1497         case MSR_IA32_PERF_STATUS:
1498         case MSR_IA32_EBL_CR_POWERON:
1499                 /* MTRR registers */
1500         case 0xfe:
1501         case 0x200 ... 0x2ff:
1502                 data = 0;
1503                 break;
1504         case 0xcd: /* fsb frequency */
1505                 data = 3;
1506                 break;
1507         case MSR_IA32_APICBASE:
1508                 data = vcpu->apic_base;
1509                 break;
1510         case MSR_IA32_MISC_ENABLE:
1511                 data = vcpu->ia32_misc_enable_msr;
1512                 break;
1513 #ifdef CONFIG_X86_64
1514         case MSR_EFER:
1515                 data = vcpu->shadow_efer;
1516                 break;
1517 #endif
1518         default:
1519                 printk(KERN_ERR "kvm: unhandled rdmsr: 0x%x\n", msr);
1520                 return 1;
1521         }
1522         *pdata = data;
1523         return 0;
1524 }
1525 EXPORT_SYMBOL_GPL(kvm_get_msr_common);
1526
1527 /*
1528  * Reads an msr value (of 'msr_index') into 'pdata'.
1529  * Returns 0 on success, non-0 otherwise.
1530  * Assumes vcpu_load() was already called.
1531  */
1532 int kvm_get_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 *pdata)
1533 {
1534         return kvm_arch_ops->get_msr(vcpu, msr_index, pdata);
1535 }
1536
1537 #ifdef CONFIG_X86_64
1538
1539 static void set_efer(struct kvm_vcpu *vcpu, u64 efer)
1540 {
1541         if (efer & EFER_RESERVED_BITS) {
1542                 printk(KERN_DEBUG "set_efer: 0x%llx #GP, reserved bits\n",
1543                        efer);
1544                 inject_gp(vcpu);
1545                 return;
1546         }
1547
1548         if (is_paging(vcpu)
1549             && (vcpu->shadow_efer & EFER_LME) != (efer & EFER_LME)) {
1550                 printk(KERN_DEBUG "set_efer: #GP, change LME while paging\n");
1551                 inject_gp(vcpu);
1552                 return;
1553         }
1554
1555         kvm_arch_ops->set_efer(vcpu, efer);
1556
1557         efer &= ~EFER_LMA;
1558         efer |= vcpu->shadow_efer & EFER_LMA;
1559
1560         vcpu->shadow_efer = efer;
1561 }
1562
1563 #endif
1564
1565 int kvm_set_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 data)
1566 {
1567         switch (msr) {
1568 #ifdef CONFIG_X86_64
1569         case MSR_EFER:
1570                 set_efer(vcpu, data);
1571                 break;
1572 #endif
1573         case MSR_IA32_MC0_STATUS:
1574                 printk(KERN_WARNING "%s: MSR_IA32_MC0_STATUS 0x%llx, nop\n",
1575                        __FUNCTION__, data);
1576                 break;
1577         case MSR_IA32_MCG_STATUS:
1578                 printk(KERN_WARNING "%s: MSR_IA32_MCG_STATUS 0x%llx, nop\n",
1579                         __FUNCTION__, data);
1580                 break;
1581         case MSR_IA32_UCODE_REV:
1582         case MSR_IA32_UCODE_WRITE:
1583         case 0x200 ... 0x2ff: /* MTRRs */
1584                 break;
1585         case MSR_IA32_APICBASE:
1586                 vcpu->apic_base = data;
1587                 break;
1588         case MSR_IA32_MISC_ENABLE:
1589                 vcpu->ia32_misc_enable_msr = data;
1590                 break;
1591         /*
1592          * This is the 'probe whether the host is KVM' logic:
1593          */
1594         case MSR_KVM_API_MAGIC:
1595                 return vcpu_register_para(vcpu, data);
1596
1597         default:
1598                 printk(KERN_ERR "kvm: unhandled wrmsr: 0x%x\n", msr);
1599                 return 1;
1600         }
1601         return 0;
1602 }
1603 EXPORT_SYMBOL_GPL(kvm_set_msr_common);
1604
1605 /*
1606  * Writes msr value into into the appropriate "register".
1607  * Returns 0 on success, non-0 otherwise.
1608  * Assumes vcpu_load() was already called.
1609  */
1610 int kvm_set_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 data)
1611 {
1612         return kvm_arch_ops->set_msr(vcpu, msr_index, data);
1613 }
1614
1615 void kvm_resched(struct kvm_vcpu *vcpu)
1616 {
1617         if (!need_resched())
1618                 return;
1619         cond_resched();
1620 }
1621 EXPORT_SYMBOL_GPL(kvm_resched);
1622
1623 void kvm_emulate_cpuid(struct kvm_vcpu *vcpu)
1624 {
1625         int i;
1626         u32 function;
1627         struct kvm_cpuid_entry *e, *best;
1628
1629         kvm_arch_ops->cache_regs(vcpu);
1630         function = vcpu->regs[VCPU_REGS_RAX];
1631         vcpu->regs[VCPU_REGS_RAX] = 0;
1632         vcpu->regs[VCPU_REGS_RBX] = 0;
1633         vcpu->regs[VCPU_REGS_RCX] = 0;
1634         vcpu->regs[VCPU_REGS_RDX] = 0;
1635         best = NULL;
1636         for (i = 0; i < vcpu->cpuid_nent; ++i) {
1637                 e = &vcpu->cpuid_entries[i];
1638                 if (e->function == function) {
1639                         best = e;
1640                         break;
1641                 }
1642                 /*
1643                  * Both basic or both extended?
1644                  */
1645                 if (((e->function ^ function) & 0x80000000) == 0)
1646                         if (!best || e->function > best->function)
1647                                 best = e;
1648         }
1649         if (best) {
1650                 vcpu->regs[VCPU_REGS_RAX] = best->eax;
1651                 vcpu->regs[VCPU_REGS_RBX] = best->ebx;
1652                 vcpu->regs[VCPU_REGS_RCX] = best->ecx;
1653                 vcpu->regs[VCPU_REGS_RDX] = best->edx;
1654         }
1655         kvm_arch_ops->decache_regs(vcpu);
1656         kvm_arch_ops->skip_emulated_instruction(vcpu);
1657 }
1658 EXPORT_SYMBOL_GPL(kvm_emulate_cpuid);
1659
1660 static int pio_copy_data(struct kvm_vcpu *vcpu)
1661 {
1662         void *p = vcpu->pio_data;
1663         void *q;
1664         unsigned bytes;
1665         int nr_pages = vcpu->pio.guest_pages[1] ? 2 : 1;
1666
1667         q = vmap(vcpu->pio.guest_pages, nr_pages, VM_READ|VM_WRITE,
1668                  PAGE_KERNEL);
1669         if (!q) {
1670                 free_pio_guest_pages(vcpu);
1671                 return -ENOMEM;
1672         }
1673         q += vcpu->pio.guest_page_offset;
1674         bytes = vcpu->pio.size * vcpu->pio.cur_count;
1675         if (vcpu->pio.in)
1676                 memcpy(q, p, bytes);
1677         else
1678                 memcpy(p, q, bytes);
1679         q -= vcpu->pio.guest_page_offset;
1680         vunmap(q);
1681         free_pio_guest_pages(vcpu);
1682         return 0;
1683 }
1684
1685 static int complete_pio(struct kvm_vcpu *vcpu)
1686 {
1687         struct kvm_pio_request *io = &vcpu->pio;
1688         long delta;
1689         int r;
1690
1691         kvm_arch_ops->cache_regs(vcpu);
1692
1693         if (!io->string) {
1694                 if (io->in)
1695                         memcpy(&vcpu->regs[VCPU_REGS_RAX], vcpu->pio_data,
1696                                io->size);
1697         } else {
1698                 if (io->in) {
1699                         r = pio_copy_data(vcpu);
1700                         if (r) {
1701                                 kvm_arch_ops->cache_regs(vcpu);
1702                                 return r;
1703                         }
1704                 }
1705
1706                 delta = 1;
1707                 if (io->rep) {
1708                         delta *= io->cur_count;
1709                         /*
1710                          * The size of the register should really depend on
1711                          * current address size.
1712                          */
1713                         vcpu->regs[VCPU_REGS_RCX] -= delta;
1714                 }
1715                 if (io->down)
1716                         delta = -delta;
1717                 delta *= io->size;
1718                 if (io->in)
1719                         vcpu->regs[VCPU_REGS_RDI] += delta;
1720                 else
1721                         vcpu->regs[VCPU_REGS_RSI] += delta;
1722         }
1723
1724         kvm_arch_ops->decache_regs(vcpu);
1725
1726         io->count -= io->cur_count;
1727         io->cur_count = 0;
1728
1729         if (!io->count)
1730                 kvm_arch_ops->skip_emulated_instruction(vcpu);
1731         return 0;
1732 }
1733
1734 static void kernel_pio(struct kvm_io_device *pio_dev,
1735                        struct kvm_vcpu *vcpu,
1736                        void *pd)
1737 {
1738         /* TODO: String I/O for in kernel device */
1739
1740         if (vcpu->pio.in)
1741                 kvm_iodevice_read(pio_dev, vcpu->pio.port,
1742                                   vcpu->pio.size,
1743                                   pd);
1744         else
1745                 kvm_iodevice_write(pio_dev, vcpu->pio.port,
1746                                    vcpu->pio.size,
1747                                    pd);
1748 }
1749
1750 static void pio_string_write(struct kvm_io_device *pio_dev,
1751                              struct kvm_vcpu *vcpu)
1752 {
1753         struct kvm_pio_request *io = &vcpu->pio;
1754         void *pd = vcpu->pio_data;
1755         int i;
1756
1757         for (i = 0; i < io->cur_count; i++) {
1758                 kvm_iodevice_write(pio_dev, io->port,
1759                                    io->size,
1760                                    pd);
1761                 pd += io->size;
1762         }
1763 }
1764
1765 int kvm_setup_pio(struct kvm_vcpu *vcpu, struct kvm_run *run, int in,
1766                   int size, unsigned long count, int string, int down,
1767                   gva_t address, int rep, unsigned port)
1768 {
1769         unsigned now, in_page;
1770         int i, ret = 0;
1771         int nr_pages = 1;
1772         struct page *page;
1773         struct kvm_io_device *pio_dev;
1774
1775         vcpu->run->exit_reason = KVM_EXIT_IO;
1776         vcpu->run->io.direction = in ? KVM_EXIT_IO_IN : KVM_EXIT_IO_OUT;
1777         vcpu->run->io.size = size;
1778         vcpu->run->io.data_offset = KVM_PIO_PAGE_OFFSET * PAGE_SIZE;
1779         vcpu->run->io.count = count;
1780         vcpu->run->io.port = port;
1781         vcpu->pio.count = count;
1782         vcpu->pio.cur_count = count;
1783         vcpu->pio.size = size;
1784         vcpu->pio.in = in;
1785         vcpu->pio.port = port;
1786         vcpu->pio.string = string;
1787         vcpu->pio.down = down;
1788         vcpu->pio.guest_page_offset = offset_in_page(address);
1789         vcpu->pio.rep = rep;
1790
1791         pio_dev = vcpu_find_pio_dev(vcpu, port);
1792         if (!string) {
1793                 kvm_arch_ops->cache_regs(vcpu);
1794                 memcpy(vcpu->pio_data, &vcpu->regs[VCPU_REGS_RAX], 4);
1795                 kvm_arch_ops->decache_regs(vcpu);
1796                 if (pio_dev) {
1797                         kernel_pio(pio_dev, vcpu, vcpu->pio_data);
1798                         complete_pio(vcpu);
1799                         return 1;
1800                 }
1801                 return 0;
1802         }
1803
1804         if (!count) {
1805                 kvm_arch_ops->skip_emulated_instruction(vcpu);
1806                 return 1;
1807         }
1808
1809         now = min(count, PAGE_SIZE / size);
1810
1811         if (!down)
1812                 in_page = PAGE_SIZE - offset_in_page(address);
1813         else
1814                 in_page = offset_in_page(address) + size;
1815         now = min(count, (unsigned long)in_page / size);
1816         if (!now) {
1817                 /*
1818                  * String I/O straddles page boundary.  Pin two guest pages
1819                  * so that we satisfy atomicity constraints.  Do just one
1820                  * transaction to avoid complexity.
1821                  */
1822                 nr_pages = 2;
1823                 now = 1;
1824         }
1825         if (down) {
1826                 /*
1827                  * String I/O in reverse.  Yuck.  Kill the guest, fix later.
1828                  */
1829                 printk(KERN_ERR "kvm: guest string pio down\n");
1830                 inject_gp(vcpu);
1831                 return 1;
1832         }
1833         vcpu->run->io.count = now;
1834         vcpu->pio.cur_count = now;
1835
1836         for (i = 0; i < nr_pages; ++i) {
1837                 mutex_lock(&vcpu->kvm->lock);
1838                 page = gva_to_page(vcpu, address + i * PAGE_SIZE);
1839                 if (page)
1840                         get_page(page);
1841                 vcpu->pio.guest_pages[i] = page;
1842                 mutex_unlock(&vcpu->kvm->lock);
1843                 if (!page) {
1844                         inject_gp(vcpu);
1845                         free_pio_guest_pages(vcpu);
1846                         return 1;
1847                 }
1848         }
1849
1850         if (!vcpu->pio.in) {
1851                 /* string PIO write */
1852                 ret = pio_copy_data(vcpu);
1853                 if (ret >= 0 && pio_dev) {
1854                         pio_string_write(pio_dev, vcpu);
1855                         complete_pio(vcpu);
1856                         if (vcpu->pio.count == 0)
1857                                 ret = 1;
1858                 }
1859         } else if (pio_dev)
1860                 printk(KERN_ERR "no string pio read support yet, "
1861                        "port %x size %d count %ld\n",
1862                         port, size, count);
1863
1864         return ret;
1865 }
1866 EXPORT_SYMBOL_GPL(kvm_setup_pio);
1867
1868 static int kvm_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
1869 {
1870         int r;
1871         sigset_t sigsaved;
1872
1873         vcpu_load(vcpu);
1874
1875         if (vcpu->sigset_active)
1876                 sigprocmask(SIG_SETMASK, &vcpu->sigset, &sigsaved);
1877
1878         /* re-sync apic's tpr */
1879         vcpu->cr8 = kvm_run->cr8;
1880
1881         if (vcpu->pio.cur_count) {
1882                 r = complete_pio(vcpu);
1883                 if (r)
1884                         goto out;
1885         }
1886
1887         if (vcpu->mmio_needed) {
1888                 memcpy(vcpu->mmio_data, kvm_run->mmio.data, 8);
1889                 vcpu->mmio_read_completed = 1;
1890                 vcpu->mmio_needed = 0;
1891                 r = emulate_instruction(vcpu, kvm_run,
1892                                         vcpu->mmio_fault_cr2, 0);
1893                 if (r == EMULATE_DO_MMIO) {
1894                         /*
1895                          * Read-modify-write.  Back to userspace.
1896                          */
1897                         r = 0;
1898                         goto out;
1899                 }
1900         }
1901
1902         if (kvm_run->exit_reason == KVM_EXIT_HYPERCALL) {
1903                 kvm_arch_ops->cache_regs(vcpu);
1904                 vcpu->regs[VCPU_REGS_RAX] = kvm_run->hypercall.ret;
1905                 kvm_arch_ops->decache_regs(vcpu);
1906         }
1907
1908         r = kvm_arch_ops->run(vcpu, kvm_run);
1909
1910 out:
1911         if (vcpu->sigset_active)
1912                 sigprocmask(SIG_SETMASK, &sigsaved, NULL);
1913
1914         vcpu_put(vcpu);
1915         return r;
1916 }
1917
1918 static int kvm_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu,
1919                                    struct kvm_regs *regs)
1920 {
1921         vcpu_load(vcpu);
1922
1923         kvm_arch_ops->cache_regs(vcpu);
1924
1925         regs->rax = vcpu->regs[VCPU_REGS_RAX];
1926         regs->rbx = vcpu->regs[VCPU_REGS_RBX];
1927         regs->rcx = vcpu->regs[VCPU_REGS_RCX];
1928         regs->rdx = vcpu->regs[VCPU_REGS_RDX];
1929         regs->rsi = vcpu->regs[VCPU_REGS_RSI];
1930         regs->rdi = vcpu->regs[VCPU_REGS_RDI];
1931         regs->rsp = vcpu->regs[VCPU_REGS_RSP];
1932         regs->rbp = vcpu->regs[VCPU_REGS_RBP];
1933 #ifdef CONFIG_X86_64
1934         regs->r8 = vcpu->regs[VCPU_REGS_R8];
1935         regs->r9 = vcpu->regs[VCPU_REGS_R9];
1936         regs->r10 = vcpu->regs[VCPU_REGS_R10];
1937         regs->r11 = vcpu->regs[VCPU_REGS_R11];
1938         regs->r12 = vcpu->regs[VCPU_REGS_R12];
1939         regs->r13 = vcpu->regs[VCPU_REGS_R13];
1940         regs->r14 = vcpu->regs[VCPU_REGS_R14];
1941         regs->r15 = vcpu->regs[VCPU_REGS_R15];
1942 #endif
1943
1944         regs->rip = vcpu->rip;
1945         regs->rflags = kvm_arch_ops->get_rflags(vcpu);
1946
1947         /*
1948          * Don't leak debug flags in case they were set for guest debugging
1949          */
1950         if (vcpu->guest_debug.enabled && vcpu->guest_debug.singlestep)
1951                 regs->rflags &= ~(X86_EFLAGS_TF | X86_EFLAGS_RF);
1952
1953         vcpu_put(vcpu);
1954
1955         return 0;
1956 }
1957
1958 static int kvm_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu,
1959                                    struct kvm_regs *regs)
1960 {
1961         vcpu_load(vcpu);
1962
1963         vcpu->regs[VCPU_REGS_RAX] = regs->rax;
1964         vcpu->regs[VCPU_REGS_RBX] = regs->rbx;
1965         vcpu->regs[VCPU_REGS_RCX] = regs->rcx;
1966         vcpu->regs[VCPU_REGS_RDX] = regs->rdx;
1967         vcpu->regs[VCPU_REGS_RSI] = regs->rsi;
1968         vcpu->regs[VCPU_REGS_RDI] = regs->rdi;
1969         vcpu->regs[VCPU_REGS_RSP] = regs->rsp;
1970         vcpu->regs[VCPU_REGS_RBP] = regs->rbp;
1971 #ifdef CONFIG_X86_64
1972         vcpu->regs[VCPU_REGS_R8] = regs->r8;
1973         vcpu->regs[VCPU_REGS_R9] = regs->r9;
1974         vcpu->regs[VCPU_REGS_R10] = regs->r10;
1975         vcpu->regs[VCPU_REGS_R11] = regs->r11;
1976         vcpu->regs[VCPU_REGS_R12] = regs->r12;
1977         vcpu->regs[VCPU_REGS_R13] = regs->r13;
1978         vcpu->regs[VCPU_REGS_R14] = regs->r14;
1979         vcpu->regs[VCPU_REGS_R15] = regs->r15;
1980 #endif
1981
1982         vcpu->rip = regs->rip;
1983         kvm_arch_ops->set_rflags(vcpu, regs->rflags);
1984
1985         kvm_arch_ops->decache_regs(vcpu);
1986
1987         vcpu_put(vcpu);
1988
1989         return 0;
1990 }
1991
1992 static void get_segment(struct kvm_vcpu *vcpu,
1993                         struct kvm_segment *var, int seg)
1994 {
1995         return kvm_arch_ops->get_segment(vcpu, var, seg);
1996 }
1997
1998 static int kvm_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
1999                                     struct kvm_sregs *sregs)
2000 {
2001         struct descriptor_table dt;
2002
2003         vcpu_load(vcpu);
2004
2005         get_segment(vcpu, &sregs->cs, VCPU_SREG_CS);
2006         get_segment(vcpu, &sregs->ds, VCPU_SREG_DS);
2007         get_segment(vcpu, &sregs->es, VCPU_SREG_ES);
2008         get_segment(vcpu, &sregs->fs, VCPU_SREG_FS);
2009         get_segment(vcpu, &sregs->gs, VCPU_SREG_GS);
2010         get_segment(vcpu, &sregs->ss, VCPU_SREG_SS);
2011
2012         get_segment(vcpu, &sregs->tr, VCPU_SREG_TR);
2013         get_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR);
2014
2015         kvm_arch_ops->get_idt(vcpu, &dt);
2016         sregs->idt.limit = dt.limit;
2017         sregs->idt.base = dt.base;
2018         kvm_arch_ops->get_gdt(vcpu, &dt);
2019         sregs->gdt.limit = dt.limit;
2020         sregs->gdt.base = dt.base;
2021
2022         kvm_arch_ops->decache_cr4_guest_bits(vcpu);
2023         sregs->cr0 = vcpu->cr0;
2024         sregs->cr2 = vcpu->cr2;
2025         sregs->cr3 = vcpu->cr3;
2026         sregs->cr4 = vcpu->cr4;
2027         sregs->cr8 = vcpu->cr8;
2028         sregs->efer = vcpu->shadow_efer;
2029         sregs->apic_base = vcpu->apic_base;
2030
2031         memcpy(sregs->interrupt_bitmap, vcpu->irq_pending,
2032                sizeof sregs->interrupt_bitmap);
2033
2034         vcpu_put(vcpu);
2035
2036         return 0;
2037 }
2038
2039 static void set_segment(struct kvm_vcpu *vcpu,
2040                         struct kvm_segment *var, int seg)
2041 {
2042         return kvm_arch_ops->set_segment(vcpu, var, seg);
2043 }
2044
2045 static int kvm_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
2046                                     struct kvm_sregs *sregs)
2047 {
2048         int mmu_reset_needed = 0;
2049         int i;
2050         struct descriptor_table dt;
2051
2052         vcpu_load(vcpu);
2053
2054         dt.limit = sregs->idt.limit;
2055         dt.base = sregs->idt.base;
2056         kvm_arch_ops->set_idt(vcpu, &dt);
2057         dt.limit = sregs->gdt.limit;
2058         dt.base = sregs->gdt.base;
2059         kvm_arch_ops->set_gdt(vcpu, &dt);
2060
2061         vcpu->cr2 = sregs->cr2;
2062         mmu_reset_needed |= vcpu->cr3 != sregs->cr3;
2063         vcpu->cr3 = sregs->cr3;
2064
2065         vcpu->cr8 = sregs->cr8;
2066
2067         mmu_reset_needed |= vcpu->shadow_efer != sregs->efer;
2068 #ifdef CONFIG_X86_64
2069         kvm_arch_ops->set_efer(vcpu, sregs->efer);
2070 #endif
2071         vcpu->apic_base = sregs->apic_base;
2072
2073         kvm_arch_ops->decache_cr4_guest_bits(vcpu);
2074
2075         mmu_reset_needed |= vcpu->cr0 != sregs->cr0;
2076         kvm_arch_ops->set_cr0(vcpu, sregs->cr0);
2077
2078         mmu_reset_needed |= vcpu->cr4 != sregs->cr4;
2079         kvm_arch_ops->set_cr4(vcpu, sregs->cr4);
2080         if (!is_long_mode(vcpu) && is_pae(vcpu))
2081                 load_pdptrs(vcpu, vcpu->cr3);
2082
2083         if (mmu_reset_needed)
2084                 kvm_mmu_reset_context(vcpu);
2085
2086         memcpy(vcpu->irq_pending, sregs->interrupt_bitmap,
2087                sizeof vcpu->irq_pending);
2088         vcpu->irq_summary = 0;
2089         for (i = 0; i < ARRAY_SIZE(vcpu->irq_pending); ++i)
2090                 if (vcpu->irq_pending[i])
2091                         __set_bit(i, &vcpu->irq_summary);
2092
2093         set_segment(vcpu, &sregs->cs, VCPU_SREG_CS);
2094         set_segment(vcpu, &sregs->ds, VCPU_SREG_DS);
2095         set_segment(vcpu, &sregs->es, VCPU_SREG_ES);
2096         set_segment(vcpu, &sregs->fs, VCPU_SREG_FS);
2097         set_segment(vcpu, &sregs->gs, VCPU_SREG_GS);
2098         set_segment(vcpu, &sregs->ss, VCPU_SREG_SS);
2099
2100         set_segment(vcpu, &sregs->tr, VCPU_SREG_TR);
2101         set_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR);
2102
2103         vcpu_put(vcpu);
2104
2105         return 0;
2106 }
2107
2108 /*
2109  * List of msr numbers which we expose to userspace through KVM_GET_MSRS
2110  * and KVM_SET_MSRS, and KVM_GET_MSR_INDEX_LIST.
2111  *
2112  * This list is modified at module load time to reflect the
2113  * capabilities of the host cpu.
2114  */
2115 static u32 msrs_to_save[] = {
2116         MSR_IA32_SYSENTER_CS, MSR_IA32_SYSENTER_ESP, MSR_IA32_SYSENTER_EIP,
2117         MSR_K6_STAR,
2118 #ifdef CONFIG_X86_64
2119         MSR_CSTAR, MSR_KERNEL_GS_BASE, MSR_SYSCALL_MASK, MSR_LSTAR,
2120 #endif
2121         MSR_IA32_TIME_STAMP_COUNTER,
2122 };
2123
2124 static unsigned num_msrs_to_save;
2125
2126 static u32 emulated_msrs[] = {
2127         MSR_IA32_MISC_ENABLE,
2128 };
2129
2130 static __init void kvm_init_msr_list(void)
2131 {
2132         u32 dummy[2];
2133         unsigned i, j;
2134
2135         for (i = j = 0; i < ARRAY_SIZE(msrs_to_save); i++) {
2136                 if (rdmsr_safe(msrs_to_save[i], &dummy[0], &dummy[1]) < 0)
2137                         continue;
2138                 if (j < i)
2139                         msrs_to_save[j] = msrs_to_save[i];
2140                 j++;
2141         }
2142         num_msrs_to_save = j;
2143 }
2144
2145 /*
2146  * Adapt set_msr() to msr_io()'s calling convention
2147  */
2148 static int do_set_msr(struct kvm_vcpu *vcpu, unsigned index, u64 *data)
2149 {
2150         return kvm_set_msr(vcpu, index, *data);
2151 }
2152
2153 /*
2154  * Read or write a bunch of msrs. All parameters are kernel addresses.
2155  *
2156  * @return number of msrs set successfully.
2157  */
2158 static int __msr_io(struct kvm_vcpu *vcpu, struct kvm_msrs *msrs,
2159                     struct kvm_msr_entry *entries,
2160                     int (*do_msr)(struct kvm_vcpu *vcpu,
2161                                   unsigned index, u64 *data))
2162 {
2163         int i;
2164
2165         vcpu_load(vcpu);
2166
2167         for (i = 0; i < msrs->nmsrs; ++i)
2168                 if (do_msr(vcpu, entries[i].index, &entries[i].data))
2169                         break;
2170
2171         vcpu_put(vcpu);
2172
2173         return i;
2174 }
2175
2176 /*
2177  * Read or write a bunch of msrs. Parameters are user addresses.
2178  *
2179  * @return number of msrs set successfully.
2180  */
2181 static int msr_io(struct kvm_vcpu *vcpu, struct kvm_msrs __user *user_msrs,
2182                   int (*do_msr)(struct kvm_vcpu *vcpu,
2183                                 unsigned index, u64 *data),
2184                   int writeback)
2185 {
2186         struct kvm_msrs msrs;
2187         struct kvm_msr_entry *entries;
2188         int r, n;
2189         unsigned size;
2190
2191         r = -EFAULT;
2192         if (copy_from_user(&msrs, user_msrs, sizeof msrs))
2193                 goto out;
2194
2195         r = -E2BIG;
2196         if (msrs.nmsrs >= MAX_IO_MSRS)
2197                 goto out;
2198
2199         r = -ENOMEM;
2200         size = sizeof(struct kvm_msr_entry) * msrs.nmsrs;
2201         entries = vmalloc(size);
2202         if (!entries)
2203                 goto out;
2204
2205         r = -EFAULT;
2206         if (copy_from_user(entries, user_msrs->entries, size))
2207                 goto out_free;
2208
2209         r = n = __msr_io(vcpu, &msrs, entries, do_msr);
2210         if (r < 0)
2211                 goto out_free;
2212
2213         r = -EFAULT;
2214         if (writeback && copy_to_user(user_msrs->entries, entries, size))
2215                 goto out_free;
2216
2217         r = n;
2218
2219 out_free:
2220         vfree(entries);
2221 out:
2222         return r;
2223 }
2224
2225 /*
2226  * Translate a guest virtual address to a guest physical address.
2227  */
2228 static int kvm_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
2229                                     struct kvm_translation *tr)
2230 {
2231         unsigned long vaddr = tr->linear_address;
2232         gpa_t gpa;
2233
2234         vcpu_load(vcpu);
2235         mutex_lock(&vcpu->kvm->lock);
2236         gpa = vcpu->mmu.gva_to_gpa(vcpu, vaddr);
2237         tr->physical_address = gpa;
2238         tr->valid = gpa != UNMAPPED_GVA;
2239         tr->writeable = 1;
2240         tr->usermode = 0;
2241         mutex_unlock(&vcpu->kvm->lock);
2242         vcpu_put(vcpu);
2243
2244         return 0;
2245 }
2246
2247 static int kvm_vcpu_ioctl_interrupt(struct kvm_vcpu *vcpu,
2248                                     struct kvm_interrupt *irq)
2249 {
2250         if (irq->irq < 0 || irq->irq >= 256)
2251                 return -EINVAL;
2252         vcpu_load(vcpu);
2253
2254         set_bit(irq->irq, vcpu->irq_pending);
2255         set_bit(irq->irq / BITS_PER_LONG, &vcpu->irq_summary);
2256
2257         vcpu_put(vcpu);
2258
2259         return 0;
2260 }
2261
2262 static int kvm_vcpu_ioctl_debug_guest(struct kvm_vcpu *vcpu,
2263                                       struct kvm_debug_guest *dbg)
2264 {
2265         int r;
2266
2267         vcpu_load(vcpu);
2268
2269         r = kvm_arch_ops->set_guest_debug(vcpu, dbg);
2270
2271         vcpu_put(vcpu);
2272
2273         return r;
2274 }
2275
2276 static struct page *kvm_vcpu_nopage(struct vm_area_struct *vma,
2277                                     unsigned long address,
2278                                     int *type)
2279 {
2280         struct kvm_vcpu *vcpu = vma->vm_file->private_data;
2281         unsigned long pgoff;
2282         struct page *page;
2283
2284         pgoff = ((address - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff;
2285         if (pgoff == 0)
2286                 page = virt_to_page(vcpu->run);
2287         else if (pgoff == KVM_PIO_PAGE_OFFSET)
2288                 page = virt_to_page(vcpu->pio_data);
2289         else
2290                 return NOPAGE_SIGBUS;
2291         get_page(page);
2292         if (type != NULL)
2293                 *type = VM_FAULT_MINOR;
2294
2295         return page;
2296 }
2297
2298 static struct vm_operations_struct kvm_vcpu_vm_ops = {
2299         .nopage = kvm_vcpu_nopage,
2300 };
2301
2302 static int kvm_vcpu_mmap(struct file *file, struct vm_area_struct *vma)
2303 {
2304         vma->vm_ops = &kvm_vcpu_vm_ops;
2305         return 0;
2306 }
2307
2308 static int kvm_vcpu_release(struct inode *inode, struct file *filp)
2309 {
2310         struct kvm_vcpu *vcpu = filp->private_data;
2311
2312         fput(vcpu->kvm->filp);
2313         return 0;
2314 }
2315
2316 static struct file_operations kvm_vcpu_fops = {
2317         .release        = kvm_vcpu_release,
2318         .unlocked_ioctl = kvm_vcpu_ioctl,
2319         .compat_ioctl   = kvm_vcpu_ioctl,
2320         .mmap           = kvm_vcpu_mmap,
2321 };
2322
2323 /*
2324  * Allocates an inode for the vcpu.
2325  */
2326 static int create_vcpu_fd(struct kvm_vcpu *vcpu)
2327 {
2328         int fd, r;
2329         struct inode *inode;
2330         struct file *file;
2331
2332         r = anon_inode_getfd(&fd, &inode, &file,
2333                              "kvm-vcpu", &kvm_vcpu_fops, vcpu);
2334         if (r)
2335                 return r;
2336         atomic_inc(&vcpu->kvm->filp->f_count);
2337         return fd;
2338 }
2339
2340 /*
2341  * Creates some virtual cpus.  Good luck creating more than one.
2342  */
2343 static int kvm_vm_ioctl_create_vcpu(struct kvm *kvm, int n)
2344 {
2345         int r;
2346         struct kvm_vcpu *vcpu;
2347
2348         if (!valid_vcpu(n))
2349                 return -EINVAL;
2350
2351         vcpu = kvm_arch_ops->vcpu_create(kvm, n);
2352         if (IS_ERR(vcpu))
2353                 return PTR_ERR(vcpu);
2354
2355         preempt_notifier_init(&vcpu->preempt_notifier, &kvm_preempt_ops);
2356
2357         vcpu_load(vcpu);
2358         r = kvm_mmu_setup(vcpu);
2359         vcpu_put(vcpu);
2360         if (r < 0)
2361                 goto free_vcpu;
2362
2363         mutex_lock(&kvm->lock);
2364         if (kvm->vcpus[n]) {
2365                 r = -EEXIST;
2366                 mutex_unlock(&kvm->lock);
2367                 goto mmu_unload;
2368         }
2369         kvm->vcpus[n] = vcpu;
2370         mutex_unlock(&kvm->lock);
2371
2372         /* Now it's all set up, let userspace reach it */
2373         r = create_vcpu_fd(vcpu);
2374         if (r < 0)
2375                 goto unlink;
2376         return r;
2377
2378 unlink:
2379         mutex_lock(&kvm->lock);
2380         kvm->vcpus[n] = NULL;
2381         mutex_unlock(&kvm->lock);
2382
2383 mmu_unload:
2384         vcpu_load(vcpu);
2385         kvm_mmu_unload(vcpu);
2386         vcpu_put(vcpu);
2387
2388 free_vcpu:
2389         kvm_arch_ops->vcpu_free(vcpu);
2390         return r;
2391 }
2392
2393 static void cpuid_fix_nx_cap(struct kvm_vcpu *vcpu)
2394 {
2395         u64 efer;
2396         int i;
2397         struct kvm_cpuid_entry *e, *entry;
2398
2399         rdmsrl(MSR_EFER, efer);
2400         entry = NULL;
2401         for (i = 0; i < vcpu->cpuid_nent; ++i) {
2402                 e = &vcpu->cpuid_entries[i];
2403                 if (e->function == 0x80000001) {
2404                         entry = e;
2405                         break;
2406                 }
2407         }
2408         if (entry && (entry->edx & (1 << 20)) && !(efer & EFER_NX)) {
2409                 entry->edx &= ~(1 << 20);
2410                 printk(KERN_INFO "kvm: guest NX capability removed\n");
2411         }
2412 }
2413
2414 static int kvm_vcpu_ioctl_set_cpuid(struct kvm_vcpu *vcpu,
2415                                     struct kvm_cpuid *cpuid,
2416                                     struct kvm_cpuid_entry __user *entries)
2417 {
2418         int r;
2419
2420         r = -E2BIG;
2421         if (cpuid->nent > KVM_MAX_CPUID_ENTRIES)
2422                 goto out;
2423         r = -EFAULT;
2424         if (copy_from_user(&vcpu->cpuid_entries, entries,
2425                            cpuid->nent * sizeof(struct kvm_cpuid_entry)))
2426                 goto out;
2427         vcpu->cpuid_nent = cpuid->nent;
2428         cpuid_fix_nx_cap(vcpu);
2429         return 0;
2430
2431 out:
2432         return r;
2433 }
2434
2435 static int kvm_vcpu_ioctl_set_sigmask(struct kvm_vcpu *vcpu, sigset_t *sigset)
2436 {
2437         if (sigset) {
2438                 sigdelsetmask(sigset, sigmask(SIGKILL)|sigmask(SIGSTOP));
2439                 vcpu->sigset_active = 1;
2440                 vcpu->sigset = *sigset;
2441         } else
2442                 vcpu->sigset_active = 0;
2443         return 0;
2444 }
2445
2446 /*
2447  * fxsave fpu state.  Taken from x86_64/processor.h.  To be killed when
2448  * we have asm/x86/processor.h
2449  */
2450 struct fxsave {
2451         u16     cwd;
2452         u16     swd;
2453         u16     twd;
2454         u16     fop;
2455         u64     rip;
2456         u64     rdp;
2457         u32     mxcsr;
2458         u32     mxcsr_mask;
2459         u32     st_space[32];   /* 8*16 bytes for each FP-reg = 128 bytes */
2460 #ifdef CONFIG_X86_64
2461         u32     xmm_space[64];  /* 16*16 bytes for each XMM-reg = 256 bytes */
2462 #else
2463         u32     xmm_space[32];  /* 8*16 bytes for each XMM-reg = 128 bytes */
2464 #endif
2465 };
2466
2467 static int kvm_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
2468 {
2469         struct fxsave *fxsave = (struct fxsave *)vcpu->guest_fx_image;
2470
2471         vcpu_load(vcpu);
2472
2473         memcpy(fpu->fpr, fxsave->st_space, 128);
2474         fpu->fcw = fxsave->cwd;
2475         fpu->fsw = fxsave->swd;
2476         fpu->ftwx = fxsave->twd;
2477         fpu->last_opcode = fxsave->fop;
2478         fpu->last_ip = fxsave->rip;
2479         fpu->last_dp = fxsave->rdp;
2480         memcpy(fpu->xmm, fxsave->xmm_space, sizeof fxsave->xmm_space);
2481
2482         vcpu_put(vcpu);
2483
2484         return 0;
2485 }
2486
2487 static int kvm_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
2488 {
2489         struct fxsave *fxsave = (struct fxsave *)vcpu->guest_fx_image;
2490
2491         vcpu_load(vcpu);
2492
2493         memcpy(fxsave->st_space, fpu->fpr, 128);
2494         fxsave->cwd = fpu->fcw;
2495         fxsave->swd = fpu->fsw;
2496         fxsave->twd = fpu->ftwx;
2497         fxsave->fop = fpu->last_opcode;
2498         fxsave->rip = fpu->last_ip;
2499         fxsave->rdp = fpu->last_dp;
2500         memcpy(fxsave->xmm_space, fpu->xmm, sizeof fxsave->xmm_space);
2501
2502         vcpu_put(vcpu);
2503
2504         return 0;
2505 }
2506
2507 static long kvm_vcpu_ioctl(struct file *filp,
2508                            unsigned int ioctl, unsigned long arg)
2509 {
2510         struct kvm_vcpu *vcpu = filp->private_data;
2511         void __user *argp = (void __user *)arg;
2512         int r = -EINVAL;
2513
2514         switch (ioctl) {
2515         case KVM_RUN:
2516                 r = -EINVAL;
2517                 if (arg)
2518                         goto out;
2519                 r = kvm_vcpu_ioctl_run(vcpu, vcpu->run);
2520                 break;
2521         case KVM_GET_REGS: {
2522                 struct kvm_regs kvm_regs;
2523
2524                 memset(&kvm_regs, 0, sizeof kvm_regs);
2525                 r = kvm_vcpu_ioctl_get_regs(vcpu, &kvm_regs);
2526                 if (r)
2527                         goto out;
2528                 r = -EFAULT;
2529                 if (copy_to_user(argp, &kvm_regs, sizeof kvm_regs))
2530                         goto out;
2531                 r = 0;
2532                 break;
2533         }
2534         case KVM_SET_REGS: {
2535                 struct kvm_regs kvm_regs;
2536
2537                 r = -EFAULT;
2538                 if (copy_from_user(&kvm_regs, argp, sizeof kvm_regs))
2539                         goto out;
2540                 r = kvm_vcpu_ioctl_set_regs(vcpu, &kvm_regs);
2541                 if (r)
2542                         goto out;
2543                 r = 0;
2544                 break;
2545         }
2546         case KVM_GET_SREGS: {
2547                 struct kvm_sregs kvm_sregs;
2548
2549                 memset(&kvm_sregs, 0, sizeof kvm_sregs);
2550                 r = kvm_vcpu_ioctl_get_sregs(vcpu, &kvm_sregs);
2551                 if (r)
2552                         goto out;
2553                 r = -EFAULT;
2554                 if (copy_to_user(argp, &kvm_sregs, sizeof kvm_sregs))
2555                         goto out;
2556                 r = 0;
2557                 break;
2558         }
2559         case KVM_SET_SREGS: {
2560                 struct kvm_sregs kvm_sregs;
2561
2562                 r = -EFAULT;
2563                 if (copy_from_user(&kvm_sregs, argp, sizeof kvm_sregs))
2564                         goto out;
2565                 r = kvm_vcpu_ioctl_set_sregs(vcpu, &kvm_sregs);
2566                 if (r)
2567                         goto out;
2568                 r = 0;
2569                 break;
2570         }
2571         case KVM_TRANSLATE: {
2572                 struct kvm_translation tr;
2573
2574                 r = -EFAULT;
2575                 if (copy_from_user(&tr, argp, sizeof tr))
2576                         goto out;
2577                 r = kvm_vcpu_ioctl_translate(vcpu, &tr);
2578                 if (r)
2579                         goto out;
2580                 r = -EFAULT;
2581                 if (copy_to_user(argp, &tr, sizeof tr))
2582                         goto out;
2583                 r = 0;
2584                 break;
2585         }
2586         case KVM_INTERRUPT: {
2587                 struct kvm_interrupt irq;
2588
2589                 r = -EFAULT;
2590                 if (copy_from_user(&irq, argp, sizeof irq))
2591                         goto out;
2592                 r = kvm_vcpu_ioctl_interrupt(vcpu, &irq);
2593                 if (r)
2594                         goto out;
2595                 r = 0;
2596                 break;
2597         }
2598         case KVM_DEBUG_GUEST: {
2599                 struct kvm_debug_guest dbg;
2600
2601                 r = -EFAULT;
2602                 if (copy_from_user(&dbg, argp, sizeof dbg))
2603                         goto out;
2604                 r = kvm_vcpu_ioctl_debug_guest(vcpu, &dbg);
2605                 if (r)
2606                         goto out;
2607                 r = 0;
2608                 break;
2609         }
2610         case KVM_GET_MSRS:
2611                 r = msr_io(vcpu, argp, kvm_get_msr, 1);
2612                 break;
2613         case KVM_SET_MSRS:
2614                 r = msr_io(vcpu, argp, do_set_msr, 0);
2615                 break;
2616         case KVM_SET_CPUID: {
2617                 struct kvm_cpuid __user *cpuid_arg = argp;
2618                 struct kvm_cpuid cpuid;
2619
2620                 r = -EFAULT;
2621                 if (copy_from_user(&cpuid, cpuid_arg, sizeof cpuid))
2622                         goto out;
2623                 r = kvm_vcpu_ioctl_set_cpuid(vcpu, &cpuid, cpuid_arg->entries);
2624                 if (r)
2625                         goto out;
2626                 break;
2627         }
2628         case KVM_SET_SIGNAL_MASK: {
2629                 struct kvm_signal_mask __user *sigmask_arg = argp;
2630                 struct kvm_signal_mask kvm_sigmask;
2631                 sigset_t sigset, *p;
2632
2633                 p = NULL;
2634                 if (argp) {
2635                         r = -EFAULT;
2636                         if (copy_from_user(&kvm_sigmask, argp,
2637                                            sizeof kvm_sigmask))
2638                                 goto out;
2639                         r = -EINVAL;
2640                         if (kvm_sigmask.len != sizeof sigset)
2641                                 goto out;
2642                         r = -EFAULT;
2643                         if (copy_from_user(&sigset, sigmask_arg->sigset,
2644                                            sizeof sigset))
2645                                 goto out;
2646                         p = &sigset;
2647                 }
2648                 r = kvm_vcpu_ioctl_set_sigmask(vcpu, &sigset);
2649                 break;
2650         }
2651         case KVM_GET_FPU: {
2652                 struct kvm_fpu fpu;
2653
2654                 memset(&fpu, 0, sizeof fpu);
2655                 r = kvm_vcpu_ioctl_get_fpu(vcpu, &fpu);
2656                 if (r)
2657                         goto out;
2658                 r = -EFAULT;
2659                 if (copy_to_user(argp, &fpu, sizeof fpu))
2660                         goto out;
2661                 r = 0;
2662                 break;
2663         }
2664         case KVM_SET_FPU: {
2665                 struct kvm_fpu fpu;
2666
2667                 r = -EFAULT;
2668                 if (copy_from_user(&fpu, argp, sizeof fpu))
2669                         goto out;
2670                 r = kvm_vcpu_ioctl_set_fpu(vcpu, &fpu);
2671                 if (r)
2672                         goto out;
2673                 r = 0;
2674                 break;
2675         }
2676         default:
2677                 ;
2678         }
2679 out:
2680         return r;
2681 }
2682
2683 static long kvm_vm_ioctl(struct file *filp,
2684                            unsigned int ioctl, unsigned long arg)
2685 {
2686         struct kvm *kvm = filp->private_data;
2687         void __user *argp = (void __user *)arg;
2688         int r = -EINVAL;
2689
2690         switch (ioctl) {
2691         case KVM_CREATE_VCPU:
2692                 r = kvm_vm_ioctl_create_vcpu(kvm, arg);
2693                 if (r < 0)
2694                         goto out;
2695                 break;
2696         case KVM_SET_MEMORY_REGION: {
2697                 struct kvm_memory_region kvm_mem;
2698
2699                 r = -EFAULT;
2700                 if (copy_from_user(&kvm_mem, argp, sizeof kvm_mem))
2701                         goto out;
2702                 r = kvm_vm_ioctl_set_memory_region(kvm, &kvm_mem);
2703                 if (r)
2704                         goto out;
2705                 break;
2706         }
2707         case KVM_GET_DIRTY_LOG: {
2708                 struct kvm_dirty_log log;
2709
2710                 r = -EFAULT;
2711                 if (copy_from_user(&log, argp, sizeof log))
2712                         goto out;
2713                 r = kvm_vm_ioctl_get_dirty_log(kvm, &log);
2714                 if (r)
2715                         goto out;
2716                 break;
2717         }
2718         case KVM_SET_MEMORY_ALIAS: {
2719                 struct kvm_memory_alias alias;
2720
2721                 r = -EFAULT;
2722                 if (copy_from_user(&alias, argp, sizeof alias))
2723                         goto out;
2724                 r = kvm_vm_ioctl_set_memory_alias(kvm, &alias);
2725                 if (r)
2726                         goto out;
2727                 break;
2728         }
2729         default:
2730                 ;
2731         }
2732 out:
2733         return r;
2734 }
2735
2736 static struct page *kvm_vm_nopage(struct vm_area_struct *vma,
2737                                   unsigned long address,
2738                                   int *type)
2739 {
2740         struct kvm *kvm = vma->vm_file->private_data;
2741         unsigned long pgoff;
2742         struct page *page;
2743
2744         pgoff = ((address - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff;
2745         page = gfn_to_page(kvm, pgoff);
2746         if (!page)
2747                 return NOPAGE_SIGBUS;
2748         get_page(page);
2749         if (type != NULL)
2750                 *type = VM_FAULT_MINOR;
2751
2752         return page;
2753 }
2754
2755 static struct vm_operations_struct kvm_vm_vm_ops = {
2756         .nopage = kvm_vm_nopage,
2757 };
2758
2759 static int kvm_vm_mmap(struct file *file, struct vm_area_struct *vma)
2760 {
2761         vma->vm_ops = &kvm_vm_vm_ops;
2762         return 0;
2763 }
2764
2765 static struct file_operations kvm_vm_fops = {
2766         .release        = kvm_vm_release,
2767         .unlocked_ioctl = kvm_vm_ioctl,
2768         .compat_ioctl   = kvm_vm_ioctl,
2769         .mmap           = kvm_vm_mmap,
2770 };
2771
2772 static int kvm_dev_ioctl_create_vm(void)
2773 {
2774         int fd, r;
2775         struct inode *inode;
2776         struct file *file;
2777         struct kvm *kvm;
2778
2779         kvm = kvm_create_vm();
2780         if (IS_ERR(kvm))
2781                 return PTR_ERR(kvm);
2782         r = anon_inode_getfd(&fd, &inode, &file, "kvm-vm", &kvm_vm_fops, kvm);
2783         if (r) {
2784                 kvm_destroy_vm(kvm);
2785                 return r;
2786         }
2787
2788         kvm->filp = file;
2789
2790         return fd;
2791 }
2792
2793 static long kvm_dev_ioctl(struct file *filp,
2794                           unsigned int ioctl, unsigned long arg)
2795 {
2796         void __user *argp = (void __user *)arg;
2797         long r = -EINVAL;
2798
2799         switch (ioctl) {
2800         case KVM_GET_API_VERSION:
2801                 r = -EINVAL;
2802                 if (arg)
2803                         goto out;
2804                 r = KVM_API_VERSION;
2805                 break;
2806         case KVM_CREATE_VM:
2807                 r = -EINVAL;
2808                 if (arg)
2809                         goto out;
2810                 r = kvm_dev_ioctl_create_vm();
2811                 break;
2812         case KVM_GET_MSR_INDEX_LIST: {
2813                 struct kvm_msr_list __user *user_msr_list = argp;
2814                 struct kvm_msr_list msr_list;
2815                 unsigned n;
2816
2817                 r = -EFAULT;
2818                 if (copy_from_user(&msr_list, user_msr_list, sizeof msr_list))
2819                         goto out;
2820                 n = msr_list.nmsrs;
2821                 msr_list.nmsrs = num_msrs_to_save + ARRAY_SIZE(emulated_msrs);
2822                 if (copy_to_user(user_msr_list, &msr_list, sizeof msr_list))
2823                         goto out;
2824                 r = -E2BIG;
2825                 if (n < num_msrs_to_save)
2826                         goto out;
2827                 r = -EFAULT;
2828                 if (copy_to_user(user_msr_list->indices, &msrs_to_save,
2829                                  num_msrs_to_save * sizeof(u32)))
2830                         goto out;
2831                 if (copy_to_user(user_msr_list->indices
2832                                  + num_msrs_to_save * sizeof(u32),
2833                                  &emulated_msrs,
2834                                  ARRAY_SIZE(emulated_msrs) * sizeof(u32)))
2835                         goto out;
2836                 r = 0;
2837                 break;
2838         }
2839         case KVM_CHECK_EXTENSION:
2840                 /*
2841                  * No extensions defined at present.
2842                  */
2843                 r = 0;
2844                 break;
2845         case KVM_GET_VCPU_MMAP_SIZE:
2846                 r = -EINVAL;
2847                 if (arg)
2848                         goto out;
2849                 r = 2 * PAGE_SIZE;
2850                 break;
2851         default:
2852                 ;
2853         }
2854 out:
2855         return r;
2856 }
2857
2858 static struct file_operations kvm_chardev_ops = {
2859         .open           = kvm_dev_open,
2860         .release        = kvm_dev_release,
2861         .unlocked_ioctl = kvm_dev_ioctl,
2862         .compat_ioctl   = kvm_dev_ioctl,
2863 };
2864
2865 static struct miscdevice kvm_dev = {
2866         KVM_MINOR,
2867         "kvm",
2868         &kvm_chardev_ops,
2869 };
2870
2871 /*
2872  * Make sure that a cpu that is being hot-unplugged does not have any vcpus
2873  * cached on it.
2874  */
2875 static void decache_vcpus_on_cpu(int cpu)
2876 {
2877         struct kvm *vm;
2878         struct kvm_vcpu *vcpu;
2879         int i;
2880
2881         spin_lock(&kvm_lock);
2882         list_for_each_entry(vm, &vm_list, vm_list)
2883                 for (i = 0; i < KVM_MAX_VCPUS; ++i) {
2884                         vcpu = vm->vcpus[i];
2885                         if (!vcpu)
2886                                 continue;
2887                         /*
2888                          * If the vcpu is locked, then it is running on some
2889                          * other cpu and therefore it is not cached on the
2890                          * cpu in question.
2891                          *
2892                          * If it's not locked, check the last cpu it executed
2893                          * on.
2894                          */
2895                         if (mutex_trylock(&vcpu->mutex)) {
2896                                 if (vcpu->cpu == cpu) {
2897                                         kvm_arch_ops->vcpu_decache(vcpu);
2898                                         vcpu->cpu = -1;
2899                                 }
2900                                 mutex_unlock(&vcpu->mutex);
2901                         }
2902                 }
2903         spin_unlock(&kvm_lock);
2904 }
2905
2906 static void hardware_enable(void *junk)
2907 {
2908         int cpu = raw_smp_processor_id();
2909
2910         if (cpu_isset(cpu, cpus_hardware_enabled))
2911                 return;
2912         cpu_set(cpu, cpus_hardware_enabled);
2913         kvm_arch_ops->hardware_enable(NULL);
2914 }
2915
2916 static void hardware_disable(void *junk)
2917 {
2918         int cpu = raw_smp_processor_id();
2919
2920         if (!cpu_isset(cpu, cpus_hardware_enabled))
2921                 return;
2922         cpu_clear(cpu, cpus_hardware_enabled);
2923         decache_vcpus_on_cpu(cpu);
2924         kvm_arch_ops->hardware_disable(NULL);
2925 }
2926
2927 static int kvm_cpu_hotplug(struct notifier_block *notifier, unsigned long val,
2928                            void *v)
2929 {
2930         int cpu = (long)v;
2931
2932         switch (val) {
2933         case CPU_DYING:
2934         case CPU_DYING_FROZEN:
2935                 printk(KERN_INFO "kvm: disabling virtualization on CPU%d\n",
2936                        cpu);
2937                 hardware_disable(NULL);
2938                 break;
2939         case CPU_UP_CANCELED:
2940         case CPU_UP_CANCELED_FROZEN:
2941                 printk(KERN_INFO "kvm: disabling virtualization on CPU%d\n",
2942                        cpu);
2943                 smp_call_function_single(cpu, hardware_disable, NULL, 0, 1);
2944                 break;
2945         case CPU_ONLINE:
2946         case CPU_ONLINE_FROZEN:
2947                 printk(KERN_INFO "kvm: enabling virtualization on CPU%d\n",
2948                        cpu);
2949                 smp_call_function_single(cpu, hardware_enable, NULL, 0, 1);
2950                 break;
2951         }
2952         return NOTIFY_OK;
2953 }
2954
2955 static int kvm_reboot(struct notifier_block *notifier, unsigned long val,
2956                        void *v)
2957 {
2958         if (val == SYS_RESTART) {
2959                 /*
2960                  * Some (well, at least mine) BIOSes hang on reboot if
2961                  * in vmx root mode.
2962                  */
2963                 printk(KERN_INFO "kvm: exiting hardware virtualization\n");
2964                 on_each_cpu(hardware_disable, NULL, 0, 1);
2965         }
2966         return NOTIFY_OK;
2967 }
2968
2969 static struct notifier_block kvm_reboot_notifier = {
2970         .notifier_call = kvm_reboot,
2971         .priority = 0,
2972 };
2973
2974 void kvm_io_bus_init(struct kvm_io_bus *bus)
2975 {
2976         memset(bus, 0, sizeof(*bus));
2977 }
2978
2979 void kvm_io_bus_destroy(struct kvm_io_bus *bus)
2980 {
2981         int i;
2982
2983         for (i = 0; i < bus->dev_count; i++) {
2984                 struct kvm_io_device *pos = bus->devs[i];
2985
2986                 kvm_iodevice_destructor(pos);
2987         }
2988 }
2989
2990 struct kvm_io_device *kvm_io_bus_find_dev(struct kvm_io_bus *bus, gpa_t addr)
2991 {
2992         int i;
2993
2994         for (i = 0; i < bus->dev_count; i++) {
2995                 struct kvm_io_device *pos = bus->devs[i];
2996
2997                 if (pos->in_range(pos, addr))
2998                         return pos;
2999         }
3000
3001         return NULL;
3002 }
3003
3004 void kvm_io_bus_register_dev(struct kvm_io_bus *bus, struct kvm_io_device *dev)
3005 {
3006         BUG_ON(bus->dev_count > (NR_IOBUS_DEVS-1));
3007
3008         bus->devs[bus->dev_count++] = dev;
3009 }
3010
3011 static struct notifier_block kvm_cpu_notifier = {
3012         .notifier_call = kvm_cpu_hotplug,
3013         .priority = 20, /* must be > scheduler priority */
3014 };
3015
3016 static u64 stat_get(void *_offset)
3017 {
3018         unsigned offset = (long)_offset;
3019         u64 total = 0;
3020         struct kvm *kvm;
3021         struct kvm_vcpu *vcpu;
3022         int i;
3023
3024         spin_lock(&kvm_lock);
3025         list_for_each_entry(kvm, &vm_list, vm_list)
3026                 for (i = 0; i < KVM_MAX_VCPUS; ++i) {
3027                         vcpu = kvm->vcpus[i];
3028                         if (vcpu)
3029                                 total += *(u32 *)((void *)vcpu + offset);
3030                 }
3031         spin_unlock(&kvm_lock);
3032         return total;
3033 }
3034
3035 static void stat_set(void *offset, u64 val)
3036 {
3037 }
3038
3039 DEFINE_SIMPLE_ATTRIBUTE(stat_fops, stat_get, stat_set, "%llu\n");
3040
3041 static __init void kvm_init_debug(void)
3042 {
3043         struct kvm_stats_debugfs_item *p;
3044
3045         debugfs_dir = debugfs_create_dir("kvm", NULL);
3046         for (p = debugfs_entries; p->name; ++p)
3047                 p->dentry = debugfs_create_file(p->name, 0444, debugfs_dir,
3048                                                 (void *)(long)p->offset,
3049                                                 &stat_fops);
3050 }
3051
3052 static void kvm_exit_debug(void)
3053 {
3054         struct kvm_stats_debugfs_item *p;
3055
3056         for (p = debugfs_entries; p->name; ++p)
3057                 debugfs_remove(p->dentry);
3058         debugfs_remove(debugfs_dir);
3059 }
3060
3061 static int kvm_suspend(struct sys_device *dev, pm_message_t state)
3062 {
3063         hardware_disable(NULL);
3064         return 0;
3065 }
3066
3067 static int kvm_resume(struct sys_device *dev)
3068 {
3069         hardware_enable(NULL);
3070         return 0;
3071 }
3072
3073 static struct sysdev_class kvm_sysdev_class = {
3074         set_kset_name("kvm"),
3075         .suspend = kvm_suspend,
3076         .resume = kvm_resume,
3077 };
3078
3079 static struct sys_device kvm_sysdev = {
3080         .id = 0,
3081         .cls = &kvm_sysdev_class,
3082 };
3083
3084 hpa_t bad_page_address;
3085
3086 static inline
3087 struct kvm_vcpu *preempt_notifier_to_vcpu(struct preempt_notifier *pn)
3088 {
3089         return container_of(pn, struct kvm_vcpu, preempt_notifier);
3090 }
3091
3092 static void kvm_sched_in(struct preempt_notifier *pn, int cpu)
3093 {
3094         struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn);
3095
3096         kvm_arch_ops->vcpu_load(vcpu, cpu);
3097 }
3098
3099 static void kvm_sched_out(struct preempt_notifier *pn,
3100                           struct task_struct *next)
3101 {
3102         struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn);
3103
3104         kvm_arch_ops->vcpu_put(vcpu);
3105 }
3106
3107 int kvm_init_arch(struct kvm_arch_ops *ops, struct module *module)
3108 {
3109         int r;
3110
3111         if (kvm_arch_ops) {
3112                 printk(KERN_ERR "kvm: already loaded the other module\n");
3113                 return -EEXIST;
3114         }
3115
3116         if (!ops->cpu_has_kvm_support()) {
3117                 printk(KERN_ERR "kvm: no hardware support\n");
3118                 return -EOPNOTSUPP;
3119         }
3120         if (ops->disabled_by_bios()) {
3121                 printk(KERN_ERR "kvm: disabled by bios\n");
3122                 return -EOPNOTSUPP;
3123         }
3124
3125         kvm_arch_ops = ops;
3126
3127         r = kvm_arch_ops->hardware_setup();
3128         if (r < 0)
3129                 goto out;
3130
3131         on_each_cpu(hardware_enable, NULL, 0, 1);
3132         r = register_cpu_notifier(&kvm_cpu_notifier);
3133         if (r)
3134                 goto out_free_1;
3135         register_reboot_notifier(&kvm_reboot_notifier);
3136
3137         r = sysdev_class_register(&kvm_sysdev_class);
3138         if (r)
3139                 goto out_free_2;
3140
3141         r = sysdev_register(&kvm_sysdev);
3142         if (r)
3143                 goto out_free_3;
3144
3145         kvm_chardev_ops.owner = module;
3146
3147         r = misc_register(&kvm_dev);
3148         if (r) {
3149                 printk (KERN_ERR "kvm: misc device register failed\n");
3150                 goto out_free;
3151         }
3152
3153         kvm_preempt_ops.sched_in = kvm_sched_in;
3154         kvm_preempt_ops.sched_out = kvm_sched_out;
3155
3156         return r;
3157
3158 out_free:
3159         sysdev_unregister(&kvm_sysdev);
3160 out_free_3:
3161         sysdev_class_unregister(&kvm_sysdev_class);
3162 out_free_2:
3163         unregister_reboot_notifier(&kvm_reboot_notifier);
3164         unregister_cpu_notifier(&kvm_cpu_notifier);
3165 out_free_1:
3166         on_each_cpu(hardware_disable, NULL, 0, 1);
3167         kvm_arch_ops->hardware_unsetup();
3168 out:
3169         kvm_arch_ops = NULL;
3170         return r;
3171 }
3172
3173 void kvm_exit_arch(void)
3174 {
3175         misc_deregister(&kvm_dev);
3176         sysdev_unregister(&kvm_sysdev);
3177         sysdev_class_unregister(&kvm_sysdev_class);
3178         unregister_reboot_notifier(&kvm_reboot_notifier);
3179         unregister_cpu_notifier(&kvm_cpu_notifier);
3180         on_each_cpu(hardware_disable, NULL, 0, 1);
3181         kvm_arch_ops->hardware_unsetup();
3182         kvm_arch_ops = NULL;
3183 }
3184
3185 static __init int kvm_init(void)
3186 {
3187         static struct page *bad_page;
3188         int r;
3189
3190         r = kvm_mmu_module_init();
3191         if (r)
3192                 goto out4;
3193
3194         kvm_init_debug();
3195
3196         kvm_init_msr_list();
3197
3198         if ((bad_page = alloc_page(GFP_KERNEL)) == NULL) {
3199                 r = -ENOMEM;
3200                 goto out;
3201         }
3202
3203         bad_page_address = page_to_pfn(bad_page) << PAGE_SHIFT;
3204         memset(__va(bad_page_address), 0, PAGE_SIZE);
3205
3206         return 0;
3207
3208 out:
3209         kvm_exit_debug();
3210         kvm_mmu_module_exit();
3211 out4:
3212         return r;
3213 }
3214
3215 static __exit void kvm_exit(void)
3216 {
3217         kvm_exit_debug();
3218         __free_page(pfn_to_page(bad_page_address >> PAGE_SHIFT));
3219         kvm_mmu_module_exit();
3220 }
3221
3222 module_init(kvm_init)
3223 module_exit(kvm_exit)
3224
3225 EXPORT_SYMBOL_GPL(kvm_init_arch);
3226 EXPORT_SYMBOL_GPL(kvm_exit_arch);