err.no Git - linux-2.6/blob - drivers/kvm/kvm_main.c

   1 /*
   2  * Kernel-based Virtual Machine driver for Linux
   3  *
   4  * This module enables machines with Intel VT-x extensions to run virtual
   5  * machines without emulation or binary translation.
   6  *
   7  * Copyright (C) 2006 Qumranet, Inc.
   8  *
   9  * Authors:
  10  *   Avi Kivity   <avi@qumranet.com>
  11  *   Yaniv Kamay  <yaniv@qumranet.com>
  12  *
  13  * This work is licensed under the terms of the GNU GPL, version 2.  See
  14  * the COPYING file in the top-level directory.
  15  *
  16  */
  17
  18 #include "kvm.h"
  19
  20 #include <linux/kvm.h>
  21 #include <linux/module.h>
  22 #include <linux/errno.h>
  23 #include <asm/processor.h>
  24 #include <linux/percpu.h>
  25 #include <linux/gfp.h>
  26 #include <asm/msr.h>
  27 #include <linux/mm.h>
  28 #include <linux/miscdevice.h>
  29 #include <linux/vmalloc.h>
  30 #include <asm/uaccess.h>
  31 #include <linux/reboot.h>
  32 #include <asm/io.h>
  33 #include <linux/debugfs.h>
  34 #include <linux/highmem.h>
  35 #include <linux/file.h>
  36 #include <asm/desc.h>
  37 #include <linux/sysdev.h>
  38 #include <linux/cpu.h>
  39 #include <linux/file.h>
  40 #include <linux/fs.h>
  41 #include <linux/mount.h>
  42
  43 #include "x86_emulate.h"
  44 #include "segment_descriptor.h"
  45
  46 MODULE_AUTHOR("Qumranet");
  47 MODULE_LICENSE("GPL");
  48
  49 static DEFINE_SPINLOCK(kvm_lock);
  50 static LIST_HEAD(vm_list);
  51
  52 struct kvm_arch_ops *kvm_arch_ops;
  53 struct kvm_stat kvm_stat;
  54 EXPORT_SYMBOL_GPL(kvm_stat);
  55
  56 static struct kvm_stats_debugfs_item {
  57         const char *name;
  58         u32 *data;
  59         struct dentry *dentry;
  60 } debugfs_entries[] = {
  61         { "pf_fixed", &kvm_stat.pf_fixed },
  62         { "pf_guest", &kvm_stat.pf_guest },
  63         { "tlb_flush", &kvm_stat.tlb_flush },
  64         { "invlpg", &kvm_stat.invlpg },
  65         { "exits", &kvm_stat.exits },
  66         { "io_exits", &kvm_stat.io_exits },
  67         { "mmio_exits", &kvm_stat.mmio_exits },
  68         { "signal_exits", &kvm_stat.signal_exits },
  69         { "irq_window", &kvm_stat.irq_window_exits },
  70         { "halt_exits", &kvm_stat.halt_exits },
  71         { "request_irq", &kvm_stat.request_irq_exits },
  72         { "irq_exits", &kvm_stat.irq_exits },
  73         { NULL, NULL }
  74 };
  75
  76 static struct dentry *debugfs_dir;
  77
  78 #define KVMFS_MAGIC 0x19700426
  79 struct vfsmount *kvmfs_mnt;
  80
  81 #define MAX_IO_MSRS 256
  82
  83 #define CR0_RESEVED_BITS 0xffffffff1ffaffc0ULL
  84 #define LMSW_GUEST_MASK 0x0eULL
  85 #define CR4_RESEVED_BITS (~((1ULL << 11) - 1))
  86 #define CR8_RESEVED_BITS (~0x0fULL)
  87 #define EFER_RESERVED_BITS 0xfffffffffffff2fe
  88
  89 #ifdef CONFIG_X86_64
  90 // LDT or TSS descriptor in the GDT. 16 bytes.
  91 struct segment_descriptor_64 {
  92         struct segment_descriptor s;
  93         u32 base_higher;
  94         u32 pad_zero;
  95 };
  96
  97 #endif
  98
  99 static struct inode *kvmfs_inode(struct file_operations *fops)
 100 {
 101         int error = -ENOMEM;
 102         struct inode *inode = new_inode(kvmfs_mnt->mnt_sb);
 103
 104         if (!inode)
 105                 goto eexit_1;
 106
 107         inode->i_fop = fops;
 108
 109         /*
 110          * Mark the inode dirty from the very beginning,
 111          * that way it will never be moved to the dirty
 112          * list because mark_inode_dirty() will think
 113          * that it already _is_ on the dirty list.
 114          */
 115         inode->i_state = I_DIRTY;
 116         inode->i_mode = S_IRUSR | S_IWUSR;
 117         inode->i_uid = current->fsuid;
 118         inode->i_gid = current->fsgid;
 119         inode->i_atime = inode->i_mtime = inode->i_ctime = CURRENT_TIME;
 120         return inode;
 121
 122 eexit_1:
 123         return ERR_PTR(error);
 124 }
 125
 126 static struct file *kvmfs_file(struct inode *inode, void *private_data)
 127 {
 128         struct file *file = get_empty_filp();
 129
 130         if (!file)
 131                 return ERR_PTR(-ENFILE);
 132
 133         file->f_path.mnt = mntget(kvmfs_mnt);
 134         file->f_path.dentry = d_alloc_anon(inode);
 135         if (!file->f_path.dentry)
 136                 return ERR_PTR(-ENOMEM);
 137         file->f_mapping = inode->i_mapping;
 138
 139         file->f_pos = 0;
 140         file->f_flags = O_RDWR;
 141         file->f_op = inode->i_fop;
 142         file->f_mode = FMODE_READ | FMODE_WRITE;
 143         file->f_version = 0;
 144         file->private_data = private_data;
 145         return file;
 146 }
 147
 148 unsigned long segment_base(u16 selector)
 149 {
 150         struct descriptor_table gdt;
 151         struct segment_descriptor *d;
 152         unsigned long table_base;
 153         typedef unsigned long ul;
 154         unsigned long v;
 155
 156         if (selector == 0)
 157                 return 0;
 158
 159         asm ("sgdt %0" : "=m"(gdt));
 160         table_base = gdt.base;
 161
 162         if (selector & 4) {           /* from ldt */
 163                 u16 ldt_selector;
 164
 165                 asm ("sldt %0" : "=g"(ldt_selector));
 166                 table_base = segment_base(ldt_selector);
 167         }
 168         d = (struct segment_descriptor *)(table_base + (selector & ~7));
 169         v = d->base_low | ((ul)d->base_mid << 16) | ((ul)d->base_high << 24);
 170 #ifdef CONFIG_X86_64
 171         if (d->system == 0
 172             && (d->type == 2 || d->type == 9 || d->type == 11))
 173                 v |= ((ul)((struct segment_descriptor_64 *)d)->base_higher) << 32;
 174 #endif
 175         return v;
 176 }
 177 EXPORT_SYMBOL_GPL(segment_base);
 178
 179 static inline int valid_vcpu(int n)
 180 {
 181         return likely(n >= 0 && n < KVM_MAX_VCPUS);
 182 }
 183
 184 int kvm_read_guest(struct kvm_vcpu *vcpu, gva_t addr, unsigned long size,
 185                    void *dest)
 186 {
 187         unsigned char *host_buf = dest;
 188         unsigned long req_size = size;
 189
 190         while (size) {
 191                 hpa_t paddr;
 192                 unsigned now;
 193                 unsigned offset;
 194                 hva_t guest_buf;
 195
 196                 paddr = gva_to_hpa(vcpu, addr);
 197
 198                 if (is_error_hpa(paddr))
 199                         break;
 200
 201                 guest_buf = (hva_t)kmap_atomic(
 202                                         pfn_to_page(paddr >> PAGE_SHIFT),
 203                                         KM_USER0);
 204                 offset = addr & ~PAGE_MASK;
 205                 guest_buf |= offset;
 206                 now = min(size, PAGE_SIZE - offset);
 207                 memcpy(host_buf, (void*)guest_buf, now);
 208                 host_buf += now;
 209                 addr += now;
 210                 size -= now;
 211                 kunmap_atomic((void *)(guest_buf & PAGE_MASK), KM_USER0);
 212         }
 213         return req_size - size;
 214 }
 215 EXPORT_SYMBOL_GPL(kvm_read_guest);
 216
 217 int kvm_write_guest(struct kvm_vcpu *vcpu, gva_t addr, unsigned long size,
 218                     void *data)
 219 {
 220         unsigned char *host_buf = data;
 221         unsigned long req_size = size;
 222
 223         while (size) {
 224                 hpa_t paddr;
 225                 unsigned now;
 226                 unsigned offset;
 227                 hva_t guest_buf;
 228
 229                 paddr = gva_to_hpa(vcpu, addr);
 230
 231                 if (is_error_hpa(paddr))
 232                         break;
 233
 234                 guest_buf = (hva_t)kmap_atomic(
 235                                 pfn_to_page(paddr >> PAGE_SHIFT), KM_USER0);
 236                 offset = addr & ~PAGE_MASK;
 237                 guest_buf |= offset;
 238                 now = min(size, PAGE_SIZE - offset);
 239                 memcpy((void*)guest_buf, host_buf, now);
 240                 host_buf += now;
 241                 addr += now;
 242                 size -= now;
 243                 kunmap_atomic((void *)(guest_buf & PAGE_MASK), KM_USER0);
 244         }
 245         return req_size - size;
 246 }
 247 EXPORT_SYMBOL_GPL(kvm_write_guest);
 248
 249 static int vcpu_slot(struct kvm_vcpu *vcpu)
 250 {
 251         return vcpu - vcpu->kvm->vcpus;
 252 }
 253
 254 /*
 255  * Switches to specified vcpu, until a matching vcpu_put()
 256  */
 257 static struct kvm_vcpu *vcpu_load(struct kvm *kvm, int vcpu_slot)
 258 {
 259         struct kvm_vcpu *vcpu = &kvm->vcpus[vcpu_slot];
 260
 261         mutex_lock(&vcpu->mutex);
 262         if (unlikely(!vcpu->vmcs)) {
 263                 mutex_unlock(&vcpu->mutex);
 264                 return NULL;
 265         }
 266         return kvm_arch_ops->vcpu_load(vcpu);
 267 }
 268
 269 static void vcpu_put(struct kvm_vcpu *vcpu)
 270 {
 271         kvm_arch_ops->vcpu_put(vcpu);
 272         mutex_unlock(&vcpu->mutex);
 273 }
 274
 275 static struct kvm *kvm_create_vm(void)
 276 {
 277         struct kvm *kvm = kzalloc(sizeof(struct kvm), GFP_KERNEL);
 278         int i;
 279
 280         if (!kvm)
 281                 return ERR_PTR(-ENOMEM);
 282
 283         spin_lock_init(&kvm->lock);
 284         INIT_LIST_HEAD(&kvm->active_mmu_pages);
 285         for (i = 0; i < KVM_MAX_VCPUS; ++i) {
 286                 struct kvm_vcpu *vcpu = &kvm->vcpus[i];
 287
 288                 mutex_init(&vcpu->mutex);
 289                 vcpu->cpu = -1;
 290                 vcpu->kvm = kvm;
 291                 vcpu->mmu.root_hpa = INVALID_PAGE;
 292                 INIT_LIST_HEAD(&vcpu->free_pages);
 293                 spin_lock(&kvm_lock);
 294                 list_add(&kvm->vm_list, &vm_list);
 295                 spin_unlock(&kvm_lock);
 296         }
 297         return kvm;
 298 }
 299
 300 static int kvm_dev_open(struct inode *inode, struct file *filp)
 301 {
 302         return 0;
 303 }
 304
 305 /*
 306  * Free any memory in @free but not in @dont.
 307  */
 308 static void kvm_free_physmem_slot(struct kvm_memory_slot *free,
 309                                   struct kvm_memory_slot *dont)
 310 {
 311         int i;
 312
 313         if (!dont || free->phys_mem != dont->phys_mem)
 314                 if (free->phys_mem) {
 315                         for (i = 0; i < free->npages; ++i)
 316                                 if (free->phys_mem[i])
 317                                         __free_page(free->phys_mem[i]);
 318                         vfree(free->phys_mem);
 319                 }
 320
 321         if (!dont || free->dirty_bitmap != dont->dirty_bitmap)
 322                 vfree(free->dirty_bitmap);
 323
 324         free->phys_mem = NULL;
 325         free->npages = 0;
 326         free->dirty_bitmap = NULL;
 327 }
 328
 329 static void kvm_free_physmem(struct kvm *kvm)
 330 {
 331         int i;
 332
 333         for (i = 0; i < kvm->nmemslots; ++i)
 334                 kvm_free_physmem_slot(&kvm->memslots[i], NULL);
 335 }
 336
 337 static void kvm_free_vcpu(struct kvm_vcpu *vcpu)
 338 {
 339         if (!vcpu_load(vcpu->kvm, vcpu_slot(vcpu)))
 340                 return;
 341
 342         kvm_mmu_destroy(vcpu);
 343         vcpu_put(vcpu);
 344         kvm_arch_ops->vcpu_free(vcpu);
 345 }
 346
 347 static void kvm_free_vcpus(struct kvm *kvm)
 348 {
 349         unsigned int i;
 350
 351         for (i = 0; i < KVM_MAX_VCPUS; ++i)
 352                 kvm_free_vcpu(&kvm->vcpus[i]);
 353 }
 354
 355 static int kvm_dev_release(struct inode *inode, struct file *filp)
 356 {
 357         return 0;
 358 }
 359
 360 static void kvm_destroy_vm(struct kvm *kvm)
 361 {
 362         spin_lock(&kvm_lock);
 363         list_del(&kvm->vm_list);
 364         spin_unlock(&kvm_lock);
 365         kvm_free_vcpus(kvm);
 366         kvm_free_physmem(kvm);
 367         kfree(kvm);
 368 }
 369
 370 static int kvm_vm_release(struct inode *inode, struct file *filp)
 371 {
 372         struct kvm *kvm = filp->private_data;
 373
 374         kvm_destroy_vm(kvm);
 375         return 0;
 376 }
 377
 378 static void inject_gp(struct kvm_vcpu *vcpu)
 379 {
 380         kvm_arch_ops->inject_gp(vcpu, 0);
 381 }
 382
 383 /*
 384  * Load the pae pdptrs.  Return true is they are all valid.
 385  */
 386 static int load_pdptrs(struct kvm_vcpu *vcpu, unsigned long cr3)
 387 {
 388         gfn_t pdpt_gfn = cr3 >> PAGE_SHIFT;
 389         unsigned offset = ((cr3 & (PAGE_SIZE-1)) >> 5) << 2;
 390         int i;
 391         u64 pdpte;
 392         u64 *pdpt;
 393         int ret;
 394         struct kvm_memory_slot *memslot;
 395
 396         spin_lock(&vcpu->kvm->lock);
 397         memslot = gfn_to_memslot(vcpu->kvm, pdpt_gfn);
 398         /* FIXME: !memslot - emulate? 0xff? */
 399         pdpt = kmap_atomic(gfn_to_page(memslot, pdpt_gfn), KM_USER0);
 400
 401         ret = 1;
 402         for (i = 0; i < 4; ++i) {
 403                 pdpte = pdpt[offset + i];
 404                 if ((pdpte & 1) && (pdpte & 0xfffffff0000001e6ull)) {
 405                         ret = 0;
 406                         goto out;
 407                 }
 408         }
 409
 410         for (i = 0; i < 4; ++i)
 411                 vcpu->pdptrs[i] = pdpt[offset + i];
 412
 413 out:
 414         kunmap_atomic(pdpt, KM_USER0);
 415         spin_unlock(&vcpu->kvm->lock);
 416
 417         return ret;
 418 }
 419
 420 void set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
 421 {
 422         if (cr0 & CR0_RESEVED_BITS) {
 423                 printk(KERN_DEBUG "set_cr0: 0x%lx #GP, reserved bits 0x%lx\n",
 424                        cr0, vcpu->cr0);
 425                 inject_gp(vcpu);
 426                 return;
 427         }
 428
 429         if ((cr0 & CR0_NW_MASK) && !(cr0 & CR0_CD_MASK)) {
 430                 printk(KERN_DEBUG "set_cr0: #GP, CD == 0 && NW == 1\n");
 431                 inject_gp(vcpu);
 432                 return;
 433         }
 434
 435         if ((cr0 & CR0_PG_MASK) && !(cr0 & CR0_PE_MASK)) {
 436                 printk(KERN_DEBUG "set_cr0: #GP, set PG flag "
 437                        "and a clear PE flag\n");
 438                 inject_gp(vcpu);
 439                 return;
 440         }
 441
 442         if (!is_paging(vcpu) && (cr0 & CR0_PG_MASK)) {
 443 #ifdef CONFIG_X86_64
 444                 if ((vcpu->shadow_efer & EFER_LME)) {
 445                         int cs_db, cs_l;
 446
 447                         if (!is_pae(vcpu)) {
 448                                 printk(KERN_DEBUG "set_cr0: #GP, start paging "
 449                                        "in long mode while PAE is disabled\n");
 450                                 inject_gp(vcpu);
 451                                 return;
 452                         }
 453                         kvm_arch_ops->get_cs_db_l_bits(vcpu, &cs_db, &cs_l);
 454                         if (cs_l) {
 455                                 printk(KERN_DEBUG "set_cr0: #GP, start paging "
 456                                        "in long mode while CS.L == 1\n");
 457                                 inject_gp(vcpu);
 458                                 return;
 459
 460                         }
 461                 } else
 462 #endif
 463                 if (is_pae(vcpu) && !load_pdptrs(vcpu, vcpu->cr3)) {
 464                         printk(KERN_DEBUG "set_cr0: #GP, pdptrs "
 465                                "reserved bits\n");
 466                         inject_gp(vcpu);
 467                         return;
 468                 }
 469
 470         }
 471
 472         kvm_arch_ops->set_cr0(vcpu, cr0);
 473         vcpu->cr0 = cr0;
 474
 475         spin_lock(&vcpu->kvm->lock);
 476         kvm_mmu_reset_context(vcpu);
 477         spin_unlock(&vcpu->kvm->lock);
 478         return;
 479 }
 480 EXPORT_SYMBOL_GPL(set_cr0);
 481
 482 void lmsw(struct kvm_vcpu *vcpu, unsigned long msw)
 483 {
 484         kvm_arch_ops->decache_cr0_cr4_guest_bits(vcpu);
 485         set_cr0(vcpu, (vcpu->cr0 & ~0x0ful) | (msw & 0x0f));
 486 }
 487 EXPORT_SYMBOL_GPL(lmsw);
 488
 489 void set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
 490 {
 491         if (cr4 & CR4_RESEVED_BITS) {
 492                 printk(KERN_DEBUG "set_cr4: #GP, reserved bits\n");
 493                 inject_gp(vcpu);
 494                 return;
 495         }
 496
 497         if (is_long_mode(vcpu)) {
 498                 if (!(cr4 & CR4_PAE_MASK)) {
 499                         printk(KERN_DEBUG "set_cr4: #GP, clearing PAE while "
 500                                "in long mode\n");
 501                         inject_gp(vcpu);
 502                         return;
 503                 }
 504         } else if (is_paging(vcpu) && !is_pae(vcpu) && (cr4 & CR4_PAE_MASK)
 505                    && !load_pdptrs(vcpu, vcpu->cr3)) {
 506                 printk(KERN_DEBUG "set_cr4: #GP, pdptrs reserved bits\n");
 507                 inject_gp(vcpu);
 508         }
 509
 510         if (cr4 & CR4_VMXE_MASK) {
 511                 printk(KERN_DEBUG "set_cr4: #GP, setting VMXE\n");
 512                 inject_gp(vcpu);
 513                 return;
 514         }
 515         kvm_arch_ops->set_cr4(vcpu, cr4);
 516         spin_lock(&vcpu->kvm->lock);
 517         kvm_mmu_reset_context(vcpu);
 518         spin_unlock(&vcpu->kvm->lock);
 519 }
 520 EXPORT_SYMBOL_GPL(set_cr4);
 521
 522 void set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3)
 523 {
 524         if (is_long_mode(vcpu)) {
 525                 if (cr3 & CR3_L_MODE_RESEVED_BITS) {
 526                         printk(KERN_DEBUG "set_cr3: #GP, reserved bits\n");
 527                         inject_gp(vcpu);
 528                         return;
 529                 }
 530         } else {
 531                 if (cr3 & CR3_RESEVED_BITS) {
 532                         printk(KERN_DEBUG "set_cr3: #GP, reserved bits\n");
 533                         inject_gp(vcpu);
 534                         return;
 535                 }
 536                 if (is_paging(vcpu) && is_pae(vcpu) &&
 537                     !load_pdptrs(vcpu, cr3)) {
 538                         printk(KERN_DEBUG "set_cr3: #GP, pdptrs "
 539                                "reserved bits\n");
 540                         inject_gp(vcpu);
 541                         return;
 542                 }
 543         }
 544
 545         vcpu->cr3 = cr3;
 546         spin_lock(&vcpu->kvm->lock);
 547         /*
 548          * Does the new cr3 value map to physical memory? (Note, we
 549          * catch an invalid cr3 even in real-mode, because it would
 550          * cause trouble later on when we turn on paging anyway.)
 551          *
 552          * A real CPU would silently accept an invalid cr3 and would
 553          * attempt to use it - with largely undefined (and often hard
 554          * to debug) behavior on the guest side.
 555          */
 556         if (unlikely(!gfn_to_memslot(vcpu->kvm, cr3 >> PAGE_SHIFT)))
 557                 inject_gp(vcpu);
 558         else
 559                 vcpu->mmu.new_cr3(vcpu);
 560         spin_unlock(&vcpu->kvm->lock);
 561 }
 562 EXPORT_SYMBOL_GPL(set_cr3);
 563
 564 void set_cr8(struct kvm_vcpu *vcpu, unsigned long cr8)
 565 {
 566         if ( cr8 & CR8_RESEVED_BITS) {
 567                 printk(KERN_DEBUG "set_cr8: #GP, reserved bits 0x%lx\n", cr8);
 568                 inject_gp(vcpu);
 569                 return;
 570         }
 571         vcpu->cr8 = cr8;
 572 }
 573 EXPORT_SYMBOL_GPL(set_cr8);
 574
 575 void fx_init(struct kvm_vcpu *vcpu)
 576 {
 577         struct __attribute__ ((__packed__)) fx_image_s {
 578                 u16 control; //fcw
 579                 u16 status; //fsw
 580                 u16 tag; // ftw
 581                 u16 opcode; //fop
 582                 u64 ip; // fpu ip
 583                 u64 operand;// fpu dp
 584                 u32 mxcsr;
 585                 u32 mxcsr_mask;
 586
 587         } *fx_image;
 588
 589         fx_save(vcpu->host_fx_image);
 590         fpu_init();
 591         fx_save(vcpu->guest_fx_image);
 592         fx_restore(vcpu->host_fx_image);
 593
 594         fx_image = (struct fx_image_s *)vcpu->guest_fx_image;
 595         fx_image->mxcsr = 0x1f80;
 596         memset(vcpu->guest_fx_image + sizeof(struct fx_image_s),
 597                0, FX_IMAGE_SIZE - sizeof(struct fx_image_s));
 598 }
 599 EXPORT_SYMBOL_GPL(fx_init);
 600
 601 /*
 602  * Allocate some memory and give it an address in the guest physical address
 603  * space.
 604  *
 605  * Discontiguous memory is allowed, mostly for framebuffers.
 606  */
 607 static int kvm_vm_ioctl_set_memory_region(struct kvm *kvm,
 608                                           struct kvm_memory_region *mem)
 609 {
 610         int r;
 611         gfn_t base_gfn;
 612         unsigned long npages;
 613         unsigned long i;
 614         struct kvm_memory_slot *memslot;
 615         struct kvm_memory_slot old, new;
 616         int memory_config_version;
 617
 618         r = -EINVAL;
 619         /* General sanity checks */
 620         if (mem->memory_size & (PAGE_SIZE - 1))
 621                 goto out;
 622         if (mem->guest_phys_addr & (PAGE_SIZE - 1))
 623                 goto out;
 624         if (mem->slot >= KVM_MEMORY_SLOTS)
 625                 goto out;
 626         if (mem->guest_phys_addr + mem->memory_size < mem->guest_phys_addr)
 627                 goto out;
 628
 629         memslot = &kvm->memslots[mem->slot];
 630         base_gfn = mem->guest_phys_addr >> PAGE_SHIFT;
 631         npages = mem->memory_size >> PAGE_SHIFT;
 632
 633         if (!npages)
 634                 mem->flags &= ~KVM_MEM_LOG_DIRTY_PAGES;
 635
 636 raced:
 637         spin_lock(&kvm->lock);
 638
 639         memory_config_version = kvm->memory_config_version;
 640         new = old = *memslot;
 641
 642         new.base_gfn = base_gfn;
 643         new.npages = npages;
 644         new.flags = mem->flags;
 645
 646         /* Disallow changing a memory slot's size. */
 647         r = -EINVAL;
 648         if (npages && old.npages && npages != old.npages)
 649                 goto out_unlock;
 650
 651         /* Check for overlaps */
 652         r = -EEXIST;
 653         for (i = 0; i < KVM_MEMORY_SLOTS; ++i) {
 654                 struct kvm_memory_slot *s = &kvm->memslots[i];
 655
 656                 if (s == memslot)
 657                         continue;
 658                 if (!((base_gfn + npages <= s->base_gfn) ||
 659                       (base_gfn >= s->base_gfn + s->npages)))
 660                         goto out_unlock;
 661         }
 662         /*
 663          * Do memory allocations outside lock.  memory_config_version will
 664          * detect any races.
 665          */
 666         spin_unlock(&kvm->lock);
 667
 668         /* Deallocate if slot is being removed */
 669         if (!npages)
 670                 new.phys_mem = NULL;
 671
 672         /* Free page dirty bitmap if unneeded */
 673         if (!(new.flags & KVM_MEM_LOG_DIRTY_PAGES))
 674                 new.dirty_bitmap = NULL;
 675
 676         r = -ENOMEM;
 677
 678         /* Allocate if a slot is being created */
 679         if (npages && !new.phys_mem) {
 680                 new.phys_mem = vmalloc(npages * sizeof(struct page *));
 681
 682                 if (!new.phys_mem)
 683                         goto out_free;
 684
 685                 memset(new.phys_mem, 0, npages * sizeof(struct page *));
 686                 for (i = 0; i < npages; ++i) {
 687                         new.phys_mem[i] = alloc_page(GFP_HIGHUSER
 688                                                      | __GFP_ZERO);
 689                         if (!new.phys_mem[i])
 690                                 goto out_free;
 691                         set_page_private(new.phys_mem[i],0);
 692                 }
 693         }
 694
 695         /* Allocate page dirty bitmap if needed */
 696         if ((new.flags & KVM_MEM_LOG_DIRTY_PAGES) && !new.dirty_bitmap) {
 697                 unsigned dirty_bytes = ALIGN(npages, BITS_PER_LONG) / 8;
 698
 699                 new.dirty_bitmap = vmalloc(dirty_bytes);
 700                 if (!new.dirty_bitmap)
 701                         goto out_free;
 702                 memset(new.dirty_bitmap, 0, dirty_bytes);
 703         }
 704
 705         spin_lock(&kvm->lock);
 706
 707         if (memory_config_version != kvm->memory_config_version) {
 708                 spin_unlock(&kvm->lock);
 709                 kvm_free_physmem_slot(&new, &old);
 710                 goto raced;
 711         }
 712
 713         r = -EAGAIN;
 714         if (kvm->busy)
 715                 goto out_unlock;
 716
 717         if (mem->slot >= kvm->nmemslots)
 718                 kvm->nmemslots = mem->slot + 1;
 719
 720         *memslot = new;
 721         ++kvm->memory_config_version;
 722
 723         spin_unlock(&kvm->lock);
 724
 725         for (i = 0; i < KVM_MAX_VCPUS; ++i) {
 726                 struct kvm_vcpu *vcpu;
 727
 728                 vcpu = vcpu_load(kvm, i);
 729                 if (!vcpu)
 730                         continue;
 731                 kvm_mmu_reset_context(vcpu);
 732                 vcpu_put(vcpu);
 733         }
 734
 735         kvm_free_physmem_slot(&old, &new);
 736         return 0;
 737
 738 out_unlock:
 739         spin_unlock(&kvm->lock);
 740 out_free:
 741         kvm_free_physmem_slot(&new, &old);
 742 out:
 743         return r;
 744 }
 745
 746 static void do_remove_write_access(struct kvm_vcpu *vcpu, int slot)
 747 {
 748         spin_lock(&vcpu->kvm->lock);
 749         kvm_mmu_slot_remove_write_access(vcpu, slot);
 750         spin_unlock(&vcpu->kvm->lock);
 751 }
 752
 753 /*
 754  * Get (and clear) the dirty memory log for a memory slot.
 755  */
 756 static int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
 757                                       struct kvm_dirty_log *log)
 758 {
 759         struct kvm_memory_slot *memslot;
 760         int r, i;
 761         int n;
 762         int cleared;
 763         unsigned long any = 0;
 764
 765         spin_lock(&kvm->lock);
 766
 767         /*
 768          * Prevent changes to guest memory configuration even while the lock
 769          * is not taken.
 770          */
 771         ++kvm->busy;
 772         spin_unlock(&kvm->lock);
 773         r = -EINVAL;
 774         if (log->slot >= KVM_MEMORY_SLOTS)
 775                 goto out;
 776
 777         memslot = &kvm->memslots[log->slot];
 778         r = -ENOENT;
 779         if (!memslot->dirty_bitmap)
 780                 goto out;
 781
 782         n = ALIGN(memslot->npages, 8) / 8;
 783
 784         for (i = 0; !any && i < n; ++i)
 785                 any = memslot->dirty_bitmap[i];
 786
 787         r = -EFAULT;
 788         if (copy_to_user(log->dirty_bitmap, memslot->dirty_bitmap, n))
 789                 goto out;
 790
 791         if (any) {
 792                 cleared = 0;
 793                 for (i = 0; i < KVM_MAX_VCPUS; ++i) {
 794                         struct kvm_vcpu *vcpu = vcpu_load(kvm, i);
 795
 796                         if (!vcpu)
 797                                 continue;
 798                         if (!cleared) {
 799                                 do_remove_write_access(vcpu, log->slot);
 800                                 memset(memslot->dirty_bitmap, 0, n);
 801                                 cleared = 1;
 802                         }
 803                         kvm_arch_ops->tlb_flush(vcpu);
 804                         vcpu_put(vcpu);
 805                 }
 806         }
 807
 808         r = 0;
 809
 810 out:
 811         spin_lock(&kvm->lock);
 812         --kvm->busy;
 813         spin_unlock(&kvm->lock);
 814         return r;
 815 }
 816
 817 struct kvm_memory_slot *gfn_to_memslot(struct kvm *kvm, gfn_t gfn)
 818 {
 819         int i;
 820
 821         for (i = 0; i < kvm->nmemslots; ++i) {
 822                 struct kvm_memory_slot *memslot = &kvm->memslots[i];
 823
 824                 if (gfn >= memslot->base_gfn
 825                     && gfn < memslot->base_gfn + memslot->npages)
 826                         return memslot;
 827         }
 828         return NULL;
 829 }
 830 EXPORT_SYMBOL_GPL(gfn_to_memslot);
 831
 832 void mark_page_dirty(struct kvm *kvm, gfn_t gfn)
 833 {
 834         int i;
 835         struct kvm_memory_slot *memslot = NULL;
 836         unsigned long rel_gfn;
 837
 838         for (i = 0; i < kvm->nmemslots; ++i) {
 839                 memslot = &kvm->memslots[i];
 840
 841                 if (gfn >= memslot->base_gfn
 842                     && gfn < memslot->base_gfn + memslot->npages) {
 843
 844                         if (!memslot || !memslot->dirty_bitmap)
 845                                 return;
 846
 847                         rel_gfn = gfn - memslot->base_gfn;
 848
 849                         /* avoid RMW */
 850                         if (!test_bit(rel_gfn, memslot->dirty_bitmap))
 851                                 set_bit(rel_gfn, memslot->dirty_bitmap);
 852                         return;
 853                 }
 854         }
 855 }
 856
 857 static int emulator_read_std(unsigned long addr,
 858                              unsigned long *val,
 859                              unsigned int bytes,
 860                              struct x86_emulate_ctxt *ctxt)
 861 {
 862         struct kvm_vcpu *vcpu = ctxt->vcpu;
 863         void *data = val;
 864
 865         while (bytes) {
 866                 gpa_t gpa = vcpu->mmu.gva_to_gpa(vcpu, addr);
 867                 unsigned offset = addr & (PAGE_SIZE-1);
 868                 unsigned tocopy = min(bytes, (unsigned)PAGE_SIZE - offset);
 869                 unsigned long pfn;
 870                 struct kvm_memory_slot *memslot;
 871                 void *page;
 872
 873                 if (gpa == UNMAPPED_GVA)
 874                         return X86EMUL_PROPAGATE_FAULT;
 875                 pfn = gpa >> PAGE_SHIFT;
 876                 memslot = gfn_to_memslot(vcpu->kvm, pfn);
 877                 if (!memslot)
 878                         return X86EMUL_UNHANDLEABLE;
 879                 page = kmap_atomic(gfn_to_page(memslot, pfn), KM_USER0);
 880
 881                 memcpy(data, page + offset, tocopy);
 882
 883                 kunmap_atomic(page, KM_USER0);
 884
 885                 bytes -= tocopy;
 886                 data += tocopy;
 887                 addr += tocopy;
 888         }
 889
 890         return X86EMUL_CONTINUE;
 891 }
 892
 893 static int emulator_write_std(unsigned long addr,
 894                               unsigned long val,
 895                               unsigned int bytes,
 896                               struct x86_emulate_ctxt *ctxt)
 897 {
 898         printk(KERN_ERR "emulator_write_std: addr %lx n %d\n",
 899                addr, bytes);
 900         return X86EMUL_UNHANDLEABLE;
 901 }
 902
 903 static int emulator_read_emulated(unsigned long addr,
 904                                   unsigned long *val,
 905                                   unsigned int bytes,
 906                                   struct x86_emulate_ctxt *ctxt)
 907 {
 908         struct kvm_vcpu *vcpu = ctxt->vcpu;
 909
 910         if (vcpu->mmio_read_completed) {
 911                 memcpy(val, vcpu->mmio_data, bytes);
 912                 vcpu->mmio_read_completed = 0;
 913                 return X86EMUL_CONTINUE;
 914         } else if (emulator_read_std(addr, val, bytes, ctxt)
 915                    == X86EMUL_CONTINUE)
 916                 return X86EMUL_CONTINUE;
 917         else {
 918                 gpa_t gpa = vcpu->mmu.gva_to_gpa(vcpu, addr);
 919
 920                 if (gpa == UNMAPPED_GVA)
 921                         return X86EMUL_PROPAGATE_FAULT;
 922                 vcpu->mmio_needed = 1;
 923                 vcpu->mmio_phys_addr = gpa;
 924                 vcpu->mmio_size = bytes;
 925                 vcpu->mmio_is_write = 0;
 926
 927                 return X86EMUL_UNHANDLEABLE;
 928         }
 929 }
 930
 931 static int emulator_write_phys(struct kvm_vcpu *vcpu, gpa_t gpa,
 932                                unsigned long val, int bytes)
 933 {
 934         struct kvm_memory_slot *m;
 935         struct page *page;
 936         void *virt;
 937
 938         if (((gpa + bytes - 1) >> PAGE_SHIFT) != (gpa >> PAGE_SHIFT))
 939                 return 0;
 940         m = gfn_to_memslot(vcpu->kvm, gpa >> PAGE_SHIFT);
 941         if (!m)
 942                 return 0;
 943         page = gfn_to_page(m, gpa >> PAGE_SHIFT);
 944         kvm_mmu_pre_write(vcpu, gpa, bytes);
 945         virt = kmap_atomic(page, KM_USER0);
 946         memcpy(virt + offset_in_page(gpa), &val, bytes);
 947         kunmap_atomic(virt, KM_USER0);
 948         kvm_mmu_post_write(vcpu, gpa, bytes);
 949         return 1;
 950 }
 951
 952 static int emulator_write_emulated(unsigned long addr,
 953                                    unsigned long val,
 954                                    unsigned int bytes,
 955                                    struct x86_emulate_ctxt *ctxt)
 956 {
 957         struct kvm_vcpu *vcpu = ctxt->vcpu;
 958         gpa_t gpa = vcpu->mmu.gva_to_gpa(vcpu, addr);
 959
 960         if (gpa == UNMAPPED_GVA)
 961                 return X86EMUL_PROPAGATE_FAULT;
 962
 963         if (emulator_write_phys(vcpu, gpa, val, bytes))
 964                 return X86EMUL_CONTINUE;
 965
 966         vcpu->mmio_needed = 1;
 967         vcpu->mmio_phys_addr = gpa;
 968         vcpu->mmio_size = bytes;
 969         vcpu->mmio_is_write = 1;
 970         memcpy(vcpu->mmio_data, &val, bytes);
 971
 972         return X86EMUL_CONTINUE;
 973 }
 974
 975 static int emulator_cmpxchg_emulated(unsigned long addr,
 976                                      unsigned long old,
 977                                      unsigned long new,
 978                                      unsigned int bytes,
 979                                      struct x86_emulate_ctxt *ctxt)
 980 {
 981         static int reported;
 982
 983         if (!reported) {
 984                 reported = 1;
 985                 printk(KERN_WARNING "kvm: emulating exchange as write\n");
 986         }
 987         return emulator_write_emulated(addr, new, bytes, ctxt);
 988 }
 989
 990 #ifdef CONFIG_X86_32
 991
 992 static int emulator_cmpxchg8b_emulated(unsigned long addr,
 993                                        unsigned long old_lo,
 994                                        unsigned long old_hi,
 995                                        unsigned long new_lo,
 996                                        unsigned long new_hi,
 997                                        struct x86_emulate_ctxt *ctxt)
 998 {
 999         static int reported;
1000         int r;
1001
1002         if (!reported) {
1003                 reported = 1;
1004                 printk(KERN_WARNING "kvm: emulating exchange8b as write\n");
1005         }
1006         r = emulator_write_emulated(addr, new_lo, 4, ctxt);
1007         if (r != X86EMUL_CONTINUE)
1008                 return r;
1009         return emulator_write_emulated(addr+4, new_hi, 4, ctxt);
1010 }
1011
1012 #endif
1013
1014 static unsigned long get_segment_base(struct kvm_vcpu *vcpu, int seg)
1015 {
1016         return kvm_arch_ops->get_segment_base(vcpu, seg);
1017 }
1018
1019 int emulate_invlpg(struct kvm_vcpu *vcpu, gva_t address)
1020 {
1021         return X86EMUL_CONTINUE;
1022 }
1023
1024 int emulate_clts(struct kvm_vcpu *vcpu)
1025 {
1026         unsigned long cr0;
1027
1028         kvm_arch_ops->decache_cr0_cr4_guest_bits(vcpu);
1029         cr0 = vcpu->cr0 & ~CR0_TS_MASK;
1030         kvm_arch_ops->set_cr0(vcpu, cr0);
1031         return X86EMUL_CONTINUE;
1032 }
1033
1034 int emulator_get_dr(struct x86_emulate_ctxt* ctxt, int dr, unsigned long *dest)
1035 {
1036         struct kvm_vcpu *vcpu = ctxt->vcpu;
1037
1038         switch (dr) {
1039         case 0 ... 3:
1040                 *dest = kvm_arch_ops->get_dr(vcpu, dr);
1041                 return X86EMUL_CONTINUE;
1042         default:
1043                 printk(KERN_DEBUG "%s: unexpected dr %u\n",
1044                        __FUNCTION__, dr);
1045                 return X86EMUL_UNHANDLEABLE;
1046         }
1047 }
1048
1049 int emulator_set_dr(struct x86_emulate_ctxt *ctxt, int dr, unsigned long value)
1050 {
1051         unsigned long mask = (ctxt->mode == X86EMUL_MODE_PROT64) ? ~0ULL : ~0U;
1052         int exception;
1053
1054         kvm_arch_ops->set_dr(ctxt->vcpu, dr, value & mask, &exception);
1055         if (exception) {
1056                 /* FIXME: better handling */
1057                 return X86EMUL_UNHANDLEABLE;
1058         }
1059         return X86EMUL_CONTINUE;
1060 }
1061
1062 static void report_emulation_failure(struct x86_emulate_ctxt *ctxt)
1063 {
1064         static int reported;
1065         u8 opcodes[4];
1066         unsigned long rip = ctxt->vcpu->rip;
1067         unsigned long rip_linear;
1068
1069         rip_linear = rip + get_segment_base(ctxt->vcpu, VCPU_SREG_CS);
1070
1071         if (reported)
1072                 return;
1073
1074         emulator_read_std(rip_linear, (void *)opcodes, 4, ctxt);
1075
1076         printk(KERN_ERR "emulation failed but !mmio_needed?"
1077                " rip %lx %02x %02x %02x %02x\n",
1078                rip, opcodes[0], opcodes[1], opcodes[2], opcodes[3]);
1079         reported = 1;
1080 }
1081
1082 struct x86_emulate_ops emulate_ops = {
1083         .read_std            = emulator_read_std,
1084         .write_std           = emulator_write_std,
1085         .read_emulated       = emulator_read_emulated,
1086         .write_emulated      = emulator_write_emulated,
1087         .cmpxchg_emulated    = emulator_cmpxchg_emulated,
1088 #ifdef CONFIG_X86_32
1089         .cmpxchg8b_emulated  = emulator_cmpxchg8b_emulated,
1090 #endif
1091 };
1092
1093 int emulate_instruction(struct kvm_vcpu *vcpu,
1094                         struct kvm_run *run,
1095                         unsigned long cr2,
1096                         u16 error_code)
1097 {
1098         struct x86_emulate_ctxt emulate_ctxt;
1099         int r;
1100         int cs_db, cs_l;
1101
1102         kvm_arch_ops->cache_regs(vcpu);
1103
1104         kvm_arch_ops->get_cs_db_l_bits(vcpu, &cs_db, &cs_l);
1105
1106         emulate_ctxt.vcpu = vcpu;
1107         emulate_ctxt.eflags = kvm_arch_ops->get_rflags(vcpu);
1108         emulate_ctxt.cr2 = cr2;
1109         emulate_ctxt.mode = (emulate_ctxt.eflags & X86_EFLAGS_VM)
1110                 ? X86EMUL_MODE_REAL : cs_l
1111                 ? X86EMUL_MODE_PROT64 : cs_db
1112                 ? X86EMUL_MODE_PROT32 : X86EMUL_MODE_PROT16;
1113
1114         if (emulate_ctxt.mode == X86EMUL_MODE_PROT64) {
1115                 emulate_ctxt.cs_base = 0;
1116                 emulate_ctxt.ds_base = 0;
1117                 emulate_ctxt.es_base = 0;
1118                 emulate_ctxt.ss_base = 0;
1119         } else {
1120                 emulate_ctxt.cs_base = get_segment_base(vcpu, VCPU_SREG_CS);
1121                 emulate_ctxt.ds_base = get_segment_base(vcpu, VCPU_SREG_DS);
1122                 emulate_ctxt.es_base = get_segment_base(vcpu, VCPU_SREG_ES);
1123                 emulate_ctxt.ss_base = get_segment_base(vcpu, VCPU_SREG_SS);
1124         }
1125
1126         emulate_ctxt.gs_base = get_segment_base(vcpu, VCPU_SREG_GS);
1127         emulate_ctxt.fs_base = get_segment_base(vcpu, VCPU_SREG_FS);
1128
1129         vcpu->mmio_is_write = 0;
1130         r = x86_emulate_memop(&emulate_ctxt, &emulate_ops);
1131
1132         if ((r || vcpu->mmio_is_write) && run) {
1133                 run->mmio.phys_addr = vcpu->mmio_phys_addr;
1134                 memcpy(run->mmio.data, vcpu->mmio_data, 8);
1135                 run->mmio.len = vcpu->mmio_size;
1136                 run->mmio.is_write = vcpu->mmio_is_write;
1137         }
1138
1139         if (r) {
1140                 if (kvm_mmu_unprotect_page_virt(vcpu, cr2))
1141                         return EMULATE_DONE;
1142                 if (!vcpu->mmio_needed) {
1143                         report_emulation_failure(&emulate_ctxt);
1144                         return EMULATE_FAIL;
1145                 }
1146                 return EMULATE_DO_MMIO;
1147         }
1148
1149         kvm_arch_ops->decache_regs(vcpu);
1150         kvm_arch_ops->set_rflags(vcpu, emulate_ctxt.eflags);
1151
1152         if (vcpu->mmio_is_write)
1153                 return EMULATE_DO_MMIO;
1154
1155         return EMULATE_DONE;
1156 }
1157 EXPORT_SYMBOL_GPL(emulate_instruction);
1158
1159 int kvm_hypercall(struct kvm_vcpu *vcpu, struct kvm_run *run)
1160 {
1161         unsigned long nr, a0, a1, a2, a3, a4, a5, ret;
1162
1163         kvm_arch_ops->decache_regs(vcpu);
1164         ret = -KVM_EINVAL;
1165 #ifdef CONFIG_X86_64
1166         if (is_long_mode(vcpu)) {
1167                 nr = vcpu->regs[VCPU_REGS_RAX];
1168                 a0 = vcpu->regs[VCPU_REGS_RDI];
1169                 a1 = vcpu->regs[VCPU_REGS_RSI];
1170                 a2 = vcpu->regs[VCPU_REGS_RDX];
1171                 a3 = vcpu->regs[VCPU_REGS_RCX];
1172                 a4 = vcpu->regs[VCPU_REGS_R8];
1173                 a5 = vcpu->regs[VCPU_REGS_R9];
1174         } else
1175 #endif
1176         {
1177                 nr = vcpu->regs[VCPU_REGS_RBX] & -1u;
1178                 a0 = vcpu->regs[VCPU_REGS_RAX] & -1u;
1179                 a1 = vcpu->regs[VCPU_REGS_RCX] & -1u;
1180                 a2 = vcpu->regs[VCPU_REGS_RDX] & -1u;
1181                 a3 = vcpu->regs[VCPU_REGS_RSI] & -1u;
1182                 a4 = vcpu->regs[VCPU_REGS_RDI] & -1u;
1183                 a5 = vcpu->regs[VCPU_REGS_RBP] & -1u;
1184         }
1185         switch (nr) {
1186         default:
1187                 ;
1188         }
1189         vcpu->regs[VCPU_REGS_RAX] = ret;
1190         kvm_arch_ops->cache_regs(vcpu);
1191         return 1;
1192 }
1193 EXPORT_SYMBOL_GPL(kvm_hypercall);
1194
1195 static u64 mk_cr_64(u64 curr_cr, u32 new_val)
1196 {
1197         return (curr_cr & ~((1ULL << 32) - 1)) | new_val;
1198 }
1199
1200 void realmode_lgdt(struct kvm_vcpu *vcpu, u16 limit, unsigned long base)
1201 {
1202         struct descriptor_table dt = { limit, base };
1203
1204         kvm_arch_ops->set_gdt(vcpu, &dt);
1205 }
1206
1207 void realmode_lidt(struct kvm_vcpu *vcpu, u16 limit, unsigned long base)
1208 {
1209         struct descriptor_table dt = { limit, base };
1210
1211         kvm_arch_ops->set_idt(vcpu, &dt);
1212 }
1213
1214 void realmode_lmsw(struct kvm_vcpu *vcpu, unsigned long msw,
1215                    unsigned long *rflags)
1216 {
1217         lmsw(vcpu, msw);
1218         *rflags = kvm_arch_ops->get_rflags(vcpu);
1219 }
1220
1221 unsigned long realmode_get_cr(struct kvm_vcpu *vcpu, int cr)
1222 {
1223         kvm_arch_ops->decache_cr0_cr4_guest_bits(vcpu);
1224         switch (cr) {
1225         case 0:
1226                 return vcpu->cr0;
1227         case 2:
1228                 return vcpu->cr2;
1229         case 3:
1230                 return vcpu->cr3;
1231         case 4:
1232                 return vcpu->cr4;
1233         default:
1234                 vcpu_printf(vcpu, "%s: unexpected cr %u\n", __FUNCTION__, cr);
1235                 return 0;
1236         }
1237 }
1238
1239 void realmode_set_cr(struct kvm_vcpu *vcpu, int cr, unsigned long val,
1240                      unsigned long *rflags)
1241 {
1242         switch (cr) {
1243         case 0:
1244                 set_cr0(vcpu, mk_cr_64(vcpu->cr0, val));
1245                 *rflags = kvm_arch_ops->get_rflags(vcpu);
1246                 break;
1247         case 2:
1248                 vcpu->cr2 = val;
1249                 break;
1250         case 3:
1251                 set_cr3(vcpu, val);
1252                 break;
1253         case 4:
1254                 set_cr4(vcpu, mk_cr_64(vcpu->cr4, val));
1255                 break;
1256         default:
1257                 vcpu_printf(vcpu, "%s: unexpected cr %u\n", __FUNCTION__, cr);
1258         }
1259 }
1260
1261 /*
1262  * Register the para guest with the host:
1263  */
1264 static int vcpu_register_para(struct kvm_vcpu *vcpu, gpa_t para_state_gpa)
1265 {
1266         struct kvm_vcpu_para_state *para_state;
1267         hpa_t para_state_hpa, hypercall_hpa;
1268         struct page *para_state_page;
1269         unsigned char *hypercall;
1270         gpa_t hypercall_gpa;
1271
1272         printk(KERN_DEBUG "kvm: guest trying to enter paravirtual mode\n");
1273         printk(KERN_DEBUG ".... para_state_gpa: %08Lx\n", para_state_gpa);
1274
1275         /*
1276          * Needs to be page aligned:
1277          */
1278         if (para_state_gpa != PAGE_ALIGN(para_state_gpa))
1279                 goto err_gp;
1280
1281         para_state_hpa = gpa_to_hpa(vcpu, para_state_gpa);
1282         printk(KERN_DEBUG ".... para_state_hpa: %08Lx\n", para_state_hpa);
1283         if (is_error_hpa(para_state_hpa))
1284                 goto err_gp;
1285
1286         para_state_page = pfn_to_page(para_state_hpa >> PAGE_SHIFT);
1287         para_state = kmap_atomic(para_state_page, KM_USER0);
1288
1289         printk(KERN_DEBUG "....  guest version: %d\n", para_state->guest_version);
1290         printk(KERN_DEBUG "....           size: %d\n", para_state->size);
1291
1292         para_state->host_version = KVM_PARA_API_VERSION;
1293         /*
1294          * We cannot support guests that try to register themselves
1295          * with a newer API version than the host supports:
1296          */
1297         if (para_state->guest_version > KVM_PARA_API_VERSION) {
1298                 para_state->ret = -KVM_EINVAL;
1299                 goto err_kunmap_skip;
1300         }
1301
1302         hypercall_gpa = para_state->hypercall_gpa;
1303         hypercall_hpa = gpa_to_hpa(vcpu, hypercall_gpa);
1304         printk(KERN_DEBUG ".... hypercall_hpa: %08Lx\n", hypercall_hpa);
1305         if (is_error_hpa(hypercall_hpa)) {
1306                 para_state->ret = -KVM_EINVAL;
1307                 goto err_kunmap_skip;
1308         }
1309
1310         printk(KERN_DEBUG "kvm: para guest successfully registered.\n");
1311         vcpu->para_state_page = para_state_page;
1312         vcpu->para_state_gpa = para_state_gpa;
1313         vcpu->hypercall_gpa = hypercall_gpa;
1314
1315         hypercall = kmap_atomic(pfn_to_page(hypercall_hpa >> PAGE_SHIFT),
1316                                 KM_USER1) + (hypercall_hpa & ~PAGE_MASK);
1317         kvm_arch_ops->patch_hypercall(vcpu, hypercall);
1318         kunmap_atomic(hypercall, KM_USER1);
1319
1320         para_state->ret = 0;
1321 err_kunmap_skip:
1322         kunmap_atomic(para_state, KM_USER0);
1323         return 0;
1324 err_gp:
1325         return 1;
1326 }
1327
1328 int kvm_get_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata)
1329 {
1330         u64 data;
1331
1332         switch (msr) {
1333         case 0xc0010010: /* SYSCFG */
1334         case 0xc0010015: /* HWCR */
1335         case MSR_IA32_PLATFORM_ID:
1336         case MSR_IA32_P5_MC_ADDR:
1337         case MSR_IA32_P5_MC_TYPE:
1338         case MSR_IA32_MC0_CTL:
1339         case MSR_IA32_MCG_STATUS:
1340         case MSR_IA32_MCG_CAP:
1341         case MSR_IA32_MC0_MISC:
1342         case MSR_IA32_MC0_MISC+4:
1343         case MSR_IA32_MC0_MISC+8:
1344         case MSR_IA32_MC0_MISC+12:
1345         case MSR_IA32_MC0_MISC+16:
1346         case MSR_IA32_UCODE_REV:
1347         case MSR_IA32_PERF_STATUS:
1348                 /* MTRR registers */
1349         case 0xfe:
1350         case 0x200 ... 0x2ff:
1351                 data = 0;
1352                 break;
1353         case 0xcd: /* fsb frequency */
1354                 data = 3;
1355                 break;
1356         case MSR_IA32_APICBASE:
1357                 data = vcpu->apic_base;
1358                 break;
1359         case MSR_IA32_MISC_ENABLE:
1360                 data = vcpu->ia32_misc_enable_msr;
1361                 break;
1362 #ifdef CONFIG_X86_64
1363         case MSR_EFER:
1364                 data = vcpu->shadow_efer;
1365                 break;
1366 #endif
1367         default:
1368                 printk(KERN_ERR "kvm: unhandled rdmsr: 0x%x\n", msr);
1369                 return 1;
1370         }
1371         *pdata = data;
1372         return 0;
1373 }
1374 EXPORT_SYMBOL_GPL(kvm_get_msr_common);
1375
1376 /*
1377  * Reads an msr value (of 'msr_index') into 'pdata'.
1378  * Returns 0 on success, non-0 otherwise.
1379  * Assumes vcpu_load() was already called.
1380  */
1381 static int get_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 *pdata)
1382 {
1383         return kvm_arch_ops->get_msr(vcpu, msr_index, pdata);
1384 }
1385
1386 #ifdef CONFIG_X86_64
1387
1388 static void set_efer(struct kvm_vcpu *vcpu, u64 efer)
1389 {
1390         if (efer & EFER_RESERVED_BITS) {
1391                 printk(KERN_DEBUG "set_efer: 0x%llx #GP, reserved bits\n",
1392                        efer);
1393                 inject_gp(vcpu);
1394                 return;
1395         }
1396
1397         if (is_paging(vcpu)
1398             && (vcpu->shadow_efer & EFER_LME) != (efer & EFER_LME)) {
1399                 printk(KERN_DEBUG "set_efer: #GP, change LME while paging\n");
1400                 inject_gp(vcpu);
1401                 return;
1402         }
1403
1404         kvm_arch_ops->set_efer(vcpu, efer);
1405
1406         efer &= ~EFER_LMA;
1407         efer |= vcpu->shadow_efer & EFER_LMA;
1408
1409         vcpu->shadow_efer = efer;
1410 }
1411
1412 #endif
1413
1414 int kvm_set_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 data)
1415 {
1416         switch (msr) {
1417 #ifdef CONFIG_X86_64
1418         case MSR_EFER:
1419                 set_efer(vcpu, data);
1420                 break;
1421 #endif
1422         case MSR_IA32_MC0_STATUS:
1423                 printk(KERN_WARNING "%s: MSR_IA32_MC0_STATUS 0x%llx, nop\n",
1424                        __FUNCTION__, data);
1425                 break;
1426         case MSR_IA32_UCODE_REV:
1427         case MSR_IA32_UCODE_WRITE:
1428         case 0x200 ... 0x2ff: /* MTRRs */
1429                 break;
1430         case MSR_IA32_APICBASE:
1431                 vcpu->apic_base = data;
1432                 break;
1433         case MSR_IA32_MISC_ENABLE:
1434                 vcpu->ia32_misc_enable_msr = data;
1435                 break;
1436         /*
1437          * This is the 'probe whether the host is KVM' logic:
1438          */
1439         case MSR_KVM_API_MAGIC:
1440                 return vcpu_register_para(vcpu, data);
1441
1442         default:
1443                 printk(KERN_ERR "kvm: unhandled wrmsr: 0x%x\n", msr);
1444                 return 1;
1445         }
1446         return 0;
1447 }
1448 EXPORT_SYMBOL_GPL(kvm_set_msr_common);
1449
1450 /*
1451  * Writes msr value into into the appropriate "register".
1452  * Returns 0 on success, non-0 otherwise.
1453  * Assumes vcpu_load() was already called.
1454  */
1455 static int set_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 data)
1456 {
1457         return kvm_arch_ops->set_msr(vcpu, msr_index, data);
1458 }
1459
1460 void kvm_resched(struct kvm_vcpu *vcpu)
1461 {
1462         vcpu_put(vcpu);
1463         cond_resched();
1464         /* Cannot fail -  no vcpu unplug yet. */
1465         vcpu_load(vcpu->kvm, vcpu_slot(vcpu));
1466 }
1467 EXPORT_SYMBOL_GPL(kvm_resched);
1468
1469 void load_msrs(struct vmx_msr_entry *e, int n)
1470 {
1471         int i;
1472
1473         for (i = 0; i < n; ++i)
1474                 wrmsrl(e[i].index, e[i].data);
1475 }
1476 EXPORT_SYMBOL_GPL(load_msrs);
1477
1478 void save_msrs(struct vmx_msr_entry *e, int n)
1479 {
1480         int i;
1481
1482         for (i = 0; i < n; ++i)
1483                 rdmsrl(e[i].index, e[i].data);
1484 }
1485 EXPORT_SYMBOL_GPL(save_msrs);
1486
1487 static int kvm_vm_ioctl_run(struct kvm *kvm, struct kvm_run *kvm_run)
1488 {
1489         struct kvm_vcpu *vcpu;
1490         int r;
1491
1492         if (!valid_vcpu(kvm_run->vcpu))
1493                 return -EINVAL;
1494
1495         vcpu = vcpu_load(kvm, kvm_run->vcpu);
1496         if (!vcpu)
1497                 return -ENOENT;
1498
1499         /* re-sync apic's tpr */
1500         vcpu->cr8 = kvm_run->cr8;
1501
1502         if (kvm_run->emulated) {
1503                 kvm_arch_ops->skip_emulated_instruction(vcpu);
1504                 kvm_run->emulated = 0;
1505         }
1506
1507         if (kvm_run->mmio_completed) {
1508                 memcpy(vcpu->mmio_data, kvm_run->mmio.data, 8);
1509                 vcpu->mmio_read_completed = 1;
1510         }
1511
1512         vcpu->mmio_needed = 0;
1513
1514         r = kvm_arch_ops->run(vcpu, kvm_run);
1515
1516         vcpu_put(vcpu);
1517         return r;
1518 }
1519
1520 static int kvm_vm_ioctl_get_regs(struct kvm *kvm, struct kvm_regs *regs)
1521 {
1522         struct kvm_vcpu *vcpu;
1523
1524         if (!valid_vcpu(regs->vcpu))
1525                 return -EINVAL;
1526
1527         vcpu = vcpu_load(kvm, regs->vcpu);
1528         if (!vcpu)
1529                 return -ENOENT;
1530
1531         kvm_arch_ops->cache_regs(vcpu);
1532
1533         regs->rax = vcpu->regs[VCPU_REGS_RAX];
1534         regs->rbx = vcpu->regs[VCPU_REGS_RBX];
1535         regs->rcx = vcpu->regs[VCPU_REGS_RCX];
1536         regs->rdx = vcpu->regs[VCPU_REGS_RDX];
1537         regs->rsi = vcpu->regs[VCPU_REGS_RSI];
1538         regs->rdi = vcpu->regs[VCPU_REGS_RDI];
1539         regs->rsp = vcpu->regs[VCPU_REGS_RSP];
1540         regs->rbp = vcpu->regs[VCPU_REGS_RBP];
1541 #ifdef CONFIG_X86_64
1542         regs->r8 = vcpu->regs[VCPU_REGS_R8];
1543         regs->r9 = vcpu->regs[VCPU_REGS_R9];
1544         regs->r10 = vcpu->regs[VCPU_REGS_R10];
1545         regs->r11 = vcpu->regs[VCPU_REGS_R11];
1546         regs->r12 = vcpu->regs[VCPU_REGS_R12];
1547         regs->r13 = vcpu->regs[VCPU_REGS_R13];
1548         regs->r14 = vcpu->regs[VCPU_REGS_R14];
1549         regs->r15 = vcpu->regs[VCPU_REGS_R15];
1550 #endif
1551
1552         regs->rip = vcpu->rip;
1553         regs->rflags = kvm_arch_ops->get_rflags(vcpu);
1554
1555         /*
1556          * Don't leak debug flags in case they were set for guest debugging
1557          */
1558         if (vcpu->guest_debug.enabled && vcpu->guest_debug.singlestep)
1559                 regs->rflags &= ~(X86_EFLAGS_TF | X86_EFLAGS_RF);
1560
1561         vcpu_put(vcpu);
1562
1563         return 0;
1564 }
1565
1566 static int kvm_vm_ioctl_set_regs(struct kvm *kvm, struct kvm_regs *regs)
1567 {
1568         struct kvm_vcpu *vcpu;
1569
1570         if (!valid_vcpu(regs->vcpu))
1571                 return -EINVAL;
1572
1573         vcpu = vcpu_load(kvm, regs->vcpu);
1574         if (!vcpu)
1575                 return -ENOENT;
1576
1577         vcpu->regs[VCPU_REGS_RAX] = regs->rax;
1578         vcpu->regs[VCPU_REGS_RBX] = regs->rbx;
1579         vcpu->regs[VCPU_REGS_RCX] = regs->rcx;
1580         vcpu->regs[VCPU_REGS_RDX] = regs->rdx;
1581         vcpu->regs[VCPU_REGS_RSI] = regs->rsi;
1582         vcpu->regs[VCPU_REGS_RDI] = regs->rdi;
1583         vcpu->regs[VCPU_REGS_RSP] = regs->rsp;
1584         vcpu->regs[VCPU_REGS_RBP] = regs->rbp;
1585 #ifdef CONFIG_X86_64
1586         vcpu->regs[VCPU_REGS_R8] = regs->r8;
1587         vcpu->regs[VCPU_REGS_R9] = regs->r9;
1588         vcpu->regs[VCPU_REGS_R10] = regs->r10;
1589         vcpu->regs[VCPU_REGS_R11] = regs->r11;
1590         vcpu->regs[VCPU_REGS_R12] = regs->r12;
1591         vcpu->regs[VCPU_REGS_R13] = regs->r13;
1592         vcpu->regs[VCPU_REGS_R14] = regs->r14;
1593         vcpu->regs[VCPU_REGS_R15] = regs->r15;
1594 #endif
1595
1596         vcpu->rip = regs->rip;
1597         kvm_arch_ops->set_rflags(vcpu, regs->rflags);
1598
1599         kvm_arch_ops->decache_regs(vcpu);
1600
1601         vcpu_put(vcpu);
1602
1603         return 0;
1604 }
1605
1606 static void get_segment(struct kvm_vcpu *vcpu,
1607                         struct kvm_segment *var, int seg)
1608 {
1609         return kvm_arch_ops->get_segment(vcpu, var, seg);
1610 }
1611
1612 static int kvm_vm_ioctl_get_sregs(struct kvm *kvm, struct kvm_sregs *sregs)
1613 {
1614         struct kvm_vcpu *vcpu;
1615         struct descriptor_table dt;
1616
1617         if (!valid_vcpu(sregs->vcpu))
1618                 return -EINVAL;
1619         vcpu = vcpu_load(kvm, sregs->vcpu);
1620         if (!vcpu)
1621                 return -ENOENT;
1622
1623         get_segment(vcpu, &sregs->cs, VCPU_SREG_CS);
1624         get_segment(vcpu, &sregs->ds, VCPU_SREG_DS);
1625         get_segment(vcpu, &sregs->es, VCPU_SREG_ES);
1626         get_segment(vcpu, &sregs->fs, VCPU_SREG_FS);
1627         get_segment(vcpu, &sregs->gs, VCPU_SREG_GS);
1628         get_segment(vcpu, &sregs->ss, VCPU_SREG_SS);
1629
1630         get_segment(vcpu, &sregs->tr, VCPU_SREG_TR);
1631         get_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR);
1632
1633         kvm_arch_ops->get_idt(vcpu, &dt);
1634         sregs->idt.limit = dt.limit;
1635         sregs->idt.base = dt.base;
1636         kvm_arch_ops->get_gdt(vcpu, &dt);
1637         sregs->gdt.limit = dt.limit;
1638         sregs->gdt.base = dt.base;
1639
1640         kvm_arch_ops->decache_cr0_cr4_guest_bits(vcpu);
1641         sregs->cr0 = vcpu->cr0;
1642         sregs->cr2 = vcpu->cr2;
1643         sregs->cr3 = vcpu->cr3;
1644         sregs->cr4 = vcpu->cr4;
1645         sregs->cr8 = vcpu->cr8;
1646         sregs->efer = vcpu->shadow_efer;
1647         sregs->apic_base = vcpu->apic_base;
1648
1649         memcpy(sregs->interrupt_bitmap, vcpu->irq_pending,
1650                sizeof sregs->interrupt_bitmap);
1651
1652         vcpu_put(vcpu);
1653
1654         return 0;
1655 }
1656
1657 static void set_segment(struct kvm_vcpu *vcpu,
1658                         struct kvm_segment *var, int seg)
1659 {
1660         return kvm_arch_ops->set_segment(vcpu, var, seg);
1661 }
1662
1663 static int kvm_vm_ioctl_set_sregs(struct kvm *kvm, struct kvm_sregs *sregs)
1664 {
1665         struct kvm_vcpu *vcpu;
1666         int mmu_reset_needed = 0;
1667         int i;
1668         struct descriptor_table dt;
1669
1670         if (!valid_vcpu(sregs->vcpu))
1671                 return -EINVAL;
1672         vcpu = vcpu_load(kvm, sregs->vcpu);
1673         if (!vcpu)
1674                 return -ENOENT;
1675
1676         set_segment(vcpu, &sregs->cs, VCPU_SREG_CS);
1677         set_segment(vcpu, &sregs->ds, VCPU_SREG_DS);
1678         set_segment(vcpu, &sregs->es, VCPU_SREG_ES);
1679         set_segment(vcpu, &sregs->fs, VCPU_SREG_FS);
1680         set_segment(vcpu, &sregs->gs, VCPU_SREG_GS);
1681         set_segment(vcpu, &sregs->ss, VCPU_SREG_SS);
1682
1683         set_segment(vcpu, &sregs->tr, VCPU_SREG_TR);
1684         set_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR);
1685
1686         dt.limit = sregs->idt.limit;
1687         dt.base = sregs->idt.base;
1688         kvm_arch_ops->set_idt(vcpu, &dt);
1689         dt.limit = sregs->gdt.limit;
1690         dt.base = sregs->gdt.base;
1691         kvm_arch_ops->set_gdt(vcpu, &dt);
1692
1693         vcpu->cr2 = sregs->cr2;
1694         mmu_reset_needed |= vcpu->cr3 != sregs->cr3;
1695         vcpu->cr3 = sregs->cr3;
1696
1697         vcpu->cr8 = sregs->cr8;
1698
1699         mmu_reset_needed |= vcpu->shadow_efer != sregs->efer;
1700 #ifdef CONFIG_X86_64
1701         kvm_arch_ops->set_efer(vcpu, sregs->efer);
1702 #endif
1703         vcpu->apic_base = sregs->apic_base;
1704
1705         kvm_arch_ops->decache_cr0_cr4_guest_bits(vcpu);
1706
1707         mmu_reset_needed |= vcpu->cr0 != sregs->cr0;
1708         kvm_arch_ops->set_cr0_no_modeswitch(vcpu, sregs->cr0);
1709
1710         mmu_reset_needed |= vcpu->cr4 != sregs->cr4;
1711         kvm_arch_ops->set_cr4(vcpu, sregs->cr4);
1712         if (!is_long_mode(vcpu) && is_pae(vcpu))
1713                 load_pdptrs(vcpu, vcpu->cr3);
1714
1715         if (mmu_reset_needed)
1716                 kvm_mmu_reset_context(vcpu);
1717
1718         memcpy(vcpu->irq_pending, sregs->interrupt_bitmap,
1719                sizeof vcpu->irq_pending);
1720         vcpu->irq_summary = 0;
1721         for (i = 0; i < NR_IRQ_WORDS; ++i)
1722                 if (vcpu->irq_pending[i])
1723                         __set_bit(i, &vcpu->irq_summary);
1724
1725         vcpu_put(vcpu);
1726
1727         return 0;
1728 }
1729
1730 /*
1731  * List of msr numbers which we expose to userspace through KVM_GET_MSRS
1732  * and KVM_SET_MSRS, and KVM_GET_MSR_INDEX_LIST.
1733  *
1734  * This list is modified at module load time to reflect the
1735  * capabilities of the host cpu.
1736  */
1737 static u32 msrs_to_save[] = {
1738         MSR_IA32_SYSENTER_CS, MSR_IA32_SYSENTER_ESP, MSR_IA32_SYSENTER_EIP,
1739         MSR_K6_STAR,
1740 #ifdef CONFIG_X86_64
1741         MSR_CSTAR, MSR_KERNEL_GS_BASE, MSR_SYSCALL_MASK, MSR_LSTAR,
1742 #endif
1743         MSR_IA32_TIME_STAMP_COUNTER,
1744 };
1745
1746 static unsigned num_msrs_to_save;
1747
1748 static u32 emulated_msrs[] = {
1749         MSR_IA32_MISC_ENABLE,
1750 };
1751
1752 static __init void kvm_init_msr_list(void)
1753 {
1754         u32 dummy[2];
1755         unsigned i, j;
1756
1757         for (i = j = 0; i < ARRAY_SIZE(msrs_to_save); i++) {
1758                 if (rdmsr_safe(msrs_to_save[i], &dummy[0], &dummy[1]) < 0)
1759                         continue;
1760                 if (j < i)
1761                         msrs_to_save[j] = msrs_to_save[i];
1762                 j++;
1763         }
1764         num_msrs_to_save = j;
1765 }
1766
1767 /*
1768  * Adapt set_msr() to msr_io()'s calling convention
1769  */
1770 static int do_set_msr(struct kvm_vcpu *vcpu, unsigned index, u64 *data)
1771 {
1772         return set_msr(vcpu, index, *data);
1773 }
1774
1775 /*
1776  * Read or write a bunch of msrs. All parameters are kernel addresses.
1777  *
1778  * @return number of msrs set successfully.
1779  */
1780 static int __msr_io(struct kvm *kvm, struct kvm_msrs *msrs,
1781                     struct kvm_msr_entry *entries,
1782                     int (*do_msr)(struct kvm_vcpu *vcpu,
1783                                   unsigned index, u64 *data))
1784 {
1785         struct kvm_vcpu *vcpu;
1786         int i;
1787
1788         if (!valid_vcpu(msrs->vcpu))
1789                 return -EINVAL;
1790
1791         vcpu = vcpu_load(kvm, msrs->vcpu);
1792         if (!vcpu)
1793                 return -ENOENT;
1794
1795         for (i = 0; i < msrs->nmsrs; ++i)
1796                 if (do_msr(vcpu, entries[i].index, &entries[i].data))
1797                         break;
1798
1799         vcpu_put(vcpu);
1800
1801         return i;
1802 }
1803
1804 /*
1805  * Read or write a bunch of msrs. Parameters are user addresses.
1806  *
1807  * @return number of msrs set successfully.
1808  */
1809 static int msr_io(struct kvm *kvm, struct kvm_msrs __user *user_msrs,
1810                   int (*do_msr)(struct kvm_vcpu *vcpu,
1811                                 unsigned index, u64 *data),
1812                   int writeback)
1813 {
1814         struct kvm_msrs msrs;
1815         struct kvm_msr_entry *entries;
1816         int r, n;
1817         unsigned size;
1818
1819         r = -EFAULT;
1820         if (copy_from_user(&msrs, user_msrs, sizeof msrs))
1821                 goto out;
1822
1823         r = -E2BIG;
1824         if (msrs.nmsrs >= MAX_IO_MSRS)
1825                 goto out;
1826
1827         r = -ENOMEM;
1828         size = sizeof(struct kvm_msr_entry) * msrs.nmsrs;
1829         entries = vmalloc(size);
1830         if (!entries)
1831                 goto out;
1832
1833         r = -EFAULT;
1834         if (copy_from_user(entries, user_msrs->entries, size))
1835                 goto out_free;
1836
1837         r = n = __msr_io(kvm, &msrs, entries, do_msr);
1838         if (r < 0)
1839                 goto out_free;
1840
1841         r = -EFAULT;
1842         if (writeback && copy_to_user(user_msrs->entries, entries, size))
1843                 goto out_free;
1844
1845         r = n;
1846
1847 out_free:
1848         vfree(entries);
1849 out:
1850         return r;
1851 }
1852
1853 /*
1854  * Translate a guest virtual address to a guest physical address.
1855  */
1856 static int kvm_vm_ioctl_translate(struct kvm *kvm, struct kvm_translation *tr)
1857 {
1858         unsigned long vaddr = tr->linear_address;
1859         struct kvm_vcpu *vcpu;
1860         gpa_t gpa;
1861
1862         vcpu = vcpu_load(kvm, tr->vcpu);
1863         if (!vcpu)
1864                 return -ENOENT;
1865         spin_lock(&kvm->lock);
1866         gpa = vcpu->mmu.gva_to_gpa(vcpu, vaddr);
1867         tr->physical_address = gpa;
1868         tr->valid = gpa != UNMAPPED_GVA;
1869         tr->writeable = 1;
1870         tr->usermode = 0;
1871         spin_unlock(&kvm->lock);
1872         vcpu_put(vcpu);
1873
1874         return 0;
1875 }
1876
1877 static int kvm_vm_ioctl_interrupt(struct kvm *kvm, struct kvm_interrupt *irq)
1878 {
1879         struct kvm_vcpu *vcpu;
1880
1881         if (!valid_vcpu(irq->vcpu))
1882                 return -EINVAL;
1883         if (irq->irq < 0 || irq->irq >= 256)
1884                 return -EINVAL;
1885         vcpu = vcpu_load(kvm, irq->vcpu);
1886         if (!vcpu)
1887                 return -ENOENT;
1888
1889         set_bit(irq->irq, vcpu->irq_pending);
1890         set_bit(irq->irq / BITS_PER_LONG, &vcpu->irq_summary);
1891
1892         vcpu_put(vcpu);
1893
1894         return 0;
1895 }
1896
1897 static int kvm_vm_ioctl_debug_guest(struct kvm *kvm,
1898                                      struct kvm_debug_guest *dbg)
1899 {
1900         struct kvm_vcpu *vcpu;
1901         int r;
1902
1903         if (!valid_vcpu(dbg->vcpu))
1904                 return -EINVAL;
1905         vcpu = vcpu_load(kvm, dbg->vcpu);
1906         if (!vcpu)
1907                 return -ENOENT;
1908
1909         r = kvm_arch_ops->set_guest_debug(vcpu, dbg);
1910
1911         vcpu_put(vcpu);
1912
1913         return r;
1914 }
1915
1916 /*
1917  * Creates some virtual cpus.  Good luck creating more than one.
1918  */
1919 static int kvm_vm_ioctl_create_vcpu(struct kvm *kvm, int n)
1920 {
1921         int r;
1922         struct kvm_vcpu *vcpu;
1923
1924         r = -EINVAL;
1925         if (!valid_vcpu(n))
1926                 goto out;
1927
1928         vcpu = &kvm->vcpus[n];
1929
1930         mutex_lock(&vcpu->mutex);
1931
1932         if (vcpu->vmcs) {
1933                 mutex_unlock(&vcpu->mutex);
1934                 return -EEXIST;
1935         }
1936
1937         vcpu->host_fx_image = (char*)ALIGN((hva_t)vcpu->fx_buf,
1938                                            FX_IMAGE_ALIGN);
1939         vcpu->guest_fx_image = vcpu->host_fx_image + FX_IMAGE_SIZE;
1940
1941         r = kvm_arch_ops->vcpu_create(vcpu);
1942         if (r < 0)
1943                 goto out_free_vcpus;
1944
1945         r = kvm_mmu_create(vcpu);
1946         if (r < 0)
1947                 goto out_free_vcpus;
1948
1949         kvm_arch_ops->vcpu_load(vcpu);
1950         r = kvm_mmu_setup(vcpu);
1951         if (r >= 0)
1952                 r = kvm_arch_ops->vcpu_setup(vcpu);
1953         vcpu_put(vcpu);
1954
1955         if (r < 0)
1956                 goto out_free_vcpus;
1957
1958         return 0;
1959
1960 out_free_vcpus:
1961         kvm_free_vcpu(vcpu);
1962         mutex_unlock(&vcpu->mutex);
1963 out:
1964         return r;
1965 }
1966
1967 static long kvm_vm_ioctl(struct file *filp,
1968                          unsigned int ioctl, unsigned long arg)
1969 {
1970         struct kvm *kvm = filp->private_data;
1971         void __user *argp = (void __user *)arg;
1972         int r = -EINVAL;
1973
1974         switch (ioctl) {
1975         case KVM_CREATE_VCPU:
1976                 r = kvm_vm_ioctl_create_vcpu(kvm, arg);
1977                 if (r)
1978                         goto out;
1979                 break;
1980         case KVM_RUN: {
1981                 struct kvm_run kvm_run;
1982
1983                 r = -EFAULT;
1984                 if (copy_from_user(&kvm_run, argp, sizeof kvm_run))
1985                         goto out;
1986                 r = kvm_vm_ioctl_run(kvm, &kvm_run);
1987                 if (r < 0 &&  r != -EINTR)
1988                         goto out;
1989                 if (copy_to_user(argp, &kvm_run, sizeof kvm_run)) {
1990                         r = -EFAULT;
1991                         goto out;
1992                 }
1993                 break;
1994         }
1995         case KVM_GET_REGS: {
1996                 struct kvm_regs kvm_regs;
1997
1998                 r = -EFAULT;
1999                 if (copy_from_user(&kvm_regs, argp, sizeof kvm_regs))
2000                         goto out;
2001                 r = kvm_vm_ioctl_get_regs(kvm, &kvm_regs);
2002                 if (r)
2003                         goto out;
2004                 r = -EFAULT;
2005                 if (copy_to_user(argp, &kvm_regs, sizeof kvm_regs))
2006                         goto out;
2007                 r = 0;
2008                 break;
2009         }
2010         case KVM_SET_REGS: {
2011                 struct kvm_regs kvm_regs;
2012
2013                 r = -EFAULT;
2014                 if (copy_from_user(&kvm_regs, argp, sizeof kvm_regs))
2015                         goto out;
2016                 r = kvm_vm_ioctl_set_regs(kvm, &kvm_regs);
2017                 if (r)
2018                         goto out;
2019                 r = 0;
2020                 break;
2021         }
2022         case KVM_GET_SREGS: {
2023                 struct kvm_sregs kvm_sregs;
2024
2025                 r = -EFAULT;
2026                 if (copy_from_user(&kvm_sregs, argp, sizeof kvm_sregs))
2027                         goto out;
2028                 r = kvm_vm_ioctl_get_sregs(kvm, &kvm_sregs);
2029                 if (r)
2030                         goto out;
2031                 r = -EFAULT;
2032                 if (copy_to_user(argp, &kvm_sregs, sizeof kvm_sregs))
2033                         goto out;
2034                 r = 0;
2035                 break;
2036         }
2037         case KVM_SET_SREGS: {
2038                 struct kvm_sregs kvm_sregs;
2039
2040                 r = -EFAULT;
2041                 if (copy_from_user(&kvm_sregs, argp, sizeof kvm_sregs))
2042                         goto out;
2043                 r = kvm_vm_ioctl_set_sregs(kvm, &kvm_sregs);
2044                 if (r)
2045                         goto out;
2046                 r = 0;
2047                 break;
2048         }
2049         case KVM_TRANSLATE: {
2050                 struct kvm_translation tr;
2051
2052                 r = -EFAULT;
2053                 if (copy_from_user(&tr, argp, sizeof tr))
2054                         goto out;
2055                 r = kvm_vm_ioctl_translate(kvm, &tr);
2056                 if (r)
2057                         goto out;
2058                 r = -EFAULT;
2059                 if (copy_to_user(argp, &tr, sizeof tr))
2060                         goto out;
2061                 r = 0;
2062                 break;
2063         }
2064         case KVM_INTERRUPT: {
2065                 struct kvm_interrupt irq;
2066
2067                 r = -EFAULT;
2068                 if (copy_from_user(&irq, argp, sizeof irq))
2069                         goto out;
2070                 r = kvm_vm_ioctl_interrupt(kvm, &irq);
2071                 if (r)
2072                         goto out;
2073                 r = 0;
2074                 break;
2075         }
2076         case KVM_DEBUG_GUEST: {
2077                 struct kvm_debug_guest dbg;
2078
2079                 r = -EFAULT;
2080                 if (copy_from_user(&dbg, argp, sizeof dbg))
2081                         goto out;
2082                 r = kvm_vm_ioctl_debug_guest(kvm, &dbg);
2083                 if (r)
2084                         goto out;
2085                 r = 0;
2086                 break;
2087         }
2088         case KVM_SET_MEMORY_REGION: {
2089                 struct kvm_memory_region kvm_mem;
2090
2091                 r = -EFAULT;
2092                 if (copy_from_user(&kvm_mem, argp, sizeof kvm_mem))
2093                         goto out;
2094                 r = kvm_vm_ioctl_set_memory_region(kvm, &kvm_mem);
2095                 if (r)
2096                         goto out;
2097                 break;
2098         }
2099         case KVM_GET_DIRTY_LOG: {
2100                 struct kvm_dirty_log log;
2101
2102                 r = -EFAULT;
2103                 if (copy_from_user(&log, argp, sizeof log))
2104                         goto out;
2105                 r = kvm_vm_ioctl_get_dirty_log(kvm, &log);
2106                 if (r)
2107                         goto out;
2108                 break;
2109         }
2110         case KVM_GET_MSRS:
2111                 r = msr_io(kvm, argp, get_msr, 1);
2112                 break;
2113         case KVM_SET_MSRS:
2114                 r = msr_io(kvm, argp, do_set_msr, 0);
2115                 break;
2116         default:
2117                 ;
2118         }
2119 out:
2120         return r;
2121 }
2122
2123 static struct page *kvm_vm_nopage(struct vm_area_struct *vma,
2124                                   unsigned long address,
2125                                   int *type)
2126 {
2127         struct kvm *kvm = vma->vm_file->private_data;
2128         unsigned long pgoff;
2129         struct kvm_memory_slot *slot;
2130         struct page *page;
2131
2132         *type = VM_FAULT_MINOR;
2133         pgoff = ((address - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff;
2134         slot = gfn_to_memslot(kvm, pgoff);
2135         if (!slot)
2136                 return NOPAGE_SIGBUS;
2137         page = gfn_to_page(slot, pgoff);
2138         if (!page)
2139                 return NOPAGE_SIGBUS;
2140         get_page(page);
2141         return page;
2142 }
2143
2144 static struct vm_operations_struct kvm_vm_vm_ops = {
2145         .nopage = kvm_vm_nopage,
2146 };
2147
2148 static int kvm_vm_mmap(struct file *file, struct vm_area_struct *vma)
2149 {
2150         vma->vm_ops = &kvm_vm_vm_ops;
2151         return 0;
2152 }
2153
2154 static struct file_operations kvm_vm_fops = {
2155         .release        = kvm_vm_release,
2156         .unlocked_ioctl = kvm_vm_ioctl,
2157         .compat_ioctl   = kvm_vm_ioctl,
2158         .mmap           = kvm_vm_mmap,
2159 };
2160
2161 static int kvm_dev_ioctl_create_vm(void)
2162 {
2163         int fd, r;
2164         struct inode *inode;
2165         struct file *file;
2166         struct kvm *kvm;
2167
2168         inode = kvmfs_inode(&kvm_vm_fops);
2169         if (IS_ERR(inode)) {
2170                 r = PTR_ERR(inode);
2171                 goto out1;
2172         }
2173
2174         kvm = kvm_create_vm();
2175         if (IS_ERR(kvm)) {
2176                 r = PTR_ERR(kvm);
2177                 goto out2;
2178         }
2179
2180         file = kvmfs_file(inode, kvm);
2181         if (IS_ERR(file)) {
2182                 r = PTR_ERR(file);
2183                 goto out3;
2184         }
2185
2186         r = get_unused_fd();
2187         if (r < 0)
2188                 goto out4;
2189         fd = r;
2190         fd_install(fd, file);
2191
2192         return fd;
2193
2194 out4:
2195         fput(file);
2196 out3:
2197         kvm_destroy_vm(kvm);
2198 out2:
2199         iput(inode);
2200 out1:
2201         return r;
2202 }
2203
2204 static long kvm_dev_ioctl(struct file *filp,
2205                           unsigned int ioctl, unsigned long arg)
2206 {
2207         void __user *argp = (void __user *)arg;
2208         int r = -EINVAL;
2209
2210         switch (ioctl) {
2211         case KVM_GET_API_VERSION:
2212                 r = KVM_API_VERSION;
2213                 break;
2214         case KVM_CREATE_VM:
2215                 r = kvm_dev_ioctl_create_vm();
2216                 break;
2217         case KVM_GET_MSR_INDEX_LIST: {
2218                 struct kvm_msr_list __user *user_msr_list = argp;
2219                 struct kvm_msr_list msr_list;
2220                 unsigned n;
2221
2222                 r = -EFAULT;
2223                 if (copy_from_user(&msr_list, user_msr_list, sizeof msr_list))
2224                         goto out;
2225                 n = msr_list.nmsrs;
2226                 msr_list.nmsrs = num_msrs_to_save + ARRAY_SIZE(emulated_msrs);
2227                 if (copy_to_user(user_msr_list, &msr_list, sizeof msr_list))
2228                         goto out;
2229                 r = -E2BIG;
2230                 if (n < num_msrs_to_save)
2231                         goto out;
2232                 r = -EFAULT;
2233                 if (copy_to_user(user_msr_list->indices, &msrs_to_save,
2234                                  num_msrs_to_save * sizeof(u32)))
2235                         goto out;
2236                 if (copy_to_user(user_msr_list->indices
2237                                  + num_msrs_to_save * sizeof(u32),
2238                                  &emulated_msrs,
2239                                  ARRAY_SIZE(emulated_msrs) * sizeof(u32)))
2240                         goto out;
2241                 r = 0;
2242                 break;
2243         }
2244         default:
2245                 ;
2246         }
2247 out:
2248         return r;
2249 }
2250
2251 static struct file_operations kvm_chardev_ops = {
2252         .open           = kvm_dev_open,
2253         .release        = kvm_dev_release,
2254         .unlocked_ioctl = kvm_dev_ioctl,
2255         .compat_ioctl   = kvm_dev_ioctl,
2256 };
2257
2258 static struct miscdevice kvm_dev = {
2259         MISC_DYNAMIC_MINOR,
2260         "kvm",
2261         &kvm_chardev_ops,
2262 };
2263
2264 static int kvm_reboot(struct notifier_block *notifier, unsigned long val,
2265                        void *v)
2266 {
2267         if (val == SYS_RESTART) {
2268                 /*
2269                  * Some (well, at least mine) BIOSes hang on reboot if
2270                  * in vmx root mode.
2271                  */
2272                 printk(KERN_INFO "kvm: exiting hardware virtualization\n");
2273                 on_each_cpu(kvm_arch_ops->hardware_disable, NULL, 0, 1);
2274         }
2275         return NOTIFY_OK;
2276 }
2277
2278 static struct notifier_block kvm_reboot_notifier = {
2279         .notifier_call = kvm_reboot,
2280         .priority = 0,
2281 };
2282
2283 /*
2284  * Make sure that a cpu that is being hot-unplugged does not have any vcpus
2285  * cached on it.
2286  */
2287 static void decache_vcpus_on_cpu(int cpu)
2288 {
2289         struct kvm *vm;
2290         struct kvm_vcpu *vcpu;
2291         int i;
2292
2293         spin_lock(&kvm_lock);
2294         list_for_each_entry(vm, &vm_list, vm_list)
2295                 for (i = 0; i < KVM_MAX_VCPUS; ++i) {
2296                         vcpu = &vm->vcpus[i];
2297                         /*
2298                          * If the vcpu is locked, then it is running on some
2299                          * other cpu and therefore it is not cached on the
2300                          * cpu in question.
2301                          *
2302                          * If it's not locked, check the last cpu it executed
2303                          * on.
2304                          */
2305                         if (mutex_trylock(&vcpu->mutex)) {
2306                                 if (vcpu->cpu == cpu) {
2307                                         kvm_arch_ops->vcpu_decache(vcpu);
2308                                         vcpu->cpu = -1;
2309                                 }
2310                                 mutex_unlock(&vcpu->mutex);
2311                         }
2312                 }
2313         spin_unlock(&kvm_lock);
2314 }
2315
2316 static int kvm_cpu_hotplug(struct notifier_block *notifier, unsigned long val,
2317                            void *v)
2318 {
2319         int cpu = (long)v;
2320
2321         switch (val) {
2322         case CPU_DOWN_PREPARE:
2323         case CPU_UP_CANCELED:
2324                 printk(KERN_INFO "kvm: disabling virtualization on CPU%d\n",
2325                        cpu);
2326                 decache_vcpus_on_cpu(cpu);
2327                 smp_call_function_single(cpu, kvm_arch_ops->hardware_disable,
2328                                          NULL, 0, 1);
2329                 break;
2330         case CPU_ONLINE:
2331                 printk(KERN_INFO "kvm: enabling virtualization on CPU%d\n",
2332                        cpu);
2333                 smp_call_function_single(cpu, kvm_arch_ops->hardware_enable,
2334                                          NULL, 0, 1);
2335                 break;
2336         }
2337         return NOTIFY_OK;
2338 }
2339
2340 static struct notifier_block kvm_cpu_notifier = {
2341         .notifier_call = kvm_cpu_hotplug,
2342         .priority = 20, /* must be > scheduler priority */
2343 };
2344
2345 static __init void kvm_init_debug(void)
2346 {
2347         struct kvm_stats_debugfs_item *p;
2348
2349         debugfs_dir = debugfs_create_dir("kvm", NULL);
2350         for (p = debugfs_entries; p->name; ++p)
2351                 p->dentry = debugfs_create_u32(p->name, 0444, debugfs_dir,
2352                                                p->data);
2353 }
2354
2355 static void kvm_exit_debug(void)
2356 {
2357         struct kvm_stats_debugfs_item *p;
2358
2359         for (p = debugfs_entries; p->name; ++p)
2360                 debugfs_remove(p->dentry);
2361         debugfs_remove(debugfs_dir);
2362 }
2363
2364 static int kvm_suspend(struct sys_device *dev, pm_message_t state)
2365 {
2366         decache_vcpus_on_cpu(raw_smp_processor_id());
2367         on_each_cpu(kvm_arch_ops->hardware_disable, NULL, 0, 1);
2368         return 0;
2369 }
2370
2371 static int kvm_resume(struct sys_device *dev)
2372 {
2373         on_each_cpu(kvm_arch_ops->hardware_enable, NULL, 0, 1);
2374         return 0;
2375 }
2376
2377 static struct sysdev_class kvm_sysdev_class = {
2378         set_kset_name("kvm"),
2379         .suspend = kvm_suspend,
2380         .resume = kvm_resume,
2381 };
2382
2383 static struct sys_device kvm_sysdev = {
2384         .id = 0,
2385         .cls = &kvm_sysdev_class,
2386 };
2387
2388 hpa_t bad_page_address;
2389
2390 static int kvmfs_get_sb(struct file_system_type *fs_type, int flags,
2391                         const char *dev_name, void *data, struct vfsmount *mnt)
2392 {
2393         return get_sb_pseudo(fs_type, "kvm:", NULL, KVMFS_MAGIC, mnt);
2394 }
2395
2396 static struct file_system_type kvm_fs_type = {
2397         .name           = "kvmfs",
2398         .get_sb         = kvmfs_get_sb,
2399         .kill_sb        = kill_anon_super,
2400 };
2401
2402 int kvm_init_arch(struct kvm_arch_ops *ops, struct module *module)
2403 {
2404         int r;
2405
2406         if (kvm_arch_ops) {
2407                 printk(KERN_ERR "kvm: already loaded the other module\n");
2408                 return -EEXIST;
2409         }
2410
2411         if (!ops->cpu_has_kvm_support()) {
2412                 printk(KERN_ERR "kvm: no hardware support\n");
2413                 return -EOPNOTSUPP;
2414         }
2415         if (ops->disabled_by_bios()) {
2416                 printk(KERN_ERR "kvm: disabled by bios\n");
2417                 return -EOPNOTSUPP;
2418         }
2419
2420         kvm_arch_ops = ops;
2421
2422         r = kvm_arch_ops->hardware_setup();
2423         if (r < 0)
2424             return r;
2425
2426         on_each_cpu(kvm_arch_ops->hardware_enable, NULL, 0, 1);
2427         r = register_cpu_notifier(&kvm_cpu_notifier);
2428         if (r)
2429                 goto out_free_1;
2430         register_reboot_notifier(&kvm_reboot_notifier);
2431
2432         r = sysdev_class_register(&kvm_sysdev_class);
2433         if (r)
2434                 goto out_free_2;
2435
2436         r = sysdev_register(&kvm_sysdev);
2437         if (r)
2438                 goto out_free_3;
2439
2440         kvm_chardev_ops.owner = module;
2441
2442         r = misc_register(&kvm_dev);
2443         if (r) {
2444                 printk (KERN_ERR "kvm: misc device register failed\n");
2445                 goto out_free;
2446         }
2447
2448         return r;
2449
2450 out_free:
2451         sysdev_unregister(&kvm_sysdev);
2452 out_free_3:
2453         sysdev_class_unregister(&kvm_sysdev_class);
2454 out_free_2:
2455         unregister_reboot_notifier(&kvm_reboot_notifier);
2456         unregister_cpu_notifier(&kvm_cpu_notifier);
2457 out_free_1:
2458         on_each_cpu(kvm_arch_ops->hardware_disable, NULL, 0, 1);
2459         kvm_arch_ops->hardware_unsetup();
2460         return r;
2461 }
2462
2463 void kvm_exit_arch(void)
2464 {
2465         misc_deregister(&kvm_dev);
2466         sysdev_unregister(&kvm_sysdev);
2467         sysdev_class_unregister(&kvm_sysdev_class);
2468         unregister_reboot_notifier(&kvm_reboot_notifier);
2469         unregister_cpu_notifier(&kvm_cpu_notifier);
2470         on_each_cpu(kvm_arch_ops->hardware_disable, NULL, 0, 1);
2471         kvm_arch_ops->hardware_unsetup();
2472         kvm_arch_ops = NULL;
2473 }
2474
2475 static __init int kvm_init(void)
2476 {
2477         static struct page *bad_page;
2478         int r;
2479
2480         r = register_filesystem(&kvm_fs_type);
2481         if (r)
2482                 goto out3;
2483
2484         kvmfs_mnt = kern_mount(&kvm_fs_type);
2485         r = PTR_ERR(kvmfs_mnt);
2486         if (IS_ERR(kvmfs_mnt))
2487                 goto out2;
2488         kvm_init_debug();
2489
2490         kvm_init_msr_list();
2491
2492         if ((bad_page = alloc_page(GFP_KERNEL)) == NULL) {
2493                 r = -ENOMEM;
2494                 goto out;
2495         }
2496
2497         bad_page_address = page_to_pfn(bad_page) << PAGE_SHIFT;
2498         memset(__va(bad_page_address), 0, PAGE_SIZE);
2499
2500         return r;
2501
2502 out:
2503         kvm_exit_debug();
2504         mntput(kvmfs_mnt);
2505 out2:
2506         unregister_filesystem(&kvm_fs_type);
2507 out3:
2508         return r;
2509 }
2510
2511 static __exit void kvm_exit(void)
2512 {
2513         kvm_exit_debug();
2514         __free_page(pfn_to_page(bad_page_address >> PAGE_SHIFT));
2515         mntput(kvmfs_mnt);
2516         unregister_filesystem(&kvm_fs_type);
2517 }
2518
2519 module_init(kvm_init)
2520 module_exit(kvm_exit)
2521
2522 EXPORT_SYMBOL_GPL(kvm_init_arch);
2523 EXPORT_SYMBOL_GPL(kvm_exit_arch);