git.oblomov.eu Git - linux-2.6/blob - drivers/kvm/kvm_main.c

   1 /*
   2  * Kernel-based Virtual Machine driver for Linux
   3  *
   4  * This module enables machines with Intel VT-x extensions to run virtual
   5  * machines without emulation or binary translation.
   6  *
   7  * Copyright (C) 2006 Qumranet, Inc.
   8  *
   9  * Authors:
  10  *   Avi Kivity   <avi@qumranet.com>
  11  *   Yaniv Kamay  <yaniv@qumranet.com>
  12  *
  13  * This work is licensed under the terms of the GNU GPL, version 2.  See
  14  * the COPYING file in the top-level directory.
  15  *
  16  */
  17
  18 #include "kvm.h"
  19
  20 #include <linux/kvm.h>
  21 #include <linux/module.h>
  22 #include <linux/errno.h>
  23 #include <asm/processor.h>
  24 #include <linux/percpu.h>
  25 #include <linux/gfp.h>
  26 #include <asm/msr.h>
  27 #include <linux/mm.h>
  28 #include <linux/miscdevice.h>
  29 #include <linux/vmalloc.h>
  30 #include <asm/uaccess.h>
  31 #include <linux/reboot.h>
  32 #include <asm/io.h>
  33 #include <linux/debugfs.h>
  34 #include <linux/highmem.h>
  35 #include <linux/file.h>
  36 #include <asm/desc.h>
  37 #include <linux/sysdev.h>
  38 #include <linux/cpu.h>
  39 #include <linux/fs.h>
  40 #include <linux/mount.h>
  41
  42 #include "x86_emulate.h"
  43 #include "segment_descriptor.h"
  44
  45 MODULE_AUTHOR("Qumranet");
  46 MODULE_LICENSE("GPL");
  47
  48 static DEFINE_SPINLOCK(kvm_lock);
  49 static LIST_HEAD(vm_list);
  50
  51 struct kvm_arch_ops *kvm_arch_ops;
  52 struct kvm_stat kvm_stat;
  53 EXPORT_SYMBOL_GPL(kvm_stat);
  54
  55 static struct kvm_stats_debugfs_item {
  56         const char *name;
  57         u32 *data;
  58         struct dentry *dentry;
  59 } debugfs_entries[] = {
  60         { "pf_fixed", &kvm_stat.pf_fixed },
  61         { "pf_guest", &kvm_stat.pf_guest },
  62         { "tlb_flush", &kvm_stat.tlb_flush },
  63         { "invlpg", &kvm_stat.invlpg },
  64         { "exits", &kvm_stat.exits },
  65         { "io_exits", &kvm_stat.io_exits },
  66         { "mmio_exits", &kvm_stat.mmio_exits },
  67         { "signal_exits", &kvm_stat.signal_exits },
  68         { "irq_window", &kvm_stat.irq_window_exits },
  69         { "halt_exits", &kvm_stat.halt_exits },
  70         { "request_irq", &kvm_stat.request_irq_exits },
  71         { "irq_exits", &kvm_stat.irq_exits },
  72         { NULL, NULL }
  73 };
  74
  75 static struct dentry *debugfs_dir;
  76
  77 #define KVMFS_MAGIC 0x19700426
  78 struct vfsmount *kvmfs_mnt;
  79
  80 #define MAX_IO_MSRS 256
  81
  82 #define CR0_RESEVED_BITS 0xffffffff1ffaffc0ULL
  83 #define LMSW_GUEST_MASK 0x0eULL
  84 #define CR4_RESEVED_BITS (~((1ULL << 11) - 1))
  85 #define CR8_RESEVED_BITS (~0x0fULL)
  86 #define EFER_RESERVED_BITS 0xfffffffffffff2fe
  87
  88 #ifdef CONFIG_X86_64
  89 // LDT or TSS descriptor in the GDT. 16 bytes.
  90 struct segment_descriptor_64 {
  91         struct segment_descriptor s;
  92         u32 base_higher;
  93         u32 pad_zero;
  94 };
  95
  96 #endif
  97
  98 unsigned long segment_base(u16 selector)
  99 {
 100         struct descriptor_table gdt;
 101         struct segment_descriptor *d;
 102         unsigned long table_base;
 103         typedef unsigned long ul;
 104         unsigned long v;
 105
 106         if (selector == 0)
 107                 return 0;
 108
 109         asm ("sgdt %0" : "=m"(gdt));
 110         table_base = gdt.base;
 111
 112         if (selector & 4) {           /* from ldt */
 113                 u16 ldt_selector;
 114
 115                 asm ("sldt %0" : "=g"(ldt_selector));
 116                 table_base = segment_base(ldt_selector);
 117         }
 118         d = (struct segment_descriptor *)(table_base + (selector & ~7));
 119         v = d->base_low | ((ul)d->base_mid << 16) | ((ul)d->base_high << 24);
 120 #ifdef CONFIG_X86_64
 121         if (d->system == 0
 122             && (d->type == 2 || d->type == 9 || d->type == 11))
 123                 v |= ((ul)((struct segment_descriptor_64 *)d)->base_higher) << 32;
 124 #endif
 125         return v;
 126 }
 127 EXPORT_SYMBOL_GPL(segment_base);
 128
 129 static inline int valid_vcpu(int n)
 130 {
 131         return likely(n >= 0 && n < KVM_MAX_VCPUS);
 132 }
 133
 134 int kvm_read_guest(struct kvm_vcpu *vcpu, gva_t addr, unsigned long size,
 135                    void *dest)
 136 {
 137         unsigned char *host_buf = dest;
 138         unsigned long req_size = size;
 139
 140         while (size) {
 141                 hpa_t paddr;
 142                 unsigned now;
 143                 unsigned offset;
 144                 hva_t guest_buf;
 145
 146                 paddr = gva_to_hpa(vcpu, addr);
 147
 148                 if (is_error_hpa(paddr))
 149                         break;
 150
 151                 guest_buf = (hva_t)kmap_atomic(
 152                                         pfn_to_page(paddr >> PAGE_SHIFT),
 153                                         KM_USER0);
 154                 offset = addr & ~PAGE_MASK;
 155                 guest_buf |= offset;
 156                 now = min(size, PAGE_SIZE - offset);
 157                 memcpy(host_buf, (void*)guest_buf, now);
 158                 host_buf += now;
 159                 addr += now;
 160                 size -= now;
 161                 kunmap_atomic((void *)(guest_buf & PAGE_MASK), KM_USER0);
 162         }
 163         return req_size - size;
 164 }
 165 EXPORT_SYMBOL_GPL(kvm_read_guest);
 166
 167 int kvm_write_guest(struct kvm_vcpu *vcpu, gva_t addr, unsigned long size,
 168                     void *data)
 169 {
 170         unsigned char *host_buf = data;
 171         unsigned long req_size = size;
 172
 173         while (size) {
 174                 hpa_t paddr;
 175                 unsigned now;
 176                 unsigned offset;
 177                 hva_t guest_buf;
 178
 179                 paddr = gva_to_hpa(vcpu, addr);
 180
 181                 if (is_error_hpa(paddr))
 182                         break;
 183
 184                 guest_buf = (hva_t)kmap_atomic(
 185                                 pfn_to_page(paddr >> PAGE_SHIFT), KM_USER0);
 186                 offset = addr & ~PAGE_MASK;
 187                 guest_buf |= offset;
 188                 now = min(size, PAGE_SIZE - offset);
 189                 memcpy((void*)guest_buf, host_buf, now);
 190                 host_buf += now;
 191                 addr += now;
 192                 size -= now;
 193                 kunmap_atomic((void *)(guest_buf & PAGE_MASK), KM_USER0);
 194         }
 195         return req_size - size;
 196 }
 197 EXPORT_SYMBOL_GPL(kvm_write_guest);
 198
 199 static int vcpu_slot(struct kvm_vcpu *vcpu)
 200 {
 201         return vcpu - vcpu->kvm->vcpus;
 202 }
 203
 204 /*
 205  * Switches to specified vcpu, until a matching vcpu_put()
 206  */
 207 static struct kvm_vcpu *vcpu_load(struct kvm *kvm, int vcpu_slot)
 208 {
 209         struct kvm_vcpu *vcpu = &kvm->vcpus[vcpu_slot];
 210
 211         mutex_lock(&vcpu->mutex);
 212         if (unlikely(!vcpu->vmcs)) {
 213                 mutex_unlock(&vcpu->mutex);
 214                 return NULL;
 215         }
 216         return kvm_arch_ops->vcpu_load(vcpu);
 217 }
 218
 219 static void vcpu_put(struct kvm_vcpu *vcpu)
 220 {
 221         kvm_arch_ops->vcpu_put(vcpu);
 222         mutex_unlock(&vcpu->mutex);
 223 }
 224
 225 static int kvm_dev_open(struct inode *inode, struct file *filp)
 226 {
 227         struct kvm *kvm = kzalloc(sizeof(struct kvm), GFP_KERNEL);
 228         int i;
 229
 230         if (!kvm)
 231                 return -ENOMEM;
 232
 233         spin_lock_init(&kvm->lock);
 234         INIT_LIST_HEAD(&kvm->active_mmu_pages);
 235         for (i = 0; i < KVM_MAX_VCPUS; ++i) {
 236                 struct kvm_vcpu *vcpu = &kvm->vcpus[i];
 237
 238                 mutex_init(&vcpu->mutex);
 239                 vcpu->cpu = -1;
 240                 vcpu->kvm = kvm;
 241                 vcpu->mmu.root_hpa = INVALID_PAGE;
 242                 INIT_LIST_HEAD(&vcpu->free_pages);
 243                 spin_lock(&kvm_lock);
 244                 list_add(&kvm->vm_list, &vm_list);
 245                 spin_unlock(&kvm_lock);
 246         }
 247         filp->private_data = kvm;
 248         return 0;
 249 }
 250
 251 /*
 252  * Free any memory in @free but not in @dont.
 253  */
 254 static void kvm_free_physmem_slot(struct kvm_memory_slot *free,
 255                                   struct kvm_memory_slot *dont)
 256 {
 257         int i;
 258
 259         if (!dont || free->phys_mem != dont->phys_mem)
 260                 if (free->phys_mem) {
 261                         for (i = 0; i < free->npages; ++i)
 262                                 if (free->phys_mem[i])
 263                                         __free_page(free->phys_mem[i]);
 264                         vfree(free->phys_mem);
 265                 }
 266
 267         if (!dont || free->dirty_bitmap != dont->dirty_bitmap)
 268                 vfree(free->dirty_bitmap);
 269
 270         free->phys_mem = NULL;
 271         free->npages = 0;
 272         free->dirty_bitmap = NULL;
 273 }
 274
 275 static void kvm_free_physmem(struct kvm *kvm)
 276 {
 277         int i;
 278
 279         for (i = 0; i < kvm->nmemslots; ++i)
 280                 kvm_free_physmem_slot(&kvm->memslots[i], NULL);
 281 }
 282
 283 static void kvm_free_vcpu(struct kvm_vcpu *vcpu)
 284 {
 285         if (!vcpu_load(vcpu->kvm, vcpu_slot(vcpu)))
 286                 return;
 287
 288         kvm_mmu_destroy(vcpu);
 289         vcpu_put(vcpu);
 290         kvm_arch_ops->vcpu_free(vcpu);
 291 }
 292
 293 static void kvm_free_vcpus(struct kvm *kvm)
 294 {
 295         unsigned int i;
 296
 297         for (i = 0; i < KVM_MAX_VCPUS; ++i)
 298                 kvm_free_vcpu(&kvm->vcpus[i]);
 299 }
 300
 301 static int kvm_dev_release(struct inode *inode, struct file *filp)
 302 {
 303         struct kvm *kvm = filp->private_data;
 304
 305         spin_lock(&kvm_lock);
 306         list_del(&kvm->vm_list);
 307         spin_unlock(&kvm_lock);
 308         kvm_free_vcpus(kvm);
 309         kvm_free_physmem(kvm);
 310         kfree(kvm);
 311         return 0;
 312 }
 313
 314 static void inject_gp(struct kvm_vcpu *vcpu)
 315 {
 316         kvm_arch_ops->inject_gp(vcpu, 0);
 317 }
 318
 319 /*
 320  * Load the pae pdptrs.  Return true is they are all valid.
 321  */
 322 static int load_pdptrs(struct kvm_vcpu *vcpu, unsigned long cr3)
 323 {
 324         gfn_t pdpt_gfn = cr3 >> PAGE_SHIFT;
 325         unsigned offset = ((cr3 & (PAGE_SIZE-1)) >> 5) << 2;
 326         int i;
 327         u64 pdpte;
 328         u64 *pdpt;
 329         int ret;
 330         struct kvm_memory_slot *memslot;
 331
 332         spin_lock(&vcpu->kvm->lock);
 333         memslot = gfn_to_memslot(vcpu->kvm, pdpt_gfn);
 334         /* FIXME: !memslot - emulate? 0xff? */
 335         pdpt = kmap_atomic(gfn_to_page(memslot, pdpt_gfn), KM_USER0);
 336
 337         ret = 1;
 338         for (i = 0; i < 4; ++i) {
 339                 pdpte = pdpt[offset + i];
 340                 if ((pdpte & 1) && (pdpte & 0xfffffff0000001e6ull)) {
 341                         ret = 0;
 342                         goto out;
 343                 }
 344         }
 345
 346         for (i = 0; i < 4; ++i)
 347                 vcpu->pdptrs[i] = pdpt[offset + i];
 348
 349 out:
 350         kunmap_atomic(pdpt, KM_USER0);
 351         spin_unlock(&vcpu->kvm->lock);
 352
 353         return ret;
 354 }
 355
 356 void set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
 357 {
 358         if (cr0 & CR0_RESEVED_BITS) {
 359                 printk(KERN_DEBUG "set_cr0: 0x%lx #GP, reserved bits 0x%lx\n",
 360                        cr0, vcpu->cr0);
 361                 inject_gp(vcpu);
 362                 return;
 363         }
 364
 365         if ((cr0 & CR0_NW_MASK) && !(cr0 & CR0_CD_MASK)) {
 366                 printk(KERN_DEBUG "set_cr0: #GP, CD == 0 && NW == 1\n");
 367                 inject_gp(vcpu);
 368                 return;
 369         }
 370
 371         if ((cr0 & CR0_PG_MASK) && !(cr0 & CR0_PE_MASK)) {
 372                 printk(KERN_DEBUG "set_cr0: #GP, set PG flag "
 373                        "and a clear PE flag\n");
 374                 inject_gp(vcpu);
 375                 return;
 376         }
 377
 378         if (!is_paging(vcpu) && (cr0 & CR0_PG_MASK)) {
 379 #ifdef CONFIG_X86_64
 380                 if ((vcpu->shadow_efer & EFER_LME)) {
 381                         int cs_db, cs_l;
 382
 383                         if (!is_pae(vcpu)) {
 384                                 printk(KERN_DEBUG "set_cr0: #GP, start paging "
 385                                        "in long mode while PAE is disabled\n");
 386                                 inject_gp(vcpu);
 387                                 return;
 388                         }
 389                         kvm_arch_ops->get_cs_db_l_bits(vcpu, &cs_db, &cs_l);
 390                         if (cs_l) {
 391                                 printk(KERN_DEBUG "set_cr0: #GP, start paging "
 392                                        "in long mode while CS.L == 1\n");
 393                                 inject_gp(vcpu);
 394                                 return;
 395
 396                         }
 397                 } else
 398 #endif
 399                 if (is_pae(vcpu) && !load_pdptrs(vcpu, vcpu->cr3)) {
 400                         printk(KERN_DEBUG "set_cr0: #GP, pdptrs "
 401                                "reserved bits\n");
 402                         inject_gp(vcpu);
 403                         return;
 404                 }
 405
 406         }
 407
 408         kvm_arch_ops->set_cr0(vcpu, cr0);
 409         vcpu->cr0 = cr0;
 410
 411         spin_lock(&vcpu->kvm->lock);
 412         kvm_mmu_reset_context(vcpu);
 413         spin_unlock(&vcpu->kvm->lock);
 414         return;
 415 }
 416 EXPORT_SYMBOL_GPL(set_cr0);
 417
 418 void lmsw(struct kvm_vcpu *vcpu, unsigned long msw)
 419 {
 420         kvm_arch_ops->decache_cr0_cr4_guest_bits(vcpu);
 421         set_cr0(vcpu, (vcpu->cr0 & ~0x0ful) | (msw & 0x0f));
 422 }
 423 EXPORT_SYMBOL_GPL(lmsw);
 424
 425 void set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
 426 {
 427         if (cr4 & CR4_RESEVED_BITS) {
 428                 printk(KERN_DEBUG "set_cr4: #GP, reserved bits\n");
 429                 inject_gp(vcpu);
 430                 return;
 431         }
 432
 433         if (is_long_mode(vcpu)) {
 434                 if (!(cr4 & CR4_PAE_MASK)) {
 435                         printk(KERN_DEBUG "set_cr4: #GP, clearing PAE while "
 436                                "in long mode\n");
 437                         inject_gp(vcpu);
 438                         return;
 439                 }
 440         } else if (is_paging(vcpu) && !is_pae(vcpu) && (cr4 & CR4_PAE_MASK)
 441                    && !load_pdptrs(vcpu, vcpu->cr3)) {
 442                 printk(KERN_DEBUG "set_cr4: #GP, pdptrs reserved bits\n");
 443                 inject_gp(vcpu);
 444         }
 445
 446         if (cr4 & CR4_VMXE_MASK) {
 447                 printk(KERN_DEBUG "set_cr4: #GP, setting VMXE\n");
 448                 inject_gp(vcpu);
 449                 return;
 450         }
 451         kvm_arch_ops->set_cr4(vcpu, cr4);
 452         spin_lock(&vcpu->kvm->lock);
 453         kvm_mmu_reset_context(vcpu);
 454         spin_unlock(&vcpu->kvm->lock);
 455 }
 456 EXPORT_SYMBOL_GPL(set_cr4);
 457
 458 void set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3)
 459 {
 460         if (is_long_mode(vcpu)) {
 461                 if (cr3 & CR3_L_MODE_RESEVED_BITS) {
 462                         printk(KERN_DEBUG "set_cr3: #GP, reserved bits\n");
 463                         inject_gp(vcpu);
 464                         return;
 465                 }
 466         } else {
 467                 if (cr3 & CR3_RESEVED_BITS) {
 468                         printk(KERN_DEBUG "set_cr3: #GP, reserved bits\n");
 469                         inject_gp(vcpu);
 470                         return;
 471                 }
 472                 if (is_paging(vcpu) && is_pae(vcpu) &&
 473                     !load_pdptrs(vcpu, cr3)) {
 474                         printk(KERN_DEBUG "set_cr3: #GP, pdptrs "
 475                                "reserved bits\n");
 476                         inject_gp(vcpu);
 477                         return;
 478                 }
 479         }
 480
 481         vcpu->cr3 = cr3;
 482         spin_lock(&vcpu->kvm->lock);
 483         /*
 484          * Does the new cr3 value map to physical memory? (Note, we
 485          * catch an invalid cr3 even in real-mode, because it would
 486          * cause trouble later on when we turn on paging anyway.)
 487          *
 488          * A real CPU would silently accept an invalid cr3 and would
 489          * attempt to use it - with largely undefined (and often hard
 490          * to debug) behavior on the guest side.
 491          */
 492         if (unlikely(!gfn_to_memslot(vcpu->kvm, cr3 >> PAGE_SHIFT)))
 493                 inject_gp(vcpu);
 494         else
 495                 vcpu->mmu.new_cr3(vcpu);
 496         spin_unlock(&vcpu->kvm->lock);
 497 }
 498 EXPORT_SYMBOL_GPL(set_cr3);
 499
 500 void set_cr8(struct kvm_vcpu *vcpu, unsigned long cr8)
 501 {
 502         if ( cr8 & CR8_RESEVED_BITS) {
 503                 printk(KERN_DEBUG "set_cr8: #GP, reserved bits 0x%lx\n", cr8);
 504                 inject_gp(vcpu);
 505                 return;
 506         }
 507         vcpu->cr8 = cr8;
 508 }
 509 EXPORT_SYMBOL_GPL(set_cr8);
 510
 511 void fx_init(struct kvm_vcpu *vcpu)
 512 {
 513         struct __attribute__ ((__packed__)) fx_image_s {
 514                 u16 control; //fcw
 515                 u16 status; //fsw
 516                 u16 tag; // ftw
 517                 u16 opcode; //fop
 518                 u64 ip; // fpu ip
 519                 u64 operand;// fpu dp
 520                 u32 mxcsr;
 521                 u32 mxcsr_mask;
 522
 523         } *fx_image;
 524
 525         fx_save(vcpu->host_fx_image);
 526         fpu_init();
 527         fx_save(vcpu->guest_fx_image);
 528         fx_restore(vcpu->host_fx_image);
 529
 530         fx_image = (struct fx_image_s *)vcpu->guest_fx_image;
 531         fx_image->mxcsr = 0x1f80;
 532         memset(vcpu->guest_fx_image + sizeof(struct fx_image_s),
 533                0, FX_IMAGE_SIZE - sizeof(struct fx_image_s));
 534 }
 535 EXPORT_SYMBOL_GPL(fx_init);
 536
 537 /*
 538  * Creates some virtual cpus.  Good luck creating more than one.
 539  */
 540 static int kvm_dev_ioctl_create_vcpu(struct kvm *kvm, int n)
 541 {
 542         int r;
 543         struct kvm_vcpu *vcpu;
 544
 545         r = -EINVAL;
 546         if (!valid_vcpu(n))
 547                 goto out;
 548
 549         vcpu = &kvm->vcpus[n];
 550
 551         mutex_lock(&vcpu->mutex);
 552
 553         if (vcpu->vmcs) {
 554                 mutex_unlock(&vcpu->mutex);
 555                 return -EEXIST;
 556         }
 557
 558         vcpu->host_fx_image = (char*)ALIGN((hva_t)vcpu->fx_buf,
 559                                            FX_IMAGE_ALIGN);
 560         vcpu->guest_fx_image = vcpu->host_fx_image + FX_IMAGE_SIZE;
 561
 562         r = kvm_arch_ops->vcpu_create(vcpu);
 563         if (r < 0)
 564                 goto out_free_vcpus;
 565
 566         r = kvm_mmu_create(vcpu);
 567         if (r < 0)
 568                 goto out_free_vcpus;
 569
 570         kvm_arch_ops->vcpu_load(vcpu);
 571         r = kvm_mmu_setup(vcpu);
 572         if (r >= 0)
 573                 r = kvm_arch_ops->vcpu_setup(vcpu);
 574         vcpu_put(vcpu);
 575
 576         if (r < 0)
 577                 goto out_free_vcpus;
 578
 579         return 0;
 580
 581 out_free_vcpus:
 582         kvm_free_vcpu(vcpu);
 583         mutex_unlock(&vcpu->mutex);
 584 out:
 585         return r;
 586 }
 587
 588 /*
 589  * Allocate some memory and give it an address in the guest physical address
 590  * space.
 591  *
 592  * Discontiguous memory is allowed, mostly for framebuffers.
 593  */
 594 static int kvm_dev_ioctl_set_memory_region(struct kvm *kvm,
 595                                            struct kvm_memory_region *mem)
 596 {
 597         int r;
 598         gfn_t base_gfn;
 599         unsigned long npages;
 600         unsigned long i;
 601         struct kvm_memory_slot *memslot;
 602         struct kvm_memory_slot old, new;
 603         int memory_config_version;
 604
 605         r = -EINVAL;
 606         /* General sanity checks */
 607         if (mem->memory_size & (PAGE_SIZE - 1))
 608                 goto out;
 609         if (mem->guest_phys_addr & (PAGE_SIZE - 1))
 610                 goto out;
 611         if (mem->slot >= KVM_MEMORY_SLOTS)
 612                 goto out;
 613         if (mem->guest_phys_addr + mem->memory_size < mem->guest_phys_addr)
 614                 goto out;
 615
 616         memslot = &kvm->memslots[mem->slot];
 617         base_gfn = mem->guest_phys_addr >> PAGE_SHIFT;
 618         npages = mem->memory_size >> PAGE_SHIFT;
 619
 620         if (!npages)
 621                 mem->flags &= ~KVM_MEM_LOG_DIRTY_PAGES;
 622
 623 raced:
 624         spin_lock(&kvm->lock);
 625
 626         memory_config_version = kvm->memory_config_version;
 627         new = old = *memslot;
 628
 629         new.base_gfn = base_gfn;
 630         new.npages = npages;
 631         new.flags = mem->flags;
 632
 633         /* Disallow changing a memory slot's size. */
 634         r = -EINVAL;
 635         if (npages && old.npages && npages != old.npages)
 636                 goto out_unlock;
 637
 638         /* Check for overlaps */
 639         r = -EEXIST;
 640         for (i = 0; i < KVM_MEMORY_SLOTS; ++i) {
 641                 struct kvm_memory_slot *s = &kvm->memslots[i];
 642
 643                 if (s == memslot)
 644                         continue;
 645                 if (!((base_gfn + npages <= s->base_gfn) ||
 646                       (base_gfn >= s->base_gfn + s->npages)))
 647                         goto out_unlock;
 648         }
 649         /*
 650          * Do memory allocations outside lock.  memory_config_version will
 651          * detect any races.
 652          */
 653         spin_unlock(&kvm->lock);
 654
 655         /* Deallocate if slot is being removed */
 656         if (!npages)
 657                 new.phys_mem = NULL;
 658
 659         /* Free page dirty bitmap if unneeded */
 660         if (!(new.flags & KVM_MEM_LOG_DIRTY_PAGES))
 661                 new.dirty_bitmap = NULL;
 662
 663         r = -ENOMEM;
 664
 665         /* Allocate if a slot is being created */
 666         if (npages && !new.phys_mem) {
 667                 new.phys_mem = vmalloc(npages * sizeof(struct page *));
 668
 669                 if (!new.phys_mem)
 670                         goto out_free;
 671
 672                 memset(new.phys_mem, 0, npages * sizeof(struct page *));
 673                 for (i = 0; i < npages; ++i) {
 674                         new.phys_mem[i] = alloc_page(GFP_HIGHUSER
 675                                                      | __GFP_ZERO);
 676                         if (!new.phys_mem[i])
 677                                 goto out_free;
 678                         set_page_private(new.phys_mem[i],0);
 679                 }
 680         }
 681
 682         /* Allocate page dirty bitmap if needed */
 683         if ((new.flags & KVM_MEM_LOG_DIRTY_PAGES) && !new.dirty_bitmap) {
 684                 unsigned dirty_bytes = ALIGN(npages, BITS_PER_LONG) / 8;
 685
 686                 new.dirty_bitmap = vmalloc(dirty_bytes);
 687                 if (!new.dirty_bitmap)
 688                         goto out_free;
 689                 memset(new.dirty_bitmap, 0, dirty_bytes);
 690         }
 691
 692         spin_lock(&kvm->lock);
 693
 694         if (memory_config_version != kvm->memory_config_version) {
 695                 spin_unlock(&kvm->lock);
 696                 kvm_free_physmem_slot(&new, &old);
 697                 goto raced;
 698         }
 699
 700         r = -EAGAIN;
 701         if (kvm->busy)
 702                 goto out_unlock;
 703
 704         if (mem->slot >= kvm->nmemslots)
 705                 kvm->nmemslots = mem->slot + 1;
 706
 707         *memslot = new;
 708         ++kvm->memory_config_version;
 709
 710         spin_unlock(&kvm->lock);
 711
 712         for (i = 0; i < KVM_MAX_VCPUS; ++i) {
 713                 struct kvm_vcpu *vcpu;
 714
 715                 vcpu = vcpu_load(kvm, i);
 716                 if (!vcpu)
 717                         continue;
 718                 kvm_mmu_reset_context(vcpu);
 719                 vcpu_put(vcpu);
 720         }
 721
 722         kvm_free_physmem_slot(&old, &new);
 723         return 0;
 724
 725 out_unlock:
 726         spin_unlock(&kvm->lock);
 727 out_free:
 728         kvm_free_physmem_slot(&new, &old);
 729 out:
 730         return r;
 731 }
 732
 733 static void do_remove_write_access(struct kvm_vcpu *vcpu, int slot)
 734 {
 735         spin_lock(&vcpu->kvm->lock);
 736         kvm_mmu_slot_remove_write_access(vcpu, slot);
 737         spin_unlock(&vcpu->kvm->lock);
 738 }
 739
 740 /*
 741  * Get (and clear) the dirty memory log for a memory slot.
 742  */
 743 static int kvm_dev_ioctl_get_dirty_log(struct kvm *kvm,
 744                                        struct kvm_dirty_log *log)
 745 {
 746         struct kvm_memory_slot *memslot;
 747         int r, i;
 748         int n;
 749         int cleared;
 750         unsigned long any = 0;
 751
 752         spin_lock(&kvm->lock);
 753
 754         /*
 755          * Prevent changes to guest memory configuration even while the lock
 756          * is not taken.
 757          */
 758         ++kvm->busy;
 759         spin_unlock(&kvm->lock);
 760         r = -EINVAL;
 761         if (log->slot >= KVM_MEMORY_SLOTS)
 762                 goto out;
 763
 764         memslot = &kvm->memslots[log->slot];
 765         r = -ENOENT;
 766         if (!memslot->dirty_bitmap)
 767                 goto out;
 768
 769         n = ALIGN(memslot->npages, 8) / 8;
 770
 771         for (i = 0; !any && i < n; ++i)
 772                 any = memslot->dirty_bitmap[i];
 773
 774         r = -EFAULT;
 775         if (copy_to_user(log->dirty_bitmap, memslot->dirty_bitmap, n))
 776                 goto out;
 777
 778         if (any) {
 779                 cleared = 0;
 780                 for (i = 0; i < KVM_MAX_VCPUS; ++i) {
 781                         struct kvm_vcpu *vcpu = vcpu_load(kvm, i);
 782
 783                         if (!vcpu)
 784                                 continue;
 785                         if (!cleared) {
 786                                 do_remove_write_access(vcpu, log->slot);
 787                                 memset(memslot->dirty_bitmap, 0, n);
 788                                 cleared = 1;
 789                         }
 790                         kvm_arch_ops->tlb_flush(vcpu);
 791                         vcpu_put(vcpu);
 792                 }
 793         }
 794
 795         r = 0;
 796
 797 out:
 798         spin_lock(&kvm->lock);
 799         --kvm->busy;
 800         spin_unlock(&kvm->lock);
 801         return r;
 802 }
 803
 804 struct kvm_memory_slot *gfn_to_memslot(struct kvm *kvm, gfn_t gfn)
 805 {
 806         int i;
 807
 808         for (i = 0; i < kvm->nmemslots; ++i) {
 809                 struct kvm_memory_slot *memslot = &kvm->memslots[i];
 810
 811                 if (gfn >= memslot->base_gfn
 812                     && gfn < memslot->base_gfn + memslot->npages)
 813                         return memslot;
 814         }
 815         return NULL;
 816 }
 817 EXPORT_SYMBOL_GPL(gfn_to_memslot);
 818
 819 void mark_page_dirty(struct kvm *kvm, gfn_t gfn)
 820 {
 821         int i;
 822         struct kvm_memory_slot *memslot = NULL;
 823         unsigned long rel_gfn;
 824
 825         for (i = 0; i < kvm->nmemslots; ++i) {
 826                 memslot = &kvm->memslots[i];
 827
 828                 if (gfn >= memslot->base_gfn
 829                     && gfn < memslot->base_gfn + memslot->npages) {
 830
 831                         if (!memslot || !memslot->dirty_bitmap)
 832                                 return;
 833
 834                         rel_gfn = gfn - memslot->base_gfn;
 835
 836                         /* avoid RMW */
 837                         if (!test_bit(rel_gfn, memslot->dirty_bitmap))
 838                                 set_bit(rel_gfn, memslot->dirty_bitmap);
 839                         return;
 840                 }
 841         }
 842 }
 843
 844 static int emulator_read_std(unsigned long addr,
 845                              unsigned long *val,
 846                              unsigned int bytes,
 847                              struct x86_emulate_ctxt *ctxt)
 848 {
 849         struct kvm_vcpu *vcpu = ctxt->vcpu;
 850         void *data = val;
 851
 852         while (bytes) {
 853                 gpa_t gpa = vcpu->mmu.gva_to_gpa(vcpu, addr);
 854                 unsigned offset = addr & (PAGE_SIZE-1);
 855                 unsigned tocopy = min(bytes, (unsigned)PAGE_SIZE - offset);
 856                 unsigned long pfn;
 857                 struct kvm_memory_slot *memslot;
 858                 void *page;
 859
 860                 if (gpa == UNMAPPED_GVA)
 861                         return X86EMUL_PROPAGATE_FAULT;
 862                 pfn = gpa >> PAGE_SHIFT;
 863                 memslot = gfn_to_memslot(vcpu->kvm, pfn);
 864                 if (!memslot)
 865                         return X86EMUL_UNHANDLEABLE;
 866                 page = kmap_atomic(gfn_to_page(memslot, pfn), KM_USER0);
 867
 868                 memcpy(data, page + offset, tocopy);
 869
 870                 kunmap_atomic(page, KM_USER0);
 871
 872                 bytes -= tocopy;
 873                 data += tocopy;
 874                 addr += tocopy;
 875         }
 876
 877         return X86EMUL_CONTINUE;
 878 }
 879
 880 static int emulator_write_std(unsigned long addr,
 881                               unsigned long val,
 882                               unsigned int bytes,
 883                               struct x86_emulate_ctxt *ctxt)
 884 {
 885         printk(KERN_ERR "emulator_write_std: addr %lx n %d\n",
 886                addr, bytes);
 887         return X86EMUL_UNHANDLEABLE;
 888 }
 889
 890 static int emulator_read_emulated(unsigned long addr,
 891                                   unsigned long *val,
 892                                   unsigned int bytes,
 893                                   struct x86_emulate_ctxt *ctxt)
 894 {
 895         struct kvm_vcpu *vcpu = ctxt->vcpu;
 896
 897         if (vcpu->mmio_read_completed) {
 898                 memcpy(val, vcpu->mmio_data, bytes);
 899                 vcpu->mmio_read_completed = 0;
 900                 return X86EMUL_CONTINUE;
 901         } else if (emulator_read_std(addr, val, bytes, ctxt)
 902                    == X86EMUL_CONTINUE)
 903                 return X86EMUL_CONTINUE;
 904         else {
 905                 gpa_t gpa = vcpu->mmu.gva_to_gpa(vcpu, addr);
 906
 907                 if (gpa == UNMAPPED_GVA)
 908                         return X86EMUL_PROPAGATE_FAULT;
 909                 vcpu->mmio_needed = 1;
 910                 vcpu->mmio_phys_addr = gpa;
 911                 vcpu->mmio_size = bytes;
 912                 vcpu->mmio_is_write = 0;
 913
 914                 return X86EMUL_UNHANDLEABLE;
 915         }
 916 }
 917
 918 static int emulator_write_phys(struct kvm_vcpu *vcpu, gpa_t gpa,
 919                                unsigned long val, int bytes)
 920 {
 921         struct kvm_memory_slot *m;
 922         struct page *page;
 923         void *virt;
 924
 925         if (((gpa + bytes - 1) >> PAGE_SHIFT) != (gpa >> PAGE_SHIFT))
 926                 return 0;
 927         m = gfn_to_memslot(vcpu->kvm, gpa >> PAGE_SHIFT);
 928         if (!m)
 929                 return 0;
 930         page = gfn_to_page(m, gpa >> PAGE_SHIFT);
 931         kvm_mmu_pre_write(vcpu, gpa, bytes);
 932         virt = kmap_atomic(page, KM_USER0);
 933         memcpy(virt + offset_in_page(gpa), &val, bytes);
 934         kunmap_atomic(virt, KM_USER0);
 935         kvm_mmu_post_write(vcpu, gpa, bytes);
 936         return 1;
 937 }
 938
 939 static int emulator_write_emulated(unsigned long addr,
 940                                    unsigned long val,
 941                                    unsigned int bytes,
 942                                    struct x86_emulate_ctxt *ctxt)
 943 {
 944         struct kvm_vcpu *vcpu = ctxt->vcpu;
 945         gpa_t gpa = vcpu->mmu.gva_to_gpa(vcpu, addr);
 946
 947         if (gpa == UNMAPPED_GVA)
 948                 return X86EMUL_PROPAGATE_FAULT;
 949
 950         if (emulator_write_phys(vcpu, gpa, val, bytes))
 951                 return X86EMUL_CONTINUE;
 952
 953         vcpu->mmio_needed = 1;
 954         vcpu->mmio_phys_addr = gpa;
 955         vcpu->mmio_size = bytes;
 956         vcpu->mmio_is_write = 1;
 957         memcpy(vcpu->mmio_data, &val, bytes);
 958
 959         return X86EMUL_CONTINUE;
 960 }
 961
 962 static int emulator_cmpxchg_emulated(unsigned long addr,
 963                                      unsigned long old,
 964                                      unsigned long new,
 965                                      unsigned int bytes,
 966                                      struct x86_emulate_ctxt *ctxt)
 967 {
 968         static int reported;
 969
 970         if (!reported) {
 971                 reported = 1;
 972                 printk(KERN_WARNING "kvm: emulating exchange as write\n");
 973         }
 974         return emulator_write_emulated(addr, new, bytes, ctxt);
 975 }
 976
 977 #ifdef CONFIG_X86_32
 978
 979 static int emulator_cmpxchg8b_emulated(unsigned long addr,
 980                                        unsigned long old_lo,
 981                                        unsigned long old_hi,
 982                                        unsigned long new_lo,
 983                                        unsigned long new_hi,
 984                                        struct x86_emulate_ctxt *ctxt)
 985 {
 986         static int reported;
 987         int r;
 988
 989         if (!reported) {
 990                 reported = 1;
 991                 printk(KERN_WARNING "kvm: emulating exchange8b as write\n");
 992         }
 993         r = emulator_write_emulated(addr, new_lo, 4, ctxt);
 994         if (r != X86EMUL_CONTINUE)
 995                 return r;
 996         return emulator_write_emulated(addr+4, new_hi, 4, ctxt);
 997 }
 998
 999 #endif
1000
1001 static unsigned long get_segment_base(struct kvm_vcpu *vcpu, int seg)
1002 {
1003         return kvm_arch_ops->get_segment_base(vcpu, seg);
1004 }
1005
1006 int emulate_invlpg(struct kvm_vcpu *vcpu, gva_t address)
1007 {
1008         return X86EMUL_CONTINUE;
1009 }
1010
1011 int emulate_clts(struct kvm_vcpu *vcpu)
1012 {
1013         unsigned long cr0;
1014
1015         kvm_arch_ops->decache_cr0_cr4_guest_bits(vcpu);
1016         cr0 = vcpu->cr0 & ~CR0_TS_MASK;
1017         kvm_arch_ops->set_cr0(vcpu, cr0);
1018         return X86EMUL_CONTINUE;
1019 }
1020
1021 int emulator_get_dr(struct x86_emulate_ctxt* ctxt, int dr, unsigned long *dest)
1022 {
1023         struct kvm_vcpu *vcpu = ctxt->vcpu;
1024
1025         switch (dr) {
1026         case 0 ... 3:
1027                 *dest = kvm_arch_ops->get_dr(vcpu, dr);
1028                 return X86EMUL_CONTINUE;
1029         default:
1030                 printk(KERN_DEBUG "%s: unexpected dr %u\n",
1031                        __FUNCTION__, dr);
1032                 return X86EMUL_UNHANDLEABLE;
1033         }
1034 }
1035
1036 int emulator_set_dr(struct x86_emulate_ctxt *ctxt, int dr, unsigned long value)
1037 {
1038         unsigned long mask = (ctxt->mode == X86EMUL_MODE_PROT64) ? ~0ULL : ~0U;
1039         int exception;
1040
1041         kvm_arch_ops->set_dr(ctxt->vcpu, dr, value & mask, &exception);
1042         if (exception) {
1043                 /* FIXME: better handling */
1044                 return X86EMUL_UNHANDLEABLE;
1045         }
1046         return X86EMUL_CONTINUE;
1047 }
1048
1049 static void report_emulation_failure(struct x86_emulate_ctxt *ctxt)
1050 {
1051         static int reported;
1052         u8 opcodes[4];
1053         unsigned long rip = ctxt->vcpu->rip;
1054         unsigned long rip_linear;
1055
1056         rip_linear = rip + get_segment_base(ctxt->vcpu, VCPU_SREG_CS);
1057
1058         if (reported)
1059                 return;
1060
1061         emulator_read_std(rip_linear, (void *)opcodes, 4, ctxt);
1062
1063         printk(KERN_ERR "emulation failed but !mmio_needed?"
1064                " rip %lx %02x %02x %02x %02x\n",
1065                rip, opcodes[0], opcodes[1], opcodes[2], opcodes[3]);
1066         reported = 1;
1067 }
1068
1069 struct x86_emulate_ops emulate_ops = {
1070         .read_std            = emulator_read_std,
1071         .write_std           = emulator_write_std,
1072         .read_emulated       = emulator_read_emulated,
1073         .write_emulated      = emulator_write_emulated,
1074         .cmpxchg_emulated    = emulator_cmpxchg_emulated,
1075 #ifdef CONFIG_X86_32
1076         .cmpxchg8b_emulated  = emulator_cmpxchg8b_emulated,
1077 #endif
1078 };
1079
1080 int emulate_instruction(struct kvm_vcpu *vcpu,
1081                         struct kvm_run *run,
1082                         unsigned long cr2,
1083                         u16 error_code)
1084 {
1085         struct x86_emulate_ctxt emulate_ctxt;
1086         int r;
1087         int cs_db, cs_l;
1088
1089         kvm_arch_ops->cache_regs(vcpu);
1090
1091         kvm_arch_ops->get_cs_db_l_bits(vcpu, &cs_db, &cs_l);
1092
1093         emulate_ctxt.vcpu = vcpu;
1094         emulate_ctxt.eflags = kvm_arch_ops->get_rflags(vcpu);
1095         emulate_ctxt.cr2 = cr2;
1096         emulate_ctxt.mode = (emulate_ctxt.eflags & X86_EFLAGS_VM)
1097                 ? X86EMUL_MODE_REAL : cs_l
1098                 ? X86EMUL_MODE_PROT64 : cs_db
1099                 ? X86EMUL_MODE_PROT32 : X86EMUL_MODE_PROT16;
1100
1101         if (emulate_ctxt.mode == X86EMUL_MODE_PROT64) {
1102                 emulate_ctxt.cs_base = 0;
1103                 emulate_ctxt.ds_base = 0;
1104                 emulate_ctxt.es_base = 0;
1105                 emulate_ctxt.ss_base = 0;
1106         } else {
1107                 emulate_ctxt.cs_base = get_segment_base(vcpu, VCPU_SREG_CS);
1108                 emulate_ctxt.ds_base = get_segment_base(vcpu, VCPU_SREG_DS);
1109                 emulate_ctxt.es_base = get_segment_base(vcpu, VCPU_SREG_ES);
1110                 emulate_ctxt.ss_base = get_segment_base(vcpu, VCPU_SREG_SS);
1111         }
1112
1113         emulate_ctxt.gs_base = get_segment_base(vcpu, VCPU_SREG_GS);
1114         emulate_ctxt.fs_base = get_segment_base(vcpu, VCPU_SREG_FS);
1115
1116         vcpu->mmio_is_write = 0;
1117         r = x86_emulate_memop(&emulate_ctxt, &emulate_ops);
1118
1119         if ((r || vcpu->mmio_is_write) && run) {
1120                 run->mmio.phys_addr = vcpu->mmio_phys_addr;
1121                 memcpy(run->mmio.data, vcpu->mmio_data, 8);
1122                 run->mmio.len = vcpu->mmio_size;
1123                 run->mmio.is_write = vcpu->mmio_is_write;
1124         }
1125
1126         if (r) {
1127                 if (kvm_mmu_unprotect_page_virt(vcpu, cr2))
1128                         return EMULATE_DONE;
1129                 if (!vcpu->mmio_needed) {
1130                         report_emulation_failure(&emulate_ctxt);
1131                         return EMULATE_FAIL;
1132                 }
1133                 return EMULATE_DO_MMIO;
1134         }
1135
1136         kvm_arch_ops->decache_regs(vcpu);
1137         kvm_arch_ops->set_rflags(vcpu, emulate_ctxt.eflags);
1138
1139         if (vcpu->mmio_is_write)
1140                 return EMULATE_DO_MMIO;
1141
1142         return EMULATE_DONE;
1143 }
1144 EXPORT_SYMBOL_GPL(emulate_instruction);
1145
1146 int kvm_hypercall(struct kvm_vcpu *vcpu, struct kvm_run *run)
1147 {
1148         unsigned long nr, a0, a1, a2, a3, a4, a5, ret;
1149
1150         kvm_arch_ops->decache_regs(vcpu);
1151         ret = -KVM_EINVAL;
1152 #ifdef CONFIG_X86_64
1153         if (is_long_mode(vcpu)) {
1154                 nr = vcpu->regs[VCPU_REGS_RAX];
1155                 a0 = vcpu->regs[VCPU_REGS_RDI];
1156                 a1 = vcpu->regs[VCPU_REGS_RSI];
1157                 a2 = vcpu->regs[VCPU_REGS_RDX];
1158                 a3 = vcpu->regs[VCPU_REGS_RCX];
1159                 a4 = vcpu->regs[VCPU_REGS_R8];
1160                 a5 = vcpu->regs[VCPU_REGS_R9];
1161         } else
1162 #endif
1163         {
1164                 nr = vcpu->regs[VCPU_REGS_RBX] & -1u;
1165                 a0 = vcpu->regs[VCPU_REGS_RAX] & -1u;
1166                 a1 = vcpu->regs[VCPU_REGS_RCX] & -1u;
1167                 a2 = vcpu->regs[VCPU_REGS_RDX] & -1u;
1168                 a3 = vcpu->regs[VCPU_REGS_RSI] & -1u;
1169                 a4 = vcpu->regs[VCPU_REGS_RDI] & -1u;
1170                 a5 = vcpu->regs[VCPU_REGS_RBP] & -1u;
1171         }
1172         switch (nr) {
1173         default:
1174                 ;
1175         }
1176         vcpu->regs[VCPU_REGS_RAX] = ret;
1177         kvm_arch_ops->cache_regs(vcpu);
1178         return 1;
1179 }
1180 EXPORT_SYMBOL_GPL(kvm_hypercall);
1181
1182 static u64 mk_cr_64(u64 curr_cr, u32 new_val)
1183 {
1184         return (curr_cr & ~((1ULL << 32) - 1)) | new_val;
1185 }
1186
1187 void realmode_lgdt(struct kvm_vcpu *vcpu, u16 limit, unsigned long base)
1188 {
1189         struct descriptor_table dt = { limit, base };
1190
1191         kvm_arch_ops->set_gdt(vcpu, &dt);
1192 }
1193
1194 void realmode_lidt(struct kvm_vcpu *vcpu, u16 limit, unsigned long base)
1195 {
1196         struct descriptor_table dt = { limit, base };
1197
1198         kvm_arch_ops->set_idt(vcpu, &dt);
1199 }
1200
1201 void realmode_lmsw(struct kvm_vcpu *vcpu, unsigned long msw,
1202                    unsigned long *rflags)
1203 {
1204         lmsw(vcpu, msw);
1205         *rflags = kvm_arch_ops->get_rflags(vcpu);
1206 }
1207
1208 unsigned long realmode_get_cr(struct kvm_vcpu *vcpu, int cr)
1209 {
1210         kvm_arch_ops->decache_cr0_cr4_guest_bits(vcpu);
1211         switch (cr) {
1212         case 0:
1213                 return vcpu->cr0;
1214         case 2:
1215                 return vcpu->cr2;
1216         case 3:
1217                 return vcpu->cr3;
1218         case 4:
1219                 return vcpu->cr4;
1220         default:
1221                 vcpu_printf(vcpu, "%s: unexpected cr %u\n", __FUNCTION__, cr);
1222                 return 0;
1223         }
1224 }
1225
1226 void realmode_set_cr(struct kvm_vcpu *vcpu, int cr, unsigned long val,
1227                      unsigned long *rflags)
1228 {
1229         switch (cr) {
1230         case 0:
1231                 set_cr0(vcpu, mk_cr_64(vcpu->cr0, val));
1232                 *rflags = kvm_arch_ops->get_rflags(vcpu);
1233                 break;
1234         case 2:
1235                 vcpu->cr2 = val;
1236                 break;
1237         case 3:
1238                 set_cr3(vcpu, val);
1239                 break;
1240         case 4:
1241                 set_cr4(vcpu, mk_cr_64(vcpu->cr4, val));
1242                 break;
1243         default:
1244                 vcpu_printf(vcpu, "%s: unexpected cr %u\n", __FUNCTION__, cr);
1245         }
1246 }
1247
1248 /*
1249  * Register the para guest with the host:
1250  */
1251 static int vcpu_register_para(struct kvm_vcpu *vcpu, gpa_t para_state_gpa)
1252 {
1253         struct kvm_vcpu_para_state *para_state;
1254         hpa_t para_state_hpa, hypercall_hpa;
1255         struct page *para_state_page;
1256         unsigned char *hypercall;
1257         gpa_t hypercall_gpa;
1258
1259         printk(KERN_DEBUG "kvm: guest trying to enter paravirtual mode\n");
1260         printk(KERN_DEBUG ".... para_state_gpa: %08Lx\n", para_state_gpa);
1261
1262         /*
1263          * Needs to be page aligned:
1264          */
1265         if (para_state_gpa != PAGE_ALIGN(para_state_gpa))
1266                 goto err_gp;
1267
1268         para_state_hpa = gpa_to_hpa(vcpu, para_state_gpa);
1269         printk(KERN_DEBUG ".... para_state_hpa: %08Lx\n", para_state_hpa);
1270         if (is_error_hpa(para_state_hpa))
1271                 goto err_gp;
1272
1273         para_state_page = pfn_to_page(para_state_hpa >> PAGE_SHIFT);
1274         para_state = kmap_atomic(para_state_page, KM_USER0);
1275
1276         printk(KERN_DEBUG "....  guest version: %d\n", para_state->guest_version);
1277         printk(KERN_DEBUG "....           size: %d\n", para_state->size);
1278
1279         para_state->host_version = KVM_PARA_API_VERSION;
1280         /*
1281          * We cannot support guests that try to register themselves
1282          * with a newer API version than the host supports:
1283          */
1284         if (para_state->guest_version > KVM_PARA_API_VERSION) {
1285                 para_state->ret = -KVM_EINVAL;
1286                 goto err_kunmap_skip;
1287         }
1288
1289         hypercall_gpa = para_state->hypercall_gpa;
1290         hypercall_hpa = gpa_to_hpa(vcpu, hypercall_gpa);
1291         printk(KERN_DEBUG ".... hypercall_hpa: %08Lx\n", hypercall_hpa);
1292         if (is_error_hpa(hypercall_hpa)) {
1293                 para_state->ret = -KVM_EINVAL;
1294                 goto err_kunmap_skip;
1295         }
1296
1297         printk(KERN_DEBUG "kvm: para guest successfully registered.\n");
1298         vcpu->para_state_page = para_state_page;
1299         vcpu->para_state_gpa = para_state_gpa;
1300         vcpu->hypercall_gpa = hypercall_gpa;
1301
1302         hypercall = kmap_atomic(pfn_to_page(hypercall_hpa >> PAGE_SHIFT),
1303                                 KM_USER1) + (hypercall_hpa & ~PAGE_MASK);
1304         kvm_arch_ops->patch_hypercall(vcpu, hypercall);
1305         kunmap_atomic(hypercall, KM_USER1);
1306
1307         para_state->ret = 0;
1308 err_kunmap_skip:
1309         kunmap_atomic(para_state, KM_USER0);
1310         return 0;
1311 err_gp:
1312         return 1;
1313 }
1314
1315 int kvm_get_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata)
1316 {
1317         u64 data;
1318
1319         switch (msr) {
1320         case 0xc0010010: /* SYSCFG */
1321         case 0xc0010015: /* HWCR */
1322         case MSR_IA32_PLATFORM_ID:
1323         case MSR_IA32_P5_MC_ADDR:
1324         case MSR_IA32_P5_MC_TYPE:
1325         case MSR_IA32_MC0_CTL:
1326         case MSR_IA32_MCG_STATUS:
1327         case MSR_IA32_MCG_CAP:
1328         case MSR_IA32_MC0_MISC:
1329         case MSR_IA32_MC0_MISC+4:
1330         case MSR_IA32_MC0_MISC+8:
1331         case MSR_IA32_MC0_MISC+12:
1332         case MSR_IA32_MC0_MISC+16:
1333         case MSR_IA32_UCODE_REV:
1334         case MSR_IA32_PERF_STATUS:
1335                 /* MTRR registers */
1336         case 0xfe:
1337         case 0x200 ... 0x2ff:
1338                 data = 0;
1339                 break;
1340         case 0xcd: /* fsb frequency */
1341                 data = 3;
1342                 break;
1343         case MSR_IA32_APICBASE:
1344                 data = vcpu->apic_base;
1345                 break;
1346         case MSR_IA32_MISC_ENABLE:
1347                 data = vcpu->ia32_misc_enable_msr;
1348                 break;
1349 #ifdef CONFIG_X86_64
1350         case MSR_EFER:
1351                 data = vcpu->shadow_efer;
1352                 break;
1353 #endif
1354         default:
1355                 printk(KERN_ERR "kvm: unhandled rdmsr: 0x%x\n", msr);
1356                 return 1;
1357         }
1358         *pdata = data;
1359         return 0;
1360 }
1361 EXPORT_SYMBOL_GPL(kvm_get_msr_common);
1362
1363 /*
1364  * Reads an msr value (of 'msr_index') into 'pdata'.
1365  * Returns 0 on success, non-0 otherwise.
1366  * Assumes vcpu_load() was already called.
1367  */
1368 static int get_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 *pdata)
1369 {
1370         return kvm_arch_ops->get_msr(vcpu, msr_index, pdata);
1371 }
1372
1373 #ifdef CONFIG_X86_64
1374
1375 static void set_efer(struct kvm_vcpu *vcpu, u64 efer)
1376 {
1377         if (efer & EFER_RESERVED_BITS) {
1378                 printk(KERN_DEBUG "set_efer: 0x%llx #GP, reserved bits\n",
1379                        efer);
1380                 inject_gp(vcpu);
1381                 return;
1382         }
1383
1384         if (is_paging(vcpu)
1385             && (vcpu->shadow_efer & EFER_LME) != (efer & EFER_LME)) {
1386                 printk(KERN_DEBUG "set_efer: #GP, change LME while paging\n");
1387                 inject_gp(vcpu);
1388                 return;
1389         }
1390
1391         kvm_arch_ops->set_efer(vcpu, efer);
1392
1393         efer &= ~EFER_LMA;
1394         efer |= vcpu->shadow_efer & EFER_LMA;
1395
1396         vcpu->shadow_efer = efer;
1397 }
1398
1399 #endif
1400
1401 int kvm_set_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 data)
1402 {
1403         switch (msr) {
1404 #ifdef CONFIG_X86_64
1405         case MSR_EFER:
1406                 set_efer(vcpu, data);
1407                 break;
1408 #endif
1409         case MSR_IA32_MC0_STATUS:
1410                 printk(KERN_WARNING "%s: MSR_IA32_MC0_STATUS 0x%llx, nop\n",
1411                        __FUNCTION__, data);
1412                 break;
1413         case MSR_IA32_UCODE_REV:
1414         case MSR_IA32_UCODE_WRITE:
1415         case 0x200 ... 0x2ff: /* MTRRs */
1416                 break;
1417         case MSR_IA32_APICBASE:
1418                 vcpu->apic_base = data;
1419                 break;
1420         case MSR_IA32_MISC_ENABLE:
1421                 vcpu->ia32_misc_enable_msr = data;
1422                 break;
1423         /*
1424          * This is the 'probe whether the host is KVM' logic:
1425          */
1426         case MSR_KVM_API_MAGIC:
1427                 return vcpu_register_para(vcpu, data);
1428
1429         default:
1430                 printk(KERN_ERR "kvm: unhandled wrmsr: 0x%x\n", msr);
1431                 return 1;
1432         }
1433         return 0;
1434 }
1435 EXPORT_SYMBOL_GPL(kvm_set_msr_common);
1436
1437 /*
1438  * Writes msr value into into the appropriate "register".
1439  * Returns 0 on success, non-0 otherwise.
1440  * Assumes vcpu_load() was already called.
1441  */
1442 static int set_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 data)
1443 {
1444         return kvm_arch_ops->set_msr(vcpu, msr_index, data);
1445 }
1446
1447 void kvm_resched(struct kvm_vcpu *vcpu)
1448 {
1449         vcpu_put(vcpu);
1450         cond_resched();
1451         /* Cannot fail -  no vcpu unplug yet. */
1452         vcpu_load(vcpu->kvm, vcpu_slot(vcpu));
1453 }
1454 EXPORT_SYMBOL_GPL(kvm_resched);
1455
1456 void load_msrs(struct vmx_msr_entry *e, int n)
1457 {
1458         int i;
1459
1460         for (i = 0; i < n; ++i)
1461                 wrmsrl(e[i].index, e[i].data);
1462 }
1463 EXPORT_SYMBOL_GPL(load_msrs);
1464
1465 void save_msrs(struct vmx_msr_entry *e, int n)
1466 {
1467         int i;
1468
1469         for (i = 0; i < n; ++i)
1470                 rdmsrl(e[i].index, e[i].data);
1471 }
1472 EXPORT_SYMBOL_GPL(save_msrs);
1473
1474 static int kvm_dev_ioctl_run(struct kvm *kvm, struct kvm_run *kvm_run)
1475 {
1476         struct kvm_vcpu *vcpu;
1477         int r;
1478
1479         if (!valid_vcpu(kvm_run->vcpu))
1480                 return -EINVAL;
1481
1482         vcpu = vcpu_load(kvm, kvm_run->vcpu);
1483         if (!vcpu)
1484                 return -ENOENT;
1485
1486         /* re-sync apic's tpr */
1487         vcpu->cr8 = kvm_run->cr8;
1488
1489         if (kvm_run->emulated) {
1490                 kvm_arch_ops->skip_emulated_instruction(vcpu);
1491                 kvm_run->emulated = 0;
1492         }
1493
1494         if (kvm_run->mmio_completed) {
1495                 memcpy(vcpu->mmio_data, kvm_run->mmio.data, 8);
1496                 vcpu->mmio_read_completed = 1;
1497         }
1498
1499         vcpu->mmio_needed = 0;
1500
1501         r = kvm_arch_ops->run(vcpu, kvm_run);
1502
1503         vcpu_put(vcpu);
1504         return r;
1505 }
1506
1507 static int kvm_dev_ioctl_get_regs(struct kvm *kvm, struct kvm_regs *regs)
1508 {
1509         struct kvm_vcpu *vcpu;
1510
1511         if (!valid_vcpu(regs->vcpu))
1512                 return -EINVAL;
1513
1514         vcpu = vcpu_load(kvm, regs->vcpu);
1515         if (!vcpu)
1516                 return -ENOENT;
1517
1518         kvm_arch_ops->cache_regs(vcpu);
1519
1520         regs->rax = vcpu->regs[VCPU_REGS_RAX];
1521         regs->rbx = vcpu->regs[VCPU_REGS_RBX];
1522         regs->rcx = vcpu->regs[VCPU_REGS_RCX];
1523         regs->rdx = vcpu->regs[VCPU_REGS_RDX];
1524         regs->rsi = vcpu->regs[VCPU_REGS_RSI];
1525         regs->rdi = vcpu->regs[VCPU_REGS_RDI];
1526         regs->rsp = vcpu->regs[VCPU_REGS_RSP];
1527         regs->rbp = vcpu->regs[VCPU_REGS_RBP];
1528 #ifdef CONFIG_X86_64
1529         regs->r8 = vcpu->regs[VCPU_REGS_R8];
1530         regs->r9 = vcpu->regs[VCPU_REGS_R9];
1531         regs->r10 = vcpu->regs[VCPU_REGS_R10];
1532         regs->r11 = vcpu->regs[VCPU_REGS_R11];
1533         regs->r12 = vcpu->regs[VCPU_REGS_R12];
1534         regs->r13 = vcpu->regs[VCPU_REGS_R13];
1535         regs->r14 = vcpu->regs[VCPU_REGS_R14];
1536         regs->r15 = vcpu->regs[VCPU_REGS_R15];
1537 #endif
1538
1539         regs->rip = vcpu->rip;
1540         regs->rflags = kvm_arch_ops->get_rflags(vcpu);
1541
1542         /*
1543          * Don't leak debug flags in case they were set for guest debugging
1544          */
1545         if (vcpu->guest_debug.enabled && vcpu->guest_debug.singlestep)
1546                 regs->rflags &= ~(X86_EFLAGS_TF | X86_EFLAGS_RF);
1547
1548         vcpu_put(vcpu);
1549
1550         return 0;
1551 }
1552
1553 static int kvm_dev_ioctl_set_regs(struct kvm *kvm, struct kvm_regs *regs)
1554 {
1555         struct kvm_vcpu *vcpu;
1556
1557         if (!valid_vcpu(regs->vcpu))
1558                 return -EINVAL;
1559
1560         vcpu = vcpu_load(kvm, regs->vcpu);
1561         if (!vcpu)
1562                 return -ENOENT;
1563
1564         vcpu->regs[VCPU_REGS_RAX] = regs->rax;
1565         vcpu->regs[VCPU_REGS_RBX] = regs->rbx;
1566         vcpu->regs[VCPU_REGS_RCX] = regs->rcx;
1567         vcpu->regs[VCPU_REGS_RDX] = regs->rdx;
1568         vcpu->regs[VCPU_REGS_RSI] = regs->rsi;
1569         vcpu->regs[VCPU_REGS_RDI] = regs->rdi;
1570         vcpu->regs[VCPU_REGS_RSP] = regs->rsp;
1571         vcpu->regs[VCPU_REGS_RBP] = regs->rbp;
1572 #ifdef CONFIG_X86_64
1573         vcpu->regs[VCPU_REGS_R8] = regs->r8;
1574         vcpu->regs[VCPU_REGS_R9] = regs->r9;
1575         vcpu->regs[VCPU_REGS_R10] = regs->r10;
1576         vcpu->regs[VCPU_REGS_R11] = regs->r11;
1577         vcpu->regs[VCPU_REGS_R12] = regs->r12;
1578         vcpu->regs[VCPU_REGS_R13] = regs->r13;
1579         vcpu->regs[VCPU_REGS_R14] = regs->r14;
1580         vcpu->regs[VCPU_REGS_R15] = regs->r15;
1581 #endif
1582
1583         vcpu->rip = regs->rip;
1584         kvm_arch_ops->set_rflags(vcpu, regs->rflags);
1585
1586         kvm_arch_ops->decache_regs(vcpu);
1587
1588         vcpu_put(vcpu);
1589
1590         return 0;
1591 }
1592
1593 static void get_segment(struct kvm_vcpu *vcpu,
1594                         struct kvm_segment *var, int seg)
1595 {
1596         return kvm_arch_ops->get_segment(vcpu, var, seg);
1597 }
1598
1599 static int kvm_dev_ioctl_get_sregs(struct kvm *kvm, struct kvm_sregs *sregs)
1600 {
1601         struct kvm_vcpu *vcpu;
1602         struct descriptor_table dt;
1603
1604         if (!valid_vcpu(sregs->vcpu))
1605                 return -EINVAL;
1606         vcpu = vcpu_load(kvm, sregs->vcpu);
1607         if (!vcpu)
1608                 return -ENOENT;
1609
1610         get_segment(vcpu, &sregs->cs, VCPU_SREG_CS);
1611         get_segment(vcpu, &sregs->ds, VCPU_SREG_DS);
1612         get_segment(vcpu, &sregs->es, VCPU_SREG_ES);
1613         get_segment(vcpu, &sregs->fs, VCPU_SREG_FS);
1614         get_segment(vcpu, &sregs->gs, VCPU_SREG_GS);
1615         get_segment(vcpu, &sregs->ss, VCPU_SREG_SS);
1616
1617         get_segment(vcpu, &sregs->tr, VCPU_SREG_TR);
1618         get_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR);
1619
1620         kvm_arch_ops->get_idt(vcpu, &dt);
1621         sregs->idt.limit = dt.limit;
1622         sregs->idt.base = dt.base;
1623         kvm_arch_ops->get_gdt(vcpu, &dt);
1624         sregs->gdt.limit = dt.limit;
1625         sregs->gdt.base = dt.base;
1626
1627         kvm_arch_ops->decache_cr0_cr4_guest_bits(vcpu);
1628         sregs->cr0 = vcpu->cr0;
1629         sregs->cr2 = vcpu->cr2;
1630         sregs->cr3 = vcpu->cr3;
1631         sregs->cr4 = vcpu->cr4;
1632         sregs->cr8 = vcpu->cr8;
1633         sregs->efer = vcpu->shadow_efer;
1634         sregs->apic_base = vcpu->apic_base;
1635
1636         memcpy(sregs->interrupt_bitmap, vcpu->irq_pending,
1637                sizeof sregs->interrupt_bitmap);
1638
1639         vcpu_put(vcpu);
1640
1641         return 0;
1642 }
1643
1644 static void set_segment(struct kvm_vcpu *vcpu,
1645                         struct kvm_segment *var, int seg)
1646 {
1647         return kvm_arch_ops->set_segment(vcpu, var, seg);
1648 }
1649
1650 static int kvm_dev_ioctl_set_sregs(struct kvm *kvm, struct kvm_sregs *sregs)
1651 {
1652         struct kvm_vcpu *vcpu;
1653         int mmu_reset_needed = 0;
1654         int i;
1655         struct descriptor_table dt;
1656
1657         if (!valid_vcpu(sregs->vcpu))
1658                 return -EINVAL;
1659         vcpu = vcpu_load(kvm, sregs->vcpu);
1660         if (!vcpu)
1661                 return -ENOENT;
1662
1663         set_segment(vcpu, &sregs->cs, VCPU_SREG_CS);
1664         set_segment(vcpu, &sregs->ds, VCPU_SREG_DS);
1665         set_segment(vcpu, &sregs->es, VCPU_SREG_ES);
1666         set_segment(vcpu, &sregs->fs, VCPU_SREG_FS);
1667         set_segment(vcpu, &sregs->gs, VCPU_SREG_GS);
1668         set_segment(vcpu, &sregs->ss, VCPU_SREG_SS);
1669
1670         set_segment(vcpu, &sregs->tr, VCPU_SREG_TR);
1671         set_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR);
1672
1673         dt.limit = sregs->idt.limit;
1674         dt.base = sregs->idt.base;
1675         kvm_arch_ops->set_idt(vcpu, &dt);
1676         dt.limit = sregs->gdt.limit;
1677         dt.base = sregs->gdt.base;
1678         kvm_arch_ops->set_gdt(vcpu, &dt);
1679
1680         vcpu->cr2 = sregs->cr2;
1681         mmu_reset_needed |= vcpu->cr3 != sregs->cr3;
1682         vcpu->cr3 = sregs->cr3;
1683
1684         vcpu->cr8 = sregs->cr8;
1685
1686         mmu_reset_needed |= vcpu->shadow_efer != sregs->efer;
1687 #ifdef CONFIG_X86_64
1688         kvm_arch_ops->set_efer(vcpu, sregs->efer);
1689 #endif
1690         vcpu->apic_base = sregs->apic_base;
1691
1692         kvm_arch_ops->decache_cr0_cr4_guest_bits(vcpu);
1693
1694         mmu_reset_needed |= vcpu->cr0 != sregs->cr0;
1695         kvm_arch_ops->set_cr0_no_modeswitch(vcpu, sregs->cr0);
1696
1697         mmu_reset_needed |= vcpu->cr4 != sregs->cr4;
1698         kvm_arch_ops->set_cr4(vcpu, sregs->cr4);
1699         if (!is_long_mode(vcpu) && is_pae(vcpu))
1700                 load_pdptrs(vcpu, vcpu->cr3);
1701
1702         if (mmu_reset_needed)
1703                 kvm_mmu_reset_context(vcpu);
1704
1705         memcpy(vcpu->irq_pending, sregs->interrupt_bitmap,
1706                sizeof vcpu->irq_pending);
1707         vcpu->irq_summary = 0;
1708         for (i = 0; i < NR_IRQ_WORDS; ++i)
1709                 if (vcpu->irq_pending[i])
1710                         __set_bit(i, &vcpu->irq_summary);
1711
1712         vcpu_put(vcpu);
1713
1714         return 0;
1715 }
1716
1717 /*
1718  * List of msr numbers which we expose to userspace through KVM_GET_MSRS
1719  * and KVM_SET_MSRS, and KVM_GET_MSR_INDEX_LIST.
1720  *
1721  * This list is modified at module load time to reflect the
1722  * capabilities of the host cpu.
1723  */
1724 static u32 msrs_to_save[] = {
1725         MSR_IA32_SYSENTER_CS, MSR_IA32_SYSENTER_ESP, MSR_IA32_SYSENTER_EIP,
1726         MSR_K6_STAR,
1727 #ifdef CONFIG_X86_64
1728         MSR_CSTAR, MSR_KERNEL_GS_BASE, MSR_SYSCALL_MASK, MSR_LSTAR,
1729 #endif
1730         MSR_IA32_TIME_STAMP_COUNTER,
1731 };
1732
1733 static unsigned num_msrs_to_save;
1734
1735 static u32 emulated_msrs[] = {
1736         MSR_IA32_MISC_ENABLE,
1737 };
1738
1739 static __init void kvm_init_msr_list(void)
1740 {
1741         u32 dummy[2];
1742         unsigned i, j;
1743
1744         for (i = j = 0; i < ARRAY_SIZE(msrs_to_save); i++) {
1745                 if (rdmsr_safe(msrs_to_save[i], &dummy[0], &dummy[1]) < 0)
1746                         continue;
1747                 if (j < i)
1748                         msrs_to_save[j] = msrs_to_save[i];
1749                 j++;
1750         }
1751         num_msrs_to_save = j;
1752 }
1753
1754 /*
1755  * Adapt set_msr() to msr_io()'s calling convention
1756  */
1757 static int do_set_msr(struct kvm_vcpu *vcpu, unsigned index, u64 *data)
1758 {
1759         return set_msr(vcpu, index, *data);
1760 }
1761
1762 /*
1763  * Read or write a bunch of msrs. All parameters are kernel addresses.
1764  *
1765  * @return number of msrs set successfully.
1766  */
1767 static int __msr_io(struct kvm *kvm, struct kvm_msrs *msrs,
1768                     struct kvm_msr_entry *entries,
1769                     int (*do_msr)(struct kvm_vcpu *vcpu,
1770                                   unsigned index, u64 *data))
1771 {
1772         struct kvm_vcpu *vcpu;
1773         int i;
1774
1775         if (!valid_vcpu(msrs->vcpu))
1776                 return -EINVAL;
1777
1778         vcpu = vcpu_load(kvm, msrs->vcpu);
1779         if (!vcpu)
1780                 return -ENOENT;
1781
1782         for (i = 0; i < msrs->nmsrs; ++i)
1783                 if (do_msr(vcpu, entries[i].index, &entries[i].data))
1784                         break;
1785
1786         vcpu_put(vcpu);
1787
1788         return i;
1789 }
1790
1791 /*
1792  * Read or write a bunch of msrs. Parameters are user addresses.
1793  *
1794  * @return number of msrs set successfully.
1795  */
1796 static int msr_io(struct kvm *kvm, struct kvm_msrs __user *user_msrs,
1797                   int (*do_msr)(struct kvm_vcpu *vcpu,
1798                                 unsigned index, u64 *data),
1799                   int writeback)
1800 {
1801         struct kvm_msrs msrs;
1802         struct kvm_msr_entry *entries;
1803         int r, n;
1804         unsigned size;
1805
1806         r = -EFAULT;
1807         if (copy_from_user(&msrs, user_msrs, sizeof msrs))
1808                 goto out;
1809
1810         r = -E2BIG;
1811         if (msrs.nmsrs >= MAX_IO_MSRS)
1812                 goto out;
1813
1814         r = -ENOMEM;
1815         size = sizeof(struct kvm_msr_entry) * msrs.nmsrs;
1816         entries = vmalloc(size);
1817         if (!entries)
1818                 goto out;
1819
1820         r = -EFAULT;
1821         if (copy_from_user(entries, user_msrs->entries, size))
1822                 goto out_free;
1823
1824         r = n = __msr_io(kvm, &msrs, entries, do_msr);
1825         if (r < 0)
1826                 goto out_free;
1827
1828         r = -EFAULT;
1829         if (writeback && copy_to_user(user_msrs->entries, entries, size))
1830                 goto out_free;
1831
1832         r = n;
1833
1834 out_free:
1835         vfree(entries);
1836 out:
1837         return r;
1838 }
1839
1840 /*
1841  * Translate a guest virtual address to a guest physical address.
1842  */
1843 static int kvm_dev_ioctl_translate(struct kvm *kvm, struct kvm_translation *tr)
1844 {
1845         unsigned long vaddr = tr->linear_address;
1846         struct kvm_vcpu *vcpu;
1847         gpa_t gpa;
1848
1849         vcpu = vcpu_load(kvm, tr->vcpu);
1850         if (!vcpu)
1851                 return -ENOENT;
1852         spin_lock(&kvm->lock);
1853         gpa = vcpu->mmu.gva_to_gpa(vcpu, vaddr);
1854         tr->physical_address = gpa;
1855         tr->valid = gpa != UNMAPPED_GVA;
1856         tr->writeable = 1;
1857         tr->usermode = 0;
1858         spin_unlock(&kvm->lock);
1859         vcpu_put(vcpu);
1860
1861         return 0;
1862 }
1863
1864 static int kvm_dev_ioctl_interrupt(struct kvm *kvm, struct kvm_interrupt *irq)
1865 {
1866         struct kvm_vcpu *vcpu;
1867
1868         if (!valid_vcpu(irq->vcpu))
1869                 return -EINVAL;
1870         if (irq->irq < 0 || irq->irq >= 256)
1871                 return -EINVAL;
1872         vcpu = vcpu_load(kvm, irq->vcpu);
1873         if (!vcpu)
1874                 return -ENOENT;
1875
1876         set_bit(irq->irq, vcpu->irq_pending);
1877         set_bit(irq->irq / BITS_PER_LONG, &vcpu->irq_summary);
1878
1879         vcpu_put(vcpu);
1880
1881         return 0;
1882 }
1883
1884 static int kvm_dev_ioctl_debug_guest(struct kvm *kvm,
1885                                      struct kvm_debug_guest *dbg)
1886 {
1887         struct kvm_vcpu *vcpu;
1888         int r;
1889
1890         if (!valid_vcpu(dbg->vcpu))
1891                 return -EINVAL;
1892         vcpu = vcpu_load(kvm, dbg->vcpu);
1893         if (!vcpu)
1894                 return -ENOENT;
1895
1896         r = kvm_arch_ops->set_guest_debug(vcpu, dbg);
1897
1898         vcpu_put(vcpu);
1899
1900         return r;
1901 }
1902
1903 static long kvm_dev_ioctl(struct file *filp,
1904                           unsigned int ioctl, unsigned long arg)
1905 {
1906         struct kvm *kvm = filp->private_data;
1907         void __user *argp = (void __user *)arg;
1908         int r = -EINVAL;
1909
1910         switch (ioctl) {
1911         case KVM_GET_API_VERSION:
1912                 r = KVM_API_VERSION;
1913                 break;
1914         case KVM_CREATE_VCPU:
1915                 r = kvm_dev_ioctl_create_vcpu(kvm, arg);
1916                 if (r)
1917                         goto out;
1918                 break;
1919         case KVM_RUN: {
1920                 struct kvm_run kvm_run;
1921
1922                 r = -EFAULT;
1923                 if (copy_from_user(&kvm_run, argp, sizeof kvm_run))
1924                         goto out;
1925                 r = kvm_dev_ioctl_run(kvm, &kvm_run);
1926                 if (r < 0 &&  r != -EINTR)
1927                         goto out;
1928                 if (copy_to_user(argp, &kvm_run, sizeof kvm_run)) {
1929                         r = -EFAULT;
1930                         goto out;
1931                 }
1932                 break;
1933         }
1934         case KVM_GET_REGS: {
1935                 struct kvm_regs kvm_regs;
1936
1937                 r = -EFAULT;
1938                 if (copy_from_user(&kvm_regs, argp, sizeof kvm_regs))
1939                         goto out;
1940                 r = kvm_dev_ioctl_get_regs(kvm, &kvm_regs);
1941                 if (r)
1942                         goto out;
1943                 r = -EFAULT;
1944                 if (copy_to_user(argp, &kvm_regs, sizeof kvm_regs))
1945                         goto out;
1946                 r = 0;
1947                 break;
1948         }
1949         case KVM_SET_REGS: {
1950                 struct kvm_regs kvm_regs;
1951
1952                 r = -EFAULT;
1953                 if (copy_from_user(&kvm_regs, argp, sizeof kvm_regs))
1954                         goto out;
1955                 r = kvm_dev_ioctl_set_regs(kvm, &kvm_regs);
1956                 if (r)
1957                         goto out;
1958                 r = 0;
1959                 break;
1960         }
1961         case KVM_GET_SREGS: {
1962                 struct kvm_sregs kvm_sregs;
1963
1964                 r = -EFAULT;
1965                 if (copy_from_user(&kvm_sregs, argp, sizeof kvm_sregs))
1966                         goto out;
1967                 r = kvm_dev_ioctl_get_sregs(kvm, &kvm_sregs);
1968                 if (r)
1969                         goto out;
1970                 r = -EFAULT;
1971                 if (copy_to_user(argp, &kvm_sregs, sizeof kvm_sregs))
1972                         goto out;
1973                 r = 0;
1974                 break;
1975         }
1976         case KVM_SET_SREGS: {
1977                 struct kvm_sregs kvm_sregs;
1978
1979                 r = -EFAULT;
1980                 if (copy_from_user(&kvm_sregs, argp, sizeof kvm_sregs))
1981                         goto out;
1982                 r = kvm_dev_ioctl_set_sregs(kvm, &kvm_sregs);
1983                 if (r)
1984                         goto out;
1985                 r = 0;
1986                 break;
1987         }
1988         case KVM_TRANSLATE: {
1989                 struct kvm_translation tr;
1990
1991                 r = -EFAULT;
1992                 if (copy_from_user(&tr, argp, sizeof tr))
1993                         goto out;
1994                 r = kvm_dev_ioctl_translate(kvm, &tr);
1995                 if (r)
1996                         goto out;
1997                 r = -EFAULT;
1998                 if (copy_to_user(argp, &tr, sizeof tr))
1999                         goto out;
2000                 r = 0;
2001                 break;
2002         }
2003         case KVM_INTERRUPT: {
2004                 struct kvm_interrupt irq;
2005
2006                 r = -EFAULT;
2007                 if (copy_from_user(&irq, argp, sizeof irq))
2008                         goto out;
2009                 r = kvm_dev_ioctl_interrupt(kvm, &irq);
2010                 if (r)
2011                         goto out;
2012                 r = 0;
2013                 break;
2014         }
2015         case KVM_DEBUG_GUEST: {
2016                 struct kvm_debug_guest dbg;
2017
2018                 r = -EFAULT;
2019                 if (copy_from_user(&dbg, argp, sizeof dbg))
2020                         goto out;
2021                 r = kvm_dev_ioctl_debug_guest(kvm, &dbg);
2022                 if (r)
2023                         goto out;
2024                 r = 0;
2025                 break;
2026         }
2027         case KVM_SET_MEMORY_REGION: {
2028                 struct kvm_memory_region kvm_mem;
2029
2030                 r = -EFAULT;
2031                 if (copy_from_user(&kvm_mem, argp, sizeof kvm_mem))
2032                         goto out;
2033                 r = kvm_dev_ioctl_set_memory_region(kvm, &kvm_mem);
2034                 if (r)
2035                         goto out;
2036                 break;
2037         }
2038         case KVM_GET_DIRTY_LOG: {
2039                 struct kvm_dirty_log log;
2040
2041                 r = -EFAULT;
2042                 if (copy_from_user(&log, argp, sizeof log))
2043                         goto out;
2044                 r = kvm_dev_ioctl_get_dirty_log(kvm, &log);
2045                 if (r)
2046                         goto out;
2047                 break;
2048         }
2049         case KVM_GET_MSRS:
2050                 r = msr_io(kvm, argp, get_msr, 1);
2051                 break;
2052         case KVM_SET_MSRS:
2053                 r = msr_io(kvm, argp, do_set_msr, 0);
2054                 break;
2055         case KVM_GET_MSR_INDEX_LIST: {
2056                 struct kvm_msr_list __user *user_msr_list = argp;
2057                 struct kvm_msr_list msr_list;
2058                 unsigned n;
2059
2060                 r = -EFAULT;
2061                 if (copy_from_user(&msr_list, user_msr_list, sizeof msr_list))
2062                         goto out;
2063                 n = msr_list.nmsrs;
2064                 msr_list.nmsrs = num_msrs_to_save + ARRAY_SIZE(emulated_msrs);
2065                 if (copy_to_user(user_msr_list, &msr_list, sizeof msr_list))
2066                         goto out;
2067                 r = -E2BIG;
2068                 if (n < num_msrs_to_save)
2069                         goto out;
2070                 r = -EFAULT;
2071                 if (copy_to_user(user_msr_list->indices, &msrs_to_save,
2072                                  num_msrs_to_save * sizeof(u32)))
2073                         goto out;
2074                 if (copy_to_user(user_msr_list->indices
2075                                  + num_msrs_to_save * sizeof(u32),
2076                                  &emulated_msrs,
2077                                  ARRAY_SIZE(emulated_msrs) * sizeof(u32)))
2078                         goto out;
2079                 r = 0;
2080                 break;
2081         }
2082         default:
2083                 ;
2084         }
2085 out:
2086         return r;
2087 }
2088
2089 static struct page *kvm_dev_nopage(struct vm_area_struct *vma,
2090                                    unsigned long address,
2091                                    int *type)
2092 {
2093         struct kvm *kvm = vma->vm_file->private_data;
2094         unsigned long pgoff;
2095         struct kvm_memory_slot *slot;
2096         struct page *page;
2097
2098         *type = VM_FAULT_MINOR;
2099         pgoff = ((address - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff;
2100         slot = gfn_to_memslot(kvm, pgoff);
2101         if (!slot)
2102                 return NOPAGE_SIGBUS;
2103         page = gfn_to_page(slot, pgoff);
2104         if (!page)
2105                 return NOPAGE_SIGBUS;
2106         get_page(page);
2107         return page;
2108 }
2109
2110 static struct vm_operations_struct kvm_dev_vm_ops = {
2111         .nopage = kvm_dev_nopage,
2112 };
2113
2114 static int kvm_dev_mmap(struct file *file, struct vm_area_struct *vma)
2115 {
2116         vma->vm_ops = &kvm_dev_vm_ops;
2117         return 0;
2118 }
2119
2120 static struct file_operations kvm_chardev_ops = {
2121         .open           = kvm_dev_open,
2122         .release        = kvm_dev_release,
2123         .unlocked_ioctl = kvm_dev_ioctl,
2124         .compat_ioctl   = kvm_dev_ioctl,
2125         .mmap           = kvm_dev_mmap,
2126 };
2127
2128 static struct miscdevice kvm_dev = {
2129         MISC_DYNAMIC_MINOR,
2130         "kvm",
2131         &kvm_chardev_ops,
2132 };
2133
2134 static int kvm_reboot(struct notifier_block *notifier, unsigned long val,
2135                        void *v)
2136 {
2137         if (val == SYS_RESTART) {
2138                 /*
2139                  * Some (well, at least mine) BIOSes hang on reboot if
2140                  * in vmx root mode.
2141                  */
2142                 printk(KERN_INFO "kvm: exiting hardware virtualization\n");
2143                 on_each_cpu(kvm_arch_ops->hardware_disable, NULL, 0, 1);
2144         }
2145         return NOTIFY_OK;
2146 }
2147
2148 static struct notifier_block kvm_reboot_notifier = {
2149         .notifier_call = kvm_reboot,
2150         .priority = 0,
2151 };
2152
2153 /*
2154  * Make sure that a cpu that is being hot-unplugged does not have any vcpus
2155  * cached on it.
2156  */
2157 static void decache_vcpus_on_cpu(int cpu)
2158 {
2159         struct kvm *vm;
2160         struct kvm_vcpu *vcpu;
2161         int i;
2162
2163         spin_lock(&kvm_lock);
2164         list_for_each_entry(vm, &vm_list, vm_list)
2165                 for (i = 0; i < KVM_MAX_VCPUS; ++i) {
2166                         vcpu = &vm->vcpus[i];
2167                         /*
2168                          * If the vcpu is locked, then it is running on some
2169                          * other cpu and therefore it is not cached on the
2170                          * cpu in question.
2171                          *
2172                          * If it's not locked, check the last cpu it executed
2173                          * on.
2174                          */
2175                         if (mutex_trylock(&vcpu->mutex)) {
2176                                 if (vcpu->cpu == cpu) {
2177                                         kvm_arch_ops->vcpu_decache(vcpu);
2178                                         vcpu->cpu = -1;
2179                                 }
2180                                 mutex_unlock(&vcpu->mutex);
2181                         }
2182                 }
2183         spin_unlock(&kvm_lock);
2184 }
2185
2186 static int kvm_cpu_hotplug(struct notifier_block *notifier, unsigned long val,
2187                            void *v)
2188 {
2189         int cpu = (long)v;
2190
2191         switch (val) {
2192         case CPU_DOWN_PREPARE:
2193         case CPU_UP_CANCELED:
2194                 printk(KERN_INFO "kvm: disabling virtualization on CPU%d\n",
2195                        cpu);
2196                 decache_vcpus_on_cpu(cpu);
2197                 smp_call_function_single(cpu, kvm_arch_ops->hardware_disable,
2198                                          NULL, 0, 1);
2199                 break;
2200         case CPU_ONLINE:
2201                 printk(KERN_INFO "kvm: enabling virtualization on CPU%d\n",
2202                        cpu);
2203                 smp_call_function_single(cpu, kvm_arch_ops->hardware_enable,
2204                                          NULL, 0, 1);
2205                 break;
2206         }
2207         return NOTIFY_OK;
2208 }
2209
2210 static struct notifier_block kvm_cpu_notifier = {
2211         .notifier_call = kvm_cpu_hotplug,
2212         .priority = 20, /* must be > scheduler priority */
2213 };
2214
2215 static __init void kvm_init_debug(void)
2216 {
2217         struct kvm_stats_debugfs_item *p;
2218
2219         debugfs_dir = debugfs_create_dir("kvm", NULL);
2220         for (p = debugfs_entries; p->name; ++p)
2221                 p->dentry = debugfs_create_u32(p->name, 0444, debugfs_dir,
2222                                                p->data);
2223 }
2224
2225 static void kvm_exit_debug(void)
2226 {
2227         struct kvm_stats_debugfs_item *p;
2228
2229         for (p = debugfs_entries; p->name; ++p)
2230                 debugfs_remove(p->dentry);
2231         debugfs_remove(debugfs_dir);
2232 }
2233
2234 static int kvm_suspend(struct sys_device *dev, pm_message_t state)
2235 {
2236         decache_vcpus_on_cpu(raw_smp_processor_id());
2237         on_each_cpu(kvm_arch_ops->hardware_disable, NULL, 0, 1);
2238         return 0;
2239 }
2240
2241 static int kvm_resume(struct sys_device *dev)
2242 {
2243         on_each_cpu(kvm_arch_ops->hardware_enable, NULL, 0, 1);
2244         return 0;
2245 }
2246
2247 static struct sysdev_class kvm_sysdev_class = {
2248         set_kset_name("kvm"),
2249         .suspend = kvm_suspend,
2250         .resume = kvm_resume,
2251 };
2252
2253 static struct sys_device kvm_sysdev = {
2254         .id = 0,
2255         .cls = &kvm_sysdev_class,
2256 };
2257
2258 hpa_t bad_page_address;
2259
2260 static int kvmfs_get_sb(struct file_system_type *fs_type, int flags,
2261                         const char *dev_name, void *data, struct vfsmount *mnt)
2262 {
2263         return get_sb_pseudo(fs_type, "kvm:", NULL, KVMFS_MAGIC, mnt);
2264 }
2265
2266 static struct file_system_type kvm_fs_type = {
2267         .name           = "kvmfs",
2268         .get_sb         = kvmfs_get_sb,
2269         .kill_sb        = kill_anon_super,
2270 };
2271
2272 int kvm_init_arch(struct kvm_arch_ops *ops, struct module *module)
2273 {
2274         int r;
2275
2276         if (kvm_arch_ops) {
2277                 printk(KERN_ERR "kvm: already loaded the other module\n");
2278                 return -EEXIST;
2279         }
2280
2281         if (!ops->cpu_has_kvm_support()) {
2282                 printk(KERN_ERR "kvm: no hardware support\n");
2283                 return -EOPNOTSUPP;
2284         }
2285         if (ops->disabled_by_bios()) {
2286                 printk(KERN_ERR "kvm: disabled by bios\n");
2287                 return -EOPNOTSUPP;
2288         }
2289
2290         kvm_arch_ops = ops;
2291
2292         r = kvm_arch_ops->hardware_setup();
2293         if (r < 0)
2294             return r;
2295
2296         on_each_cpu(kvm_arch_ops->hardware_enable, NULL, 0, 1);
2297         r = register_cpu_notifier(&kvm_cpu_notifier);
2298         if (r)
2299                 goto out_free_1;
2300         register_reboot_notifier(&kvm_reboot_notifier);
2301
2302         r = sysdev_class_register(&kvm_sysdev_class);
2303         if (r)
2304                 goto out_free_2;
2305
2306         r = sysdev_register(&kvm_sysdev);
2307         if (r)
2308                 goto out_free_3;
2309
2310         kvm_chardev_ops.owner = module;
2311
2312         r = misc_register(&kvm_dev);
2313         if (r) {
2314                 printk (KERN_ERR "kvm: misc device register failed\n");
2315                 goto out_free;
2316         }
2317
2318         return r;
2319
2320 out_free:
2321         sysdev_unregister(&kvm_sysdev);
2322 out_free_3:
2323         sysdev_class_unregister(&kvm_sysdev_class);
2324 out_free_2:
2325         unregister_reboot_notifier(&kvm_reboot_notifier);
2326         unregister_cpu_notifier(&kvm_cpu_notifier);
2327 out_free_1:
2328         on_each_cpu(kvm_arch_ops->hardware_disable, NULL, 0, 1);
2329         kvm_arch_ops->hardware_unsetup();
2330         return r;
2331 }
2332
2333 void kvm_exit_arch(void)
2334 {
2335         misc_deregister(&kvm_dev);
2336         sysdev_unregister(&kvm_sysdev);
2337         sysdev_class_unregister(&kvm_sysdev_class);
2338         unregister_reboot_notifier(&kvm_reboot_notifier);
2339         unregister_cpu_notifier(&kvm_cpu_notifier);
2340         on_each_cpu(kvm_arch_ops->hardware_disable, NULL, 0, 1);
2341         kvm_arch_ops->hardware_unsetup();
2342         kvm_arch_ops = NULL;
2343 }
2344
2345 static __init int kvm_init(void)
2346 {
2347         static struct page *bad_page;
2348         int r;
2349
2350         r = register_filesystem(&kvm_fs_type);
2351         if (r)
2352                 goto out3;
2353
2354         kvmfs_mnt = kern_mount(&kvm_fs_type);
2355         r = PTR_ERR(kvmfs_mnt);
2356         if (IS_ERR(kvmfs_mnt))
2357                 goto out2;
2358         kvm_init_debug();
2359
2360         kvm_init_msr_list();
2361
2362         if ((bad_page = alloc_page(GFP_KERNEL)) == NULL) {
2363                 r = -ENOMEM;
2364                 goto out;
2365         }
2366
2367         bad_page_address = page_to_pfn(bad_page) << PAGE_SHIFT;
2368         memset(__va(bad_page_address), 0, PAGE_SIZE);
2369
2370         return r;
2371
2372 out:
2373         kvm_exit_debug();
2374         mntput(kvmfs_mnt);
2375 out2:
2376         unregister_filesystem(&kvm_fs_type);
2377 out3:
2378         return r;
2379 }
2380
2381 static __exit void kvm_exit(void)
2382 {
2383         kvm_exit_debug();
2384         __free_page(pfn_to_page(bad_page_address >> PAGE_SHIFT));
2385         mntput(kvmfs_mnt);
2386         unregister_filesystem(&kvm_fs_type);
2387 }
2388
2389 module_init(kvm_init)
2390 module_exit(kvm_exit)
2391
2392 EXPORT_SYMBOL_GPL(kvm_init_arch);
2393 EXPORT_SYMBOL_GPL(kvm_exit_arch);