git.oblomov.eu Git - linux-2.6/blob - drivers/kvm/kvm_main.c

   1 /*
   2  * Kernel-based Virtual Machine driver for Linux
   3  *
   4  * This module enables machines with Intel VT-x extensions to run virtual
   5  * machines without emulation or binary translation.
   6  *
   7  * Copyright (C) 2006 Qumranet, Inc.
   8  *
   9  * Authors:
  10  *   Avi Kivity   <avi@qumranet.com>
  11  *   Yaniv Kamay  <yaniv@qumranet.com>
  12  *
  13  * This work is licensed under the terms of the GNU GPL, version 2.  See
  14  * the COPYING file in the top-level directory.
  15  *
  16  */
  17
  18 #include "kvm.h"
  19
  20 #include <linux/kvm.h>
  21 #include <linux/module.h>
  22 #include <linux/errno.h>
  23 #include <asm/processor.h>
  24 #include <linux/percpu.h>
  25 #include <linux/gfp.h>
  26 #include <asm/msr.h>
  27 #include <linux/mm.h>
  28 #include <linux/miscdevice.h>
  29 #include <linux/vmalloc.h>
  30 #include <asm/uaccess.h>
  31 #include <linux/reboot.h>
  32 #include <asm/io.h>
  33 #include <linux/debugfs.h>
  34 #include <linux/highmem.h>
  35 #include <linux/file.h>
  36 #include <asm/desc.h>
  37
  38 #include "x86_emulate.h"
  39 #include "segment_descriptor.h"
  40
  41 MODULE_AUTHOR("Qumranet");
  42 MODULE_LICENSE("GPL");
  43
  44 struct kvm_arch_ops *kvm_arch_ops;
  45 struct kvm_stat kvm_stat;
  46 EXPORT_SYMBOL_GPL(kvm_stat);
  47
  48 static struct kvm_stats_debugfs_item {
  49         const char *name;
  50         u32 *data;
  51         struct dentry *dentry;
  52 } debugfs_entries[] = {
  53         { "pf_fixed", &kvm_stat.pf_fixed },
  54         { "pf_guest", &kvm_stat.pf_guest },
  55         { "tlb_flush", &kvm_stat.tlb_flush },
  56         { "invlpg", &kvm_stat.invlpg },
  57         { "exits", &kvm_stat.exits },
  58         { "io_exits", &kvm_stat.io_exits },
  59         { "mmio_exits", &kvm_stat.mmio_exits },
  60         { "signal_exits", &kvm_stat.signal_exits },
  61         { "irq_exits", &kvm_stat.irq_exits },
  62         { 0, 0 }
  63 };
  64
  65 static struct dentry *debugfs_dir;
  66
  67 #define MAX_IO_MSRS 256
  68
  69 #define CR0_RESEVED_BITS 0xffffffff1ffaffc0ULL
  70 #define LMSW_GUEST_MASK 0x0eULL
  71 #define CR4_RESEVED_BITS (~((1ULL << 11) - 1))
  72 #define CR8_RESEVED_BITS (~0x0fULL)
  73 #define EFER_RESERVED_BITS 0xfffffffffffff2fe
  74
  75 #ifdef CONFIG_X86_64
  76 // LDT or TSS descriptor in the GDT. 16 bytes.
  77 struct segment_descriptor_64 {
  78         struct segment_descriptor s;
  79         u32 base_higher;
  80         u32 pad_zero;
  81 };
  82
  83 #endif
  84
  85 unsigned long segment_base(u16 selector)
  86 {
  87         struct descriptor_table gdt;
  88         struct segment_descriptor *d;
  89         unsigned long table_base;
  90         typedef unsigned long ul;
  91         unsigned long v;
  92
  93         if (selector == 0)
  94                 return 0;
  95
  96         asm ("sgdt %0" : "=m"(gdt));
  97         table_base = gdt.base;
  98
  99         if (selector & 4) {           /* from ldt */
 100                 u16 ldt_selector;
 101
 102                 asm ("sldt %0" : "=g"(ldt_selector));
 103                 table_base = segment_base(ldt_selector);
 104         }
 105         d = (struct segment_descriptor *)(table_base + (selector & ~7));
 106         v = d->base_low | ((ul)d->base_mid << 16) | ((ul)d->base_high << 24);
 107 #ifdef CONFIG_X86_64
 108         if (d->system == 0
 109             && (d->type == 2 || d->type == 9 || d->type == 11))
 110                 v |= ((ul)((struct segment_descriptor_64 *)d)->base_higher) << 32;
 111 #endif
 112         return v;
 113 }
 114 EXPORT_SYMBOL_GPL(segment_base);
 115
 116 static inline int valid_vcpu(int n)
 117 {
 118         return likely(n >= 0 && n < KVM_MAX_VCPUS);
 119 }
 120
 121 int kvm_read_guest(struct kvm_vcpu *vcpu,
 122                              gva_t addr,
 123                              unsigned long size,
 124                              void *dest)
 125 {
 126         unsigned char *host_buf = dest;
 127         unsigned long req_size = size;
 128
 129         while (size) {
 130                 hpa_t paddr;
 131                 unsigned now;
 132                 unsigned offset;
 133                 hva_t guest_buf;
 134
 135                 paddr = gva_to_hpa(vcpu, addr);
 136
 137                 if (is_error_hpa(paddr))
 138                         break;
 139
 140                 guest_buf = (hva_t)kmap_atomic(
 141                                         pfn_to_page(paddr >> PAGE_SHIFT),
 142                                         KM_USER0);
 143                 offset = addr & ~PAGE_MASK;
 144                 guest_buf |= offset;
 145                 now = min(size, PAGE_SIZE - offset);
 146                 memcpy(host_buf, (void*)guest_buf, now);
 147                 host_buf += now;
 148                 addr += now;
 149                 size -= now;
 150                 kunmap_atomic((void *)(guest_buf & PAGE_MASK), KM_USER0);
 151         }
 152         return req_size - size;
 153 }
 154 EXPORT_SYMBOL_GPL(kvm_read_guest);
 155
 156 int kvm_write_guest(struct kvm_vcpu *vcpu,
 157                              gva_t addr,
 158                              unsigned long size,
 159                              void *data)
 160 {
 161         unsigned char *host_buf = data;
 162         unsigned long req_size = size;
 163
 164         while (size) {
 165                 hpa_t paddr;
 166                 unsigned now;
 167                 unsigned offset;
 168                 hva_t guest_buf;
 169
 170                 paddr = gva_to_hpa(vcpu, addr);
 171
 172                 if (is_error_hpa(paddr))
 173                         break;
 174
 175                 guest_buf = (hva_t)kmap_atomic(
 176                                 pfn_to_page(paddr >> PAGE_SHIFT), KM_USER0);
 177                 offset = addr & ~PAGE_MASK;
 178                 guest_buf |= offset;
 179                 now = min(size, PAGE_SIZE - offset);
 180                 memcpy((void*)guest_buf, host_buf, now);
 181                 host_buf += now;
 182                 addr += now;
 183                 size -= now;
 184                 kunmap_atomic((void *)(guest_buf & PAGE_MASK), KM_USER0);
 185         }
 186         return req_size - size;
 187 }
 188 EXPORT_SYMBOL_GPL(kvm_write_guest);
 189
 190 static int vcpu_slot(struct kvm_vcpu *vcpu)
 191 {
 192         return vcpu - vcpu->kvm->vcpus;
 193 }
 194
 195 /*
 196  * Switches to specified vcpu, until a matching vcpu_put()
 197  */
 198 static struct kvm_vcpu *vcpu_load(struct kvm *kvm, int vcpu_slot)
 199 {
 200         struct kvm_vcpu *vcpu = &kvm->vcpus[vcpu_slot];
 201
 202         mutex_lock(&vcpu->mutex);
 203         if (unlikely(!vcpu->vmcs)) {
 204                 mutex_unlock(&vcpu->mutex);
 205                 return 0;
 206         }
 207         return kvm_arch_ops->vcpu_load(vcpu);
 208 }
 209
 210 static void vcpu_put(struct kvm_vcpu *vcpu)
 211 {
 212         kvm_arch_ops->vcpu_put(vcpu);
 213         mutex_unlock(&vcpu->mutex);
 214 }
 215
 216 static int kvm_dev_open(struct inode *inode, struct file *filp)
 217 {
 218         struct kvm *kvm = kzalloc(sizeof(struct kvm), GFP_KERNEL);
 219         int i;
 220
 221         if (!kvm)
 222                 return -ENOMEM;
 223
 224         spin_lock_init(&kvm->lock);
 225         INIT_LIST_HEAD(&kvm->active_mmu_pages);
 226         for (i = 0; i < KVM_MAX_VCPUS; ++i) {
 227                 struct kvm_vcpu *vcpu = &kvm->vcpus[i];
 228
 229                 mutex_init(&vcpu->mutex);
 230                 vcpu->mmu.root_hpa = INVALID_PAGE;
 231                 INIT_LIST_HEAD(&vcpu->free_pages);
 232         }
 233         filp->private_data = kvm;
 234         return 0;
 235 }
 236
 237 /*
 238  * Free any memory in @free but not in @dont.
 239  */
 240 static void kvm_free_physmem_slot(struct kvm_memory_slot *free,
 241                                   struct kvm_memory_slot *dont)
 242 {
 243         int i;
 244
 245         if (!dont || free->phys_mem != dont->phys_mem)
 246                 if (free->phys_mem) {
 247                         for (i = 0; i < free->npages; ++i)
 248                                 if (free->phys_mem[i])
 249                                         __free_page(free->phys_mem[i]);
 250                         vfree(free->phys_mem);
 251                 }
 252
 253         if (!dont || free->dirty_bitmap != dont->dirty_bitmap)
 254                 vfree(free->dirty_bitmap);
 255
 256         free->phys_mem = 0;
 257         free->npages = 0;
 258         free->dirty_bitmap = 0;
 259 }
 260
 261 static void kvm_free_physmem(struct kvm *kvm)
 262 {
 263         int i;
 264
 265         for (i = 0; i < kvm->nmemslots; ++i)
 266                 kvm_free_physmem_slot(&kvm->memslots[i], 0);
 267 }
 268
 269 static void kvm_free_vcpu(struct kvm_vcpu *vcpu)
 270 {
 271         kvm_arch_ops->vcpu_free(vcpu);
 272         kvm_mmu_destroy(vcpu);
 273 }
 274
 275 static void kvm_free_vcpus(struct kvm *kvm)
 276 {
 277         unsigned int i;
 278
 279         for (i = 0; i < KVM_MAX_VCPUS; ++i)
 280                 kvm_free_vcpu(&kvm->vcpus[i]);
 281 }
 282
 283 static int kvm_dev_release(struct inode *inode, struct file *filp)
 284 {
 285         struct kvm *kvm = filp->private_data;
 286
 287         kvm_free_vcpus(kvm);
 288         kvm_free_physmem(kvm);
 289         kfree(kvm);
 290         return 0;
 291 }
 292
 293 static void inject_gp(struct kvm_vcpu *vcpu)
 294 {
 295         kvm_arch_ops->inject_gp(vcpu, 0);
 296 }
 297
 298 static int pdptrs_have_reserved_bits_set(struct kvm_vcpu *vcpu,
 299                                          unsigned long cr3)
 300 {
 301         gfn_t pdpt_gfn = cr3 >> PAGE_SHIFT;
 302         unsigned offset = (cr3 & (PAGE_SIZE-1)) >> 5;
 303         int i;
 304         u64 pdpte;
 305         u64 *pdpt;
 306         struct kvm_memory_slot *memslot;
 307
 308         spin_lock(&vcpu->kvm->lock);
 309         memslot = gfn_to_memslot(vcpu->kvm, pdpt_gfn);
 310         /* FIXME: !memslot - emulate? 0xff? */
 311         pdpt = kmap_atomic(gfn_to_page(memslot, pdpt_gfn), KM_USER0);
 312
 313         for (i = 0; i < 4; ++i) {
 314                 pdpte = pdpt[offset + i];
 315                 if ((pdpte & 1) && (pdpte & 0xfffffff0000001e6ull))
 316                         break;
 317         }
 318
 319         kunmap_atomic(pdpt, KM_USER0);
 320         spin_unlock(&vcpu->kvm->lock);
 321
 322         return i != 4;
 323 }
 324
 325 void set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
 326 {
 327         if (cr0 & CR0_RESEVED_BITS) {
 328                 printk(KERN_DEBUG "set_cr0: 0x%lx #GP, reserved bits 0x%lx\n",
 329                        cr0, vcpu->cr0);
 330                 inject_gp(vcpu);
 331                 return;
 332         }
 333
 334         if ((cr0 & CR0_NW_MASK) && !(cr0 & CR0_CD_MASK)) {
 335                 printk(KERN_DEBUG "set_cr0: #GP, CD == 0 && NW == 1\n");
 336                 inject_gp(vcpu);
 337                 return;
 338         }
 339
 340         if ((cr0 & CR0_PG_MASK) && !(cr0 & CR0_PE_MASK)) {
 341                 printk(KERN_DEBUG "set_cr0: #GP, set PG flag "
 342                        "and a clear PE flag\n");
 343                 inject_gp(vcpu);
 344                 return;
 345         }
 346
 347         if (!is_paging(vcpu) && (cr0 & CR0_PG_MASK)) {
 348 #ifdef CONFIG_X86_64
 349                 if ((vcpu->shadow_efer & EFER_LME)) {
 350                         int cs_db, cs_l;
 351
 352                         if (!is_pae(vcpu)) {
 353                                 printk(KERN_DEBUG "set_cr0: #GP, start paging "
 354                                        "in long mode while PAE is disabled\n");
 355                                 inject_gp(vcpu);
 356                                 return;
 357                         }
 358                         kvm_arch_ops->get_cs_db_l_bits(vcpu, &cs_db, &cs_l);
 359                         if (cs_l) {
 360                                 printk(KERN_DEBUG "set_cr0: #GP, start paging "
 361                                        "in long mode while CS.L == 1\n");
 362                                 inject_gp(vcpu);
 363                                 return;
 364
 365                         }
 366                 } else
 367 #endif
 368                 if (is_pae(vcpu) &&
 369                             pdptrs_have_reserved_bits_set(vcpu, vcpu->cr3)) {
 370                         printk(KERN_DEBUG "set_cr0: #GP, pdptrs "
 371                                "reserved bits\n");
 372                         inject_gp(vcpu);
 373                         return;
 374                 }
 375
 376         }
 377
 378         kvm_arch_ops->set_cr0(vcpu, cr0);
 379         vcpu->cr0 = cr0;
 380
 381         spin_lock(&vcpu->kvm->lock);
 382         kvm_mmu_reset_context(vcpu);
 383         spin_unlock(&vcpu->kvm->lock);
 384         return;
 385 }
 386 EXPORT_SYMBOL_GPL(set_cr0);
 387
 388 void lmsw(struct kvm_vcpu *vcpu, unsigned long msw)
 389 {
 390         set_cr0(vcpu, (vcpu->cr0 & ~0x0ful) | (msw & 0x0f));
 391 }
 392 EXPORT_SYMBOL_GPL(lmsw);
 393
 394 void set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
 395 {
 396         if (cr4 & CR4_RESEVED_BITS) {
 397                 printk(KERN_DEBUG "set_cr4: #GP, reserved bits\n");
 398                 inject_gp(vcpu);
 399                 return;
 400         }
 401
 402         if (is_long_mode(vcpu)) {
 403                 if (!(cr4 & CR4_PAE_MASK)) {
 404                         printk(KERN_DEBUG "set_cr4: #GP, clearing PAE while "
 405                                "in long mode\n");
 406                         inject_gp(vcpu);
 407                         return;
 408                 }
 409         } else if (is_paging(vcpu) && !is_pae(vcpu) && (cr4 & CR4_PAE_MASK)
 410                    && pdptrs_have_reserved_bits_set(vcpu, vcpu->cr3)) {
 411                 printk(KERN_DEBUG "set_cr4: #GP, pdptrs reserved bits\n");
 412                 inject_gp(vcpu);
 413         }
 414
 415         if (cr4 & CR4_VMXE_MASK) {
 416                 printk(KERN_DEBUG "set_cr4: #GP, setting VMXE\n");
 417                 inject_gp(vcpu);
 418                 return;
 419         }
 420         kvm_arch_ops->set_cr4(vcpu, cr4);
 421         spin_lock(&vcpu->kvm->lock);
 422         kvm_mmu_reset_context(vcpu);
 423         spin_unlock(&vcpu->kvm->lock);
 424 }
 425 EXPORT_SYMBOL_GPL(set_cr4);
 426
 427 void set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3)
 428 {
 429         if (is_long_mode(vcpu)) {
 430                 if ( cr3 & CR3_L_MODE_RESEVED_BITS) {
 431                         printk(KERN_DEBUG "set_cr3: #GP, reserved bits\n");
 432                         inject_gp(vcpu);
 433                         return;
 434                 }
 435         } else {
 436                 if (cr3 & CR3_RESEVED_BITS) {
 437                         printk(KERN_DEBUG "set_cr3: #GP, reserved bits\n");
 438                         inject_gp(vcpu);
 439                         return;
 440                 }
 441                 if (is_paging(vcpu) && is_pae(vcpu) &&
 442                     pdptrs_have_reserved_bits_set(vcpu, cr3)) {
 443                         printk(KERN_DEBUG "set_cr3: #GP, pdptrs "
 444                                "reserved bits\n");
 445                         inject_gp(vcpu);
 446                         return;
 447                 }
 448         }
 449
 450         vcpu->cr3 = cr3;
 451         spin_lock(&vcpu->kvm->lock);
 452         vcpu->mmu.new_cr3(vcpu);
 453         spin_unlock(&vcpu->kvm->lock);
 454 }
 455 EXPORT_SYMBOL_GPL(set_cr3);
 456
 457 void set_cr8(struct kvm_vcpu *vcpu, unsigned long cr8)
 458 {
 459         if ( cr8 & CR8_RESEVED_BITS) {
 460                 printk(KERN_DEBUG "set_cr8: #GP, reserved bits 0x%lx\n", cr8);
 461                 inject_gp(vcpu);
 462                 return;
 463         }
 464         vcpu->cr8 = cr8;
 465 }
 466 EXPORT_SYMBOL_GPL(set_cr8);
 467
 468 void fx_init(struct kvm_vcpu *vcpu)
 469 {
 470         struct __attribute__ ((__packed__)) fx_image_s {
 471                 u16 control; //fcw
 472                 u16 status; //fsw
 473                 u16 tag; // ftw
 474                 u16 opcode; //fop
 475                 u64 ip; // fpu ip
 476                 u64 operand;// fpu dp
 477                 u32 mxcsr;
 478                 u32 mxcsr_mask;
 479
 480         } *fx_image;
 481
 482         fx_save(vcpu->host_fx_image);
 483         fpu_init();
 484         fx_save(vcpu->guest_fx_image);
 485         fx_restore(vcpu->host_fx_image);
 486
 487         fx_image = (struct fx_image_s *)vcpu->guest_fx_image;
 488         fx_image->mxcsr = 0x1f80;
 489         memset(vcpu->guest_fx_image + sizeof(struct fx_image_s),
 490                0, FX_IMAGE_SIZE - sizeof(struct fx_image_s));
 491 }
 492 EXPORT_SYMBOL_GPL(fx_init);
 493
 494 /*
 495  * Creates some virtual cpus.  Good luck creating more than one.
 496  */
 497 static int kvm_dev_ioctl_create_vcpu(struct kvm *kvm, int n)
 498 {
 499         int r;
 500         struct kvm_vcpu *vcpu;
 501
 502         r = -EINVAL;
 503         if (!valid_vcpu(n))
 504                 goto out;
 505
 506         vcpu = &kvm->vcpus[n];
 507
 508         mutex_lock(&vcpu->mutex);
 509
 510         if (vcpu->vmcs) {
 511                 mutex_unlock(&vcpu->mutex);
 512                 return -EEXIST;
 513         }
 514
 515         vcpu->host_fx_image = (char*)ALIGN((hva_t)vcpu->fx_buf,
 516                                            FX_IMAGE_ALIGN);
 517         vcpu->guest_fx_image = vcpu->host_fx_image + FX_IMAGE_SIZE;
 518
 519         vcpu->cpu = -1;  /* First load will set up TR */
 520         vcpu->kvm = kvm;
 521         r = kvm_arch_ops->vcpu_create(vcpu);
 522         if (r < 0)
 523                 goto out_free_vcpus;
 524
 525         kvm_arch_ops->vcpu_load(vcpu);
 526
 527         r = kvm_arch_ops->vcpu_setup(vcpu);
 528         if (r >= 0)
 529                 r = kvm_mmu_init(vcpu);
 530
 531         vcpu_put(vcpu);
 532
 533         if (r < 0)
 534                 goto out_free_vcpus;
 535
 536         return 0;
 537
 538 out_free_vcpus:
 539         kvm_free_vcpu(vcpu);
 540         mutex_unlock(&vcpu->mutex);
 541 out:
 542         return r;
 543 }
 544
 545 /*
 546  * Allocate some memory and give it an address in the guest physical address
 547  * space.
 548  *
 549  * Discontiguous memory is allowed, mostly for framebuffers.
 550  */
 551 static int kvm_dev_ioctl_set_memory_region(struct kvm *kvm,
 552                                            struct kvm_memory_region *mem)
 553 {
 554         int r;
 555         gfn_t base_gfn;
 556         unsigned long npages;
 557         unsigned long i;
 558         struct kvm_memory_slot *memslot;
 559         struct kvm_memory_slot old, new;
 560         int memory_config_version;
 561
 562         r = -EINVAL;
 563         /* General sanity checks */
 564         if (mem->memory_size & (PAGE_SIZE - 1))
 565                 goto out;
 566         if (mem->guest_phys_addr & (PAGE_SIZE - 1))
 567                 goto out;
 568         if (mem->slot >= KVM_MEMORY_SLOTS)
 569                 goto out;
 570         if (mem->guest_phys_addr + mem->memory_size < mem->guest_phys_addr)
 571                 goto out;
 572
 573         memslot = &kvm->memslots[mem->slot];
 574         base_gfn = mem->guest_phys_addr >> PAGE_SHIFT;
 575         npages = mem->memory_size >> PAGE_SHIFT;
 576
 577         if (!npages)
 578                 mem->flags &= ~KVM_MEM_LOG_DIRTY_PAGES;
 579
 580 raced:
 581         spin_lock(&kvm->lock);
 582
 583         memory_config_version = kvm->memory_config_version;
 584         new = old = *memslot;
 585
 586         new.base_gfn = base_gfn;
 587         new.npages = npages;
 588         new.flags = mem->flags;
 589
 590         /* Disallow changing a memory slot's size. */
 591         r = -EINVAL;
 592         if (npages && old.npages && npages != old.npages)
 593                 goto out_unlock;
 594
 595         /* Check for overlaps */
 596         r = -EEXIST;
 597         for (i = 0; i < KVM_MEMORY_SLOTS; ++i) {
 598                 struct kvm_memory_slot *s = &kvm->memslots[i];
 599
 600                 if (s == memslot)
 601                         continue;
 602                 if (!((base_gfn + npages <= s->base_gfn) ||
 603                       (base_gfn >= s->base_gfn + s->npages)))
 604                         goto out_unlock;
 605         }
 606         /*
 607          * Do memory allocations outside lock.  memory_config_version will
 608          * detect any races.
 609          */
 610         spin_unlock(&kvm->lock);
 611
 612         /* Deallocate if slot is being removed */
 613         if (!npages)
 614                 new.phys_mem = 0;
 615
 616         /* Free page dirty bitmap if unneeded */
 617         if (!(new.flags & KVM_MEM_LOG_DIRTY_PAGES))
 618                 new.dirty_bitmap = 0;
 619
 620         r = -ENOMEM;
 621
 622         /* Allocate if a slot is being created */
 623         if (npages && !new.phys_mem) {
 624                 new.phys_mem = vmalloc(npages * sizeof(struct page *));
 625
 626                 if (!new.phys_mem)
 627                         goto out_free;
 628
 629                 memset(new.phys_mem, 0, npages * sizeof(struct page *));
 630                 for (i = 0; i < npages; ++i) {
 631                         new.phys_mem[i] = alloc_page(GFP_HIGHUSER
 632                                                      | __GFP_ZERO);
 633                         if (!new.phys_mem[i])
 634                                 goto out_free;
 635                 }
 636         }
 637
 638         /* Allocate page dirty bitmap if needed */
 639         if ((new.flags & KVM_MEM_LOG_DIRTY_PAGES) && !new.dirty_bitmap) {
 640                 unsigned dirty_bytes = ALIGN(npages, BITS_PER_LONG) / 8;
 641
 642                 new.dirty_bitmap = vmalloc(dirty_bytes);
 643                 if (!new.dirty_bitmap)
 644                         goto out_free;
 645                 memset(new.dirty_bitmap, 0, dirty_bytes);
 646         }
 647
 648         spin_lock(&kvm->lock);
 649
 650         if (memory_config_version != kvm->memory_config_version) {
 651                 spin_unlock(&kvm->lock);
 652                 kvm_free_physmem_slot(&new, &old);
 653                 goto raced;
 654         }
 655
 656         r = -EAGAIN;
 657         if (kvm->busy)
 658                 goto out_unlock;
 659
 660         if (mem->slot >= kvm->nmemslots)
 661                 kvm->nmemslots = mem->slot + 1;
 662
 663         *memslot = new;
 664         ++kvm->memory_config_version;
 665
 666         spin_unlock(&kvm->lock);
 667
 668         for (i = 0; i < KVM_MAX_VCPUS; ++i) {
 669                 struct kvm_vcpu *vcpu;
 670
 671                 vcpu = vcpu_load(kvm, i);
 672                 if (!vcpu)
 673                         continue;
 674                 kvm_mmu_reset_context(vcpu);
 675                 vcpu_put(vcpu);
 676         }
 677
 678         kvm_free_physmem_slot(&old, &new);
 679         return 0;
 680
 681 out_unlock:
 682         spin_unlock(&kvm->lock);
 683 out_free:
 684         kvm_free_physmem_slot(&new, &old);
 685 out:
 686         return r;
 687 }
 688
 689 /*
 690  * Get (and clear) the dirty memory log for a memory slot.
 691  */
 692 static int kvm_dev_ioctl_get_dirty_log(struct kvm *kvm,
 693                                        struct kvm_dirty_log *log)
 694 {
 695         struct kvm_memory_slot *memslot;
 696         int r, i;
 697         int n;
 698         unsigned long any = 0;
 699
 700         spin_lock(&kvm->lock);
 701
 702         /*
 703          * Prevent changes to guest memory configuration even while the lock
 704          * is not taken.
 705          */
 706         ++kvm->busy;
 707         spin_unlock(&kvm->lock);
 708         r = -EINVAL;
 709         if (log->slot >= KVM_MEMORY_SLOTS)
 710                 goto out;
 711
 712         memslot = &kvm->memslots[log->slot];
 713         r = -ENOENT;
 714         if (!memslot->dirty_bitmap)
 715                 goto out;
 716
 717         n = ALIGN(memslot->npages, 8) / 8;
 718
 719         for (i = 0; !any && i < n; ++i)
 720                 any = memslot->dirty_bitmap[i];
 721
 722         r = -EFAULT;
 723         if (copy_to_user(log->dirty_bitmap, memslot->dirty_bitmap, n))
 724                 goto out;
 725
 726
 727         if (any) {
 728                 spin_lock(&kvm->lock);
 729                 kvm_mmu_slot_remove_write_access(kvm, log->slot);
 730                 spin_unlock(&kvm->lock);
 731                 memset(memslot->dirty_bitmap, 0, n);
 732                 for (i = 0; i < KVM_MAX_VCPUS; ++i) {
 733                         struct kvm_vcpu *vcpu = vcpu_load(kvm, i);
 734
 735                         if (!vcpu)
 736                                 continue;
 737                         kvm_arch_ops->tlb_flush(vcpu);
 738                         vcpu_put(vcpu);
 739                 }
 740         }
 741
 742         r = 0;
 743
 744 out:
 745         spin_lock(&kvm->lock);
 746         --kvm->busy;
 747         spin_unlock(&kvm->lock);
 748         return r;
 749 }
 750
 751 struct kvm_memory_slot *gfn_to_memslot(struct kvm *kvm, gfn_t gfn)
 752 {
 753         int i;
 754
 755         for (i = 0; i < kvm->nmemslots; ++i) {
 756                 struct kvm_memory_slot *memslot = &kvm->memslots[i];
 757
 758                 if (gfn >= memslot->base_gfn
 759                     && gfn < memslot->base_gfn + memslot->npages)
 760                         return memslot;
 761         }
 762         return 0;
 763 }
 764 EXPORT_SYMBOL_GPL(gfn_to_memslot);
 765
 766 void mark_page_dirty(struct kvm *kvm, gfn_t gfn)
 767 {
 768         int i;
 769         struct kvm_memory_slot *memslot = 0;
 770         unsigned long rel_gfn;
 771
 772         for (i = 0; i < kvm->nmemslots; ++i) {
 773                 memslot = &kvm->memslots[i];
 774
 775                 if (gfn >= memslot->base_gfn
 776                     && gfn < memslot->base_gfn + memslot->npages) {
 777
 778                         if (!memslot || !memslot->dirty_bitmap)
 779                                 return;
 780
 781                         rel_gfn = gfn - memslot->base_gfn;
 782
 783                         /* avoid RMW */
 784                         if (!test_bit(rel_gfn, memslot->dirty_bitmap))
 785                                 set_bit(rel_gfn, memslot->dirty_bitmap);
 786                         return;
 787                 }
 788         }
 789 }
 790
 791 static int emulator_read_std(unsigned long addr,
 792                              unsigned long *val,
 793                              unsigned int bytes,
 794                              struct x86_emulate_ctxt *ctxt)
 795 {
 796         struct kvm_vcpu *vcpu = ctxt->vcpu;
 797         void *data = val;
 798
 799         while (bytes) {
 800                 gpa_t gpa = vcpu->mmu.gva_to_gpa(vcpu, addr);
 801                 unsigned offset = addr & (PAGE_SIZE-1);
 802                 unsigned tocopy = min(bytes, (unsigned)PAGE_SIZE - offset);
 803                 unsigned long pfn;
 804                 struct kvm_memory_slot *memslot;
 805                 void *page;
 806
 807                 if (gpa == UNMAPPED_GVA)
 808                         return X86EMUL_PROPAGATE_FAULT;
 809                 pfn = gpa >> PAGE_SHIFT;
 810                 memslot = gfn_to_memslot(vcpu->kvm, pfn);
 811                 if (!memslot)
 812                         return X86EMUL_UNHANDLEABLE;
 813                 page = kmap_atomic(gfn_to_page(memslot, pfn), KM_USER0);
 814
 815                 memcpy(data, page + offset, tocopy);
 816
 817                 kunmap_atomic(page, KM_USER0);
 818
 819                 bytes -= tocopy;
 820                 data += tocopy;
 821                 addr += tocopy;
 822         }
 823
 824         return X86EMUL_CONTINUE;
 825 }
 826
 827 static int emulator_write_std(unsigned long addr,
 828                               unsigned long val,
 829                               unsigned int bytes,
 830                               struct x86_emulate_ctxt *ctxt)
 831 {
 832         printk(KERN_ERR "emulator_write_std: addr %lx n %d\n",
 833                addr, bytes);
 834         return X86EMUL_UNHANDLEABLE;
 835 }
 836
 837 static int emulator_read_emulated(unsigned long addr,
 838                                   unsigned long *val,
 839                                   unsigned int bytes,
 840                                   struct x86_emulate_ctxt *ctxt)
 841 {
 842         struct kvm_vcpu *vcpu = ctxt->vcpu;
 843
 844         if (vcpu->mmio_read_completed) {
 845                 memcpy(val, vcpu->mmio_data, bytes);
 846                 vcpu->mmio_read_completed = 0;
 847                 return X86EMUL_CONTINUE;
 848         } else if (emulator_read_std(addr, val, bytes, ctxt)
 849                    == X86EMUL_CONTINUE)
 850                 return X86EMUL_CONTINUE;
 851         else {
 852                 gpa_t gpa = vcpu->mmu.gva_to_gpa(vcpu, addr);
 853                 if (gpa == UNMAPPED_GVA)
 854                         return vcpu_printf(vcpu, "not present\n"), X86EMUL_PROPAGATE_FAULT;
 855                 vcpu->mmio_needed = 1;
 856                 vcpu->mmio_phys_addr = gpa;
 857                 vcpu->mmio_size = bytes;
 858                 vcpu->mmio_is_write = 0;
 859
 860                 return X86EMUL_UNHANDLEABLE;
 861         }
 862 }
 863
 864 static int emulator_write_emulated(unsigned long addr,
 865                                    unsigned long val,
 866                                    unsigned int bytes,
 867                                    struct x86_emulate_ctxt *ctxt)
 868 {
 869         struct kvm_vcpu *vcpu = ctxt->vcpu;
 870         gpa_t gpa = vcpu->mmu.gva_to_gpa(vcpu, addr);
 871
 872         if (gpa == UNMAPPED_GVA)
 873                 return X86EMUL_PROPAGATE_FAULT;
 874
 875         vcpu->mmio_needed = 1;
 876         vcpu->mmio_phys_addr = gpa;
 877         vcpu->mmio_size = bytes;
 878         vcpu->mmio_is_write = 1;
 879         memcpy(vcpu->mmio_data, &val, bytes);
 880
 881         return X86EMUL_CONTINUE;
 882 }
 883
 884 static int emulator_cmpxchg_emulated(unsigned long addr,
 885                                      unsigned long old,
 886                                      unsigned long new,
 887                                      unsigned int bytes,
 888                                      struct x86_emulate_ctxt *ctxt)
 889 {
 890         static int reported;
 891
 892         if (!reported) {
 893                 reported = 1;
 894                 printk(KERN_WARNING "kvm: emulating exchange as write\n");
 895         }
 896         return emulator_write_emulated(addr, new, bytes, ctxt);
 897 }
 898
 899 static unsigned long get_segment_base(struct kvm_vcpu *vcpu, int seg)
 900 {
 901         return kvm_arch_ops->get_segment_base(vcpu, seg);
 902 }
 903
 904 int emulate_invlpg(struct kvm_vcpu *vcpu, gva_t address)
 905 {
 906         spin_lock(&vcpu->kvm->lock);
 907         vcpu->mmu.inval_page(vcpu, address);
 908         spin_unlock(&vcpu->kvm->lock);
 909         kvm_arch_ops->invlpg(vcpu, address);
 910         return X86EMUL_CONTINUE;
 911 }
 912
 913 int emulate_clts(struct kvm_vcpu *vcpu)
 914 {
 915         unsigned long cr0 = vcpu->cr0;
 916
 917         cr0 &= ~CR0_TS_MASK;
 918         kvm_arch_ops->set_cr0(vcpu, cr0);
 919         return X86EMUL_CONTINUE;
 920 }
 921
 922 int emulator_get_dr(struct x86_emulate_ctxt* ctxt, int dr, unsigned long *dest)
 923 {
 924         struct kvm_vcpu *vcpu = ctxt->vcpu;
 925
 926         switch (dr) {
 927         case 0 ... 3:
 928                 *dest = kvm_arch_ops->get_dr(vcpu, dr);
 929                 return X86EMUL_CONTINUE;
 930         default:
 931                 printk(KERN_DEBUG "%s: unexpected dr %u\n",
 932                        __FUNCTION__, dr);
 933                 return X86EMUL_UNHANDLEABLE;
 934         }
 935 }
 936
 937 int emulator_set_dr(struct x86_emulate_ctxt *ctxt, int dr, unsigned long value)
 938 {
 939         unsigned long mask = (ctxt->mode == X86EMUL_MODE_PROT64) ? ~0ULL : ~0U;
 940         int exception;
 941
 942         kvm_arch_ops->set_dr(ctxt->vcpu, dr, value & mask, &exception);
 943         if (exception) {
 944                 /* FIXME: better handling */
 945                 return X86EMUL_UNHANDLEABLE;
 946         }
 947         return X86EMUL_CONTINUE;
 948 }
 949
 950 static void report_emulation_failure(struct x86_emulate_ctxt *ctxt)
 951 {
 952         static int reported;
 953         u8 opcodes[4];
 954         unsigned long rip = ctxt->vcpu->rip;
 955         unsigned long rip_linear;
 956
 957         rip_linear = rip + get_segment_base(ctxt->vcpu, VCPU_SREG_CS);
 958
 959         if (reported)
 960                 return;
 961
 962         emulator_read_std(rip_linear, (void *)opcodes, 4, ctxt);
 963
 964         printk(KERN_ERR "emulation failed but !mmio_needed?"
 965                " rip %lx %02x %02x %02x %02x\n",
 966                rip, opcodes[0], opcodes[1], opcodes[2], opcodes[3]);
 967         reported = 1;
 968 }
 969
 970 struct x86_emulate_ops emulate_ops = {
 971         .read_std            = emulator_read_std,
 972         .write_std           = emulator_write_std,
 973         .read_emulated       = emulator_read_emulated,
 974         .write_emulated      = emulator_write_emulated,
 975         .cmpxchg_emulated    = emulator_cmpxchg_emulated,
 976 };
 977
 978 int emulate_instruction(struct kvm_vcpu *vcpu,
 979                         struct kvm_run *run,
 980                         unsigned long cr2,
 981                         u16 error_code)
 982 {
 983         struct x86_emulate_ctxt emulate_ctxt;
 984         int r;
 985         int cs_db, cs_l;
 986
 987         kvm_arch_ops->cache_regs(vcpu);
 988
 989         kvm_arch_ops->get_cs_db_l_bits(vcpu, &cs_db, &cs_l);
 990
 991         emulate_ctxt.vcpu = vcpu;
 992         emulate_ctxt.eflags = kvm_arch_ops->get_rflags(vcpu);
 993         emulate_ctxt.cr2 = cr2;
 994         emulate_ctxt.mode = (emulate_ctxt.eflags & X86_EFLAGS_VM)
 995                 ? X86EMUL_MODE_REAL : cs_l
 996                 ? X86EMUL_MODE_PROT64 : cs_db
 997                 ? X86EMUL_MODE_PROT32 : X86EMUL_MODE_PROT16;
 998
 999         if (emulate_ctxt.mode == X86EMUL_MODE_PROT64) {
1000                 emulate_ctxt.cs_base = 0;
1001                 emulate_ctxt.ds_base = 0;
1002                 emulate_ctxt.es_base = 0;
1003                 emulate_ctxt.ss_base = 0;
1004         } else {
1005                 emulate_ctxt.cs_base = get_segment_base(vcpu, VCPU_SREG_CS);
1006                 emulate_ctxt.ds_base = get_segment_base(vcpu, VCPU_SREG_DS);
1007                 emulate_ctxt.es_base = get_segment_base(vcpu, VCPU_SREG_ES);
1008                 emulate_ctxt.ss_base = get_segment_base(vcpu, VCPU_SREG_SS);
1009         }
1010
1011         emulate_ctxt.gs_base = get_segment_base(vcpu, VCPU_SREG_GS);
1012         emulate_ctxt.fs_base = get_segment_base(vcpu, VCPU_SREG_FS);
1013
1014         vcpu->mmio_is_write = 0;
1015         r = x86_emulate_memop(&emulate_ctxt, &emulate_ops);
1016
1017         if ((r || vcpu->mmio_is_write) && run) {
1018                 run->mmio.phys_addr = vcpu->mmio_phys_addr;
1019                 memcpy(run->mmio.data, vcpu->mmio_data, 8);
1020                 run->mmio.len = vcpu->mmio_size;
1021                 run->mmio.is_write = vcpu->mmio_is_write;
1022         }
1023
1024         if (r) {
1025                 if (!vcpu->mmio_needed) {
1026                         report_emulation_failure(&emulate_ctxt);
1027                         return EMULATE_FAIL;
1028                 }
1029                 return EMULATE_DO_MMIO;
1030         }
1031
1032         kvm_arch_ops->decache_regs(vcpu);
1033         kvm_arch_ops->set_rflags(vcpu, emulate_ctxt.eflags);
1034
1035         if (vcpu->mmio_is_write)
1036                 return EMULATE_DO_MMIO;
1037
1038         return EMULATE_DONE;
1039 }
1040 EXPORT_SYMBOL_GPL(emulate_instruction);
1041
1042 static u64 mk_cr_64(u64 curr_cr, u32 new_val)
1043 {
1044         return (curr_cr & ~((1ULL << 32) - 1)) | new_val;
1045 }
1046
1047 void realmode_lgdt(struct kvm_vcpu *vcpu, u16 limit, unsigned long base)
1048 {
1049         struct descriptor_table dt = { limit, base };
1050
1051         kvm_arch_ops->set_gdt(vcpu, &dt);
1052 }
1053
1054 void realmode_lidt(struct kvm_vcpu *vcpu, u16 limit, unsigned long base)
1055 {
1056         struct descriptor_table dt = { limit, base };
1057
1058         kvm_arch_ops->set_idt(vcpu, &dt);
1059 }
1060
1061 void realmode_lmsw(struct kvm_vcpu *vcpu, unsigned long msw,
1062                    unsigned long *rflags)
1063 {
1064         lmsw(vcpu, msw);
1065         *rflags = kvm_arch_ops->get_rflags(vcpu);
1066 }
1067
1068 unsigned long realmode_get_cr(struct kvm_vcpu *vcpu, int cr)
1069 {
1070         switch (cr) {
1071         case 0:
1072                 return vcpu->cr0;
1073         case 2:
1074                 return vcpu->cr2;
1075         case 3:
1076                 return vcpu->cr3;
1077         case 4:
1078                 return vcpu->cr4;
1079         default:
1080                 vcpu_printf(vcpu, "%s: unexpected cr %u\n", __FUNCTION__, cr);
1081                 return 0;
1082         }
1083 }
1084
1085 void realmode_set_cr(struct kvm_vcpu *vcpu, int cr, unsigned long val,
1086                      unsigned long *rflags)
1087 {
1088         switch (cr) {
1089         case 0:
1090                 set_cr0(vcpu, mk_cr_64(vcpu->cr0, val));
1091                 *rflags = kvm_arch_ops->get_rflags(vcpu);
1092                 break;
1093         case 2:
1094                 vcpu->cr2 = val;
1095                 break;
1096         case 3:
1097                 set_cr3(vcpu, val);
1098                 break;
1099         case 4:
1100                 set_cr4(vcpu, mk_cr_64(vcpu->cr4, val));
1101                 break;
1102         default:
1103                 vcpu_printf(vcpu, "%s: unexpected cr %u\n", __FUNCTION__, cr);
1104         }
1105 }
1106
1107 int kvm_get_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata)
1108 {
1109         u64 data;
1110
1111         switch (msr) {
1112         case 0xc0010010: /* SYSCFG */
1113         case 0xc0010015: /* HWCR */
1114         case MSR_IA32_PLATFORM_ID:
1115         case MSR_IA32_P5_MC_ADDR:
1116         case MSR_IA32_P5_MC_TYPE:
1117         case MSR_IA32_MC0_CTL:
1118         case MSR_IA32_MCG_STATUS:
1119         case MSR_IA32_MCG_CAP:
1120         case MSR_IA32_MC0_MISC:
1121         case MSR_IA32_MC0_MISC+4:
1122         case MSR_IA32_MC0_MISC+8:
1123         case MSR_IA32_MC0_MISC+12:
1124         case MSR_IA32_MC0_MISC+16:
1125         case MSR_IA32_UCODE_REV:
1126         case MSR_IA32_PERF_STATUS:
1127                 /* MTRR registers */
1128         case 0xfe:
1129         case 0x200 ... 0x2ff:
1130                 data = 0;
1131                 break;
1132         case 0xcd: /* fsb frequency */
1133                 data = 3;
1134                 break;
1135         case MSR_IA32_APICBASE:
1136                 data = vcpu->apic_base;
1137                 break;
1138 #ifdef CONFIG_X86_64
1139         case MSR_EFER:
1140                 data = vcpu->shadow_efer;
1141                 break;
1142 #endif
1143         default:
1144                 printk(KERN_ERR "kvm: unhandled rdmsr: 0x%x\n", msr);
1145                 return 1;
1146         }
1147         *pdata = data;
1148         return 0;
1149 }
1150 EXPORT_SYMBOL_GPL(kvm_get_msr_common);
1151
1152 /*
1153  * Reads an msr value (of 'msr_index') into 'pdata'.
1154  * Returns 0 on success, non-0 otherwise.
1155  * Assumes vcpu_load() was already called.
1156  */
1157 static int get_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 *pdata)
1158 {
1159         return kvm_arch_ops->get_msr(vcpu, msr_index, pdata);
1160 }
1161
1162 #ifdef CONFIG_X86_64
1163
1164 static void set_efer(struct kvm_vcpu *vcpu, u64 efer)
1165 {
1166         if (efer & EFER_RESERVED_BITS) {
1167                 printk(KERN_DEBUG "set_efer: 0x%llx #GP, reserved bits\n",
1168                        efer);
1169                 inject_gp(vcpu);
1170                 return;
1171         }
1172
1173         if (is_paging(vcpu)
1174             && (vcpu->shadow_efer & EFER_LME) != (efer & EFER_LME)) {
1175                 printk(KERN_DEBUG "set_efer: #GP, change LME while paging\n");
1176                 inject_gp(vcpu);
1177                 return;
1178         }
1179
1180         kvm_arch_ops->set_efer(vcpu, efer);
1181
1182         efer &= ~EFER_LMA;
1183         efer |= vcpu->shadow_efer & EFER_LMA;
1184
1185         vcpu->shadow_efer = efer;
1186 }
1187
1188 #endif
1189
1190 int kvm_set_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 data)
1191 {
1192         switch (msr) {
1193 #ifdef CONFIG_X86_64
1194         case MSR_EFER:
1195                 set_efer(vcpu, data);
1196                 break;
1197 #endif
1198         case MSR_IA32_MC0_STATUS:
1199                 printk(KERN_WARNING "%s: MSR_IA32_MC0_STATUS 0x%llx, nop\n",
1200                        __FUNCTION__, data);
1201                 break;
1202         case MSR_IA32_UCODE_REV:
1203         case MSR_IA32_UCODE_WRITE:
1204         case 0x200 ... 0x2ff: /* MTRRs */
1205                 break;
1206         case MSR_IA32_APICBASE:
1207                 vcpu->apic_base = data;
1208                 break;
1209         default:
1210                 printk(KERN_ERR "kvm: unhandled wrmsr: 0x%x\n", msr);
1211                 return 1;
1212         }
1213         return 0;
1214 }
1215 EXPORT_SYMBOL_GPL(kvm_set_msr_common);
1216
1217 /*
1218  * Writes msr value into into the appropriate "register".
1219  * Returns 0 on success, non-0 otherwise.
1220  * Assumes vcpu_load() was already called.
1221  */
1222 static int set_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 data)
1223 {
1224         return kvm_arch_ops->set_msr(vcpu, msr_index, data);
1225 }
1226
1227 void kvm_resched(struct kvm_vcpu *vcpu)
1228 {
1229         vcpu_put(vcpu);
1230         cond_resched();
1231         /* Cannot fail -  no vcpu unplug yet. */
1232         vcpu_load(vcpu->kvm, vcpu_slot(vcpu));
1233 }
1234 EXPORT_SYMBOL_GPL(kvm_resched);
1235
1236 void load_msrs(struct vmx_msr_entry *e, int n)
1237 {
1238         int i;
1239
1240         for (i = 0; i < n; ++i)
1241                 wrmsrl(e[i].index, e[i].data);
1242 }
1243 EXPORT_SYMBOL_GPL(load_msrs);
1244
1245 void save_msrs(struct vmx_msr_entry *e, int n)
1246 {
1247         int i;
1248
1249         for (i = 0; i < n; ++i)
1250                 rdmsrl(e[i].index, e[i].data);
1251 }
1252 EXPORT_SYMBOL_GPL(save_msrs);
1253
1254 static int kvm_dev_ioctl_run(struct kvm *kvm, struct kvm_run *kvm_run)
1255 {
1256         struct kvm_vcpu *vcpu;
1257         int r;
1258
1259         if (!valid_vcpu(kvm_run->vcpu))
1260                 return -EINVAL;
1261
1262         vcpu = vcpu_load(kvm, kvm_run->vcpu);
1263         if (!vcpu)
1264                 return -ENOENT;
1265
1266         if (kvm_run->emulated) {
1267                 kvm_arch_ops->skip_emulated_instruction(vcpu);
1268                 kvm_run->emulated = 0;
1269         }
1270
1271         if (kvm_run->mmio_completed) {
1272                 memcpy(vcpu->mmio_data, kvm_run->mmio.data, 8);
1273                 vcpu->mmio_read_completed = 1;
1274         }
1275
1276         vcpu->mmio_needed = 0;
1277
1278         r = kvm_arch_ops->run(vcpu, kvm_run);
1279
1280         vcpu_put(vcpu);
1281         return r;
1282 }
1283
1284 static int kvm_dev_ioctl_get_regs(struct kvm *kvm, struct kvm_regs *regs)
1285 {
1286         struct kvm_vcpu *vcpu;
1287
1288         if (!valid_vcpu(regs->vcpu))
1289                 return -EINVAL;
1290
1291         vcpu = vcpu_load(kvm, regs->vcpu);
1292         if (!vcpu)
1293                 return -ENOENT;
1294
1295         kvm_arch_ops->cache_regs(vcpu);
1296
1297         regs->rax = vcpu->regs[VCPU_REGS_RAX];
1298         regs->rbx = vcpu->regs[VCPU_REGS_RBX];
1299         regs->rcx = vcpu->regs[VCPU_REGS_RCX];
1300         regs->rdx = vcpu->regs[VCPU_REGS_RDX];
1301         regs->rsi = vcpu->regs[VCPU_REGS_RSI];
1302         regs->rdi = vcpu->regs[VCPU_REGS_RDI];
1303         regs->rsp = vcpu->regs[VCPU_REGS_RSP];
1304         regs->rbp = vcpu->regs[VCPU_REGS_RBP];
1305 #ifdef CONFIG_X86_64
1306         regs->r8 = vcpu->regs[VCPU_REGS_R8];
1307         regs->r9 = vcpu->regs[VCPU_REGS_R9];
1308         regs->r10 = vcpu->regs[VCPU_REGS_R10];
1309         regs->r11 = vcpu->regs[VCPU_REGS_R11];
1310         regs->r12 = vcpu->regs[VCPU_REGS_R12];
1311         regs->r13 = vcpu->regs[VCPU_REGS_R13];
1312         regs->r14 = vcpu->regs[VCPU_REGS_R14];
1313         regs->r15 = vcpu->regs[VCPU_REGS_R15];
1314 #endif
1315
1316         regs->rip = vcpu->rip;
1317         regs->rflags = kvm_arch_ops->get_rflags(vcpu);
1318
1319         /*
1320          * Don't leak debug flags in case they were set for guest debugging
1321          */
1322         if (vcpu->guest_debug.enabled && vcpu->guest_debug.singlestep)
1323                 regs->rflags &= ~(X86_EFLAGS_TF | X86_EFLAGS_RF);
1324
1325         vcpu_put(vcpu);
1326
1327         return 0;
1328 }
1329
1330 static int kvm_dev_ioctl_set_regs(struct kvm *kvm, struct kvm_regs *regs)
1331 {
1332         struct kvm_vcpu *vcpu;
1333
1334         if (!valid_vcpu(regs->vcpu))
1335                 return -EINVAL;
1336
1337         vcpu = vcpu_load(kvm, regs->vcpu);
1338         if (!vcpu)
1339                 return -ENOENT;
1340
1341         vcpu->regs[VCPU_REGS_RAX] = regs->rax;
1342         vcpu->regs[VCPU_REGS_RBX] = regs->rbx;
1343         vcpu->regs[VCPU_REGS_RCX] = regs->rcx;
1344         vcpu->regs[VCPU_REGS_RDX] = regs->rdx;
1345         vcpu->regs[VCPU_REGS_RSI] = regs->rsi;
1346         vcpu->regs[VCPU_REGS_RDI] = regs->rdi;
1347         vcpu->regs[VCPU_REGS_RSP] = regs->rsp;
1348         vcpu->regs[VCPU_REGS_RBP] = regs->rbp;
1349 #ifdef CONFIG_X86_64
1350         vcpu->regs[VCPU_REGS_R8] = regs->r8;
1351         vcpu->regs[VCPU_REGS_R9] = regs->r9;
1352         vcpu->regs[VCPU_REGS_R10] = regs->r10;
1353         vcpu->regs[VCPU_REGS_R11] = regs->r11;
1354         vcpu->regs[VCPU_REGS_R12] = regs->r12;
1355         vcpu->regs[VCPU_REGS_R13] = regs->r13;
1356         vcpu->regs[VCPU_REGS_R14] = regs->r14;
1357         vcpu->regs[VCPU_REGS_R15] = regs->r15;
1358 #endif
1359
1360         vcpu->rip = regs->rip;
1361         kvm_arch_ops->set_rflags(vcpu, regs->rflags);
1362
1363         kvm_arch_ops->decache_regs(vcpu);
1364
1365         vcpu_put(vcpu);
1366
1367         return 0;
1368 }
1369
1370 static void get_segment(struct kvm_vcpu *vcpu,
1371                         struct kvm_segment *var, int seg)
1372 {
1373         return kvm_arch_ops->get_segment(vcpu, var, seg);
1374 }
1375
1376 static int kvm_dev_ioctl_get_sregs(struct kvm *kvm, struct kvm_sregs *sregs)
1377 {
1378         struct kvm_vcpu *vcpu;
1379         struct descriptor_table dt;
1380
1381         if (!valid_vcpu(sregs->vcpu))
1382                 return -EINVAL;
1383         vcpu = vcpu_load(kvm, sregs->vcpu);
1384         if (!vcpu)
1385                 return -ENOENT;
1386
1387         get_segment(vcpu, &sregs->cs, VCPU_SREG_CS);
1388         get_segment(vcpu, &sregs->ds, VCPU_SREG_DS);
1389         get_segment(vcpu, &sregs->es, VCPU_SREG_ES);
1390         get_segment(vcpu, &sregs->fs, VCPU_SREG_FS);
1391         get_segment(vcpu, &sregs->gs, VCPU_SREG_GS);
1392         get_segment(vcpu, &sregs->ss, VCPU_SREG_SS);
1393
1394         get_segment(vcpu, &sregs->tr, VCPU_SREG_TR);
1395         get_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR);
1396
1397         kvm_arch_ops->get_idt(vcpu, &dt);
1398         sregs->idt.limit = dt.limit;
1399         sregs->idt.base = dt.base;
1400         kvm_arch_ops->get_gdt(vcpu, &dt);
1401         sregs->gdt.limit = dt.limit;
1402         sregs->gdt.base = dt.base;
1403
1404         sregs->cr0 = vcpu->cr0;
1405         sregs->cr2 = vcpu->cr2;
1406         sregs->cr3 = vcpu->cr3;
1407         sregs->cr4 = vcpu->cr4;
1408         sregs->cr8 = vcpu->cr8;
1409         sregs->efer = vcpu->shadow_efer;
1410         sregs->apic_base = vcpu->apic_base;
1411
1412         memcpy(sregs->interrupt_bitmap, vcpu->irq_pending,
1413                sizeof sregs->interrupt_bitmap);
1414
1415         vcpu_put(vcpu);
1416
1417         return 0;
1418 }
1419
1420 static void set_segment(struct kvm_vcpu *vcpu,
1421                         struct kvm_segment *var, int seg)
1422 {
1423         return kvm_arch_ops->set_segment(vcpu, var, seg);
1424 }
1425
1426 static int kvm_dev_ioctl_set_sregs(struct kvm *kvm, struct kvm_sregs *sregs)
1427 {
1428         struct kvm_vcpu *vcpu;
1429         int mmu_reset_needed = 0;
1430         int i;
1431         struct descriptor_table dt;
1432
1433         if (!valid_vcpu(sregs->vcpu))
1434                 return -EINVAL;
1435         vcpu = vcpu_load(kvm, sregs->vcpu);
1436         if (!vcpu)
1437                 return -ENOENT;
1438
1439         set_segment(vcpu, &sregs->cs, VCPU_SREG_CS);
1440         set_segment(vcpu, &sregs->ds, VCPU_SREG_DS);
1441         set_segment(vcpu, &sregs->es, VCPU_SREG_ES);
1442         set_segment(vcpu, &sregs->fs, VCPU_SREG_FS);
1443         set_segment(vcpu, &sregs->gs, VCPU_SREG_GS);
1444         set_segment(vcpu, &sregs->ss, VCPU_SREG_SS);
1445
1446         set_segment(vcpu, &sregs->tr, VCPU_SREG_TR);
1447         set_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR);
1448
1449         dt.limit = sregs->idt.limit;
1450         dt.base = sregs->idt.base;
1451         kvm_arch_ops->set_idt(vcpu, &dt);
1452         dt.limit = sregs->gdt.limit;
1453         dt.base = sregs->gdt.base;
1454         kvm_arch_ops->set_gdt(vcpu, &dt);
1455
1456         vcpu->cr2 = sregs->cr2;
1457         mmu_reset_needed |= vcpu->cr3 != sregs->cr3;
1458         vcpu->cr3 = sregs->cr3;
1459
1460         vcpu->cr8 = sregs->cr8;
1461
1462         mmu_reset_needed |= vcpu->shadow_efer != sregs->efer;
1463 #ifdef CONFIG_X86_64
1464         kvm_arch_ops->set_efer(vcpu, sregs->efer);
1465 #endif
1466         vcpu->apic_base = sregs->apic_base;
1467
1468         mmu_reset_needed |= vcpu->cr0 != sregs->cr0;
1469         kvm_arch_ops->set_cr0_no_modeswitch(vcpu, sregs->cr0);
1470
1471         mmu_reset_needed |= vcpu->cr4 != sregs->cr4;
1472         kvm_arch_ops->set_cr4(vcpu, sregs->cr4);
1473
1474         if (mmu_reset_needed)
1475                 kvm_mmu_reset_context(vcpu);
1476
1477         memcpy(vcpu->irq_pending, sregs->interrupt_bitmap,
1478                sizeof vcpu->irq_pending);
1479         vcpu->irq_summary = 0;
1480         for (i = 0; i < NR_IRQ_WORDS; ++i)
1481                 if (vcpu->irq_pending[i])
1482                         __set_bit(i, &vcpu->irq_summary);
1483
1484         vcpu_put(vcpu);
1485
1486         return 0;
1487 }
1488
1489 /*
1490  * List of msr numbers which we expose to userspace through KVM_GET_MSRS
1491  * and KVM_SET_MSRS, and KVM_GET_MSR_INDEX_LIST.
1492  *
1493  * This list is modified at module load time to reflect the
1494  * capabilities of the host cpu.
1495  */
1496 static u32 msrs_to_save[] = {
1497         MSR_IA32_SYSENTER_CS, MSR_IA32_SYSENTER_ESP, MSR_IA32_SYSENTER_EIP,
1498         MSR_K6_STAR,
1499 #ifdef CONFIG_X86_64
1500         MSR_CSTAR, MSR_KERNEL_GS_BASE, MSR_SYSCALL_MASK, MSR_LSTAR,
1501 #endif
1502         MSR_IA32_TIME_STAMP_COUNTER,
1503 };
1504
1505 static unsigned num_msrs_to_save;
1506
1507 static __init void kvm_init_msr_list(void)
1508 {
1509         u32 dummy[2];
1510         unsigned i, j;
1511
1512         for (i = j = 0; i < ARRAY_SIZE(msrs_to_save); i++) {
1513                 if (rdmsr_safe(msrs_to_save[i], &dummy[0], &dummy[1]) < 0)
1514                         continue;
1515                 if (j < i)
1516                         msrs_to_save[j] = msrs_to_save[i];
1517                 j++;
1518         }
1519         num_msrs_to_save = j;
1520 }
1521
1522 /*
1523  * Adapt set_msr() to msr_io()'s calling convention
1524  */
1525 static int do_set_msr(struct kvm_vcpu *vcpu, unsigned index, u64 *data)
1526 {
1527         return set_msr(vcpu, index, *data);
1528 }
1529
1530 /*
1531  * Read or write a bunch of msrs. All parameters are kernel addresses.
1532  *
1533  * @return number of msrs set successfully.
1534  */
1535 static int __msr_io(struct kvm *kvm, struct kvm_msrs *msrs,
1536                     struct kvm_msr_entry *entries,
1537                     int (*do_msr)(struct kvm_vcpu *vcpu,
1538                                   unsigned index, u64 *data))
1539 {
1540         struct kvm_vcpu *vcpu;
1541         int i;
1542
1543         if (!valid_vcpu(msrs->vcpu))
1544                 return -EINVAL;
1545
1546         vcpu = vcpu_load(kvm, msrs->vcpu);
1547         if (!vcpu)
1548                 return -ENOENT;
1549
1550         for (i = 0; i < msrs->nmsrs; ++i)
1551                 if (do_msr(vcpu, entries[i].index, &entries[i].data))
1552                         break;
1553
1554         vcpu_put(vcpu);
1555
1556         return i;
1557 }
1558
1559 /*
1560  * Read or write a bunch of msrs. Parameters are user addresses.
1561  *
1562  * @return number of msrs set successfully.
1563  */
1564 static int msr_io(struct kvm *kvm, struct kvm_msrs __user *user_msrs,
1565                   int (*do_msr)(struct kvm_vcpu *vcpu,
1566                                 unsigned index, u64 *data),
1567                   int writeback)
1568 {
1569         struct kvm_msrs msrs;
1570         struct kvm_msr_entry *entries;
1571         int r, n;
1572         unsigned size;
1573
1574         r = -EFAULT;
1575         if (copy_from_user(&msrs, user_msrs, sizeof msrs))
1576                 goto out;
1577
1578         r = -E2BIG;
1579         if (msrs.nmsrs >= MAX_IO_MSRS)
1580                 goto out;
1581
1582         r = -ENOMEM;
1583         size = sizeof(struct kvm_msr_entry) * msrs.nmsrs;
1584         entries = vmalloc(size);
1585         if (!entries)
1586                 goto out;
1587
1588         r = -EFAULT;
1589         if (copy_from_user(entries, user_msrs->entries, size))
1590                 goto out_free;
1591
1592         r = n = __msr_io(kvm, &msrs, entries, do_msr);
1593         if (r < 0)
1594                 goto out_free;
1595
1596         r = -EFAULT;
1597         if (writeback && copy_to_user(user_msrs->entries, entries, size))
1598                 goto out_free;
1599
1600         r = n;
1601
1602 out_free:
1603         vfree(entries);
1604 out:
1605         return r;
1606 }
1607
1608 /*
1609  * Translate a guest virtual address to a guest physical address.
1610  */
1611 static int kvm_dev_ioctl_translate(struct kvm *kvm, struct kvm_translation *tr)
1612 {
1613         unsigned long vaddr = tr->linear_address;
1614         struct kvm_vcpu *vcpu;
1615         gpa_t gpa;
1616
1617         vcpu = vcpu_load(kvm, tr->vcpu);
1618         if (!vcpu)
1619                 return -ENOENT;
1620         spin_lock(&kvm->lock);
1621         gpa = vcpu->mmu.gva_to_gpa(vcpu, vaddr);
1622         tr->physical_address = gpa;
1623         tr->valid = gpa != UNMAPPED_GVA;
1624         tr->writeable = 1;
1625         tr->usermode = 0;
1626         spin_unlock(&kvm->lock);
1627         vcpu_put(vcpu);
1628
1629         return 0;
1630 }
1631
1632 static int kvm_dev_ioctl_interrupt(struct kvm *kvm, struct kvm_interrupt *irq)
1633 {
1634         struct kvm_vcpu *vcpu;
1635
1636         if (!valid_vcpu(irq->vcpu))
1637                 return -EINVAL;
1638         if (irq->irq < 0 || irq->irq >= 256)
1639                 return -EINVAL;
1640         vcpu = vcpu_load(kvm, irq->vcpu);
1641         if (!vcpu)
1642                 return -ENOENT;
1643
1644         set_bit(irq->irq, vcpu->irq_pending);
1645         set_bit(irq->irq / BITS_PER_LONG, &vcpu->irq_summary);
1646
1647         vcpu_put(vcpu);
1648
1649         return 0;
1650 }
1651
1652 static int kvm_dev_ioctl_debug_guest(struct kvm *kvm,
1653                                      struct kvm_debug_guest *dbg)
1654 {
1655         struct kvm_vcpu *vcpu;
1656         int r;
1657
1658         if (!valid_vcpu(dbg->vcpu))
1659                 return -EINVAL;
1660         vcpu = vcpu_load(kvm, dbg->vcpu);
1661         if (!vcpu)
1662                 return -ENOENT;
1663
1664         r = kvm_arch_ops->set_guest_debug(vcpu, dbg);
1665
1666         vcpu_put(vcpu);
1667
1668         return r;
1669 }
1670
1671 static long kvm_dev_ioctl(struct file *filp,
1672                           unsigned int ioctl, unsigned long arg)
1673 {
1674         struct kvm *kvm = filp->private_data;
1675         int r = -EINVAL;
1676
1677         switch (ioctl) {
1678         case KVM_GET_API_VERSION:
1679                 r = KVM_API_VERSION;
1680                 break;
1681         case KVM_CREATE_VCPU: {
1682                 r = kvm_dev_ioctl_create_vcpu(kvm, arg);
1683                 if (r)
1684                         goto out;
1685                 break;
1686         }
1687         case KVM_RUN: {
1688                 struct kvm_run kvm_run;
1689
1690                 r = -EFAULT;
1691                 if (copy_from_user(&kvm_run, (void *)arg, sizeof kvm_run))
1692                         goto out;
1693                 r = kvm_dev_ioctl_run(kvm, &kvm_run);
1694                 if (r < 0)
1695                         goto out;
1696                 r = -EFAULT;
1697                 if (copy_to_user((void *)arg, &kvm_run, sizeof kvm_run))
1698                         goto out;
1699                 r = 0;
1700                 break;
1701         }
1702         case KVM_GET_REGS: {
1703                 struct kvm_regs kvm_regs;
1704
1705                 r = -EFAULT;
1706                 if (copy_from_user(&kvm_regs, (void *)arg, sizeof kvm_regs))
1707                         goto out;
1708                 r = kvm_dev_ioctl_get_regs(kvm, &kvm_regs);
1709                 if (r)
1710                         goto out;
1711                 r = -EFAULT;
1712                 if (copy_to_user((void *)arg, &kvm_regs, sizeof kvm_regs))
1713                         goto out;
1714                 r = 0;
1715                 break;
1716         }
1717         case KVM_SET_REGS: {
1718                 struct kvm_regs kvm_regs;
1719
1720                 r = -EFAULT;
1721                 if (copy_from_user(&kvm_regs, (void *)arg, sizeof kvm_regs))
1722                         goto out;
1723                 r = kvm_dev_ioctl_set_regs(kvm, &kvm_regs);
1724                 if (r)
1725                         goto out;
1726                 r = 0;
1727                 break;
1728         }
1729         case KVM_GET_SREGS: {
1730                 struct kvm_sregs kvm_sregs;
1731
1732                 r = -EFAULT;
1733                 if (copy_from_user(&kvm_sregs, (void *)arg, sizeof kvm_sregs))
1734                         goto out;
1735                 r = kvm_dev_ioctl_get_sregs(kvm, &kvm_sregs);
1736                 if (r)
1737                         goto out;
1738                 r = -EFAULT;
1739                 if (copy_to_user((void *)arg, &kvm_sregs, sizeof kvm_sregs))
1740                         goto out;
1741                 r = 0;
1742                 break;
1743         }
1744         case KVM_SET_SREGS: {
1745                 struct kvm_sregs kvm_sregs;
1746
1747                 r = -EFAULT;
1748                 if (copy_from_user(&kvm_sregs, (void *)arg, sizeof kvm_sregs))
1749                         goto out;
1750                 r = kvm_dev_ioctl_set_sregs(kvm, &kvm_sregs);
1751                 if (r)
1752                         goto out;
1753                 r = 0;
1754                 break;
1755         }
1756         case KVM_TRANSLATE: {
1757                 struct kvm_translation tr;
1758
1759                 r = -EFAULT;
1760                 if (copy_from_user(&tr, (void *)arg, sizeof tr))
1761                         goto out;
1762                 r = kvm_dev_ioctl_translate(kvm, &tr);
1763                 if (r)
1764                         goto out;
1765                 r = -EFAULT;
1766                 if (copy_to_user((void *)arg, &tr, sizeof tr))
1767                         goto out;
1768                 r = 0;
1769                 break;
1770         }
1771         case KVM_INTERRUPT: {
1772                 struct kvm_interrupt irq;
1773
1774                 r = -EFAULT;
1775                 if (copy_from_user(&irq, (void *)arg, sizeof irq))
1776                         goto out;
1777                 r = kvm_dev_ioctl_interrupt(kvm, &irq);
1778                 if (r)
1779                         goto out;
1780                 r = 0;
1781                 break;
1782         }
1783         case KVM_DEBUG_GUEST: {
1784                 struct kvm_debug_guest dbg;
1785
1786                 r = -EFAULT;
1787                 if (copy_from_user(&dbg, (void *)arg, sizeof dbg))
1788                         goto out;
1789                 r = kvm_dev_ioctl_debug_guest(kvm, &dbg);
1790                 if (r)
1791                         goto out;
1792                 r = 0;
1793                 break;
1794         }
1795         case KVM_SET_MEMORY_REGION: {
1796                 struct kvm_memory_region kvm_mem;
1797
1798                 r = -EFAULT;
1799                 if (copy_from_user(&kvm_mem, (void *)arg, sizeof kvm_mem))
1800                         goto out;
1801                 r = kvm_dev_ioctl_set_memory_region(kvm, &kvm_mem);
1802                 if (r)
1803                         goto out;
1804                 break;
1805         }
1806         case KVM_GET_DIRTY_LOG: {
1807                 struct kvm_dirty_log log;
1808
1809                 r = -EFAULT;
1810                 if (copy_from_user(&log, (void *)arg, sizeof log))
1811                         goto out;
1812                 r = kvm_dev_ioctl_get_dirty_log(kvm, &log);
1813                 if (r)
1814                         goto out;
1815                 break;
1816         }
1817         case KVM_GET_MSRS:
1818                 r = msr_io(kvm, (void __user *)arg, get_msr, 1);
1819                 break;
1820         case KVM_SET_MSRS:
1821                 r = msr_io(kvm, (void __user *)arg, do_set_msr, 0);
1822                 break;
1823         case KVM_GET_MSR_INDEX_LIST: {
1824                 struct kvm_msr_list __user *user_msr_list = (void __user *)arg;
1825                 struct kvm_msr_list msr_list;
1826                 unsigned n;
1827
1828                 r = -EFAULT;
1829                 if (copy_from_user(&msr_list, user_msr_list, sizeof msr_list))
1830                         goto out;
1831                 n = msr_list.nmsrs;
1832                 msr_list.nmsrs = num_msrs_to_save;
1833                 if (copy_to_user(user_msr_list, &msr_list, sizeof msr_list))
1834                         goto out;
1835                 r = -E2BIG;
1836                 if (n < num_msrs_to_save)
1837                         goto out;
1838                 r = -EFAULT;
1839                 if (copy_to_user(user_msr_list->indices, &msrs_to_save,
1840                                  num_msrs_to_save * sizeof(u32)))
1841                         goto out;
1842                 r = 0;
1843         }
1844         default:
1845                 ;
1846         }
1847 out:
1848         return r;
1849 }
1850
1851 static struct page *kvm_dev_nopage(struct vm_area_struct *vma,
1852                                    unsigned long address,
1853                                    int *type)
1854 {
1855         struct kvm *kvm = vma->vm_file->private_data;
1856         unsigned long pgoff;
1857         struct kvm_memory_slot *slot;
1858         struct page *page;
1859
1860         *type = VM_FAULT_MINOR;
1861         pgoff = ((address - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff;
1862         slot = gfn_to_memslot(kvm, pgoff);
1863         if (!slot)
1864                 return NOPAGE_SIGBUS;
1865         page = gfn_to_page(slot, pgoff);
1866         if (!page)
1867                 return NOPAGE_SIGBUS;
1868         get_page(page);
1869         return page;
1870 }
1871
1872 static struct vm_operations_struct kvm_dev_vm_ops = {
1873         .nopage = kvm_dev_nopage,
1874 };
1875
1876 static int kvm_dev_mmap(struct file *file, struct vm_area_struct *vma)
1877 {
1878         vma->vm_ops = &kvm_dev_vm_ops;
1879         return 0;
1880 }
1881
1882 static struct file_operations kvm_chardev_ops = {
1883         .open           = kvm_dev_open,
1884         .release        = kvm_dev_release,
1885         .unlocked_ioctl = kvm_dev_ioctl,
1886         .compat_ioctl   = kvm_dev_ioctl,
1887         .mmap           = kvm_dev_mmap,
1888 };
1889
1890 static struct miscdevice kvm_dev = {
1891         MISC_DYNAMIC_MINOR,
1892         "kvm",
1893         &kvm_chardev_ops,
1894 };
1895
1896 static int kvm_reboot(struct notifier_block *notifier, unsigned long val,
1897                        void *v)
1898 {
1899         if (val == SYS_RESTART) {
1900                 /*
1901                  * Some (well, at least mine) BIOSes hang on reboot if
1902                  * in vmx root mode.
1903                  */
1904                 printk(KERN_INFO "kvm: exiting hardware virtualization\n");
1905                 on_each_cpu(kvm_arch_ops->hardware_disable, 0, 0, 1);
1906         }
1907         return NOTIFY_OK;
1908 }
1909
1910 static struct notifier_block kvm_reboot_notifier = {
1911         .notifier_call = kvm_reboot,
1912         .priority = 0,
1913 };
1914
1915 static __init void kvm_init_debug(void)
1916 {
1917         struct kvm_stats_debugfs_item *p;
1918
1919         debugfs_dir = debugfs_create_dir("kvm", 0);
1920         for (p = debugfs_entries; p->name; ++p)
1921                 p->dentry = debugfs_create_u32(p->name, 0444, debugfs_dir,
1922                                                p->data);
1923 }
1924
1925 static void kvm_exit_debug(void)
1926 {
1927         struct kvm_stats_debugfs_item *p;
1928
1929         for (p = debugfs_entries; p->name; ++p)
1930                 debugfs_remove(p->dentry);
1931         debugfs_remove(debugfs_dir);
1932 }
1933
1934 hpa_t bad_page_address;
1935
1936 int kvm_init_arch(struct kvm_arch_ops *ops, struct module *module)
1937 {
1938         int r;
1939
1940         if (kvm_arch_ops) {
1941                 printk(KERN_ERR "kvm: already loaded the other module\n");
1942                 return -EEXIST;
1943         }
1944
1945         kvm_arch_ops = ops;
1946
1947         if (!kvm_arch_ops->cpu_has_kvm_support()) {
1948                 printk(KERN_ERR "kvm: no hardware support\n");
1949                 return -EOPNOTSUPP;
1950         }
1951         if (kvm_arch_ops->disabled_by_bios()) {
1952                 printk(KERN_ERR "kvm: disabled by bios\n");
1953                 return -EOPNOTSUPP;
1954         }
1955
1956         r = kvm_arch_ops->hardware_setup();
1957         if (r < 0)
1958             return r;
1959
1960         on_each_cpu(kvm_arch_ops->hardware_enable, 0, 0, 1);
1961         register_reboot_notifier(&kvm_reboot_notifier);
1962
1963         kvm_chardev_ops.owner = module;
1964
1965         r = misc_register(&kvm_dev);
1966         if (r) {
1967                 printk (KERN_ERR "kvm: misc device register failed\n");
1968                 goto out_free;
1969         }
1970
1971         return r;
1972
1973 out_free:
1974         unregister_reboot_notifier(&kvm_reboot_notifier);
1975         on_each_cpu(kvm_arch_ops->hardware_disable, 0, 0, 1);
1976         kvm_arch_ops->hardware_unsetup();
1977         return r;
1978 }
1979
1980 void kvm_exit_arch(void)
1981 {
1982         misc_deregister(&kvm_dev);
1983
1984         unregister_reboot_notifier(&kvm_reboot_notifier);
1985         on_each_cpu(kvm_arch_ops->hardware_disable, 0, 0, 1);
1986         kvm_arch_ops->hardware_unsetup();
1987         kvm_arch_ops = NULL;
1988 }
1989
1990 static __init int kvm_init(void)
1991 {
1992         static struct page *bad_page;
1993         int r = 0;
1994
1995         kvm_init_debug();
1996
1997         kvm_init_msr_list();
1998
1999         if ((bad_page = alloc_page(GFP_KERNEL)) == NULL) {
2000                 r = -ENOMEM;
2001                 goto out;
2002         }
2003
2004         bad_page_address = page_to_pfn(bad_page) << PAGE_SHIFT;
2005         memset(__va(bad_page_address), 0, PAGE_SIZE);
2006
2007         return r;
2008
2009 out:
2010         kvm_exit_debug();
2011         return r;
2012 }
2013
2014 static __exit void kvm_exit(void)
2015 {
2016         kvm_exit_debug();
2017         __free_page(pfn_to_page(bad_page_address >> PAGE_SHIFT));
2018 }
2019
2020 module_init(kvm_init)
2021 module_exit(kvm_exit)
2022
2023 EXPORT_SYMBOL_GPL(kvm_init_arch);
2024 EXPORT_SYMBOL_GPL(kvm_exit_arch);