[PATCH] kvm: fix GFP_KERNEL allocation in atomic section in kvm_dev_ioctl_create_vcpu()
[linux-2.6] / drivers / kvm / kvm_main.c
1 /*
2  * Kernel-based Virtual Machine driver for Linux
3  *
4  * This module enables machines with Intel VT-x extensions to run virtual
5  * machines without emulation or binary translation.
6  *
7  * Copyright (C) 2006 Qumranet, Inc.
8  *
9  * Authors:
10  *   Avi Kivity   <avi@qumranet.com>
11  *   Yaniv Kamay  <yaniv@qumranet.com>
12  *
13  * This work is licensed under the terms of the GNU GPL, version 2.  See
14  * the COPYING file in the top-level directory.
15  *
16  */
17
18 #include "kvm.h"
19
20 #include <linux/kvm.h>
21 #include <linux/module.h>
22 #include <linux/errno.h>
23 #include <asm/processor.h>
24 #include <linux/percpu.h>
25 #include <linux/gfp.h>
26 #include <asm/msr.h>
27 #include <linux/mm.h>
28 #include <linux/miscdevice.h>
29 #include <linux/vmalloc.h>
30 #include <asm/uaccess.h>
31 #include <linux/reboot.h>
32 #include <asm/io.h>
33 #include <linux/debugfs.h>
34 #include <linux/highmem.h>
35 #include <linux/file.h>
36 #include <asm/desc.h>
37
38 #include "x86_emulate.h"
39 #include "segment_descriptor.h"
40
41 MODULE_AUTHOR("Qumranet");
42 MODULE_LICENSE("GPL");
43
44 struct kvm_arch_ops *kvm_arch_ops;
45 struct kvm_stat kvm_stat;
46 EXPORT_SYMBOL_GPL(kvm_stat);
47
48 static struct kvm_stats_debugfs_item {
49         const char *name;
50         u32 *data;
51         struct dentry *dentry;
52 } debugfs_entries[] = {
53         { "pf_fixed", &kvm_stat.pf_fixed },
54         { "pf_guest", &kvm_stat.pf_guest },
55         { "tlb_flush", &kvm_stat.tlb_flush },
56         { "invlpg", &kvm_stat.invlpg },
57         { "exits", &kvm_stat.exits },
58         { "io_exits", &kvm_stat.io_exits },
59         { "mmio_exits", &kvm_stat.mmio_exits },
60         { "signal_exits", &kvm_stat.signal_exits },
61         { "irq_exits", &kvm_stat.irq_exits },
62         { 0, 0 }
63 };
64
65 static struct dentry *debugfs_dir;
66
67 #define MAX_IO_MSRS 256
68
69 #define CR0_RESEVED_BITS 0xffffffff1ffaffc0ULL
70 #define LMSW_GUEST_MASK 0x0eULL
71 #define CR4_RESEVED_BITS (~((1ULL << 11) - 1))
72 #define CR8_RESEVED_BITS (~0x0fULL)
73 #define EFER_RESERVED_BITS 0xfffffffffffff2fe
74
75 #ifdef CONFIG_X86_64
76 // LDT or TSS descriptor in the GDT. 16 bytes.
77 struct segment_descriptor_64 {
78         struct segment_descriptor s;
79         u32 base_higher;
80         u32 pad_zero;
81 };
82
83 #endif
84
85 unsigned long segment_base(u16 selector)
86 {
87         struct descriptor_table gdt;
88         struct segment_descriptor *d;
89         unsigned long table_base;
90         typedef unsigned long ul;
91         unsigned long v;
92
93         if (selector == 0)
94                 return 0;
95
96         asm ("sgdt %0" : "=m"(gdt));
97         table_base = gdt.base;
98
99         if (selector & 4) {           /* from ldt */
100                 u16 ldt_selector;
101
102                 asm ("sldt %0" : "=g"(ldt_selector));
103                 table_base = segment_base(ldt_selector);
104         }
105         d = (struct segment_descriptor *)(table_base + (selector & ~7));
106         v = d->base_low | ((ul)d->base_mid << 16) | ((ul)d->base_high << 24);
107 #ifdef CONFIG_X86_64
108         if (d->system == 0
109             && (d->type == 2 || d->type == 9 || d->type == 11))
110                 v |= ((ul)((struct segment_descriptor_64 *)d)->base_higher) << 32;
111 #endif
112         return v;
113 }
114 EXPORT_SYMBOL_GPL(segment_base);
115
116 static inline int valid_vcpu(int n)
117 {
118         return likely(n >= 0 && n < KVM_MAX_VCPUS);
119 }
120
121 int kvm_read_guest(struct kvm_vcpu *vcpu,
122                              gva_t addr,
123                              unsigned long size,
124                              void *dest)
125 {
126         unsigned char *host_buf = dest;
127         unsigned long req_size = size;
128
129         while (size) {
130                 hpa_t paddr;
131                 unsigned now;
132                 unsigned offset;
133                 hva_t guest_buf;
134
135                 paddr = gva_to_hpa(vcpu, addr);
136
137                 if (is_error_hpa(paddr))
138                         break;
139
140                 guest_buf = (hva_t)kmap_atomic(
141                                         pfn_to_page(paddr >> PAGE_SHIFT),
142                                         KM_USER0);
143                 offset = addr & ~PAGE_MASK;
144                 guest_buf |= offset;
145                 now = min(size, PAGE_SIZE - offset);
146                 memcpy(host_buf, (void*)guest_buf, now);
147                 host_buf += now;
148                 addr += now;
149                 size -= now;
150                 kunmap_atomic((void *)(guest_buf & PAGE_MASK), KM_USER0);
151         }
152         return req_size - size;
153 }
154 EXPORT_SYMBOL_GPL(kvm_read_guest);
155
156 int kvm_write_guest(struct kvm_vcpu *vcpu,
157                              gva_t addr,
158                              unsigned long size,
159                              void *data)
160 {
161         unsigned char *host_buf = data;
162         unsigned long req_size = size;
163
164         while (size) {
165                 hpa_t paddr;
166                 unsigned now;
167                 unsigned offset;
168                 hva_t guest_buf;
169
170                 paddr = gva_to_hpa(vcpu, addr);
171
172                 if (is_error_hpa(paddr))
173                         break;
174
175                 guest_buf = (hva_t)kmap_atomic(
176                                 pfn_to_page(paddr >> PAGE_SHIFT), KM_USER0);
177                 offset = addr & ~PAGE_MASK;
178                 guest_buf |= offset;
179                 now = min(size, PAGE_SIZE - offset);
180                 memcpy((void*)guest_buf, host_buf, now);
181                 host_buf += now;
182                 addr += now;
183                 size -= now;
184                 kunmap_atomic((void *)(guest_buf & PAGE_MASK), KM_USER0);
185         }
186         return req_size - size;
187 }
188 EXPORT_SYMBOL_GPL(kvm_write_guest);
189
190 static int vcpu_slot(struct kvm_vcpu *vcpu)
191 {
192         return vcpu - vcpu->kvm->vcpus;
193 }
194
195 /*
196  * Switches to specified vcpu, until a matching vcpu_put()
197  */
198 static struct kvm_vcpu *vcpu_load(struct kvm *kvm, int vcpu_slot)
199 {
200         struct kvm_vcpu *vcpu = &kvm->vcpus[vcpu_slot];
201
202         mutex_lock(&vcpu->mutex);
203         if (unlikely(!vcpu->vmcs)) {
204                 mutex_unlock(&vcpu->mutex);
205                 return 0;
206         }
207         return kvm_arch_ops->vcpu_load(vcpu);
208 }
209
210 static void vcpu_put(struct kvm_vcpu *vcpu)
211 {
212         kvm_arch_ops->vcpu_put(vcpu);
213         mutex_unlock(&vcpu->mutex);
214 }
215
216 static int kvm_dev_open(struct inode *inode, struct file *filp)
217 {
218         struct kvm *kvm = kzalloc(sizeof(struct kvm), GFP_KERNEL);
219         int i;
220
221         if (!kvm)
222                 return -ENOMEM;
223
224         spin_lock_init(&kvm->lock);
225         INIT_LIST_HEAD(&kvm->active_mmu_pages);
226         for (i = 0; i < KVM_MAX_VCPUS; ++i) {
227                 struct kvm_vcpu *vcpu = &kvm->vcpus[i];
228
229                 mutex_init(&vcpu->mutex);
230                 vcpu->mmu.root_hpa = INVALID_PAGE;
231                 INIT_LIST_HEAD(&vcpu->free_pages);
232         }
233         filp->private_data = kvm;
234         return 0;
235 }
236
237 /*
238  * Free any memory in @free but not in @dont.
239  */
240 static void kvm_free_physmem_slot(struct kvm_memory_slot *free,
241                                   struct kvm_memory_slot *dont)
242 {
243         int i;
244
245         if (!dont || free->phys_mem != dont->phys_mem)
246                 if (free->phys_mem) {
247                         for (i = 0; i < free->npages; ++i)
248                                 if (free->phys_mem[i])
249                                         __free_page(free->phys_mem[i]);
250                         vfree(free->phys_mem);
251                 }
252
253         if (!dont || free->dirty_bitmap != dont->dirty_bitmap)
254                 vfree(free->dirty_bitmap);
255
256         free->phys_mem = 0;
257         free->npages = 0;
258         free->dirty_bitmap = 0;
259 }
260
261 static void kvm_free_physmem(struct kvm *kvm)
262 {
263         int i;
264
265         for (i = 0; i < kvm->nmemslots; ++i)
266                 kvm_free_physmem_slot(&kvm->memslots[i], 0);
267 }
268
269 static void kvm_free_vcpu(struct kvm_vcpu *vcpu)
270 {
271         kvm_arch_ops->vcpu_free(vcpu);
272         kvm_mmu_destroy(vcpu);
273 }
274
275 static void kvm_free_vcpus(struct kvm *kvm)
276 {
277         unsigned int i;
278
279         for (i = 0; i < KVM_MAX_VCPUS; ++i)
280                 kvm_free_vcpu(&kvm->vcpus[i]);
281 }
282
283 static int kvm_dev_release(struct inode *inode, struct file *filp)
284 {
285         struct kvm *kvm = filp->private_data;
286
287         kvm_free_vcpus(kvm);
288         kvm_free_physmem(kvm);
289         kfree(kvm);
290         return 0;
291 }
292
293 static void inject_gp(struct kvm_vcpu *vcpu)
294 {
295         kvm_arch_ops->inject_gp(vcpu, 0);
296 }
297
298 static int pdptrs_have_reserved_bits_set(struct kvm_vcpu *vcpu,
299                                          unsigned long cr3)
300 {
301         gfn_t pdpt_gfn = cr3 >> PAGE_SHIFT;
302         unsigned offset = (cr3 & (PAGE_SIZE-1)) >> 5;
303         int i;
304         u64 pdpte;
305         u64 *pdpt;
306         struct kvm_memory_slot *memslot;
307
308         spin_lock(&vcpu->kvm->lock);
309         memslot = gfn_to_memslot(vcpu->kvm, pdpt_gfn);
310         /* FIXME: !memslot - emulate? 0xff? */
311         pdpt = kmap_atomic(gfn_to_page(memslot, pdpt_gfn), KM_USER0);
312
313         for (i = 0; i < 4; ++i) {
314                 pdpte = pdpt[offset + i];
315                 if ((pdpte & 1) && (pdpte & 0xfffffff0000001e6ull))
316                         break;
317         }
318
319         kunmap_atomic(pdpt, KM_USER0);
320         spin_unlock(&vcpu->kvm->lock);
321
322         return i != 4;
323 }
324
325 void set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
326 {
327         if (cr0 & CR0_RESEVED_BITS) {
328                 printk(KERN_DEBUG "set_cr0: 0x%lx #GP, reserved bits 0x%lx\n",
329                        cr0, vcpu->cr0);
330                 inject_gp(vcpu);
331                 return;
332         }
333
334         if ((cr0 & CR0_NW_MASK) && !(cr0 & CR0_CD_MASK)) {
335                 printk(KERN_DEBUG "set_cr0: #GP, CD == 0 && NW == 1\n");
336                 inject_gp(vcpu);
337                 return;
338         }
339
340         if ((cr0 & CR0_PG_MASK) && !(cr0 & CR0_PE_MASK)) {
341                 printk(KERN_DEBUG "set_cr0: #GP, set PG flag "
342                        "and a clear PE flag\n");
343                 inject_gp(vcpu);
344                 return;
345         }
346
347         if (!is_paging(vcpu) && (cr0 & CR0_PG_MASK)) {
348 #ifdef CONFIG_X86_64
349                 if ((vcpu->shadow_efer & EFER_LME)) {
350                         int cs_db, cs_l;
351
352                         if (!is_pae(vcpu)) {
353                                 printk(KERN_DEBUG "set_cr0: #GP, start paging "
354                                        "in long mode while PAE is disabled\n");
355                                 inject_gp(vcpu);
356                                 return;
357                         }
358                         kvm_arch_ops->get_cs_db_l_bits(vcpu, &cs_db, &cs_l);
359                         if (cs_l) {
360                                 printk(KERN_DEBUG "set_cr0: #GP, start paging "
361                                        "in long mode while CS.L == 1\n");
362                                 inject_gp(vcpu);
363                                 return;
364
365                         }
366                 } else
367 #endif
368                 if (is_pae(vcpu) &&
369                             pdptrs_have_reserved_bits_set(vcpu, vcpu->cr3)) {
370                         printk(KERN_DEBUG "set_cr0: #GP, pdptrs "
371                                "reserved bits\n");
372                         inject_gp(vcpu);
373                         return;
374                 }
375
376         }
377
378         kvm_arch_ops->set_cr0(vcpu, cr0);
379         vcpu->cr0 = cr0;
380
381         spin_lock(&vcpu->kvm->lock);
382         kvm_mmu_reset_context(vcpu);
383         spin_unlock(&vcpu->kvm->lock);
384         return;
385 }
386 EXPORT_SYMBOL_GPL(set_cr0);
387
388 void lmsw(struct kvm_vcpu *vcpu, unsigned long msw)
389 {
390         set_cr0(vcpu, (vcpu->cr0 & ~0x0ful) | (msw & 0x0f));
391 }
392 EXPORT_SYMBOL_GPL(lmsw);
393
394 void set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
395 {
396         if (cr4 & CR4_RESEVED_BITS) {
397                 printk(KERN_DEBUG "set_cr4: #GP, reserved bits\n");
398                 inject_gp(vcpu);
399                 return;
400         }
401
402         if (is_long_mode(vcpu)) {
403                 if (!(cr4 & CR4_PAE_MASK)) {
404                         printk(KERN_DEBUG "set_cr4: #GP, clearing PAE while "
405                                "in long mode\n");
406                         inject_gp(vcpu);
407                         return;
408                 }
409         } else if (is_paging(vcpu) && !is_pae(vcpu) && (cr4 & CR4_PAE_MASK)
410                    && pdptrs_have_reserved_bits_set(vcpu, vcpu->cr3)) {
411                 printk(KERN_DEBUG "set_cr4: #GP, pdptrs reserved bits\n");
412                 inject_gp(vcpu);
413         }
414
415         if (cr4 & CR4_VMXE_MASK) {
416                 printk(KERN_DEBUG "set_cr4: #GP, setting VMXE\n");
417                 inject_gp(vcpu);
418                 return;
419         }
420         kvm_arch_ops->set_cr4(vcpu, cr4);
421         spin_lock(&vcpu->kvm->lock);
422         kvm_mmu_reset_context(vcpu);
423         spin_unlock(&vcpu->kvm->lock);
424 }
425 EXPORT_SYMBOL_GPL(set_cr4);
426
427 void set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3)
428 {
429         if (is_long_mode(vcpu)) {
430                 if ( cr3 & CR3_L_MODE_RESEVED_BITS) {
431                         printk(KERN_DEBUG "set_cr3: #GP, reserved bits\n");
432                         inject_gp(vcpu);
433                         return;
434                 }
435         } else {
436                 if (cr3 & CR3_RESEVED_BITS) {
437                         printk(KERN_DEBUG "set_cr3: #GP, reserved bits\n");
438                         inject_gp(vcpu);
439                         return;
440                 }
441                 if (is_paging(vcpu) && is_pae(vcpu) &&
442                     pdptrs_have_reserved_bits_set(vcpu, cr3)) {
443                         printk(KERN_DEBUG "set_cr3: #GP, pdptrs "
444                                "reserved bits\n");
445                         inject_gp(vcpu);
446                         return;
447                 }
448         }
449
450         vcpu->cr3 = cr3;
451         spin_lock(&vcpu->kvm->lock);
452         vcpu->mmu.new_cr3(vcpu);
453         spin_unlock(&vcpu->kvm->lock);
454 }
455 EXPORT_SYMBOL_GPL(set_cr3);
456
457 void set_cr8(struct kvm_vcpu *vcpu, unsigned long cr8)
458 {
459         if ( cr8 & CR8_RESEVED_BITS) {
460                 printk(KERN_DEBUG "set_cr8: #GP, reserved bits 0x%lx\n", cr8);
461                 inject_gp(vcpu);
462                 return;
463         }
464         vcpu->cr8 = cr8;
465 }
466 EXPORT_SYMBOL_GPL(set_cr8);
467
468 void fx_init(struct kvm_vcpu *vcpu)
469 {
470         struct __attribute__ ((__packed__)) fx_image_s {
471                 u16 control; //fcw
472                 u16 status; //fsw
473                 u16 tag; // ftw
474                 u16 opcode; //fop
475                 u64 ip; // fpu ip
476                 u64 operand;// fpu dp
477                 u32 mxcsr;
478                 u32 mxcsr_mask;
479
480         } *fx_image;
481
482         fx_save(vcpu->host_fx_image);
483         fpu_init();
484         fx_save(vcpu->guest_fx_image);
485         fx_restore(vcpu->host_fx_image);
486
487         fx_image = (struct fx_image_s *)vcpu->guest_fx_image;
488         fx_image->mxcsr = 0x1f80;
489         memset(vcpu->guest_fx_image + sizeof(struct fx_image_s),
490                0, FX_IMAGE_SIZE - sizeof(struct fx_image_s));
491 }
492 EXPORT_SYMBOL_GPL(fx_init);
493
494 /*
495  * Creates some virtual cpus.  Good luck creating more than one.
496  */
497 static int kvm_dev_ioctl_create_vcpu(struct kvm *kvm, int n)
498 {
499         int r;
500         struct kvm_vcpu *vcpu;
501
502         r = -EINVAL;
503         if (!valid_vcpu(n))
504                 goto out;
505
506         vcpu = &kvm->vcpus[n];
507
508         mutex_lock(&vcpu->mutex);
509
510         if (vcpu->vmcs) {
511                 mutex_unlock(&vcpu->mutex);
512                 return -EEXIST;
513         }
514
515         vcpu->host_fx_image = (char*)ALIGN((hva_t)vcpu->fx_buf,
516                                            FX_IMAGE_ALIGN);
517         vcpu->guest_fx_image = vcpu->host_fx_image + FX_IMAGE_SIZE;
518
519         vcpu->cpu = -1;  /* First load will set up TR */
520         vcpu->kvm = kvm;
521         r = kvm_arch_ops->vcpu_create(vcpu);
522         if (r < 0)
523                 goto out_free_vcpus;
524
525         r = kvm_mmu_create(vcpu);
526         if (r < 0)
527                 goto out_free_vcpus;
528
529         kvm_arch_ops->vcpu_load(vcpu);
530         r = kvm_mmu_setup(vcpu);
531         if (r >= 0)
532                 r = kvm_arch_ops->vcpu_setup(vcpu);
533         vcpu_put(vcpu);
534
535         if (r < 0)
536                 goto out_free_vcpus;
537
538         return 0;
539
540 out_free_vcpus:
541         kvm_free_vcpu(vcpu);
542         mutex_unlock(&vcpu->mutex);
543 out:
544         return r;
545 }
546
547 /*
548  * Allocate some memory and give it an address in the guest physical address
549  * space.
550  *
551  * Discontiguous memory is allowed, mostly for framebuffers.
552  */
553 static int kvm_dev_ioctl_set_memory_region(struct kvm *kvm,
554                                            struct kvm_memory_region *mem)
555 {
556         int r;
557         gfn_t base_gfn;
558         unsigned long npages;
559         unsigned long i;
560         struct kvm_memory_slot *memslot;
561         struct kvm_memory_slot old, new;
562         int memory_config_version;
563
564         r = -EINVAL;
565         /* General sanity checks */
566         if (mem->memory_size & (PAGE_SIZE - 1))
567                 goto out;
568         if (mem->guest_phys_addr & (PAGE_SIZE - 1))
569                 goto out;
570         if (mem->slot >= KVM_MEMORY_SLOTS)
571                 goto out;
572         if (mem->guest_phys_addr + mem->memory_size < mem->guest_phys_addr)
573                 goto out;
574
575         memslot = &kvm->memslots[mem->slot];
576         base_gfn = mem->guest_phys_addr >> PAGE_SHIFT;
577         npages = mem->memory_size >> PAGE_SHIFT;
578
579         if (!npages)
580                 mem->flags &= ~KVM_MEM_LOG_DIRTY_PAGES;
581
582 raced:
583         spin_lock(&kvm->lock);
584
585         memory_config_version = kvm->memory_config_version;
586         new = old = *memslot;
587
588         new.base_gfn = base_gfn;
589         new.npages = npages;
590         new.flags = mem->flags;
591
592         /* Disallow changing a memory slot's size. */
593         r = -EINVAL;
594         if (npages && old.npages && npages != old.npages)
595                 goto out_unlock;
596
597         /* Check for overlaps */
598         r = -EEXIST;
599         for (i = 0; i < KVM_MEMORY_SLOTS; ++i) {
600                 struct kvm_memory_slot *s = &kvm->memslots[i];
601
602                 if (s == memslot)
603                         continue;
604                 if (!((base_gfn + npages <= s->base_gfn) ||
605                       (base_gfn >= s->base_gfn + s->npages)))
606                         goto out_unlock;
607         }
608         /*
609          * Do memory allocations outside lock.  memory_config_version will
610          * detect any races.
611          */
612         spin_unlock(&kvm->lock);
613
614         /* Deallocate if slot is being removed */
615         if (!npages)
616                 new.phys_mem = 0;
617
618         /* Free page dirty bitmap if unneeded */
619         if (!(new.flags & KVM_MEM_LOG_DIRTY_PAGES))
620                 new.dirty_bitmap = 0;
621
622         r = -ENOMEM;
623
624         /* Allocate if a slot is being created */
625         if (npages && !new.phys_mem) {
626                 new.phys_mem = vmalloc(npages * sizeof(struct page *));
627
628                 if (!new.phys_mem)
629                         goto out_free;
630
631                 memset(new.phys_mem, 0, npages * sizeof(struct page *));
632                 for (i = 0; i < npages; ++i) {
633                         new.phys_mem[i] = alloc_page(GFP_HIGHUSER
634                                                      | __GFP_ZERO);
635                         if (!new.phys_mem[i])
636                                 goto out_free;
637                 }
638         }
639
640         /* Allocate page dirty bitmap if needed */
641         if ((new.flags & KVM_MEM_LOG_DIRTY_PAGES) && !new.dirty_bitmap) {
642                 unsigned dirty_bytes = ALIGN(npages, BITS_PER_LONG) / 8;
643
644                 new.dirty_bitmap = vmalloc(dirty_bytes);
645                 if (!new.dirty_bitmap)
646                         goto out_free;
647                 memset(new.dirty_bitmap, 0, dirty_bytes);
648         }
649
650         spin_lock(&kvm->lock);
651
652         if (memory_config_version != kvm->memory_config_version) {
653                 spin_unlock(&kvm->lock);
654                 kvm_free_physmem_slot(&new, &old);
655                 goto raced;
656         }
657
658         r = -EAGAIN;
659         if (kvm->busy)
660                 goto out_unlock;
661
662         if (mem->slot >= kvm->nmemslots)
663                 kvm->nmemslots = mem->slot + 1;
664
665         *memslot = new;
666         ++kvm->memory_config_version;
667
668         spin_unlock(&kvm->lock);
669
670         for (i = 0; i < KVM_MAX_VCPUS; ++i) {
671                 struct kvm_vcpu *vcpu;
672
673                 vcpu = vcpu_load(kvm, i);
674                 if (!vcpu)
675                         continue;
676                 kvm_mmu_reset_context(vcpu);
677                 vcpu_put(vcpu);
678         }
679
680         kvm_free_physmem_slot(&old, &new);
681         return 0;
682
683 out_unlock:
684         spin_unlock(&kvm->lock);
685 out_free:
686         kvm_free_physmem_slot(&new, &old);
687 out:
688         return r;
689 }
690
691 /*
692  * Get (and clear) the dirty memory log for a memory slot.
693  */
694 static int kvm_dev_ioctl_get_dirty_log(struct kvm *kvm,
695                                        struct kvm_dirty_log *log)
696 {
697         struct kvm_memory_slot *memslot;
698         int r, i;
699         int n;
700         unsigned long any = 0;
701
702         spin_lock(&kvm->lock);
703
704         /*
705          * Prevent changes to guest memory configuration even while the lock
706          * is not taken.
707          */
708         ++kvm->busy;
709         spin_unlock(&kvm->lock);
710         r = -EINVAL;
711         if (log->slot >= KVM_MEMORY_SLOTS)
712                 goto out;
713
714         memslot = &kvm->memslots[log->slot];
715         r = -ENOENT;
716         if (!memslot->dirty_bitmap)
717                 goto out;
718
719         n = ALIGN(memslot->npages, 8) / 8;
720
721         for (i = 0; !any && i < n; ++i)
722                 any = memslot->dirty_bitmap[i];
723
724         r = -EFAULT;
725         if (copy_to_user(log->dirty_bitmap, memslot->dirty_bitmap, n))
726                 goto out;
727
728
729         if (any) {
730                 spin_lock(&kvm->lock);
731                 kvm_mmu_slot_remove_write_access(kvm, log->slot);
732                 spin_unlock(&kvm->lock);
733                 memset(memslot->dirty_bitmap, 0, n);
734                 for (i = 0; i < KVM_MAX_VCPUS; ++i) {
735                         struct kvm_vcpu *vcpu = vcpu_load(kvm, i);
736
737                         if (!vcpu)
738                                 continue;
739                         kvm_arch_ops->tlb_flush(vcpu);
740                         vcpu_put(vcpu);
741                 }
742         }
743
744         r = 0;
745
746 out:
747         spin_lock(&kvm->lock);
748         --kvm->busy;
749         spin_unlock(&kvm->lock);
750         return r;
751 }
752
753 struct kvm_memory_slot *gfn_to_memslot(struct kvm *kvm, gfn_t gfn)
754 {
755         int i;
756
757         for (i = 0; i < kvm->nmemslots; ++i) {
758                 struct kvm_memory_slot *memslot = &kvm->memslots[i];
759
760                 if (gfn >= memslot->base_gfn
761                     && gfn < memslot->base_gfn + memslot->npages)
762                         return memslot;
763         }
764         return 0;
765 }
766 EXPORT_SYMBOL_GPL(gfn_to_memslot);
767
768 void mark_page_dirty(struct kvm *kvm, gfn_t gfn)
769 {
770         int i;
771         struct kvm_memory_slot *memslot = 0;
772         unsigned long rel_gfn;
773
774         for (i = 0; i < kvm->nmemslots; ++i) {
775                 memslot = &kvm->memslots[i];
776
777                 if (gfn >= memslot->base_gfn
778                     && gfn < memslot->base_gfn + memslot->npages) {
779
780                         if (!memslot || !memslot->dirty_bitmap)
781                                 return;
782
783                         rel_gfn = gfn - memslot->base_gfn;
784
785                         /* avoid RMW */
786                         if (!test_bit(rel_gfn, memslot->dirty_bitmap))
787                                 set_bit(rel_gfn, memslot->dirty_bitmap);
788                         return;
789                 }
790         }
791 }
792
793 static int emulator_read_std(unsigned long addr,
794                              unsigned long *val,
795                              unsigned int bytes,
796                              struct x86_emulate_ctxt *ctxt)
797 {
798         struct kvm_vcpu *vcpu = ctxt->vcpu;
799         void *data = val;
800
801         while (bytes) {
802                 gpa_t gpa = vcpu->mmu.gva_to_gpa(vcpu, addr);
803                 unsigned offset = addr & (PAGE_SIZE-1);
804                 unsigned tocopy = min(bytes, (unsigned)PAGE_SIZE - offset);
805                 unsigned long pfn;
806                 struct kvm_memory_slot *memslot;
807                 void *page;
808
809                 if (gpa == UNMAPPED_GVA)
810                         return X86EMUL_PROPAGATE_FAULT;
811                 pfn = gpa >> PAGE_SHIFT;
812                 memslot = gfn_to_memslot(vcpu->kvm, pfn);
813                 if (!memslot)
814                         return X86EMUL_UNHANDLEABLE;
815                 page = kmap_atomic(gfn_to_page(memslot, pfn), KM_USER0);
816
817                 memcpy(data, page + offset, tocopy);
818
819                 kunmap_atomic(page, KM_USER0);
820
821                 bytes -= tocopy;
822                 data += tocopy;
823                 addr += tocopy;
824         }
825
826         return X86EMUL_CONTINUE;
827 }
828
829 static int emulator_write_std(unsigned long addr,
830                               unsigned long val,
831                               unsigned int bytes,
832                               struct x86_emulate_ctxt *ctxt)
833 {
834         printk(KERN_ERR "emulator_write_std: addr %lx n %d\n",
835                addr, bytes);
836         return X86EMUL_UNHANDLEABLE;
837 }
838
839 static int emulator_read_emulated(unsigned long addr,
840                                   unsigned long *val,
841                                   unsigned int bytes,
842                                   struct x86_emulate_ctxt *ctxt)
843 {
844         struct kvm_vcpu *vcpu = ctxt->vcpu;
845
846         if (vcpu->mmio_read_completed) {
847                 memcpy(val, vcpu->mmio_data, bytes);
848                 vcpu->mmio_read_completed = 0;
849                 return X86EMUL_CONTINUE;
850         } else if (emulator_read_std(addr, val, bytes, ctxt)
851                    == X86EMUL_CONTINUE)
852                 return X86EMUL_CONTINUE;
853         else {
854                 gpa_t gpa = vcpu->mmu.gva_to_gpa(vcpu, addr);
855                 if (gpa == UNMAPPED_GVA)
856                         return vcpu_printf(vcpu, "not present\n"), X86EMUL_PROPAGATE_FAULT;
857                 vcpu->mmio_needed = 1;
858                 vcpu->mmio_phys_addr = gpa;
859                 vcpu->mmio_size = bytes;
860                 vcpu->mmio_is_write = 0;
861
862                 return X86EMUL_UNHANDLEABLE;
863         }
864 }
865
866 static int emulator_write_emulated(unsigned long addr,
867                                    unsigned long val,
868                                    unsigned int bytes,
869                                    struct x86_emulate_ctxt *ctxt)
870 {
871         struct kvm_vcpu *vcpu = ctxt->vcpu;
872         gpa_t gpa = vcpu->mmu.gva_to_gpa(vcpu, addr);
873
874         if (gpa == UNMAPPED_GVA)
875                 return X86EMUL_PROPAGATE_FAULT;
876
877         vcpu->mmio_needed = 1;
878         vcpu->mmio_phys_addr = gpa;
879         vcpu->mmio_size = bytes;
880         vcpu->mmio_is_write = 1;
881         memcpy(vcpu->mmio_data, &val, bytes);
882
883         return X86EMUL_CONTINUE;
884 }
885
886 static int emulator_cmpxchg_emulated(unsigned long addr,
887                                      unsigned long old,
888                                      unsigned long new,
889                                      unsigned int bytes,
890                                      struct x86_emulate_ctxt *ctxt)
891 {
892         static int reported;
893
894         if (!reported) {
895                 reported = 1;
896                 printk(KERN_WARNING "kvm: emulating exchange as write\n");
897         }
898         return emulator_write_emulated(addr, new, bytes, ctxt);
899 }
900
901 static unsigned long get_segment_base(struct kvm_vcpu *vcpu, int seg)
902 {
903         return kvm_arch_ops->get_segment_base(vcpu, seg);
904 }
905
906 int emulate_invlpg(struct kvm_vcpu *vcpu, gva_t address)
907 {
908         spin_lock(&vcpu->kvm->lock);
909         vcpu->mmu.inval_page(vcpu, address);
910         spin_unlock(&vcpu->kvm->lock);
911         kvm_arch_ops->invlpg(vcpu, address);
912         return X86EMUL_CONTINUE;
913 }
914
915 int emulate_clts(struct kvm_vcpu *vcpu)
916 {
917         unsigned long cr0 = vcpu->cr0;
918
919         cr0 &= ~CR0_TS_MASK;
920         kvm_arch_ops->set_cr0(vcpu, cr0);
921         return X86EMUL_CONTINUE;
922 }
923
924 int emulator_get_dr(struct x86_emulate_ctxt* ctxt, int dr, unsigned long *dest)
925 {
926         struct kvm_vcpu *vcpu = ctxt->vcpu;
927
928         switch (dr) {
929         case 0 ... 3:
930                 *dest = kvm_arch_ops->get_dr(vcpu, dr);
931                 return X86EMUL_CONTINUE;
932         default:
933                 printk(KERN_DEBUG "%s: unexpected dr %u\n",
934                        __FUNCTION__, dr);
935                 return X86EMUL_UNHANDLEABLE;
936         }
937 }
938
939 int emulator_set_dr(struct x86_emulate_ctxt *ctxt, int dr, unsigned long value)
940 {
941         unsigned long mask = (ctxt->mode == X86EMUL_MODE_PROT64) ? ~0ULL : ~0U;
942         int exception;
943
944         kvm_arch_ops->set_dr(ctxt->vcpu, dr, value & mask, &exception);
945         if (exception) {
946                 /* FIXME: better handling */
947                 return X86EMUL_UNHANDLEABLE;
948         }
949         return X86EMUL_CONTINUE;
950 }
951
952 static void report_emulation_failure(struct x86_emulate_ctxt *ctxt)
953 {
954         static int reported;
955         u8 opcodes[4];
956         unsigned long rip = ctxt->vcpu->rip;
957         unsigned long rip_linear;
958
959         rip_linear = rip + get_segment_base(ctxt->vcpu, VCPU_SREG_CS);
960
961         if (reported)
962                 return;
963
964         emulator_read_std(rip_linear, (void *)opcodes, 4, ctxt);
965
966         printk(KERN_ERR "emulation failed but !mmio_needed?"
967                " rip %lx %02x %02x %02x %02x\n",
968                rip, opcodes[0], opcodes[1], opcodes[2], opcodes[3]);
969         reported = 1;
970 }
971
972 struct x86_emulate_ops emulate_ops = {
973         .read_std            = emulator_read_std,
974         .write_std           = emulator_write_std,
975         .read_emulated       = emulator_read_emulated,
976         .write_emulated      = emulator_write_emulated,
977         .cmpxchg_emulated    = emulator_cmpxchg_emulated,
978 };
979
980 int emulate_instruction(struct kvm_vcpu *vcpu,
981                         struct kvm_run *run,
982                         unsigned long cr2,
983                         u16 error_code)
984 {
985         struct x86_emulate_ctxt emulate_ctxt;
986         int r;
987         int cs_db, cs_l;
988
989         kvm_arch_ops->cache_regs(vcpu);
990
991         kvm_arch_ops->get_cs_db_l_bits(vcpu, &cs_db, &cs_l);
992
993         emulate_ctxt.vcpu = vcpu;
994         emulate_ctxt.eflags = kvm_arch_ops->get_rflags(vcpu);
995         emulate_ctxt.cr2 = cr2;
996         emulate_ctxt.mode = (emulate_ctxt.eflags & X86_EFLAGS_VM)
997                 ? X86EMUL_MODE_REAL : cs_l
998                 ? X86EMUL_MODE_PROT64 : cs_db
999                 ? X86EMUL_MODE_PROT32 : X86EMUL_MODE_PROT16;
1000
1001         if (emulate_ctxt.mode == X86EMUL_MODE_PROT64) {
1002                 emulate_ctxt.cs_base = 0;
1003                 emulate_ctxt.ds_base = 0;
1004                 emulate_ctxt.es_base = 0;
1005                 emulate_ctxt.ss_base = 0;
1006         } else {
1007                 emulate_ctxt.cs_base = get_segment_base(vcpu, VCPU_SREG_CS);
1008                 emulate_ctxt.ds_base = get_segment_base(vcpu, VCPU_SREG_DS);
1009                 emulate_ctxt.es_base = get_segment_base(vcpu, VCPU_SREG_ES);
1010                 emulate_ctxt.ss_base = get_segment_base(vcpu, VCPU_SREG_SS);
1011         }
1012
1013         emulate_ctxt.gs_base = get_segment_base(vcpu, VCPU_SREG_GS);
1014         emulate_ctxt.fs_base = get_segment_base(vcpu, VCPU_SREG_FS);
1015
1016         vcpu->mmio_is_write = 0;
1017         r = x86_emulate_memop(&emulate_ctxt, &emulate_ops);
1018
1019         if ((r || vcpu->mmio_is_write) && run) {
1020                 run->mmio.phys_addr = vcpu->mmio_phys_addr;
1021                 memcpy(run->mmio.data, vcpu->mmio_data, 8);
1022                 run->mmio.len = vcpu->mmio_size;
1023                 run->mmio.is_write = vcpu->mmio_is_write;
1024         }
1025
1026         if (r) {
1027                 if (!vcpu->mmio_needed) {
1028                         report_emulation_failure(&emulate_ctxt);
1029                         return EMULATE_FAIL;
1030                 }
1031                 return EMULATE_DO_MMIO;
1032         }
1033
1034         kvm_arch_ops->decache_regs(vcpu);
1035         kvm_arch_ops->set_rflags(vcpu, emulate_ctxt.eflags);
1036
1037         if (vcpu->mmio_is_write)
1038                 return EMULATE_DO_MMIO;
1039
1040         return EMULATE_DONE;
1041 }
1042 EXPORT_SYMBOL_GPL(emulate_instruction);
1043
1044 static u64 mk_cr_64(u64 curr_cr, u32 new_val)
1045 {
1046         return (curr_cr & ~((1ULL << 32) - 1)) | new_val;
1047 }
1048
1049 void realmode_lgdt(struct kvm_vcpu *vcpu, u16 limit, unsigned long base)
1050 {
1051         struct descriptor_table dt = { limit, base };
1052
1053         kvm_arch_ops->set_gdt(vcpu, &dt);
1054 }
1055
1056 void realmode_lidt(struct kvm_vcpu *vcpu, u16 limit, unsigned long base)
1057 {
1058         struct descriptor_table dt = { limit, base };
1059
1060         kvm_arch_ops->set_idt(vcpu, &dt);
1061 }
1062
1063 void realmode_lmsw(struct kvm_vcpu *vcpu, unsigned long msw,
1064                    unsigned long *rflags)
1065 {
1066         lmsw(vcpu, msw);
1067         *rflags = kvm_arch_ops->get_rflags(vcpu);
1068 }
1069
1070 unsigned long realmode_get_cr(struct kvm_vcpu *vcpu, int cr)
1071 {
1072         switch (cr) {
1073         case 0:
1074                 return vcpu->cr0;
1075         case 2:
1076                 return vcpu->cr2;
1077         case 3:
1078                 return vcpu->cr3;
1079         case 4:
1080                 return vcpu->cr4;
1081         default:
1082                 vcpu_printf(vcpu, "%s: unexpected cr %u\n", __FUNCTION__, cr);
1083                 return 0;
1084         }
1085 }
1086
1087 void realmode_set_cr(struct kvm_vcpu *vcpu, int cr, unsigned long val,
1088                      unsigned long *rflags)
1089 {
1090         switch (cr) {
1091         case 0:
1092                 set_cr0(vcpu, mk_cr_64(vcpu->cr0, val));
1093                 *rflags = kvm_arch_ops->get_rflags(vcpu);
1094                 break;
1095         case 2:
1096                 vcpu->cr2 = val;
1097                 break;
1098         case 3:
1099                 set_cr3(vcpu, val);
1100                 break;
1101         case 4:
1102                 set_cr4(vcpu, mk_cr_64(vcpu->cr4, val));
1103                 break;
1104         default:
1105                 vcpu_printf(vcpu, "%s: unexpected cr %u\n", __FUNCTION__, cr);
1106         }
1107 }
1108
1109 int kvm_get_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata)
1110 {
1111         u64 data;
1112
1113         switch (msr) {
1114         case 0xc0010010: /* SYSCFG */
1115         case 0xc0010015: /* HWCR */
1116         case MSR_IA32_PLATFORM_ID:
1117         case MSR_IA32_P5_MC_ADDR:
1118         case MSR_IA32_P5_MC_TYPE:
1119         case MSR_IA32_MC0_CTL:
1120         case MSR_IA32_MCG_STATUS:
1121         case MSR_IA32_MCG_CAP:
1122         case MSR_IA32_MC0_MISC:
1123         case MSR_IA32_MC0_MISC+4:
1124         case MSR_IA32_MC0_MISC+8:
1125         case MSR_IA32_MC0_MISC+12:
1126         case MSR_IA32_MC0_MISC+16:
1127         case MSR_IA32_UCODE_REV:
1128         case MSR_IA32_PERF_STATUS:
1129                 /* MTRR registers */
1130         case 0xfe:
1131         case 0x200 ... 0x2ff:
1132                 data = 0;
1133                 break;
1134         case 0xcd: /* fsb frequency */
1135                 data = 3;
1136                 break;
1137         case MSR_IA32_APICBASE:
1138                 data = vcpu->apic_base;
1139                 break;
1140 #ifdef CONFIG_X86_64
1141         case MSR_EFER:
1142                 data = vcpu->shadow_efer;
1143                 break;
1144 #endif
1145         default:
1146                 printk(KERN_ERR "kvm: unhandled rdmsr: 0x%x\n", msr);
1147                 return 1;
1148         }
1149         *pdata = data;
1150         return 0;
1151 }
1152 EXPORT_SYMBOL_GPL(kvm_get_msr_common);
1153
1154 /*
1155  * Reads an msr value (of 'msr_index') into 'pdata'.
1156  * Returns 0 on success, non-0 otherwise.
1157  * Assumes vcpu_load() was already called.
1158  */
1159 static int get_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 *pdata)
1160 {
1161         return kvm_arch_ops->get_msr(vcpu, msr_index, pdata);
1162 }
1163
1164 #ifdef CONFIG_X86_64
1165
1166 static void set_efer(struct kvm_vcpu *vcpu, u64 efer)
1167 {
1168         if (efer & EFER_RESERVED_BITS) {
1169                 printk(KERN_DEBUG "set_efer: 0x%llx #GP, reserved bits\n",
1170                        efer);
1171                 inject_gp(vcpu);
1172                 return;
1173         }
1174
1175         if (is_paging(vcpu)
1176             && (vcpu->shadow_efer & EFER_LME) != (efer & EFER_LME)) {
1177                 printk(KERN_DEBUG "set_efer: #GP, change LME while paging\n");
1178                 inject_gp(vcpu);
1179                 return;
1180         }
1181
1182         kvm_arch_ops->set_efer(vcpu, efer);
1183
1184         efer &= ~EFER_LMA;
1185         efer |= vcpu->shadow_efer & EFER_LMA;
1186
1187         vcpu->shadow_efer = efer;
1188 }
1189
1190 #endif
1191
1192 int kvm_set_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 data)
1193 {
1194         switch (msr) {
1195 #ifdef CONFIG_X86_64
1196         case MSR_EFER:
1197                 set_efer(vcpu, data);
1198                 break;
1199 #endif
1200         case MSR_IA32_MC0_STATUS:
1201                 printk(KERN_WARNING "%s: MSR_IA32_MC0_STATUS 0x%llx, nop\n",
1202                        __FUNCTION__, data);
1203                 break;
1204         case MSR_IA32_UCODE_REV:
1205         case MSR_IA32_UCODE_WRITE:
1206         case 0x200 ... 0x2ff: /* MTRRs */
1207                 break;
1208         case MSR_IA32_APICBASE:
1209                 vcpu->apic_base = data;
1210                 break;
1211         default:
1212                 printk(KERN_ERR "kvm: unhandled wrmsr: 0x%x\n", msr);
1213                 return 1;
1214         }
1215         return 0;
1216 }
1217 EXPORT_SYMBOL_GPL(kvm_set_msr_common);
1218
1219 /*
1220  * Writes msr value into into the appropriate "register".
1221  * Returns 0 on success, non-0 otherwise.
1222  * Assumes vcpu_load() was already called.
1223  */
1224 static int set_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 data)
1225 {
1226         return kvm_arch_ops->set_msr(vcpu, msr_index, data);
1227 }
1228
1229 void kvm_resched(struct kvm_vcpu *vcpu)
1230 {
1231         vcpu_put(vcpu);
1232         cond_resched();
1233         /* Cannot fail -  no vcpu unplug yet. */
1234         vcpu_load(vcpu->kvm, vcpu_slot(vcpu));
1235 }
1236 EXPORT_SYMBOL_GPL(kvm_resched);
1237
1238 void load_msrs(struct vmx_msr_entry *e, int n)
1239 {
1240         int i;
1241
1242         for (i = 0; i < n; ++i)
1243                 wrmsrl(e[i].index, e[i].data);
1244 }
1245 EXPORT_SYMBOL_GPL(load_msrs);
1246
1247 void save_msrs(struct vmx_msr_entry *e, int n)
1248 {
1249         int i;
1250
1251         for (i = 0; i < n; ++i)
1252                 rdmsrl(e[i].index, e[i].data);
1253 }
1254 EXPORT_SYMBOL_GPL(save_msrs);
1255
1256 static int kvm_dev_ioctl_run(struct kvm *kvm, struct kvm_run *kvm_run)
1257 {
1258         struct kvm_vcpu *vcpu;
1259         int r;
1260
1261         if (!valid_vcpu(kvm_run->vcpu))
1262                 return -EINVAL;
1263
1264         vcpu = vcpu_load(kvm, kvm_run->vcpu);
1265         if (!vcpu)
1266                 return -ENOENT;
1267
1268         if (kvm_run->emulated) {
1269                 kvm_arch_ops->skip_emulated_instruction(vcpu);
1270                 kvm_run->emulated = 0;
1271         }
1272
1273         if (kvm_run->mmio_completed) {
1274                 memcpy(vcpu->mmio_data, kvm_run->mmio.data, 8);
1275                 vcpu->mmio_read_completed = 1;
1276         }
1277
1278         vcpu->mmio_needed = 0;
1279
1280         r = kvm_arch_ops->run(vcpu, kvm_run);
1281
1282         vcpu_put(vcpu);
1283         return r;
1284 }
1285
1286 static int kvm_dev_ioctl_get_regs(struct kvm *kvm, struct kvm_regs *regs)
1287 {
1288         struct kvm_vcpu *vcpu;
1289
1290         if (!valid_vcpu(regs->vcpu))
1291                 return -EINVAL;
1292
1293         vcpu = vcpu_load(kvm, regs->vcpu);
1294         if (!vcpu)
1295                 return -ENOENT;
1296
1297         kvm_arch_ops->cache_regs(vcpu);
1298
1299         regs->rax = vcpu->regs[VCPU_REGS_RAX];
1300         regs->rbx = vcpu->regs[VCPU_REGS_RBX];
1301         regs->rcx = vcpu->regs[VCPU_REGS_RCX];
1302         regs->rdx = vcpu->regs[VCPU_REGS_RDX];
1303         regs->rsi = vcpu->regs[VCPU_REGS_RSI];
1304         regs->rdi = vcpu->regs[VCPU_REGS_RDI];
1305         regs->rsp = vcpu->regs[VCPU_REGS_RSP];
1306         regs->rbp = vcpu->regs[VCPU_REGS_RBP];
1307 #ifdef CONFIG_X86_64
1308         regs->r8 = vcpu->regs[VCPU_REGS_R8];
1309         regs->r9 = vcpu->regs[VCPU_REGS_R9];
1310         regs->r10 = vcpu->regs[VCPU_REGS_R10];
1311         regs->r11 = vcpu->regs[VCPU_REGS_R11];
1312         regs->r12 = vcpu->regs[VCPU_REGS_R12];
1313         regs->r13 = vcpu->regs[VCPU_REGS_R13];
1314         regs->r14 = vcpu->regs[VCPU_REGS_R14];
1315         regs->r15 = vcpu->regs[VCPU_REGS_R15];
1316 #endif
1317
1318         regs->rip = vcpu->rip;
1319         regs->rflags = kvm_arch_ops->get_rflags(vcpu);
1320
1321         /*
1322          * Don't leak debug flags in case they were set for guest debugging
1323          */
1324         if (vcpu->guest_debug.enabled && vcpu->guest_debug.singlestep)
1325                 regs->rflags &= ~(X86_EFLAGS_TF | X86_EFLAGS_RF);
1326
1327         vcpu_put(vcpu);
1328
1329         return 0;
1330 }
1331
1332 static int kvm_dev_ioctl_set_regs(struct kvm *kvm, struct kvm_regs *regs)
1333 {
1334         struct kvm_vcpu *vcpu;
1335
1336         if (!valid_vcpu(regs->vcpu))
1337                 return -EINVAL;
1338
1339         vcpu = vcpu_load(kvm, regs->vcpu);
1340         if (!vcpu)
1341                 return -ENOENT;
1342
1343         vcpu->regs[VCPU_REGS_RAX] = regs->rax;
1344         vcpu->regs[VCPU_REGS_RBX] = regs->rbx;
1345         vcpu->regs[VCPU_REGS_RCX] = regs->rcx;
1346         vcpu->regs[VCPU_REGS_RDX] = regs->rdx;
1347         vcpu->regs[VCPU_REGS_RSI] = regs->rsi;
1348         vcpu->regs[VCPU_REGS_RDI] = regs->rdi;
1349         vcpu->regs[VCPU_REGS_RSP] = regs->rsp;
1350         vcpu->regs[VCPU_REGS_RBP] = regs->rbp;
1351 #ifdef CONFIG_X86_64
1352         vcpu->regs[VCPU_REGS_R8] = regs->r8;
1353         vcpu->regs[VCPU_REGS_R9] = regs->r9;
1354         vcpu->regs[VCPU_REGS_R10] = regs->r10;
1355         vcpu->regs[VCPU_REGS_R11] = regs->r11;
1356         vcpu->regs[VCPU_REGS_R12] = regs->r12;
1357         vcpu->regs[VCPU_REGS_R13] = regs->r13;
1358         vcpu->regs[VCPU_REGS_R14] = regs->r14;
1359         vcpu->regs[VCPU_REGS_R15] = regs->r15;
1360 #endif
1361
1362         vcpu->rip = regs->rip;
1363         kvm_arch_ops->set_rflags(vcpu, regs->rflags);
1364
1365         kvm_arch_ops->decache_regs(vcpu);
1366
1367         vcpu_put(vcpu);
1368
1369         return 0;
1370 }
1371
1372 static void get_segment(struct kvm_vcpu *vcpu,
1373                         struct kvm_segment *var, int seg)
1374 {
1375         return kvm_arch_ops->get_segment(vcpu, var, seg);
1376 }
1377
1378 static int kvm_dev_ioctl_get_sregs(struct kvm *kvm, struct kvm_sregs *sregs)
1379 {
1380         struct kvm_vcpu *vcpu;
1381         struct descriptor_table dt;
1382
1383         if (!valid_vcpu(sregs->vcpu))
1384                 return -EINVAL;
1385         vcpu = vcpu_load(kvm, sregs->vcpu);
1386         if (!vcpu)
1387                 return -ENOENT;
1388
1389         get_segment(vcpu, &sregs->cs, VCPU_SREG_CS);
1390         get_segment(vcpu, &sregs->ds, VCPU_SREG_DS);
1391         get_segment(vcpu, &sregs->es, VCPU_SREG_ES);
1392         get_segment(vcpu, &sregs->fs, VCPU_SREG_FS);
1393         get_segment(vcpu, &sregs->gs, VCPU_SREG_GS);
1394         get_segment(vcpu, &sregs->ss, VCPU_SREG_SS);
1395
1396         get_segment(vcpu, &sregs->tr, VCPU_SREG_TR);
1397         get_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR);
1398
1399         kvm_arch_ops->get_idt(vcpu, &dt);
1400         sregs->idt.limit = dt.limit;
1401         sregs->idt.base = dt.base;
1402         kvm_arch_ops->get_gdt(vcpu, &dt);
1403         sregs->gdt.limit = dt.limit;
1404         sregs->gdt.base = dt.base;
1405
1406         sregs->cr0 = vcpu->cr0;
1407         sregs->cr2 = vcpu->cr2;
1408         sregs->cr3 = vcpu->cr3;
1409         sregs->cr4 = vcpu->cr4;
1410         sregs->cr8 = vcpu->cr8;
1411         sregs->efer = vcpu->shadow_efer;
1412         sregs->apic_base = vcpu->apic_base;
1413
1414         memcpy(sregs->interrupt_bitmap, vcpu->irq_pending,
1415                sizeof sregs->interrupt_bitmap);
1416
1417         vcpu_put(vcpu);
1418
1419         return 0;
1420 }
1421
1422 static void set_segment(struct kvm_vcpu *vcpu,
1423                         struct kvm_segment *var, int seg)
1424 {
1425         return kvm_arch_ops->set_segment(vcpu, var, seg);
1426 }
1427
1428 static int kvm_dev_ioctl_set_sregs(struct kvm *kvm, struct kvm_sregs *sregs)
1429 {
1430         struct kvm_vcpu *vcpu;
1431         int mmu_reset_needed = 0;
1432         int i;
1433         struct descriptor_table dt;
1434
1435         if (!valid_vcpu(sregs->vcpu))
1436                 return -EINVAL;
1437         vcpu = vcpu_load(kvm, sregs->vcpu);
1438         if (!vcpu)
1439                 return -ENOENT;
1440
1441         set_segment(vcpu, &sregs->cs, VCPU_SREG_CS);
1442         set_segment(vcpu, &sregs->ds, VCPU_SREG_DS);
1443         set_segment(vcpu, &sregs->es, VCPU_SREG_ES);
1444         set_segment(vcpu, &sregs->fs, VCPU_SREG_FS);
1445         set_segment(vcpu, &sregs->gs, VCPU_SREG_GS);
1446         set_segment(vcpu, &sregs->ss, VCPU_SREG_SS);
1447
1448         set_segment(vcpu, &sregs->tr, VCPU_SREG_TR);
1449         set_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR);
1450
1451         dt.limit = sregs->idt.limit;
1452         dt.base = sregs->idt.base;
1453         kvm_arch_ops->set_idt(vcpu, &dt);
1454         dt.limit = sregs->gdt.limit;
1455         dt.base = sregs->gdt.base;
1456         kvm_arch_ops->set_gdt(vcpu, &dt);
1457
1458         vcpu->cr2 = sregs->cr2;
1459         mmu_reset_needed |= vcpu->cr3 != sregs->cr3;
1460         vcpu->cr3 = sregs->cr3;
1461
1462         vcpu->cr8 = sregs->cr8;
1463
1464         mmu_reset_needed |= vcpu->shadow_efer != sregs->efer;
1465 #ifdef CONFIG_X86_64
1466         kvm_arch_ops->set_efer(vcpu, sregs->efer);
1467 #endif
1468         vcpu->apic_base = sregs->apic_base;
1469
1470         mmu_reset_needed |= vcpu->cr0 != sregs->cr0;
1471         kvm_arch_ops->set_cr0_no_modeswitch(vcpu, sregs->cr0);
1472
1473         mmu_reset_needed |= vcpu->cr4 != sregs->cr4;
1474         kvm_arch_ops->set_cr4(vcpu, sregs->cr4);
1475
1476         if (mmu_reset_needed)
1477                 kvm_mmu_reset_context(vcpu);
1478
1479         memcpy(vcpu->irq_pending, sregs->interrupt_bitmap,
1480                sizeof vcpu->irq_pending);
1481         vcpu->irq_summary = 0;
1482         for (i = 0; i < NR_IRQ_WORDS; ++i)
1483                 if (vcpu->irq_pending[i])
1484                         __set_bit(i, &vcpu->irq_summary);
1485
1486         vcpu_put(vcpu);
1487
1488         return 0;
1489 }
1490
1491 /*
1492  * List of msr numbers which we expose to userspace through KVM_GET_MSRS
1493  * and KVM_SET_MSRS, and KVM_GET_MSR_INDEX_LIST.
1494  *
1495  * This list is modified at module load time to reflect the
1496  * capabilities of the host cpu.
1497  */
1498 static u32 msrs_to_save[] = {
1499         MSR_IA32_SYSENTER_CS, MSR_IA32_SYSENTER_ESP, MSR_IA32_SYSENTER_EIP,
1500         MSR_K6_STAR,
1501 #ifdef CONFIG_X86_64
1502         MSR_CSTAR, MSR_KERNEL_GS_BASE, MSR_SYSCALL_MASK, MSR_LSTAR,
1503 #endif
1504         MSR_IA32_TIME_STAMP_COUNTER,
1505 };
1506
1507 static unsigned num_msrs_to_save;
1508
1509 static __init void kvm_init_msr_list(void)
1510 {
1511         u32 dummy[2];
1512         unsigned i, j;
1513
1514         for (i = j = 0; i < ARRAY_SIZE(msrs_to_save); i++) {
1515                 if (rdmsr_safe(msrs_to_save[i], &dummy[0], &dummy[1]) < 0)
1516                         continue;
1517                 if (j < i)
1518                         msrs_to_save[j] = msrs_to_save[i];
1519                 j++;
1520         }
1521         num_msrs_to_save = j;
1522 }
1523
1524 /*
1525  * Adapt set_msr() to msr_io()'s calling convention
1526  */
1527 static int do_set_msr(struct kvm_vcpu *vcpu, unsigned index, u64 *data)
1528 {
1529         return set_msr(vcpu, index, *data);
1530 }
1531
1532 /*
1533  * Read or write a bunch of msrs. All parameters are kernel addresses.
1534  *
1535  * @return number of msrs set successfully.
1536  */
1537 static int __msr_io(struct kvm *kvm, struct kvm_msrs *msrs,
1538                     struct kvm_msr_entry *entries,
1539                     int (*do_msr)(struct kvm_vcpu *vcpu,
1540                                   unsigned index, u64 *data))
1541 {
1542         struct kvm_vcpu *vcpu;
1543         int i;
1544
1545         if (!valid_vcpu(msrs->vcpu))
1546                 return -EINVAL;
1547
1548         vcpu = vcpu_load(kvm, msrs->vcpu);
1549         if (!vcpu)
1550                 return -ENOENT;
1551
1552         for (i = 0; i < msrs->nmsrs; ++i)
1553                 if (do_msr(vcpu, entries[i].index, &entries[i].data))
1554                         break;
1555
1556         vcpu_put(vcpu);
1557
1558         return i;
1559 }
1560
1561 /*
1562  * Read or write a bunch of msrs. Parameters are user addresses.
1563  *
1564  * @return number of msrs set successfully.
1565  */
1566 static int msr_io(struct kvm *kvm, struct kvm_msrs __user *user_msrs,
1567                   int (*do_msr)(struct kvm_vcpu *vcpu,
1568                                 unsigned index, u64 *data),
1569                   int writeback)
1570 {
1571         struct kvm_msrs msrs;
1572         struct kvm_msr_entry *entries;
1573         int r, n;
1574         unsigned size;
1575
1576         r = -EFAULT;
1577         if (copy_from_user(&msrs, user_msrs, sizeof msrs))
1578                 goto out;
1579
1580         r = -E2BIG;
1581         if (msrs.nmsrs >= MAX_IO_MSRS)
1582                 goto out;
1583
1584         r = -ENOMEM;
1585         size = sizeof(struct kvm_msr_entry) * msrs.nmsrs;
1586         entries = vmalloc(size);
1587         if (!entries)
1588                 goto out;
1589
1590         r = -EFAULT;
1591         if (copy_from_user(entries, user_msrs->entries, size))
1592                 goto out_free;
1593
1594         r = n = __msr_io(kvm, &msrs, entries, do_msr);
1595         if (r < 0)
1596                 goto out_free;
1597
1598         r = -EFAULT;
1599         if (writeback && copy_to_user(user_msrs->entries, entries, size))
1600                 goto out_free;
1601
1602         r = n;
1603
1604 out_free:
1605         vfree(entries);
1606 out:
1607         return r;
1608 }
1609
1610 /*
1611  * Translate a guest virtual address to a guest physical address.
1612  */
1613 static int kvm_dev_ioctl_translate(struct kvm *kvm, struct kvm_translation *tr)
1614 {
1615         unsigned long vaddr = tr->linear_address;
1616         struct kvm_vcpu *vcpu;
1617         gpa_t gpa;
1618
1619         vcpu = vcpu_load(kvm, tr->vcpu);
1620         if (!vcpu)
1621                 return -ENOENT;
1622         spin_lock(&kvm->lock);
1623         gpa = vcpu->mmu.gva_to_gpa(vcpu, vaddr);
1624         tr->physical_address = gpa;
1625         tr->valid = gpa != UNMAPPED_GVA;
1626         tr->writeable = 1;
1627         tr->usermode = 0;
1628         spin_unlock(&kvm->lock);
1629         vcpu_put(vcpu);
1630
1631         return 0;
1632 }
1633
1634 static int kvm_dev_ioctl_interrupt(struct kvm *kvm, struct kvm_interrupt *irq)
1635 {
1636         struct kvm_vcpu *vcpu;
1637
1638         if (!valid_vcpu(irq->vcpu))
1639                 return -EINVAL;
1640         if (irq->irq < 0 || irq->irq >= 256)
1641                 return -EINVAL;
1642         vcpu = vcpu_load(kvm, irq->vcpu);
1643         if (!vcpu)
1644                 return -ENOENT;
1645
1646         set_bit(irq->irq, vcpu->irq_pending);
1647         set_bit(irq->irq / BITS_PER_LONG, &vcpu->irq_summary);
1648
1649         vcpu_put(vcpu);
1650
1651         return 0;
1652 }
1653
1654 static int kvm_dev_ioctl_debug_guest(struct kvm *kvm,
1655                                      struct kvm_debug_guest *dbg)
1656 {
1657         struct kvm_vcpu *vcpu;
1658         int r;
1659
1660         if (!valid_vcpu(dbg->vcpu))
1661                 return -EINVAL;
1662         vcpu = vcpu_load(kvm, dbg->vcpu);
1663         if (!vcpu)
1664                 return -ENOENT;
1665
1666         r = kvm_arch_ops->set_guest_debug(vcpu, dbg);
1667
1668         vcpu_put(vcpu);
1669
1670         return r;
1671 }
1672
1673 static long kvm_dev_ioctl(struct file *filp,
1674                           unsigned int ioctl, unsigned long arg)
1675 {
1676         struct kvm *kvm = filp->private_data;
1677         int r = -EINVAL;
1678
1679         switch (ioctl) {
1680         case KVM_GET_API_VERSION:
1681                 r = KVM_API_VERSION;
1682                 break;
1683         case KVM_CREATE_VCPU: {
1684                 r = kvm_dev_ioctl_create_vcpu(kvm, arg);
1685                 if (r)
1686                         goto out;
1687                 break;
1688         }
1689         case KVM_RUN: {
1690                 struct kvm_run kvm_run;
1691
1692                 r = -EFAULT;
1693                 if (copy_from_user(&kvm_run, (void *)arg, sizeof kvm_run))
1694                         goto out;
1695                 r = kvm_dev_ioctl_run(kvm, &kvm_run);
1696                 if (r < 0)
1697                         goto out;
1698                 r = -EFAULT;
1699                 if (copy_to_user((void *)arg, &kvm_run, sizeof kvm_run))
1700                         goto out;
1701                 r = 0;
1702                 break;
1703         }
1704         case KVM_GET_REGS: {
1705                 struct kvm_regs kvm_regs;
1706
1707                 r = -EFAULT;
1708                 if (copy_from_user(&kvm_regs, (void *)arg, sizeof kvm_regs))
1709                         goto out;
1710                 r = kvm_dev_ioctl_get_regs(kvm, &kvm_regs);
1711                 if (r)
1712                         goto out;
1713                 r = -EFAULT;
1714                 if (copy_to_user((void *)arg, &kvm_regs, sizeof kvm_regs))
1715                         goto out;
1716                 r = 0;
1717                 break;
1718         }
1719         case KVM_SET_REGS: {
1720                 struct kvm_regs kvm_regs;
1721
1722                 r = -EFAULT;
1723                 if (copy_from_user(&kvm_regs, (void *)arg, sizeof kvm_regs))
1724                         goto out;
1725                 r = kvm_dev_ioctl_set_regs(kvm, &kvm_regs);
1726                 if (r)
1727                         goto out;
1728                 r = 0;
1729                 break;
1730         }
1731         case KVM_GET_SREGS: {
1732                 struct kvm_sregs kvm_sregs;
1733
1734                 r = -EFAULT;
1735                 if (copy_from_user(&kvm_sregs, (void *)arg, sizeof kvm_sregs))
1736                         goto out;
1737                 r = kvm_dev_ioctl_get_sregs(kvm, &kvm_sregs);
1738                 if (r)
1739                         goto out;
1740                 r = -EFAULT;
1741                 if (copy_to_user((void *)arg, &kvm_sregs, sizeof kvm_sregs))
1742                         goto out;
1743                 r = 0;
1744                 break;
1745         }
1746         case KVM_SET_SREGS: {
1747                 struct kvm_sregs kvm_sregs;
1748
1749                 r = -EFAULT;
1750                 if (copy_from_user(&kvm_sregs, (void *)arg, sizeof kvm_sregs))
1751                         goto out;
1752                 r = kvm_dev_ioctl_set_sregs(kvm, &kvm_sregs);
1753                 if (r)
1754                         goto out;
1755                 r = 0;
1756                 break;
1757         }
1758         case KVM_TRANSLATE: {
1759                 struct kvm_translation tr;
1760
1761                 r = -EFAULT;
1762                 if (copy_from_user(&tr, (void *)arg, sizeof tr))
1763                         goto out;
1764                 r = kvm_dev_ioctl_translate(kvm, &tr);
1765                 if (r)
1766                         goto out;
1767                 r = -EFAULT;
1768                 if (copy_to_user((void *)arg, &tr, sizeof tr))
1769                         goto out;
1770                 r = 0;
1771                 break;
1772         }
1773         case KVM_INTERRUPT: {
1774                 struct kvm_interrupt irq;
1775
1776                 r = -EFAULT;
1777                 if (copy_from_user(&irq, (void *)arg, sizeof irq))
1778                         goto out;
1779                 r = kvm_dev_ioctl_interrupt(kvm, &irq);
1780                 if (r)
1781                         goto out;
1782                 r = 0;
1783                 break;
1784         }
1785         case KVM_DEBUG_GUEST: {
1786                 struct kvm_debug_guest dbg;
1787
1788                 r = -EFAULT;
1789                 if (copy_from_user(&dbg, (void *)arg, sizeof dbg))
1790                         goto out;
1791                 r = kvm_dev_ioctl_debug_guest(kvm, &dbg);
1792                 if (r)
1793                         goto out;
1794                 r = 0;
1795                 break;
1796         }
1797         case KVM_SET_MEMORY_REGION: {
1798                 struct kvm_memory_region kvm_mem;
1799
1800                 r = -EFAULT;
1801                 if (copy_from_user(&kvm_mem, (void *)arg, sizeof kvm_mem))
1802                         goto out;
1803                 r = kvm_dev_ioctl_set_memory_region(kvm, &kvm_mem);
1804                 if (r)
1805                         goto out;
1806                 break;
1807         }
1808         case KVM_GET_DIRTY_LOG: {
1809                 struct kvm_dirty_log log;
1810
1811                 r = -EFAULT;
1812                 if (copy_from_user(&log, (void *)arg, sizeof log))
1813                         goto out;
1814                 r = kvm_dev_ioctl_get_dirty_log(kvm, &log);
1815                 if (r)
1816                         goto out;
1817                 break;
1818         }
1819         case KVM_GET_MSRS:
1820                 r = msr_io(kvm, (void __user *)arg, get_msr, 1);
1821                 break;
1822         case KVM_SET_MSRS:
1823                 r = msr_io(kvm, (void __user *)arg, do_set_msr, 0);
1824                 break;
1825         case KVM_GET_MSR_INDEX_LIST: {
1826                 struct kvm_msr_list __user *user_msr_list = (void __user *)arg;
1827                 struct kvm_msr_list msr_list;
1828                 unsigned n;
1829
1830                 r = -EFAULT;
1831                 if (copy_from_user(&msr_list, user_msr_list, sizeof msr_list))
1832                         goto out;
1833                 n = msr_list.nmsrs;
1834                 msr_list.nmsrs = num_msrs_to_save;
1835                 if (copy_to_user(user_msr_list, &msr_list, sizeof msr_list))
1836                         goto out;
1837                 r = -E2BIG;
1838                 if (n < num_msrs_to_save)
1839                         goto out;
1840                 r = -EFAULT;
1841                 if (copy_to_user(user_msr_list->indices, &msrs_to_save,
1842                                  num_msrs_to_save * sizeof(u32)))
1843                         goto out;
1844                 r = 0;
1845         }
1846         default:
1847                 ;
1848         }
1849 out:
1850         return r;
1851 }
1852
1853 static struct page *kvm_dev_nopage(struct vm_area_struct *vma,
1854                                    unsigned long address,
1855                                    int *type)
1856 {
1857         struct kvm *kvm = vma->vm_file->private_data;
1858         unsigned long pgoff;
1859         struct kvm_memory_slot *slot;
1860         struct page *page;
1861
1862         *type = VM_FAULT_MINOR;
1863         pgoff = ((address - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff;
1864         slot = gfn_to_memslot(kvm, pgoff);
1865         if (!slot)
1866                 return NOPAGE_SIGBUS;
1867         page = gfn_to_page(slot, pgoff);
1868         if (!page)
1869                 return NOPAGE_SIGBUS;
1870         get_page(page);
1871         return page;
1872 }
1873
1874 static struct vm_operations_struct kvm_dev_vm_ops = {
1875         .nopage = kvm_dev_nopage,
1876 };
1877
1878 static int kvm_dev_mmap(struct file *file, struct vm_area_struct *vma)
1879 {
1880         vma->vm_ops = &kvm_dev_vm_ops;
1881         return 0;
1882 }
1883
1884 static struct file_operations kvm_chardev_ops = {
1885         .open           = kvm_dev_open,
1886         .release        = kvm_dev_release,
1887         .unlocked_ioctl = kvm_dev_ioctl,
1888         .compat_ioctl   = kvm_dev_ioctl,
1889         .mmap           = kvm_dev_mmap,
1890 };
1891
1892 static struct miscdevice kvm_dev = {
1893         MISC_DYNAMIC_MINOR,
1894         "kvm",
1895         &kvm_chardev_ops,
1896 };
1897
1898 static int kvm_reboot(struct notifier_block *notifier, unsigned long val,
1899                        void *v)
1900 {
1901         if (val == SYS_RESTART) {
1902                 /*
1903                  * Some (well, at least mine) BIOSes hang on reboot if
1904                  * in vmx root mode.
1905                  */
1906                 printk(KERN_INFO "kvm: exiting hardware virtualization\n");
1907                 on_each_cpu(kvm_arch_ops->hardware_disable, 0, 0, 1);
1908         }
1909         return NOTIFY_OK;
1910 }
1911
1912 static struct notifier_block kvm_reboot_notifier = {
1913         .notifier_call = kvm_reboot,
1914         .priority = 0,
1915 };
1916
1917 static __init void kvm_init_debug(void)
1918 {
1919         struct kvm_stats_debugfs_item *p;
1920
1921         debugfs_dir = debugfs_create_dir("kvm", 0);
1922         for (p = debugfs_entries; p->name; ++p)
1923                 p->dentry = debugfs_create_u32(p->name, 0444, debugfs_dir,
1924                                                p->data);
1925 }
1926
1927 static void kvm_exit_debug(void)
1928 {
1929         struct kvm_stats_debugfs_item *p;
1930
1931         for (p = debugfs_entries; p->name; ++p)
1932                 debugfs_remove(p->dentry);
1933         debugfs_remove(debugfs_dir);
1934 }
1935
1936 hpa_t bad_page_address;
1937
1938 int kvm_init_arch(struct kvm_arch_ops *ops, struct module *module)
1939 {
1940         int r;
1941
1942         if (kvm_arch_ops) {
1943                 printk(KERN_ERR "kvm: already loaded the other module\n");
1944                 return -EEXIST;
1945         }
1946
1947         kvm_arch_ops = ops;
1948
1949         if (!kvm_arch_ops->cpu_has_kvm_support()) {
1950                 printk(KERN_ERR "kvm: no hardware support\n");
1951                 return -EOPNOTSUPP;
1952         }
1953         if (kvm_arch_ops->disabled_by_bios()) {
1954                 printk(KERN_ERR "kvm: disabled by bios\n");
1955                 return -EOPNOTSUPP;
1956         }
1957
1958         r = kvm_arch_ops->hardware_setup();
1959         if (r < 0)
1960             return r;
1961
1962         on_each_cpu(kvm_arch_ops->hardware_enable, 0, 0, 1);
1963         register_reboot_notifier(&kvm_reboot_notifier);
1964
1965         kvm_chardev_ops.owner = module;
1966
1967         r = misc_register(&kvm_dev);
1968         if (r) {
1969                 printk (KERN_ERR "kvm: misc device register failed\n");
1970                 goto out_free;
1971         }
1972
1973         return r;
1974
1975 out_free:
1976         unregister_reboot_notifier(&kvm_reboot_notifier);
1977         on_each_cpu(kvm_arch_ops->hardware_disable, 0, 0, 1);
1978         kvm_arch_ops->hardware_unsetup();
1979         return r;
1980 }
1981
1982 void kvm_exit_arch(void)
1983 {
1984         misc_deregister(&kvm_dev);
1985
1986         unregister_reboot_notifier(&kvm_reboot_notifier);
1987         on_each_cpu(kvm_arch_ops->hardware_disable, 0, 0, 1);
1988         kvm_arch_ops->hardware_unsetup();
1989         kvm_arch_ops = NULL;
1990 }
1991
1992 static __init int kvm_init(void)
1993 {
1994         static struct page *bad_page;
1995         int r = 0;
1996
1997         kvm_init_debug();
1998
1999         kvm_init_msr_list();
2000
2001         if ((bad_page = alloc_page(GFP_KERNEL)) == NULL) {
2002                 r = -ENOMEM;
2003                 goto out;
2004         }
2005
2006         bad_page_address = page_to_pfn(bad_page) << PAGE_SHIFT;
2007         memset(__va(bad_page_address), 0, PAGE_SIZE);
2008
2009         return r;
2010
2011 out:
2012         kvm_exit_debug();
2013         return r;
2014 }
2015
2016 static __exit void kvm_exit(void)
2017 {
2018         kvm_exit_debug();
2019         __free_page(pfn_to_page(bad_page_address >> PAGE_SHIFT));
2020 }
2021
2022 module_init(kvm_init)
2023 module_exit(kvm_exit)
2024
2025 EXPORT_SYMBOL_GPL(kvm_init_arch);
2026 EXPORT_SYMBOL_GPL(kvm_exit_arch);