git.oblomov.eu Git - linux-2.6/blob - arch/x86/kvm/mmu.c

   1 /*
   2  * Kernel-based Virtual Machine driver for Linux
   3  *
   4  * This module enables machines with Intel VT-x extensions to run virtual
   5  * machines without emulation or binary translation.
   6  *
   7  * MMU support
   8  *
   9  * Copyright (C) 2006 Qumranet, Inc.
  10  *
  11  * Authors:
  12  *   Yaniv Kamay  <yaniv@qumranet.com>
  13  *   Avi Kivity   <avi@qumranet.com>
  14  *
  15  * This work is licensed under the terms of the GNU GPL, version 2.  See
  16  * the COPYING file in the top-level directory.
  17  *
  18  */
  19
  20 #include "vmx.h"
  21 #include "mmu.h"
  22
  23 #include <linux/kvm_host.h>
  24 #include <linux/types.h>
  25 #include <linux/string.h>
  26 #include <linux/mm.h>
  27 #include <linux/highmem.h>
  28 #include <linux/module.h>
  29 #include <linux/swap.h>
  30 #include <linux/hugetlb.h>
  31 #include <linux/compiler.h>
  32
  33 #include <asm/page.h>
  34 #include <asm/cmpxchg.h>
  35 #include <asm/io.h>
  36
  37 /*
  38  * When setting this variable to true it enables Two-Dimensional-Paging
  39  * where the hardware walks 2 page tables:
  40  * 1. the guest-virtual to guest-physical
  41  * 2. while doing 1. it walks guest-physical to host-physical
  42  * If the hardware supports that we don't need to do shadow paging.
  43  */
  44 bool tdp_enabled = false;
  45
  46 #undef MMU_DEBUG
  47
  48 #undef AUDIT
  49
  50 #ifdef AUDIT
  51 static void kvm_mmu_audit(struct kvm_vcpu *vcpu, const char *msg);
  52 #else
  53 static void kvm_mmu_audit(struct kvm_vcpu *vcpu, const char *msg) {}
  54 #endif
  55
  56 #ifdef MMU_DEBUG
  57
  58 #define pgprintk(x...) do { if (dbg) printk(x); } while (0)
  59 #define rmap_printk(x...) do { if (dbg) printk(x); } while (0)
  60
  61 #else
  62
  63 #define pgprintk(x...) do { } while (0)
  64 #define rmap_printk(x...) do { } while (0)
  65
  66 #endif
  67
  68 #if defined(MMU_DEBUG) || defined(AUDIT)
  69 static int dbg = 0;
  70 module_param(dbg, bool, 0644);
  71 #endif
  72
  73 #ifndef MMU_DEBUG
  74 #define ASSERT(x) do { } while (0)
  75 #else
  76 #define ASSERT(x)                                                       \
  77         if (!(x)) {                                                     \
  78                 printk(KERN_WARNING "assertion failed %s:%d: %s\n",     \
  79                        __FILE__, __LINE__, #x);                         \
  80         }
  81 #endif
  82
  83 #define PT_FIRST_AVAIL_BITS_SHIFT 9
  84 #define PT64_SECOND_AVAIL_BITS_SHIFT 52
  85
  86 #define VALID_PAGE(x) ((x) != INVALID_PAGE)
  87
  88 #define PT64_LEVEL_BITS 9
  89
  90 #define PT64_LEVEL_SHIFT(level) \
  91                 (PAGE_SHIFT + (level - 1) * PT64_LEVEL_BITS)
  92
  93 #define PT64_LEVEL_MASK(level) \
  94                 (((1ULL << PT64_LEVEL_BITS) - 1) << PT64_LEVEL_SHIFT(level))
  95
  96 #define PT64_INDEX(address, level)\
  97         (((address) >> PT64_LEVEL_SHIFT(level)) & ((1 << PT64_LEVEL_BITS) - 1))
  98
  99
 100 #define PT32_LEVEL_BITS 10
 101
 102 #define PT32_LEVEL_SHIFT(level) \
 103                 (PAGE_SHIFT + (level - 1) * PT32_LEVEL_BITS)
 104
 105 #define PT32_LEVEL_MASK(level) \
 106                 (((1ULL << PT32_LEVEL_BITS) - 1) << PT32_LEVEL_SHIFT(level))
 107
 108 #define PT32_INDEX(address, level)\
 109         (((address) >> PT32_LEVEL_SHIFT(level)) & ((1 << PT32_LEVEL_BITS) - 1))
 110
 111
 112 #define PT64_BASE_ADDR_MASK (((1ULL << 52) - 1) & ~(u64)(PAGE_SIZE-1))
 113 #define PT64_DIR_BASE_ADDR_MASK \
 114         (PT64_BASE_ADDR_MASK & ~((1ULL << (PAGE_SHIFT + PT64_LEVEL_BITS)) - 1))
 115
 116 #define PT32_BASE_ADDR_MASK PAGE_MASK
 117 #define PT32_DIR_BASE_ADDR_MASK \
 118         (PAGE_MASK & ~((1ULL << (PAGE_SHIFT + PT32_LEVEL_BITS)) - 1))
 119
 120 #define PT64_PERM_MASK (PT_PRESENT_MASK | PT_WRITABLE_MASK | PT_USER_MASK \
 121                         | PT64_NX_MASK)
 122
 123 #define PFERR_PRESENT_MASK (1U << 0)
 124 #define PFERR_WRITE_MASK (1U << 1)
 125 #define PFERR_USER_MASK (1U << 2)
 126 #define PFERR_FETCH_MASK (1U << 4)
 127
 128 #define PT_DIRECTORY_LEVEL 2
 129 #define PT_PAGE_TABLE_LEVEL 1
 130
 131 #define RMAP_EXT 4
 132
 133 #define ACC_EXEC_MASK    1
 134 #define ACC_WRITE_MASK   PT_WRITABLE_MASK
 135 #define ACC_USER_MASK    PT_USER_MASK
 136 #define ACC_ALL          (ACC_EXEC_MASK | ACC_WRITE_MASK | ACC_USER_MASK)
 137
 138 struct kvm_pv_mmu_op_buffer {
 139         void *ptr;
 140         unsigned len;
 141         unsigned processed;
 142         char buf[512] __aligned(sizeof(long));
 143 };
 144
 145 struct kvm_rmap_desc {
 146         u64 *shadow_ptes[RMAP_EXT];
 147         struct kvm_rmap_desc *more;
 148 };
 149
 150 static struct kmem_cache *pte_chain_cache;
 151 static struct kmem_cache *rmap_desc_cache;
 152 static struct kmem_cache *mmu_page_header_cache;
 153
 154 static u64 __read_mostly shadow_trap_nonpresent_pte;
 155 static u64 __read_mostly shadow_notrap_nonpresent_pte;
 156 static u64 __read_mostly shadow_base_present_pte;
 157 static u64 __read_mostly shadow_nx_mask;
 158 static u64 __read_mostly shadow_x_mask; /* mutual exclusive with nx_mask */
 159 static u64 __read_mostly shadow_user_mask;
 160 static u64 __read_mostly shadow_accessed_mask;
 161 static u64 __read_mostly shadow_dirty_mask;
 162
 163 void kvm_mmu_set_nonpresent_ptes(u64 trap_pte, u64 notrap_pte)
 164 {
 165         shadow_trap_nonpresent_pte = trap_pte;
 166         shadow_notrap_nonpresent_pte = notrap_pte;
 167 }
 168 EXPORT_SYMBOL_GPL(kvm_mmu_set_nonpresent_ptes);
 169
 170 void kvm_mmu_set_base_ptes(u64 base_pte)
 171 {
 172         shadow_base_present_pte = base_pte;
 173 }
 174 EXPORT_SYMBOL_GPL(kvm_mmu_set_base_ptes);
 175
 176 void kvm_mmu_set_mask_ptes(u64 user_mask, u64 accessed_mask,
 177                 u64 dirty_mask, u64 nx_mask, u64 x_mask)
 178 {
 179         shadow_user_mask = user_mask;
 180         shadow_accessed_mask = accessed_mask;
 181         shadow_dirty_mask = dirty_mask;
 182         shadow_nx_mask = nx_mask;
 183         shadow_x_mask = x_mask;
 184 }
 185 EXPORT_SYMBOL_GPL(kvm_mmu_set_mask_ptes);
 186
 187 static int is_write_protection(struct kvm_vcpu *vcpu)
 188 {
 189         return vcpu->arch.cr0 & X86_CR0_WP;
 190 }
 191
 192 static int is_cpuid_PSE36(void)
 193 {
 194         return 1;
 195 }
 196
 197 static int is_nx(struct kvm_vcpu *vcpu)
 198 {
 199         return vcpu->arch.shadow_efer & EFER_NX;
 200 }
 201
 202 static int is_present_pte(unsigned long pte)
 203 {
 204         return pte & PT_PRESENT_MASK;
 205 }
 206
 207 static int is_shadow_present_pte(u64 pte)
 208 {
 209         return pte != shadow_trap_nonpresent_pte
 210                 && pte != shadow_notrap_nonpresent_pte;
 211 }
 212
 213 static int is_large_pte(u64 pte)
 214 {
 215         return pte & PT_PAGE_SIZE_MASK;
 216 }
 217
 218 static int is_writeble_pte(unsigned long pte)
 219 {
 220         return pte & PT_WRITABLE_MASK;
 221 }
 222
 223 static int is_dirty_pte(unsigned long pte)
 224 {
 225         return pte & shadow_dirty_mask;
 226 }
 227
 228 static int is_rmap_pte(u64 pte)
 229 {
 230         return is_shadow_present_pte(pte);
 231 }
 232
 233 static pfn_t spte_to_pfn(u64 pte)
 234 {
 235         return (pte & PT64_BASE_ADDR_MASK) >> PAGE_SHIFT;
 236 }
 237
 238 static gfn_t pse36_gfn_delta(u32 gpte)
 239 {
 240         int shift = 32 - PT32_DIR_PSE36_SHIFT - PAGE_SHIFT;
 241
 242         return (gpte & PT32_DIR_PSE36_MASK) << shift;
 243 }
 244
 245 static void set_shadow_pte(u64 *sptep, u64 spte)
 246 {
 247 #ifdef CONFIG_X86_64
 248         set_64bit((unsigned long *)sptep, spte);
 249 #else
 250         set_64bit((unsigned long long *)sptep, spte);
 251 #endif
 252 }
 253
 254 static int mmu_topup_memory_cache(struct kvm_mmu_memory_cache *cache,
 255                                   struct kmem_cache *base_cache, int min)
 256 {
 257         void *obj;
 258
 259         if (cache->nobjs >= min)
 260                 return 0;
 261         while (cache->nobjs < ARRAY_SIZE(cache->objects)) {
 262                 obj = kmem_cache_zalloc(base_cache, GFP_KERNEL);
 263                 if (!obj)
 264                         return -ENOMEM;
 265                 cache->objects[cache->nobjs++] = obj;
 266         }
 267         return 0;
 268 }
 269
 270 static void mmu_free_memory_cache(struct kvm_mmu_memory_cache *mc)
 271 {
 272         while (mc->nobjs)
 273                 kfree(mc->objects[--mc->nobjs]);
 274 }
 275
 276 static int mmu_topup_memory_cache_page(struct kvm_mmu_memory_cache *cache,
 277                                        int min)
 278 {
 279         struct page *page;
 280
 281         if (cache->nobjs >= min)
 282                 return 0;
 283         while (cache->nobjs < ARRAY_SIZE(cache->objects)) {
 284                 page = alloc_page(GFP_KERNEL);
 285                 if (!page)
 286                         return -ENOMEM;
 287                 set_page_private(page, 0);
 288                 cache->objects[cache->nobjs++] = page_address(page);
 289         }
 290         return 0;
 291 }
 292
 293 static void mmu_free_memory_cache_page(struct kvm_mmu_memory_cache *mc)
 294 {
 295         while (mc->nobjs)
 296                 free_page((unsigned long)mc->objects[--mc->nobjs]);
 297 }
 298
 299 static int mmu_topup_memory_caches(struct kvm_vcpu *vcpu)
 300 {
 301         int r;
 302
 303         r = mmu_topup_memory_cache(&vcpu->arch.mmu_pte_chain_cache,
 304                                    pte_chain_cache, 4);
 305         if (r)
 306                 goto out;
 307         r = mmu_topup_memory_cache(&vcpu->arch.mmu_rmap_desc_cache,
 308                                    rmap_desc_cache, 1);
 309         if (r)
 310                 goto out;
 311         r = mmu_topup_memory_cache_page(&vcpu->arch.mmu_page_cache, 8);
 312         if (r)
 313                 goto out;
 314         r = mmu_topup_memory_cache(&vcpu->arch.mmu_page_header_cache,
 315                                    mmu_page_header_cache, 4);
 316 out:
 317         return r;
 318 }
 319
 320 static void mmu_free_memory_caches(struct kvm_vcpu *vcpu)
 321 {
 322         mmu_free_memory_cache(&vcpu->arch.mmu_pte_chain_cache);
 323         mmu_free_memory_cache(&vcpu->arch.mmu_rmap_desc_cache);
 324         mmu_free_memory_cache_page(&vcpu->arch.mmu_page_cache);
 325         mmu_free_memory_cache(&vcpu->arch.mmu_page_header_cache);
 326 }
 327
 328 static void *mmu_memory_cache_alloc(struct kvm_mmu_memory_cache *mc,
 329                                     size_t size)
 330 {
 331         void *p;
 332
 333         BUG_ON(!mc->nobjs);
 334         p = mc->objects[--mc->nobjs];
 335         memset(p, 0, size);
 336         return p;
 337 }
 338
 339 static struct kvm_pte_chain *mmu_alloc_pte_chain(struct kvm_vcpu *vcpu)
 340 {
 341         return mmu_memory_cache_alloc(&vcpu->arch.mmu_pte_chain_cache,
 342                                       sizeof(struct kvm_pte_chain));
 343 }
 344
 345 static void mmu_free_pte_chain(struct kvm_pte_chain *pc)
 346 {
 347         kfree(pc);
 348 }
 349
 350 static struct kvm_rmap_desc *mmu_alloc_rmap_desc(struct kvm_vcpu *vcpu)
 351 {
 352         return mmu_memory_cache_alloc(&vcpu->arch.mmu_rmap_desc_cache,
 353                                       sizeof(struct kvm_rmap_desc));
 354 }
 355
 356 static void mmu_free_rmap_desc(struct kvm_rmap_desc *rd)
 357 {
 358         kfree(rd);
 359 }
 360
 361 /*
 362  * Return the pointer to the largepage write count for a given
 363  * gfn, handling slots that are not large page aligned.
 364  */
 365 static int *slot_largepage_idx(gfn_t gfn, struct kvm_memory_slot *slot)
 366 {
 367         unsigned long idx;
 368
 369         idx = (gfn / KVM_PAGES_PER_HPAGE) -
 370               (slot->base_gfn / KVM_PAGES_PER_HPAGE);
 371         return &slot->lpage_info[idx].write_count;
 372 }
 373
 374 static void account_shadowed(struct kvm *kvm, gfn_t gfn)
 375 {
 376         int *write_count;
 377
 378         write_count = slot_largepage_idx(gfn, gfn_to_memslot(kvm, gfn));
 379         *write_count += 1;
 380 }
 381
 382 static void unaccount_shadowed(struct kvm *kvm, gfn_t gfn)
 383 {
 384         int *write_count;
 385
 386         write_count = slot_largepage_idx(gfn, gfn_to_memslot(kvm, gfn));
 387         *write_count -= 1;
 388         WARN_ON(*write_count < 0);
 389 }
 390
 391 static int has_wrprotected_page(struct kvm *kvm, gfn_t gfn)
 392 {
 393         struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn);
 394         int *largepage_idx;
 395
 396         if (slot) {
 397                 largepage_idx = slot_largepage_idx(gfn, slot);
 398                 return *largepage_idx;
 399         }
 400
 401         return 1;
 402 }
 403
 404 static int host_largepage_backed(struct kvm *kvm, gfn_t gfn)
 405 {
 406         struct vm_area_struct *vma;
 407         unsigned long addr;
 408
 409         addr = gfn_to_hva(kvm, gfn);
 410         if (kvm_is_error_hva(addr))
 411                 return 0;
 412
 413         vma = find_vma(current->mm, addr);
 414         if (vma && is_vm_hugetlb_page(vma))
 415                 return 1;
 416
 417         return 0;
 418 }
 419
 420 static int is_largepage_backed(struct kvm_vcpu *vcpu, gfn_t large_gfn)
 421 {
 422         struct kvm_memory_slot *slot;
 423
 424         if (has_wrprotected_page(vcpu->kvm, large_gfn))
 425                 return 0;
 426
 427         if (!host_largepage_backed(vcpu->kvm, large_gfn))
 428                 return 0;
 429
 430         slot = gfn_to_memslot(vcpu->kvm, large_gfn);
 431         if (slot && slot->dirty_bitmap)
 432                 return 0;
 433
 434         return 1;
 435 }
 436
 437 /*
 438  * Take gfn and return the reverse mapping to it.
 439  * Note: gfn must be unaliased before this function get called
 440  */
 441
 442 static unsigned long *gfn_to_rmap(struct kvm *kvm, gfn_t gfn, int lpage)
 443 {
 444         struct kvm_memory_slot *slot;
 445         unsigned long idx;
 446
 447         slot = gfn_to_memslot(kvm, gfn);
 448         if (!lpage)
 449                 return &slot->rmap[gfn - slot->base_gfn];
 450
 451         idx = (gfn / KVM_PAGES_PER_HPAGE) -
 452               (slot->base_gfn / KVM_PAGES_PER_HPAGE);
 453
 454         return &slot->lpage_info[idx].rmap_pde;
 455 }
 456
 457 /*
 458  * Reverse mapping data structures:
 459  *
 460  * If rmapp bit zero is zero, then rmapp point to the shadw page table entry
 461  * that points to page_address(page).
 462  *
 463  * If rmapp bit zero is one, (then rmap & ~1) points to a struct kvm_rmap_desc
 464  * containing more mappings.
 465  */
 466 static void rmap_add(struct kvm_vcpu *vcpu, u64 *spte, gfn_t gfn, int lpage)
 467 {
 468         struct kvm_mmu_page *sp;
 469         struct kvm_rmap_desc *desc;
 470         unsigned long *rmapp;
 471         int i;
 472
 473         if (!is_rmap_pte(*spte))
 474                 return;
 475         gfn = unalias_gfn(vcpu->kvm, gfn);
 476         sp = page_header(__pa(spte));
 477         sp->gfns[spte - sp->spt] = gfn;
 478         rmapp = gfn_to_rmap(vcpu->kvm, gfn, lpage);
 479         if (!*rmapp) {
 480                 rmap_printk("rmap_add: %p %llx 0->1\n", spte, *spte);
 481                 *rmapp = (unsigned long)spte;
 482         } else if (!(*rmapp & 1)) {
 483                 rmap_printk("rmap_add: %p %llx 1->many\n", spte, *spte);
 484                 desc = mmu_alloc_rmap_desc(vcpu);
 485                 desc->shadow_ptes[0] = (u64 *)*rmapp;
 486                 desc->shadow_ptes[1] = spte;
 487                 *rmapp = (unsigned long)desc | 1;
 488         } else {
 489                 rmap_printk("rmap_add: %p %llx many->many\n", spte, *spte);
 490                 desc = (struct kvm_rmap_desc *)(*rmapp & ~1ul);
 491                 while (desc->shadow_ptes[RMAP_EXT-1] && desc->more)
 492                         desc = desc->more;
 493                 if (desc->shadow_ptes[RMAP_EXT-1]) {
 494                         desc->more = mmu_alloc_rmap_desc(vcpu);
 495                         desc = desc->more;
 496                 }
 497                 for (i = 0; desc->shadow_ptes[i]; ++i)
 498                         ;
 499                 desc->shadow_ptes[i] = spte;
 500         }
 501 }
 502
 503 static void rmap_desc_remove_entry(unsigned long *rmapp,
 504                                    struct kvm_rmap_desc *desc,
 505                                    int i,
 506                                    struct kvm_rmap_desc *prev_desc)
 507 {
 508         int j;
 509
 510         for (j = RMAP_EXT - 1; !desc->shadow_ptes[j] && j > i; --j)
 511                 ;
 512         desc->shadow_ptes[i] = desc->shadow_ptes[j];
 513         desc->shadow_ptes[j] = NULL;
 514         if (j != 0)
 515                 return;
 516         if (!prev_desc && !desc->more)
 517                 *rmapp = (unsigned long)desc->shadow_ptes[0];
 518         else
 519                 if (prev_desc)
 520                         prev_desc->more = desc->more;
 521                 else
 522                         *rmapp = (unsigned long)desc->more | 1;
 523         mmu_free_rmap_desc(desc);
 524 }
 525
 526 static void rmap_remove(struct kvm *kvm, u64 *spte)
 527 {
 528         struct kvm_rmap_desc *desc;
 529         struct kvm_rmap_desc *prev_desc;
 530         struct kvm_mmu_page *sp;
 531         pfn_t pfn;
 532         unsigned long *rmapp;
 533         int i;
 534
 535         if (!is_rmap_pte(*spte))
 536                 return;
 537         sp = page_header(__pa(spte));
 538         pfn = spte_to_pfn(*spte);
 539         if (*spte & shadow_accessed_mask)
 540                 kvm_set_pfn_accessed(pfn);
 541         if (is_writeble_pte(*spte))
 542                 kvm_release_pfn_dirty(pfn);
 543         else
 544                 kvm_release_pfn_clean(pfn);
 545         rmapp = gfn_to_rmap(kvm, sp->gfns[spte - sp->spt], is_large_pte(*spte));
 546         if (!*rmapp) {
 547                 printk(KERN_ERR "rmap_remove: %p %llx 0->BUG\n", spte, *spte);
 548                 BUG();
 549         } else if (!(*rmapp & 1)) {
 550                 rmap_printk("rmap_remove:  %p %llx 1->0\n", spte, *spte);
 551                 if ((u64 *)*rmapp != spte) {
 552                         printk(KERN_ERR "rmap_remove:  %p %llx 1->BUG\n",
 553                                spte, *spte);
 554                         BUG();
 555                 }
 556                 *rmapp = 0;
 557         } else {
 558                 rmap_printk("rmap_remove:  %p %llx many->many\n", spte, *spte);
 559                 desc = (struct kvm_rmap_desc *)(*rmapp & ~1ul);
 560                 prev_desc = NULL;
 561                 while (desc) {
 562                         for (i = 0; i < RMAP_EXT && desc->shadow_ptes[i]; ++i)
 563                                 if (desc->shadow_ptes[i] == spte) {
 564                                         rmap_desc_remove_entry(rmapp,
 565                                                                desc, i,
 566                                                                prev_desc);
 567                                         return;
 568                                 }
 569                         prev_desc = desc;
 570                         desc = desc->more;
 571                 }
 572                 BUG();
 573         }
 574 }
 575
 576 static u64 *rmap_next(struct kvm *kvm, unsigned long *rmapp, u64 *spte)
 577 {
 578         struct kvm_rmap_desc *desc;
 579         struct kvm_rmap_desc *prev_desc;
 580         u64 *prev_spte;
 581         int i;
 582
 583         if (!*rmapp)
 584                 return NULL;
 585         else if (!(*rmapp & 1)) {
 586                 if (!spte)
 587                         return (u64 *)*rmapp;
 588                 return NULL;
 589         }
 590         desc = (struct kvm_rmap_desc *)(*rmapp & ~1ul);
 591         prev_desc = NULL;
 592         prev_spte = NULL;
 593         while (desc) {
 594                 for (i = 0; i < RMAP_EXT && desc->shadow_ptes[i]; ++i) {
 595                         if (prev_spte == spte)
 596                                 return desc->shadow_ptes[i];
 597                         prev_spte = desc->shadow_ptes[i];
 598                 }
 599                 desc = desc->more;
 600         }
 601         return NULL;
 602 }
 603
 604 static void rmap_write_protect(struct kvm *kvm, u64 gfn)
 605 {
 606         unsigned long *rmapp;
 607         u64 *spte;
 608         int write_protected = 0;
 609
 610         gfn = unalias_gfn(kvm, gfn);
 611         rmapp = gfn_to_rmap(kvm, gfn, 0);
 612
 613         spte = rmap_next(kvm, rmapp, NULL);
 614         while (spte) {
 615                 BUG_ON(!spte);
 616                 BUG_ON(!(*spte & PT_PRESENT_MASK));
 617                 rmap_printk("rmap_write_protect: spte %p %llx\n", spte, *spte);
 618                 if (is_writeble_pte(*spte)) {
 619                         set_shadow_pte(spte, *spte & ~PT_WRITABLE_MASK);
 620                         write_protected = 1;
 621                 }
 622                 spte = rmap_next(kvm, rmapp, spte);
 623         }
 624         if (write_protected) {
 625                 pfn_t pfn;
 626
 627                 spte = rmap_next(kvm, rmapp, NULL);
 628                 pfn = spte_to_pfn(*spte);
 629                 kvm_set_pfn_dirty(pfn);
 630         }
 631
 632         /* check for huge page mappings */
 633         rmapp = gfn_to_rmap(kvm, gfn, 1);
 634         spte = rmap_next(kvm, rmapp, NULL);
 635         while (spte) {
 636                 BUG_ON(!spte);
 637                 BUG_ON(!(*spte & PT_PRESENT_MASK));
 638                 BUG_ON((*spte & (PT_PAGE_SIZE_MASK|PT_PRESENT_MASK)) != (PT_PAGE_SIZE_MASK|PT_PRESENT_MASK));
 639                 pgprintk("rmap_write_protect(large): spte %p %llx %lld\n", spte, *spte, gfn);
 640                 if (is_writeble_pte(*spte)) {
 641                         rmap_remove(kvm, spte);
 642                         --kvm->stat.lpages;
 643                         set_shadow_pte(spte, shadow_trap_nonpresent_pte);
 644                         spte = NULL;
 645                         write_protected = 1;
 646                 }
 647                 spte = rmap_next(kvm, rmapp, spte);
 648         }
 649
 650         if (write_protected)
 651                 kvm_flush_remote_tlbs(kvm);
 652
 653         account_shadowed(kvm, gfn);
 654 }
 655
 656 static int kvm_unmap_rmapp(struct kvm *kvm, unsigned long *rmapp)
 657 {
 658         u64 *spte;
 659         int need_tlb_flush = 0;
 660
 661         while ((spte = rmap_next(kvm, rmapp, NULL))) {
 662                 BUG_ON(!(*spte & PT_PRESENT_MASK));
 663                 rmap_printk("kvm_rmap_unmap_hva: spte %p %llx\n", spte, *spte);
 664                 rmap_remove(kvm, spte);
 665                 set_shadow_pte(spte, shadow_trap_nonpresent_pte);
 666                 need_tlb_flush = 1;
 667         }
 668         return need_tlb_flush;
 669 }
 670
 671 static int kvm_handle_hva(struct kvm *kvm, unsigned long hva,
 672                           int (*handler)(struct kvm *kvm, unsigned long *rmapp))
 673 {
 674         int i;
 675         int retval = 0;
 676
 677         /*
 678          * If mmap_sem isn't taken, we can look the memslots with only
 679          * the mmu_lock by skipping over the slots with userspace_addr == 0.
 680          */
 681         for (i = 0; i < kvm->nmemslots; i++) {
 682                 struct kvm_memory_slot *memslot = &kvm->memslots[i];
 683                 unsigned long start = memslot->userspace_addr;
 684                 unsigned long end;
 685
 686                 /* mmu_lock protects userspace_addr */
 687                 if (!start)
 688                         continue;
 689
 690                 end = start + (memslot->npages << PAGE_SHIFT);
 691                 if (hva >= start && hva < end) {
 692                         gfn_t gfn_offset = (hva - start) >> PAGE_SHIFT;
 693                         retval |= handler(kvm, &memslot->rmap[gfn_offset]);
 694                         retval |= handler(kvm,
 695                                           &memslot->lpage_info[
 696                                                   gfn_offset /
 697                                                   KVM_PAGES_PER_HPAGE].rmap_pde);
 698                 }
 699         }
 700
 701         return retval;
 702 }
 703
 704 int kvm_unmap_hva(struct kvm *kvm, unsigned long hva)
 705 {
 706         return kvm_handle_hva(kvm, hva, kvm_unmap_rmapp);
 707 }
 708
 709 static int kvm_age_rmapp(struct kvm *kvm, unsigned long *rmapp)
 710 {
 711         u64 *spte;
 712         int young = 0;
 713
 714         /* always return old for EPT */
 715         if (!shadow_accessed_mask)
 716                 return 0;
 717
 718         spte = rmap_next(kvm, rmapp, NULL);
 719         while (spte) {
 720                 int _young;
 721                 u64 _spte = *spte;
 722                 BUG_ON(!(_spte & PT_PRESENT_MASK));
 723                 _young = _spte & PT_ACCESSED_MASK;
 724                 if (_young) {
 725                         young = 1;
 726                         clear_bit(PT_ACCESSED_SHIFT, (unsigned long *)spte);
 727                 }
 728                 spte = rmap_next(kvm, rmapp, spte);
 729         }
 730         return young;
 731 }
 732
 733 int kvm_age_hva(struct kvm *kvm, unsigned long hva)
 734 {
 735         return kvm_handle_hva(kvm, hva, kvm_age_rmapp);
 736 }
 737
 738 #ifdef MMU_DEBUG
 739 static int is_empty_shadow_page(u64 *spt)
 740 {
 741         u64 *pos;
 742         u64 *end;
 743
 744         for (pos = spt, end = pos + PAGE_SIZE / sizeof(u64); pos != end; pos++)
 745                 if (is_shadow_present_pte(*pos)) {
 746                         printk(KERN_ERR "%s: %p %llx\n", __func__,
 747                                pos, *pos);
 748                         return 0;
 749                 }
 750         return 1;
 751 }
 752 #endif
 753
 754 static void kvm_mmu_free_page(struct kvm *kvm, struct kvm_mmu_page *sp)
 755 {
 756         ASSERT(is_empty_shadow_page(sp->spt));
 757         list_del(&sp->link);
 758         __free_page(virt_to_page(sp->spt));
 759         __free_page(virt_to_page(sp->gfns));
 760         kfree(sp);
 761         ++kvm->arch.n_free_mmu_pages;
 762 }
 763
 764 static unsigned kvm_page_table_hashfn(gfn_t gfn)
 765 {
 766         return gfn & ((1 << KVM_MMU_HASH_SHIFT) - 1);
 767 }
 768
 769 static struct kvm_mmu_page *kvm_mmu_alloc_page(struct kvm_vcpu *vcpu,
 770                                                u64 *parent_pte)
 771 {
 772         struct kvm_mmu_page *sp;
 773
 774         sp = mmu_memory_cache_alloc(&vcpu->arch.mmu_page_header_cache, sizeof *sp);
 775         sp->spt = mmu_memory_cache_alloc(&vcpu->arch.mmu_page_cache, PAGE_SIZE);
 776         sp->gfns = mmu_memory_cache_alloc(&vcpu->arch.mmu_page_cache, PAGE_SIZE);
 777         set_page_private(virt_to_page(sp->spt), (unsigned long)sp);
 778         list_add(&sp->link, &vcpu->kvm->arch.active_mmu_pages);
 779         ASSERT(is_empty_shadow_page(sp->spt));
 780         sp->slot_bitmap = 0;
 781         sp->multimapped = 0;
 782         sp->parent_pte = parent_pte;
 783         --vcpu->kvm->arch.n_free_mmu_pages;
 784         return sp;
 785 }
 786
 787 static void mmu_page_add_parent_pte(struct kvm_vcpu *vcpu,
 788                                     struct kvm_mmu_page *sp, u64 *parent_pte)
 789 {
 790         struct kvm_pte_chain *pte_chain;
 791         struct hlist_node *node;
 792         int i;
 793
 794         if (!parent_pte)
 795                 return;
 796         if (!sp->multimapped) {
 797                 u64 *old = sp->parent_pte;
 798
 799                 if (!old) {
 800                         sp->parent_pte = parent_pte;
 801                         return;
 802                 }
 803                 sp->multimapped = 1;
 804                 pte_chain = mmu_alloc_pte_chain(vcpu);
 805                 INIT_HLIST_HEAD(&sp->parent_ptes);
 806                 hlist_add_head(&pte_chain->link, &sp->parent_ptes);
 807                 pte_chain->parent_ptes[0] = old;
 808         }
 809         hlist_for_each_entry(pte_chain, node, &sp->parent_ptes, link) {
 810                 if (pte_chain->parent_ptes[NR_PTE_CHAIN_ENTRIES-1])
 811                         continue;
 812                 for (i = 0; i < NR_PTE_CHAIN_ENTRIES; ++i)
 813                         if (!pte_chain->parent_ptes[i]) {
 814                                 pte_chain->parent_ptes[i] = parent_pte;
 815                                 return;
 816                         }
 817         }
 818         pte_chain = mmu_alloc_pte_chain(vcpu);
 819         BUG_ON(!pte_chain);
 820         hlist_add_head(&pte_chain->link, &sp->parent_ptes);
 821         pte_chain->parent_ptes[0] = parent_pte;
 822 }
 823
 824 static void mmu_page_remove_parent_pte(struct kvm_mmu_page *sp,
 825                                        u64 *parent_pte)
 826 {
 827         struct kvm_pte_chain *pte_chain;
 828         struct hlist_node *node;
 829         int i;
 830
 831         if (!sp->multimapped) {
 832                 BUG_ON(sp->parent_pte != parent_pte);
 833                 sp->parent_pte = NULL;
 834                 return;
 835         }
 836         hlist_for_each_entry(pte_chain, node, &sp->parent_ptes, link)
 837                 for (i = 0; i < NR_PTE_CHAIN_ENTRIES; ++i) {
 838                         if (!pte_chain->parent_ptes[i])
 839                                 break;
 840                         if (pte_chain->parent_ptes[i] != parent_pte)
 841                                 continue;
 842                         while (i + 1 < NR_PTE_CHAIN_ENTRIES
 843                                 && pte_chain->parent_ptes[i + 1]) {
 844                                 pte_chain->parent_ptes[i]
 845                                         = pte_chain->parent_ptes[i + 1];
 846                                 ++i;
 847                         }
 848                         pte_chain->parent_ptes[i] = NULL;
 849                         if (i == 0) {
 850                                 hlist_del(&pte_chain->link);
 851                                 mmu_free_pte_chain(pte_chain);
 852                                 if (hlist_empty(&sp->parent_ptes)) {
 853                                         sp->multimapped = 0;
 854                                         sp->parent_pte = NULL;
 855                                 }
 856                         }
 857                         return;
 858                 }
 859         BUG();
 860 }
 861
 862 static void nonpaging_prefetch_page(struct kvm_vcpu *vcpu,
 863                                     struct kvm_mmu_page *sp)
 864 {
 865         int i;
 866
 867         for (i = 0; i < PT64_ENT_PER_PAGE; ++i)
 868                 sp->spt[i] = shadow_trap_nonpresent_pte;
 869 }
 870
 871 static struct kvm_mmu_page *kvm_mmu_lookup_page(struct kvm *kvm, gfn_t gfn)
 872 {
 873         unsigned index;
 874         struct hlist_head *bucket;
 875         struct kvm_mmu_page *sp;
 876         struct hlist_node *node;
 877
 878         pgprintk("%s: looking for gfn %lx\n", __func__, gfn);
 879         index = kvm_page_table_hashfn(gfn);
 880         bucket = &kvm->arch.mmu_page_hash[index];
 881         hlist_for_each_entry(sp, node, bucket, hash_link)
 882                 if (sp->gfn == gfn && !sp->role.metaphysical
 883                     && !sp->role.invalid) {
 884                         pgprintk("%s: found role %x\n",
 885                                  __func__, sp->role.word);
 886                         return sp;
 887                 }
 888         return NULL;
 889 }
 890
 891 static struct kvm_mmu_page *kvm_mmu_get_page(struct kvm_vcpu *vcpu,
 892                                              gfn_t gfn,
 893                                              gva_t gaddr,
 894                                              unsigned level,
 895                                              int metaphysical,
 896                                              unsigned access,
 897                                              u64 *parent_pte)
 898 {
 899         union kvm_mmu_page_role role;
 900         unsigned index;
 901         unsigned quadrant;
 902         struct hlist_head *bucket;
 903         struct kvm_mmu_page *sp;
 904         struct hlist_node *node;
 905
 906         role.word = 0;
 907         role.glevels = vcpu->arch.mmu.root_level;
 908         role.level = level;
 909         role.metaphysical = metaphysical;
 910         role.access = access;
 911         if (vcpu->arch.mmu.root_level <= PT32_ROOT_LEVEL) {
 912                 quadrant = gaddr >> (PAGE_SHIFT + (PT64_PT_BITS * level));
 913                 quadrant &= (1 << ((PT32_PT_BITS - PT64_PT_BITS) * level)) - 1;
 914                 role.quadrant = quadrant;
 915         }
 916         pgprintk("%s: looking gfn %lx role %x\n", __func__,
 917                  gfn, role.word);
 918         index = kvm_page_table_hashfn(gfn);
 919         bucket = &vcpu->kvm->arch.mmu_page_hash[index];
 920         hlist_for_each_entry(sp, node, bucket, hash_link)
 921                 if (sp->gfn == gfn && sp->role.word == role.word) {
 922                         mmu_page_add_parent_pte(vcpu, sp, parent_pte);
 923                         pgprintk("%s: found\n", __func__);
 924                         return sp;
 925                 }
 926         ++vcpu->kvm->stat.mmu_cache_miss;
 927         sp = kvm_mmu_alloc_page(vcpu, parent_pte);
 928         if (!sp)
 929                 return sp;
 930         pgprintk("%s: adding gfn %lx role %x\n", __func__, gfn, role.word);
 931         sp->gfn = gfn;
 932         sp->role = role;
 933         hlist_add_head(&sp->hash_link, bucket);
 934         if (!metaphysical)
 935                 rmap_write_protect(vcpu->kvm, gfn);
 936         if (shadow_trap_nonpresent_pte != shadow_notrap_nonpresent_pte)
 937                 vcpu->arch.mmu.prefetch_page(vcpu, sp);
 938         else
 939                 nonpaging_prefetch_page(vcpu, sp);
 940         return sp;
 941 }
 942
 943 static void kvm_mmu_page_unlink_children(struct kvm *kvm,
 944                                          struct kvm_mmu_page *sp)
 945 {
 946         unsigned i;
 947         u64 *pt;
 948         u64 ent;
 949
 950         pt = sp->spt;
 951
 952         if (sp->role.level == PT_PAGE_TABLE_LEVEL) {
 953                 for (i = 0; i < PT64_ENT_PER_PAGE; ++i) {
 954                         if (is_shadow_present_pte(pt[i]))
 955                                 rmap_remove(kvm, &pt[i]);
 956                         pt[i] = shadow_trap_nonpresent_pte;
 957                 }
 958                 kvm_flush_remote_tlbs(kvm);
 959                 return;
 960         }
 961
 962         for (i = 0; i < PT64_ENT_PER_PAGE; ++i) {
 963                 ent = pt[i];
 964
 965                 if (is_shadow_present_pte(ent)) {
 966                         if (!is_large_pte(ent)) {
 967                                 ent &= PT64_BASE_ADDR_MASK;
 968                                 mmu_page_remove_parent_pte(page_header(ent),
 969                                                            &pt[i]);
 970                         } else {
 971                                 --kvm->stat.lpages;
 972                                 rmap_remove(kvm, &pt[i]);
 973                         }
 974                 }
 975                 pt[i] = shadow_trap_nonpresent_pte;
 976         }
 977         kvm_flush_remote_tlbs(kvm);
 978 }
 979
 980 static void kvm_mmu_put_page(struct kvm_mmu_page *sp, u64 *parent_pte)
 981 {
 982         mmu_page_remove_parent_pte(sp, parent_pte);
 983 }
 984
 985 static void kvm_mmu_reset_last_pte_updated(struct kvm *kvm)
 986 {
 987         int i;
 988
 989         for (i = 0; i < KVM_MAX_VCPUS; ++i)
 990                 if (kvm->vcpus[i])
 991                         kvm->vcpus[i]->arch.last_pte_updated = NULL;
 992 }
 993
 994 static void kvm_mmu_unlink_parents(struct kvm *kvm, struct kvm_mmu_page *sp)
 995 {
 996         u64 *parent_pte;
 997
 998         while (sp->multimapped || sp->parent_pte) {
 999                 if (!sp->multimapped)
1000                         parent_pte = sp->parent_pte;
1001                 else {
1002                         struct kvm_pte_chain *chain;
1003
1004                         chain = container_of(sp->parent_ptes.first,
1005                                              struct kvm_pte_chain, link);
1006                         parent_pte = chain->parent_ptes[0];
1007                 }
1008                 BUG_ON(!parent_pte);
1009                 kvm_mmu_put_page(sp, parent_pte);
1010                 set_shadow_pte(parent_pte, shadow_trap_nonpresent_pte);
1011         }
1012 }
1013
1014 static void kvm_mmu_zap_page(struct kvm *kvm, struct kvm_mmu_page *sp)
1015 {
1016         ++kvm->stat.mmu_shadow_zapped;
1017         kvm_mmu_page_unlink_children(kvm, sp);
1018         kvm_mmu_unlink_parents(kvm, sp);
1019         if (!sp->root_count) {
1020                 if (!sp->role.metaphysical && !sp->role.invalid)
1021                         unaccount_shadowed(kvm, sp->gfn);
1022                 hlist_del(&sp->hash_link);
1023                 kvm_mmu_free_page(kvm, sp);
1024         } else {
1025                 int invalid = sp->role.invalid;
1026                 list_move(&sp->link, &kvm->arch.active_mmu_pages);
1027                 sp->role.invalid = 1;
1028                 kvm_reload_remote_mmus(kvm);
1029                 if (!sp->role.metaphysical && !invalid)
1030                         unaccount_shadowed(kvm, sp->gfn);
1031         }
1032         kvm_mmu_reset_last_pte_updated(kvm);
1033 }
1034
1035 /*
1036  * Changing the number of mmu pages allocated to the vm
1037  * Note: if kvm_nr_mmu_pages is too small, you will get dead lock
1038  */
1039 void kvm_mmu_change_mmu_pages(struct kvm *kvm, unsigned int kvm_nr_mmu_pages)
1040 {
1041         /*
1042          * If we set the number of mmu pages to be smaller be than the
1043          * number of actived pages , we must to free some mmu pages before we
1044          * change the value
1045          */
1046
1047         if ((kvm->arch.n_alloc_mmu_pages - kvm->arch.n_free_mmu_pages) >
1048             kvm_nr_mmu_pages) {
1049                 int n_used_mmu_pages = kvm->arch.n_alloc_mmu_pages
1050                                        - kvm->arch.n_free_mmu_pages;
1051
1052                 while (n_used_mmu_pages > kvm_nr_mmu_pages) {
1053                         struct kvm_mmu_page *page;
1054
1055                         page = container_of(kvm->arch.active_mmu_pages.prev,
1056                                             struct kvm_mmu_page, link);
1057                         kvm_mmu_zap_page(kvm, page);
1058                         n_used_mmu_pages--;
1059                 }
1060                 kvm->arch.n_free_mmu_pages = 0;
1061         }
1062         else
1063                 kvm->arch.n_free_mmu_pages += kvm_nr_mmu_pages
1064                                          - kvm->arch.n_alloc_mmu_pages;
1065
1066         kvm->arch.n_alloc_mmu_pages = kvm_nr_mmu_pages;
1067 }
1068
1069 static int kvm_mmu_unprotect_page(struct kvm *kvm, gfn_t gfn)
1070 {
1071         unsigned index;
1072         struct hlist_head *bucket;
1073         struct kvm_mmu_page *sp;
1074         struct hlist_node *node, *n;
1075         int r;
1076
1077         pgprintk("%s: looking for gfn %lx\n", __func__, gfn);
1078         r = 0;
1079         index = kvm_page_table_hashfn(gfn);
1080         bucket = &kvm->arch.mmu_page_hash[index];
1081         hlist_for_each_entry_safe(sp, node, n, bucket, hash_link)
1082                 if (sp->gfn == gfn && !sp->role.metaphysical) {
1083                         pgprintk("%s: gfn %lx role %x\n", __func__, gfn,
1084                                  sp->role.word);
1085                         kvm_mmu_zap_page(kvm, sp);
1086                         r = 1;
1087                 }
1088         return r;
1089 }
1090
1091 static void mmu_unshadow(struct kvm *kvm, gfn_t gfn)
1092 {
1093         struct kvm_mmu_page *sp;
1094
1095         while ((sp = kvm_mmu_lookup_page(kvm, gfn)) != NULL) {
1096                 pgprintk("%s: zap %lx %x\n", __func__, gfn, sp->role.word);
1097                 kvm_mmu_zap_page(kvm, sp);
1098         }
1099 }
1100
1101 static void page_header_update_slot(struct kvm *kvm, void *pte, gfn_t gfn)
1102 {
1103         int slot = memslot_id(kvm, gfn_to_memslot(kvm, gfn));
1104         struct kvm_mmu_page *sp = page_header(__pa(pte));
1105
1106         __set_bit(slot, &sp->slot_bitmap);
1107 }
1108
1109 struct page *gva_to_page(struct kvm_vcpu *vcpu, gva_t gva)
1110 {
1111         struct page *page;
1112
1113         gpa_t gpa = vcpu->arch.mmu.gva_to_gpa(vcpu, gva);
1114
1115         if (gpa == UNMAPPED_GVA)
1116                 return NULL;
1117
1118         down_read(&current->mm->mmap_sem);
1119         page = gfn_to_page(vcpu->kvm, gpa >> PAGE_SHIFT);
1120         up_read(&current->mm->mmap_sem);
1121
1122         return page;
1123 }
1124
1125 static void mmu_set_spte(struct kvm_vcpu *vcpu, u64 *shadow_pte,
1126                          unsigned pt_access, unsigned pte_access,
1127                          int user_fault, int write_fault, int dirty,
1128                          int *ptwrite, int largepage, gfn_t gfn,
1129                          pfn_t pfn, bool speculative)
1130 {
1131         u64 spte;
1132         int was_rmapped = 0;
1133         int was_writeble = is_writeble_pte(*shadow_pte);
1134
1135         pgprintk("%s: spte %llx access %x write_fault %d"
1136                  " user_fault %d gfn %lx\n",
1137                  __func__, *shadow_pte, pt_access,
1138                  write_fault, user_fault, gfn);
1139
1140         if (is_rmap_pte(*shadow_pte)) {
1141                 /*
1142                  * If we overwrite a PTE page pointer with a 2MB PMD, unlink
1143                  * the parent of the now unreachable PTE.
1144                  */
1145                 if (largepage && !is_large_pte(*shadow_pte)) {
1146                         struct kvm_mmu_page *child;
1147                         u64 pte = *shadow_pte;
1148
1149                         child = page_header(pte & PT64_BASE_ADDR_MASK);
1150                         mmu_page_remove_parent_pte(child, shadow_pte);
1151                 } else if (pfn != spte_to_pfn(*shadow_pte)) {
1152                         pgprintk("hfn old %lx new %lx\n",
1153                                  spte_to_pfn(*shadow_pte), pfn);
1154                         rmap_remove(vcpu->kvm, shadow_pte);
1155                 } else {
1156                         if (largepage)
1157                                 was_rmapped = is_large_pte(*shadow_pte);
1158                         else
1159                                 was_rmapped = 1;
1160                 }
1161         }
1162
1163         /*
1164          * We don't set the accessed bit, since we sometimes want to see
1165          * whether the guest actually used the pte (in order to detect
1166          * demand paging).
1167          */
1168         spte = shadow_base_present_pte | shadow_dirty_mask;
1169         if (!speculative)
1170                 pte_access |= PT_ACCESSED_MASK;
1171         if (!dirty)
1172                 pte_access &= ~ACC_WRITE_MASK;
1173         if (pte_access & ACC_EXEC_MASK)
1174                 spte |= shadow_x_mask;
1175         else
1176                 spte |= shadow_nx_mask;
1177         if (pte_access & ACC_USER_MASK)
1178                 spte |= shadow_user_mask;
1179         if (largepage)
1180                 spte |= PT_PAGE_SIZE_MASK;
1181
1182         spte |= (u64)pfn << PAGE_SHIFT;
1183
1184         if ((pte_access & ACC_WRITE_MASK)
1185             || (write_fault && !is_write_protection(vcpu) && !user_fault)) {
1186                 struct kvm_mmu_page *shadow;
1187
1188                 spte |= PT_WRITABLE_MASK;
1189
1190                 shadow = kvm_mmu_lookup_page(vcpu->kvm, gfn);
1191                 if (shadow ||
1192                    (largepage && has_wrprotected_page(vcpu->kvm, gfn))) {
1193                         pgprintk("%s: found shadow page for %lx, marking ro\n",
1194                                  __func__, gfn);
1195                         pte_access &= ~ACC_WRITE_MASK;
1196                         if (is_writeble_pte(spte)) {
1197                                 spte &= ~PT_WRITABLE_MASK;
1198                                 kvm_x86_ops->tlb_flush(vcpu);
1199                         }
1200                         if (write_fault)
1201                                 *ptwrite = 1;
1202                 }
1203         }
1204
1205         if (pte_access & ACC_WRITE_MASK)
1206                 mark_page_dirty(vcpu->kvm, gfn);
1207
1208         pgprintk("%s: setting spte %llx\n", __func__, spte);
1209         pgprintk("instantiating %s PTE (%s) at %ld (%llx) addr %p\n",
1210                  (spte&PT_PAGE_SIZE_MASK)? "2MB" : "4kB",
1211                  (spte&PT_WRITABLE_MASK)?"RW":"R", gfn, spte, shadow_pte);
1212         set_shadow_pte(shadow_pte, spte);
1213         if (!was_rmapped && (spte & PT_PAGE_SIZE_MASK)
1214             && (spte & PT_PRESENT_MASK))
1215                 ++vcpu->kvm->stat.lpages;
1216
1217         page_header_update_slot(vcpu->kvm, shadow_pte, gfn);
1218         if (!was_rmapped) {
1219                 rmap_add(vcpu, shadow_pte, gfn, largepage);
1220                 if (!is_rmap_pte(*shadow_pte))
1221                         kvm_release_pfn_clean(pfn);
1222         } else {
1223                 if (was_writeble)
1224                         kvm_release_pfn_dirty(pfn);
1225                 else
1226                         kvm_release_pfn_clean(pfn);
1227         }
1228         if (speculative) {
1229                 vcpu->arch.last_pte_updated = shadow_pte;
1230                 vcpu->arch.last_pte_gfn = gfn;
1231         }
1232 }
1233
1234 static void nonpaging_new_cr3(struct kvm_vcpu *vcpu)
1235 {
1236 }
1237
1238 static int __direct_map(struct kvm_vcpu *vcpu, gpa_t v, int write,
1239                            int largepage, gfn_t gfn, pfn_t pfn,
1240                            int level)
1241 {
1242         hpa_t table_addr = vcpu->arch.mmu.root_hpa;
1243         int pt_write = 0;
1244
1245         for (; ; level--) {
1246                 u32 index = PT64_INDEX(v, level);
1247                 u64 *table;
1248
1249                 ASSERT(VALID_PAGE(table_addr));
1250                 table = __va(table_addr);
1251
1252                 if (level == 1) {
1253                         mmu_set_spte(vcpu, &table[index], ACC_ALL, ACC_ALL,
1254                                      0, write, 1, &pt_write, 0, gfn, pfn, false);
1255                         return pt_write;
1256                 }
1257
1258                 if (largepage && level == 2) {
1259                         mmu_set_spte(vcpu, &table[index], ACC_ALL, ACC_ALL,
1260                                      0, write, 1, &pt_write, 1, gfn, pfn, false);
1261                         return pt_write;
1262                 }
1263
1264                 if (table[index] == shadow_trap_nonpresent_pte) {
1265                         struct kvm_mmu_page *new_table;
1266                         gfn_t pseudo_gfn;
1267
1268                         pseudo_gfn = (v & PT64_DIR_BASE_ADDR_MASK)
1269                                 >> PAGE_SHIFT;
1270                         new_table = kvm_mmu_get_page(vcpu, pseudo_gfn,
1271                                                      v, level - 1,
1272                                                      1, ACC_ALL, &table[index]);
1273                         if (!new_table) {
1274                                 pgprintk("nonpaging_map: ENOMEM\n");
1275                                 kvm_release_pfn_clean(pfn);
1276                                 return -ENOMEM;
1277                         }
1278
1279                         set_shadow_pte(&table[index],
1280                                        __pa(new_table->spt)
1281                                        | PT_PRESENT_MASK | PT_WRITABLE_MASK
1282                                        | shadow_user_mask | shadow_x_mask);
1283                 }
1284                 table_addr = table[index] & PT64_BASE_ADDR_MASK;
1285         }
1286 }
1287
1288 static int nonpaging_map(struct kvm_vcpu *vcpu, gva_t v, int write, gfn_t gfn)
1289 {
1290         int r;
1291         int largepage = 0;
1292         pfn_t pfn;
1293         unsigned long mmu_seq;
1294
1295         down_read(&current->mm->mmap_sem);
1296         if (is_largepage_backed(vcpu, gfn & ~(KVM_PAGES_PER_HPAGE-1))) {
1297                 gfn &= ~(KVM_PAGES_PER_HPAGE-1);
1298                 largepage = 1;
1299         }
1300
1301         mmu_seq = vcpu->kvm->mmu_notifier_seq;
1302         /* implicit mb(), we'll read before PT lock is unlocked */
1303         pfn = gfn_to_pfn(vcpu->kvm, gfn);
1304         up_read(&current->mm->mmap_sem);
1305
1306         /* mmio */
1307         if (is_error_pfn(pfn)) {
1308                 kvm_release_pfn_clean(pfn);
1309                 return 1;
1310         }
1311
1312         spin_lock(&vcpu->kvm->mmu_lock);
1313         if (mmu_notifier_retry(vcpu, mmu_seq))
1314                 goto out_unlock;
1315         kvm_mmu_free_some_pages(vcpu);
1316         r = __direct_map(vcpu, v, write, largepage, gfn, pfn,
1317                          PT32E_ROOT_LEVEL);
1318         spin_unlock(&vcpu->kvm->mmu_lock);
1319
1320
1321         return r;
1322
1323 out_unlock:
1324         spin_unlock(&vcpu->kvm->mmu_lock);
1325         kvm_release_pfn_clean(pfn);
1326         return 0;
1327 }
1328
1329
1330 static void mmu_free_roots(struct kvm_vcpu *vcpu)
1331 {
1332         int i;
1333         struct kvm_mmu_page *sp;
1334
1335         if (!VALID_PAGE(vcpu->arch.mmu.root_hpa))
1336                 return;
1337         spin_lock(&vcpu->kvm->mmu_lock);
1338         if (vcpu->arch.mmu.shadow_root_level == PT64_ROOT_LEVEL) {
1339                 hpa_t root = vcpu->arch.mmu.root_hpa;
1340
1341                 sp = page_header(root);
1342                 --sp->root_count;
1343                 if (!sp->root_count && sp->role.invalid)
1344                         kvm_mmu_zap_page(vcpu->kvm, sp);
1345                 vcpu->arch.mmu.root_hpa = INVALID_PAGE;
1346                 spin_unlock(&vcpu->kvm->mmu_lock);
1347                 return;
1348         }
1349         for (i = 0; i < 4; ++i) {
1350                 hpa_t root = vcpu->arch.mmu.pae_root[i];
1351
1352                 if (root) {
1353                         root &= PT64_BASE_ADDR_MASK;
1354                         sp = page_header(root);
1355                         --sp->root_count;
1356                         if (!sp->root_count && sp->role.invalid)
1357                                 kvm_mmu_zap_page(vcpu->kvm, sp);
1358                 }
1359                 vcpu->arch.mmu.pae_root[i] = INVALID_PAGE;
1360         }
1361         spin_unlock(&vcpu->kvm->mmu_lock);
1362         vcpu->arch.mmu.root_hpa = INVALID_PAGE;
1363 }
1364
1365 static void mmu_alloc_roots(struct kvm_vcpu *vcpu)
1366 {
1367         int i;
1368         gfn_t root_gfn;
1369         struct kvm_mmu_page *sp;
1370         int metaphysical = 0;
1371
1372         root_gfn = vcpu->arch.cr3 >> PAGE_SHIFT;
1373
1374         if (vcpu->arch.mmu.shadow_root_level == PT64_ROOT_LEVEL) {
1375                 hpa_t root = vcpu->arch.mmu.root_hpa;
1376
1377                 ASSERT(!VALID_PAGE(root));
1378                 if (tdp_enabled)
1379                         metaphysical = 1;
1380                 sp = kvm_mmu_get_page(vcpu, root_gfn, 0,
1381                                       PT64_ROOT_LEVEL, metaphysical,
1382                                       ACC_ALL, NULL);
1383                 root = __pa(sp->spt);
1384                 ++sp->root_count;
1385                 vcpu->arch.mmu.root_hpa = root;
1386                 return;
1387         }
1388         metaphysical = !is_paging(vcpu);
1389         if (tdp_enabled)
1390                 metaphysical = 1;
1391         for (i = 0; i < 4; ++i) {
1392                 hpa_t root = vcpu->arch.mmu.pae_root[i];
1393
1394                 ASSERT(!VALID_PAGE(root));
1395                 if (vcpu->arch.mmu.root_level == PT32E_ROOT_LEVEL) {
1396                         if (!is_present_pte(vcpu->arch.pdptrs[i])) {
1397                                 vcpu->arch.mmu.pae_root[i] = 0;
1398                                 continue;
1399                         }
1400                         root_gfn = vcpu->arch.pdptrs[i] >> PAGE_SHIFT;
1401                 } else if (vcpu->arch.mmu.root_level == 0)
1402                         root_gfn = 0;
1403                 sp = kvm_mmu_get_page(vcpu, root_gfn, i << 30,
1404                                       PT32_ROOT_LEVEL, metaphysical,
1405                                       ACC_ALL, NULL);
1406                 root = __pa(sp->spt);
1407                 ++sp->root_count;
1408                 vcpu->arch.mmu.pae_root[i] = root | PT_PRESENT_MASK;
1409         }
1410         vcpu->arch.mmu.root_hpa = __pa(vcpu->arch.mmu.pae_root);
1411 }
1412
1413 static gpa_t nonpaging_gva_to_gpa(struct kvm_vcpu *vcpu, gva_t vaddr)
1414 {
1415         return vaddr;
1416 }
1417
1418 static int nonpaging_page_fault(struct kvm_vcpu *vcpu, gva_t gva,
1419                                 u32 error_code)
1420 {
1421         gfn_t gfn;
1422         int r;
1423
1424         pgprintk("%s: gva %lx error %x\n", __func__, gva, error_code);
1425         r = mmu_topup_memory_caches(vcpu);
1426         if (r)
1427                 return r;
1428
1429         ASSERT(vcpu);
1430         ASSERT(VALID_PAGE(vcpu->arch.mmu.root_hpa));
1431
1432         gfn = gva >> PAGE_SHIFT;
1433
1434         return nonpaging_map(vcpu, gva & PAGE_MASK,
1435                              error_code & PFERR_WRITE_MASK, gfn);
1436 }
1437
1438 static int tdp_page_fault(struct kvm_vcpu *vcpu, gva_t gpa,
1439                                 u32 error_code)
1440 {
1441         pfn_t pfn;
1442         int r;
1443         int largepage = 0;
1444         gfn_t gfn = gpa >> PAGE_SHIFT;
1445         unsigned long mmu_seq;
1446
1447         ASSERT(vcpu);
1448         ASSERT(VALID_PAGE(vcpu->arch.mmu.root_hpa));
1449
1450         r = mmu_topup_memory_caches(vcpu);
1451         if (r)
1452                 return r;
1453
1454         down_read(&current->mm->mmap_sem);
1455         if (is_largepage_backed(vcpu, gfn & ~(KVM_PAGES_PER_HPAGE-1))) {
1456                 gfn &= ~(KVM_PAGES_PER_HPAGE-1);
1457                 largepage = 1;
1458         }
1459         mmu_seq = vcpu->kvm->mmu_notifier_seq;
1460         /* implicit mb(), we'll read before PT lock is unlocked */
1461         pfn = gfn_to_pfn(vcpu->kvm, gfn);
1462         up_read(&current->mm->mmap_sem);
1463         if (is_error_pfn(pfn)) {
1464                 kvm_release_pfn_clean(pfn);
1465                 return 1;
1466         }
1467         spin_lock(&vcpu->kvm->mmu_lock);
1468         if (mmu_notifier_retry(vcpu, mmu_seq))
1469                 goto out_unlock;
1470         kvm_mmu_free_some_pages(vcpu);
1471         r = __direct_map(vcpu, gpa, error_code & PFERR_WRITE_MASK,
1472                          largepage, gfn, pfn, kvm_x86_ops->get_tdp_level());
1473         spin_unlock(&vcpu->kvm->mmu_lock);
1474
1475         return r;
1476
1477 out_unlock:
1478         spin_unlock(&vcpu->kvm->mmu_lock);
1479         kvm_release_pfn_clean(pfn);
1480         return 0;
1481 }
1482
1483 static void nonpaging_free(struct kvm_vcpu *vcpu)
1484 {
1485         mmu_free_roots(vcpu);
1486 }
1487
1488 static int nonpaging_init_context(struct kvm_vcpu *vcpu)
1489 {
1490         struct kvm_mmu *context = &vcpu->arch.mmu;
1491
1492         context->new_cr3 = nonpaging_new_cr3;
1493         context->page_fault = nonpaging_page_fault;
1494         context->gva_to_gpa = nonpaging_gva_to_gpa;
1495         context->free = nonpaging_free;
1496         context->prefetch_page = nonpaging_prefetch_page;
1497         context->root_level = 0;
1498         context->shadow_root_level = PT32E_ROOT_LEVEL;
1499         context->root_hpa = INVALID_PAGE;
1500         return 0;
1501 }
1502
1503 void kvm_mmu_flush_tlb(struct kvm_vcpu *vcpu)
1504 {
1505         ++vcpu->stat.tlb_flush;
1506         kvm_x86_ops->tlb_flush(vcpu);
1507 }
1508
1509 static void paging_new_cr3(struct kvm_vcpu *vcpu)
1510 {
1511         pgprintk("%s: cr3 %lx\n", __func__, vcpu->arch.cr3);
1512         mmu_free_roots(vcpu);
1513 }
1514
1515 static void inject_page_fault(struct kvm_vcpu *vcpu,
1516                               u64 addr,
1517                               u32 err_code)
1518 {
1519         kvm_inject_page_fault(vcpu, addr, err_code);
1520 }
1521
1522 static void paging_free(struct kvm_vcpu *vcpu)
1523 {
1524         nonpaging_free(vcpu);
1525 }
1526
1527 #define PTTYPE 64
1528 #include "paging_tmpl.h"
1529 #undef PTTYPE
1530
1531 #define PTTYPE 32
1532 #include "paging_tmpl.h"
1533 #undef PTTYPE
1534
1535 static int paging64_init_context_common(struct kvm_vcpu *vcpu, int level)
1536 {
1537         struct kvm_mmu *context = &vcpu->arch.mmu;
1538
1539         ASSERT(is_pae(vcpu));
1540         context->new_cr3 = paging_new_cr3;
1541         context->page_fault = paging64_page_fault;
1542         context->gva_to_gpa = paging64_gva_to_gpa;
1543         context->prefetch_page = paging64_prefetch_page;
1544         context->free = paging_free;
1545         context->root_level = level;
1546         context->shadow_root_level = level;
1547         context->root_hpa = INVALID_PAGE;
1548         return 0;
1549 }
1550
1551 static int paging64_init_context(struct kvm_vcpu *vcpu)
1552 {
1553         return paging64_init_context_common(vcpu, PT64_ROOT_LEVEL);
1554 }
1555
1556 static int paging32_init_context(struct kvm_vcpu *vcpu)
1557 {
1558         struct kvm_mmu *context = &vcpu->arch.mmu;
1559
1560         context->new_cr3 = paging_new_cr3;
1561         context->page_fault = paging32_page_fault;
1562         context->gva_to_gpa = paging32_gva_to_gpa;
1563         context->free = paging_free;
1564         context->prefetch_page = paging32_prefetch_page;
1565         context->root_level = PT32_ROOT_LEVEL;
1566         context->shadow_root_level = PT32E_ROOT_LEVEL;
1567         context->root_hpa = INVALID_PAGE;
1568         return 0;
1569 }
1570
1571 static int paging32E_init_context(struct kvm_vcpu *vcpu)
1572 {
1573         return paging64_init_context_common(vcpu, PT32E_ROOT_LEVEL);
1574 }
1575
1576 static int init_kvm_tdp_mmu(struct kvm_vcpu *vcpu)
1577 {
1578         struct kvm_mmu *context = &vcpu->arch.mmu;
1579
1580         context->new_cr3 = nonpaging_new_cr3;
1581         context->page_fault = tdp_page_fault;
1582         context->free = nonpaging_free;
1583         context->prefetch_page = nonpaging_prefetch_page;
1584         context->shadow_root_level = kvm_x86_ops->get_tdp_level();
1585         context->root_hpa = INVALID_PAGE;
1586
1587         if (!is_paging(vcpu)) {
1588                 context->gva_to_gpa = nonpaging_gva_to_gpa;
1589                 context->root_level = 0;
1590         } else if (is_long_mode(vcpu)) {
1591                 context->gva_to_gpa = paging64_gva_to_gpa;
1592                 context->root_level = PT64_ROOT_LEVEL;
1593         } else if (is_pae(vcpu)) {
1594                 context->gva_to_gpa = paging64_gva_to_gpa;
1595                 context->root_level = PT32E_ROOT_LEVEL;
1596         } else {
1597                 context->gva_to_gpa = paging32_gva_to_gpa;
1598                 context->root_level = PT32_ROOT_LEVEL;
1599         }
1600
1601         return 0;
1602 }
1603
1604 static int init_kvm_softmmu(struct kvm_vcpu *vcpu)
1605 {
1606         ASSERT(vcpu);
1607         ASSERT(!VALID_PAGE(vcpu->arch.mmu.root_hpa));
1608
1609         if (!is_paging(vcpu))
1610                 return nonpaging_init_context(vcpu);
1611         else if (is_long_mode(vcpu))
1612                 return paging64_init_context(vcpu);
1613         else if (is_pae(vcpu))
1614                 return paging32E_init_context(vcpu);
1615         else
1616                 return paging32_init_context(vcpu);
1617 }
1618
1619 static int init_kvm_mmu(struct kvm_vcpu *vcpu)
1620 {
1621         vcpu->arch.update_pte.pfn = bad_pfn;
1622
1623         if (tdp_enabled)
1624                 return init_kvm_tdp_mmu(vcpu);
1625         else
1626                 return init_kvm_softmmu(vcpu);
1627 }
1628
1629 static void destroy_kvm_mmu(struct kvm_vcpu *vcpu)
1630 {
1631         ASSERT(vcpu);
1632         if (VALID_PAGE(vcpu->arch.mmu.root_hpa)) {
1633                 vcpu->arch.mmu.free(vcpu);
1634                 vcpu->arch.mmu.root_hpa = INVALID_PAGE;
1635         }
1636 }
1637
1638 int kvm_mmu_reset_context(struct kvm_vcpu *vcpu)
1639 {
1640         destroy_kvm_mmu(vcpu);
1641         return init_kvm_mmu(vcpu);
1642 }
1643 EXPORT_SYMBOL_GPL(kvm_mmu_reset_context);
1644
1645 int kvm_mmu_load(struct kvm_vcpu *vcpu)
1646 {
1647         int r;
1648
1649         r = mmu_topup_memory_caches(vcpu);
1650         if (r)
1651                 goto out;
1652         spin_lock(&vcpu->kvm->mmu_lock);
1653         kvm_mmu_free_some_pages(vcpu);
1654         mmu_alloc_roots(vcpu);
1655         spin_unlock(&vcpu->kvm->mmu_lock);
1656         kvm_x86_ops->set_cr3(vcpu, vcpu->arch.mmu.root_hpa);
1657         kvm_mmu_flush_tlb(vcpu);
1658 out:
1659         return r;
1660 }
1661 EXPORT_SYMBOL_GPL(kvm_mmu_load);
1662
1663 void kvm_mmu_unload(struct kvm_vcpu *vcpu)
1664 {
1665         mmu_free_roots(vcpu);
1666 }
1667
1668 static void mmu_pte_write_zap_pte(struct kvm_vcpu *vcpu,
1669                                   struct kvm_mmu_page *sp,
1670                                   u64 *spte)
1671 {
1672         u64 pte;
1673         struct kvm_mmu_page *child;
1674
1675         pte = *spte;
1676         if (is_shadow_present_pte(pte)) {
1677                 if (sp->role.level == PT_PAGE_TABLE_LEVEL ||
1678                     is_large_pte(pte))
1679                         rmap_remove(vcpu->kvm, spte);
1680                 else {
1681                         child = page_header(pte & PT64_BASE_ADDR_MASK);
1682                         mmu_page_remove_parent_pte(child, spte);
1683                 }
1684         }
1685         set_shadow_pte(spte, shadow_trap_nonpresent_pte);
1686         if (is_large_pte(pte))
1687                 --vcpu->kvm->stat.lpages;
1688 }
1689
1690 static void mmu_pte_write_new_pte(struct kvm_vcpu *vcpu,
1691                                   struct kvm_mmu_page *sp,
1692                                   u64 *spte,
1693                                   const void *new)
1694 {
1695         if (sp->role.level != PT_PAGE_TABLE_LEVEL) {
1696                 if (!vcpu->arch.update_pte.largepage ||
1697                     sp->role.glevels == PT32_ROOT_LEVEL) {
1698                         ++vcpu->kvm->stat.mmu_pde_zapped;
1699                         return;
1700                 }
1701         }
1702
1703         ++vcpu->kvm->stat.mmu_pte_updated;
1704         if (sp->role.glevels == PT32_ROOT_LEVEL)
1705                 paging32_update_pte(vcpu, sp, spte, new);
1706         else
1707                 paging64_update_pte(vcpu, sp, spte, new);
1708 }
1709
1710 static bool need_remote_flush(u64 old, u64 new)
1711 {
1712         if (!is_shadow_present_pte(old))
1713                 return false;
1714         if (!is_shadow_present_pte(new))
1715                 return true;
1716         if ((old ^ new) & PT64_BASE_ADDR_MASK)
1717                 return true;
1718         old ^= PT64_NX_MASK;
1719         new ^= PT64_NX_MASK;
1720         return (old & ~new & PT64_PERM_MASK) != 0;
1721 }
1722
1723 static void mmu_pte_write_flush_tlb(struct kvm_vcpu *vcpu, u64 old, u64 new)
1724 {
1725         if (need_remote_flush(old, new))
1726                 kvm_flush_remote_tlbs(vcpu->kvm);
1727         else
1728                 kvm_mmu_flush_tlb(vcpu);
1729 }
1730
1731 static bool last_updated_pte_accessed(struct kvm_vcpu *vcpu)
1732 {
1733         u64 *spte = vcpu->arch.last_pte_updated;
1734
1735         return !!(spte && (*spte & shadow_accessed_mask));
1736 }
1737
1738 static void mmu_guess_page_from_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa,
1739                                           const u8 *new, int bytes)
1740 {
1741         gfn_t gfn;
1742         int r;
1743         u64 gpte = 0;
1744         pfn_t pfn;
1745
1746         vcpu->arch.update_pte.largepage = 0;
1747
1748         if (bytes != 4 && bytes != 8)
1749                 return;
1750
1751         /*
1752          * Assume that the pte write on a page table of the same type
1753          * as the current vcpu paging mode.  This is nearly always true
1754          * (might be false while changing modes).  Note it is verified later
1755          * by update_pte().
1756          */
1757         if (is_pae(vcpu)) {
1758                 /* Handle a 32-bit guest writing two halves of a 64-bit gpte */
1759                 if ((bytes == 4) && (gpa % 4 == 0)) {
1760                         r = kvm_read_guest(vcpu->kvm, gpa & ~(u64)7, &gpte, 8);
1761                         if (r)
1762                                 return;
1763                         memcpy((void *)&gpte + (gpa % 8), new, 4);
1764                 } else if ((bytes == 8) && (gpa % 8 == 0)) {
1765                         memcpy((void *)&gpte, new, 8);
1766                 }
1767         } else {
1768                 if ((bytes == 4) && (gpa % 4 == 0))
1769                         memcpy((void *)&gpte, new, 4);
1770         }
1771         if (!is_present_pte(gpte))
1772                 return;
1773         gfn = (gpte & PT64_BASE_ADDR_MASK) >> PAGE_SHIFT;
1774
1775         down_read(&current->mm->mmap_sem);
1776         if (is_large_pte(gpte) && is_largepage_backed(vcpu, gfn)) {
1777                 gfn &= ~(KVM_PAGES_PER_HPAGE-1);
1778                 vcpu->arch.update_pte.largepage = 1;
1779         }
1780         vcpu->arch.update_pte.mmu_seq = vcpu->kvm->mmu_notifier_seq;
1781         /* implicit mb(), we'll read before PT lock is unlocked */
1782         pfn = gfn_to_pfn(vcpu->kvm, gfn);
1783         up_read(&current->mm->mmap_sem);
1784
1785         if (is_error_pfn(pfn)) {
1786                 kvm_release_pfn_clean(pfn);
1787                 return;
1788         }
1789         vcpu->arch.update_pte.gfn = gfn;
1790         vcpu->arch.update_pte.pfn = pfn;
1791 }
1792
1793 static void kvm_mmu_access_page(struct kvm_vcpu *vcpu, gfn_t gfn)
1794 {
1795         u64 *spte = vcpu->arch.last_pte_updated;
1796
1797         if (spte
1798             && vcpu->arch.last_pte_gfn == gfn
1799             && shadow_accessed_mask
1800             && !(*spte & shadow_accessed_mask)
1801             && is_shadow_present_pte(*spte))
1802                 set_bit(PT_ACCESSED_SHIFT, (unsigned long *)spte);
1803 }
1804
1805 void kvm_mmu_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa,
1806                        const u8 *new, int bytes)
1807 {
1808         gfn_t gfn = gpa >> PAGE_SHIFT;
1809         struct kvm_mmu_page *sp;
1810         struct hlist_node *node, *n;
1811         struct hlist_head *bucket;
1812         unsigned index;
1813         u64 entry, gentry;
1814         u64 *spte;
1815         unsigned offset = offset_in_page(gpa);
1816         unsigned pte_size;
1817         unsigned page_offset;
1818         unsigned misaligned;
1819         unsigned quadrant;
1820         int level;
1821         int flooded = 0;
1822         int npte;
1823         int r;
1824
1825         pgprintk("%s: gpa %llx bytes %d\n", __func__, gpa, bytes);
1826         mmu_guess_page_from_pte_write(vcpu, gpa, new, bytes);
1827         spin_lock(&vcpu->kvm->mmu_lock);
1828         kvm_mmu_access_page(vcpu, gfn);
1829         kvm_mmu_free_some_pages(vcpu);
1830         ++vcpu->kvm->stat.mmu_pte_write;
1831         kvm_mmu_audit(vcpu, "pre pte write");
1832         if (gfn == vcpu->arch.last_pt_write_gfn
1833             && !last_updated_pte_accessed(vcpu)) {
1834                 ++vcpu->arch.last_pt_write_count;
1835                 if (vcpu->arch.last_pt_write_count >= 3)
1836                         flooded = 1;
1837         } else {
1838                 vcpu->arch.last_pt_write_gfn = gfn;
1839                 vcpu->arch.last_pt_write_count = 1;
1840                 vcpu->arch.last_pte_updated = NULL;
1841         }
1842         index = kvm_page_table_hashfn(gfn);
1843         bucket = &vcpu->kvm->arch.mmu_page_hash[index];
1844         hlist_for_each_entry_safe(sp, node, n, bucket, hash_link) {
1845                 if (sp->gfn != gfn || sp->role.metaphysical)
1846                         continue;
1847                 pte_size = sp->role.glevels == PT32_ROOT_LEVEL ? 4 : 8;
1848                 misaligned = (offset ^ (offset + bytes - 1)) & ~(pte_size - 1);
1849                 misaligned |= bytes < 4;
1850                 if (misaligned || flooded) {
1851                         /*
1852                          * Misaligned accesses are too much trouble to fix
1853                          * up; also, they usually indicate a page is not used
1854                          * as a page table.
1855                          *
1856                          * If we're seeing too many writes to a page,
1857                          * it may no longer be a page table, or we may be
1858                          * forking, in which case it is better to unmap the
1859                          * page.
1860                          */
1861                         pgprintk("misaligned: gpa %llx bytes %d role %x\n",
1862                                  gpa, bytes, sp->role.word);
1863                         kvm_mmu_zap_page(vcpu->kvm, sp);
1864                         ++vcpu->kvm->stat.mmu_flooded;
1865                         continue;
1866                 }
1867                 page_offset = offset;
1868                 level = sp->role.level;
1869                 npte = 1;
1870                 if (sp->role.glevels == PT32_ROOT_LEVEL) {
1871                         page_offset <<= 1;      /* 32->64 */
1872                         /*
1873                          * A 32-bit pde maps 4MB while the shadow pdes map
1874                          * only 2MB.  So we need to double the offset again
1875                          * and zap two pdes instead of one.
1876                          */
1877                         if (level == PT32_ROOT_LEVEL) {
1878                                 page_offset &= ~7; /* kill rounding error */
1879                                 page_offset <<= 1;
1880                                 npte = 2;
1881                         }
1882                         quadrant = page_offset >> PAGE_SHIFT;
1883                         page_offset &= ~PAGE_MASK;
1884                         if (quadrant != sp->role.quadrant)
1885                                 continue;
1886                 }
1887                 spte = &sp->spt[page_offset / sizeof(*spte)];
1888                 if ((gpa & (pte_size - 1)) || (bytes < pte_size)) {
1889                         gentry = 0;
1890                         r = kvm_read_guest_atomic(vcpu->kvm,
1891                                                   gpa & ~(u64)(pte_size - 1),
1892                                                   &gentry, pte_size);
1893                         new = (const void *)&gentry;
1894                         if (r < 0)
1895                                 new = NULL;
1896                 }
1897                 while (npte--) {
1898                         entry = *spte;
1899                         mmu_pte_write_zap_pte(vcpu, sp, spte);
1900                         if (new)
1901                                 mmu_pte_write_new_pte(vcpu, sp, spte, new);
1902                         mmu_pte_write_flush_tlb(vcpu, entry, *spte);
1903                         ++spte;
1904                 }
1905         }
1906         kvm_mmu_audit(vcpu, "post pte write");
1907         spin_unlock(&vcpu->kvm->mmu_lock);
1908         if (!is_error_pfn(vcpu->arch.update_pte.pfn)) {
1909                 kvm_release_pfn_clean(vcpu->arch.update_pte.pfn);
1910                 vcpu->arch.update_pte.pfn = bad_pfn;
1911         }
1912 }
1913
1914 int kvm_mmu_unprotect_page_virt(struct kvm_vcpu *vcpu, gva_t gva)
1915 {
1916         gpa_t gpa;
1917         int r;
1918
1919         gpa = vcpu->arch.mmu.gva_to_gpa(vcpu, gva);
1920
1921         spin_lock(&vcpu->kvm->mmu_lock);
1922         r = kvm_mmu_unprotect_page(vcpu->kvm, gpa >> PAGE_SHIFT);
1923         spin_unlock(&vcpu->kvm->mmu_lock);
1924         return r;
1925 }
1926 EXPORT_SYMBOL_GPL(kvm_mmu_unprotect_page_virt);
1927
1928 void __kvm_mmu_free_some_pages(struct kvm_vcpu *vcpu)
1929 {
1930         while (vcpu->kvm->arch.n_free_mmu_pages < KVM_REFILL_PAGES) {
1931                 struct kvm_mmu_page *sp;
1932
1933                 sp = container_of(vcpu->kvm->arch.active_mmu_pages.prev,
1934                                   struct kvm_mmu_page, link);
1935                 kvm_mmu_zap_page(vcpu->kvm, sp);
1936                 ++vcpu->kvm->stat.mmu_recycled;
1937         }
1938 }
1939
1940 int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gva_t cr2, u32 error_code)
1941 {
1942         int r;
1943         enum emulation_result er;
1944
1945         r = vcpu->arch.mmu.page_fault(vcpu, cr2, error_code);
1946         if (r < 0)
1947                 goto out;
1948
1949         if (!r) {
1950                 r = 1;
1951                 goto out;
1952         }
1953
1954         r = mmu_topup_memory_caches(vcpu);
1955         if (r)
1956                 goto out;
1957
1958         er = emulate_instruction(vcpu, vcpu->run, cr2, error_code, 0);
1959
1960         switch (er) {
1961         case EMULATE_DONE:
1962                 return 1;
1963         case EMULATE_DO_MMIO:
1964                 ++vcpu->stat.mmio_exits;
1965                 return 0;
1966         case EMULATE_FAIL:
1967                 kvm_report_emulation_failure(vcpu, "pagetable");
1968                 return 1;
1969         default:
1970                 BUG();
1971         }
1972 out:
1973         return r;
1974 }
1975 EXPORT_SYMBOL_GPL(kvm_mmu_page_fault);
1976
1977 void kvm_enable_tdp(void)
1978 {
1979         tdp_enabled = true;
1980 }
1981 EXPORT_SYMBOL_GPL(kvm_enable_tdp);
1982
1983 void kvm_disable_tdp(void)
1984 {
1985         tdp_enabled = false;
1986 }
1987 EXPORT_SYMBOL_GPL(kvm_disable_tdp);
1988
1989 static void free_mmu_pages(struct kvm_vcpu *vcpu)
1990 {
1991         struct kvm_mmu_page *sp;
1992
1993         while (!list_empty(&vcpu->kvm->arch.active_mmu_pages)) {
1994                 sp = container_of(vcpu->kvm->arch.active_mmu_pages.next,
1995                                   struct kvm_mmu_page, link);
1996                 kvm_mmu_zap_page(vcpu->kvm, sp);
1997                 cond_resched();
1998         }
1999         free_page((unsigned long)vcpu->arch.mmu.pae_root);
2000 }
2001
2002 static int alloc_mmu_pages(struct kvm_vcpu *vcpu)
2003 {
2004         struct page *page;
2005         int i;
2006
2007         ASSERT(vcpu);
2008
2009         if (vcpu->kvm->arch.n_requested_mmu_pages)
2010                 vcpu->kvm->arch.n_free_mmu_pages =
2011                                         vcpu->kvm->arch.n_requested_mmu_pages;
2012         else
2013                 vcpu->kvm->arch.n_free_mmu_pages =
2014                                         vcpu->kvm->arch.n_alloc_mmu_pages;
2015         /*
2016          * When emulating 32-bit mode, cr3 is only 32 bits even on x86_64.
2017          * Therefore we need to allocate shadow page tables in the first
2018          * 4GB of memory, which happens to fit the DMA32 zone.
2019          */
2020         page = alloc_page(GFP_KERNEL | __GFP_DMA32);
2021         if (!page)
2022                 goto error_1;
2023         vcpu->arch.mmu.pae_root = page_address(page);
2024         for (i = 0; i < 4; ++i)
2025                 vcpu->arch.mmu.pae_root[i] = INVALID_PAGE;
2026
2027         return 0;
2028
2029 error_1:
2030         free_mmu_pages(vcpu);
2031         return -ENOMEM;
2032 }
2033
2034 int kvm_mmu_create(struct kvm_vcpu *vcpu)
2035 {
2036         ASSERT(vcpu);
2037         ASSERT(!VALID_PAGE(vcpu->arch.mmu.root_hpa));
2038
2039         return alloc_mmu_pages(vcpu);
2040 }
2041
2042 int kvm_mmu_setup(struct kvm_vcpu *vcpu)
2043 {
2044         ASSERT(vcpu);
2045         ASSERT(!VALID_PAGE(vcpu->arch.mmu.root_hpa));
2046
2047         return init_kvm_mmu(vcpu);
2048 }
2049
2050 void kvm_mmu_destroy(struct kvm_vcpu *vcpu)
2051 {
2052         ASSERT(vcpu);
2053
2054         destroy_kvm_mmu(vcpu);
2055         free_mmu_pages(vcpu);
2056         mmu_free_memory_caches(vcpu);
2057 }
2058
2059 void kvm_mmu_slot_remove_write_access(struct kvm *kvm, int slot)
2060 {
2061         struct kvm_mmu_page *sp;
2062
2063         list_for_each_entry(sp, &kvm->arch.active_mmu_pages, link) {
2064                 int i;
2065                 u64 *pt;
2066
2067                 if (!test_bit(slot, &sp->slot_bitmap))
2068                         continue;
2069
2070                 pt = sp->spt;
2071                 for (i = 0; i < PT64_ENT_PER_PAGE; ++i)
2072                         /* avoid RMW */
2073                         if (pt[i] & PT_WRITABLE_MASK)
2074                                 pt[i] &= ~PT_WRITABLE_MASK;
2075         }
2076 }
2077
2078 void kvm_mmu_zap_all(struct kvm *kvm)
2079 {
2080         struct kvm_mmu_page *sp, *node;
2081
2082         spin_lock(&kvm->mmu_lock);
2083         list_for_each_entry_safe(sp, node, &kvm->arch.active_mmu_pages, link)
2084                 kvm_mmu_zap_page(kvm, sp);
2085         spin_unlock(&kvm->mmu_lock);
2086
2087         kvm_flush_remote_tlbs(kvm);
2088 }
2089
2090 static void kvm_mmu_remove_one_alloc_mmu_page(struct kvm *kvm)
2091 {
2092         struct kvm_mmu_page *page;
2093
2094         page = container_of(kvm->arch.active_mmu_pages.prev,
2095                             struct kvm_mmu_page, link);
2096         kvm_mmu_zap_page(kvm, page);
2097 }
2098
2099 static int mmu_shrink(int nr_to_scan, gfp_t gfp_mask)
2100 {
2101         struct kvm *kvm;
2102         struct kvm *kvm_freed = NULL;
2103         int cache_count = 0;
2104
2105         spin_lock(&kvm_lock);
2106
2107         list_for_each_entry(kvm, &vm_list, vm_list) {
2108                 int npages;
2109
2110                 if (!down_read_trylock(&kvm->slots_lock))
2111                         continue;
2112                 spin_lock(&kvm->mmu_lock);
2113                 npages = kvm->arch.n_alloc_mmu_pages -
2114                          kvm->arch.n_free_mmu_pages;
2115                 cache_count += npages;
2116                 if (!kvm_freed && nr_to_scan > 0 && npages > 0) {
2117                         kvm_mmu_remove_one_alloc_mmu_page(kvm);
2118                         cache_count--;
2119                         kvm_freed = kvm;
2120                 }
2121                 nr_to_scan--;
2122
2123                 spin_unlock(&kvm->mmu_lock);
2124                 up_read(&kvm->slots_lock);
2125         }
2126         if (kvm_freed)
2127                 list_move_tail(&kvm_freed->vm_list, &vm_list);
2128
2129         spin_unlock(&kvm_lock);
2130
2131         return cache_count;
2132 }
2133
2134 static struct shrinker mmu_shrinker = {
2135         .shrink = mmu_shrink,
2136         .seeks = DEFAULT_SEEKS * 10,
2137 };
2138
2139 static void mmu_destroy_caches(void)
2140 {
2141         if (pte_chain_cache)
2142                 kmem_cache_destroy(pte_chain_cache);
2143         if (rmap_desc_cache)
2144                 kmem_cache_destroy(rmap_desc_cache);
2145         if (mmu_page_header_cache)
2146                 kmem_cache_destroy(mmu_page_header_cache);
2147 }
2148
2149 void kvm_mmu_module_exit(void)
2150 {
2151         mmu_destroy_caches();
2152         unregister_shrinker(&mmu_shrinker);
2153 }
2154
2155 int kvm_mmu_module_init(void)
2156 {
2157         pte_chain_cache = kmem_cache_create("kvm_pte_chain",
2158                                             sizeof(struct kvm_pte_chain),
2159                                             0, 0, NULL);
2160         if (!pte_chain_cache)
2161                 goto nomem;
2162         rmap_desc_cache = kmem_cache_create("kvm_rmap_desc",
2163                                             sizeof(struct kvm_rmap_desc),
2164                                             0, 0, NULL);
2165         if (!rmap_desc_cache)
2166                 goto nomem;
2167
2168         mmu_page_header_cache = kmem_cache_create("kvm_mmu_page_header",
2169                                                   sizeof(struct kvm_mmu_page),
2170                                                   0, 0, NULL);
2171         if (!mmu_page_header_cache)
2172                 goto nomem;
2173
2174         register_shrinker(&mmu_shrinker);
2175
2176         return 0;
2177
2178 nomem:
2179         mmu_destroy_caches();
2180         return -ENOMEM;
2181 }
2182
2183 /*
2184  * Caculate mmu pages needed for kvm.
2185  */
2186 unsigned int kvm_mmu_calculate_mmu_pages(struct kvm *kvm)
2187 {
2188         int i;
2189         unsigned int nr_mmu_pages;
2190         unsigned int  nr_pages = 0;
2191
2192         for (i = 0; i < kvm->nmemslots; i++)
2193                 nr_pages += kvm->memslots[i].npages;
2194
2195         nr_mmu_pages = nr_pages * KVM_PERMILLE_MMU_PAGES / 1000;
2196         nr_mmu_pages = max(nr_mmu_pages,
2197                         (unsigned int) KVM_MIN_ALLOC_MMU_PAGES);
2198
2199         return nr_mmu_pages;
2200 }
2201
2202 static void *pv_mmu_peek_buffer(struct kvm_pv_mmu_op_buffer *buffer,
2203                                 unsigned len)
2204 {
2205         if (len > buffer->len)
2206                 return NULL;
2207         return buffer->ptr;
2208 }
2209
2210 static void *pv_mmu_read_buffer(struct kvm_pv_mmu_op_buffer *buffer,
2211                                 unsigned len)
2212 {
2213         void *ret;
2214
2215         ret = pv_mmu_peek_buffer(buffer, len);
2216         if (!ret)
2217                 return ret;
2218         buffer->ptr += len;
2219         buffer->len -= len;
2220         buffer->processed += len;
2221         return ret;
2222 }
2223
2224 static int kvm_pv_mmu_write(struct kvm_vcpu *vcpu,
2225                              gpa_t addr, gpa_t value)
2226 {
2227         int bytes = 8;
2228         int r;
2229
2230         if (!is_long_mode(vcpu) && !is_pae(vcpu))
2231                 bytes = 4;
2232
2233         r = mmu_topup_memory_caches(vcpu);
2234         if (r)
2235                 return r;
2236
2237         if (!emulator_write_phys(vcpu, addr, &value, bytes))
2238                 return -EFAULT;
2239
2240         return 1;
2241 }
2242
2243 static int kvm_pv_mmu_flush_tlb(struct kvm_vcpu *vcpu)
2244 {
2245         kvm_x86_ops->tlb_flush(vcpu);
2246         return 1;
2247 }
2248
2249 static int kvm_pv_mmu_release_pt(struct kvm_vcpu *vcpu, gpa_t addr)
2250 {
2251         spin_lock(&vcpu->kvm->mmu_lock);
2252         mmu_unshadow(vcpu->kvm, addr >> PAGE_SHIFT);
2253         spin_unlock(&vcpu->kvm->mmu_lock);
2254         return 1;
2255 }
2256
2257 static int kvm_pv_mmu_op_one(struct kvm_vcpu *vcpu,
2258                              struct kvm_pv_mmu_op_buffer *buffer)
2259 {
2260         struct kvm_mmu_op_header *header;
2261
2262         header = pv_mmu_peek_buffer(buffer, sizeof *header);
2263         if (!header)
2264                 return 0;
2265         switch (header->op) {
2266         case KVM_MMU_OP_WRITE_PTE: {
2267                 struct kvm_mmu_op_write_pte *wpte;
2268
2269                 wpte = pv_mmu_read_buffer(buffer, sizeof *wpte);
2270                 if (!wpte)
2271                         return 0;
2272                 return kvm_pv_mmu_write(vcpu, wpte->pte_phys,
2273                                         wpte->pte_val);
2274         }
2275         case KVM_MMU_OP_FLUSH_TLB: {
2276                 struct kvm_mmu_op_flush_tlb *ftlb;
2277
2278                 ftlb = pv_mmu_read_buffer(buffer, sizeof *ftlb);
2279                 if (!ftlb)
2280                         return 0;
2281                 return kvm_pv_mmu_flush_tlb(vcpu);
2282         }
2283         case KVM_MMU_OP_RELEASE_PT: {
2284                 struct kvm_mmu_op_release_pt *rpt;
2285
2286                 rpt = pv_mmu_read_buffer(buffer, sizeof *rpt);
2287                 if (!rpt)
2288                         return 0;
2289                 return kvm_pv_mmu_release_pt(vcpu, rpt->pt_phys);
2290         }
2291         default: return 0;
2292         }
2293 }
2294
2295 int kvm_pv_mmu_op(struct kvm_vcpu *vcpu, unsigned long bytes,
2296                   gpa_t addr, unsigned long *ret)
2297 {
2298         int r;
2299         struct kvm_pv_mmu_op_buffer buffer;
2300
2301         buffer.ptr = buffer.buf;
2302         buffer.len = min_t(unsigned long, bytes, sizeof buffer.buf);
2303         buffer.processed = 0;
2304
2305         r = kvm_read_guest(vcpu->kvm, addr, buffer.buf, buffer.len);
2306         if (r)
2307                 goto out;
2308
2309         while (buffer.len) {
2310                 r = kvm_pv_mmu_op_one(vcpu, &buffer);
2311                 if (r < 0)
2312                         goto out;
2313                 if (r == 0)
2314                         break;
2315         }
2316
2317         r = 1;
2318 out:
2319         *ret = buffer.processed;
2320         return r;
2321 }
2322
2323 #ifdef AUDIT
2324
2325 static const char *audit_msg;
2326
2327 static gva_t canonicalize(gva_t gva)
2328 {
2329 #ifdef CONFIG_X86_64
2330         gva = (long long)(gva << 16) >> 16;
2331 #endif
2332         return gva;
2333 }
2334
2335 static void audit_mappings_page(struct kvm_vcpu *vcpu, u64 page_pte,
2336                                 gva_t va, int level)
2337 {
2338         u64 *pt = __va(page_pte & PT64_BASE_ADDR_MASK);
2339         int i;
2340         gva_t va_delta = 1ul << (PAGE_SHIFT + 9 * (level - 1));
2341
2342         for (i = 0; i < PT64_ENT_PER_PAGE; ++i, va += va_delta) {
2343                 u64 ent = pt[i];
2344
2345                 if (ent == shadow_trap_nonpresent_pte)
2346                         continue;
2347
2348                 va = canonicalize(va);
2349                 if (level > 1) {
2350                         if (ent == shadow_notrap_nonpresent_pte)
2351                                 printk(KERN_ERR "audit: (%s) nontrapping pte"
2352                                        " in nonleaf level: levels %d gva %lx"
2353                                        " level %d pte %llx\n", audit_msg,
2354                                        vcpu->arch.mmu.root_level, va, level, ent);
2355
2356                         audit_mappings_page(vcpu, ent, va, level - 1);
2357                 } else {
2358                         gpa_t gpa = vcpu->arch.mmu.gva_to_gpa(vcpu, va);
2359                         hpa_t hpa = (hpa_t)gpa_to_pfn(vcpu, gpa) << PAGE_SHIFT;
2360
2361                         if (is_shadow_present_pte(ent)
2362                             && (ent & PT64_BASE_ADDR_MASK) != hpa)
2363                                 printk(KERN_ERR "xx audit error: (%s) levels %d"
2364                                        " gva %lx gpa %llx hpa %llx ent %llx %d\n",
2365                                        audit_msg, vcpu->arch.mmu.root_level,
2366                                        va, gpa, hpa, ent,
2367                                        is_shadow_present_pte(ent));
2368                         else if (ent == shadow_notrap_nonpresent_pte
2369                                  && !is_error_hpa(hpa))
2370                                 printk(KERN_ERR "audit: (%s) notrap shadow,"
2371                                        " valid guest gva %lx\n", audit_msg, va);
2372                         kvm_release_pfn_clean(pfn);
2373
2374                 }
2375         }
2376 }
2377
2378 static void audit_mappings(struct kvm_vcpu *vcpu)
2379 {
2380         unsigned i;
2381
2382         if (vcpu->arch.mmu.root_level == 4)
2383                 audit_mappings_page(vcpu, vcpu->arch.mmu.root_hpa, 0, 4);
2384         else
2385                 for (i = 0; i < 4; ++i)
2386                         if (vcpu->arch.mmu.pae_root[i] & PT_PRESENT_MASK)
2387                                 audit_mappings_page(vcpu,
2388                                                     vcpu->arch.mmu.pae_root[i],
2389                                                     i << 30,
2390                                                     2);
2391 }
2392
2393 static int count_rmaps(struct kvm_vcpu *vcpu)
2394 {
2395         int nmaps = 0;
2396         int i, j, k;
2397
2398         for (i = 0; i < KVM_MEMORY_SLOTS; ++i) {
2399                 struct kvm_memory_slot *m = &vcpu->kvm->memslots[i];
2400                 struct kvm_rmap_desc *d;
2401
2402                 for (j = 0; j < m->npages; ++j) {
2403                         unsigned long *rmapp = &m->rmap[j];
2404
2405                         if (!*rmapp)
2406                                 continue;
2407                         if (!(*rmapp & 1)) {
2408                                 ++nmaps;
2409                                 continue;
2410                         }
2411                         d = (struct kvm_rmap_desc *)(*rmapp & ~1ul);
2412                         while (d) {
2413                                 for (k = 0; k < RMAP_EXT; ++k)
2414                                         if (d->shadow_ptes[k])
2415                                                 ++nmaps;
2416                                         else
2417                                                 break;
2418                                 d = d->more;
2419                         }
2420                 }
2421         }
2422         return nmaps;
2423 }
2424
2425 static int count_writable_mappings(struct kvm_vcpu *vcpu)
2426 {
2427         int nmaps = 0;
2428         struct kvm_mmu_page *sp;
2429         int i;
2430
2431         list_for_each_entry(sp, &vcpu->kvm->arch.active_mmu_pages, link) {
2432                 u64 *pt = sp->spt;
2433
2434                 if (sp->role.level != PT_PAGE_TABLE_LEVEL)
2435                         continue;
2436
2437                 for (i = 0; i < PT64_ENT_PER_PAGE; ++i) {
2438                         u64 ent = pt[i];
2439
2440                         if (!(ent & PT_PRESENT_MASK))
2441                                 continue;
2442                         if (!(ent & PT_WRITABLE_MASK))
2443                                 continue;
2444                         ++nmaps;
2445                 }
2446         }
2447         return nmaps;
2448 }
2449
2450 static void audit_rmap(struct kvm_vcpu *vcpu)
2451 {
2452         int n_rmap = count_rmaps(vcpu);
2453         int n_actual = count_writable_mappings(vcpu);
2454
2455         if (n_rmap != n_actual)
2456                 printk(KERN_ERR "%s: (%s) rmap %d actual %d\n",
2457                        __func__, audit_msg, n_rmap, n_actual);
2458 }
2459
2460 static void audit_write_protection(struct kvm_vcpu *vcpu)
2461 {
2462         struct kvm_mmu_page *sp;
2463         struct kvm_memory_slot *slot;
2464         unsigned long *rmapp;
2465         gfn_t gfn;
2466
2467         list_for_each_entry(sp, &vcpu->kvm->arch.active_mmu_pages, link) {
2468                 if (sp->role.metaphysical)
2469                         continue;
2470
2471                 slot = gfn_to_memslot(vcpu->kvm, sp->gfn);
2472                 gfn = unalias_gfn(vcpu->kvm, sp->gfn);
2473                 rmapp = &slot->rmap[gfn - slot->base_gfn];
2474                 if (*rmapp)
2475                         printk(KERN_ERR "%s: (%s) shadow page has writable"
2476                                " mappings: gfn %lx role %x\n",
2477                                __func__, audit_msg, sp->gfn,
2478                                sp->role.word);
2479         }
2480 }
2481
2482 static void kvm_mmu_audit(struct kvm_vcpu *vcpu, const char *msg)
2483 {
2484         int olddbg = dbg;
2485
2486         dbg = 0;
2487         audit_msg = msg;
2488         audit_rmap(vcpu);
2489         audit_write_protection(vcpu);
2490         audit_mappings(vcpu);
2491         dbg = olddbg;
2492 }
2493
2494 #endif