git.oblomov.eu Git - linux-2.6/blob - drivers/pci/intel-iommu.c

   1 /*
   2  * Copyright (c) 2006, Intel Corporation.
   3  *
   4  * This program is free software; you can redistribute it and/or modify it
   5  * under the terms and conditions of the GNU General Public License,
   6  * version 2, as published by the Free Software Foundation.
   7  *
   8  * This program is distributed in the hope it will be useful, but WITHOUT
   9  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  10  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  11  * more details.
  12  *
  13  * You should have received a copy of the GNU General Public License along with
  14  * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
  15  * Place - Suite 330, Boston, MA 02111-1307 USA.
  16  *
  17  * Copyright (C) 2006-2008 Intel Corporation
  18  * Author: Ashok Raj <ashok.raj@intel.com>
  19  * Author: Shaohua Li <shaohua.li@intel.com>
  20  * Author: Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>
  21  * Author: Fenghua Yu <fenghua.yu@intel.com>
  22  */
  23
  24 #include <linux/init.h>
  25 #include <linux/bitmap.h>
  26 #include <linux/debugfs.h>
  27 #include <linux/slab.h>
  28 #include <linux/irq.h>
  29 #include <linux/interrupt.h>
  30 #include <linux/spinlock.h>
  31 #include <linux/pci.h>
  32 #include <linux/dmar.h>
  33 #include <linux/dma-mapping.h>
  34 #include <linux/mempool.h>
  35 #include <linux/timer.h>
  36 #include <linux/iova.h>
  37 #include <linux/iommu.h>
  38 #include <linux/intel-iommu.h>
  39 #include <linux/sysdev.h>
  40 #include <asm/cacheflush.h>
  41 #include <asm/iommu.h>
  42 #include "pci.h"
  43
  44 #define ROOT_SIZE               VTD_PAGE_SIZE
  45 #define CONTEXT_SIZE            VTD_PAGE_SIZE
  46
  47 #define IS_GFX_DEVICE(pdev) ((pdev->class >> 16) == PCI_BASE_CLASS_DISPLAY)
  48 #define IS_ISA_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_BRIDGE_ISA)
  49
  50 #define IOAPIC_RANGE_START      (0xfee00000)
  51 #define IOAPIC_RANGE_END        (0xfeefffff)
  52 #define IOVA_START_ADDR         (0x1000)
  53
  54 #define DEFAULT_DOMAIN_ADDRESS_WIDTH 48
  55
  56 #define DOMAIN_MAX_ADDR(gaw) ((((u64)1) << gaw) - 1)
  57
  58 #define IOVA_PFN(addr)          ((addr) >> PAGE_SHIFT)
  59 #define DMA_32BIT_PFN           IOVA_PFN(DMA_32BIT_MASK)
  60 #define DMA_64BIT_PFN           IOVA_PFN(DMA_64BIT_MASK)
  61
  62 /* global iommu list, set NULL for ignored DMAR units */
  63 static struct intel_iommu **g_iommus;
  64
  65 static int rwbf_quirk;
  66
  67 /*
  68  * 0: Present
  69  * 1-11: Reserved
  70  * 12-63: Context Ptr (12 - (haw-1))
  71  * 64-127: Reserved
  72  */
  73 struct root_entry {
  74         u64     val;
  75         u64     rsvd1;
  76 };
  77 #define ROOT_ENTRY_NR (VTD_PAGE_SIZE/sizeof(struct root_entry))
  78 static inline bool root_present(struct root_entry *root)
  79 {
  80         return (root->val & 1);
  81 }
  82 static inline void set_root_present(struct root_entry *root)
  83 {
  84         root->val |= 1;
  85 }
  86 static inline void set_root_value(struct root_entry *root, unsigned long value)
  87 {
  88         root->val |= value & VTD_PAGE_MASK;
  89 }
  90
  91 static inline struct context_entry *
  92 get_context_addr_from_root(struct root_entry *root)
  93 {
  94         return (struct context_entry *)
  95                 (root_present(root)?phys_to_virt(
  96                 root->val & VTD_PAGE_MASK) :
  97                 NULL);
  98 }
  99
 100 /*
 101  * low 64 bits:
 102  * 0: present
 103  * 1: fault processing disable
 104  * 2-3: translation type
 105  * 12-63: address space root
 106  * high 64 bits:
 107  * 0-2: address width
 108  * 3-6: aval
 109  * 8-23: domain id
 110  */
 111 struct context_entry {
 112         u64 lo;
 113         u64 hi;
 114 };
 115
 116 static inline bool context_present(struct context_entry *context)
 117 {
 118         return (context->lo & 1);
 119 }
 120 static inline void context_set_present(struct context_entry *context)
 121 {
 122         context->lo |= 1;
 123 }
 124
 125 static inline void context_set_fault_enable(struct context_entry *context)
 126 {
 127         context->lo &= (((u64)-1) << 2) | 1;
 128 }
 129
 130 #define CONTEXT_TT_MULTI_LEVEL 0
 131
 132 static inline void context_set_translation_type(struct context_entry *context,
 133                                                 unsigned long value)
 134 {
 135         context->lo &= (((u64)-1) << 4) | 3;
 136         context->lo |= (value & 3) << 2;
 137 }
 138
 139 static inline void context_set_address_root(struct context_entry *context,
 140                                             unsigned long value)
 141 {
 142         context->lo |= value & VTD_PAGE_MASK;
 143 }
 144
 145 static inline void context_set_address_width(struct context_entry *context,
 146                                              unsigned long value)
 147 {
 148         context->hi |= value & 7;
 149 }
 150
 151 static inline void context_set_domain_id(struct context_entry *context,
 152                                          unsigned long value)
 153 {
 154         context->hi |= (value & ((1 << 16) - 1)) << 8;
 155 }
 156
 157 static inline void context_clear_entry(struct context_entry *context)
 158 {
 159         context->lo = 0;
 160         context->hi = 0;
 161 }
 162
 163 /*
 164  * 0: readable
 165  * 1: writable
 166  * 2-6: reserved
 167  * 7: super page
 168  * 8-10: available
 169  * 11: snoop behavior
 170  * 12-63: Host physcial address
 171  */
 172 struct dma_pte {
 173         u64 val;
 174 };
 175
 176 static inline void dma_clear_pte(struct dma_pte *pte)
 177 {
 178         pte->val = 0;
 179 }
 180
 181 static inline void dma_set_pte_readable(struct dma_pte *pte)
 182 {
 183         pte->val |= DMA_PTE_READ;
 184 }
 185
 186 static inline void dma_set_pte_writable(struct dma_pte *pte)
 187 {
 188         pte->val |= DMA_PTE_WRITE;
 189 }
 190
 191 static inline void dma_set_pte_snp(struct dma_pte *pte)
 192 {
 193         pte->val |= DMA_PTE_SNP;
 194 }
 195
 196 static inline void dma_set_pte_prot(struct dma_pte *pte, unsigned long prot)
 197 {
 198         pte->val = (pte->val & ~3) | (prot & 3);
 199 }
 200
 201 static inline u64 dma_pte_addr(struct dma_pte *pte)
 202 {
 203         return (pte->val & VTD_PAGE_MASK);
 204 }
 205
 206 static inline void dma_set_pte_addr(struct dma_pte *pte, u64 addr)
 207 {
 208         pte->val |= (addr & VTD_PAGE_MASK);
 209 }
 210
 211 static inline bool dma_pte_present(struct dma_pte *pte)
 212 {
 213         return (pte->val & 3) != 0;
 214 }
 215
 216 /* devices under the same p2p bridge are owned in one domain */
 217 #define DOMAIN_FLAG_P2P_MULTIPLE_DEVICES (1 << 0)
 218
 219 /* domain represents a virtual machine, more than one devices
 220  * across iommus may be owned in one domain, e.g. kvm guest.
 221  */
 222 #define DOMAIN_FLAG_VIRTUAL_MACHINE     (1 << 1)
 223
 224 struct dmar_domain {
 225         int     id;                     /* domain id */
 226         unsigned long iommu_bmp;        /* bitmap of iommus this domain uses*/
 227
 228         struct list_head devices;       /* all devices' list */
 229         struct iova_domain iovad;       /* iova's that belong to this domain */
 230
 231         struct dma_pte  *pgd;           /* virtual address */
 232         spinlock_t      mapping_lock;   /* page table lock */
 233         int             gaw;            /* max guest address width */
 234
 235         /* adjusted guest address width, 0 is level 2 30-bit */
 236         int             agaw;
 237
 238         int             flags;          /* flags to find out type of domain */
 239
 240         int             iommu_coherency;/* indicate coherency of iommu access */
 241         int             iommu_snooping; /* indicate snooping control feature*/
 242         int             iommu_count;    /* reference count of iommu */
 243         spinlock_t      iommu_lock;     /* protect iommu set in domain */
 244         u64             max_addr;       /* maximum mapped address */
 245 };
 246
 247 /* PCI domain-device relationship */
 248 struct device_domain_info {
 249         struct list_head link;  /* link to domain siblings */
 250         struct list_head global; /* link to global list */
 251         u8 bus;                 /* PCI bus numer */
 252         u8 devfn;               /* PCI devfn number */
 253         struct pci_dev *dev; /* it's NULL for PCIE-to-PCI bridge */
 254         struct dmar_domain *domain; /* pointer to domain */
 255 };
 256
 257 static void flush_unmaps_timeout(unsigned long data);
 258
 259 DEFINE_TIMER(unmap_timer,  flush_unmaps_timeout, 0, 0);
 260
 261 #define HIGH_WATER_MARK 250
 262 struct deferred_flush_tables {
 263         int next;
 264         struct iova *iova[HIGH_WATER_MARK];
 265         struct dmar_domain *domain[HIGH_WATER_MARK];
 266 };
 267
 268 static struct deferred_flush_tables *deferred_flush;
 269
 270 /* bitmap for indexing intel_iommus */
 271 static int g_num_of_iommus;
 272
 273 static DEFINE_SPINLOCK(async_umap_flush_lock);
 274 static LIST_HEAD(unmaps_to_do);
 275
 276 static int timer_on;
 277 static long list_size;
 278
 279 static void domain_remove_dev_info(struct dmar_domain *domain);
 280
 281 #ifdef CONFIG_DMAR_DEFAULT_ON
 282 int dmar_disabled = 0;
 283 #else
 284 int dmar_disabled = 1;
 285 #endif /*CONFIG_DMAR_DEFAULT_ON*/
 286
 287 static int __initdata dmar_map_gfx = 1;
 288 static int dmar_forcedac;
 289 static int intel_iommu_strict;
 290
 291 #define DUMMY_DEVICE_DOMAIN_INFO ((struct device_domain_info *)(-1))
 292 static DEFINE_SPINLOCK(device_domain_lock);
 293 static LIST_HEAD(device_domain_list);
 294
 295 static struct iommu_ops intel_iommu_ops;
 296
 297 static int __init intel_iommu_setup(char *str)
 298 {
 299         if (!str)
 300                 return -EINVAL;
 301         while (*str) {
 302                 if (!strncmp(str, "on", 2)) {
 303                         dmar_disabled = 0;
 304                         printk(KERN_INFO "Intel-IOMMU: enabled\n");
 305                 } else if (!strncmp(str, "off", 3)) {
 306                         dmar_disabled = 1;
 307                         printk(KERN_INFO "Intel-IOMMU: disabled\n");
 308                 } else if (!strncmp(str, "igfx_off", 8)) {
 309                         dmar_map_gfx = 0;
 310                         printk(KERN_INFO
 311                                 "Intel-IOMMU: disable GFX device mapping\n");
 312                 } else if (!strncmp(str, "forcedac", 8)) {
 313                         printk(KERN_INFO
 314                                 "Intel-IOMMU: Forcing DAC for PCI devices\n");
 315                         dmar_forcedac = 1;
 316                 } else if (!strncmp(str, "strict", 6)) {
 317                         printk(KERN_INFO
 318                                 "Intel-IOMMU: disable batched IOTLB flush\n");
 319                         intel_iommu_strict = 1;
 320                 }
 321
 322                 str += strcspn(str, ",");
 323                 while (*str == ',')
 324                         str++;
 325         }
 326         return 0;
 327 }
 328 __setup("intel_iommu=", intel_iommu_setup);
 329
 330 static struct kmem_cache *iommu_domain_cache;
 331 static struct kmem_cache *iommu_devinfo_cache;
 332 static struct kmem_cache *iommu_iova_cache;
 333
 334 static inline void *iommu_kmem_cache_alloc(struct kmem_cache *cachep)
 335 {
 336         unsigned int flags;
 337         void *vaddr;
 338
 339         /* trying to avoid low memory issues */
 340         flags = current->flags & PF_MEMALLOC;
 341         current->flags |= PF_MEMALLOC;
 342         vaddr = kmem_cache_alloc(cachep, GFP_ATOMIC);
 343         current->flags &= (~PF_MEMALLOC | flags);
 344         return vaddr;
 345 }
 346
 347
 348 static inline void *alloc_pgtable_page(void)
 349 {
 350         unsigned int flags;
 351         void *vaddr;
 352
 353         /* trying to avoid low memory issues */
 354         flags = current->flags & PF_MEMALLOC;
 355         current->flags |= PF_MEMALLOC;
 356         vaddr = (void *)get_zeroed_page(GFP_ATOMIC);
 357         current->flags &= (~PF_MEMALLOC | flags);
 358         return vaddr;
 359 }
 360
 361 static inline void free_pgtable_page(void *vaddr)
 362 {
 363         free_page((unsigned long)vaddr);
 364 }
 365
 366 static inline void *alloc_domain_mem(void)
 367 {
 368         return iommu_kmem_cache_alloc(iommu_domain_cache);
 369 }
 370
 371 static void free_domain_mem(void *vaddr)
 372 {
 373         kmem_cache_free(iommu_domain_cache, vaddr);
 374 }
 375
 376 static inline void * alloc_devinfo_mem(void)
 377 {
 378         return iommu_kmem_cache_alloc(iommu_devinfo_cache);
 379 }
 380
 381 static inline void free_devinfo_mem(void *vaddr)
 382 {
 383         kmem_cache_free(iommu_devinfo_cache, vaddr);
 384 }
 385
 386 struct iova *alloc_iova_mem(void)
 387 {
 388         return iommu_kmem_cache_alloc(iommu_iova_cache);
 389 }
 390
 391 void free_iova_mem(struct iova *iova)
 392 {
 393         kmem_cache_free(iommu_iova_cache, iova);
 394 }
 395
 396
 397 static inline int width_to_agaw(int width);
 398
 399 /* calculate agaw for each iommu.
 400  * "SAGAW" may be different across iommus, use a default agaw, and
 401  * get a supported less agaw for iommus that don't support the default agaw.
 402  */
 403 int iommu_calculate_agaw(struct intel_iommu *iommu)
 404 {
 405         unsigned long sagaw;
 406         int agaw = -1;
 407
 408         sagaw = cap_sagaw(iommu->cap);
 409         for (agaw = width_to_agaw(DEFAULT_DOMAIN_ADDRESS_WIDTH);
 410              agaw >= 0; agaw--) {
 411                 if (test_bit(agaw, &sagaw))
 412                         break;
 413         }
 414
 415         return agaw;
 416 }
 417
 418 /* in native case, each domain is related to only one iommu */
 419 static struct intel_iommu *domain_get_iommu(struct dmar_domain *domain)
 420 {
 421         int iommu_id;
 422
 423         BUG_ON(domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE);
 424
 425         iommu_id = find_first_bit(&domain->iommu_bmp, g_num_of_iommus);
 426         if (iommu_id < 0 || iommu_id >= g_num_of_iommus)
 427                 return NULL;
 428
 429         return g_iommus[iommu_id];
 430 }
 431
 432 static void domain_update_iommu_coherency(struct dmar_domain *domain)
 433 {
 434         int i;
 435
 436         domain->iommu_coherency = 1;
 437
 438         i = find_first_bit(&domain->iommu_bmp, g_num_of_iommus);
 439         for (; i < g_num_of_iommus; ) {
 440                 if (!ecap_coherent(g_iommus[i]->ecap)) {
 441                         domain->iommu_coherency = 0;
 442                         break;
 443                 }
 444                 i = find_next_bit(&domain->iommu_bmp, g_num_of_iommus, i+1);
 445         }
 446 }
 447
 448 static void domain_update_iommu_snooping(struct dmar_domain *domain)
 449 {
 450         int i;
 451
 452         domain->iommu_snooping = 1;
 453
 454         i = find_first_bit(&domain->iommu_bmp, g_num_of_iommus);
 455         for (; i < g_num_of_iommus; ) {
 456                 if (!ecap_sc_support(g_iommus[i]->ecap)) {
 457                         domain->iommu_snooping = 0;
 458                         break;
 459                 }
 460                 i = find_next_bit(&domain->iommu_bmp, g_num_of_iommus, i+1);
 461         }
 462 }
 463
 464 /* Some capabilities may be different across iommus */
 465 static void domain_update_iommu_cap(struct dmar_domain *domain)
 466 {
 467         domain_update_iommu_coherency(domain);
 468         domain_update_iommu_snooping(domain);
 469 }
 470
 471 static struct intel_iommu *device_to_iommu(u8 bus, u8 devfn)
 472 {
 473         struct dmar_drhd_unit *drhd = NULL;
 474         int i;
 475
 476         for_each_drhd_unit(drhd) {
 477                 if (drhd->ignored)
 478                         continue;
 479
 480                 for (i = 0; i < drhd->devices_cnt; i++)
 481                         if (drhd->devices[i] &&
 482                             drhd->devices[i]->bus->number == bus &&
 483                             drhd->devices[i]->devfn == devfn)
 484                                 return drhd->iommu;
 485
 486                 if (drhd->include_all)
 487                         return drhd->iommu;
 488         }
 489
 490         return NULL;
 491 }
 492
 493 static void domain_flush_cache(struct dmar_domain *domain,
 494                                void *addr, int size)
 495 {
 496         if (!domain->iommu_coherency)
 497                 clflush_cache_range(addr, size);
 498 }
 499
 500 /* Gets context entry for a given bus and devfn */
 501 static struct context_entry * device_to_context_entry(struct intel_iommu *iommu,
 502                 u8 bus, u8 devfn)
 503 {
 504         struct root_entry *root;
 505         struct context_entry *context;
 506         unsigned long phy_addr;
 507         unsigned long flags;
 508
 509         spin_lock_irqsave(&iommu->lock, flags);
 510         root = &iommu->root_entry[bus];
 511         context = get_context_addr_from_root(root);
 512         if (!context) {
 513                 context = (struct context_entry *)alloc_pgtable_page();
 514                 if (!context) {
 515                         spin_unlock_irqrestore(&iommu->lock, flags);
 516                         return NULL;
 517                 }
 518                 __iommu_flush_cache(iommu, (void *)context, CONTEXT_SIZE);
 519                 phy_addr = virt_to_phys((void *)context);
 520                 set_root_value(root, phy_addr);
 521                 set_root_present(root);
 522                 __iommu_flush_cache(iommu, root, sizeof(*root));
 523         }
 524         spin_unlock_irqrestore(&iommu->lock, flags);
 525         return &context[devfn];
 526 }
 527
 528 static int device_context_mapped(struct intel_iommu *iommu, u8 bus, u8 devfn)
 529 {
 530         struct root_entry *root;
 531         struct context_entry *context;
 532         int ret;
 533         unsigned long flags;
 534
 535         spin_lock_irqsave(&iommu->lock, flags);
 536         root = &iommu->root_entry[bus];
 537         context = get_context_addr_from_root(root);
 538         if (!context) {
 539                 ret = 0;
 540                 goto out;
 541         }
 542         ret = context_present(&context[devfn]);
 543 out:
 544         spin_unlock_irqrestore(&iommu->lock, flags);
 545         return ret;
 546 }
 547
 548 static void clear_context_table(struct intel_iommu *iommu, u8 bus, u8 devfn)
 549 {
 550         struct root_entry *root;
 551         struct context_entry *context;
 552         unsigned long flags;
 553
 554         spin_lock_irqsave(&iommu->lock, flags);
 555         root = &iommu->root_entry[bus];
 556         context = get_context_addr_from_root(root);
 557         if (context) {
 558                 context_clear_entry(&context[devfn]);
 559                 __iommu_flush_cache(iommu, &context[devfn], \
 560                         sizeof(*context));
 561         }
 562         spin_unlock_irqrestore(&iommu->lock, flags);
 563 }
 564
 565 static void free_context_table(struct intel_iommu *iommu)
 566 {
 567         struct root_entry *root;
 568         int i;
 569         unsigned long flags;
 570         struct context_entry *context;
 571
 572         spin_lock_irqsave(&iommu->lock, flags);
 573         if (!iommu->root_entry) {
 574                 goto out;
 575         }
 576         for (i = 0; i < ROOT_ENTRY_NR; i++) {
 577                 root = &iommu->root_entry[i];
 578                 context = get_context_addr_from_root(root);
 579                 if (context)
 580                         free_pgtable_page(context);
 581         }
 582         free_pgtable_page(iommu->root_entry);
 583         iommu->root_entry = NULL;
 584 out:
 585         spin_unlock_irqrestore(&iommu->lock, flags);
 586 }
 587
 588 /* page table handling */
 589 #define LEVEL_STRIDE            (9)
 590 #define LEVEL_MASK              (((u64)1 << LEVEL_STRIDE) - 1)
 591
 592 static inline int agaw_to_level(int agaw)
 593 {
 594         return agaw + 2;
 595 }
 596
 597 static inline int agaw_to_width(int agaw)
 598 {
 599         return 30 + agaw * LEVEL_STRIDE;
 600
 601 }
 602
 603 static inline int width_to_agaw(int width)
 604 {
 605         return (width - 30) / LEVEL_STRIDE;
 606 }
 607
 608 static inline unsigned int level_to_offset_bits(int level)
 609 {
 610         return (12 + (level - 1) * LEVEL_STRIDE);
 611 }
 612
 613 static inline int address_level_offset(u64 addr, int level)
 614 {
 615         return ((addr >> level_to_offset_bits(level)) & LEVEL_MASK);
 616 }
 617
 618 static inline u64 level_mask(int level)
 619 {
 620         return ((u64)-1 << level_to_offset_bits(level));
 621 }
 622
 623 static inline u64 level_size(int level)
 624 {
 625         return ((u64)1 << level_to_offset_bits(level));
 626 }
 627
 628 static inline u64 align_to_level(u64 addr, int level)
 629 {
 630         return ((addr + level_size(level) - 1) & level_mask(level));
 631 }
 632
 633 static struct dma_pte * addr_to_dma_pte(struct dmar_domain *domain, u64 addr)
 634 {
 635         int addr_width = agaw_to_width(domain->agaw);
 636         struct dma_pte *parent, *pte = NULL;
 637         int level = agaw_to_level(domain->agaw);
 638         int offset;
 639         unsigned long flags;
 640
 641         BUG_ON(!domain->pgd);
 642
 643         addr &= (((u64)1) << addr_width) - 1;
 644         parent = domain->pgd;
 645
 646         spin_lock_irqsave(&domain->mapping_lock, flags);
 647         while (level > 0) {
 648                 void *tmp_page;
 649
 650                 offset = address_level_offset(addr, level);
 651                 pte = &parent[offset];
 652                 if (level == 1)
 653                         break;
 654
 655                 if (!dma_pte_present(pte)) {
 656                         tmp_page = alloc_pgtable_page();
 657
 658                         if (!tmp_page) {
 659                                 spin_unlock_irqrestore(&domain->mapping_lock,
 660                                         flags);
 661                                 return NULL;
 662                         }
 663                         domain_flush_cache(domain, tmp_page, PAGE_SIZE);
 664                         dma_set_pte_addr(pte, virt_to_phys(tmp_page));
 665                         /*
 666                          * high level table always sets r/w, last level page
 667                          * table control read/write
 668                          */
 669                         dma_set_pte_readable(pte);
 670                         dma_set_pte_writable(pte);
 671                         domain_flush_cache(domain, pte, sizeof(*pte));
 672                 }
 673                 parent = phys_to_virt(dma_pte_addr(pte));
 674                 level--;
 675         }
 676
 677         spin_unlock_irqrestore(&domain->mapping_lock, flags);
 678         return pte;
 679 }
 680
 681 /* return address's pte at specific level */
 682 static struct dma_pte *dma_addr_level_pte(struct dmar_domain *domain, u64 addr,
 683                 int level)
 684 {
 685         struct dma_pte *parent, *pte = NULL;
 686         int total = agaw_to_level(domain->agaw);
 687         int offset;
 688
 689         parent = domain->pgd;
 690         while (level <= total) {
 691                 offset = address_level_offset(addr, total);
 692                 pte = &parent[offset];
 693                 if (level == total)
 694                         return pte;
 695
 696                 if (!dma_pte_present(pte))
 697                         break;
 698                 parent = phys_to_virt(dma_pte_addr(pte));
 699                 total--;
 700         }
 701         return NULL;
 702 }
 703
 704 /* clear one page's page table */
 705 static void dma_pte_clear_one(struct dmar_domain *domain, u64 addr)
 706 {
 707         struct dma_pte *pte = NULL;
 708
 709         /* get last level pte */
 710         pte = dma_addr_level_pte(domain, addr, 1);
 711
 712         if (pte) {
 713                 dma_clear_pte(pte);
 714                 domain_flush_cache(domain, pte, sizeof(*pte));
 715         }
 716 }
 717
 718 /* clear last level pte, a tlb flush should be followed */
 719 static void dma_pte_clear_range(struct dmar_domain *domain, u64 start, u64 end)
 720 {
 721         int addr_width = agaw_to_width(domain->agaw);
 722         int npages;
 723
 724         start &= (((u64)1) << addr_width) - 1;
 725         end &= (((u64)1) << addr_width) - 1;
 726         /* in case it's partial page */
 727         start = PAGE_ALIGN(start);
 728         end &= PAGE_MASK;
 729         npages = (end - start) / VTD_PAGE_SIZE;
 730
 731         /* we don't need lock here, nobody else touches the iova range */
 732         while (npages--) {
 733                 dma_pte_clear_one(domain, start);
 734                 start += VTD_PAGE_SIZE;
 735         }
 736 }
 737
 738 /* free page table pages. last level pte should already be cleared */
 739 static void dma_pte_free_pagetable(struct dmar_domain *domain,
 740         u64 start, u64 end)
 741 {
 742         int addr_width = agaw_to_width(domain->agaw);
 743         struct dma_pte *pte;
 744         int total = agaw_to_level(domain->agaw);
 745         int level;
 746         u64 tmp;
 747
 748         start &= (((u64)1) << addr_width) - 1;
 749         end &= (((u64)1) << addr_width) - 1;
 750
 751         /* we don't need lock here, nobody else touches the iova range */
 752         level = 2;
 753         while (level <= total) {
 754                 tmp = align_to_level(start, level);
 755                 if (tmp >= end || (tmp + level_size(level) > end))
 756                         return;
 757
 758                 while (tmp < end) {
 759                         pte = dma_addr_level_pte(domain, tmp, level);
 760                         if (pte) {
 761                                 free_pgtable_page(
 762                                         phys_to_virt(dma_pte_addr(pte)));
 763                                 dma_clear_pte(pte);
 764                                 domain_flush_cache(domain, pte, sizeof(*pte));
 765                         }
 766                         tmp += level_size(level);
 767                 }
 768                 level++;
 769         }
 770         /* free pgd */
 771         if (start == 0 && end >= ((((u64)1) << addr_width) - 1)) {
 772                 free_pgtable_page(domain->pgd);
 773                 domain->pgd = NULL;
 774         }
 775 }
 776
 777 /* iommu handling */
 778 static int iommu_alloc_root_entry(struct intel_iommu *iommu)
 779 {
 780         struct root_entry *root;
 781         unsigned long flags;
 782
 783         root = (struct root_entry *)alloc_pgtable_page();
 784         if (!root)
 785                 return -ENOMEM;
 786
 787         __iommu_flush_cache(iommu, root, ROOT_SIZE);
 788
 789         spin_lock_irqsave(&iommu->lock, flags);
 790         iommu->root_entry = root;
 791         spin_unlock_irqrestore(&iommu->lock, flags);
 792
 793         return 0;
 794 }
 795
 796 static void iommu_set_root_entry(struct intel_iommu *iommu)
 797 {
 798         void *addr;
 799         u32 cmd, sts;
 800         unsigned long flag;
 801
 802         addr = iommu->root_entry;
 803
 804         spin_lock_irqsave(&iommu->register_lock, flag);
 805         dmar_writeq(iommu->reg + DMAR_RTADDR_REG, virt_to_phys(addr));
 806
 807         cmd = iommu->gcmd | DMA_GCMD_SRTP;
 808         writel(cmd, iommu->reg + DMAR_GCMD_REG);
 809
 810         /* Make sure hardware complete it */
 811         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
 812                 readl, (sts & DMA_GSTS_RTPS), sts);
 813
 814         spin_unlock_irqrestore(&iommu->register_lock, flag);
 815 }
 816
 817 static void iommu_flush_write_buffer(struct intel_iommu *iommu)
 818 {
 819         u32 val;
 820         unsigned long flag;
 821
 822         if (!rwbf_quirk && !cap_rwbf(iommu->cap))
 823                 return;
 824         val = iommu->gcmd | DMA_GCMD_WBF;
 825
 826         spin_lock_irqsave(&iommu->register_lock, flag);
 827         writel(val, iommu->reg + DMAR_GCMD_REG);
 828
 829         /* Make sure hardware complete it */
 830         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
 831                         readl, (!(val & DMA_GSTS_WBFS)), val);
 832
 833         spin_unlock_irqrestore(&iommu->register_lock, flag);
 834 }
 835
 836 /* return value determine if we need a write buffer flush */
 837 static int __iommu_flush_context(struct intel_iommu *iommu,
 838         u16 did, u16 source_id, u8 function_mask, u64 type,
 839         int non_present_entry_flush)
 840 {
 841         u64 val = 0;
 842         unsigned long flag;
 843
 844         /*
 845          * In the non-present entry flush case, if hardware doesn't cache
 846          * non-present entry we do nothing and if hardware cache non-present
 847          * entry, we flush entries of domain 0 (the domain id is used to cache
 848          * any non-present entries)
 849          */
 850         if (non_present_entry_flush) {
 851                 if (!cap_caching_mode(iommu->cap))
 852                         return 1;
 853                 else
 854                         did = 0;
 855         }
 856
 857         switch (type) {
 858         case DMA_CCMD_GLOBAL_INVL:
 859                 val = DMA_CCMD_GLOBAL_INVL;
 860                 break;
 861         case DMA_CCMD_DOMAIN_INVL:
 862                 val = DMA_CCMD_DOMAIN_INVL|DMA_CCMD_DID(did);
 863                 break;
 864         case DMA_CCMD_DEVICE_INVL:
 865                 val = DMA_CCMD_DEVICE_INVL|DMA_CCMD_DID(did)
 866                         | DMA_CCMD_SID(source_id) | DMA_CCMD_FM(function_mask);
 867                 break;
 868         default:
 869                 BUG();
 870         }
 871         val |= DMA_CCMD_ICC;
 872
 873         spin_lock_irqsave(&iommu->register_lock, flag);
 874         dmar_writeq(iommu->reg + DMAR_CCMD_REG, val);
 875
 876         /* Make sure hardware complete it */
 877         IOMMU_WAIT_OP(iommu, DMAR_CCMD_REG,
 878                 dmar_readq, (!(val & DMA_CCMD_ICC)), val);
 879
 880         spin_unlock_irqrestore(&iommu->register_lock, flag);
 881
 882         /* flush context entry will implicitly flush write buffer */
 883         return 0;
 884 }
 885
 886 /* return value determine if we need a write buffer flush */
 887 static int __iommu_flush_iotlb(struct intel_iommu *iommu, u16 did,
 888         u64 addr, unsigned int size_order, u64 type,
 889         int non_present_entry_flush)
 890 {
 891         int tlb_offset = ecap_iotlb_offset(iommu->ecap);
 892         u64 val = 0, val_iva = 0;
 893         unsigned long flag;
 894
 895         /*
 896          * In the non-present entry flush case, if hardware doesn't cache
 897          * non-present entry we do nothing and if hardware cache non-present
 898          * entry, we flush entries of domain 0 (the domain id is used to cache
 899          * any non-present entries)
 900          */
 901         if (non_present_entry_flush) {
 902                 if (!cap_caching_mode(iommu->cap))
 903                         return 1;
 904                 else
 905                         did = 0;
 906         }
 907
 908         switch (type) {
 909         case DMA_TLB_GLOBAL_FLUSH:
 910                 /* global flush doesn't need set IVA_REG */
 911                 val = DMA_TLB_GLOBAL_FLUSH|DMA_TLB_IVT;
 912                 break;
 913         case DMA_TLB_DSI_FLUSH:
 914                 val = DMA_TLB_DSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
 915                 break;
 916         case DMA_TLB_PSI_FLUSH:
 917                 val = DMA_TLB_PSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
 918                 /* Note: always flush non-leaf currently */
 919                 val_iva = size_order | addr;
 920                 break;
 921         default:
 922                 BUG();
 923         }
 924         /* Note: set drain read/write */
 925 #if 0
 926         /*
 927          * This is probably to be super secure.. Looks like we can
 928          * ignore it without any impact.
 929          */
 930         if (cap_read_drain(iommu->cap))
 931                 val |= DMA_TLB_READ_DRAIN;
 932 #endif
 933         if (cap_write_drain(iommu->cap))
 934                 val |= DMA_TLB_WRITE_DRAIN;
 935
 936         spin_lock_irqsave(&iommu->register_lock, flag);
 937         /* Note: Only uses first TLB reg currently */
 938         if (val_iva)
 939                 dmar_writeq(iommu->reg + tlb_offset, val_iva);
 940         dmar_writeq(iommu->reg + tlb_offset + 8, val);
 941
 942         /* Make sure hardware complete it */
 943         IOMMU_WAIT_OP(iommu, tlb_offset + 8,
 944                 dmar_readq, (!(val & DMA_TLB_IVT)), val);
 945
 946         spin_unlock_irqrestore(&iommu->register_lock, flag);
 947
 948         /* check IOTLB invalidation granularity */
 949         if (DMA_TLB_IAIG(val) == 0)
 950                 printk(KERN_ERR"IOMMU: flush IOTLB failed\n");
 951         if (DMA_TLB_IAIG(val) != DMA_TLB_IIRG(type))
 952                 pr_debug("IOMMU: tlb flush request %Lx, actual %Lx\n",
 953                         (unsigned long long)DMA_TLB_IIRG(type),
 954                         (unsigned long long)DMA_TLB_IAIG(val));
 955         /* flush iotlb entry will implicitly flush write buffer */
 956         return 0;
 957 }
 958
 959 static int iommu_flush_iotlb_psi(struct intel_iommu *iommu, u16 did,
 960         u64 addr, unsigned int pages, int non_present_entry_flush)
 961 {
 962         unsigned int mask;
 963
 964         BUG_ON(addr & (~VTD_PAGE_MASK));
 965         BUG_ON(pages == 0);
 966
 967         /* Fallback to domain selective flush if no PSI support */
 968         if (!cap_pgsel_inv(iommu->cap))
 969                 return iommu->flush.flush_iotlb(iommu, did, 0, 0,
 970                                                 DMA_TLB_DSI_FLUSH,
 971                                                 non_present_entry_flush);
 972
 973         /*
 974          * PSI requires page size to be 2 ^ x, and the base address is naturally
 975          * aligned to the size
 976          */
 977         mask = ilog2(__roundup_pow_of_two(pages));
 978         /* Fallback to domain selective flush if size is too big */
 979         if (mask > cap_max_amask_val(iommu->cap))
 980                 return iommu->flush.flush_iotlb(iommu, did, 0, 0,
 981                         DMA_TLB_DSI_FLUSH, non_present_entry_flush);
 982
 983         return iommu->flush.flush_iotlb(iommu, did, addr, mask,
 984                                         DMA_TLB_PSI_FLUSH,
 985                                         non_present_entry_flush);
 986 }
 987
 988 static void iommu_disable_protect_mem_regions(struct intel_iommu *iommu)
 989 {
 990         u32 pmen;
 991         unsigned long flags;
 992
 993         spin_lock_irqsave(&iommu->register_lock, flags);
 994         pmen = readl(iommu->reg + DMAR_PMEN_REG);
 995         pmen &= ~DMA_PMEN_EPM;
 996         writel(pmen, iommu->reg + DMAR_PMEN_REG);
 997
 998         /* wait for the protected region status bit to clear */
 999         IOMMU_WAIT_OP(iommu, DMAR_PMEN_REG,
1000                 readl, !(pmen & DMA_PMEN_PRS), pmen);
1001
1002         spin_unlock_irqrestore(&iommu->register_lock, flags);
1003 }
1004
1005 static int iommu_enable_translation(struct intel_iommu *iommu)
1006 {
1007         u32 sts;
1008         unsigned long flags;
1009
1010         spin_lock_irqsave(&iommu->register_lock, flags);
1011         writel(iommu->gcmd|DMA_GCMD_TE, iommu->reg + DMAR_GCMD_REG);
1012
1013         /* Make sure hardware complete it */
1014         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1015                 readl, (sts & DMA_GSTS_TES), sts);
1016
1017         iommu->gcmd |= DMA_GCMD_TE;
1018         spin_unlock_irqrestore(&iommu->register_lock, flags);
1019         return 0;
1020 }
1021
1022 static int iommu_disable_translation(struct intel_iommu *iommu)
1023 {
1024         u32 sts;
1025         unsigned long flag;
1026
1027         spin_lock_irqsave(&iommu->register_lock, flag);
1028         iommu->gcmd &= ~DMA_GCMD_TE;
1029         writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1030
1031         /* Make sure hardware complete it */
1032         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1033                 readl, (!(sts & DMA_GSTS_TES)), sts);
1034
1035         spin_unlock_irqrestore(&iommu->register_lock, flag);
1036         return 0;
1037 }
1038
1039
1040 static int iommu_init_domains(struct intel_iommu *iommu)
1041 {
1042         unsigned long ndomains;
1043         unsigned long nlongs;
1044
1045         ndomains = cap_ndoms(iommu->cap);
1046         pr_debug("Number of Domains supportd <%ld>\n", ndomains);
1047         nlongs = BITS_TO_LONGS(ndomains);
1048
1049         /* TBD: there might be 64K domains,
1050          * consider other allocation for future chip
1051          */
1052         iommu->domain_ids = kcalloc(nlongs, sizeof(unsigned long), GFP_KERNEL);
1053         if (!iommu->domain_ids) {
1054                 printk(KERN_ERR "Allocating domain id array failed\n");
1055                 return -ENOMEM;
1056         }
1057         iommu->domains = kcalloc(ndomains, sizeof(struct dmar_domain *),
1058                         GFP_KERNEL);
1059         if (!iommu->domains) {
1060                 printk(KERN_ERR "Allocating domain array failed\n");
1061                 kfree(iommu->domain_ids);
1062                 return -ENOMEM;
1063         }
1064
1065         spin_lock_init(&iommu->lock);
1066
1067         /*
1068          * if Caching mode is set, then invalid translations are tagged
1069          * with domainid 0. Hence we need to pre-allocate it.
1070          */
1071         if (cap_caching_mode(iommu->cap))
1072                 set_bit(0, iommu->domain_ids);
1073         return 0;
1074 }
1075
1076
1077 static void domain_exit(struct dmar_domain *domain);
1078 static void vm_domain_exit(struct dmar_domain *domain);
1079
1080 void free_dmar_iommu(struct intel_iommu *iommu)
1081 {
1082         struct dmar_domain *domain;
1083         int i;
1084         unsigned long flags;
1085
1086         i = find_first_bit(iommu->domain_ids, cap_ndoms(iommu->cap));
1087         for (; i < cap_ndoms(iommu->cap); ) {
1088                 domain = iommu->domains[i];
1089                 clear_bit(i, iommu->domain_ids);
1090
1091                 spin_lock_irqsave(&domain->iommu_lock, flags);
1092                 if (--domain->iommu_count == 0) {
1093                         if (domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE)
1094                                 vm_domain_exit(domain);
1095                         else
1096                                 domain_exit(domain);
1097                 }
1098                 spin_unlock_irqrestore(&domain->iommu_lock, flags);
1099
1100                 i = find_next_bit(iommu->domain_ids,
1101                         cap_ndoms(iommu->cap), i+1);
1102         }
1103
1104         if (iommu->gcmd & DMA_GCMD_TE)
1105                 iommu_disable_translation(iommu);
1106
1107         if (iommu->irq) {
1108                 set_irq_data(iommu->irq, NULL);
1109                 /* This will mask the irq */
1110                 free_irq(iommu->irq, iommu);
1111                 destroy_irq(iommu->irq);
1112         }
1113
1114         kfree(iommu->domains);
1115         kfree(iommu->domain_ids);
1116
1117         g_iommus[iommu->seq_id] = NULL;
1118
1119         /* if all iommus are freed, free g_iommus */
1120         for (i = 0; i < g_num_of_iommus; i++) {
1121                 if (g_iommus[i])
1122                         break;
1123         }
1124
1125         if (i == g_num_of_iommus)
1126                 kfree(g_iommus);
1127
1128         /* free context mapping */
1129         free_context_table(iommu);
1130 }
1131
1132 static struct dmar_domain * iommu_alloc_domain(struct intel_iommu *iommu)
1133 {
1134         unsigned long num;
1135         unsigned long ndomains;
1136         struct dmar_domain *domain;
1137         unsigned long flags;
1138
1139         domain = alloc_domain_mem();
1140         if (!domain)
1141                 return NULL;
1142
1143         ndomains = cap_ndoms(iommu->cap);
1144
1145         spin_lock_irqsave(&iommu->lock, flags);
1146         num = find_first_zero_bit(iommu->domain_ids, ndomains);
1147         if (num >= ndomains) {
1148                 spin_unlock_irqrestore(&iommu->lock, flags);
1149                 free_domain_mem(domain);
1150                 printk(KERN_ERR "IOMMU: no free domain ids\n");
1151                 return NULL;
1152         }
1153
1154         set_bit(num, iommu->domain_ids);
1155         domain->id = num;
1156         memset(&domain->iommu_bmp, 0, sizeof(unsigned long));
1157         set_bit(iommu->seq_id, &domain->iommu_bmp);
1158         domain->flags = 0;
1159         iommu->domains[num] = domain;
1160         spin_unlock_irqrestore(&iommu->lock, flags);
1161
1162         return domain;
1163 }
1164
1165 static void iommu_free_domain(struct dmar_domain *domain)
1166 {
1167         unsigned long flags;
1168         struct intel_iommu *iommu;
1169
1170         iommu = domain_get_iommu(domain);
1171
1172         spin_lock_irqsave(&iommu->lock, flags);
1173         clear_bit(domain->id, iommu->domain_ids);
1174         spin_unlock_irqrestore(&iommu->lock, flags);
1175 }
1176
1177 static struct iova_domain reserved_iova_list;
1178 static struct lock_class_key reserved_alloc_key;
1179 static struct lock_class_key reserved_rbtree_key;
1180
1181 static void dmar_init_reserved_ranges(void)
1182 {
1183         struct pci_dev *pdev = NULL;
1184         struct iova *iova;
1185         int i;
1186         u64 addr, size;
1187
1188         init_iova_domain(&reserved_iova_list, DMA_32BIT_PFN);
1189
1190         lockdep_set_class(&reserved_iova_list.iova_alloc_lock,
1191                 &reserved_alloc_key);
1192         lockdep_set_class(&reserved_iova_list.iova_rbtree_lock,
1193                 &reserved_rbtree_key);
1194
1195         /* IOAPIC ranges shouldn't be accessed by DMA */
1196         iova = reserve_iova(&reserved_iova_list, IOVA_PFN(IOAPIC_RANGE_START),
1197                 IOVA_PFN(IOAPIC_RANGE_END));
1198         if (!iova)
1199                 printk(KERN_ERR "Reserve IOAPIC range failed\n");
1200
1201         /* Reserve all PCI MMIO to avoid peer-to-peer access */
1202         for_each_pci_dev(pdev) {
1203                 struct resource *r;
1204
1205                 for (i = 0; i < PCI_NUM_RESOURCES; i++) {
1206                         r = &pdev->resource[i];
1207                         if (!r->flags || !(r->flags & IORESOURCE_MEM))
1208                                 continue;
1209                         addr = r->start;
1210                         addr &= PAGE_MASK;
1211                         size = r->end - addr;
1212                         size = PAGE_ALIGN(size);
1213                         iova = reserve_iova(&reserved_iova_list, IOVA_PFN(addr),
1214                                 IOVA_PFN(size + addr) - 1);
1215                         if (!iova)
1216                                 printk(KERN_ERR "Reserve iova failed\n");
1217                 }
1218         }
1219
1220 }
1221
1222 static void domain_reserve_special_ranges(struct dmar_domain *domain)
1223 {
1224         copy_reserved_iova(&reserved_iova_list, &domain->iovad);
1225 }
1226
1227 static inline int guestwidth_to_adjustwidth(int gaw)
1228 {
1229         int agaw;
1230         int r = (gaw - 12) % 9;
1231
1232         if (r == 0)
1233                 agaw = gaw;
1234         else
1235                 agaw = gaw + 9 - r;
1236         if (agaw > 64)
1237                 agaw = 64;
1238         return agaw;
1239 }
1240
1241 static int domain_init(struct dmar_domain *domain, int guest_width)
1242 {
1243         struct intel_iommu *iommu;
1244         int adjust_width, agaw;
1245         unsigned long sagaw;
1246
1247         init_iova_domain(&domain->iovad, DMA_32BIT_PFN);
1248         spin_lock_init(&domain->mapping_lock);
1249         spin_lock_init(&domain->iommu_lock);
1250
1251         domain_reserve_special_ranges(domain);
1252
1253         /* calculate AGAW */
1254         iommu = domain_get_iommu(domain);
1255         if (guest_width > cap_mgaw(iommu->cap))
1256                 guest_width = cap_mgaw(iommu->cap);
1257         domain->gaw = guest_width;
1258         adjust_width = guestwidth_to_adjustwidth(guest_width);
1259         agaw = width_to_agaw(adjust_width);
1260         sagaw = cap_sagaw(iommu->cap);
1261         if (!test_bit(agaw, &sagaw)) {
1262                 /* hardware doesn't support it, choose a bigger one */
1263                 pr_debug("IOMMU: hardware doesn't support agaw %d\n", agaw);
1264                 agaw = find_next_bit(&sagaw, 5, agaw);
1265                 if (agaw >= 5)
1266                         return -ENODEV;
1267         }
1268         domain->agaw = agaw;
1269         INIT_LIST_HEAD(&domain->devices);
1270
1271         if (ecap_coherent(iommu->ecap))
1272                 domain->iommu_coherency = 1;
1273         else
1274                 domain->iommu_coherency = 0;
1275
1276         if (ecap_sc_support(iommu->ecap))
1277                 domain->iommu_snooping = 1;
1278         else
1279                 domain->iommu_snooping = 0;
1280
1281         domain->iommu_count = 1;
1282
1283         /* always allocate the top pgd */
1284         domain->pgd = (struct dma_pte *)alloc_pgtable_page();
1285         if (!domain->pgd)
1286                 return -ENOMEM;
1287         __iommu_flush_cache(iommu, domain->pgd, PAGE_SIZE);
1288         return 0;
1289 }
1290
1291 static void domain_exit(struct dmar_domain *domain)
1292 {
1293         u64 end;
1294
1295         /* Domain 0 is reserved, so dont process it */
1296         if (!domain)
1297                 return;
1298
1299         domain_remove_dev_info(domain);
1300         /* destroy iovas */
1301         put_iova_domain(&domain->iovad);
1302         end = DOMAIN_MAX_ADDR(domain->gaw);
1303         end = end & (~PAGE_MASK);
1304
1305         /* clear ptes */
1306         dma_pte_clear_range(domain, 0, end);
1307
1308         /* free page tables */
1309         dma_pte_free_pagetable(domain, 0, end);
1310
1311         iommu_free_domain(domain);
1312         free_domain_mem(domain);
1313 }
1314
1315 static int domain_context_mapping_one(struct dmar_domain *domain,
1316                 u8 bus, u8 devfn)
1317 {
1318         struct context_entry *context;
1319         unsigned long flags;
1320         struct intel_iommu *iommu;
1321         struct dma_pte *pgd;
1322         unsigned long num;
1323         unsigned long ndomains;
1324         int id;
1325         int agaw;
1326
1327         pr_debug("Set context mapping for %02x:%02x.%d\n",
1328                 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
1329         BUG_ON(!domain->pgd);
1330
1331         iommu = device_to_iommu(bus, devfn);
1332         if (!iommu)
1333                 return -ENODEV;
1334
1335         context = device_to_context_entry(iommu, bus, devfn);
1336         if (!context)
1337                 return -ENOMEM;
1338         spin_lock_irqsave(&iommu->lock, flags);
1339         if (context_present(context)) {
1340                 spin_unlock_irqrestore(&iommu->lock, flags);
1341                 return 0;
1342         }
1343
1344         id = domain->id;
1345         pgd = domain->pgd;
1346
1347         if (domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE) {
1348                 int found = 0;
1349
1350                 /* find an available domain id for this device in iommu */
1351                 ndomains = cap_ndoms(iommu->cap);
1352                 num = find_first_bit(iommu->domain_ids, ndomains);
1353                 for (; num < ndomains; ) {
1354                         if (iommu->domains[num] == domain) {
1355                                 id = num;
1356                                 found = 1;
1357                                 break;
1358                         }
1359                         num = find_next_bit(iommu->domain_ids,
1360                                             cap_ndoms(iommu->cap), num+1);
1361                 }
1362
1363                 if (found == 0) {
1364                         num = find_first_zero_bit(iommu->domain_ids, ndomains);
1365                         if (num >= ndomains) {
1366                                 spin_unlock_irqrestore(&iommu->lock, flags);
1367                                 printk(KERN_ERR "IOMMU: no free domain ids\n");
1368                                 return -EFAULT;
1369                         }
1370
1371                         set_bit(num, iommu->domain_ids);
1372                         iommu->domains[num] = domain;
1373                         id = num;
1374                 }
1375
1376                 /* Skip top levels of page tables for
1377                  * iommu which has less agaw than default.
1378                  */
1379                 for (agaw = domain->agaw; agaw != iommu->agaw; agaw--) {
1380                         pgd = phys_to_virt(dma_pte_addr(pgd));
1381                         if (!dma_pte_present(pgd)) {
1382                                 spin_unlock_irqrestore(&iommu->lock, flags);
1383                                 return -ENOMEM;
1384                         }
1385                 }
1386         }
1387
1388         context_set_domain_id(context, id);
1389         context_set_address_width(context, iommu->agaw);
1390         context_set_address_root(context, virt_to_phys(pgd));
1391         context_set_translation_type(context, CONTEXT_TT_MULTI_LEVEL);
1392         context_set_fault_enable(context);
1393         context_set_present(context);
1394         domain_flush_cache(domain, context, sizeof(*context));
1395
1396         /* it's a non-present to present mapping */
1397         if (iommu->flush.flush_context(iommu, domain->id,
1398                 (((u16)bus) << 8) | devfn, DMA_CCMD_MASK_NOBIT,
1399                 DMA_CCMD_DEVICE_INVL, 1))
1400                 iommu_flush_write_buffer(iommu);
1401         else
1402                 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_DSI_FLUSH, 0);
1403
1404         spin_unlock_irqrestore(&iommu->lock, flags);
1405
1406         spin_lock_irqsave(&domain->iommu_lock, flags);
1407         if (!test_and_set_bit(iommu->seq_id, &domain->iommu_bmp)) {
1408                 domain->iommu_count++;
1409                 domain_update_iommu_cap(domain);
1410         }
1411         spin_unlock_irqrestore(&domain->iommu_lock, flags);
1412         return 0;
1413 }
1414
1415 static int
1416 domain_context_mapping(struct dmar_domain *domain, struct pci_dev *pdev)
1417 {
1418         int ret;
1419         struct pci_dev *tmp, *parent;
1420
1421         ret = domain_context_mapping_one(domain, pdev->bus->number,
1422                 pdev->devfn);
1423         if (ret)
1424                 return ret;
1425
1426         /* dependent device mapping */
1427         tmp = pci_find_upstream_pcie_bridge(pdev);
1428         if (!tmp)
1429                 return 0;
1430         /* Secondary interface's bus number and devfn 0 */
1431         parent = pdev->bus->self;
1432         while (parent != tmp) {
1433                 ret = domain_context_mapping_one(domain, parent->bus->number,
1434                         parent->devfn);
1435                 if (ret)
1436                         return ret;
1437                 parent = parent->bus->self;
1438         }
1439         if (tmp->is_pcie) /* this is a PCIE-to-PCI bridge */
1440                 return domain_context_mapping_one(domain,
1441                         tmp->subordinate->number, 0);
1442         else /* this is a legacy PCI bridge */
1443                 return domain_context_mapping_one(domain,
1444                         tmp->bus->number, tmp->devfn);
1445 }
1446
1447 static int domain_context_mapped(struct pci_dev *pdev)
1448 {
1449         int ret;
1450         struct pci_dev *tmp, *parent;
1451         struct intel_iommu *iommu;
1452
1453         iommu = device_to_iommu(pdev->bus->number, pdev->devfn);
1454         if (!iommu)
1455                 return -ENODEV;
1456
1457         ret = device_context_mapped(iommu,
1458                 pdev->bus->number, pdev->devfn);
1459         if (!ret)
1460                 return ret;
1461         /* dependent device mapping */
1462         tmp = pci_find_upstream_pcie_bridge(pdev);
1463         if (!tmp)
1464                 return ret;
1465         /* Secondary interface's bus number and devfn 0 */
1466         parent = pdev->bus->self;
1467         while (parent != tmp) {
1468                 ret = device_context_mapped(iommu, parent->bus->number,
1469                         parent->devfn);
1470                 if (!ret)
1471                         return ret;
1472                 parent = parent->bus->self;
1473         }
1474         if (tmp->is_pcie)
1475                 return device_context_mapped(iommu,
1476                         tmp->subordinate->number, 0);
1477         else
1478                 return device_context_mapped(iommu,
1479                         tmp->bus->number, tmp->devfn);
1480 }
1481
1482 static int
1483 domain_page_mapping(struct dmar_domain *domain, dma_addr_t iova,
1484                         u64 hpa, size_t size, int prot)
1485 {
1486         u64 start_pfn, end_pfn;
1487         struct dma_pte *pte;
1488         int index;
1489         int addr_width = agaw_to_width(domain->agaw);
1490
1491         hpa &= (((u64)1) << addr_width) - 1;
1492
1493         if ((prot & (DMA_PTE_READ|DMA_PTE_WRITE)) == 0)
1494                 return -EINVAL;
1495         iova &= PAGE_MASK;
1496         start_pfn = ((u64)hpa) >> VTD_PAGE_SHIFT;
1497         end_pfn = (VTD_PAGE_ALIGN(((u64)hpa) + size)) >> VTD_PAGE_SHIFT;
1498         index = 0;
1499         while (start_pfn < end_pfn) {
1500                 pte = addr_to_dma_pte(domain, iova + VTD_PAGE_SIZE * index);
1501                 if (!pte)
1502                         return -ENOMEM;
1503                 /* We don't need lock here, nobody else
1504                  * touches the iova range
1505                  */
1506                 BUG_ON(dma_pte_addr(pte));
1507                 dma_set_pte_addr(pte, start_pfn << VTD_PAGE_SHIFT);
1508                 dma_set_pte_prot(pte, prot);
1509                 if (prot & DMA_PTE_SNP)
1510                         dma_set_pte_snp(pte);
1511                 domain_flush_cache(domain, pte, sizeof(*pte));
1512                 start_pfn++;
1513                 index++;
1514         }
1515         return 0;
1516 }
1517
1518 static void iommu_detach_dev(struct intel_iommu *iommu, u8 bus, u8 devfn)
1519 {
1520         if (!iommu)
1521                 return;
1522
1523         clear_context_table(iommu, bus, devfn);
1524         iommu->flush.flush_context(iommu, 0, 0, 0,
1525                                            DMA_CCMD_GLOBAL_INVL, 0);
1526         iommu->flush.flush_iotlb(iommu, 0, 0, 0,
1527                                          DMA_TLB_GLOBAL_FLUSH, 0);
1528 }
1529
1530 static void domain_remove_dev_info(struct dmar_domain *domain)
1531 {
1532         struct device_domain_info *info;
1533         unsigned long flags;
1534         struct intel_iommu *iommu;
1535
1536         spin_lock_irqsave(&device_domain_lock, flags);
1537         while (!list_empty(&domain->devices)) {
1538                 info = list_entry(domain->devices.next,
1539                         struct device_domain_info, link);
1540                 list_del(&info->link);
1541                 list_del(&info->global);
1542                 if (info->dev)
1543                         info->dev->dev.archdata.iommu = NULL;
1544                 spin_unlock_irqrestore(&device_domain_lock, flags);
1545
1546                 iommu = device_to_iommu(info->bus, info->devfn);
1547                 iommu_detach_dev(iommu, info->bus, info->devfn);
1548                 free_devinfo_mem(info);
1549
1550                 spin_lock_irqsave(&device_domain_lock, flags);
1551         }
1552         spin_unlock_irqrestore(&device_domain_lock, flags);
1553 }
1554
1555 /*
1556  * find_domain
1557  * Note: we use struct pci_dev->dev.archdata.iommu stores the info
1558  */
1559 static struct dmar_domain *
1560 find_domain(struct pci_dev *pdev)
1561 {
1562         struct device_domain_info *info;
1563
1564         /* No lock here, assumes no domain exit in normal case */
1565         info = pdev->dev.archdata.iommu;
1566         if (info)
1567                 return info->domain;
1568         return NULL;
1569 }
1570
1571 /* domain is initialized */
1572 static struct dmar_domain *get_domain_for_dev(struct pci_dev *pdev, int gaw)
1573 {
1574         struct dmar_domain *domain, *found = NULL;
1575         struct intel_iommu *iommu;
1576         struct dmar_drhd_unit *drhd;
1577         struct device_domain_info *info, *tmp;
1578         struct pci_dev *dev_tmp;
1579         unsigned long flags;
1580         int bus = 0, devfn = 0;
1581
1582         domain = find_domain(pdev);
1583         if (domain)
1584                 return domain;
1585
1586         dev_tmp = pci_find_upstream_pcie_bridge(pdev);
1587         if (dev_tmp) {
1588                 if (dev_tmp->is_pcie) {
1589                         bus = dev_tmp->subordinate->number;
1590                         devfn = 0;
1591                 } else {
1592                         bus = dev_tmp->bus->number;
1593                         devfn = dev_tmp->devfn;
1594                 }
1595                 spin_lock_irqsave(&device_domain_lock, flags);
1596                 list_for_each_entry(info, &device_domain_list, global) {
1597                         if (info->bus == bus && info->devfn == devfn) {
1598                                 found = info->domain;
1599                                 break;
1600                         }
1601                 }
1602                 spin_unlock_irqrestore(&device_domain_lock, flags);
1603                 /* pcie-pci bridge already has a domain, uses it */
1604                 if (found) {
1605                         domain = found;
1606                         goto found_domain;
1607                 }
1608         }
1609
1610         /* Allocate new domain for the device */
1611         drhd = dmar_find_matched_drhd_unit(pdev);
1612         if (!drhd) {
1613                 printk(KERN_ERR "IOMMU: can't find DMAR for device %s\n",
1614                         pci_name(pdev));
1615                 return NULL;
1616         }
1617         iommu = drhd->iommu;
1618
1619         domain = iommu_alloc_domain(iommu);
1620         if (!domain)
1621                 goto error;
1622
1623         if (domain_init(domain, gaw)) {
1624                 domain_exit(domain);
1625                 goto error;
1626         }
1627
1628         /* register pcie-to-pci device */
1629         if (dev_tmp) {
1630                 info = alloc_devinfo_mem();
1631                 if (!info) {
1632                         domain_exit(domain);
1633                         goto error;
1634                 }
1635                 info->bus = bus;
1636                 info->devfn = devfn;
1637                 info->dev = NULL;
1638                 info->domain = domain;
1639                 /* This domain is shared by devices under p2p bridge */
1640                 domain->flags |= DOMAIN_FLAG_P2P_MULTIPLE_DEVICES;
1641
1642                 /* pcie-to-pci bridge already has a domain, uses it */
1643                 found = NULL;
1644                 spin_lock_irqsave(&device_domain_lock, flags);
1645                 list_for_each_entry(tmp, &device_domain_list, global) {
1646                         if (tmp->bus == bus && tmp->devfn == devfn) {
1647                                 found = tmp->domain;
1648                                 break;
1649                         }
1650                 }
1651                 if (found) {
1652                         free_devinfo_mem(info);
1653                         domain_exit(domain);
1654                         domain = found;
1655                 } else {
1656                         list_add(&info->link, &domain->devices);
1657                         list_add(&info->global, &device_domain_list);
1658                 }
1659                 spin_unlock_irqrestore(&device_domain_lock, flags);
1660         }
1661
1662 found_domain:
1663         info = alloc_devinfo_mem();
1664         if (!info)
1665                 goto error;
1666         info->bus = pdev->bus->number;
1667         info->devfn = pdev->devfn;
1668         info->dev = pdev;
1669         info->domain = domain;
1670         spin_lock_irqsave(&device_domain_lock, flags);
1671         /* somebody is fast */
1672         found = find_domain(pdev);
1673         if (found != NULL) {
1674                 spin_unlock_irqrestore(&device_domain_lock, flags);
1675                 if (found != domain) {
1676                         domain_exit(domain);
1677                         domain = found;
1678                 }
1679                 free_devinfo_mem(info);
1680                 return domain;
1681         }
1682         list_add(&info->link, &domain->devices);
1683         list_add(&info->global, &device_domain_list);
1684         pdev->dev.archdata.iommu = info;
1685         spin_unlock_irqrestore(&device_domain_lock, flags);
1686         return domain;
1687 error:
1688         /* recheck it here, maybe others set it */
1689         return find_domain(pdev);
1690 }
1691
1692 static int iommu_prepare_identity_map(struct pci_dev *pdev,
1693                                       unsigned long long start,
1694                                       unsigned long long end)
1695 {
1696         struct dmar_domain *domain;
1697         unsigned long size;
1698         unsigned long long base;
1699         int ret;
1700
1701         printk(KERN_INFO
1702                 "IOMMU: Setting identity map for device %s [0x%Lx - 0x%Lx]\n",
1703                 pci_name(pdev), start, end);
1704         /* page table init */
1705         domain = get_domain_for_dev(pdev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
1706         if (!domain)
1707                 return -ENOMEM;
1708
1709         /* The address might not be aligned */
1710         base = start & PAGE_MASK;
1711         size = end - base;
1712         size = PAGE_ALIGN(size);
1713         if (!reserve_iova(&domain->iovad, IOVA_PFN(base),
1714                         IOVA_PFN(base + size) - 1)) {
1715                 printk(KERN_ERR "IOMMU: reserve iova failed\n");
1716                 ret = -ENOMEM;
1717                 goto error;
1718         }
1719
1720         pr_debug("Mapping reserved region %lx@%llx for %s\n",
1721                 size, base, pci_name(pdev));
1722         /*
1723          * RMRR range might have overlap with physical memory range,
1724          * clear it first
1725          */
1726         dma_pte_clear_range(domain, base, base + size);
1727
1728         ret = domain_page_mapping(domain, base, base, size,
1729                 DMA_PTE_READ|DMA_PTE_WRITE);
1730         if (ret)
1731                 goto error;
1732
1733         /* context entry init */
1734         ret = domain_context_mapping(domain, pdev);
1735         if (!ret)
1736                 return 0;
1737 error:
1738         domain_exit(domain);
1739         return ret;
1740
1741 }
1742
1743 static inline int iommu_prepare_rmrr_dev(struct dmar_rmrr_unit *rmrr,
1744         struct pci_dev *pdev)
1745 {
1746         if (pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
1747                 return 0;
1748         return iommu_prepare_identity_map(pdev, rmrr->base_address,
1749                 rmrr->end_address + 1);
1750 }
1751
1752 #ifdef CONFIG_DMAR_GFX_WA
1753 struct iommu_prepare_data {
1754         struct pci_dev *pdev;
1755         int ret;
1756 };
1757
1758 static int __init iommu_prepare_work_fn(unsigned long start_pfn,
1759                                          unsigned long end_pfn, void *datax)
1760 {
1761         struct iommu_prepare_data *data;
1762
1763         data = (struct iommu_prepare_data *)datax;
1764
1765         data->ret = iommu_prepare_identity_map(data->pdev,
1766                                 start_pfn<<PAGE_SHIFT, end_pfn<<PAGE_SHIFT);
1767         return data->ret;
1768
1769 }
1770
1771 static int __init iommu_prepare_with_active_regions(struct pci_dev *pdev)
1772 {
1773         int nid;
1774         struct iommu_prepare_data data;
1775
1776         data.pdev = pdev;
1777         data.ret = 0;
1778
1779         for_each_online_node(nid) {
1780                 work_with_active_regions(nid, iommu_prepare_work_fn, &data);
1781                 if (data.ret)
1782                         return data.ret;
1783         }
1784         return data.ret;
1785 }
1786
1787 static void __init iommu_prepare_gfx_mapping(void)
1788 {
1789         struct pci_dev *pdev = NULL;
1790         int ret;
1791
1792         for_each_pci_dev(pdev) {
1793                 if (pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO ||
1794                                 !IS_GFX_DEVICE(pdev))
1795                         continue;
1796                 printk(KERN_INFO "IOMMU: gfx device %s 1-1 mapping\n",
1797                         pci_name(pdev));
1798                 ret = iommu_prepare_with_active_regions(pdev);
1799                 if (ret)
1800                         printk(KERN_ERR "IOMMU: mapping reserved region failed\n");
1801         }
1802 }
1803 #else /* !CONFIG_DMAR_GFX_WA */
1804 static inline void iommu_prepare_gfx_mapping(void)
1805 {
1806         return;
1807 }
1808 #endif
1809
1810 #ifdef CONFIG_DMAR_FLOPPY_WA
1811 static inline void iommu_prepare_isa(void)
1812 {
1813         struct pci_dev *pdev;
1814         int ret;
1815
1816         pdev = pci_get_class(PCI_CLASS_BRIDGE_ISA << 8, NULL);
1817         if (!pdev)
1818                 return;
1819
1820         printk(KERN_INFO "IOMMU: Prepare 0-16M unity mapping for LPC\n");
1821         ret = iommu_prepare_identity_map(pdev, 0, 16*1024*1024);
1822
1823         if (ret)
1824                 printk(KERN_ERR "IOMMU: Failed to create 0-64M identity map, "
1825                         "floppy might not work\n");
1826
1827 }
1828 #else
1829 static inline void iommu_prepare_isa(void)
1830 {
1831         return;
1832 }
1833 #endif /* !CONFIG_DMAR_FLPY_WA */
1834
1835 static int __init init_dmars(void)
1836 {
1837         struct dmar_drhd_unit *drhd;
1838         struct dmar_rmrr_unit *rmrr;
1839         struct pci_dev *pdev;
1840         struct intel_iommu *iommu;
1841         int i, ret;
1842
1843         /*
1844          * for each drhd
1845          *    allocate root
1846          *    initialize and program root entry to not present
1847          * endfor
1848          */
1849         for_each_drhd_unit(drhd) {
1850                 g_num_of_iommus++;
1851                 /*
1852                  * lock not needed as this is only incremented in the single
1853                  * threaded kernel __init code path all other access are read
1854                  * only
1855                  */
1856         }
1857
1858         g_iommus = kcalloc(g_num_of_iommus, sizeof(struct intel_iommu *),
1859                         GFP_KERNEL);
1860         if (!g_iommus) {
1861                 printk(KERN_ERR "Allocating global iommu array failed\n");
1862                 ret = -ENOMEM;
1863                 goto error;
1864         }
1865
1866         deferred_flush = kzalloc(g_num_of_iommus *
1867                 sizeof(struct deferred_flush_tables), GFP_KERNEL);
1868         if (!deferred_flush) {
1869                 kfree(g_iommus);
1870                 ret = -ENOMEM;
1871                 goto error;
1872         }
1873
1874         for_each_drhd_unit(drhd) {
1875                 if (drhd->ignored)
1876                         continue;
1877
1878                 iommu = drhd->iommu;
1879                 g_iommus[iommu->seq_id] = iommu;
1880
1881                 ret = iommu_init_domains(iommu);
1882                 if (ret)
1883                         goto error;
1884
1885                 /*
1886                  * TBD:
1887                  * we could share the same root & context tables
1888                  * amoung all IOMMU's. Need to Split it later.
1889                  */
1890                 ret = iommu_alloc_root_entry(iommu);
1891                 if (ret) {
1892                         printk(KERN_ERR "IOMMU: allocate root entry failed\n");
1893                         goto error;
1894                 }
1895         }
1896
1897         /*
1898          * Start from the sane iommu hardware state.
1899          */
1900         for_each_drhd_unit(drhd) {
1901                 if (drhd->ignored)
1902                         continue;
1903
1904                 iommu = drhd->iommu;
1905
1906                 /*
1907                  * If the queued invalidation is already initialized by us
1908                  * (for example, while enabling interrupt-remapping) then
1909                  * we got the things already rolling from a sane state.
1910                  */
1911                 if (iommu->qi)
1912                         continue;
1913
1914                 /*
1915                  * Clear any previous faults.
1916                  */
1917                 dmar_fault(-1, iommu);
1918                 /*
1919                  * Disable queued invalidation if supported and already enabled
1920                  * before OS handover.
1921                  */
1922                 dmar_disable_qi(iommu);
1923         }
1924
1925         for_each_drhd_unit(drhd) {
1926                 if (drhd->ignored)
1927                         continue;
1928
1929                 iommu = drhd->iommu;
1930
1931                 if (dmar_enable_qi(iommu)) {
1932                         /*
1933                          * Queued Invalidate not enabled, use Register Based
1934                          * Invalidate
1935                          */
1936                         iommu->flush.flush_context = __iommu_flush_context;
1937                         iommu->flush.flush_iotlb = __iommu_flush_iotlb;
1938                         printk(KERN_INFO "IOMMU 0x%Lx: using Register based "
1939                                "invalidation\n",
1940                                (unsigned long long)drhd->reg_base_addr);
1941                 } else {
1942                         iommu->flush.flush_context = qi_flush_context;
1943                         iommu->flush.flush_iotlb = qi_flush_iotlb;
1944                         printk(KERN_INFO "IOMMU 0x%Lx: using Queued "
1945                                "invalidation\n",
1946                                (unsigned long long)drhd->reg_base_addr);
1947                 }
1948         }
1949
1950         /*
1951          * For each rmrr
1952          *   for each dev attached to rmrr
1953          *   do
1954          *     locate drhd for dev, alloc domain for dev
1955          *     allocate free domain
1956          *     allocate page table entries for rmrr
1957          *     if context not allocated for bus
1958          *           allocate and init context
1959          *           set present in root table for this bus
1960          *     init context with domain, translation etc
1961          *    endfor
1962          * endfor
1963          */
1964         for_each_rmrr_units(rmrr) {
1965                 for (i = 0; i < rmrr->devices_cnt; i++) {
1966                         pdev = rmrr->devices[i];
1967                         /* some BIOS lists non-exist devices in DMAR table */
1968                         if (!pdev)
1969                                 continue;
1970                         ret = iommu_prepare_rmrr_dev(rmrr, pdev);
1971                         if (ret)
1972                                 printk(KERN_ERR
1973                                  "IOMMU: mapping reserved region failed\n");
1974                 }
1975         }
1976
1977         iommu_prepare_gfx_mapping();
1978
1979         iommu_prepare_isa();
1980
1981         /*
1982          * for each drhd
1983          *   enable fault log
1984          *   global invalidate context cache
1985          *   global invalidate iotlb
1986          *   enable translation
1987          */
1988         for_each_drhd_unit(drhd) {
1989                 if (drhd->ignored)
1990                         continue;
1991                 iommu = drhd->iommu;
1992
1993                 iommu_flush_write_buffer(iommu);
1994
1995                 ret = dmar_set_interrupt(iommu);
1996                 if (ret)
1997                         goto error;
1998
1999                 iommu_set_root_entry(iommu);
2000
2001                 iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL,
2002                                            0);
2003                 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH,
2004                                          0);
2005                 iommu_disable_protect_mem_regions(iommu);
2006
2007                 ret = iommu_enable_translation(iommu);
2008                 if (ret)
2009                         goto error;
2010         }
2011
2012         return 0;
2013 error:
2014         for_each_drhd_unit(drhd) {
2015                 if (drhd->ignored)
2016                         continue;
2017                 iommu = drhd->iommu;
2018                 free_iommu(iommu);
2019         }
2020         kfree(g_iommus);
2021         return ret;
2022 }
2023
2024 static inline u64 aligned_size(u64 host_addr, size_t size)
2025 {
2026         u64 addr;
2027         addr = (host_addr & (~PAGE_MASK)) + size;
2028         return PAGE_ALIGN(addr);
2029 }
2030
2031 struct iova *
2032 iommu_alloc_iova(struct dmar_domain *domain, size_t size, u64 end)
2033 {
2034         struct iova *piova;
2035
2036         /* Make sure it's in range */
2037         end = min_t(u64, DOMAIN_MAX_ADDR(domain->gaw), end);
2038         if (!size || (IOVA_START_ADDR + size > end))
2039                 return NULL;
2040
2041         piova = alloc_iova(&domain->iovad,
2042                         size >> PAGE_SHIFT, IOVA_PFN(end), 1);
2043         return piova;
2044 }
2045
2046 static struct iova *
2047 __intel_alloc_iova(struct device *dev, struct dmar_domain *domain,
2048                    size_t size, u64 dma_mask)
2049 {
2050         struct pci_dev *pdev = to_pci_dev(dev);
2051         struct iova *iova = NULL;
2052
2053         if (dma_mask <= DMA_32BIT_MASK || dmar_forcedac)
2054                 iova = iommu_alloc_iova(domain, size, dma_mask);
2055         else {
2056                 /*
2057                  * First try to allocate an io virtual address in
2058                  * DMA_32BIT_MASK and if that fails then try allocating
2059                  * from higher range
2060                  */
2061                 iova = iommu_alloc_iova(domain, size, DMA_32BIT_MASK);
2062                 if (!iova)
2063                         iova = iommu_alloc_iova(domain, size, dma_mask);
2064         }
2065
2066         if (!iova) {
2067                 printk(KERN_ERR"Allocating iova for %s failed", pci_name(pdev));
2068                 return NULL;
2069         }
2070
2071         return iova;
2072 }
2073
2074 static struct dmar_domain *
2075 get_valid_domain_for_dev(struct pci_dev *pdev)
2076 {
2077         struct dmar_domain *domain;
2078         int ret;
2079
2080         domain = get_domain_for_dev(pdev,
2081                         DEFAULT_DOMAIN_ADDRESS_WIDTH);
2082         if (!domain) {
2083                 printk(KERN_ERR
2084                         "Allocating domain for %s failed", pci_name(pdev));
2085                 return NULL;
2086         }
2087
2088         /* make sure context mapping is ok */
2089         if (unlikely(!domain_context_mapped(pdev))) {
2090                 ret = domain_context_mapping(domain, pdev);
2091                 if (ret) {
2092                         printk(KERN_ERR
2093                                 "Domain context map for %s failed",
2094                                 pci_name(pdev));
2095                         return NULL;
2096                 }
2097         }
2098
2099         return domain;
2100 }
2101
2102 static dma_addr_t __intel_map_single(struct device *hwdev, phys_addr_t paddr,
2103                                      size_t size, int dir, u64 dma_mask)
2104 {
2105         struct pci_dev *pdev = to_pci_dev(hwdev);
2106         struct dmar_domain *domain;
2107         phys_addr_t start_paddr;
2108         struct iova *iova;
2109         int prot = 0;
2110         int ret;
2111         struct intel_iommu *iommu;
2112
2113         BUG_ON(dir == DMA_NONE);
2114         if (pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
2115                 return paddr;
2116
2117         domain = get_valid_domain_for_dev(pdev);
2118         if (!domain)
2119                 return 0;
2120
2121         iommu = domain_get_iommu(domain);
2122         size = aligned_size((u64)paddr, size);
2123
2124         iova = __intel_alloc_iova(hwdev, domain, size, pdev->dma_mask);
2125         if (!iova)
2126                 goto error;
2127
2128         start_paddr = (phys_addr_t)iova->pfn_lo << PAGE_SHIFT;
2129
2130         /*
2131          * Check if DMAR supports zero-length reads on write only
2132          * mappings..
2133          */
2134         if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
2135                         !cap_zlr(iommu->cap))
2136                 prot |= DMA_PTE_READ;
2137         if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
2138                 prot |= DMA_PTE_WRITE;
2139         /*
2140          * paddr - (paddr + size) might be partial page, we should map the whole
2141          * page.  Note: if two part of one page are separately mapped, we
2142          * might have two guest_addr mapping to the same host paddr, but this
2143          * is not a big problem
2144          */
2145         ret = domain_page_mapping(domain, start_paddr,
2146                 ((u64)paddr) & PAGE_MASK, size, prot);
2147         if (ret)
2148                 goto error;
2149
2150         /* it's a non-present to present mapping */
2151         ret = iommu_flush_iotlb_psi(iommu, domain->id,
2152                         start_paddr, size >> VTD_PAGE_SHIFT, 1);
2153         if (ret)
2154                 iommu_flush_write_buffer(iommu);
2155
2156         return start_paddr + ((u64)paddr & (~PAGE_MASK));
2157
2158 error:
2159         if (iova)
2160                 __free_iova(&domain->iovad, iova);
2161         printk(KERN_ERR"Device %s request: %zx@%llx dir %d --- failed\n",
2162                 pci_name(pdev), size, (unsigned long long)paddr, dir);
2163         return 0;
2164 }
2165
2166 static dma_addr_t intel_map_page(struct device *dev, struct page *page,
2167                                  unsigned long offset, size_t size,
2168                                  enum dma_data_direction dir,
2169                                  struct dma_attrs *attrs)
2170 {
2171         return __intel_map_single(dev, page_to_phys(page) + offset, size,
2172                                   dir, to_pci_dev(dev)->dma_mask);
2173 }
2174
2175 static void flush_unmaps(void)
2176 {
2177         int i, j;
2178
2179         timer_on = 0;
2180
2181         /* just flush them all */
2182         for (i = 0; i < g_num_of_iommus; i++) {
2183                 struct intel_iommu *iommu = g_iommus[i];
2184                 if (!iommu)
2185                         continue;
2186
2187                 if (deferred_flush[i].next) {
2188                         iommu->flush.flush_iotlb(iommu, 0, 0, 0,
2189                                                  DMA_TLB_GLOBAL_FLUSH, 0);
2190                         for (j = 0; j < deferred_flush[i].next; j++) {
2191                                 __free_iova(&deferred_flush[i].domain[j]->iovad,
2192                                                 deferred_flush[i].iova[j]);
2193                         }
2194                         deferred_flush[i].next = 0;
2195                 }
2196         }
2197
2198         list_size = 0;
2199 }
2200
2201 static void flush_unmaps_timeout(unsigned long data)
2202 {
2203         unsigned long flags;
2204
2205         spin_lock_irqsave(&async_umap_flush_lock, flags);
2206         flush_unmaps();
2207         spin_unlock_irqrestore(&async_umap_flush_lock, flags);
2208 }
2209
2210 static void add_unmap(struct dmar_domain *dom, struct iova *iova)
2211 {
2212         unsigned long flags;
2213         int next, iommu_id;
2214         struct intel_iommu *iommu;
2215
2216         spin_lock_irqsave(&async_umap_flush_lock, flags);
2217         if (list_size == HIGH_WATER_MARK)
2218                 flush_unmaps();
2219
2220         iommu = domain_get_iommu(dom);
2221         iommu_id = iommu->seq_id;
2222
2223         next = deferred_flush[iommu_id].next;
2224         deferred_flush[iommu_id].domain[next] = dom;
2225         deferred_flush[iommu_id].iova[next] = iova;
2226         deferred_flush[iommu_id].next++;
2227
2228         if (!timer_on) {
2229                 mod_timer(&unmap_timer, jiffies + msecs_to_jiffies(10));
2230                 timer_on = 1;
2231         }
2232         list_size++;
2233         spin_unlock_irqrestore(&async_umap_flush_lock, flags);
2234 }
2235
2236 static void intel_unmap_page(struct device *dev, dma_addr_t dev_addr,
2237                              size_t size, enum dma_data_direction dir,
2238                              struct dma_attrs *attrs)
2239 {
2240         struct pci_dev *pdev = to_pci_dev(dev);
2241         struct dmar_domain *domain;
2242         unsigned long start_addr;
2243         struct iova *iova;
2244         struct intel_iommu *iommu;
2245
2246         if (pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
2247                 return;
2248         domain = find_domain(pdev);
2249         BUG_ON(!domain);
2250
2251         iommu = domain_get_iommu(domain);
2252
2253         iova = find_iova(&domain->iovad, IOVA_PFN(dev_addr));
2254         if (!iova)
2255                 return;
2256
2257         start_addr = iova->pfn_lo << PAGE_SHIFT;
2258         size = aligned_size((u64)dev_addr, size);
2259
2260         pr_debug("Device %s unmapping: %zx@%llx\n",
2261                 pci_name(pdev), size, (unsigned long long)start_addr);
2262
2263         /*  clear the whole page */
2264         dma_pte_clear_range(domain, start_addr, start_addr + size);
2265         /* free page tables */
2266         dma_pte_free_pagetable(domain, start_addr, start_addr + size);
2267         if (intel_iommu_strict) {
2268                 if (iommu_flush_iotlb_psi(iommu,
2269                         domain->id, start_addr, size >> VTD_PAGE_SHIFT, 0))
2270                         iommu_flush_write_buffer(iommu);
2271                 /* free iova */
2272                 __free_iova(&domain->iovad, iova);
2273         } else {
2274                 add_unmap(domain, iova);
2275                 /*
2276                  * queue up the release of the unmap to save the 1/6th of the
2277                  * cpu used up by the iotlb flush operation...
2278                  */
2279         }
2280 }
2281
2282 static void intel_unmap_single(struct device *dev, dma_addr_t dev_addr, size_t size,
2283                                int dir)
2284 {
2285         intel_unmap_page(dev, dev_addr, size, dir, NULL);
2286 }
2287
2288 static void *intel_alloc_coherent(struct device *hwdev, size_t size,
2289                                   dma_addr_t *dma_handle, gfp_t flags)
2290 {
2291         void *vaddr;
2292         int order;
2293
2294         size = PAGE_ALIGN(size);
2295         order = get_order(size);
2296         flags &= ~(GFP_DMA | GFP_DMA32);
2297
2298         vaddr = (void *)__get_free_pages(flags, order);
2299         if (!vaddr)
2300                 return NULL;
2301         memset(vaddr, 0, size);
2302
2303         *dma_handle = __intel_map_single(hwdev, virt_to_bus(vaddr), size,
2304                                          DMA_BIDIRECTIONAL,
2305                                          hwdev->coherent_dma_mask);
2306         if (*dma_handle)
2307                 return vaddr;
2308         free_pages((unsigned long)vaddr, order);
2309         return NULL;
2310 }
2311
2312 static void intel_free_coherent(struct device *hwdev, size_t size, void *vaddr,
2313                                 dma_addr_t dma_handle)
2314 {
2315         int order;
2316
2317         size = PAGE_ALIGN(size);
2318         order = get_order(size);
2319
2320         intel_unmap_single(hwdev, dma_handle, size, DMA_BIDIRECTIONAL);
2321         free_pages((unsigned long)vaddr, order);
2322 }
2323
2324 static void intel_unmap_sg(struct device *hwdev, struct scatterlist *sglist,
2325                            int nelems, enum dma_data_direction dir,
2326                            struct dma_attrs *attrs)
2327 {
2328         int i;
2329         struct pci_dev *pdev = to_pci_dev(hwdev);
2330         struct dmar_domain *domain;
2331         unsigned long start_addr;
2332         struct iova *iova;
2333         size_t size = 0;
2334         phys_addr_t addr;
2335         struct scatterlist *sg;
2336         struct intel_iommu *iommu;
2337
2338         if (pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
2339                 return;
2340
2341         domain = find_domain(pdev);
2342         BUG_ON(!domain);
2343
2344         iommu = domain_get_iommu(domain);
2345
2346         iova = find_iova(&domain->iovad, IOVA_PFN(sglist[0].dma_address));
2347         if (!iova)
2348                 return;
2349         for_each_sg(sglist, sg, nelems, i) {
2350                 addr = page_to_phys(sg_page(sg)) + sg->offset;
2351                 size += aligned_size((u64)addr, sg->length);
2352         }
2353
2354         start_addr = iova->pfn_lo << PAGE_SHIFT;
2355
2356         /*  clear the whole page */
2357         dma_pte_clear_range(domain, start_addr, start_addr + size);
2358         /* free page tables */
2359         dma_pte_free_pagetable(domain, start_addr, start_addr + size);
2360
2361         if (iommu_flush_iotlb_psi(iommu, domain->id, start_addr,
2362                         size >> VTD_PAGE_SHIFT, 0))
2363                 iommu_flush_write_buffer(iommu);
2364
2365         /* free iova */
2366         __free_iova(&domain->iovad, iova);
2367 }
2368
2369 static int intel_nontranslate_map_sg(struct device *hddev,
2370         struct scatterlist *sglist, int nelems, int dir)
2371 {
2372         int i;
2373         struct scatterlist *sg;
2374
2375         for_each_sg(sglist, sg, nelems, i) {
2376                 BUG_ON(!sg_page(sg));
2377                 sg->dma_address = page_to_phys(sg_page(sg)) + sg->offset;
2378                 sg->dma_length = sg->length;
2379         }
2380         return nelems;
2381 }
2382
2383 static int intel_map_sg(struct device *hwdev, struct scatterlist *sglist, int nelems,
2384                         enum dma_data_direction dir, struct dma_attrs *attrs)
2385 {
2386         phys_addr_t addr;
2387         int i;
2388         struct pci_dev *pdev = to_pci_dev(hwdev);
2389         struct dmar_domain *domain;
2390         size_t size = 0;
2391         int prot = 0;
2392         size_t offset = 0;
2393         struct iova *iova = NULL;
2394         int ret;
2395         struct scatterlist *sg;
2396         unsigned long start_addr;
2397         struct intel_iommu *iommu;
2398
2399         BUG_ON(dir == DMA_NONE);
2400         if (pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
2401                 return intel_nontranslate_map_sg(hwdev, sglist, nelems, dir);
2402
2403         domain = get_valid_domain_for_dev(pdev);
2404         if (!domain)
2405                 return 0;
2406
2407         iommu = domain_get_iommu(domain);
2408
2409         for_each_sg(sglist, sg, nelems, i) {
2410                 addr = page_to_phys(sg_page(sg)) + sg->offset;
2411                 size += aligned_size((u64)addr, sg->length);
2412         }
2413
2414         iova = __intel_alloc_iova(hwdev, domain, size, pdev->dma_mask);
2415         if (!iova) {
2416                 sglist->dma_length = 0;
2417                 return 0;
2418         }
2419
2420         /*
2421          * Check if DMAR supports zero-length reads on write only
2422          * mappings..
2423          */
2424         if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
2425                         !cap_zlr(iommu->cap))
2426                 prot |= DMA_PTE_READ;
2427         if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
2428                 prot |= DMA_PTE_WRITE;
2429
2430         start_addr = iova->pfn_lo << PAGE_SHIFT;
2431         offset = 0;
2432         for_each_sg(sglist, sg, nelems, i) {
2433                 addr = page_to_phys(sg_page(sg)) + sg->offset;
2434                 size = aligned_size((u64)addr, sg->length);
2435                 ret = domain_page_mapping(domain, start_addr + offset,
2436                         ((u64)addr) & PAGE_MASK,
2437                         size, prot);
2438                 if (ret) {
2439                         /*  clear the page */
2440                         dma_pte_clear_range(domain, start_addr,
2441                                   start_addr + offset);
2442                         /* free page tables */
2443                         dma_pte_free_pagetable(domain, start_addr,
2444                                   start_addr + offset);
2445                         /* free iova */
2446                         __free_iova(&domain->iovad, iova);
2447                         return 0;
2448                 }
2449                 sg->dma_address = start_addr + offset +
2450                                 ((u64)addr & (~PAGE_MASK));
2451                 sg->dma_length = sg->length;
2452                 offset += size;
2453         }
2454
2455         /* it's a non-present to present mapping */
2456         if (iommu_flush_iotlb_psi(iommu, domain->id,
2457                         start_addr, offset >> VTD_PAGE_SHIFT, 1))
2458                 iommu_flush_write_buffer(iommu);
2459         return nelems;
2460 }
2461
2462 static int intel_mapping_error(struct device *dev, dma_addr_t dma_addr)
2463 {
2464         return !dma_addr;
2465 }
2466
2467 struct dma_map_ops intel_dma_ops = {
2468         .alloc_coherent = intel_alloc_coherent,
2469         .free_coherent = intel_free_coherent,
2470         .map_sg = intel_map_sg,
2471         .unmap_sg = intel_unmap_sg,
2472         .map_page = intel_map_page,
2473         .unmap_page = intel_unmap_page,
2474         .mapping_error = intel_mapping_error,
2475 };
2476
2477 static inline int iommu_domain_cache_init(void)
2478 {
2479         int ret = 0;
2480
2481         iommu_domain_cache = kmem_cache_create("iommu_domain",
2482                                          sizeof(struct dmar_domain),
2483                                          0,
2484                                          SLAB_HWCACHE_ALIGN,
2485
2486                                          NULL);
2487         if (!iommu_domain_cache) {
2488                 printk(KERN_ERR "Couldn't create iommu_domain cache\n");
2489                 ret = -ENOMEM;
2490         }
2491
2492         return ret;
2493 }
2494
2495 static inline int iommu_devinfo_cache_init(void)
2496 {
2497         int ret = 0;
2498
2499         iommu_devinfo_cache = kmem_cache_create("iommu_devinfo",
2500                                          sizeof(struct device_domain_info),
2501                                          0,
2502                                          SLAB_HWCACHE_ALIGN,
2503                                          NULL);
2504         if (!iommu_devinfo_cache) {
2505                 printk(KERN_ERR "Couldn't create devinfo cache\n");
2506                 ret = -ENOMEM;
2507         }
2508
2509         return ret;
2510 }
2511
2512 static inline int iommu_iova_cache_init(void)
2513 {
2514         int ret = 0;
2515
2516         iommu_iova_cache = kmem_cache_create("iommu_iova",
2517                                          sizeof(struct iova),
2518                                          0,
2519                                          SLAB_HWCACHE_ALIGN,
2520                                          NULL);
2521         if (!iommu_iova_cache) {
2522                 printk(KERN_ERR "Couldn't create iova cache\n");
2523                 ret = -ENOMEM;
2524         }
2525
2526         return ret;
2527 }
2528
2529 static int __init iommu_init_mempool(void)
2530 {
2531         int ret;
2532         ret = iommu_iova_cache_init();
2533         if (ret)
2534                 return ret;
2535
2536         ret = iommu_domain_cache_init();
2537         if (ret)
2538                 goto domain_error;
2539
2540         ret = iommu_devinfo_cache_init();
2541         if (!ret)
2542                 return ret;
2543
2544         kmem_cache_destroy(iommu_domain_cache);
2545 domain_error:
2546         kmem_cache_destroy(iommu_iova_cache);
2547
2548         return -ENOMEM;
2549 }
2550
2551 static void __init iommu_exit_mempool(void)
2552 {
2553         kmem_cache_destroy(iommu_devinfo_cache);
2554         kmem_cache_destroy(iommu_domain_cache);
2555         kmem_cache_destroy(iommu_iova_cache);
2556
2557 }
2558
2559 static void __init init_no_remapping_devices(void)
2560 {
2561         struct dmar_drhd_unit *drhd;
2562
2563         for_each_drhd_unit(drhd) {
2564                 if (!drhd->include_all) {
2565                         int i;
2566                         for (i = 0; i < drhd->devices_cnt; i++)
2567                                 if (drhd->devices[i] != NULL)
2568                                         break;
2569                         /* ignore DMAR unit if no pci devices exist */
2570                         if (i == drhd->devices_cnt)
2571                                 drhd->ignored = 1;
2572                 }
2573         }
2574
2575         if (dmar_map_gfx)
2576                 return;
2577
2578         for_each_drhd_unit(drhd) {
2579                 int i;
2580                 if (drhd->ignored || drhd->include_all)
2581                         continue;
2582
2583                 for (i = 0; i < drhd->devices_cnt; i++)
2584                         if (drhd->devices[i] &&
2585                                 !IS_GFX_DEVICE(drhd->devices[i]))
2586                                 break;
2587
2588                 if (i < drhd->devices_cnt)
2589                         continue;
2590
2591                 /* bypass IOMMU if it is just for gfx devices */
2592                 drhd->ignored = 1;
2593                 for (i = 0; i < drhd->devices_cnt; i++) {
2594                         if (!drhd->devices[i])
2595                                 continue;
2596                         drhd->devices[i]->dev.archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
2597                 }
2598         }
2599 }
2600
2601 #ifdef CONFIG_SUSPEND
2602 static int init_iommu_hw(void)
2603 {
2604         struct dmar_drhd_unit *drhd;
2605         struct intel_iommu *iommu = NULL;
2606
2607         for_each_active_iommu(iommu, drhd)
2608                 if (iommu->qi)
2609                         dmar_reenable_qi(iommu);
2610
2611         for_each_active_iommu(iommu, drhd) {
2612                 iommu_flush_write_buffer(iommu);
2613
2614                 iommu_set_root_entry(iommu);
2615
2616                 iommu->flush.flush_context(iommu, 0, 0, 0,
2617                                                 DMA_CCMD_GLOBAL_INVL, 0);
2618                 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
2619                                                 DMA_TLB_GLOBAL_FLUSH, 0);
2620                 iommu_disable_protect_mem_regions(iommu);
2621                 iommu_enable_translation(iommu);
2622         }
2623
2624         return 0;
2625 }
2626
2627 static void iommu_flush_all(void)
2628 {
2629         struct dmar_drhd_unit *drhd;
2630         struct intel_iommu *iommu;
2631
2632         for_each_active_iommu(iommu, drhd) {
2633                 iommu->flush.flush_context(iommu, 0, 0, 0,
2634                                                 DMA_CCMD_GLOBAL_INVL, 0);
2635                 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
2636                                                 DMA_TLB_GLOBAL_FLUSH, 0);
2637         }
2638 }
2639
2640 static int iommu_suspend(struct sys_device *dev, pm_message_t state)
2641 {
2642         struct dmar_drhd_unit *drhd;
2643         struct intel_iommu *iommu = NULL;
2644         unsigned long flag;
2645
2646         for_each_active_iommu(iommu, drhd) {
2647                 iommu->iommu_state = kzalloc(sizeof(u32) * MAX_SR_DMAR_REGS,
2648                                                  GFP_ATOMIC);
2649                 if (!iommu->iommu_state)
2650                         goto nomem;
2651         }
2652
2653         iommu_flush_all();
2654
2655         for_each_active_iommu(iommu, drhd) {
2656                 iommu_disable_translation(iommu);
2657
2658                 spin_lock_irqsave(&iommu->register_lock, flag);
2659
2660                 iommu->iommu_state[SR_DMAR_FECTL_REG] =
2661                         readl(iommu->reg + DMAR_FECTL_REG);
2662                 iommu->iommu_state[SR_DMAR_FEDATA_REG] =
2663                         readl(iommu->reg + DMAR_FEDATA_REG);
2664                 iommu->iommu_state[SR_DMAR_FEADDR_REG] =
2665                         readl(iommu->reg + DMAR_FEADDR_REG);
2666                 iommu->iommu_state[SR_DMAR_FEUADDR_REG] =
2667                         readl(iommu->reg + DMAR_FEUADDR_REG);
2668
2669                 spin_unlock_irqrestore(&iommu->register_lock, flag);
2670         }
2671         return 0;
2672
2673 nomem:
2674         for_each_active_iommu(iommu, drhd)
2675                 kfree(iommu->iommu_state);
2676
2677         return -ENOMEM;
2678 }
2679
2680 static int iommu_resume(struct sys_device *dev)
2681 {
2682         struct dmar_drhd_unit *drhd;
2683         struct intel_iommu *iommu = NULL;
2684         unsigned long flag;
2685
2686         if (init_iommu_hw()) {
2687                 WARN(1, "IOMMU setup failed, DMAR can not resume!\n");
2688                 return -EIO;
2689         }
2690
2691         for_each_active_iommu(iommu, drhd) {
2692
2693                 spin_lock_irqsave(&iommu->register_lock, flag);
2694
2695                 writel(iommu->iommu_state[SR_DMAR_FECTL_REG],
2696                         iommu->reg + DMAR_FECTL_REG);
2697                 writel(iommu->iommu_state[SR_DMAR_FEDATA_REG],
2698                         iommu->reg + DMAR_FEDATA_REG);
2699                 writel(iommu->iommu_state[SR_DMAR_FEADDR_REG],
2700                         iommu->reg + DMAR_FEADDR_REG);
2701                 writel(iommu->iommu_state[SR_DMAR_FEUADDR_REG],
2702                         iommu->reg + DMAR_FEUADDR_REG);
2703
2704                 spin_unlock_irqrestore(&iommu->register_lock, flag);
2705         }
2706
2707         for_each_active_iommu(iommu, drhd)
2708                 kfree(iommu->iommu_state);
2709
2710         return 0;
2711 }
2712
2713 static struct sysdev_class iommu_sysclass = {
2714         .name           = "iommu",
2715         .resume         = iommu_resume,
2716         .suspend        = iommu_suspend,
2717 };
2718
2719 static struct sys_device device_iommu = {
2720         .cls    = &iommu_sysclass,
2721 };
2722
2723 static int __init init_iommu_sysfs(void)
2724 {
2725         int error;
2726
2727         error = sysdev_class_register(&iommu_sysclass);
2728         if (error)
2729                 return error;
2730
2731         error = sysdev_register(&device_iommu);
2732         if (error)
2733                 sysdev_class_unregister(&iommu_sysclass);
2734
2735         return error;
2736 }
2737
2738 #else
2739 static int __init init_iommu_sysfs(void)
2740 {
2741         return 0;
2742 }
2743 #endif  /* CONFIG_PM */
2744
2745 int __init intel_iommu_init(void)
2746 {
2747         int ret = 0;
2748
2749         if (dmar_table_init())
2750                 return  -ENODEV;
2751
2752         if (dmar_dev_scope_init())
2753                 return  -ENODEV;
2754
2755         /*
2756          * Check the need for DMA-remapping initialization now.
2757          * Above initialization will also be used by Interrupt-remapping.
2758          */
2759         if (no_iommu || swiotlb || dmar_disabled)
2760                 return -ENODEV;
2761
2762         iommu_init_mempool();
2763         dmar_init_reserved_ranges();
2764
2765         init_no_remapping_devices();
2766
2767         ret = init_dmars();
2768         if (ret) {
2769                 printk(KERN_ERR "IOMMU: dmar init failed\n");
2770                 put_iova_domain(&reserved_iova_list);
2771                 iommu_exit_mempool();
2772                 return ret;
2773         }
2774         printk(KERN_INFO
2775         "PCI-DMA: Intel(R) Virtualization Technology for Directed I/O\n");
2776
2777         init_timer(&unmap_timer);
2778         force_iommu = 1;
2779         dma_ops = &intel_dma_ops;
2780         init_iommu_sysfs();
2781
2782         register_iommu(&intel_iommu_ops);
2783
2784         return 0;
2785 }
2786
2787 static int vm_domain_add_dev_info(struct dmar_domain *domain,
2788                                   struct pci_dev *pdev)
2789 {
2790         struct device_domain_info *info;
2791         unsigned long flags;
2792
2793         info = alloc_devinfo_mem();
2794         if (!info)
2795                 return -ENOMEM;
2796
2797         info->bus = pdev->bus->number;
2798         info->devfn = pdev->devfn;
2799         info->dev = pdev;
2800         info->domain = domain;
2801
2802         spin_lock_irqsave(&device_domain_lock, flags);
2803         list_add(&info->link, &domain->devices);
2804         list_add(&info->global, &device_domain_list);
2805         pdev->dev.archdata.iommu = info;
2806         spin_unlock_irqrestore(&device_domain_lock, flags);
2807
2808         return 0;
2809 }
2810
2811 static void iommu_detach_dependent_devices(struct intel_iommu *iommu,
2812                                            struct pci_dev *pdev)
2813 {
2814         struct pci_dev *tmp, *parent;
2815
2816         if (!iommu || !pdev)
2817                 return;
2818
2819         /* dependent device detach */
2820         tmp = pci_find_upstream_pcie_bridge(pdev);
2821         /* Secondary interface's bus number and devfn 0 */
2822         if (tmp) {
2823                 parent = pdev->bus->self;
2824                 while (parent != tmp) {
2825                         iommu_detach_dev(iommu, parent->bus->number,
2826                                 parent->devfn);
2827                         parent = parent->bus->self;
2828                 }
2829                 if (tmp->is_pcie) /* this is a PCIE-to-PCI bridge */
2830                         iommu_detach_dev(iommu,
2831                                 tmp->subordinate->number, 0);
2832                 else /* this is a legacy PCI bridge */
2833                         iommu_detach_dev(iommu,
2834                                 tmp->bus->number, tmp->devfn);
2835         }
2836 }
2837
2838 static void vm_domain_remove_one_dev_info(struct dmar_domain *domain,
2839                                           struct pci_dev *pdev)
2840 {
2841         struct device_domain_info *info;
2842         struct intel_iommu *iommu;
2843         unsigned long flags;
2844         int found = 0;
2845         struct list_head *entry, *tmp;
2846
2847         iommu = device_to_iommu(pdev->bus->number, pdev->devfn);
2848         if (!iommu)
2849                 return;
2850
2851         spin_lock_irqsave(&device_domain_lock, flags);
2852         list_for_each_safe(entry, tmp, &domain->devices) {
2853                 info = list_entry(entry, struct device_domain_info, link);
2854                 if (info->bus == pdev->bus->number &&
2855                     info->devfn == pdev->devfn) {
2856                         list_del(&info->link);
2857                         list_del(&info->global);
2858                         if (info->dev)
2859                                 info->dev->dev.archdata.iommu = NULL;
2860                         spin_unlock_irqrestore(&device_domain_lock, flags);
2861
2862                         iommu_detach_dev(iommu, info->bus, info->devfn);
2863                         iommu_detach_dependent_devices(iommu, pdev);
2864                         free_devinfo_mem(info);
2865
2866                         spin_lock_irqsave(&device_domain_lock, flags);
2867
2868                         if (found)
2869                                 break;
2870                         else
2871                                 continue;
2872                 }
2873
2874                 /* if there is no other devices under the same iommu
2875                  * owned by this domain, clear this iommu in iommu_bmp
2876                  * update iommu count and coherency
2877                  */
2878                 if (device_to_iommu(info->bus, info->devfn) == iommu)
2879                         found = 1;
2880         }
2881
2882         if (found == 0) {
2883                 unsigned long tmp_flags;
2884                 spin_lock_irqsave(&domain->iommu_lock, tmp_flags);
2885                 clear_bit(iommu->seq_id, &domain->iommu_bmp);
2886                 domain->iommu_count--;
2887                 domain_update_iommu_cap(domain);
2888                 spin_unlock_irqrestore(&domain->iommu_lock, tmp_flags);
2889         }
2890
2891         spin_unlock_irqrestore(&device_domain_lock, flags);
2892 }
2893
2894 static void vm_domain_remove_all_dev_info(struct dmar_domain *domain)
2895 {
2896         struct device_domain_info *info;
2897         struct intel_iommu *iommu;
2898         unsigned long flags1, flags2;
2899
2900         spin_lock_irqsave(&device_domain_lock, flags1);
2901         while (!list_empty(&domain->devices)) {
2902                 info = list_entry(domain->devices.next,
2903                         struct device_domain_info, link);
2904                 list_del(&info->link);
2905                 list_del(&info->global);
2906                 if (info->dev)
2907                         info->dev->dev.archdata.iommu = NULL;
2908
2909                 spin_unlock_irqrestore(&device_domain_lock, flags1);
2910
2911                 iommu = device_to_iommu(info->bus, info->devfn);
2912                 iommu_detach_dev(iommu, info->bus, info->devfn);
2913                 iommu_detach_dependent_devices(iommu, info->dev);
2914
2915                 /* clear this iommu in iommu_bmp, update iommu count
2916                  * and capabilities
2917                  */
2918                 spin_lock_irqsave(&domain->iommu_lock, flags2);
2919                 if (test_and_clear_bit(iommu->seq_id,
2920                                        &domain->iommu_bmp)) {
2921                         domain->iommu_count--;
2922                         domain_update_iommu_cap(domain);
2923                 }
2924                 spin_unlock_irqrestore(&domain->iommu_lock, flags2);
2925
2926                 free_devinfo_mem(info);
2927                 spin_lock_irqsave(&device_domain_lock, flags1);
2928         }
2929         spin_unlock_irqrestore(&device_domain_lock, flags1);
2930 }
2931
2932 /* domain id for virtual machine, it won't be set in context */
2933 static unsigned long vm_domid;
2934
2935 static int vm_domain_min_agaw(struct dmar_domain *domain)
2936 {
2937         int i;
2938         int min_agaw = domain->agaw;
2939
2940         i = find_first_bit(&domain->iommu_bmp, g_num_of_iommus);
2941         for (; i < g_num_of_iommus; ) {
2942                 if (min_agaw > g_iommus[i]->agaw)
2943                         min_agaw = g_iommus[i]->agaw;
2944
2945                 i = find_next_bit(&domain->iommu_bmp, g_num_of_iommus, i+1);
2946         }
2947
2948         return min_agaw;
2949 }
2950
2951 static struct dmar_domain *iommu_alloc_vm_domain(void)
2952 {
2953         struct dmar_domain *domain;
2954
2955         domain = alloc_domain_mem();
2956         if (!domain)
2957                 return NULL;
2958
2959         domain->id = vm_domid++;
2960         memset(&domain->iommu_bmp, 0, sizeof(unsigned long));
2961         domain->flags = DOMAIN_FLAG_VIRTUAL_MACHINE;
2962
2963         return domain;
2964 }
2965
2966 static int vm_domain_init(struct dmar_domain *domain, int guest_width)
2967 {
2968         int adjust_width;
2969
2970         init_iova_domain(&domain->iovad, DMA_32BIT_PFN);
2971         spin_lock_init(&domain->mapping_lock);
2972         spin_lock_init(&domain->iommu_lock);
2973
2974         domain_reserve_special_ranges(domain);
2975
2976         /* calculate AGAW */
2977         domain->gaw = guest_width;
2978         adjust_width = guestwidth_to_adjustwidth(guest_width);
2979         domain->agaw = width_to_agaw(adjust_width);
2980
2981         INIT_LIST_HEAD(&domain->devices);
2982
2983         domain->iommu_count = 0;
2984         domain->iommu_coherency = 0;
2985         domain->max_addr = 0;
2986
2987         /* always allocate the top pgd */
2988         domain->pgd = (struct dma_pte *)alloc_pgtable_page();
2989         if (!domain->pgd)
2990                 return -ENOMEM;
2991         domain_flush_cache(domain, domain->pgd, PAGE_SIZE);
2992         return 0;
2993 }
2994
2995 static void iommu_free_vm_domain(struct dmar_domain *domain)
2996 {
2997         unsigned long flags;
2998         struct dmar_drhd_unit *drhd;
2999         struct intel_iommu *iommu;
3000         unsigned long i;
3001         unsigned long ndomains;
3002
3003         for_each_drhd_unit(drhd) {
3004                 if (drhd->ignored)
3005                         continue;
3006                 iommu = drhd->iommu;
3007
3008                 ndomains = cap_ndoms(iommu->cap);
3009                 i = find_first_bit(iommu->domain_ids, ndomains);
3010                 for (; i < ndomains; ) {
3011                         if (iommu->domains[i] == domain) {
3012                                 spin_lock_irqsave(&iommu->lock, flags);
3013                                 clear_bit(i, iommu->domain_ids);
3014                                 iommu->domains[i] = NULL;
3015                                 spin_unlock_irqrestore(&iommu->lock, flags);
3016                                 break;
3017                         }
3018                         i = find_next_bit(iommu->domain_ids, ndomains, i+1);
3019                 }
3020         }
3021 }
3022
3023 static void vm_domain_exit(struct dmar_domain *domain)
3024 {
3025         u64 end;
3026
3027         /* Domain 0 is reserved, so dont process it */
3028         if (!domain)
3029                 return;
3030
3031         vm_domain_remove_all_dev_info(domain);
3032         /* destroy iovas */
3033         put_iova_domain(&domain->iovad);
3034         end = DOMAIN_MAX_ADDR(domain->gaw);
3035         end = end & (~VTD_PAGE_MASK);
3036
3037         /* clear ptes */
3038         dma_pte_clear_range(domain, 0, end);
3039
3040         /* free page tables */
3041         dma_pte_free_pagetable(domain, 0, end);
3042
3043         iommu_free_vm_domain(domain);
3044         free_domain_mem(domain);
3045 }
3046
3047 static int intel_iommu_domain_init(struct iommu_domain *domain)
3048 {
3049         struct dmar_domain *dmar_domain;
3050
3051         dmar_domain = iommu_alloc_vm_domain();
3052         if (!dmar_domain) {
3053                 printk(KERN_ERR
3054                         "intel_iommu_domain_init: dmar_domain == NULL\n");
3055                 return -ENOMEM;
3056         }
3057         if (vm_domain_init(dmar_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
3058                 printk(KERN_ERR
3059                         "intel_iommu_domain_init() failed\n");
3060                 vm_domain_exit(dmar_domain);
3061                 return -ENOMEM;
3062         }
3063         domain->priv = dmar_domain;
3064
3065         return 0;
3066 }
3067
3068 static void intel_iommu_domain_destroy(struct iommu_domain *domain)
3069 {
3070         struct dmar_domain *dmar_domain = domain->priv;
3071
3072         domain->priv = NULL;
3073         vm_domain_exit(dmar_domain);
3074 }
3075
3076 static int intel_iommu_attach_device(struct iommu_domain *domain,
3077                                      struct device *dev)
3078 {
3079         struct dmar_domain *dmar_domain = domain->priv;
3080         struct pci_dev *pdev = to_pci_dev(dev);
3081         struct intel_iommu *iommu;
3082         int addr_width;
3083         u64 end;
3084         int ret;
3085
3086         /* normally pdev is not mapped */
3087         if (unlikely(domain_context_mapped(pdev))) {
3088                 struct dmar_domain *old_domain;
3089
3090                 old_domain = find_domain(pdev);
3091                 if (old_domain) {
3092                         if (dmar_domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE)
3093                                 vm_domain_remove_one_dev_info(old_domain, pdev);
3094                         else
3095                                 domain_remove_dev_info(old_domain);
3096                 }
3097         }
3098
3099         iommu = device_to_iommu(pdev->bus->number, pdev->devfn);
3100         if (!iommu)
3101                 return -ENODEV;
3102
3103         /* check if this iommu agaw is sufficient for max mapped address */
3104         addr_width = agaw_to_width(iommu->agaw);
3105         end = DOMAIN_MAX_ADDR(addr_width);
3106         end = end & VTD_PAGE_MASK;
3107         if (end < dmar_domain->max_addr) {
3108                 printk(KERN_ERR "%s: iommu agaw (%d) is not "
3109                        "sufficient for the mapped address (%llx)\n",
3110                        __func__, iommu->agaw, dmar_domain->max_addr);
3111                 return -EFAULT;
3112         }
3113
3114         ret = domain_context_mapping(dmar_domain, pdev);
3115         if (ret)
3116                 return ret;
3117
3118         ret = vm_domain_add_dev_info(dmar_domain, pdev);
3119         return ret;
3120 }
3121
3122 static void intel_iommu_detach_device(struct iommu_domain *domain,
3123                                       struct device *dev)
3124 {
3125         struct dmar_domain *dmar_domain = domain->priv;
3126         struct pci_dev *pdev = to_pci_dev(dev);
3127
3128         vm_domain_remove_one_dev_info(dmar_domain, pdev);
3129 }
3130
3131 static int intel_iommu_map_range(struct iommu_domain *domain,
3132                                  unsigned long iova, phys_addr_t hpa,
3133                                  size_t size, int iommu_prot)
3134 {
3135         struct dmar_domain *dmar_domain = domain->priv;
3136         u64 max_addr;
3137         int addr_width;
3138         int prot = 0;
3139         int ret;
3140
3141         if (iommu_prot & IOMMU_READ)
3142                 prot |= DMA_PTE_READ;
3143         if (iommu_prot & IOMMU_WRITE)
3144                 prot |= DMA_PTE_WRITE;
3145         if ((iommu_prot & IOMMU_CACHE) && dmar_domain->iommu_snooping)
3146                 prot |= DMA_PTE_SNP;
3147
3148         max_addr = (iova & VTD_PAGE_MASK) + VTD_PAGE_ALIGN(size);
3149         if (dmar_domain->max_addr < max_addr) {
3150                 int min_agaw;
3151                 u64 end;
3152
3153                 /* check if minimum agaw is sufficient for mapped address */
3154                 min_agaw = vm_domain_min_agaw(dmar_domain);
3155                 addr_width = agaw_to_width(min_agaw);
3156                 end = DOMAIN_MAX_ADDR(addr_width);
3157                 end = end & VTD_PAGE_MASK;
3158                 if (end < max_addr) {
3159                         printk(KERN_ERR "%s: iommu agaw (%d) is not "
3160                                "sufficient for the mapped address (%llx)\n",
3161                                __func__, min_agaw, max_addr);
3162                         return -EFAULT;
3163                 }
3164                 dmar_domain->max_addr = max_addr;
3165         }
3166
3167         ret = domain_page_mapping(dmar_domain, iova, hpa, size, prot);
3168         return ret;
3169 }
3170
3171 static void intel_iommu_unmap_range(struct iommu_domain *domain,
3172                                     unsigned long iova, size_t size)
3173 {
3174         struct dmar_domain *dmar_domain = domain->priv;
3175         dma_addr_t base;
3176
3177         /* The address might not be aligned */
3178         base = iova & VTD_PAGE_MASK;
3179         size = VTD_PAGE_ALIGN(size);
3180         dma_pte_clear_range(dmar_domain, base, base + size);
3181
3182         if (dmar_domain->max_addr == base + size)
3183                 dmar_domain->max_addr = base;
3184 }
3185
3186 static phys_addr_t intel_iommu_iova_to_phys(struct iommu_domain *domain,
3187                                             unsigned long iova)
3188 {
3189         struct dmar_domain *dmar_domain = domain->priv;
3190         struct dma_pte *pte;
3191         u64 phys = 0;
3192
3193         pte = addr_to_dma_pte(dmar_domain, iova);
3194         if (pte)
3195                 phys = dma_pte_addr(pte);
3196
3197         return phys;
3198 }
3199
3200 static int intel_iommu_domain_has_cap(struct iommu_domain *domain,
3201                                       unsigned long cap)
3202 {
3203         struct dmar_domain *dmar_domain = domain->priv;
3204
3205         if (cap == IOMMU_CAP_CACHE_COHERENCY)
3206                 return dmar_domain->iommu_snooping;
3207
3208         return 0;
3209 }
3210
3211 static struct iommu_ops intel_iommu_ops = {
3212         .domain_init    = intel_iommu_domain_init,
3213         .domain_destroy = intel_iommu_domain_destroy,
3214         .attach_dev     = intel_iommu_attach_device,
3215         .detach_dev     = intel_iommu_detach_device,
3216         .map            = intel_iommu_map_range,
3217         .unmap          = intel_iommu_unmap_range,
3218         .iova_to_phys   = intel_iommu_iova_to_phys,
3219         .domain_has_cap = intel_iommu_domain_has_cap,
3220 };
3221
3222 static void __devinit quirk_iommu_rwbf(struct pci_dev *dev)
3223 {
3224         /*
3225          * Mobile 4 Series Chipset neglects to set RWBF capability,
3226          * but needs it:
3227          */
3228         printk(KERN_INFO "DMAR: Forcing write-buffer flush capability\n");
3229         rwbf_quirk = 1;
3230 }
3231
3232 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_rwbf);