Merge commit 'origin' into master
[linux-2.6] / drivers / pci / dmar.c
1 /*
2  * Copyright (c) 2006, Intel Corporation.
3  *
4  * This program is free software; you can redistribute it and/or modify it
5  * under the terms and conditions of the GNU General Public License,
6  * version 2, as published by the Free Software Foundation.
7  *
8  * This program is distributed in the hope it will be useful, but WITHOUT
9  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
11  * more details.
12  *
13  * You should have received a copy of the GNU General Public License along with
14  * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
15  * Place - Suite 330, Boston, MA 02111-1307 USA.
16  *
17  * Copyright (C) 2006-2008 Intel Corporation
18  * Author: Ashok Raj <ashok.raj@intel.com>
19  * Author: Shaohua Li <shaohua.li@intel.com>
20  * Author: Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>
21  *
22  * This file implements early detection/parsing of Remapping Devices
23  * reported to OS through BIOS via DMA remapping reporting (DMAR) ACPI
24  * tables.
25  *
26  * These routines are used by both DMA-remapping and Interrupt-remapping
27  */
28
29 #include <linux/pci.h>
30 #include <linux/dmar.h>
31 #include <linux/iova.h>
32 #include <linux/intel-iommu.h>
33 #include <linux/timer.h>
34
35 #undef PREFIX
36 #define PREFIX "DMAR:"
37
38 /* No locks are needed as DMA remapping hardware unit
39  * list is constructed at boot time and hotplug of
40  * these units are not supported by the architecture.
41  */
42 LIST_HEAD(dmar_drhd_units);
43
44 static struct acpi_table_header * __initdata dmar_tbl;
45
46 static void __init dmar_register_drhd_unit(struct dmar_drhd_unit *drhd)
47 {
48         /*
49          * add INCLUDE_ALL at the tail, so scan the list will find it at
50          * the very end.
51          */
52         if (drhd->include_all)
53                 list_add_tail(&drhd->list, &dmar_drhd_units);
54         else
55                 list_add(&drhd->list, &dmar_drhd_units);
56 }
57
58 static int __init dmar_parse_one_dev_scope(struct acpi_dmar_device_scope *scope,
59                                            struct pci_dev **dev, u16 segment)
60 {
61         struct pci_bus *bus;
62         struct pci_dev *pdev = NULL;
63         struct acpi_dmar_pci_path *path;
64         int count;
65
66         bus = pci_find_bus(segment, scope->bus);
67         path = (struct acpi_dmar_pci_path *)(scope + 1);
68         count = (scope->length - sizeof(struct acpi_dmar_device_scope))
69                 / sizeof(struct acpi_dmar_pci_path);
70
71         while (count) {
72                 if (pdev)
73                         pci_dev_put(pdev);
74                 /*
75                  * Some BIOSes list non-exist devices in DMAR table, just
76                  * ignore it
77                  */
78                 if (!bus) {
79                         printk(KERN_WARNING
80                         PREFIX "Device scope bus [%d] not found\n",
81                         scope->bus);
82                         break;
83                 }
84                 pdev = pci_get_slot(bus, PCI_DEVFN(path->dev, path->fn));
85                 if (!pdev) {
86                         printk(KERN_WARNING PREFIX
87                         "Device scope device [%04x:%02x:%02x.%02x] not found\n",
88                                 segment, bus->number, path->dev, path->fn);
89                         break;
90                 }
91                 path ++;
92                 count --;
93                 bus = pdev->subordinate;
94         }
95         if (!pdev) {
96                 printk(KERN_WARNING PREFIX
97                 "Device scope device [%04x:%02x:%02x.%02x] not found\n",
98                 segment, scope->bus, path->dev, path->fn);
99                 *dev = NULL;
100                 return 0;
101         }
102         if ((scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT && \
103                         pdev->subordinate) || (scope->entry_type == \
104                         ACPI_DMAR_SCOPE_TYPE_BRIDGE && !pdev->subordinate)) {
105                 pci_dev_put(pdev);
106                 printk(KERN_WARNING PREFIX
107                         "Device scope type does not match for %s\n",
108                          pci_name(pdev));
109                 return -EINVAL;
110         }
111         *dev = pdev;
112         return 0;
113 }
114
115 static int __init dmar_parse_dev_scope(void *start, void *end, int *cnt,
116                                        struct pci_dev ***devices, u16 segment)
117 {
118         struct acpi_dmar_device_scope *scope;
119         void * tmp = start;
120         int index;
121         int ret;
122
123         *cnt = 0;
124         while (start < end) {
125                 scope = start;
126                 if (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT ||
127                     scope->entry_type == ACPI_DMAR_SCOPE_TYPE_BRIDGE)
128                         (*cnt)++;
129                 else
130                         printk(KERN_WARNING PREFIX
131                                 "Unsupported device scope\n");
132                 start += scope->length;
133         }
134         if (*cnt == 0)
135                 return 0;
136
137         *devices = kcalloc(*cnt, sizeof(struct pci_dev *), GFP_KERNEL);
138         if (!*devices)
139                 return -ENOMEM;
140
141         start = tmp;
142         index = 0;
143         while (start < end) {
144                 scope = start;
145                 if (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT ||
146                     scope->entry_type == ACPI_DMAR_SCOPE_TYPE_BRIDGE) {
147                         ret = dmar_parse_one_dev_scope(scope,
148                                 &(*devices)[index], segment);
149                         if (ret) {
150                                 kfree(*devices);
151                                 return ret;
152                         }
153                         index ++;
154                 }
155                 start += scope->length;
156         }
157
158         return 0;
159 }
160
161 /**
162  * dmar_parse_one_drhd - parses exactly one DMA remapping hardware definition
163  * structure which uniquely represent one DMA remapping hardware unit
164  * present in the platform
165  */
166 static int __init
167 dmar_parse_one_drhd(struct acpi_dmar_header *header)
168 {
169         struct acpi_dmar_hardware_unit *drhd;
170         struct dmar_drhd_unit *dmaru;
171         int ret = 0;
172
173         dmaru = kzalloc(sizeof(*dmaru), GFP_KERNEL);
174         if (!dmaru)
175                 return -ENOMEM;
176
177         dmaru->hdr = header;
178         drhd = (struct acpi_dmar_hardware_unit *)header;
179         dmaru->reg_base_addr = drhd->address;
180         dmaru->include_all = drhd->flags & 0x1; /* BIT0: INCLUDE_ALL */
181
182         ret = alloc_iommu(dmaru);
183         if (ret) {
184                 kfree(dmaru);
185                 return ret;
186         }
187         dmar_register_drhd_unit(dmaru);
188         return 0;
189 }
190
191 static int __init
192 dmar_parse_dev(struct dmar_drhd_unit *dmaru)
193 {
194         struct acpi_dmar_hardware_unit *drhd;
195         static int include_all;
196         int ret = 0;
197
198         drhd = (struct acpi_dmar_hardware_unit *) dmaru->hdr;
199
200         if (!dmaru->include_all)
201                 ret = dmar_parse_dev_scope((void *)(drhd + 1),
202                                 ((void *)drhd) + drhd->header.length,
203                                 &dmaru->devices_cnt, &dmaru->devices,
204                                 drhd->segment);
205         else {
206                 /* Only allow one INCLUDE_ALL */
207                 if (include_all) {
208                         printk(KERN_WARNING PREFIX "Only one INCLUDE_ALL "
209                                 "device scope is allowed\n");
210                         ret = -EINVAL;
211                 }
212                 include_all = 1;
213         }
214
215         if (ret) {
216                 list_del(&dmaru->list);
217                 kfree(dmaru);
218         }
219         return ret;
220 }
221
222 #ifdef CONFIG_DMAR
223 LIST_HEAD(dmar_rmrr_units);
224
225 static void __init dmar_register_rmrr_unit(struct dmar_rmrr_unit *rmrr)
226 {
227         list_add(&rmrr->list, &dmar_rmrr_units);
228 }
229
230
231 static int __init
232 dmar_parse_one_rmrr(struct acpi_dmar_header *header)
233 {
234         struct acpi_dmar_reserved_memory *rmrr;
235         struct dmar_rmrr_unit *rmrru;
236
237         rmrru = kzalloc(sizeof(*rmrru), GFP_KERNEL);
238         if (!rmrru)
239                 return -ENOMEM;
240
241         rmrru->hdr = header;
242         rmrr = (struct acpi_dmar_reserved_memory *)header;
243         rmrru->base_address = rmrr->base_address;
244         rmrru->end_address = rmrr->end_address;
245
246         dmar_register_rmrr_unit(rmrru);
247         return 0;
248 }
249
250 static int __init
251 rmrr_parse_dev(struct dmar_rmrr_unit *rmrru)
252 {
253         struct acpi_dmar_reserved_memory *rmrr;
254         int ret;
255
256         rmrr = (struct acpi_dmar_reserved_memory *) rmrru->hdr;
257         ret = dmar_parse_dev_scope((void *)(rmrr + 1),
258                 ((void *)rmrr) + rmrr->header.length,
259                 &rmrru->devices_cnt, &rmrru->devices, rmrr->segment);
260
261         if (ret || (rmrru->devices_cnt == 0)) {
262                 list_del(&rmrru->list);
263                 kfree(rmrru);
264         }
265         return ret;
266 }
267 #endif
268
269 static void __init
270 dmar_table_print_dmar_entry(struct acpi_dmar_header *header)
271 {
272         struct acpi_dmar_hardware_unit *drhd;
273         struct acpi_dmar_reserved_memory *rmrr;
274
275         switch (header->type) {
276         case ACPI_DMAR_TYPE_HARDWARE_UNIT:
277                 drhd = (struct acpi_dmar_hardware_unit *)header;
278                 printk (KERN_INFO PREFIX
279                         "DRHD (flags: 0x%08x)base: 0x%016Lx\n",
280                         drhd->flags, drhd->address);
281                 break;
282         case ACPI_DMAR_TYPE_RESERVED_MEMORY:
283                 rmrr = (struct acpi_dmar_reserved_memory *)header;
284
285                 printk (KERN_INFO PREFIX
286                         "RMRR base: 0x%016Lx end: 0x%016Lx\n",
287                         rmrr->base_address, rmrr->end_address);
288                 break;
289         }
290 }
291
292 /**
293  * dmar_table_detect - checks to see if the platform supports DMAR devices
294  */
295 static int __init dmar_table_detect(void)
296 {
297         acpi_status status = AE_OK;
298
299         /* if we could find DMAR table, then there are DMAR devices */
300         status = acpi_get_table(ACPI_SIG_DMAR, 0,
301                                 (struct acpi_table_header **)&dmar_tbl);
302
303         if (ACPI_SUCCESS(status) && !dmar_tbl) {
304                 printk (KERN_WARNING PREFIX "Unable to map DMAR\n");
305                 status = AE_NOT_FOUND;
306         }
307
308         return (ACPI_SUCCESS(status) ? 1 : 0);
309 }
310
311 /**
312  * parse_dmar_table - parses the DMA reporting table
313  */
314 static int __init
315 parse_dmar_table(void)
316 {
317         struct acpi_table_dmar *dmar;
318         struct acpi_dmar_header *entry_header;
319         int ret = 0;
320
321         /*
322          * Do it again, earlier dmar_tbl mapping could be mapped with
323          * fixed map.
324          */
325         dmar_table_detect();
326
327         dmar = (struct acpi_table_dmar *)dmar_tbl;
328         if (!dmar)
329                 return -ENODEV;
330
331         if (dmar->width < PAGE_SHIFT_4K - 1) {
332                 printk(KERN_WARNING PREFIX "Invalid DMAR haw\n");
333                 return -EINVAL;
334         }
335
336         printk (KERN_INFO PREFIX "Host address width %d\n",
337                 dmar->width + 1);
338
339         entry_header = (struct acpi_dmar_header *)(dmar + 1);
340         while (((unsigned long)entry_header) <
341                         (((unsigned long)dmar) + dmar_tbl->length)) {
342                 dmar_table_print_dmar_entry(entry_header);
343
344                 switch (entry_header->type) {
345                 case ACPI_DMAR_TYPE_HARDWARE_UNIT:
346                         ret = dmar_parse_one_drhd(entry_header);
347                         break;
348                 case ACPI_DMAR_TYPE_RESERVED_MEMORY:
349 #ifdef CONFIG_DMAR
350                         ret = dmar_parse_one_rmrr(entry_header);
351 #endif
352                         break;
353                 default:
354                         printk(KERN_WARNING PREFIX
355                                 "Unknown DMAR structure type\n");
356                         ret = 0; /* for forward compatibility */
357                         break;
358                 }
359                 if (ret)
360                         break;
361
362                 entry_header = ((void *)entry_header + entry_header->length);
363         }
364         return ret;
365 }
366
367 int dmar_pci_device_match(struct pci_dev *devices[], int cnt,
368                           struct pci_dev *dev)
369 {
370         int index;
371
372         while (dev) {
373                 for (index = 0; index < cnt; index++)
374                         if (dev == devices[index])
375                                 return 1;
376
377                 /* Check our parent */
378                 dev = dev->bus->self;
379         }
380
381         return 0;
382 }
383
384 struct dmar_drhd_unit *
385 dmar_find_matched_drhd_unit(struct pci_dev *dev)
386 {
387         struct dmar_drhd_unit *drhd = NULL;
388
389         list_for_each_entry(drhd, &dmar_drhd_units, list) {
390                 if (drhd->include_all || dmar_pci_device_match(drhd->devices,
391                                                 drhd->devices_cnt, dev))
392                         return drhd;
393         }
394
395         return NULL;
396 }
397
398 int __init dmar_dev_scope_init(void)
399 {
400         struct dmar_drhd_unit *drhd, *drhd_n;
401         int ret = -ENODEV;
402
403         list_for_each_entry_safe(drhd, drhd_n, &dmar_drhd_units, list) {
404                 ret = dmar_parse_dev(drhd);
405                 if (ret)
406                         return ret;
407         }
408
409 #ifdef CONFIG_DMAR
410         {
411                 struct dmar_rmrr_unit *rmrr, *rmrr_n;
412                 list_for_each_entry_safe(rmrr, rmrr_n, &dmar_rmrr_units, list) {
413                         ret = rmrr_parse_dev(rmrr);
414                         if (ret)
415                                 return ret;
416                 }
417         }
418 #endif
419
420         return ret;
421 }
422
423
424 int __init dmar_table_init(void)
425 {
426         static int dmar_table_initialized;
427         int ret;
428
429         if (dmar_table_initialized)
430                 return 0;
431
432         dmar_table_initialized = 1;
433
434         ret = parse_dmar_table();
435         if (ret) {
436                 if (ret != -ENODEV)
437                         printk(KERN_INFO PREFIX "parse DMAR table failure.\n");
438                 return ret;
439         }
440
441         if (list_empty(&dmar_drhd_units)) {
442                 printk(KERN_INFO PREFIX "No DMAR devices found\n");
443                 return -ENODEV;
444         }
445
446 #ifdef CONFIG_DMAR
447         if (list_empty(&dmar_rmrr_units))
448                 printk(KERN_INFO PREFIX "No RMRR found\n");
449 #endif
450
451 #ifdef CONFIG_INTR_REMAP
452         parse_ioapics_under_ir();
453 #endif
454         return 0;
455 }
456
457 void __init detect_intel_iommu(void)
458 {
459         int ret;
460
461         ret = dmar_table_detect();
462
463 #ifdef CONFIG_DMAR
464         {
465                 struct acpi_table_dmar *dmar;
466                 /*
467                  * for now we will disable dma-remapping when interrupt
468                  * remapping is enabled.
469                  * When support for queued invalidation for IOTLB invalidation
470                  * is added, we will not need this any more.
471                  */
472                 dmar = (struct acpi_table_dmar *) dmar_tbl;
473                 if (ret && cpu_has_x2apic && dmar->flags & 0x1) {
474                         printk(KERN_INFO
475                                "Queued invalidation will be enabled to support "
476                                "x2apic and Intr-remapping.\n");
477                         printk(KERN_INFO
478                                "Disabling IOMMU detection, because of missing "
479                                "queued invalidation support for IOTLB "
480                                "invalidation\n");
481                         printk(KERN_INFO
482                                "Use \"nox2apic\", if you want to use Intel "
483                                " IOMMU for DMA-remapping and don't care about "
484                                " x2apic support\n");
485
486                         dmar_disabled = 1;
487                         goto end;
488                 }
489
490                 if (ret && !no_iommu && !iommu_detected && !swiotlb &&
491                     !dmar_disabled)
492                         iommu_detected = 1;
493         }
494 end:
495 #endif
496         dmar_tbl = NULL;
497 }
498
499
500 int alloc_iommu(struct dmar_drhd_unit *drhd)
501 {
502         struct intel_iommu *iommu;
503         int map_size;
504         u32 ver;
505         static int iommu_allocated = 0;
506
507         iommu = kzalloc(sizeof(*iommu), GFP_KERNEL);
508         if (!iommu)
509                 return -ENOMEM;
510
511         iommu->seq_id = iommu_allocated++;
512
513         iommu->reg = ioremap(drhd->reg_base_addr, PAGE_SIZE_4K);
514         if (!iommu->reg) {
515                 printk(KERN_ERR "IOMMU: can't map the region\n");
516                 goto error;
517         }
518         iommu->cap = dmar_readq(iommu->reg + DMAR_CAP_REG);
519         iommu->ecap = dmar_readq(iommu->reg + DMAR_ECAP_REG);
520
521         /* the registers might be more than one page */
522         map_size = max_t(int, ecap_max_iotlb_offset(iommu->ecap),
523                 cap_max_fault_reg_offset(iommu->cap));
524         map_size = PAGE_ALIGN_4K(map_size);
525         if (map_size > PAGE_SIZE_4K) {
526                 iounmap(iommu->reg);
527                 iommu->reg = ioremap(drhd->reg_base_addr, map_size);
528                 if (!iommu->reg) {
529                         printk(KERN_ERR "IOMMU: can't map the region\n");
530                         goto error;
531                 }
532         }
533
534         ver = readl(iommu->reg + DMAR_VER_REG);
535         pr_debug("IOMMU %llx: ver %d:%d cap %llx ecap %llx\n",
536                 drhd->reg_base_addr, DMAR_VER_MAJOR(ver), DMAR_VER_MINOR(ver),
537                 iommu->cap, iommu->ecap);
538
539         spin_lock_init(&iommu->register_lock);
540
541         drhd->iommu = iommu;
542         return 0;
543 error:
544         kfree(iommu);
545         return -1;
546 }
547
548 void free_iommu(struct intel_iommu *iommu)
549 {
550         if (!iommu)
551                 return;
552
553 #ifdef CONFIG_DMAR
554         free_dmar_iommu(iommu);
555 #endif
556
557         if (iommu->reg)
558                 iounmap(iommu->reg);
559         kfree(iommu);
560 }
561
562 /*
563  * Reclaim all the submitted descriptors which have completed its work.
564  */
565 static inline void reclaim_free_desc(struct q_inval *qi)
566 {
567         while (qi->desc_status[qi->free_tail] == QI_DONE) {
568                 qi->desc_status[qi->free_tail] = QI_FREE;
569                 qi->free_tail = (qi->free_tail + 1) % QI_LENGTH;
570                 qi->free_cnt++;
571         }
572 }
573
574 /*
575  * Submit the queued invalidation descriptor to the remapping
576  * hardware unit and wait for its completion.
577  */
578 void qi_submit_sync(struct qi_desc *desc, struct intel_iommu *iommu)
579 {
580         struct q_inval *qi = iommu->qi;
581         struct qi_desc *hw, wait_desc;
582         int wait_index, index;
583         unsigned long flags;
584
585         if (!qi)
586                 return;
587
588         hw = qi->desc;
589
590         spin_lock(&qi->q_lock);
591         while (qi->free_cnt < 3) {
592                 spin_unlock(&qi->q_lock);
593                 cpu_relax();
594                 spin_lock(&qi->q_lock);
595         }
596
597         index = qi->free_head;
598         wait_index = (index + 1) % QI_LENGTH;
599
600         qi->desc_status[index] = qi->desc_status[wait_index] = QI_IN_USE;
601
602         hw[index] = *desc;
603
604         wait_desc.low = QI_IWD_STATUS_DATA(2) | QI_IWD_STATUS_WRITE | QI_IWD_TYPE;
605         wait_desc.high = virt_to_phys(&qi->desc_status[wait_index]);
606
607         hw[wait_index] = wait_desc;
608
609         __iommu_flush_cache(iommu, &hw[index], sizeof(struct qi_desc));
610         __iommu_flush_cache(iommu, &hw[wait_index], sizeof(struct qi_desc));
611
612         qi->free_head = (qi->free_head + 2) % QI_LENGTH;
613         qi->free_cnt -= 2;
614
615         spin_lock_irqsave(&iommu->register_lock, flags);
616         /*
617          * update the HW tail register indicating the presence of
618          * new descriptors.
619          */
620         writel(qi->free_head << 4, iommu->reg + DMAR_IQT_REG);
621         spin_unlock_irqrestore(&iommu->register_lock, flags);
622
623         while (qi->desc_status[wait_index] != QI_DONE) {
624                 spin_unlock(&qi->q_lock);
625                 cpu_relax();
626                 spin_lock(&qi->q_lock);
627         }
628
629         qi->desc_status[index] = QI_DONE;
630
631         reclaim_free_desc(qi);
632         spin_unlock(&qi->q_lock);
633 }
634
635 /*
636  * Flush the global interrupt entry cache.
637  */
638 void qi_global_iec(struct intel_iommu *iommu)
639 {
640         struct qi_desc desc;
641
642         desc.low = QI_IEC_TYPE;
643         desc.high = 0;
644
645         qi_submit_sync(&desc, iommu);
646 }
647
648 /*
649  * Enable Queued Invalidation interface. This is a must to support
650  * interrupt-remapping. Also used by DMA-remapping, which replaces
651  * register based IOTLB invalidation.
652  */
653 int dmar_enable_qi(struct intel_iommu *iommu)
654 {
655         u32 cmd, sts;
656         unsigned long flags;
657         struct q_inval *qi;
658
659         if (!ecap_qis(iommu->ecap))
660                 return -ENOENT;
661
662         /*
663          * queued invalidation is already setup and enabled.
664          */
665         if (iommu->qi)
666                 return 0;
667
668         iommu->qi = kmalloc(sizeof(*qi), GFP_KERNEL);
669         if (!iommu->qi)
670                 return -ENOMEM;
671
672         qi = iommu->qi;
673
674         qi->desc = (void *)(get_zeroed_page(GFP_KERNEL));
675         if (!qi->desc) {
676                 kfree(qi);
677                 iommu->qi = 0;
678                 return -ENOMEM;
679         }
680
681         qi->desc_status = kmalloc(QI_LENGTH * sizeof(int), GFP_KERNEL);
682         if (!qi->desc_status) {
683                 free_page((unsigned long) qi->desc);
684                 kfree(qi);
685                 iommu->qi = 0;
686                 return -ENOMEM;
687         }
688
689         qi->free_head = qi->free_tail = 0;
690         qi->free_cnt = QI_LENGTH;
691
692         spin_lock_init(&qi->q_lock);
693
694         spin_lock_irqsave(&iommu->register_lock, flags);
695         /* write zero to the tail reg */
696         writel(0, iommu->reg + DMAR_IQT_REG);
697
698         dmar_writeq(iommu->reg + DMAR_IQA_REG, virt_to_phys(qi->desc));
699
700         cmd = iommu->gcmd | DMA_GCMD_QIE;
701         iommu->gcmd |= DMA_GCMD_QIE;
702         writel(cmd, iommu->reg + DMAR_GCMD_REG);
703
704         /* Make sure hardware complete it */
705         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl, (sts & DMA_GSTS_QIES), sts);
706         spin_unlock_irqrestore(&iommu->register_lock, flags);
707
708         return 0;
709 }