Merge branches 'irq/sparseirq', 'x86/quirks' and 'x86/reboot' into cpus4096
[linux-2.6] / drivers / pci / intr_remapping.c
1 #include <linux/interrupt.h>
2 #include <linux/dmar.h>
3 #include <linux/spinlock.h>
4 #include <linux/jiffies.h>
5 #include <linux/pci.h>
6 #include <linux/irq.h>
7 #include <asm/io_apic.h>
8 #include <linux/intel-iommu.h>
9 #include "intr_remapping.h"
10
11 static struct ioapic_scope ir_ioapic[MAX_IO_APICS];
12 static int ir_ioapic_num;
13 int intr_remapping_enabled;
14
15 struct irq_2_iommu {
16         struct intel_iommu *iommu;
17         u16 irte_index;
18         u16 sub_handle;
19         u8  irte_mask;
20 };
21
22 #ifdef CONFIG_SPARSE_IRQ
23 static struct irq_2_iommu *get_one_free_irq_2_iommu(int cpu)
24 {
25         struct irq_2_iommu *iommu;
26         int node;
27
28         node = cpu_to_node(cpu);
29
30         iommu = kzalloc_node(sizeof(*iommu), GFP_ATOMIC, node);
31         printk(KERN_DEBUG "alloc irq_2_iommu on cpu %d node %d\n", cpu, node);
32
33         return iommu;
34 }
35
36 static struct irq_2_iommu *irq_2_iommu(unsigned int irq)
37 {
38         struct irq_desc *desc;
39
40         desc = irq_to_desc(irq);
41
42         if (WARN_ON_ONCE(!desc))
43                 return NULL;
44
45         return desc->irq_2_iommu;
46 }
47
48 static struct irq_2_iommu *irq_2_iommu_alloc_cpu(unsigned int irq, int cpu)
49 {
50         struct irq_desc *desc;
51         struct irq_2_iommu *irq_iommu;
52
53         /*
54          * alloc irq desc if not allocated already.
55          */
56         desc = irq_to_desc_alloc_cpu(irq, cpu);
57         if (!desc) {
58                 printk(KERN_INFO "can not get irq_desc for %d\n", irq);
59                 return NULL;
60         }
61
62         irq_iommu = desc->irq_2_iommu;
63
64         if (!irq_iommu)
65                 desc->irq_2_iommu = get_one_free_irq_2_iommu(cpu);
66
67         return desc->irq_2_iommu;
68 }
69
70 static struct irq_2_iommu *irq_2_iommu_alloc(unsigned int irq)
71 {
72         return irq_2_iommu_alloc_cpu(irq, boot_cpu_id);
73 }
74
75 #else /* !CONFIG_SPARSE_IRQ */
76
77 static struct irq_2_iommu irq_2_iommuX[NR_IRQS];
78
79 static struct irq_2_iommu *irq_2_iommu(unsigned int irq)
80 {
81         if (irq < nr_irqs)
82                 return &irq_2_iommuX[irq];
83
84         return NULL;
85 }
86 static struct irq_2_iommu *irq_2_iommu_alloc(unsigned int irq)
87 {
88         return irq_2_iommu(irq);
89 }
90 #endif
91
92 static DEFINE_SPINLOCK(irq_2_ir_lock);
93
94 static struct irq_2_iommu *valid_irq_2_iommu(unsigned int irq)
95 {
96         struct irq_2_iommu *irq_iommu;
97
98         irq_iommu = irq_2_iommu(irq);
99
100         if (!irq_iommu)
101                 return NULL;
102
103         if (!irq_iommu->iommu)
104                 return NULL;
105
106         return irq_iommu;
107 }
108
109 int irq_remapped(int irq)
110 {
111         return valid_irq_2_iommu(irq) != NULL;
112 }
113
114 int get_irte(int irq, struct irte *entry)
115 {
116         int index;
117         struct irq_2_iommu *irq_iommu;
118
119         if (!entry)
120                 return -1;
121
122         spin_lock(&irq_2_ir_lock);
123         irq_iommu = valid_irq_2_iommu(irq);
124         if (!irq_iommu) {
125                 spin_unlock(&irq_2_ir_lock);
126                 return -1;
127         }
128
129         index = irq_iommu->irte_index + irq_iommu->sub_handle;
130         *entry = *(irq_iommu->iommu->ir_table->base + index);
131
132         spin_unlock(&irq_2_ir_lock);
133         return 0;
134 }
135
136 int alloc_irte(struct intel_iommu *iommu, int irq, u16 count)
137 {
138         struct ir_table *table = iommu->ir_table;
139         struct irq_2_iommu *irq_iommu;
140         u16 index, start_index;
141         unsigned int mask = 0;
142         int i;
143
144         if (!count)
145                 return -1;
146
147 #ifndef CONFIG_SPARSE_IRQ
148         /* protect irq_2_iommu_alloc later */
149         if (irq >= nr_irqs)
150                 return -1;
151 #endif
152
153         /*
154          * start the IRTE search from index 0.
155          */
156         index = start_index = 0;
157
158         if (count > 1) {
159                 count = __roundup_pow_of_two(count);
160                 mask = ilog2(count);
161         }
162
163         if (mask > ecap_max_handle_mask(iommu->ecap)) {
164                 printk(KERN_ERR
165                        "Requested mask %x exceeds the max invalidation handle"
166                        " mask value %Lx\n", mask,
167                        ecap_max_handle_mask(iommu->ecap));
168                 return -1;
169         }
170
171         spin_lock(&irq_2_ir_lock);
172         do {
173                 for (i = index; i < index + count; i++)
174                         if  (table->base[i].present)
175                                 break;
176                 /* empty index found */
177                 if (i == index + count)
178                         break;
179
180                 index = (index + count) % INTR_REMAP_TABLE_ENTRIES;
181
182                 if (index == start_index) {
183                         spin_unlock(&irq_2_ir_lock);
184                         printk(KERN_ERR "can't allocate an IRTE\n");
185                         return -1;
186                 }
187         } while (1);
188
189         for (i = index; i < index + count; i++)
190                 table->base[i].present = 1;
191
192         irq_iommu = irq_2_iommu_alloc(irq);
193         if (!irq_iommu) {
194                 spin_unlock(&irq_2_ir_lock);
195                 printk(KERN_ERR "can't allocate irq_2_iommu\n");
196                 return -1;
197         }
198
199         irq_iommu->iommu = iommu;
200         irq_iommu->irte_index =  index;
201         irq_iommu->sub_handle = 0;
202         irq_iommu->irte_mask = mask;
203
204         spin_unlock(&irq_2_ir_lock);
205
206         return index;
207 }
208
209 static void qi_flush_iec(struct intel_iommu *iommu, int index, int mask)
210 {
211         struct qi_desc desc;
212
213         desc.low = QI_IEC_IIDEX(index) | QI_IEC_TYPE | QI_IEC_IM(mask)
214                    | QI_IEC_SELECTIVE;
215         desc.high = 0;
216
217         qi_submit_sync(&desc, iommu);
218 }
219
220 int map_irq_to_irte_handle(int irq, u16 *sub_handle)
221 {
222         int index;
223         struct irq_2_iommu *irq_iommu;
224
225         spin_lock(&irq_2_ir_lock);
226         irq_iommu = valid_irq_2_iommu(irq);
227         if (!irq_iommu) {
228                 spin_unlock(&irq_2_ir_lock);
229                 return -1;
230         }
231
232         *sub_handle = irq_iommu->sub_handle;
233         index = irq_iommu->irte_index;
234         spin_unlock(&irq_2_ir_lock);
235         return index;
236 }
237
238 int set_irte_irq(int irq, struct intel_iommu *iommu, u16 index, u16 subhandle)
239 {
240         struct irq_2_iommu *irq_iommu;
241
242         spin_lock(&irq_2_ir_lock);
243
244         irq_iommu = irq_2_iommu_alloc(irq);
245
246         if (!irq_iommu) {
247                 spin_unlock(&irq_2_ir_lock);
248                 printk(KERN_ERR "can't allocate irq_2_iommu\n");
249                 return -1;
250         }
251
252         irq_iommu->iommu = iommu;
253         irq_iommu->irte_index = index;
254         irq_iommu->sub_handle = subhandle;
255         irq_iommu->irte_mask = 0;
256
257         spin_unlock(&irq_2_ir_lock);
258
259         return 0;
260 }
261
262 int clear_irte_irq(int irq, struct intel_iommu *iommu, u16 index)
263 {
264         struct irq_2_iommu *irq_iommu;
265
266         spin_lock(&irq_2_ir_lock);
267         irq_iommu = valid_irq_2_iommu(irq);
268         if (!irq_iommu) {
269                 spin_unlock(&irq_2_ir_lock);
270                 return -1;
271         }
272
273         irq_iommu->iommu = NULL;
274         irq_iommu->irte_index = 0;
275         irq_iommu->sub_handle = 0;
276         irq_2_iommu(irq)->irte_mask = 0;
277
278         spin_unlock(&irq_2_ir_lock);
279
280         return 0;
281 }
282
283 int modify_irte(int irq, struct irte *irte_modified)
284 {
285         int index;
286         struct irte *irte;
287         struct intel_iommu *iommu;
288         struct irq_2_iommu *irq_iommu;
289
290         spin_lock(&irq_2_ir_lock);
291         irq_iommu = valid_irq_2_iommu(irq);
292         if (!irq_iommu) {
293                 spin_unlock(&irq_2_ir_lock);
294                 return -1;
295         }
296
297         iommu = irq_iommu->iommu;
298
299         index = irq_iommu->irte_index + irq_iommu->sub_handle;
300         irte = &iommu->ir_table->base[index];
301
302         set_64bit((unsigned long *)irte, irte_modified->low | (1 << 1));
303         __iommu_flush_cache(iommu, irte, sizeof(*irte));
304
305         qi_flush_iec(iommu, index, 0);
306
307         spin_unlock(&irq_2_ir_lock);
308         return 0;
309 }
310
311 int flush_irte(int irq)
312 {
313         int index;
314         struct intel_iommu *iommu;
315         struct irq_2_iommu *irq_iommu;
316
317         spin_lock(&irq_2_ir_lock);
318         irq_iommu = valid_irq_2_iommu(irq);
319         if (!irq_iommu) {
320                 spin_unlock(&irq_2_ir_lock);
321                 return -1;
322         }
323
324         iommu = irq_iommu->iommu;
325
326         index = irq_iommu->irte_index + irq_iommu->sub_handle;
327
328         qi_flush_iec(iommu, index, irq_iommu->irte_mask);
329         spin_unlock(&irq_2_ir_lock);
330
331         return 0;
332 }
333
334 struct intel_iommu *map_ioapic_to_ir(int apic)
335 {
336         int i;
337
338         for (i = 0; i < MAX_IO_APICS; i++)
339                 if (ir_ioapic[i].id == apic)
340                         return ir_ioapic[i].iommu;
341         return NULL;
342 }
343
344 struct intel_iommu *map_dev_to_ir(struct pci_dev *dev)
345 {
346         struct dmar_drhd_unit *drhd;
347
348         drhd = dmar_find_matched_drhd_unit(dev);
349         if (!drhd)
350                 return NULL;
351
352         return drhd->iommu;
353 }
354
355 int free_irte(int irq)
356 {
357         int index, i;
358         struct irte *irte;
359         struct intel_iommu *iommu;
360         struct irq_2_iommu *irq_iommu;
361
362         spin_lock(&irq_2_ir_lock);
363         irq_iommu = valid_irq_2_iommu(irq);
364         if (!irq_iommu) {
365                 spin_unlock(&irq_2_ir_lock);
366                 return -1;
367         }
368
369         iommu = irq_iommu->iommu;
370
371         index = irq_iommu->irte_index + irq_iommu->sub_handle;
372         irte = &iommu->ir_table->base[index];
373
374         if (!irq_iommu->sub_handle) {
375                 for (i = 0; i < (1 << irq_iommu->irte_mask); i++)
376                         set_64bit((unsigned long *)irte, 0);
377                 qi_flush_iec(iommu, index, irq_iommu->irte_mask);
378         }
379
380         irq_iommu->iommu = NULL;
381         irq_iommu->irte_index = 0;
382         irq_iommu->sub_handle = 0;
383         irq_iommu->irte_mask = 0;
384
385         spin_unlock(&irq_2_ir_lock);
386
387         return 0;
388 }
389
390 static void iommu_set_intr_remapping(struct intel_iommu *iommu, int mode)
391 {
392         u64 addr;
393         u32 cmd, sts;
394         unsigned long flags;
395
396         addr = virt_to_phys((void *)iommu->ir_table->base);
397
398         spin_lock_irqsave(&iommu->register_lock, flags);
399
400         dmar_writeq(iommu->reg + DMAR_IRTA_REG,
401                     (addr) | IR_X2APIC_MODE(mode) | INTR_REMAP_TABLE_REG_SIZE);
402
403         /* Set interrupt-remapping table pointer */
404         cmd = iommu->gcmd | DMA_GCMD_SIRTP;
405         writel(cmd, iommu->reg + DMAR_GCMD_REG);
406
407         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
408                       readl, (sts & DMA_GSTS_IRTPS), sts);
409         spin_unlock_irqrestore(&iommu->register_lock, flags);
410
411         /*
412          * global invalidation of interrupt entry cache before enabling
413          * interrupt-remapping.
414          */
415         qi_global_iec(iommu);
416
417         spin_lock_irqsave(&iommu->register_lock, flags);
418
419         /* Enable interrupt-remapping */
420         cmd = iommu->gcmd | DMA_GCMD_IRE;
421         iommu->gcmd |= DMA_GCMD_IRE;
422         writel(cmd, iommu->reg + DMAR_GCMD_REG);
423
424         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
425                       readl, (sts & DMA_GSTS_IRES), sts);
426
427         spin_unlock_irqrestore(&iommu->register_lock, flags);
428 }
429
430
431 static int setup_intr_remapping(struct intel_iommu *iommu, int mode)
432 {
433         struct ir_table *ir_table;
434         struct page *pages;
435
436         ir_table = iommu->ir_table = kzalloc(sizeof(struct ir_table),
437                                              GFP_KERNEL);
438
439         if (!iommu->ir_table)
440                 return -ENOMEM;
441
442         pages = alloc_pages(GFP_KERNEL | __GFP_ZERO, INTR_REMAP_PAGE_ORDER);
443
444         if (!pages) {
445                 printk(KERN_ERR "failed to allocate pages of order %d\n",
446                        INTR_REMAP_PAGE_ORDER);
447                 kfree(iommu->ir_table);
448                 return -ENOMEM;
449         }
450
451         ir_table->base = page_address(pages);
452
453         iommu_set_intr_remapping(iommu, mode);
454         return 0;
455 }
456
457 int __init enable_intr_remapping(int eim)
458 {
459         struct dmar_drhd_unit *drhd;
460         int setup = 0;
461
462         /*
463          * check for the Interrupt-remapping support
464          */
465         for_each_drhd_unit(drhd) {
466                 struct intel_iommu *iommu = drhd->iommu;
467
468                 if (!ecap_ir_support(iommu->ecap))
469                         continue;
470
471                 if (eim && !ecap_eim_support(iommu->ecap)) {
472                         printk(KERN_INFO "DRHD %Lx: EIM not supported by DRHD, "
473                                " ecap %Lx\n", drhd->reg_base_addr, iommu->ecap);
474                         return -1;
475                 }
476         }
477
478         /*
479          * Enable queued invalidation for all the DRHD's.
480          */
481         for_each_drhd_unit(drhd) {
482                 int ret;
483                 struct intel_iommu *iommu = drhd->iommu;
484                 ret = dmar_enable_qi(iommu);
485
486                 if (ret) {
487                         printk(KERN_ERR "DRHD %Lx: failed to enable queued, "
488                                " invalidation, ecap %Lx, ret %d\n",
489                                drhd->reg_base_addr, iommu->ecap, ret);
490                         return -1;
491                 }
492         }
493
494         /*
495          * Setup Interrupt-remapping for all the DRHD's now.
496          */
497         for_each_drhd_unit(drhd) {
498                 struct intel_iommu *iommu = drhd->iommu;
499
500                 if (!ecap_ir_support(iommu->ecap))
501                         continue;
502
503                 if (setup_intr_remapping(iommu, eim))
504                         goto error;
505
506                 setup = 1;
507         }
508
509         if (!setup)
510                 goto error;
511
512         intr_remapping_enabled = 1;
513
514         return 0;
515
516 error:
517         /*
518          * handle error condition gracefully here!
519          */
520         return -1;
521 }
522
523 static int ir_parse_ioapic_scope(struct acpi_dmar_header *header,
524                                  struct intel_iommu *iommu)
525 {
526         struct acpi_dmar_hardware_unit *drhd;
527         struct acpi_dmar_device_scope *scope;
528         void *start, *end;
529
530         drhd = (struct acpi_dmar_hardware_unit *)header;
531
532         start = (void *)(drhd + 1);
533         end = ((void *)drhd) + header->length;
534
535         while (start < end) {
536                 scope = start;
537                 if (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_IOAPIC) {
538                         if (ir_ioapic_num == MAX_IO_APICS) {
539                                 printk(KERN_WARNING "Exceeded Max IO APICS\n");
540                                 return -1;
541                         }
542
543                         printk(KERN_INFO "IOAPIC id %d under DRHD base"
544                                " 0x%Lx\n", scope->enumeration_id,
545                                drhd->address);
546
547                         ir_ioapic[ir_ioapic_num].iommu = iommu;
548                         ir_ioapic[ir_ioapic_num].id = scope->enumeration_id;
549                         ir_ioapic_num++;
550                 }
551                 start += scope->length;
552         }
553
554         return 0;
555 }
556
557 /*
558  * Finds the assocaition between IOAPIC's and its Interrupt-remapping
559  * hardware unit.
560  */
561 int __init parse_ioapics_under_ir(void)
562 {
563         struct dmar_drhd_unit *drhd;
564         int ir_supported = 0;
565
566         for_each_drhd_unit(drhd) {
567                 struct intel_iommu *iommu = drhd->iommu;
568
569                 if (ecap_ir_support(iommu->ecap)) {
570                         if (ir_parse_ioapic_scope(drhd->hdr, iommu))
571                                 return -1;
572
573                         ir_supported = 1;
574                 }
575         }
576
577         if (ir_supported && ir_ioapic_num != nr_ioapics) {
578                 printk(KERN_WARNING
579                        "Not all IO-APIC's listed under remapping hardware\n");
580                 return -1;
581         }
582
583         return ir_supported;
584 }