Merge branch 'master' into for-upstream
[linux-2.6] / drivers / pci / intr_remapping.c
1 #include <linux/dmar.h>
2 #include <linux/spinlock.h>
3 #include <linux/jiffies.h>
4 #include <linux/pci.h>
5 #include <linux/irq.h>
6 #include <asm/io_apic.h>
7 #include <linux/intel-iommu.h>
8 #include "intr_remapping.h"
9
10 static struct ioapic_scope ir_ioapic[MAX_IO_APICS];
11 static int ir_ioapic_num;
12 int intr_remapping_enabled;
13
14 static struct {
15         struct intel_iommu *iommu;
16         u16 irte_index;
17         u16 sub_handle;
18         u8  irte_mask;
19 } irq_2_iommu[NR_IRQS];
20
21 static DEFINE_SPINLOCK(irq_2_ir_lock);
22
23 int irq_remapped(int irq)
24 {
25         if (irq > NR_IRQS)
26                 return 0;
27
28         if (!irq_2_iommu[irq].iommu)
29                 return 0;
30
31         return 1;
32 }
33
34 int get_irte(int irq, struct irte *entry)
35 {
36         int index;
37
38         if (!entry || irq > NR_IRQS)
39                 return -1;
40
41         spin_lock(&irq_2_ir_lock);
42         if (!irq_2_iommu[irq].iommu) {
43                 spin_unlock(&irq_2_ir_lock);
44                 return -1;
45         }
46
47         index = irq_2_iommu[irq].irte_index + irq_2_iommu[irq].sub_handle;
48         *entry = *(irq_2_iommu[irq].iommu->ir_table->base + index);
49
50         spin_unlock(&irq_2_ir_lock);
51         return 0;
52 }
53
54 int alloc_irte(struct intel_iommu *iommu, int irq, u16 count)
55 {
56         struct ir_table *table = iommu->ir_table;
57         u16 index, start_index;
58         unsigned int mask = 0;
59         int i;
60
61         if (!count)
62                 return -1;
63
64         /*
65          * start the IRTE search from index 0.
66          */
67         index = start_index = 0;
68
69         if (count > 1) {
70                 count = __roundup_pow_of_two(count);
71                 mask = ilog2(count);
72         }
73
74         if (mask > ecap_max_handle_mask(iommu->ecap)) {
75                 printk(KERN_ERR
76                        "Requested mask %x exceeds the max invalidation handle"
77                        " mask value %Lx\n", mask,
78                        ecap_max_handle_mask(iommu->ecap));
79                 return -1;
80         }
81
82         spin_lock(&irq_2_ir_lock);
83         do {
84                 for (i = index; i < index + count; i++)
85                         if  (table->base[i].present)
86                                 break;
87                 /* empty index found */
88                 if (i == index + count)
89                         break;
90
91                 index = (index + count) % INTR_REMAP_TABLE_ENTRIES;
92
93                 if (index == start_index) {
94                         spin_unlock(&irq_2_ir_lock);
95                         printk(KERN_ERR "can't allocate an IRTE\n");
96                         return -1;
97                 }
98         } while (1);
99
100         for (i = index; i < index + count; i++)
101                 table->base[i].present = 1;
102
103         irq_2_iommu[irq].iommu = iommu;
104         irq_2_iommu[irq].irte_index =  index;
105         irq_2_iommu[irq].sub_handle = 0;
106         irq_2_iommu[irq].irte_mask = mask;
107
108         spin_unlock(&irq_2_ir_lock);
109
110         return index;
111 }
112
113 static void qi_flush_iec(struct intel_iommu *iommu, int index, int mask)
114 {
115         struct qi_desc desc;
116
117         desc.low = QI_IEC_IIDEX(index) | QI_IEC_TYPE | QI_IEC_IM(mask)
118                    | QI_IEC_SELECTIVE;
119         desc.high = 0;
120
121         qi_submit_sync(&desc, iommu);
122 }
123
124 int map_irq_to_irte_handle(int irq, u16 *sub_handle)
125 {
126         int index;
127
128         spin_lock(&irq_2_ir_lock);
129         if (irq >= NR_IRQS || !irq_2_iommu[irq].iommu) {
130                 spin_unlock(&irq_2_ir_lock);
131                 return -1;
132         }
133
134         *sub_handle = irq_2_iommu[irq].sub_handle;
135         index = irq_2_iommu[irq].irte_index;
136         spin_unlock(&irq_2_ir_lock);
137         return index;
138 }
139
140 int set_irte_irq(int irq, struct intel_iommu *iommu, u16 index, u16 subhandle)
141 {
142         spin_lock(&irq_2_ir_lock);
143         if (irq >= NR_IRQS || irq_2_iommu[irq].iommu) {
144                 spin_unlock(&irq_2_ir_lock);
145                 return -1;
146         }
147
148         irq_2_iommu[irq].iommu = iommu;
149         irq_2_iommu[irq].irte_index = index;
150         irq_2_iommu[irq].sub_handle = subhandle;
151         irq_2_iommu[irq].irte_mask = 0;
152
153         spin_unlock(&irq_2_ir_lock);
154
155         return 0;
156 }
157
158 int clear_irte_irq(int irq, struct intel_iommu *iommu, u16 index)
159 {
160         spin_lock(&irq_2_ir_lock);
161         if (irq >= NR_IRQS || !irq_2_iommu[irq].iommu) {
162                 spin_unlock(&irq_2_ir_lock);
163                 return -1;
164         }
165
166         irq_2_iommu[irq].iommu = NULL;
167         irq_2_iommu[irq].irte_index = 0;
168         irq_2_iommu[irq].sub_handle = 0;
169         irq_2_iommu[irq].irte_mask = 0;
170
171         spin_unlock(&irq_2_ir_lock);
172
173         return 0;
174 }
175
176 int modify_irte(int irq, struct irte *irte_modified)
177 {
178         int index;
179         struct irte *irte;
180         struct intel_iommu *iommu;
181
182         spin_lock(&irq_2_ir_lock);
183         if (irq >= NR_IRQS || !irq_2_iommu[irq].iommu) {
184                 spin_unlock(&irq_2_ir_lock);
185                 return -1;
186         }
187
188         iommu = irq_2_iommu[irq].iommu;
189
190         index = irq_2_iommu[irq].irte_index + irq_2_iommu[irq].sub_handle;
191         irte = &iommu->ir_table->base[index];
192
193         set_64bit((unsigned long *)irte, irte_modified->low | (1 << 1));
194         __iommu_flush_cache(iommu, irte, sizeof(*irte));
195
196         qi_flush_iec(iommu, index, 0);
197
198         spin_unlock(&irq_2_ir_lock);
199         return 0;
200 }
201
202 int flush_irte(int irq)
203 {
204         int index;
205         struct intel_iommu *iommu;
206
207         spin_lock(&irq_2_ir_lock);
208         if (irq >= NR_IRQS || !irq_2_iommu[irq].iommu) {
209                 spin_unlock(&irq_2_ir_lock);
210                 return -1;
211         }
212
213         iommu = irq_2_iommu[irq].iommu;
214
215         index = irq_2_iommu[irq].irte_index + irq_2_iommu[irq].sub_handle;
216
217         qi_flush_iec(iommu, index, irq_2_iommu[irq].irte_mask);
218         spin_unlock(&irq_2_ir_lock);
219
220         return 0;
221 }
222
223 struct intel_iommu *map_ioapic_to_ir(int apic)
224 {
225         int i;
226
227         for (i = 0; i < MAX_IO_APICS; i++)
228                 if (ir_ioapic[i].id == apic)
229                         return ir_ioapic[i].iommu;
230         return NULL;
231 }
232
233 struct intel_iommu *map_dev_to_ir(struct pci_dev *dev)
234 {
235         struct dmar_drhd_unit *drhd;
236
237         drhd = dmar_find_matched_drhd_unit(dev);
238         if (!drhd)
239                 return NULL;
240
241         return drhd->iommu;
242 }
243
244 int free_irte(int irq)
245 {
246         int index, i;
247         struct irte *irte;
248         struct intel_iommu *iommu;
249
250         spin_lock(&irq_2_ir_lock);
251         if (irq >= NR_IRQS || !irq_2_iommu[irq].iommu) {
252                 spin_unlock(&irq_2_ir_lock);
253                 return -1;
254         }
255
256         iommu = irq_2_iommu[irq].iommu;
257
258         index = irq_2_iommu[irq].irte_index + irq_2_iommu[irq].sub_handle;
259         irte = &iommu->ir_table->base[index];
260
261         if (!irq_2_iommu[irq].sub_handle) {
262                 for (i = 0; i < (1 << irq_2_iommu[irq].irte_mask); i++)
263                         set_64bit((unsigned long *)irte, 0);
264                 qi_flush_iec(iommu, index, irq_2_iommu[irq].irte_mask);
265         }
266
267         irq_2_iommu[irq].iommu = NULL;
268         irq_2_iommu[irq].irte_index = 0;
269         irq_2_iommu[irq].sub_handle = 0;
270         irq_2_iommu[irq].irte_mask = 0;
271
272         spin_unlock(&irq_2_ir_lock);
273
274         return 0;
275 }
276
277 static void iommu_set_intr_remapping(struct intel_iommu *iommu, int mode)
278 {
279         u64 addr;
280         u32 cmd, sts;
281         unsigned long flags;
282
283         addr = virt_to_phys((void *)iommu->ir_table->base);
284
285         spin_lock_irqsave(&iommu->register_lock, flags);
286
287         dmar_writeq(iommu->reg + DMAR_IRTA_REG,
288                     (addr) | IR_X2APIC_MODE(mode) | INTR_REMAP_TABLE_REG_SIZE);
289
290         /* Set interrupt-remapping table pointer */
291         cmd = iommu->gcmd | DMA_GCMD_SIRTP;
292         writel(cmd, iommu->reg + DMAR_GCMD_REG);
293
294         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
295                       readl, (sts & DMA_GSTS_IRTPS), sts);
296         spin_unlock_irqrestore(&iommu->register_lock, flags);
297
298         /*
299          * global invalidation of interrupt entry cache before enabling
300          * interrupt-remapping.
301          */
302         qi_global_iec(iommu);
303
304         spin_lock_irqsave(&iommu->register_lock, flags);
305
306         /* Enable interrupt-remapping */
307         cmd = iommu->gcmd | DMA_GCMD_IRE;
308         iommu->gcmd |= DMA_GCMD_IRE;
309         writel(cmd, iommu->reg + DMAR_GCMD_REG);
310
311         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
312                       readl, (sts & DMA_GSTS_IRES), sts);
313
314         spin_unlock_irqrestore(&iommu->register_lock, flags);
315 }
316
317
318 static int setup_intr_remapping(struct intel_iommu *iommu, int mode)
319 {
320         struct ir_table *ir_table;
321         struct page *pages;
322
323         ir_table = iommu->ir_table = kzalloc(sizeof(struct ir_table),
324                                              GFP_KERNEL);
325
326         if (!iommu->ir_table)
327                 return -ENOMEM;
328
329         pages = alloc_pages(GFP_KERNEL | __GFP_ZERO, INTR_REMAP_PAGE_ORDER);
330
331         if (!pages) {
332                 printk(KERN_ERR "failed to allocate pages of order %d\n",
333                        INTR_REMAP_PAGE_ORDER);
334                 kfree(iommu->ir_table);
335                 return -ENOMEM;
336         }
337
338         ir_table->base = page_address(pages);
339
340         iommu_set_intr_remapping(iommu, mode);
341         return 0;
342 }
343
344 int __init enable_intr_remapping(int eim)
345 {
346         struct dmar_drhd_unit *drhd;
347         int setup = 0;
348
349         /*
350          * check for the Interrupt-remapping support
351          */
352         for_each_drhd_unit(drhd) {
353                 struct intel_iommu *iommu = drhd->iommu;
354
355                 if (!ecap_ir_support(iommu->ecap))
356                         continue;
357
358                 if (eim && !ecap_eim_support(iommu->ecap)) {
359                         printk(KERN_INFO "DRHD %Lx: EIM not supported by DRHD, "
360                                " ecap %Lx\n", drhd->reg_base_addr, iommu->ecap);
361                         return -1;
362                 }
363         }
364
365         /*
366          * Enable queued invalidation for all the DRHD's.
367          */
368         for_each_drhd_unit(drhd) {
369                 int ret;
370                 struct intel_iommu *iommu = drhd->iommu;
371                 ret = dmar_enable_qi(iommu);
372
373                 if (ret) {
374                         printk(KERN_ERR "DRHD %Lx: failed to enable queued, "
375                                " invalidation, ecap %Lx, ret %d\n",
376                                drhd->reg_base_addr, iommu->ecap, ret);
377                         return -1;
378                 }
379         }
380
381         /*
382          * Setup Interrupt-remapping for all the DRHD's now.
383          */
384         for_each_drhd_unit(drhd) {
385                 struct intel_iommu *iommu = drhd->iommu;
386
387                 if (!ecap_ir_support(iommu->ecap))
388                         continue;
389
390                 if (setup_intr_remapping(iommu, eim))
391                         goto error;
392
393                 setup = 1;
394         }
395
396         if (!setup)
397                 goto error;
398
399         intr_remapping_enabled = 1;
400
401         return 0;
402
403 error:
404         /*
405          * handle error condition gracefully here!
406          */
407         return -1;
408 }
409
410 static int ir_parse_ioapic_scope(struct acpi_dmar_header *header,
411                                  struct intel_iommu *iommu)
412 {
413         struct acpi_dmar_hardware_unit *drhd;
414         struct acpi_dmar_device_scope *scope;
415         void *start, *end;
416
417         drhd = (struct acpi_dmar_hardware_unit *)header;
418
419         start = (void *)(drhd + 1);
420         end = ((void *)drhd) + header->length;
421
422         while (start < end) {
423                 scope = start;
424                 if (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_IOAPIC) {
425                         if (ir_ioapic_num == MAX_IO_APICS) {
426                                 printk(KERN_WARNING "Exceeded Max IO APICS\n");
427                                 return -1;
428                         }
429
430                         printk(KERN_INFO "IOAPIC id %d under DRHD base"
431                                " 0x%Lx\n", scope->enumeration_id,
432                                drhd->address);
433
434                         ir_ioapic[ir_ioapic_num].iommu = iommu;
435                         ir_ioapic[ir_ioapic_num].id = scope->enumeration_id;
436                         ir_ioapic_num++;
437                 }
438                 start += scope->length;
439         }
440
441         return 0;
442 }
443
444 /*
445  * Finds the assocaition between IOAPIC's and its Interrupt-remapping
446  * hardware unit.
447  */
448 int __init parse_ioapics_under_ir(void)
449 {
450         struct dmar_drhd_unit *drhd;
451         int ir_supported = 0;
452
453         for_each_drhd_unit(drhd) {
454                 struct intel_iommu *iommu = drhd->iommu;
455
456                 if (ecap_ir_support(iommu->ecap)) {
457                         if (ir_parse_ioapic_scope(drhd->hdr, iommu))
458                                 return -1;
459
460                         ir_supported = 1;
461                 }
462         }
463
464         if (ir_supported && ir_ioapic_num != nr_ioapics) {
465                 printk(KERN_WARNING
466                        "Not all IO-APIC's listed under remapping hardware\n");
467                 return -1;
468         }
469
470         return ir_supported;
471 }