Merge branch 'linus' into perfcounters/urgent
[linux-2.6] / arch / blackfin / kernel / ipipe.c
1 /* -*- linux-c -*-
2  * linux/arch/blackfin/kernel/ipipe.c
3  *
4  * Copyright (C) 2005-2007 Philippe Gerum.
5  *
6  * This program is free software; you can redistribute it and/or modify
7  * it under the terms of the GNU General Public License as published by
8  * the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139,
9  * USA; either version 2 of the License, or (at your option) any later
10  * version.
11  *
12  * This program is distributed in the hope that it will be useful,
13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15  * GNU General Public License for more details.
16  *
17  * You should have received a copy of the GNU General Public License
18  * along with this program; if not, write to the Free Software
19  * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
20  *
21  * Architecture-dependent I-pipe support for the Blackfin.
22  */
23
24 #include <linux/kernel.h>
25 #include <linux/sched.h>
26 #include <linux/module.h>
27 #include <linux/interrupt.h>
28 #include <linux/percpu.h>
29 #include <linux/bitops.h>
30 #include <linux/slab.h>
31 #include <linux/errno.h>
32 #include <linux/kthread.h>
33 #include <asm/unistd.h>
34 #include <asm/system.h>
35 #include <asm/atomic.h>
36 #include <asm/io.h>
37
38 DEFINE_PER_CPU(struct pt_regs, __ipipe_tick_regs);
39
40 asmlinkage void asm_do_IRQ(unsigned int irq, struct pt_regs *regs);
41
42 static void __ipipe_no_irqtail(void);
43
44 unsigned long __ipipe_irq_tail_hook = (unsigned long)&__ipipe_no_irqtail;
45 EXPORT_SYMBOL(__ipipe_irq_tail_hook);
46
47 unsigned long __ipipe_core_clock;
48 EXPORT_SYMBOL(__ipipe_core_clock);
49
50 unsigned long __ipipe_freq_scale;
51 EXPORT_SYMBOL(__ipipe_freq_scale);
52
53 atomic_t __ipipe_irq_lvdepth[IVG15 + 1];
54
55 unsigned long __ipipe_irq_lvmask = bfin_no_irqs;
56 EXPORT_SYMBOL(__ipipe_irq_lvmask);
57
58 static void __ipipe_ack_irq(unsigned irq, struct irq_desc *desc)
59 {
60         desc->ipipe_ack(irq, desc);
61 }
62
63 /*
64  * __ipipe_enable_pipeline() -- We are running on the boot CPU, hw
65  * interrupts are off, and secondary CPUs are still lost in space.
66  */
67 void __ipipe_enable_pipeline(void)
68 {
69         unsigned irq;
70
71         __ipipe_core_clock = get_cclk(); /* Fetch this once. */
72         __ipipe_freq_scale = 1000000000UL / __ipipe_core_clock;
73
74         for (irq = 0; irq < NR_IRQS; ++irq)
75                 ipipe_virtualize_irq(ipipe_root_domain,
76                                      irq,
77                                      (ipipe_irq_handler_t)&asm_do_IRQ,
78                                      NULL,
79                                      &__ipipe_ack_irq,
80                                      IPIPE_HANDLE_MASK | IPIPE_PASS_MASK);
81 }
82
83 /*
84  * __ipipe_handle_irq() -- IPIPE's generic IRQ handler. An optimistic
85  * interrupt protection log is maintained here for each domain. Hw
86  * interrupts are masked on entry.
87  */
88 void __ipipe_handle_irq(unsigned irq, struct pt_regs *regs)
89 {
90         struct ipipe_percpu_domain_data *p = ipipe_root_cpudom_ptr();
91         struct ipipe_domain *this_domain, *next_domain;
92         struct list_head *head, *pos;
93         int m_ack, s = -1;
94
95         /*
96          * Software-triggered IRQs do not need any ack.  The contents
97          * of the register frame should only be used when processing
98          * the timer interrupt, but not for handling any other
99          * interrupt.
100          */
101         m_ack = (regs == NULL || irq == IRQ_SYSTMR || irq == IRQ_CORETMR);
102         this_domain = __ipipe_current_domain;
103
104         if (unlikely(test_bit(IPIPE_STICKY_FLAG, &this_domain->irqs[irq].control)))
105                 head = &this_domain->p_link;
106         else {
107                 head = __ipipe_pipeline.next;
108                 next_domain = list_entry(head, struct ipipe_domain, p_link);
109                 if (likely(test_bit(IPIPE_WIRED_FLAG, &next_domain->irqs[irq].control))) {
110                         if (!m_ack && next_domain->irqs[irq].acknowledge != NULL)
111                                 next_domain->irqs[irq].acknowledge(irq, irq_to_desc(irq));
112                         if (test_bit(IPIPE_SYNCDEFER_FLAG, &p->status))
113                                 s = __test_and_set_bit(IPIPE_STALL_FLAG, &p->status);
114                         __ipipe_dispatch_wired(next_domain, irq);
115                         goto out;
116                 }
117         }
118
119         /* Ack the interrupt. */
120
121         pos = head;
122         while (pos != &__ipipe_pipeline) {
123                 next_domain = list_entry(pos, struct ipipe_domain, p_link);
124                 if (test_bit(IPIPE_HANDLE_FLAG, &next_domain->irqs[irq].control)) {
125                         __ipipe_set_irq_pending(next_domain, irq);
126                         if (!m_ack && next_domain->irqs[irq].acknowledge != NULL) {
127                                 next_domain->irqs[irq].acknowledge(irq, irq_to_desc(irq));
128                                 m_ack = 1;
129                         }
130                 }
131                 if (!test_bit(IPIPE_PASS_FLAG, &next_domain->irqs[irq].control))
132                         break;
133                 pos = next_domain->p_link.next;
134         }
135
136         /*
137          * Now walk the pipeline, yielding control to the highest
138          * priority domain that has pending interrupt(s) or
139          * immediately to the current domain if the interrupt has been
140          * marked as 'sticky'. This search does not go beyond the
141          * current domain in the pipeline. We also enforce the
142          * additional root stage lock (blackfin-specific).
143          */
144         if (test_bit(IPIPE_SYNCDEFER_FLAG, &p->status))
145                 s = __test_and_set_bit(IPIPE_STALL_FLAG, &p->status);
146
147         /*
148          * If the interrupt preempted the head domain, then do not
149          * even try to walk the pipeline, unless an interrupt is
150          * pending for it.
151          */
152         if (test_bit(IPIPE_AHEAD_FLAG, &this_domain->flags) &&
153             ipipe_head_cpudom_var(irqpend_himask) == 0)
154                 goto out;
155
156         __ipipe_walk_pipeline(head);
157 out:
158         if (!s)
159                 __clear_bit(IPIPE_STALL_FLAG, &p->status);
160 }
161
162 int __ipipe_check_root(void)
163 {
164         return ipipe_root_domain_p;
165 }
166
167 void __ipipe_enable_irqdesc(struct ipipe_domain *ipd, unsigned irq)
168 {
169         struct irq_desc *desc = irq_to_desc(irq);
170         int prio = __ipipe_get_irq_priority(irq);
171
172         desc->depth = 0;
173         if (ipd != &ipipe_root &&
174             atomic_inc_return(&__ipipe_irq_lvdepth[prio]) == 1)
175                 __set_bit(prio, &__ipipe_irq_lvmask);
176 }
177 EXPORT_SYMBOL(__ipipe_enable_irqdesc);
178
179 void __ipipe_disable_irqdesc(struct ipipe_domain *ipd, unsigned irq)
180 {
181         int prio = __ipipe_get_irq_priority(irq);
182
183         if (ipd != &ipipe_root &&
184             atomic_dec_and_test(&__ipipe_irq_lvdepth[prio]))
185                 __clear_bit(prio, &__ipipe_irq_lvmask);
186 }
187 EXPORT_SYMBOL(__ipipe_disable_irqdesc);
188
189 void __ipipe_stall_root_raw(void)
190 {
191         /*
192          * This code is called by the ins{bwl} routines (see
193          * arch/blackfin/lib/ins.S), which are heavily used by the
194          * network stack. It masks all interrupts but those handled by
195          * non-root domains, so that we keep decent network transfer
196          * rates for Linux without inducing pathological jitter for
197          * the real-time domain.
198          */
199         __asm__ __volatile__ ("sti %0;" : : "d"(__ipipe_irq_lvmask));
200
201         __set_bit(IPIPE_STALL_FLAG,
202                   &ipipe_root_cpudom_var(status));
203 }
204
205 void __ipipe_unstall_root_raw(void)
206 {
207         __clear_bit(IPIPE_STALL_FLAG,
208                     &ipipe_root_cpudom_var(status));
209
210         __asm__ __volatile__ ("sti %0;" : : "d"(bfin_irq_flags));
211 }
212
213 int __ipipe_syscall_root(struct pt_regs *regs)
214 {
215         struct ipipe_percpu_domain_data *p;
216         unsigned long flags;
217         int ret;
218
219         /*
220          * We need to run the IRQ tail hook whenever we don't
221          * propagate a syscall to higher domains, because we know that
222          * important operations might be pending there (e.g. Xenomai
223          * deferred rescheduling).
224          */
225
226         if (regs->orig_p0 < NR_syscalls) {
227                 void (*hook)(void) = (void (*)(void))__ipipe_irq_tail_hook;
228                 hook();
229                 if ((current->flags & PF_EVNOTIFY) == 0)
230                         return 0;
231         }
232
233         /*
234          * This routine either returns:
235          * 0 -- if the syscall is to be passed to Linux;
236          * >0 -- if the syscall should not be passed to Linux, and no
237          * tail work should be performed;
238          * <0 -- if the syscall should not be passed to Linux but the
239          * tail work has to be performed (for handling signals etc).
240          */
241
242         if (!__ipipe_event_monitored_p(IPIPE_EVENT_SYSCALL))
243                 return 0;
244
245         ret = __ipipe_dispatch_event(IPIPE_EVENT_SYSCALL, regs);
246
247         local_irq_save_hw(flags);
248
249         if (!__ipipe_root_domain_p) {
250                 local_irq_restore_hw(flags);
251                 return 1;
252         }
253
254         p = ipipe_root_cpudom_ptr();
255         if ((p->irqpend_himask & IPIPE_IRQMASK_VIRT) != 0)
256                 __ipipe_sync_pipeline(IPIPE_IRQMASK_VIRT);
257
258         local_irq_restore_hw(flags);
259
260         return -ret;
261 }
262
263 unsigned long ipipe_critical_enter(void (*syncfn) (void))
264 {
265         unsigned long flags;
266
267         local_irq_save_hw(flags);
268
269         return flags;
270 }
271
272 void ipipe_critical_exit(unsigned long flags)
273 {
274         local_irq_restore_hw(flags);
275 }
276
277 static void __ipipe_no_irqtail(void)
278 {
279 }
280
281 int ipipe_get_sysinfo(struct ipipe_sysinfo *info)
282 {
283         info->ncpus = num_online_cpus();
284         info->cpufreq = ipipe_cpu_freq();
285         info->archdep.tmirq = IPIPE_TIMER_IRQ;
286         info->archdep.tmfreq = info->cpufreq;
287
288         return 0;
289 }
290
291 /*
292  * ipipe_trigger_irq() -- Push the interrupt at front of the pipeline
293  * just like if it has been actually received from a hw source. Also
294  * works for virtual interrupts.
295  */
296 int ipipe_trigger_irq(unsigned irq)
297 {
298         unsigned long flags;
299
300 #ifdef CONFIG_IPIPE_DEBUG
301         if (irq >= IPIPE_NR_IRQS ||
302             (ipipe_virtual_irq_p(irq)
303              && !test_bit(irq - IPIPE_VIRQ_BASE, &__ipipe_virtual_irq_map)))
304                 return -EINVAL;
305 #endif
306
307         local_irq_save_hw(flags);
308         __ipipe_handle_irq(irq, NULL);
309         local_irq_restore_hw(flags);
310
311         return 1;
312 }
313
314 asmlinkage void __ipipe_sync_root(void)
315 {
316         void (*irq_tail_hook)(void) = (void (*)(void))__ipipe_irq_tail_hook;
317         unsigned long flags;
318
319         BUG_ON(irqs_disabled());
320
321         local_irq_save_hw(flags);
322
323         if (irq_tail_hook)
324                 irq_tail_hook();
325
326         clear_thread_flag(TIF_IRQ_SYNC);
327
328         if (ipipe_root_cpudom_var(irqpend_himask) != 0)
329                 __ipipe_sync_pipeline(IPIPE_IRQMASK_ANY);
330
331         local_irq_restore_hw(flags);
332 }
333
334 void ___ipipe_sync_pipeline(unsigned long syncmask)
335 {
336         if (__ipipe_root_domain_p) {
337                 if (test_bit(IPIPE_SYNCDEFER_FLAG, &ipipe_root_cpudom_var(status)))
338                         return;
339         }
340
341         __ipipe_sync_stage(syncmask);
342 }
343
344 EXPORT_SYMBOL(show_stack);