[ARM] Feroceon: small cleanups to L2 cache code
[linux-2.6] / arch / arm / vfp / vfpdouble.c
1 /*
2  *  linux/arch/arm/vfp/vfpdouble.c
3  *
4  * This code is derived in part from John R. Housers softfloat library, which
5  * carries the following notice:
6  *
7  * ===========================================================================
8  * This C source file is part of the SoftFloat IEC/IEEE Floating-point
9  * Arithmetic Package, Release 2.
10  *
11  * Written by John R. Hauser.  This work was made possible in part by the
12  * International Computer Science Institute, located at Suite 600, 1947 Center
13  * Street, Berkeley, California 94704.  Funding was partially provided by the
14  * National Science Foundation under grant MIP-9311980.  The original version
15  * of this code was written as part of a project to build a fixed-point vector
16  * processor in collaboration with the University of California at Berkeley,
17  * overseen by Profs. Nelson Morgan and John Wawrzynek.  More information
18  * is available through the web page `http://HTTP.CS.Berkeley.EDU/~jhauser/
19  * arithmetic/softfloat.html'.
20  *
21  * THIS SOFTWARE IS DISTRIBUTED AS IS, FOR FREE.  Although reasonable effort
22  * has been made to avoid it, THIS SOFTWARE MAY CONTAIN FAULTS THAT WILL AT
23  * TIMES RESULT IN INCORRECT BEHAVIOR.  USE OF THIS SOFTWARE IS RESTRICTED TO
24  * PERSONS AND ORGANIZATIONS WHO CAN AND WILL TAKE FULL RESPONSIBILITY FOR ANY
25  * AND ALL LOSSES, COSTS, OR OTHER PROBLEMS ARISING FROM ITS USE.
26  *
27  * Derivative works are acceptable, even for commercial purposes, so long as
28  * (1) they include prominent notice that the work is derivative, and (2) they
29  * include prominent notice akin to these three paragraphs for those parts of
30  * this code that are retained.
31  * ===========================================================================
32  */
33 #include <linux/kernel.h>
34 #include <linux/bitops.h>
35
36 #include <asm/div64.h>
37 #include <asm/vfp.h>
38
39 #include "vfpinstr.h"
40 #include "vfp.h"
41
42 static struct vfp_double vfp_double_default_qnan = {
43         .exponent       = 2047,
44         .sign           = 0,
45         .significand    = VFP_DOUBLE_SIGNIFICAND_QNAN,
46 };
47
48 static void vfp_double_dump(const char *str, struct vfp_double *d)
49 {
50         pr_debug("VFP: %s: sign=%d exponent=%d significand=%016llx\n",
51                  str, d->sign != 0, d->exponent, d->significand);
52 }
53
54 static void vfp_double_normalise_denormal(struct vfp_double *vd)
55 {
56         int bits = 31 - fls(vd->significand >> 32);
57         if (bits == 31)
58                 bits = 63 - fls(vd->significand);
59
60         vfp_double_dump("normalise_denormal: in", vd);
61
62         if (bits) {
63                 vd->exponent -= bits - 1;
64                 vd->significand <<= bits;
65         }
66
67         vfp_double_dump("normalise_denormal: out", vd);
68 }
69
70 u32 vfp_double_normaliseround(int dd, struct vfp_double *vd, u32 fpscr, u32 exceptions, const char *func)
71 {
72         u64 significand, incr;
73         int exponent, shift, underflow;
74         u32 rmode;
75
76         vfp_double_dump("pack: in", vd);
77
78         /*
79          * Infinities and NaNs are a special case.
80          */
81         if (vd->exponent == 2047 && (vd->significand == 0 || exceptions))
82                 goto pack;
83
84         /*
85          * Special-case zero.
86          */
87         if (vd->significand == 0) {
88                 vd->exponent = 0;
89                 goto pack;
90         }
91
92         exponent = vd->exponent;
93         significand = vd->significand;
94
95         shift = 32 - fls(significand >> 32);
96         if (shift == 32)
97                 shift = 64 - fls(significand);
98         if (shift) {
99                 exponent -= shift;
100                 significand <<= shift;
101         }
102
103 #ifdef DEBUG
104         vd->exponent = exponent;
105         vd->significand = significand;
106         vfp_double_dump("pack: normalised", vd);
107 #endif
108
109         /*
110          * Tiny number?
111          */
112         underflow = exponent < 0;
113         if (underflow) {
114                 significand = vfp_shiftright64jamming(significand, -exponent);
115                 exponent = 0;
116 #ifdef DEBUG
117                 vd->exponent = exponent;
118                 vd->significand = significand;
119                 vfp_double_dump("pack: tiny number", vd);
120 #endif
121                 if (!(significand & ((1ULL << (VFP_DOUBLE_LOW_BITS + 1)) - 1)))
122                         underflow = 0;
123         }
124
125         /*
126          * Select rounding increment.
127          */
128         incr = 0;
129         rmode = fpscr & FPSCR_RMODE_MASK;
130
131         if (rmode == FPSCR_ROUND_NEAREST) {
132                 incr = 1ULL << VFP_DOUBLE_LOW_BITS;
133                 if ((significand & (1ULL << (VFP_DOUBLE_LOW_BITS + 1))) == 0)
134                         incr -= 1;
135         } else if (rmode == FPSCR_ROUND_TOZERO) {
136                 incr = 0;
137         } else if ((rmode == FPSCR_ROUND_PLUSINF) ^ (vd->sign != 0))
138                 incr = (1ULL << (VFP_DOUBLE_LOW_BITS + 1)) - 1;
139
140         pr_debug("VFP: rounding increment = 0x%08llx\n", incr);
141
142         /*
143          * Is our rounding going to overflow?
144          */
145         if ((significand + incr) < significand) {
146                 exponent += 1;
147                 significand = (significand >> 1) | (significand & 1);
148                 incr >>= 1;
149 #ifdef DEBUG
150                 vd->exponent = exponent;
151                 vd->significand = significand;
152                 vfp_double_dump("pack: overflow", vd);
153 #endif
154         }
155
156         /*
157          * If any of the low bits (which will be shifted out of the
158          * number) are non-zero, the result is inexact.
159          */
160         if (significand & ((1 << (VFP_DOUBLE_LOW_BITS + 1)) - 1))
161                 exceptions |= FPSCR_IXC;
162
163         /*
164          * Do our rounding.
165          */
166         significand += incr;
167
168         /*
169          * Infinity?
170          */
171         if (exponent >= 2046) {
172                 exceptions |= FPSCR_OFC | FPSCR_IXC;
173                 if (incr == 0) {
174                         vd->exponent = 2045;
175                         vd->significand = 0x7fffffffffffffffULL;
176                 } else {
177                         vd->exponent = 2047;            /* infinity */
178                         vd->significand = 0;
179                 }
180         } else {
181                 if (significand >> (VFP_DOUBLE_LOW_BITS + 1) == 0)
182                         exponent = 0;
183                 if (exponent || significand > 0x8000000000000000ULL)
184                         underflow = 0;
185                 if (underflow)
186                         exceptions |= FPSCR_UFC;
187                 vd->exponent = exponent;
188                 vd->significand = significand >> 1;
189         }
190
191  pack:
192         vfp_double_dump("pack: final", vd);
193         {
194                 s64 d = vfp_double_pack(vd);
195                 pr_debug("VFP: %s: d(d%d)=%016llx exceptions=%08x\n", func,
196                          dd, d, exceptions);
197                 vfp_put_double(d, dd);
198         }
199         return exceptions;
200 }
201
202 /*
203  * Propagate the NaN, setting exceptions if it is signalling.
204  * 'n' is always a NaN.  'm' may be a number, NaN or infinity.
205  */
206 static u32
207 vfp_propagate_nan(struct vfp_double *vdd, struct vfp_double *vdn,
208                   struct vfp_double *vdm, u32 fpscr)
209 {
210         struct vfp_double *nan;
211         int tn, tm = 0;
212
213         tn = vfp_double_type(vdn);
214
215         if (vdm)
216                 tm = vfp_double_type(vdm);
217
218         if (fpscr & FPSCR_DEFAULT_NAN)
219                 /*
220                  * Default NaN mode - always returns a quiet NaN
221                  */
222                 nan = &vfp_double_default_qnan;
223         else {
224                 /*
225                  * Contemporary mode - select the first signalling
226                  * NAN, or if neither are signalling, the first
227                  * quiet NAN.
228                  */
229                 if (tn == VFP_SNAN || (tm != VFP_SNAN && tn == VFP_QNAN))
230                         nan = vdn;
231                 else
232                         nan = vdm;
233                 /*
234                  * Make the NaN quiet.
235                  */
236                 nan->significand |= VFP_DOUBLE_SIGNIFICAND_QNAN;
237         }
238
239         *vdd = *nan;
240
241         /*
242          * If one was a signalling NAN, raise invalid operation.
243          */
244         return tn == VFP_SNAN || tm == VFP_SNAN ? FPSCR_IOC : VFP_NAN_FLAG;
245 }
246
247 /*
248  * Extended operations
249  */
250 static u32 vfp_double_fabs(int dd, int unused, int dm, u32 fpscr)
251 {
252         vfp_put_double(vfp_double_packed_abs(vfp_get_double(dm)), dd);
253         return 0;
254 }
255
256 static u32 vfp_double_fcpy(int dd, int unused, int dm, u32 fpscr)
257 {
258         vfp_put_double(vfp_get_double(dm), dd);
259         return 0;
260 }
261
262 static u32 vfp_double_fneg(int dd, int unused, int dm, u32 fpscr)
263 {
264         vfp_put_double(vfp_double_packed_negate(vfp_get_double(dm)), dd);
265         return 0;
266 }
267
268 static u32 vfp_double_fsqrt(int dd, int unused, int dm, u32 fpscr)
269 {
270         struct vfp_double vdm, vdd;
271         int ret, tm;
272
273         vfp_double_unpack(&vdm, vfp_get_double(dm));
274         tm = vfp_double_type(&vdm);
275         if (tm & (VFP_NAN|VFP_INFINITY)) {
276                 struct vfp_double *vdp = &vdd;
277
278                 if (tm & VFP_NAN)
279                         ret = vfp_propagate_nan(vdp, &vdm, NULL, fpscr);
280                 else if (vdm.sign == 0) {
281  sqrt_copy:
282                         vdp = &vdm;
283                         ret = 0;
284                 } else {
285  sqrt_invalid:
286                         vdp = &vfp_double_default_qnan;
287                         ret = FPSCR_IOC;
288                 }
289                 vfp_put_double(vfp_double_pack(vdp), dd);
290                 return ret;
291         }
292
293         /*
294          * sqrt(+/- 0) == +/- 0
295          */
296         if (tm & VFP_ZERO)
297                 goto sqrt_copy;
298
299         /*
300          * Normalise a denormalised number
301          */
302         if (tm & VFP_DENORMAL)
303                 vfp_double_normalise_denormal(&vdm);
304
305         /*
306          * sqrt(<0) = invalid
307          */
308         if (vdm.sign)
309                 goto sqrt_invalid;
310
311         vfp_double_dump("sqrt", &vdm);
312
313         /*
314          * Estimate the square root.
315          */
316         vdd.sign = 0;
317         vdd.exponent = ((vdm.exponent - 1023) >> 1) + 1023;
318         vdd.significand = (u64)vfp_estimate_sqrt_significand(vdm.exponent, vdm.significand >> 32) << 31;
319
320         vfp_double_dump("sqrt estimate1", &vdd);
321
322         vdm.significand >>= 1 + (vdm.exponent & 1);
323         vdd.significand += 2 + vfp_estimate_div128to64(vdm.significand, 0, vdd.significand);
324
325         vfp_double_dump("sqrt estimate2", &vdd);
326
327         /*
328          * And now adjust.
329          */
330         if ((vdd.significand & VFP_DOUBLE_LOW_BITS_MASK) <= 5) {
331                 if (vdd.significand < 2) {
332                         vdd.significand = ~0ULL;
333                 } else {
334                         u64 termh, terml, remh, reml;
335                         vdm.significand <<= 2;
336                         mul64to128(&termh, &terml, vdd.significand, vdd.significand);
337                         sub128(&remh, &reml, vdm.significand, 0, termh, terml);
338                         while ((s64)remh < 0) {
339                                 vdd.significand -= 1;
340                                 shift64left(&termh, &terml, vdd.significand);
341                                 terml |= 1;
342                                 add128(&remh, &reml, remh, reml, termh, terml);
343                         }
344                         vdd.significand |= (remh | reml) != 0;
345                 }
346         }
347         vdd.significand = vfp_shiftright64jamming(vdd.significand, 1);
348
349         return vfp_double_normaliseround(dd, &vdd, fpscr, 0, "fsqrt");
350 }
351
352 /*
353  * Equal        := ZC
354  * Less than    := N
355  * Greater than := C
356  * Unordered    := CV
357  */
358 static u32 vfp_compare(int dd, int signal_on_qnan, int dm, u32 fpscr)
359 {
360         s64 d, m;
361         u32 ret = 0;
362
363         m = vfp_get_double(dm);
364         if (vfp_double_packed_exponent(m) == 2047 && vfp_double_packed_mantissa(m)) {
365                 ret |= FPSCR_C | FPSCR_V;
366                 if (signal_on_qnan || !(vfp_double_packed_mantissa(m) & (1ULL << (VFP_DOUBLE_MANTISSA_BITS - 1))))
367                         /*
368                          * Signalling NaN, or signalling on quiet NaN
369                          */
370                         ret |= FPSCR_IOC;
371         }
372
373         d = vfp_get_double(dd);
374         if (vfp_double_packed_exponent(d) == 2047 && vfp_double_packed_mantissa(d)) {
375                 ret |= FPSCR_C | FPSCR_V;
376                 if (signal_on_qnan || !(vfp_double_packed_mantissa(d) & (1ULL << (VFP_DOUBLE_MANTISSA_BITS - 1))))
377                         /*
378                          * Signalling NaN, or signalling on quiet NaN
379                          */
380                         ret |= FPSCR_IOC;
381         }
382
383         if (ret == 0) {
384                 if (d == m || vfp_double_packed_abs(d | m) == 0) {
385                         /*
386                          * equal
387                          */
388                         ret |= FPSCR_Z | FPSCR_C;
389                 } else if (vfp_double_packed_sign(d ^ m)) {
390                         /*
391                          * different signs
392                          */
393                         if (vfp_double_packed_sign(d))
394                                 /*
395                                  * d is negative, so d < m
396                                  */
397                                 ret |= FPSCR_N;
398                         else
399                                 /*
400                                  * d is positive, so d > m
401                                  */
402                                 ret |= FPSCR_C;
403                 } else if ((vfp_double_packed_sign(d) != 0) ^ (d < m)) {
404                         /*
405                          * d < m
406                          */
407                         ret |= FPSCR_N;
408                 } else if ((vfp_double_packed_sign(d) != 0) ^ (d > m)) {
409                         /*
410                          * d > m
411                          */
412                         ret |= FPSCR_C;
413                 }
414         }
415
416         return ret;
417 }
418
419 static u32 vfp_double_fcmp(int dd, int unused, int dm, u32 fpscr)
420 {
421         return vfp_compare(dd, 0, dm, fpscr);
422 }
423
424 static u32 vfp_double_fcmpe(int dd, int unused, int dm, u32 fpscr)
425 {
426         return vfp_compare(dd, 1, dm, fpscr);
427 }
428
429 static u32 vfp_double_fcmpz(int dd, int unused, int dm, u32 fpscr)
430 {
431         return vfp_compare(dd, 0, VFP_REG_ZERO, fpscr);
432 }
433
434 static u32 vfp_double_fcmpez(int dd, int unused, int dm, u32 fpscr)
435 {
436         return vfp_compare(dd, 1, VFP_REG_ZERO, fpscr);
437 }
438
439 static u32 vfp_double_fcvts(int sd, int unused, int dm, u32 fpscr)
440 {
441         struct vfp_double vdm;
442         struct vfp_single vsd;
443         int tm;
444         u32 exceptions = 0;
445
446         vfp_double_unpack(&vdm, vfp_get_double(dm));
447
448         tm = vfp_double_type(&vdm);
449
450         /*
451          * If we have a signalling NaN, signal invalid operation.
452          */
453         if (tm == VFP_SNAN)
454                 exceptions = FPSCR_IOC;
455
456         if (tm & VFP_DENORMAL)
457                 vfp_double_normalise_denormal(&vdm);
458
459         vsd.sign = vdm.sign;
460         vsd.significand = vfp_hi64to32jamming(vdm.significand);
461
462         /*
463          * If we have an infinity or a NaN, the exponent must be 255
464          */
465         if (tm & (VFP_INFINITY|VFP_NAN)) {
466                 vsd.exponent = 255;
467                 if (tm == VFP_QNAN)
468                         vsd.significand |= VFP_SINGLE_SIGNIFICAND_QNAN;
469                 goto pack_nan;
470         } else if (tm & VFP_ZERO)
471                 vsd.exponent = 0;
472         else
473                 vsd.exponent = vdm.exponent - (1023 - 127);
474
475         return vfp_single_normaliseround(sd, &vsd, fpscr, exceptions, "fcvts");
476
477  pack_nan:
478         vfp_put_float(vfp_single_pack(&vsd), sd);
479         return exceptions;
480 }
481
482 static u32 vfp_double_fuito(int dd, int unused, int dm, u32 fpscr)
483 {
484         struct vfp_double vdm;
485         u32 m = vfp_get_float(dm);
486
487         vdm.sign = 0;
488         vdm.exponent = 1023 + 63 - 1;
489         vdm.significand = (u64)m;
490
491         return vfp_double_normaliseround(dd, &vdm, fpscr, 0, "fuito");
492 }
493
494 static u32 vfp_double_fsito(int dd, int unused, int dm, u32 fpscr)
495 {
496         struct vfp_double vdm;
497         u32 m = vfp_get_float(dm);
498
499         vdm.sign = (m & 0x80000000) >> 16;
500         vdm.exponent = 1023 + 63 - 1;
501         vdm.significand = vdm.sign ? -m : m;
502
503         return vfp_double_normaliseround(dd, &vdm, fpscr, 0, "fsito");
504 }
505
506 static u32 vfp_double_ftoui(int sd, int unused, int dm, u32 fpscr)
507 {
508         struct vfp_double vdm;
509         u32 d, exceptions = 0;
510         int rmode = fpscr & FPSCR_RMODE_MASK;
511         int tm;
512
513         vfp_double_unpack(&vdm, vfp_get_double(dm));
514
515         /*
516          * Do we have a denormalised number?
517          */
518         tm = vfp_double_type(&vdm);
519         if (tm & VFP_DENORMAL)
520                 exceptions |= FPSCR_IDC;
521
522         if (tm & VFP_NAN)
523                 vdm.sign = 0;
524
525         if (vdm.exponent >= 1023 + 32) {
526                 d = vdm.sign ? 0 : 0xffffffff;
527                 exceptions = FPSCR_IOC;
528         } else if (vdm.exponent >= 1023 - 1) {
529                 int shift = 1023 + 63 - vdm.exponent;
530                 u64 rem, incr = 0;
531
532                 /*
533                  * 2^0 <= m < 2^32-2^8
534                  */
535                 d = (vdm.significand << 1) >> shift;
536                 rem = vdm.significand << (65 - shift);
537
538                 if (rmode == FPSCR_ROUND_NEAREST) {
539                         incr = 0x8000000000000000ULL;
540                         if ((d & 1) == 0)
541                                 incr -= 1;
542                 } else if (rmode == FPSCR_ROUND_TOZERO) {
543                         incr = 0;
544                 } else if ((rmode == FPSCR_ROUND_PLUSINF) ^ (vdm.sign != 0)) {
545                         incr = ~0ULL;
546                 }
547
548                 if ((rem + incr) < rem) {
549                         if (d < 0xffffffff)
550                                 d += 1;
551                         else
552                                 exceptions |= FPSCR_IOC;
553                 }
554
555                 if (d && vdm.sign) {
556                         d = 0;
557                         exceptions |= FPSCR_IOC;
558                 } else if (rem)
559                         exceptions |= FPSCR_IXC;
560         } else {
561                 d = 0;
562                 if (vdm.exponent | vdm.significand) {
563                         exceptions |= FPSCR_IXC;
564                         if (rmode == FPSCR_ROUND_PLUSINF && vdm.sign == 0)
565                                 d = 1;
566                         else if (rmode == FPSCR_ROUND_MINUSINF && vdm.sign) {
567                                 d = 0;
568                                 exceptions |= FPSCR_IOC;
569                         }
570                 }
571         }
572
573         pr_debug("VFP: ftoui: d(s%d)=%08x exceptions=%08x\n", sd, d, exceptions);
574
575         vfp_put_float(d, sd);
576
577         return exceptions;
578 }
579
580 static u32 vfp_double_ftouiz(int sd, int unused, int dm, u32 fpscr)
581 {
582         return vfp_double_ftoui(sd, unused, dm, FPSCR_ROUND_TOZERO);
583 }
584
585 static u32 vfp_double_ftosi(int sd, int unused, int dm, u32 fpscr)
586 {
587         struct vfp_double vdm;
588         u32 d, exceptions = 0;
589         int rmode = fpscr & FPSCR_RMODE_MASK;
590         int tm;
591
592         vfp_double_unpack(&vdm, vfp_get_double(dm));
593         vfp_double_dump("VDM", &vdm);
594
595         /*
596          * Do we have denormalised number?
597          */
598         tm = vfp_double_type(&vdm);
599         if (tm & VFP_DENORMAL)
600                 exceptions |= FPSCR_IDC;
601
602         if (tm & VFP_NAN) {
603                 d = 0;
604                 exceptions |= FPSCR_IOC;
605         } else if (vdm.exponent >= 1023 + 32) {
606                 d = 0x7fffffff;
607                 if (vdm.sign)
608                         d = ~d;
609                 exceptions |= FPSCR_IOC;
610         } else if (vdm.exponent >= 1023 - 1) {
611                 int shift = 1023 + 63 - vdm.exponent;   /* 58 */
612                 u64 rem, incr = 0;
613
614                 d = (vdm.significand << 1) >> shift;
615                 rem = vdm.significand << (65 - shift);
616
617                 if (rmode == FPSCR_ROUND_NEAREST) {
618                         incr = 0x8000000000000000ULL;
619                         if ((d & 1) == 0)
620                                 incr -= 1;
621                 } else if (rmode == FPSCR_ROUND_TOZERO) {
622                         incr = 0;
623                 } else if ((rmode == FPSCR_ROUND_PLUSINF) ^ (vdm.sign != 0)) {
624                         incr = ~0ULL;
625                 }
626
627                 if ((rem + incr) < rem && d < 0xffffffff)
628                         d += 1;
629                 if (d > 0x7fffffff + (vdm.sign != 0)) {
630                         d = 0x7fffffff + (vdm.sign != 0);
631                         exceptions |= FPSCR_IOC;
632                 } else if (rem)
633                         exceptions |= FPSCR_IXC;
634
635                 if (vdm.sign)
636                         d = -d;
637         } else {
638                 d = 0;
639                 if (vdm.exponent | vdm.significand) {
640                         exceptions |= FPSCR_IXC;
641                         if (rmode == FPSCR_ROUND_PLUSINF && vdm.sign == 0)
642                                 d = 1;
643                         else if (rmode == FPSCR_ROUND_MINUSINF && vdm.sign)
644                                 d = -1;
645                 }
646         }
647
648         pr_debug("VFP: ftosi: d(s%d)=%08x exceptions=%08x\n", sd, d, exceptions);
649
650         vfp_put_float((s32)d, sd);
651
652         return exceptions;
653 }
654
655 static u32 vfp_double_ftosiz(int dd, int unused, int dm, u32 fpscr)
656 {
657         return vfp_double_ftosi(dd, unused, dm, FPSCR_ROUND_TOZERO);
658 }
659
660
661 static struct op fops_ext[32] = {
662         [FEXT_TO_IDX(FEXT_FCPY)]        = { vfp_double_fcpy,   0 },
663         [FEXT_TO_IDX(FEXT_FABS)]        = { vfp_double_fabs,   0 },
664         [FEXT_TO_IDX(FEXT_FNEG)]        = { vfp_double_fneg,   0 },
665         [FEXT_TO_IDX(FEXT_FSQRT)]       = { vfp_double_fsqrt,  0 },
666         [FEXT_TO_IDX(FEXT_FCMP)]        = { vfp_double_fcmp,   OP_SCALAR },
667         [FEXT_TO_IDX(FEXT_FCMPE)]       = { vfp_double_fcmpe,  OP_SCALAR },
668         [FEXT_TO_IDX(FEXT_FCMPZ)]       = { vfp_double_fcmpz,  OP_SCALAR },
669         [FEXT_TO_IDX(FEXT_FCMPEZ)]      = { vfp_double_fcmpez, OP_SCALAR },
670         [FEXT_TO_IDX(FEXT_FCVT)]        = { vfp_double_fcvts,  OP_SCALAR|OP_SD },
671         [FEXT_TO_IDX(FEXT_FUITO)]       = { vfp_double_fuito,  OP_SCALAR|OP_SM },
672         [FEXT_TO_IDX(FEXT_FSITO)]       = { vfp_double_fsito,  OP_SCALAR|OP_SM },
673         [FEXT_TO_IDX(FEXT_FTOUI)]       = { vfp_double_ftoui,  OP_SCALAR|OP_SD },
674         [FEXT_TO_IDX(FEXT_FTOUIZ)]      = { vfp_double_ftouiz, OP_SCALAR|OP_SD },
675         [FEXT_TO_IDX(FEXT_FTOSI)]       = { vfp_double_ftosi,  OP_SCALAR|OP_SD },
676         [FEXT_TO_IDX(FEXT_FTOSIZ)]      = { vfp_double_ftosiz, OP_SCALAR|OP_SD },
677 };
678
679
680
681
682 static u32
683 vfp_double_fadd_nonnumber(struct vfp_double *vdd, struct vfp_double *vdn,
684                           struct vfp_double *vdm, u32 fpscr)
685 {
686         struct vfp_double *vdp;
687         u32 exceptions = 0;
688         int tn, tm;
689
690         tn = vfp_double_type(vdn);
691         tm = vfp_double_type(vdm);
692
693         if (tn & tm & VFP_INFINITY) {
694                 /*
695                  * Two infinities.  Are they different signs?
696                  */
697                 if (vdn->sign ^ vdm->sign) {
698                         /*
699                          * different signs -> invalid
700                          */
701                         exceptions = FPSCR_IOC;
702                         vdp = &vfp_double_default_qnan;
703                 } else {
704                         /*
705                          * same signs -> valid
706                          */
707                         vdp = vdn;
708                 }
709         } else if (tn & VFP_INFINITY && tm & VFP_NUMBER) {
710                 /*
711                  * One infinity and one number -> infinity
712                  */
713                 vdp = vdn;
714         } else {
715                 /*
716                  * 'n' is a NaN of some type
717                  */
718                 return vfp_propagate_nan(vdd, vdn, vdm, fpscr);
719         }
720         *vdd = *vdp;
721         return exceptions;
722 }
723
724 static u32
725 vfp_double_add(struct vfp_double *vdd, struct vfp_double *vdn,
726                struct vfp_double *vdm, u32 fpscr)
727 {
728         u32 exp_diff;
729         u64 m_sig;
730
731         if (vdn->significand & (1ULL << 63) ||
732             vdm->significand & (1ULL << 63)) {
733                 pr_info("VFP: bad FP values in %s\n", __func__);
734                 vfp_double_dump("VDN", vdn);
735                 vfp_double_dump("VDM", vdm);
736         }
737
738         /*
739          * Ensure that 'n' is the largest magnitude number.  Note that
740          * if 'n' and 'm' have equal exponents, we do not swap them.
741          * This ensures that NaN propagation works correctly.
742          */
743         if (vdn->exponent < vdm->exponent) {
744                 struct vfp_double *t = vdn;
745                 vdn = vdm;
746                 vdm = t;
747         }
748
749         /*
750          * Is 'n' an infinity or a NaN?  Note that 'm' may be a number,
751          * infinity or a NaN here.
752          */
753         if (vdn->exponent == 2047)
754                 return vfp_double_fadd_nonnumber(vdd, vdn, vdm, fpscr);
755
756         /*
757          * We have two proper numbers, where 'vdn' is the larger magnitude.
758          *
759          * Copy 'n' to 'd' before doing the arithmetic.
760          */
761         *vdd = *vdn;
762
763         /*
764          * Align 'm' with the result.
765          */
766         exp_diff = vdn->exponent - vdm->exponent;
767         m_sig = vfp_shiftright64jamming(vdm->significand, exp_diff);
768
769         /*
770          * If the signs are different, we are really subtracting.
771          */
772         if (vdn->sign ^ vdm->sign) {
773                 m_sig = vdn->significand - m_sig;
774                 if ((s64)m_sig < 0) {
775                         vdd->sign = vfp_sign_negate(vdd->sign);
776                         m_sig = -m_sig;
777                 } else if (m_sig == 0) {
778                         vdd->sign = (fpscr & FPSCR_RMODE_MASK) ==
779                                       FPSCR_ROUND_MINUSINF ? 0x8000 : 0;
780                 }
781         } else {
782                 m_sig += vdn->significand;
783         }
784         vdd->significand = m_sig;
785
786         return 0;
787 }
788
789 static u32
790 vfp_double_multiply(struct vfp_double *vdd, struct vfp_double *vdn,
791                     struct vfp_double *vdm, u32 fpscr)
792 {
793         vfp_double_dump("VDN", vdn);
794         vfp_double_dump("VDM", vdm);
795
796         /*
797          * Ensure that 'n' is the largest magnitude number.  Note that
798          * if 'n' and 'm' have equal exponents, we do not swap them.
799          * This ensures that NaN propagation works correctly.
800          */
801         if (vdn->exponent < vdm->exponent) {
802                 struct vfp_double *t = vdn;
803                 vdn = vdm;
804                 vdm = t;
805                 pr_debug("VFP: swapping M <-> N\n");
806         }
807
808         vdd->sign = vdn->sign ^ vdm->sign;
809
810         /*
811          * If 'n' is an infinity or NaN, handle it.  'm' may be anything.
812          */
813         if (vdn->exponent == 2047) {
814                 if (vdn->significand || (vdm->exponent == 2047 && vdm->significand))
815                         return vfp_propagate_nan(vdd, vdn, vdm, fpscr);
816                 if ((vdm->exponent | vdm->significand) == 0) {
817                         *vdd = vfp_double_default_qnan;
818                         return FPSCR_IOC;
819                 }
820                 vdd->exponent = vdn->exponent;
821                 vdd->significand = 0;
822                 return 0;
823         }
824
825         /*
826          * If 'm' is zero, the result is always zero.  In this case,
827          * 'n' may be zero or a number, but it doesn't matter which.
828          */
829         if ((vdm->exponent | vdm->significand) == 0) {
830                 vdd->exponent = 0;
831                 vdd->significand = 0;
832                 return 0;
833         }
834
835         /*
836          * We add 2 to the destination exponent for the same reason
837          * as the addition case - though this time we have +1 from
838          * each input operand.
839          */
840         vdd->exponent = vdn->exponent + vdm->exponent - 1023 + 2;
841         vdd->significand = vfp_hi64multiply64(vdn->significand, vdm->significand);
842
843         vfp_double_dump("VDD", vdd);
844         return 0;
845 }
846
847 #define NEG_MULTIPLY    (1 << 0)
848 #define NEG_SUBTRACT    (1 << 1)
849
850 static u32
851 vfp_double_multiply_accumulate(int dd, int dn, int dm, u32 fpscr, u32 negate, char *func)
852 {
853         struct vfp_double vdd, vdp, vdn, vdm;
854         u32 exceptions;
855
856         vfp_double_unpack(&vdn, vfp_get_double(dn));
857         if (vdn.exponent == 0 && vdn.significand)
858                 vfp_double_normalise_denormal(&vdn);
859
860         vfp_double_unpack(&vdm, vfp_get_double(dm));
861         if (vdm.exponent == 0 && vdm.significand)
862                 vfp_double_normalise_denormal(&vdm);
863
864         exceptions = vfp_double_multiply(&vdp, &vdn, &vdm, fpscr);
865         if (negate & NEG_MULTIPLY)
866                 vdp.sign = vfp_sign_negate(vdp.sign);
867
868         vfp_double_unpack(&vdn, vfp_get_double(dd));
869         if (negate & NEG_SUBTRACT)
870                 vdn.sign = vfp_sign_negate(vdn.sign);
871
872         exceptions |= vfp_double_add(&vdd, &vdn, &vdp, fpscr);
873
874         return vfp_double_normaliseround(dd, &vdd, fpscr, exceptions, func);
875 }
876
877 /*
878  * Standard operations
879  */
880
881 /*
882  * sd = sd + (sn * sm)
883  */
884 static u32 vfp_double_fmac(int dd, int dn, int dm, u32 fpscr)
885 {
886         return vfp_double_multiply_accumulate(dd, dn, dm, fpscr, 0, "fmac");
887 }
888
889 /*
890  * sd = sd - (sn * sm)
891  */
892 static u32 vfp_double_fnmac(int dd, int dn, int dm, u32 fpscr)
893 {
894         return vfp_double_multiply_accumulate(dd, dn, dm, fpscr, NEG_MULTIPLY, "fnmac");
895 }
896
897 /*
898  * sd = -sd + (sn * sm)
899  */
900 static u32 vfp_double_fmsc(int dd, int dn, int dm, u32 fpscr)
901 {
902         return vfp_double_multiply_accumulate(dd, dn, dm, fpscr, NEG_SUBTRACT, "fmsc");
903 }
904
905 /*
906  * sd = -sd - (sn * sm)
907  */
908 static u32 vfp_double_fnmsc(int dd, int dn, int dm, u32 fpscr)
909 {
910         return vfp_double_multiply_accumulate(dd, dn, dm, fpscr, NEG_SUBTRACT | NEG_MULTIPLY, "fnmsc");
911 }
912
913 /*
914  * sd = sn * sm
915  */
916 static u32 vfp_double_fmul(int dd, int dn, int dm, u32 fpscr)
917 {
918         struct vfp_double vdd, vdn, vdm;
919         u32 exceptions;
920
921         vfp_double_unpack(&vdn, vfp_get_double(dn));
922         if (vdn.exponent == 0 && vdn.significand)
923                 vfp_double_normalise_denormal(&vdn);
924
925         vfp_double_unpack(&vdm, vfp_get_double(dm));
926         if (vdm.exponent == 0 && vdm.significand)
927                 vfp_double_normalise_denormal(&vdm);
928
929         exceptions = vfp_double_multiply(&vdd, &vdn, &vdm, fpscr);
930         return vfp_double_normaliseround(dd, &vdd, fpscr, exceptions, "fmul");
931 }
932
933 /*
934  * sd = -(sn * sm)
935  */
936 static u32 vfp_double_fnmul(int dd, int dn, int dm, u32 fpscr)
937 {
938         struct vfp_double vdd, vdn, vdm;
939         u32 exceptions;
940
941         vfp_double_unpack(&vdn, vfp_get_double(dn));
942         if (vdn.exponent == 0 && vdn.significand)
943                 vfp_double_normalise_denormal(&vdn);
944
945         vfp_double_unpack(&vdm, vfp_get_double(dm));
946         if (vdm.exponent == 0 && vdm.significand)
947                 vfp_double_normalise_denormal(&vdm);
948
949         exceptions = vfp_double_multiply(&vdd, &vdn, &vdm, fpscr);
950         vdd.sign = vfp_sign_negate(vdd.sign);
951
952         return vfp_double_normaliseround(dd, &vdd, fpscr, exceptions, "fnmul");
953 }
954
955 /*
956  * sd = sn + sm
957  */
958 static u32 vfp_double_fadd(int dd, int dn, int dm, u32 fpscr)
959 {
960         struct vfp_double vdd, vdn, vdm;
961         u32 exceptions;
962
963         vfp_double_unpack(&vdn, vfp_get_double(dn));
964         if (vdn.exponent == 0 && vdn.significand)
965                 vfp_double_normalise_denormal(&vdn);
966
967         vfp_double_unpack(&vdm, vfp_get_double(dm));
968         if (vdm.exponent == 0 && vdm.significand)
969                 vfp_double_normalise_denormal(&vdm);
970
971         exceptions = vfp_double_add(&vdd, &vdn, &vdm, fpscr);
972
973         return vfp_double_normaliseround(dd, &vdd, fpscr, exceptions, "fadd");
974 }
975
976 /*
977  * sd = sn - sm
978  */
979 static u32 vfp_double_fsub(int dd, int dn, int dm, u32 fpscr)
980 {
981         struct vfp_double vdd, vdn, vdm;
982         u32 exceptions;
983
984         vfp_double_unpack(&vdn, vfp_get_double(dn));
985         if (vdn.exponent == 0 && vdn.significand)
986                 vfp_double_normalise_denormal(&vdn);
987
988         vfp_double_unpack(&vdm, vfp_get_double(dm));
989         if (vdm.exponent == 0 && vdm.significand)
990                 vfp_double_normalise_denormal(&vdm);
991
992         /*
993          * Subtraction is like addition, but with a negated operand.
994          */
995         vdm.sign = vfp_sign_negate(vdm.sign);
996
997         exceptions = vfp_double_add(&vdd, &vdn, &vdm, fpscr);
998
999         return vfp_double_normaliseround(dd, &vdd, fpscr, exceptions, "fsub");
1000 }
1001
1002 /*
1003  * sd = sn / sm
1004  */
1005 static u32 vfp_double_fdiv(int dd, int dn, int dm, u32 fpscr)
1006 {
1007         struct vfp_double vdd, vdn, vdm;
1008         u32 exceptions = 0;
1009         int tm, tn;
1010
1011         vfp_double_unpack(&vdn, vfp_get_double(dn));
1012         vfp_double_unpack(&vdm, vfp_get_double(dm));
1013
1014         vdd.sign = vdn.sign ^ vdm.sign;
1015
1016         tn = vfp_double_type(&vdn);
1017         tm = vfp_double_type(&vdm);
1018
1019         /*
1020          * Is n a NAN?
1021          */
1022         if (tn & VFP_NAN)
1023                 goto vdn_nan;
1024
1025         /*
1026          * Is m a NAN?
1027          */
1028         if (tm & VFP_NAN)
1029                 goto vdm_nan;
1030
1031         /*
1032          * If n and m are infinity, the result is invalid
1033          * If n and m are zero, the result is invalid
1034          */
1035         if (tm & tn & (VFP_INFINITY|VFP_ZERO))
1036                 goto invalid;
1037
1038         /*
1039          * If n is infinity, the result is infinity
1040          */
1041         if (tn & VFP_INFINITY)
1042                 goto infinity;
1043
1044         /*
1045          * If m is zero, raise div0 exceptions
1046          */
1047         if (tm & VFP_ZERO)
1048                 goto divzero;
1049
1050         /*
1051          * If m is infinity, or n is zero, the result is zero
1052          */
1053         if (tm & VFP_INFINITY || tn & VFP_ZERO)
1054                 goto zero;
1055
1056         if (tn & VFP_DENORMAL)
1057                 vfp_double_normalise_denormal(&vdn);
1058         if (tm & VFP_DENORMAL)
1059                 vfp_double_normalise_denormal(&vdm);
1060
1061         /*
1062          * Ok, we have two numbers, we can perform division.
1063          */
1064         vdd.exponent = vdn.exponent - vdm.exponent + 1023 - 1;
1065         vdm.significand <<= 1;
1066         if (vdm.significand <= (2 * vdn.significand)) {
1067                 vdn.significand >>= 1;
1068                 vdd.exponent++;
1069         }
1070         vdd.significand = vfp_estimate_div128to64(vdn.significand, 0, vdm.significand);
1071         if ((vdd.significand & 0x1ff) <= 2) {
1072                 u64 termh, terml, remh, reml;
1073                 mul64to128(&termh, &terml, vdm.significand, vdd.significand);
1074                 sub128(&remh, &reml, vdn.significand, 0, termh, terml);
1075                 while ((s64)remh < 0) {
1076                         vdd.significand -= 1;
1077                         add128(&remh, &reml, remh, reml, 0, vdm.significand);
1078                 }
1079                 vdd.significand |= (reml != 0);
1080         }
1081         return vfp_double_normaliseround(dd, &vdd, fpscr, 0, "fdiv");
1082
1083  vdn_nan:
1084         exceptions = vfp_propagate_nan(&vdd, &vdn, &vdm, fpscr);
1085  pack:
1086         vfp_put_double(vfp_double_pack(&vdd), dd);
1087         return exceptions;
1088
1089  vdm_nan:
1090         exceptions = vfp_propagate_nan(&vdd, &vdm, &vdn, fpscr);
1091         goto pack;
1092
1093  zero:
1094         vdd.exponent = 0;
1095         vdd.significand = 0;
1096         goto pack;
1097
1098  divzero:
1099         exceptions = FPSCR_DZC;
1100  infinity:
1101         vdd.exponent = 2047;
1102         vdd.significand = 0;
1103         goto pack;
1104
1105  invalid:
1106         vfp_put_double(vfp_double_pack(&vfp_double_default_qnan), dd);
1107         return FPSCR_IOC;
1108 }
1109
1110 static struct op fops[16] = {
1111         [FOP_TO_IDX(FOP_FMAC)]  = { vfp_double_fmac,  0 },
1112         [FOP_TO_IDX(FOP_FNMAC)] = { vfp_double_fnmac, 0 },
1113         [FOP_TO_IDX(FOP_FMSC)]  = { vfp_double_fmsc,  0 },
1114         [FOP_TO_IDX(FOP_FNMSC)] = { vfp_double_fnmsc, 0 },
1115         [FOP_TO_IDX(FOP_FMUL)]  = { vfp_double_fmul,  0 },
1116         [FOP_TO_IDX(FOP_FNMUL)] = { vfp_double_fnmul, 0 },
1117         [FOP_TO_IDX(FOP_FADD)]  = { vfp_double_fadd,  0 },
1118         [FOP_TO_IDX(FOP_FSUB)]  = { vfp_double_fsub,  0 },
1119         [FOP_TO_IDX(FOP_FDIV)]  = { vfp_double_fdiv,  0 },
1120 };
1121
1122 #define FREG_BANK(x)    ((x) & 0x0c)
1123 #define FREG_IDX(x)     ((x) & 3)
1124
1125 u32 vfp_double_cpdo(u32 inst, u32 fpscr)
1126 {
1127         u32 op = inst & FOP_MASK;
1128         u32 exceptions = 0;
1129         unsigned int dest;
1130         unsigned int dn = vfp_get_dn(inst);
1131         unsigned int dm;
1132         unsigned int vecitr, veclen, vecstride;
1133         struct op *fop;
1134
1135         vecstride = (1 + ((fpscr & FPSCR_STRIDE_MASK) == FPSCR_STRIDE_MASK));
1136
1137         fop = (op == FOP_EXT) ? &fops_ext[FEXT_TO_IDX(inst)] : &fops[FOP_TO_IDX(op)];
1138
1139         /*
1140          * fcvtds takes an sN register number as destination, not dN.
1141          * It also always operates on scalars.
1142          */
1143         if (fop->flags & OP_SD)
1144                 dest = vfp_get_sd(inst);
1145         else
1146                 dest = vfp_get_dd(inst);
1147
1148         /*
1149          * f[us]ito takes a sN operand, not a dN operand.
1150          */
1151         if (fop->flags & OP_SM)
1152                 dm = vfp_get_sm(inst);
1153         else
1154                 dm = vfp_get_dm(inst);
1155
1156         /*
1157          * If destination bank is zero, vector length is always '1'.
1158          * ARM DDI0100F C5.1.3, C5.3.2.
1159          */
1160         if ((fop->flags & OP_SCALAR) || (FREG_BANK(dest) == 0))
1161                 veclen = 0;
1162         else
1163                 veclen = fpscr & FPSCR_LENGTH_MASK;
1164
1165         pr_debug("VFP: vecstride=%u veclen=%u\n", vecstride,
1166                  (veclen >> FPSCR_LENGTH_BIT) + 1);
1167
1168         if (!fop->fn)
1169                 goto invalid;
1170
1171         for (vecitr = 0; vecitr <= veclen; vecitr += 1 << FPSCR_LENGTH_BIT) {
1172                 u32 except;
1173                 char type;
1174
1175                 type = fop->flags & OP_SD ? 's' : 'd';
1176                 if (op == FOP_EXT)
1177                         pr_debug("VFP: itr%d (%c%u) = op[%u] (d%u)\n",
1178                                  vecitr >> FPSCR_LENGTH_BIT,
1179                                  type, dest, dn, dm);
1180                 else
1181                         pr_debug("VFP: itr%d (%c%u) = (d%u) op[%u] (d%u)\n",
1182                                  vecitr >> FPSCR_LENGTH_BIT,
1183                                  type, dest, dn, FOP_TO_IDX(op), dm);
1184
1185                 except = fop->fn(dest, dn, dm, fpscr);
1186                 pr_debug("VFP: itr%d: exceptions=%08x\n",
1187                          vecitr >> FPSCR_LENGTH_BIT, except);
1188
1189                 exceptions |= except;
1190
1191                 /*
1192                  * CHECK: It appears to be undefined whether we stop when
1193                  * we encounter an exception.  We continue.
1194                  */
1195                 dest = FREG_BANK(dest) + ((FREG_IDX(dest) + vecstride) & 3);
1196                 dn = FREG_BANK(dn) + ((FREG_IDX(dn) + vecstride) & 3);
1197                 if (FREG_BANK(dm) != 0)
1198                         dm = FREG_BANK(dm) + ((FREG_IDX(dm) + vecstride) & 3);
1199         }
1200         return exceptions;
1201
1202  invalid:
1203         return ~0;
1204 }