[PATCH] md: break out of a loop that doesn't need to run to completion
[linux-2.6] / drivers / md / raid10.c
1 /*
2  * raid10.c : Multiple Devices driver for Linux
3  *
4  * Copyright (C) 2000-2004 Neil Brown
5  *
6  * RAID-10 support for md.
7  *
8  * Base on code in raid1.c.  See raid1.c for futher copyright information.
9  *
10  *
11  * This program is free software; you can redistribute it and/or modify
12  * it under the terms of the GNU General Public License as published by
13  * the Free Software Foundation; either version 2, or (at your option)
14  * any later version.
15  *
16  * You should have received a copy of the GNU General Public License
17  * (for example /usr/src/linux/COPYING); if not, write to the Free
18  * Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
19  */
20
21 #include "dm-bio-list.h"
22 #include <linux/raid/raid10.h>
23 #include <linux/raid/bitmap.h>
24
25 /*
26  * RAID10 provides a combination of RAID0 and RAID1 functionality.
27  * The layout of data is defined by
28  *    chunk_size
29  *    raid_disks
30  *    near_copies (stored in low byte of layout)
31  *    far_copies (stored in second byte of layout)
32  *
33  * The data to be stored is divided into chunks using chunksize.
34  * Each device is divided into far_copies sections.
35  * In each section, chunks are laid out in a style similar to raid0, but
36  * near_copies copies of each chunk is stored (each on a different drive).
37  * The starting device for each section is offset near_copies from the starting
38  * device of the previous section.
39  * Thus there are (near_copies*far_copies) of each chunk, and each is on a different
40  * drive.
41  * near_copies and far_copies must be at least one, and their product is at most
42  * raid_disks.
43  */
44
45 /*
46  * Number of guaranteed r10bios in case of extreme VM load:
47  */
48 #define NR_RAID10_BIOS 256
49
50 static void unplug_slaves(mddev_t *mddev);
51
52 static void allow_barrier(conf_t *conf);
53 static void lower_barrier(conf_t *conf);
54
55 static void * r10bio_pool_alloc(gfp_t gfp_flags, void *data)
56 {
57         conf_t *conf = data;
58         r10bio_t *r10_bio;
59         int size = offsetof(struct r10bio_s, devs[conf->copies]);
60
61         /* allocate a r10bio with room for raid_disks entries in the bios array */
62         r10_bio = kzalloc(size, gfp_flags);
63         if (!r10_bio)
64                 unplug_slaves(conf->mddev);
65
66         return r10_bio;
67 }
68
69 static void r10bio_pool_free(void *r10_bio, void *data)
70 {
71         kfree(r10_bio);
72 }
73
74 #define RESYNC_BLOCK_SIZE (64*1024)
75 //#define RESYNC_BLOCK_SIZE PAGE_SIZE
76 #define RESYNC_SECTORS (RESYNC_BLOCK_SIZE >> 9)
77 #define RESYNC_PAGES ((RESYNC_BLOCK_SIZE + PAGE_SIZE-1) / PAGE_SIZE)
78 #define RESYNC_WINDOW (2048*1024)
79
80 /*
81  * When performing a resync, we need to read and compare, so
82  * we need as many pages are there are copies.
83  * When performing a recovery, we need 2 bios, one for read,
84  * one for write (we recover only one drive per r10buf)
85  *
86  */
87 static void * r10buf_pool_alloc(gfp_t gfp_flags, void *data)
88 {
89         conf_t *conf = data;
90         struct page *page;
91         r10bio_t *r10_bio;
92         struct bio *bio;
93         int i, j;
94         int nalloc;
95
96         r10_bio = r10bio_pool_alloc(gfp_flags, conf);
97         if (!r10_bio) {
98                 unplug_slaves(conf->mddev);
99                 return NULL;
100         }
101
102         if (test_bit(MD_RECOVERY_SYNC, &conf->mddev->recovery))
103                 nalloc = conf->copies; /* resync */
104         else
105                 nalloc = 2; /* recovery */
106
107         /*
108          * Allocate bios.
109          */
110         for (j = nalloc ; j-- ; ) {
111                 bio = bio_alloc(gfp_flags, RESYNC_PAGES);
112                 if (!bio)
113                         goto out_free_bio;
114                 r10_bio->devs[j].bio = bio;
115         }
116         /*
117          * Allocate RESYNC_PAGES data pages and attach them
118          * where needed.
119          */
120         for (j = 0 ; j < nalloc; j++) {
121                 bio = r10_bio->devs[j].bio;
122                 for (i = 0; i < RESYNC_PAGES; i++) {
123                         page = alloc_page(gfp_flags);
124                         if (unlikely(!page))
125                                 goto out_free_pages;
126
127                         bio->bi_io_vec[i].bv_page = page;
128                 }
129         }
130
131         return r10_bio;
132
133 out_free_pages:
134         for ( ; i > 0 ; i--)
135                 put_page(bio->bi_io_vec[i-1].bv_page);
136         while (j--)
137                 for (i = 0; i < RESYNC_PAGES ; i++)
138                         put_page(r10_bio->devs[j].bio->bi_io_vec[i].bv_page);
139         j = -1;
140 out_free_bio:
141         while ( ++j < nalloc )
142                 bio_put(r10_bio->devs[j].bio);
143         r10bio_pool_free(r10_bio, conf);
144         return NULL;
145 }
146
147 static void r10buf_pool_free(void *__r10_bio, void *data)
148 {
149         int i;
150         conf_t *conf = data;
151         r10bio_t *r10bio = __r10_bio;
152         int j;
153
154         for (j=0; j < conf->copies; j++) {
155                 struct bio *bio = r10bio->devs[j].bio;
156                 if (bio) {
157                         for (i = 0; i < RESYNC_PAGES; i++) {
158                                 put_page(bio->bi_io_vec[i].bv_page);
159                                 bio->bi_io_vec[i].bv_page = NULL;
160                         }
161                         bio_put(bio);
162                 }
163         }
164         r10bio_pool_free(r10bio, conf);
165 }
166
167 static void put_all_bios(conf_t *conf, r10bio_t *r10_bio)
168 {
169         int i;
170
171         for (i = 0; i < conf->copies; i++) {
172                 struct bio **bio = & r10_bio->devs[i].bio;
173                 if (*bio && *bio != IO_BLOCKED)
174                         bio_put(*bio);
175                 *bio = NULL;
176         }
177 }
178
179 static inline void free_r10bio(r10bio_t *r10_bio)
180 {
181         conf_t *conf = mddev_to_conf(r10_bio->mddev);
182
183         /*
184          * Wake up any possible resync thread that waits for the device
185          * to go idle.
186          */
187         allow_barrier(conf);
188
189         put_all_bios(conf, r10_bio);
190         mempool_free(r10_bio, conf->r10bio_pool);
191 }
192
193 static inline void put_buf(r10bio_t *r10_bio)
194 {
195         conf_t *conf = mddev_to_conf(r10_bio->mddev);
196
197         mempool_free(r10_bio, conf->r10buf_pool);
198
199         lower_barrier(conf);
200 }
201
202 static void reschedule_retry(r10bio_t *r10_bio)
203 {
204         unsigned long flags;
205         mddev_t *mddev = r10_bio->mddev;
206         conf_t *conf = mddev_to_conf(mddev);
207
208         spin_lock_irqsave(&conf->device_lock, flags);
209         list_add(&r10_bio->retry_list, &conf->retry_list);
210         conf->nr_queued ++;
211         spin_unlock_irqrestore(&conf->device_lock, flags);
212
213         md_wakeup_thread(mddev->thread);
214 }
215
216 /*
217  * raid_end_bio_io() is called when we have finished servicing a mirrored
218  * operation and are ready to return a success/failure code to the buffer
219  * cache layer.
220  */
221 static void raid_end_bio_io(r10bio_t *r10_bio)
222 {
223         struct bio *bio = r10_bio->master_bio;
224
225         bio_endio(bio, bio->bi_size,
226                 test_bit(R10BIO_Uptodate, &r10_bio->state) ? 0 : -EIO);
227         free_r10bio(r10_bio);
228 }
229
230 /*
231  * Update disk head position estimator based on IRQ completion info.
232  */
233 static inline void update_head_pos(int slot, r10bio_t *r10_bio)
234 {
235         conf_t *conf = mddev_to_conf(r10_bio->mddev);
236
237         conf->mirrors[r10_bio->devs[slot].devnum].head_position =
238                 r10_bio->devs[slot].addr + (r10_bio->sectors);
239 }
240
241 static int raid10_end_read_request(struct bio *bio, unsigned int bytes_done, int error)
242 {
243         int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags);
244         r10bio_t * r10_bio = (r10bio_t *)(bio->bi_private);
245         int slot, dev;
246         conf_t *conf = mddev_to_conf(r10_bio->mddev);
247
248         if (bio->bi_size)
249                 return 1;
250
251         slot = r10_bio->read_slot;
252         dev = r10_bio->devs[slot].devnum;
253         /*
254          * this branch is our 'one mirror IO has finished' event handler:
255          */
256         update_head_pos(slot, r10_bio);
257
258         if (uptodate) {
259                 /*
260                  * Set R10BIO_Uptodate in our master bio, so that
261                  * we will return a good error code to the higher
262                  * levels even if IO on some other mirrored buffer fails.
263                  *
264                  * The 'master' represents the composite IO operation to
265                  * user-side. So if something waits for IO, then it will
266                  * wait for the 'master' bio.
267                  */
268                 set_bit(R10BIO_Uptodate, &r10_bio->state);
269                 raid_end_bio_io(r10_bio);
270         } else {
271                 /*
272                  * oops, read error:
273                  */
274                 char b[BDEVNAME_SIZE];
275                 if (printk_ratelimit())
276                         printk(KERN_ERR "raid10: %s: rescheduling sector %llu\n",
277                                bdevname(conf->mirrors[dev].rdev->bdev,b), (unsigned long long)r10_bio->sector);
278                 reschedule_retry(r10_bio);
279         }
280
281         rdev_dec_pending(conf->mirrors[dev].rdev, conf->mddev);
282         return 0;
283 }
284
285 static int raid10_end_write_request(struct bio *bio, unsigned int bytes_done, int error)
286 {
287         int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags);
288         r10bio_t * r10_bio = (r10bio_t *)(bio->bi_private);
289         int slot, dev;
290         conf_t *conf = mddev_to_conf(r10_bio->mddev);
291
292         if (bio->bi_size)
293                 return 1;
294
295         for (slot = 0; slot < conf->copies; slot++)
296                 if (r10_bio->devs[slot].bio == bio)
297                         break;
298         dev = r10_bio->devs[slot].devnum;
299
300         /*
301          * this branch is our 'one mirror IO has finished' event handler:
302          */
303         if (!uptodate) {
304                 md_error(r10_bio->mddev, conf->mirrors[dev].rdev);
305                 /* an I/O failed, we can't clear the bitmap */
306                 set_bit(R10BIO_Degraded, &r10_bio->state);
307         } else
308                 /*
309                  * Set R10BIO_Uptodate in our master bio, so that
310                  * we will return a good error code for to the higher
311                  * levels even if IO on some other mirrored buffer fails.
312                  *
313                  * The 'master' represents the composite IO operation to
314                  * user-side. So if something waits for IO, then it will
315                  * wait for the 'master' bio.
316                  */
317                 set_bit(R10BIO_Uptodate, &r10_bio->state);
318
319         update_head_pos(slot, r10_bio);
320
321         /*
322          *
323          * Let's see if all mirrored write operations have finished
324          * already.
325          */
326         if (atomic_dec_and_test(&r10_bio->remaining)) {
327                 /* clear the bitmap if all writes complete successfully */
328                 bitmap_endwrite(r10_bio->mddev->bitmap, r10_bio->sector,
329                                 r10_bio->sectors,
330                                 !test_bit(R10BIO_Degraded, &r10_bio->state),
331                                 0);
332                 md_write_end(r10_bio->mddev);
333                 raid_end_bio_io(r10_bio);
334         }
335
336         rdev_dec_pending(conf->mirrors[dev].rdev, conf->mddev);
337         return 0;
338 }
339
340
341 /*
342  * RAID10 layout manager
343  * Aswell as the chunksize and raid_disks count, there are two
344  * parameters: near_copies and far_copies.
345  * near_copies * far_copies must be <= raid_disks.
346  * Normally one of these will be 1.
347  * If both are 1, we get raid0.
348  * If near_copies == raid_disks, we get raid1.
349  *
350  * Chunks are layed out in raid0 style with near_copies copies of the
351  * first chunk, followed by near_copies copies of the next chunk and
352  * so on.
353  * If far_copies > 1, then after 1/far_copies of the array has been assigned
354  * as described above, we start again with a device offset of near_copies.
355  * So we effectively have another copy of the whole array further down all
356  * the drives, but with blocks on different drives.
357  * With this layout, and block is never stored twice on the one device.
358  *
359  * raid10_find_phys finds the sector offset of a given virtual sector
360  * on each device that it is on. If a block isn't on a device,
361  * that entry in the array is set to MaxSector.
362  *
363  * raid10_find_virt does the reverse mapping, from a device and a
364  * sector offset to a virtual address
365  */
366
367 static void raid10_find_phys(conf_t *conf, r10bio_t *r10bio)
368 {
369         int n,f;
370         sector_t sector;
371         sector_t chunk;
372         sector_t stripe;
373         int dev;
374
375         int slot = 0;
376
377         /* now calculate first sector/dev */
378         chunk = r10bio->sector >> conf->chunk_shift;
379         sector = r10bio->sector & conf->chunk_mask;
380
381         chunk *= conf->near_copies;
382         stripe = chunk;
383         dev = sector_div(stripe, conf->raid_disks);
384
385         sector += stripe << conf->chunk_shift;
386
387         /* and calculate all the others */
388         for (n=0; n < conf->near_copies; n++) {
389                 int d = dev;
390                 sector_t s = sector;
391                 r10bio->devs[slot].addr = sector;
392                 r10bio->devs[slot].devnum = d;
393                 slot++;
394
395                 for (f = 1; f < conf->far_copies; f++) {
396                         d += conf->near_copies;
397                         if (d >= conf->raid_disks)
398                                 d -= conf->raid_disks;
399                         s += conf->stride;
400                         r10bio->devs[slot].devnum = d;
401                         r10bio->devs[slot].addr = s;
402                         slot++;
403                 }
404                 dev++;
405                 if (dev >= conf->raid_disks) {
406                         dev = 0;
407                         sector += (conf->chunk_mask + 1);
408                 }
409         }
410         BUG_ON(slot != conf->copies);
411 }
412
413 static sector_t raid10_find_virt(conf_t *conf, sector_t sector, int dev)
414 {
415         sector_t offset, chunk, vchunk;
416
417         while (sector > conf->stride) {
418                 sector -= conf->stride;
419                 if (dev < conf->near_copies)
420                         dev += conf->raid_disks - conf->near_copies;
421                 else
422                         dev -= conf->near_copies;
423         }
424
425         offset = sector & conf->chunk_mask;
426         chunk = sector >> conf->chunk_shift;
427         vchunk = chunk * conf->raid_disks + dev;
428         sector_div(vchunk, conf->near_copies);
429         return (vchunk << conf->chunk_shift) + offset;
430 }
431
432 /**
433  *      raid10_mergeable_bvec -- tell bio layer if a two requests can be merged
434  *      @q: request queue
435  *      @bio: the buffer head that's been built up so far
436  *      @biovec: the request that could be merged to it.
437  *
438  *      Return amount of bytes we can accept at this offset
439  *      If near_copies == raid_disk, there are no striping issues,
440  *      but in that case, the function isn't called at all.
441  */
442 static int raid10_mergeable_bvec(request_queue_t *q, struct bio *bio,
443                                 struct bio_vec *bio_vec)
444 {
445         mddev_t *mddev = q->queuedata;
446         sector_t sector = bio->bi_sector + get_start_sect(bio->bi_bdev);
447         int max;
448         unsigned int chunk_sectors = mddev->chunk_size >> 9;
449         unsigned int bio_sectors = bio->bi_size >> 9;
450
451         max =  (chunk_sectors - ((sector & (chunk_sectors - 1)) + bio_sectors)) << 9;
452         if (max < 0) max = 0; /* bio_add cannot handle a negative return */
453         if (max <= bio_vec->bv_len && bio_sectors == 0)
454                 return bio_vec->bv_len;
455         else
456                 return max;
457 }
458
459 /*
460  * This routine returns the disk from which the requested read should
461  * be done. There is a per-array 'next expected sequential IO' sector
462  * number - if this matches on the next IO then we use the last disk.
463  * There is also a per-disk 'last know head position' sector that is
464  * maintained from IRQ contexts, both the normal and the resync IO
465  * completion handlers update this position correctly. If there is no
466  * perfect sequential match then we pick the disk whose head is closest.
467  *
468  * If there are 2 mirrors in the same 2 devices, performance degrades
469  * because position is mirror, not device based.
470  *
471  * The rdev for the device selected will have nr_pending incremented.
472  */
473
474 /*
475  * FIXME: possibly should rethink readbalancing and do it differently
476  * depending on near_copies / far_copies geometry.
477  */
478 static int read_balance(conf_t *conf, r10bio_t *r10_bio)
479 {
480         const unsigned long this_sector = r10_bio->sector;
481         int disk, slot, nslot;
482         const int sectors = r10_bio->sectors;
483         sector_t new_distance, current_distance;
484         mdk_rdev_t *rdev;
485
486         raid10_find_phys(conf, r10_bio);
487         rcu_read_lock();
488         /*
489          * Check if we can balance. We can balance on the whole
490          * device if no resync is going on (recovery is ok), or below
491          * the resync window. We take the first readable disk when
492          * above the resync window.
493          */
494         if (conf->mddev->recovery_cp < MaxSector
495             && (this_sector + sectors >= conf->next_resync)) {
496                 /* make sure that disk is operational */
497                 slot = 0;
498                 disk = r10_bio->devs[slot].devnum;
499
500                 while ((rdev = rcu_dereference(conf->mirrors[disk].rdev)) == NULL ||
501                        r10_bio->devs[slot].bio == IO_BLOCKED ||
502                        !test_bit(In_sync, &rdev->flags)) {
503                         slot++;
504                         if (slot == conf->copies) {
505                                 slot = 0;
506                                 disk = -1;
507                                 break;
508                         }
509                         disk = r10_bio->devs[slot].devnum;
510                 }
511                 goto rb_out;
512         }
513
514
515         /* make sure the disk is operational */
516         slot = 0;
517         disk = r10_bio->devs[slot].devnum;
518         while ((rdev=rcu_dereference(conf->mirrors[disk].rdev)) == NULL ||
519                r10_bio->devs[slot].bio == IO_BLOCKED ||
520                !test_bit(In_sync, &rdev->flags)) {
521                 slot ++;
522                 if (slot == conf->copies) {
523                         disk = -1;
524                         goto rb_out;
525                 }
526                 disk = r10_bio->devs[slot].devnum;
527         }
528
529
530         current_distance = abs(r10_bio->devs[slot].addr -
531                                conf->mirrors[disk].head_position);
532
533         /* Find the disk whose head is closest */
534
535         for (nslot = slot; nslot < conf->copies; nslot++) {
536                 int ndisk = r10_bio->devs[nslot].devnum;
537
538
539                 if ((rdev=rcu_dereference(conf->mirrors[ndisk].rdev)) == NULL ||
540                     r10_bio->devs[nslot].bio == IO_BLOCKED ||
541                     !test_bit(In_sync, &rdev->flags))
542                         continue;
543
544                 /* This optimisation is debatable, and completely destroys
545                  * sequential read speed for 'far copies' arrays.  So only
546                  * keep it for 'near' arrays, and review those later.
547                  */
548                 if (conf->near_copies > 1 && !atomic_read(&rdev->nr_pending)) {
549                         disk = ndisk;
550                         slot = nslot;
551                         break;
552                 }
553                 new_distance = abs(r10_bio->devs[nslot].addr -
554                                    conf->mirrors[ndisk].head_position);
555                 if (new_distance < current_distance) {
556                         current_distance = new_distance;
557                         disk = ndisk;
558                         slot = nslot;
559                 }
560         }
561
562 rb_out:
563         r10_bio->read_slot = slot;
564 /*      conf->next_seq_sect = this_sector + sectors;*/
565
566         if (disk >= 0 && (rdev=rcu_dereference(conf->mirrors[disk].rdev))!= NULL)
567                 atomic_inc(&conf->mirrors[disk].rdev->nr_pending);
568         rcu_read_unlock();
569
570         return disk;
571 }
572
573 static void unplug_slaves(mddev_t *mddev)
574 {
575         conf_t *conf = mddev_to_conf(mddev);
576         int i;
577
578         rcu_read_lock();
579         for (i=0; i<mddev->raid_disks; i++) {
580                 mdk_rdev_t *rdev = rcu_dereference(conf->mirrors[i].rdev);
581                 if (rdev && !test_bit(Faulty, &rdev->flags) && atomic_read(&rdev->nr_pending)) {
582                         request_queue_t *r_queue = bdev_get_queue(rdev->bdev);
583
584                         atomic_inc(&rdev->nr_pending);
585                         rcu_read_unlock();
586
587                         if (r_queue->unplug_fn)
588                                 r_queue->unplug_fn(r_queue);
589
590                         rdev_dec_pending(rdev, mddev);
591                         rcu_read_lock();
592                 }
593         }
594         rcu_read_unlock();
595 }
596
597 static void raid10_unplug(request_queue_t *q)
598 {
599         mddev_t *mddev = q->queuedata;
600
601         unplug_slaves(q->queuedata);
602         md_wakeup_thread(mddev->thread);
603 }
604
605 static int raid10_issue_flush(request_queue_t *q, struct gendisk *disk,
606                              sector_t *error_sector)
607 {
608         mddev_t *mddev = q->queuedata;
609         conf_t *conf = mddev_to_conf(mddev);
610         int i, ret = 0;
611
612         rcu_read_lock();
613         for (i=0; i<mddev->raid_disks && ret == 0; i++) {
614                 mdk_rdev_t *rdev = rcu_dereference(conf->mirrors[i].rdev);
615                 if (rdev && !test_bit(Faulty, &rdev->flags)) {
616                         struct block_device *bdev = rdev->bdev;
617                         request_queue_t *r_queue = bdev_get_queue(bdev);
618
619                         if (!r_queue->issue_flush_fn)
620                                 ret = -EOPNOTSUPP;
621                         else {
622                                 atomic_inc(&rdev->nr_pending);
623                                 rcu_read_unlock();
624                                 ret = r_queue->issue_flush_fn(r_queue, bdev->bd_disk,
625                                                               error_sector);
626                                 rdev_dec_pending(rdev, mddev);
627                                 rcu_read_lock();
628                         }
629                 }
630         }
631         rcu_read_unlock();
632         return ret;
633 }
634
635 /* Barriers....
636  * Sometimes we need to suspend IO while we do something else,
637  * either some resync/recovery, or reconfigure the array.
638  * To do this we raise a 'barrier'.
639  * The 'barrier' is a counter that can be raised multiple times
640  * to count how many activities are happening which preclude
641  * normal IO.
642  * We can only raise the barrier if there is no pending IO.
643  * i.e. if nr_pending == 0.
644  * We choose only to raise the barrier if no-one is waiting for the
645  * barrier to go down.  This means that as soon as an IO request
646  * is ready, no other operations which require a barrier will start
647  * until the IO request has had a chance.
648  *
649  * So: regular IO calls 'wait_barrier'.  When that returns there
650  *    is no backgroup IO happening,  It must arrange to call
651  *    allow_barrier when it has finished its IO.
652  * backgroup IO calls must call raise_barrier.  Once that returns
653  *    there is no normal IO happeing.  It must arrange to call
654  *    lower_barrier when the particular background IO completes.
655  */
656 #define RESYNC_DEPTH 32
657
658 static void raise_barrier(conf_t *conf, int force)
659 {
660         BUG_ON(force && !conf->barrier);
661         spin_lock_irq(&conf->resync_lock);
662
663         /* Wait until no block IO is waiting (unless 'force') */
664         wait_event_lock_irq(conf->wait_barrier, force || !conf->nr_waiting,
665                             conf->resync_lock,
666                             raid10_unplug(conf->mddev->queue));
667
668         /* block any new IO from starting */
669         conf->barrier++;
670
671         /* No wait for all pending IO to complete */
672         wait_event_lock_irq(conf->wait_barrier,
673                             !conf->nr_pending && conf->barrier < RESYNC_DEPTH,
674                             conf->resync_lock,
675                             raid10_unplug(conf->mddev->queue));
676
677         spin_unlock_irq(&conf->resync_lock);
678 }
679
680 static void lower_barrier(conf_t *conf)
681 {
682         unsigned long flags;
683         spin_lock_irqsave(&conf->resync_lock, flags);
684         conf->barrier--;
685         spin_unlock_irqrestore(&conf->resync_lock, flags);
686         wake_up(&conf->wait_barrier);
687 }
688
689 static void wait_barrier(conf_t *conf)
690 {
691         spin_lock_irq(&conf->resync_lock);
692         if (conf->barrier) {
693                 conf->nr_waiting++;
694                 wait_event_lock_irq(conf->wait_barrier, !conf->barrier,
695                                     conf->resync_lock,
696                                     raid10_unplug(conf->mddev->queue));
697                 conf->nr_waiting--;
698         }
699         conf->nr_pending++;
700         spin_unlock_irq(&conf->resync_lock);
701 }
702
703 static void allow_barrier(conf_t *conf)
704 {
705         unsigned long flags;
706         spin_lock_irqsave(&conf->resync_lock, flags);
707         conf->nr_pending--;
708         spin_unlock_irqrestore(&conf->resync_lock, flags);
709         wake_up(&conf->wait_barrier);
710 }
711
712 static void freeze_array(conf_t *conf)
713 {
714         /* stop syncio and normal IO and wait for everything to
715          * go quite.
716          * We increment barrier and nr_waiting, and then
717          * wait until barrier+nr_pending match nr_queued+2
718          */
719         spin_lock_irq(&conf->resync_lock);
720         conf->barrier++;
721         conf->nr_waiting++;
722         wait_event_lock_irq(conf->wait_barrier,
723                             conf->barrier+conf->nr_pending == conf->nr_queued+2,
724                             conf->resync_lock,
725                             raid10_unplug(conf->mddev->queue));
726         spin_unlock_irq(&conf->resync_lock);
727 }
728
729 static void unfreeze_array(conf_t *conf)
730 {
731         /* reverse the effect of the freeze */
732         spin_lock_irq(&conf->resync_lock);
733         conf->barrier--;
734         conf->nr_waiting--;
735         wake_up(&conf->wait_barrier);
736         spin_unlock_irq(&conf->resync_lock);
737 }
738
739 static int make_request(request_queue_t *q, struct bio * bio)
740 {
741         mddev_t *mddev = q->queuedata;
742         conf_t *conf = mddev_to_conf(mddev);
743         mirror_info_t *mirror;
744         r10bio_t *r10_bio;
745         struct bio *read_bio;
746         int i;
747         int chunk_sects = conf->chunk_mask + 1;
748         const int rw = bio_data_dir(bio);
749         struct bio_list bl;
750         unsigned long flags;
751
752         if (unlikely(bio_barrier(bio))) {
753                 bio_endio(bio, bio->bi_size, -EOPNOTSUPP);
754                 return 0;
755         }
756
757         /* If this request crosses a chunk boundary, we need to
758          * split it.  This will only happen for 1 PAGE (or less) requests.
759          */
760         if (unlikely( (bio->bi_sector & conf->chunk_mask) + (bio->bi_size >> 9)
761                       > chunk_sects &&
762                     conf->near_copies < conf->raid_disks)) {
763                 struct bio_pair *bp;
764                 /* Sanity check -- queue functions should prevent this happening */
765                 if (bio->bi_vcnt != 1 ||
766                     bio->bi_idx != 0)
767                         goto bad_map;
768                 /* This is a one page bio that upper layers
769                  * refuse to split for us, so we need to split it.
770                  */
771                 bp = bio_split(bio, bio_split_pool,
772                                chunk_sects - (bio->bi_sector & (chunk_sects - 1)) );
773                 if (make_request(q, &bp->bio1))
774                         generic_make_request(&bp->bio1);
775                 if (make_request(q, &bp->bio2))
776                         generic_make_request(&bp->bio2);
777
778                 bio_pair_release(bp);
779                 return 0;
780         bad_map:
781                 printk("raid10_make_request bug: can't convert block across chunks"
782                        " or bigger than %dk %llu %d\n", chunk_sects/2,
783                        (unsigned long long)bio->bi_sector, bio->bi_size >> 10);
784
785                 bio_io_error(bio, bio->bi_size);
786                 return 0;
787         }
788
789         md_write_start(mddev, bio);
790
791         /*
792          * Register the new request and wait if the reconstruction
793          * thread has put up a bar for new requests.
794          * Continue immediately if no resync is active currently.
795          */
796         wait_barrier(conf);
797
798         disk_stat_inc(mddev->gendisk, ios[rw]);
799         disk_stat_add(mddev->gendisk, sectors[rw], bio_sectors(bio));
800
801         r10_bio = mempool_alloc(conf->r10bio_pool, GFP_NOIO);
802
803         r10_bio->master_bio = bio;
804         r10_bio->sectors = bio->bi_size >> 9;
805
806         r10_bio->mddev = mddev;
807         r10_bio->sector = bio->bi_sector;
808         r10_bio->state = 0;
809
810         if (rw == READ) {
811                 /*
812                  * read balancing logic:
813                  */
814                 int disk = read_balance(conf, r10_bio);
815                 int slot = r10_bio->read_slot;
816                 if (disk < 0) {
817                         raid_end_bio_io(r10_bio);
818                         return 0;
819                 }
820                 mirror = conf->mirrors + disk;
821
822                 read_bio = bio_clone(bio, GFP_NOIO);
823
824                 r10_bio->devs[slot].bio = read_bio;
825
826                 read_bio->bi_sector = r10_bio->devs[slot].addr +
827                         mirror->rdev->data_offset;
828                 read_bio->bi_bdev = mirror->rdev->bdev;
829                 read_bio->bi_end_io = raid10_end_read_request;
830                 read_bio->bi_rw = READ;
831                 read_bio->bi_private = r10_bio;
832
833                 generic_make_request(read_bio);
834                 return 0;
835         }
836
837         /*
838          * WRITE:
839          */
840         /* first select target devices under spinlock and
841          * inc refcount on their rdev.  Record them by setting
842          * bios[x] to bio
843          */
844         raid10_find_phys(conf, r10_bio);
845         rcu_read_lock();
846         for (i = 0;  i < conf->copies; i++) {
847                 int d = r10_bio->devs[i].devnum;
848                 mdk_rdev_t *rdev = rcu_dereference(conf->mirrors[d].rdev);
849                 if (rdev &&
850                     !test_bit(Faulty, &rdev->flags)) {
851                         atomic_inc(&rdev->nr_pending);
852                         r10_bio->devs[i].bio = bio;
853                 } else {
854                         r10_bio->devs[i].bio = NULL;
855                         set_bit(R10BIO_Degraded, &r10_bio->state);
856                 }
857         }
858         rcu_read_unlock();
859
860         atomic_set(&r10_bio->remaining, 0);
861
862         bio_list_init(&bl);
863         for (i = 0; i < conf->copies; i++) {
864                 struct bio *mbio;
865                 int d = r10_bio->devs[i].devnum;
866                 if (!r10_bio->devs[i].bio)
867                         continue;
868
869                 mbio = bio_clone(bio, GFP_NOIO);
870                 r10_bio->devs[i].bio = mbio;
871
872                 mbio->bi_sector = r10_bio->devs[i].addr+
873                         conf->mirrors[d].rdev->data_offset;
874                 mbio->bi_bdev = conf->mirrors[d].rdev->bdev;
875                 mbio->bi_end_io = raid10_end_write_request;
876                 mbio->bi_rw = WRITE;
877                 mbio->bi_private = r10_bio;
878
879                 atomic_inc(&r10_bio->remaining);
880                 bio_list_add(&bl, mbio);
881         }
882
883         bitmap_startwrite(mddev->bitmap, bio->bi_sector, r10_bio->sectors, 0);
884         spin_lock_irqsave(&conf->device_lock, flags);
885         bio_list_merge(&conf->pending_bio_list, &bl);
886         blk_plug_device(mddev->queue);
887         spin_unlock_irqrestore(&conf->device_lock, flags);
888
889         return 0;
890 }
891
892 static void status(struct seq_file *seq, mddev_t *mddev)
893 {
894         conf_t *conf = mddev_to_conf(mddev);
895         int i;
896
897         if (conf->near_copies < conf->raid_disks)
898                 seq_printf(seq, " %dK chunks", mddev->chunk_size/1024);
899         if (conf->near_copies > 1)
900                 seq_printf(seq, " %d near-copies", conf->near_copies);
901         if (conf->far_copies > 1)
902                 seq_printf(seq, " %d far-copies", conf->far_copies);
903
904         seq_printf(seq, " [%d/%d] [", conf->raid_disks,
905                                                 conf->working_disks);
906         for (i = 0; i < conf->raid_disks; i++)
907                 seq_printf(seq, "%s",
908                               conf->mirrors[i].rdev &&
909                               test_bit(In_sync, &conf->mirrors[i].rdev->flags) ? "U" : "_");
910         seq_printf(seq, "]");
911 }
912
913 static void error(mddev_t *mddev, mdk_rdev_t *rdev)
914 {
915         char b[BDEVNAME_SIZE];
916         conf_t *conf = mddev_to_conf(mddev);
917
918         /*
919          * If it is not operational, then we have already marked it as dead
920          * else if it is the last working disks, ignore the error, let the
921          * next level up know.
922          * else mark the drive as failed
923          */
924         if (test_bit(In_sync, &rdev->flags)
925             && conf->working_disks == 1)
926                 /*
927                  * Don't fail the drive, just return an IO error.
928                  * The test should really be more sophisticated than
929                  * "working_disks == 1", but it isn't critical, and
930                  * can wait until we do more sophisticated "is the drive
931                  * really dead" tests...
932                  */
933                 return;
934         if (test_bit(In_sync, &rdev->flags)) {
935                 mddev->degraded++;
936                 conf->working_disks--;
937                 /*
938                  * if recovery is running, make sure it aborts.
939                  */
940                 set_bit(MD_RECOVERY_ERR, &mddev->recovery);
941         }
942         clear_bit(In_sync, &rdev->flags);
943         set_bit(Faulty, &rdev->flags);
944         mddev->sb_dirty = 1;
945         printk(KERN_ALERT "raid10: Disk failure on %s, disabling device. \n"
946                 "       Operation continuing on %d devices\n",
947                 bdevname(rdev->bdev,b), conf->working_disks);
948 }
949
950 static void print_conf(conf_t *conf)
951 {
952         int i;
953         mirror_info_t *tmp;
954
955         printk("RAID10 conf printout:\n");
956         if (!conf) {
957                 printk("(!conf)\n");
958                 return;
959         }
960         printk(" --- wd:%d rd:%d\n", conf->working_disks,
961                 conf->raid_disks);
962
963         for (i = 0; i < conf->raid_disks; i++) {
964                 char b[BDEVNAME_SIZE];
965                 tmp = conf->mirrors + i;
966                 if (tmp->rdev)
967                         printk(" disk %d, wo:%d, o:%d, dev:%s\n",
968                                 i, !test_bit(In_sync, &tmp->rdev->flags),
969                                 !test_bit(Faulty, &tmp->rdev->flags),
970                                 bdevname(tmp->rdev->bdev,b));
971         }
972 }
973
974 static void close_sync(conf_t *conf)
975 {
976         wait_barrier(conf);
977         allow_barrier(conf);
978
979         mempool_destroy(conf->r10buf_pool);
980         conf->r10buf_pool = NULL;
981 }
982
983 /* check if there are enough drives for
984  * every block to appear on atleast one
985  */
986 static int enough(conf_t *conf)
987 {
988         int first = 0;
989
990         do {
991                 int n = conf->copies;
992                 int cnt = 0;
993                 while (n--) {
994                         if (conf->mirrors[first].rdev)
995                                 cnt++;
996                         first = (first+1) % conf->raid_disks;
997                 }
998                 if (cnt == 0)
999                         return 0;
1000         } while (first != 0);
1001         return 1;
1002 }
1003
1004 static int raid10_spare_active(mddev_t *mddev)
1005 {
1006         int i;
1007         conf_t *conf = mddev->private;
1008         mirror_info_t *tmp;
1009
1010         /*
1011          * Find all non-in_sync disks within the RAID10 configuration
1012          * and mark them in_sync
1013          */
1014         for (i = 0; i < conf->raid_disks; i++) {
1015                 tmp = conf->mirrors + i;
1016                 if (tmp->rdev
1017                     && !test_bit(Faulty, &tmp->rdev->flags)
1018                     && !test_bit(In_sync, &tmp->rdev->flags)) {
1019                         conf->working_disks++;
1020                         mddev->degraded--;
1021                         set_bit(In_sync, &tmp->rdev->flags);
1022                 }
1023         }
1024
1025         print_conf(conf);
1026         return 0;
1027 }
1028
1029
1030 static int raid10_add_disk(mddev_t *mddev, mdk_rdev_t *rdev)
1031 {
1032         conf_t *conf = mddev->private;
1033         int found = 0;
1034         int mirror;
1035         mirror_info_t *p;
1036
1037         if (mddev->recovery_cp < MaxSector)
1038                 /* only hot-add to in-sync arrays, as recovery is
1039                  * very different from resync
1040                  */
1041                 return 0;
1042         if (!enough(conf))
1043                 return 0;
1044
1045         if (rdev->saved_raid_disk >= 0 &&
1046             conf->mirrors[rdev->saved_raid_disk].rdev == NULL)
1047                 mirror = rdev->saved_raid_disk;
1048         else
1049                 mirror = 0;
1050         for ( ; mirror < mddev->raid_disks; mirror++)
1051                 if ( !(p=conf->mirrors+mirror)->rdev) {
1052
1053                         blk_queue_stack_limits(mddev->queue,
1054                                                rdev->bdev->bd_disk->queue);
1055                         /* as we don't honour merge_bvec_fn, we must never risk
1056                          * violating it, so limit ->max_sector to one PAGE, as
1057                          * a one page request is never in violation.
1058                          */
1059                         if (rdev->bdev->bd_disk->queue->merge_bvec_fn &&
1060                             mddev->queue->max_sectors > (PAGE_SIZE>>9))
1061                                 mddev->queue->max_sectors = (PAGE_SIZE>>9);
1062
1063                         p->head_position = 0;
1064                         rdev->raid_disk = mirror;
1065                         found = 1;
1066                         if (rdev->saved_raid_disk != mirror)
1067                                 conf->fullsync = 1;
1068                         rcu_assign_pointer(p->rdev, rdev);
1069                         break;
1070                 }
1071
1072         print_conf(conf);
1073         return found;
1074 }
1075
1076 static int raid10_remove_disk(mddev_t *mddev, int number)
1077 {
1078         conf_t *conf = mddev->private;
1079         int err = 0;
1080         mdk_rdev_t *rdev;
1081         mirror_info_t *p = conf->mirrors+ number;
1082
1083         print_conf(conf);
1084         rdev = p->rdev;
1085         if (rdev) {
1086                 if (test_bit(In_sync, &rdev->flags) ||
1087                     atomic_read(&rdev->nr_pending)) {
1088                         err = -EBUSY;
1089                         goto abort;
1090                 }
1091                 p->rdev = NULL;
1092                 synchronize_rcu();
1093                 if (atomic_read(&rdev->nr_pending)) {
1094                         /* lost the race, try later */
1095                         err = -EBUSY;
1096                         p->rdev = rdev;
1097                 }
1098         }
1099 abort:
1100
1101         print_conf(conf);
1102         return err;
1103 }
1104
1105
1106 static int end_sync_read(struct bio *bio, unsigned int bytes_done, int error)
1107 {
1108         r10bio_t * r10_bio = (r10bio_t *)(bio->bi_private);
1109         conf_t *conf = mddev_to_conf(r10_bio->mddev);
1110         int i,d;
1111
1112         if (bio->bi_size)
1113                 return 1;
1114
1115         for (i=0; i<conf->copies; i++)
1116                 if (r10_bio->devs[i].bio == bio)
1117                         break;
1118         if (i == conf->copies)
1119                 BUG();
1120         update_head_pos(i, r10_bio);
1121         d = r10_bio->devs[i].devnum;
1122
1123         if (test_bit(BIO_UPTODATE, &bio->bi_flags))
1124                 set_bit(R10BIO_Uptodate, &r10_bio->state);
1125         else if (!test_bit(MD_RECOVERY_SYNC, &conf->mddev->recovery))
1126                 md_error(r10_bio->mddev,
1127                          conf->mirrors[d].rdev);
1128
1129         /* for reconstruct, we always reschedule after a read.
1130          * for resync, only after all reads
1131          */
1132         if (test_bit(R10BIO_IsRecover, &r10_bio->state) ||
1133             atomic_dec_and_test(&r10_bio->remaining)) {
1134                 /* we have read all the blocks,
1135                  * do the comparison in process context in raid10d
1136                  */
1137                 reschedule_retry(r10_bio);
1138         }
1139         rdev_dec_pending(conf->mirrors[d].rdev, conf->mddev);
1140         return 0;
1141 }
1142
1143 static int end_sync_write(struct bio *bio, unsigned int bytes_done, int error)
1144 {
1145         int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags);
1146         r10bio_t * r10_bio = (r10bio_t *)(bio->bi_private);
1147         mddev_t *mddev = r10_bio->mddev;
1148         conf_t *conf = mddev_to_conf(mddev);
1149         int i,d;
1150
1151         if (bio->bi_size)
1152                 return 1;
1153
1154         for (i = 0; i < conf->copies; i++)
1155                 if (r10_bio->devs[i].bio == bio)
1156                         break;
1157         d = r10_bio->devs[i].devnum;
1158
1159         if (!uptodate)
1160                 md_error(mddev, conf->mirrors[d].rdev);
1161         update_head_pos(i, r10_bio);
1162
1163         while (atomic_dec_and_test(&r10_bio->remaining)) {
1164                 if (r10_bio->master_bio == NULL) {
1165                         /* the primary of several recovery bios */
1166                         md_done_sync(mddev, r10_bio->sectors, 1);
1167                         put_buf(r10_bio);
1168                         break;
1169                 } else {
1170                         r10bio_t *r10_bio2 = (r10bio_t *)r10_bio->master_bio;
1171                         put_buf(r10_bio);
1172                         r10_bio = r10_bio2;
1173                 }
1174         }
1175         rdev_dec_pending(conf->mirrors[d].rdev, mddev);
1176         return 0;
1177 }
1178
1179 /*
1180  * Note: sync and recover and handled very differently for raid10
1181  * This code is for resync.
1182  * For resync, we read through virtual addresses and read all blocks.
1183  * If there is any error, we schedule a write.  The lowest numbered
1184  * drive is authoritative.
1185  * However requests come for physical address, so we need to map.
1186  * For every physical address there are raid_disks/copies virtual addresses,
1187  * which is always are least one, but is not necessarly an integer.
1188  * This means that a physical address can span multiple chunks, so we may
1189  * have to submit multiple io requests for a single sync request.
1190  */
1191 /*
1192  * We check if all blocks are in-sync and only write to blocks that
1193  * aren't in sync
1194  */
1195 static void sync_request_write(mddev_t *mddev, r10bio_t *r10_bio)
1196 {
1197         conf_t *conf = mddev_to_conf(mddev);
1198         int i, first;
1199         struct bio *tbio, *fbio;
1200
1201         atomic_set(&r10_bio->remaining, 1);
1202
1203         /* find the first device with a block */
1204         for (i=0; i<conf->copies; i++)
1205                 if (test_bit(BIO_UPTODATE, &r10_bio->devs[i].bio->bi_flags))
1206                         break;
1207
1208         if (i == conf->copies)
1209                 goto done;
1210
1211         first = i;
1212         fbio = r10_bio->devs[i].bio;
1213
1214         /* now find blocks with errors */
1215         for (i=0 ; i < conf->copies ; i++) {
1216                 int  j, d;
1217                 int vcnt = r10_bio->sectors >> (PAGE_SHIFT-9);
1218
1219                 tbio = r10_bio->devs[i].bio;
1220
1221                 if (tbio->bi_end_io != end_sync_read)
1222                         continue;
1223                 if (i == first)
1224                         continue;
1225                 if (test_bit(BIO_UPTODATE, &r10_bio->devs[i].bio->bi_flags)) {
1226                         /* We know that the bi_io_vec layout is the same for
1227                          * both 'first' and 'i', so we just compare them.
1228                          * All vec entries are PAGE_SIZE;
1229                          */
1230                         for (j = 0; j < vcnt; j++)
1231                                 if (memcmp(page_address(fbio->bi_io_vec[j].bv_page),
1232                                            page_address(tbio->bi_io_vec[j].bv_page),
1233                                            PAGE_SIZE))
1234                                         break;
1235                         if (j == vcnt)
1236                                 continue;
1237                         mddev->resync_mismatches += r10_bio->sectors;
1238                 }
1239                 if (test_bit(MD_RECOVERY_CHECK, &mddev->recovery))
1240                         /* Don't fix anything. */
1241                         continue;
1242                 /* Ok, we need to write this bio
1243                  * First we need to fixup bv_offset, bv_len and
1244                  * bi_vecs, as the read request might have corrupted these
1245                  */
1246                 tbio->bi_vcnt = vcnt;
1247                 tbio->bi_size = r10_bio->sectors << 9;
1248                 tbio->bi_idx = 0;
1249                 tbio->bi_phys_segments = 0;
1250                 tbio->bi_hw_segments = 0;
1251                 tbio->bi_hw_front_size = 0;
1252                 tbio->bi_hw_back_size = 0;
1253                 tbio->bi_flags &= ~(BIO_POOL_MASK - 1);
1254                 tbio->bi_flags |= 1 << BIO_UPTODATE;
1255                 tbio->bi_next = NULL;
1256                 tbio->bi_rw = WRITE;
1257                 tbio->bi_private = r10_bio;
1258                 tbio->bi_sector = r10_bio->devs[i].addr;
1259
1260                 for (j=0; j < vcnt ; j++) {
1261                         tbio->bi_io_vec[j].bv_offset = 0;
1262                         tbio->bi_io_vec[j].bv_len = PAGE_SIZE;
1263
1264                         memcpy(page_address(tbio->bi_io_vec[j].bv_page),
1265                                page_address(fbio->bi_io_vec[j].bv_page),
1266                                PAGE_SIZE);
1267                 }
1268                 tbio->bi_end_io = end_sync_write;
1269
1270                 d = r10_bio->devs[i].devnum;
1271                 atomic_inc(&conf->mirrors[d].rdev->nr_pending);
1272                 atomic_inc(&r10_bio->remaining);
1273                 md_sync_acct(conf->mirrors[d].rdev->bdev, tbio->bi_size >> 9);
1274
1275                 tbio->bi_sector += conf->mirrors[d].rdev->data_offset;
1276                 tbio->bi_bdev = conf->mirrors[d].rdev->bdev;
1277                 generic_make_request(tbio);
1278         }
1279
1280 done:
1281         if (atomic_dec_and_test(&r10_bio->remaining)) {
1282                 md_done_sync(mddev, r10_bio->sectors, 1);
1283                 put_buf(r10_bio);
1284         }
1285 }
1286
1287 /*
1288  * Now for the recovery code.
1289  * Recovery happens across physical sectors.
1290  * We recover all non-is_sync drives by finding the virtual address of
1291  * each, and then choose a working drive that also has that virt address.
1292  * There is a separate r10_bio for each non-in_sync drive.
1293  * Only the first two slots are in use. The first for reading,
1294  * The second for writing.
1295  *
1296  */
1297
1298 static void recovery_request_write(mddev_t *mddev, r10bio_t *r10_bio)
1299 {
1300         conf_t *conf = mddev_to_conf(mddev);
1301         int i, d;
1302         struct bio *bio, *wbio;
1303
1304
1305         /* move the pages across to the second bio
1306          * and submit the write request
1307          */
1308         bio = r10_bio->devs[0].bio;
1309         wbio = r10_bio->devs[1].bio;
1310         for (i=0; i < wbio->bi_vcnt; i++) {
1311                 struct page *p = bio->bi_io_vec[i].bv_page;
1312                 bio->bi_io_vec[i].bv_page = wbio->bi_io_vec[i].bv_page;
1313                 wbio->bi_io_vec[i].bv_page = p;
1314         }
1315         d = r10_bio->devs[1].devnum;
1316
1317         atomic_inc(&conf->mirrors[d].rdev->nr_pending);
1318         md_sync_acct(conf->mirrors[d].rdev->bdev, wbio->bi_size >> 9);
1319         if (test_bit(R10BIO_Uptodate, &r10_bio->state))
1320                 generic_make_request(wbio);
1321         else
1322                 bio_endio(wbio, wbio->bi_size, -EIO);
1323 }
1324
1325
1326 /*
1327  * This is a kernel thread which:
1328  *
1329  *      1.      Retries failed read operations on working mirrors.
1330  *      2.      Updates the raid superblock when problems encounter.
1331  *      3.      Performs writes following reads for array syncronising.
1332  */
1333
1334 static void raid10d(mddev_t *mddev)
1335 {
1336         r10bio_t *r10_bio;
1337         struct bio *bio;
1338         unsigned long flags;
1339         conf_t *conf = mddev_to_conf(mddev);
1340         struct list_head *head = &conf->retry_list;
1341         int unplug=0;
1342         mdk_rdev_t *rdev;
1343
1344         md_check_recovery(mddev);
1345
1346         for (;;) {
1347                 char b[BDEVNAME_SIZE];
1348                 spin_lock_irqsave(&conf->device_lock, flags);
1349
1350                 if (conf->pending_bio_list.head) {
1351                         bio = bio_list_get(&conf->pending_bio_list);
1352                         blk_remove_plug(mddev->queue);
1353                         spin_unlock_irqrestore(&conf->device_lock, flags);
1354                         /* flush any pending bitmap writes to disk before proceeding w/ I/O */
1355                         if (bitmap_unplug(mddev->bitmap) != 0)
1356                                 printk("%s: bitmap file write failed!\n", mdname(mddev));
1357
1358                         while (bio) { /* submit pending writes */
1359                                 struct bio *next = bio->bi_next;
1360                                 bio->bi_next = NULL;
1361                                 generic_make_request(bio);
1362                                 bio = next;
1363                         }
1364                         unplug = 1;
1365
1366                         continue;
1367                 }
1368
1369                 if (list_empty(head))
1370                         break;
1371                 r10_bio = list_entry(head->prev, r10bio_t, retry_list);
1372                 list_del(head->prev);
1373                 conf->nr_queued--;
1374                 spin_unlock_irqrestore(&conf->device_lock, flags);
1375
1376                 mddev = r10_bio->mddev;
1377                 conf = mddev_to_conf(mddev);
1378                 if (test_bit(R10BIO_IsSync, &r10_bio->state)) {
1379                         sync_request_write(mddev, r10_bio);
1380                         unplug = 1;
1381                 } else  if (test_bit(R10BIO_IsRecover, &r10_bio->state)) {
1382                         recovery_request_write(mddev, r10_bio);
1383                         unplug = 1;
1384                 } else {
1385                         int mirror;
1386                         /* we got a read error. Maybe the drive is bad.  Maybe just
1387                          * the block and we can fix it.
1388                          * We freeze all other IO, and try reading the block from
1389                          * other devices.  When we find one, we re-write
1390                          * and check it that fixes the read error.
1391                          * This is all done synchronously while the array is
1392                          * frozen.
1393                          */
1394                         int sect = 0; /* Offset from r10_bio->sector */
1395                         int sectors = r10_bio->sectors;
1396                         freeze_array(conf);
1397                         if (mddev->ro == 0) while(sectors) {
1398                                 int s = sectors;
1399                                 int sl = r10_bio->read_slot;
1400                                 int success = 0;
1401
1402                                 if (s > (PAGE_SIZE>>9))
1403                                         s = PAGE_SIZE >> 9;
1404
1405                                 do {
1406                                         int d = r10_bio->devs[sl].devnum;
1407                                         rdev = conf->mirrors[d].rdev;
1408                                         if (rdev &&
1409                                             test_bit(In_sync, &rdev->flags) &&
1410                                             sync_page_io(rdev->bdev,
1411                                                          r10_bio->devs[sl].addr +
1412                                                          sect + rdev->data_offset,
1413                                                          s<<9,
1414                                                          conf->tmppage, READ))
1415                                                 success = 1;
1416                                         else {
1417                                                 sl++;
1418                                                 if (sl == conf->copies)
1419                                                         sl = 0;
1420                                         }
1421                                 } while (!success && sl != r10_bio->read_slot);
1422
1423                                 if (success) {
1424                                         /* write it back and re-read */
1425                                         while (sl != r10_bio->read_slot) {
1426                                                 int d;
1427                                                 if (sl==0)
1428                                                         sl = conf->copies;
1429                                                 sl--;
1430                                                 d = r10_bio->devs[sl].devnum;
1431                                                 rdev = conf->mirrors[d].rdev;
1432                                                 if (rdev &&
1433                                                     test_bit(In_sync, &rdev->flags)) {
1434                                                         if (sync_page_io(rdev->bdev,
1435                                                                          r10_bio->devs[sl].addr +
1436                                                                          sect + rdev->data_offset,
1437                                                                          s<<9, conf->tmppage, WRITE) == 0 ||
1438                                                             sync_page_io(rdev->bdev,
1439                                                                          r10_bio->devs[sl].addr +
1440                                                                          sect + rdev->data_offset,
1441                                                                          s<<9, conf->tmppage, READ) == 0) {
1442                                                                 /* Well, this device is dead */
1443                                                                 md_error(mddev, rdev);
1444                                                         }
1445                                                 }
1446                                         }
1447                                 } else {
1448                                         /* Cannot read from anywhere -- bye bye array */
1449                                         md_error(mddev, conf->mirrors[r10_bio->devs[r10_bio->read_slot].devnum].rdev);
1450                                         break;
1451                                 }
1452                                 sectors -= s;
1453                                 sect += s;
1454                         }
1455
1456                         unfreeze_array(conf);
1457
1458                         bio = r10_bio->devs[r10_bio->read_slot].bio;
1459                         r10_bio->devs[r10_bio->read_slot].bio =
1460                                 mddev->ro ? IO_BLOCKED : NULL;
1461                         bio_put(bio);
1462                         mirror = read_balance(conf, r10_bio);
1463                         if (mirror == -1) {
1464                                 printk(KERN_ALERT "raid10: %s: unrecoverable I/O"
1465                                        " read error for block %llu\n",
1466                                        bdevname(bio->bi_bdev,b),
1467                                        (unsigned long long)r10_bio->sector);
1468                                 raid_end_bio_io(r10_bio);
1469                         } else {
1470                                 rdev = conf->mirrors[mirror].rdev;
1471                                 if (printk_ratelimit())
1472                                         printk(KERN_ERR "raid10: %s: redirecting sector %llu to"
1473                                                " another mirror\n",
1474                                                bdevname(rdev->bdev,b),
1475                                                (unsigned long long)r10_bio->sector);
1476                                 bio = bio_clone(r10_bio->master_bio, GFP_NOIO);
1477                                 r10_bio->devs[r10_bio->read_slot].bio = bio;
1478                                 bio->bi_sector = r10_bio->devs[r10_bio->read_slot].addr
1479                                         + rdev->data_offset;
1480                                 bio->bi_bdev = rdev->bdev;
1481                                 bio->bi_rw = READ;
1482                                 bio->bi_private = r10_bio;
1483                                 bio->bi_end_io = raid10_end_read_request;
1484                                 unplug = 1;
1485                                 generic_make_request(bio);
1486                         }
1487                 }
1488         }
1489         spin_unlock_irqrestore(&conf->device_lock, flags);
1490         if (unplug)
1491                 unplug_slaves(mddev);
1492 }
1493
1494
1495 static int init_resync(conf_t *conf)
1496 {
1497         int buffs;
1498
1499         buffs = RESYNC_WINDOW / RESYNC_BLOCK_SIZE;
1500         if (conf->r10buf_pool)
1501                 BUG();
1502         conf->r10buf_pool = mempool_create(buffs, r10buf_pool_alloc, r10buf_pool_free, conf);
1503         if (!conf->r10buf_pool)
1504                 return -ENOMEM;
1505         conf->next_resync = 0;
1506         return 0;
1507 }
1508
1509 /*
1510  * perform a "sync" on one "block"
1511  *
1512  * We need to make sure that no normal I/O request - particularly write
1513  * requests - conflict with active sync requests.
1514  *
1515  * This is achieved by tracking pending requests and a 'barrier' concept
1516  * that can be installed to exclude normal IO requests.
1517  *
1518  * Resync and recovery are handled very differently.
1519  * We differentiate by looking at MD_RECOVERY_SYNC in mddev->recovery.
1520  *
1521  * For resync, we iterate over virtual addresses, read all copies,
1522  * and update if there are differences.  If only one copy is live,
1523  * skip it.
1524  * For recovery, we iterate over physical addresses, read a good
1525  * value for each non-in_sync drive, and over-write.
1526  *
1527  * So, for recovery we may have several outstanding complex requests for a
1528  * given address, one for each out-of-sync device.  We model this by allocating
1529  * a number of r10_bio structures, one for each out-of-sync device.
1530  * As we setup these structures, we collect all bio's together into a list
1531  * which we then process collectively to add pages, and then process again
1532  * to pass to generic_make_request.
1533  *
1534  * The r10_bio structures are linked using a borrowed master_bio pointer.
1535  * This link is counted in ->remaining.  When the r10_bio that points to NULL
1536  * has its remaining count decremented to 0, the whole complex operation
1537  * is complete.
1538  *
1539  */
1540
1541 static sector_t sync_request(mddev_t *mddev, sector_t sector_nr, int *skipped, int go_faster)
1542 {
1543         conf_t *conf = mddev_to_conf(mddev);
1544         r10bio_t *r10_bio;
1545         struct bio *biolist = NULL, *bio;
1546         sector_t max_sector, nr_sectors;
1547         int disk;
1548         int i;
1549         int max_sync;
1550         int sync_blocks;
1551
1552         sector_t sectors_skipped = 0;
1553         int chunks_skipped = 0;
1554
1555         if (!conf->r10buf_pool)
1556                 if (init_resync(conf))
1557                         return 0;
1558
1559  skipped:
1560         max_sector = mddev->size << 1;
1561         if (test_bit(MD_RECOVERY_SYNC, &mddev->recovery))
1562                 max_sector = mddev->resync_max_sectors;
1563         if (sector_nr >= max_sector) {
1564                 /* If we aborted, we need to abort the
1565                  * sync on the 'current' bitmap chucks (there can
1566                  * be several when recovering multiple devices).
1567                  * as we may have started syncing it but not finished.
1568                  * We can find the current address in
1569                  * mddev->curr_resync, but for recovery,
1570                  * we need to convert that to several
1571                  * virtual addresses.
1572                  */
1573                 if (mddev->curr_resync < max_sector) { /* aborted */
1574                         if (test_bit(MD_RECOVERY_SYNC, &mddev->recovery))
1575                                 bitmap_end_sync(mddev->bitmap, mddev->curr_resync,
1576                                                 &sync_blocks, 1);
1577                         else for (i=0; i<conf->raid_disks; i++) {
1578                                 sector_t sect =
1579                                         raid10_find_virt(conf, mddev->curr_resync, i);
1580                                 bitmap_end_sync(mddev->bitmap, sect,
1581                                                 &sync_blocks, 1);
1582                         }
1583                 } else /* completed sync */
1584                         conf->fullsync = 0;
1585
1586                 bitmap_close_sync(mddev->bitmap);
1587                 close_sync(conf);
1588                 *skipped = 1;
1589                 return sectors_skipped;
1590         }
1591         if (chunks_skipped >= conf->raid_disks) {
1592                 /* if there has been nothing to do on any drive,
1593                  * then there is nothing to do at all..
1594                  */
1595                 *skipped = 1;
1596                 return (max_sector - sector_nr) + sectors_skipped;
1597         }
1598
1599         /* make sure whole request will fit in a chunk - if chunks
1600          * are meaningful
1601          */
1602         if (conf->near_copies < conf->raid_disks &&
1603             max_sector > (sector_nr | conf->chunk_mask))
1604                 max_sector = (sector_nr | conf->chunk_mask) + 1;
1605         /*
1606          * If there is non-resync activity waiting for us then
1607          * put in a delay to throttle resync.
1608          */
1609         if (!go_faster && conf->nr_waiting)
1610                 msleep_interruptible(1000);
1611
1612         /* Again, very different code for resync and recovery.
1613          * Both must result in an r10bio with a list of bios that
1614          * have bi_end_io, bi_sector, bi_bdev set,
1615          * and bi_private set to the r10bio.
1616          * For recovery, we may actually create several r10bios
1617          * with 2 bios in each, that correspond to the bios in the main one.
1618          * In this case, the subordinate r10bios link back through a
1619          * borrowed master_bio pointer, and the counter in the master
1620          * includes a ref from each subordinate.
1621          */
1622         /* First, we decide what to do and set ->bi_end_io
1623          * To end_sync_read if we want to read, and
1624          * end_sync_write if we will want to write.
1625          */
1626
1627         max_sync = RESYNC_PAGES << (PAGE_SHIFT-9);
1628         if (!test_bit(MD_RECOVERY_SYNC, &mddev->recovery)) {
1629                 /* recovery... the complicated one */
1630                 int i, j, k;
1631                 r10_bio = NULL;
1632
1633                 for (i=0 ; i<conf->raid_disks; i++)
1634                         if (conf->mirrors[i].rdev &&
1635                             !test_bit(In_sync, &conf->mirrors[i].rdev->flags)) {
1636                                 int still_degraded = 0;
1637                                 /* want to reconstruct this device */
1638                                 r10bio_t *rb2 = r10_bio;
1639                                 sector_t sect = raid10_find_virt(conf, sector_nr, i);
1640                                 int must_sync;
1641                                 /* Unless we are doing a full sync, we only need
1642                                  * to recover the block if it is set in the bitmap
1643                                  */
1644                                 must_sync = bitmap_start_sync(mddev->bitmap, sect,
1645                                                               &sync_blocks, 1);
1646                                 if (sync_blocks < max_sync)
1647                                         max_sync = sync_blocks;
1648                                 if (!must_sync &&
1649                                     !conf->fullsync) {
1650                                         /* yep, skip the sync_blocks here, but don't assume
1651                                          * that there will never be anything to do here
1652                                          */
1653                                         chunks_skipped = -1;
1654                                         continue;
1655                                 }
1656
1657                                 r10_bio = mempool_alloc(conf->r10buf_pool, GFP_NOIO);
1658                                 raise_barrier(conf, rb2 != NULL);
1659                                 atomic_set(&r10_bio->remaining, 0);
1660
1661                                 r10_bio->master_bio = (struct bio*)rb2;
1662                                 if (rb2)
1663                                         atomic_inc(&rb2->remaining);
1664                                 r10_bio->mddev = mddev;
1665                                 set_bit(R10BIO_IsRecover, &r10_bio->state);
1666                                 r10_bio->sector = sect;
1667
1668                                 raid10_find_phys(conf, r10_bio);
1669                                 /* Need to check if this section will still be
1670                                  * degraded
1671                                  */
1672                                 for (j=0; j<conf->copies;j++) {
1673                                         int d = r10_bio->devs[j].devnum;
1674                                         if (conf->mirrors[d].rdev == NULL ||
1675                                             test_bit(Faulty, &conf->mirrors[d].rdev->flags)) {
1676                                                 still_degraded = 1;
1677                                                 break;
1678                                         }
1679                                 }
1680                                 must_sync = bitmap_start_sync(mddev->bitmap, sect,
1681                                                               &sync_blocks, still_degraded);
1682
1683                                 for (j=0; j<conf->copies;j++) {
1684                                         int d = r10_bio->devs[j].devnum;
1685                                         if (conf->mirrors[d].rdev &&
1686                                             test_bit(In_sync, &conf->mirrors[d].rdev->flags)) {
1687                                                 /* This is where we read from */
1688                                                 bio = r10_bio->devs[0].bio;
1689                                                 bio->bi_next = biolist;
1690                                                 biolist = bio;
1691                                                 bio->bi_private = r10_bio;
1692                                                 bio->bi_end_io = end_sync_read;
1693                                                 bio->bi_rw = 0;
1694                                                 bio->bi_sector = r10_bio->devs[j].addr +
1695                                                         conf->mirrors[d].rdev->data_offset;
1696                                                 bio->bi_bdev = conf->mirrors[d].rdev->bdev;
1697                                                 atomic_inc(&conf->mirrors[d].rdev->nr_pending);
1698                                                 atomic_inc(&r10_bio->remaining);
1699                                                 /* and we write to 'i' */
1700
1701                                                 for (k=0; k<conf->copies; k++)
1702                                                         if (r10_bio->devs[k].devnum == i)
1703                                                                 break;
1704                                                 bio = r10_bio->devs[1].bio;
1705                                                 bio->bi_next = biolist;
1706                                                 biolist = bio;
1707                                                 bio->bi_private = r10_bio;
1708                                                 bio->bi_end_io = end_sync_write;
1709                                                 bio->bi_rw = 1;
1710                                                 bio->bi_sector = r10_bio->devs[k].addr +
1711                                                         conf->mirrors[i].rdev->data_offset;
1712                                                 bio->bi_bdev = conf->mirrors[i].rdev->bdev;
1713
1714                                                 r10_bio->devs[0].devnum = d;
1715                                                 r10_bio->devs[1].devnum = i;
1716
1717                                                 break;
1718                                         }
1719                                 }
1720                                 if (j == conf->copies) {
1721                                         /* Cannot recover, so abort the recovery */
1722                                         put_buf(r10_bio);
1723                                         r10_bio = rb2;
1724                                         if (!test_and_set_bit(MD_RECOVERY_ERR, &mddev->recovery))
1725                                                 printk(KERN_INFO "raid10: %s: insufficient working devices for recovery.\n",
1726                                                        mdname(mddev));
1727                                         break;
1728                                 }
1729                         }
1730                 if (biolist == NULL) {
1731                         while (r10_bio) {
1732                                 r10bio_t *rb2 = r10_bio;
1733                                 r10_bio = (r10bio_t*) rb2->master_bio;
1734                                 rb2->master_bio = NULL;
1735                                 put_buf(rb2);
1736                         }
1737                         goto giveup;
1738                 }
1739         } else {
1740                 /* resync. Schedule a read for every block at this virt offset */
1741                 int count = 0;
1742
1743                 if (!bitmap_start_sync(mddev->bitmap, sector_nr,
1744                                        &sync_blocks, mddev->degraded) &&
1745                     !conf->fullsync && !test_bit(MD_RECOVERY_REQUESTED, &mddev->recovery)) {
1746                         /* We can skip this block */
1747                         *skipped = 1;
1748                         return sync_blocks + sectors_skipped;
1749                 }
1750                 if (sync_blocks < max_sync)
1751                         max_sync = sync_blocks;
1752                 r10_bio = mempool_alloc(conf->r10buf_pool, GFP_NOIO);
1753
1754                 r10_bio->mddev = mddev;
1755                 atomic_set(&r10_bio->remaining, 0);
1756                 raise_barrier(conf, 0);
1757                 conf->next_resync = sector_nr;
1758
1759                 r10_bio->master_bio = NULL;
1760                 r10_bio->sector = sector_nr;
1761                 set_bit(R10BIO_IsSync, &r10_bio->state);
1762                 raid10_find_phys(conf, r10_bio);
1763                 r10_bio->sectors = (sector_nr | conf->chunk_mask) - sector_nr +1;
1764
1765                 for (i=0; i<conf->copies; i++) {
1766                         int d = r10_bio->devs[i].devnum;
1767                         bio = r10_bio->devs[i].bio;
1768                         bio->bi_end_io = NULL;
1769                         if (conf->mirrors[d].rdev == NULL ||
1770                             test_bit(Faulty, &conf->mirrors[d].rdev->flags))
1771                                 continue;
1772                         atomic_inc(&conf->mirrors[d].rdev->nr_pending);
1773                         atomic_inc(&r10_bio->remaining);
1774                         bio->bi_next = biolist;
1775                         biolist = bio;
1776                         bio->bi_private = r10_bio;
1777                         bio->bi_end_io = end_sync_read;
1778                         bio->bi_rw = 0;
1779                         bio->bi_sector = r10_bio->devs[i].addr +
1780                                 conf->mirrors[d].rdev->data_offset;
1781                         bio->bi_bdev = conf->mirrors[d].rdev->bdev;
1782                         count++;
1783                 }
1784
1785                 if (count < 2) {
1786                         for (i=0; i<conf->copies; i++) {
1787                                 int d = r10_bio->devs[i].devnum;
1788                                 if (r10_bio->devs[i].bio->bi_end_io)
1789                                         rdev_dec_pending(conf->mirrors[d].rdev, mddev);
1790                         }
1791                         put_buf(r10_bio);
1792                         biolist = NULL;
1793                         goto giveup;
1794                 }
1795         }
1796
1797         for (bio = biolist; bio ; bio=bio->bi_next) {
1798
1799                 bio->bi_flags &= ~(BIO_POOL_MASK - 1);
1800                 if (bio->bi_end_io)
1801                         bio->bi_flags |= 1 << BIO_UPTODATE;
1802                 bio->bi_vcnt = 0;
1803                 bio->bi_idx = 0;
1804                 bio->bi_phys_segments = 0;
1805                 bio->bi_hw_segments = 0;
1806                 bio->bi_size = 0;
1807         }
1808
1809         nr_sectors = 0;
1810         if (sector_nr + max_sync < max_sector)
1811                 max_sector = sector_nr + max_sync;
1812         do {
1813                 struct page *page;
1814                 int len = PAGE_SIZE;
1815                 disk = 0;
1816                 if (sector_nr + (len>>9) > max_sector)
1817                         len = (max_sector - sector_nr) << 9;
1818                 if (len == 0)
1819                         break;
1820                 for (bio= biolist ; bio ; bio=bio->bi_next) {
1821                         page = bio->bi_io_vec[bio->bi_vcnt].bv_page;
1822                         if (bio_add_page(bio, page, len, 0) == 0) {
1823                                 /* stop here */
1824                                 struct bio *bio2;
1825                                 bio->bi_io_vec[bio->bi_vcnt].bv_page = page;
1826                                 for (bio2 = biolist; bio2 && bio2 != bio; bio2 = bio2->bi_next) {
1827                                         /* remove last page from this bio */
1828                                         bio2->bi_vcnt--;
1829                                         bio2->bi_size -= len;
1830                                         bio2->bi_flags &= ~(1<< BIO_SEG_VALID);
1831                                 }
1832                                 goto bio_full;
1833                         }
1834                         disk = i;
1835                 }
1836                 nr_sectors += len>>9;
1837                 sector_nr += len>>9;
1838         } while (biolist->bi_vcnt < RESYNC_PAGES);
1839  bio_full:
1840         r10_bio->sectors = nr_sectors;
1841
1842         while (biolist) {
1843                 bio = biolist;
1844                 biolist = biolist->bi_next;
1845
1846                 bio->bi_next = NULL;
1847                 r10_bio = bio->bi_private;
1848                 r10_bio->sectors = nr_sectors;
1849
1850                 if (bio->bi_end_io == end_sync_read) {
1851                         md_sync_acct(bio->bi_bdev, nr_sectors);
1852                         generic_make_request(bio);
1853                 }
1854         }
1855
1856         if (sectors_skipped)
1857                 /* pretend they weren't skipped, it makes
1858                  * no important difference in this case
1859                  */
1860                 md_done_sync(mddev, sectors_skipped, 1);
1861
1862         return sectors_skipped + nr_sectors;
1863  giveup:
1864         /* There is nowhere to write, so all non-sync
1865          * drives must be failed, so try the next chunk...
1866          */
1867         {
1868         sector_t sec = max_sector - sector_nr;
1869         sectors_skipped += sec;
1870         chunks_skipped ++;
1871         sector_nr = max_sector;
1872         goto skipped;
1873         }
1874 }
1875
1876 static int run(mddev_t *mddev)
1877 {
1878         conf_t *conf;
1879         int i, disk_idx;
1880         mirror_info_t *disk;
1881         mdk_rdev_t *rdev;
1882         struct list_head *tmp;
1883         int nc, fc;
1884         sector_t stride, size;
1885
1886         if (mddev->level != 10) {
1887                 printk(KERN_ERR "raid10: %s: raid level not set correctly... (%d)\n",
1888                        mdname(mddev), mddev->level);
1889                 goto out;
1890         }
1891         nc = mddev->layout & 255;
1892         fc = (mddev->layout >> 8) & 255;
1893         if ((nc*fc) <2 || (nc*fc) > mddev->raid_disks ||
1894             (mddev->layout >> 16)) {
1895                 printk(KERN_ERR "raid10: %s: unsupported raid10 layout: 0x%8x\n",
1896                        mdname(mddev), mddev->layout);
1897                 goto out;
1898         }
1899         /*
1900          * copy the already verified devices into our private RAID10
1901          * bookkeeping area. [whatever we allocate in run(),
1902          * should be freed in stop()]
1903          */
1904         conf = kzalloc(sizeof(conf_t), GFP_KERNEL);
1905         mddev->private = conf;
1906         if (!conf) {
1907                 printk(KERN_ERR "raid10: couldn't allocate memory for %s\n",
1908                         mdname(mddev));
1909                 goto out;
1910         }
1911         conf->mirrors = kzalloc(sizeof(struct mirror_info)*mddev->raid_disks,
1912                                  GFP_KERNEL);
1913         if (!conf->mirrors) {
1914                 printk(KERN_ERR "raid10: couldn't allocate memory for %s\n",
1915                        mdname(mddev));
1916                 goto out_free_conf;
1917         }
1918
1919         conf->tmppage = alloc_page(GFP_KERNEL);
1920         if (!conf->tmppage)
1921                 goto out_free_conf;
1922
1923         conf->near_copies = nc;
1924         conf->far_copies = fc;
1925         conf->copies = nc*fc;
1926         conf->chunk_mask = (sector_t)(mddev->chunk_size>>9)-1;
1927         conf->chunk_shift = ffz(~mddev->chunk_size) - 9;
1928         stride = mddev->size >> (conf->chunk_shift-1);
1929         sector_div(stride, fc);
1930         conf->stride = stride << conf->chunk_shift;
1931
1932         conf->r10bio_pool = mempool_create(NR_RAID10_BIOS, r10bio_pool_alloc,
1933                                                 r10bio_pool_free, conf);
1934         if (!conf->r10bio_pool) {
1935                 printk(KERN_ERR "raid10: couldn't allocate memory for %s\n",
1936                         mdname(mddev));
1937                 goto out_free_conf;
1938         }
1939
1940         ITERATE_RDEV(mddev, rdev, tmp) {
1941                 disk_idx = rdev->raid_disk;
1942                 if (disk_idx >= mddev->raid_disks
1943                     || disk_idx < 0)
1944                         continue;
1945                 disk = conf->mirrors + disk_idx;
1946
1947                 disk->rdev = rdev;
1948
1949                 blk_queue_stack_limits(mddev->queue,
1950                                        rdev->bdev->bd_disk->queue);
1951                 /* as we don't honour merge_bvec_fn, we must never risk
1952                  * violating it, so limit ->max_sector to one PAGE, as
1953                  * a one page request is never in violation.
1954                  */
1955                 if (rdev->bdev->bd_disk->queue->merge_bvec_fn &&
1956                     mddev->queue->max_sectors > (PAGE_SIZE>>9))
1957                         mddev->queue->max_sectors = (PAGE_SIZE>>9);
1958
1959                 disk->head_position = 0;
1960                 if (!test_bit(Faulty, &rdev->flags) && test_bit(In_sync, &rdev->flags))
1961                         conf->working_disks++;
1962         }
1963         conf->raid_disks = mddev->raid_disks;
1964         conf->mddev = mddev;
1965         spin_lock_init(&conf->device_lock);
1966         INIT_LIST_HEAD(&conf->retry_list);
1967
1968         spin_lock_init(&conf->resync_lock);
1969         init_waitqueue_head(&conf->wait_barrier);
1970
1971         /* need to check that every block has at least one working mirror */
1972         if (!enough(conf)) {
1973                 printk(KERN_ERR "raid10: not enough operational mirrors for %s\n",
1974                        mdname(mddev));
1975                 goto out_free_conf;
1976         }
1977
1978         mddev->degraded = 0;
1979         for (i = 0; i < conf->raid_disks; i++) {
1980
1981                 disk = conf->mirrors + i;
1982
1983                 if (!disk->rdev) {
1984                         disk->head_position = 0;
1985                         mddev->degraded++;
1986                 }
1987         }
1988
1989
1990         mddev->thread = md_register_thread(raid10d, mddev, "%s_raid10");
1991         if (!mddev->thread) {
1992                 printk(KERN_ERR
1993                        "raid10: couldn't allocate thread for %s\n",
1994                        mdname(mddev));
1995                 goto out_free_conf;
1996         }
1997
1998         printk(KERN_INFO
1999                 "raid10: raid set %s active with %d out of %d devices\n",
2000                 mdname(mddev), mddev->raid_disks - mddev->degraded,
2001                 mddev->raid_disks);
2002         /*
2003          * Ok, everything is just fine now
2004          */
2005         size = conf->stride * conf->raid_disks;
2006         sector_div(size, conf->near_copies);
2007         mddev->array_size = size/2;
2008         mddev->resync_max_sectors = size;
2009
2010         mddev->queue->unplug_fn = raid10_unplug;
2011         mddev->queue->issue_flush_fn = raid10_issue_flush;
2012
2013         /* Calculate max read-ahead size.
2014          * We need to readahead at least twice a whole stripe....
2015          * maybe...
2016          */
2017         {
2018                 int stripe = conf->raid_disks * mddev->chunk_size / PAGE_SIZE;
2019                 stripe /= conf->near_copies;
2020                 if (mddev->queue->backing_dev_info.ra_pages < 2* stripe)
2021                         mddev->queue->backing_dev_info.ra_pages = 2* stripe;
2022         }
2023
2024         if (conf->near_copies < mddev->raid_disks)
2025                 blk_queue_merge_bvec(mddev->queue, raid10_mergeable_bvec);
2026         return 0;
2027
2028 out_free_conf:
2029         if (conf->r10bio_pool)
2030                 mempool_destroy(conf->r10bio_pool);
2031         put_page(conf->tmppage);
2032         kfree(conf->mirrors);
2033         kfree(conf);
2034         mddev->private = NULL;
2035 out:
2036         return -EIO;
2037 }
2038
2039 static int stop(mddev_t *mddev)
2040 {
2041         conf_t *conf = mddev_to_conf(mddev);
2042
2043         md_unregister_thread(mddev->thread);
2044         mddev->thread = NULL;
2045         blk_sync_queue(mddev->queue); /* the unplug fn references 'conf'*/
2046         if (conf->r10bio_pool)
2047                 mempool_destroy(conf->r10bio_pool);
2048         kfree(conf->mirrors);
2049         kfree(conf);
2050         mddev->private = NULL;
2051         return 0;
2052 }
2053
2054 static void raid10_quiesce(mddev_t *mddev, int state)
2055 {
2056         conf_t *conf = mddev_to_conf(mddev);
2057
2058         switch(state) {
2059         case 1:
2060                 raise_barrier(conf, 0);
2061                 break;
2062         case 0:
2063                 lower_barrier(conf);
2064                 break;
2065         }
2066         if (mddev->thread) {
2067                 if (mddev->bitmap)
2068                         mddev->thread->timeout = mddev->bitmap->daemon_sleep * HZ;
2069                 else
2070                         mddev->thread->timeout = MAX_SCHEDULE_TIMEOUT;
2071                 md_wakeup_thread(mddev->thread);
2072         }
2073 }
2074
2075 static mdk_personality_t raid10_personality =
2076 {
2077         .name           = "raid10",
2078         .owner          = THIS_MODULE,
2079         .make_request   = make_request,
2080         .run            = run,
2081         .stop           = stop,
2082         .status         = status,
2083         .error_handler  = error,
2084         .hot_add_disk   = raid10_add_disk,
2085         .hot_remove_disk= raid10_remove_disk,
2086         .spare_active   = raid10_spare_active,
2087         .sync_request   = sync_request,
2088         .quiesce        = raid10_quiesce,
2089 };
2090
2091 static int __init raid_init(void)
2092 {
2093         return register_md_personality(RAID10, &raid10_personality);
2094 }
2095
2096 static void raid_exit(void)
2097 {
2098         unregister_md_personality(RAID10);
2099 }
2100
2101 module_init(raid_init);
2102 module_exit(raid_exit);
2103 MODULE_LICENSE("GPL");
2104 MODULE_ALIAS("md-personality-9"); /* RAID10 */