xref: /openbmc/linux/drivers/md/raid10.c (revision e868d61272caa648214046a096e5a6bfc068dc8c)
1 /*
2  * raid10.c : Multiple Devices driver for Linux
3  *
4  * Copyright (C) 2000-2004 Neil Brown
5  *
6  * RAID-10 support for md.
7  *
8  * Base on code in raid1.c.  See raid1.c for futher copyright information.
9  *
10  *
11  * This program is free software; you can redistribute it and/or modify
12  * it under the terms of the GNU General Public License as published by
13  * the Free Software Foundation; either version 2, or (at your option)
14  * any later version.
15  *
16  * You should have received a copy of the GNU General Public License
17  * (for example /usr/src/linux/COPYING); if not, write to the Free
18  * Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
19  */
20 
21 #include "dm-bio-list.h"
22 #include <linux/raid/raid10.h>
23 #include <linux/raid/bitmap.h>
24 
25 /*
26  * RAID10 provides a combination of RAID0 and RAID1 functionality.
27  * The layout of data is defined by
28  *    chunk_size
29  *    raid_disks
30  *    near_copies (stored in low byte of layout)
31  *    far_copies (stored in second byte of layout)
32  *    far_offset (stored in bit 16 of layout )
33  *
34  * The data to be stored is divided into chunks using chunksize.
35  * Each device is divided into far_copies sections.
36  * In each section, chunks are laid out in a style similar to raid0, but
37  * near_copies copies of each chunk is stored (each on a different drive).
38  * The starting device for each section is offset near_copies from the starting
39  * device of the previous section.
40  * Thus they are (near_copies*far_copies) of each chunk, and each is on a different
41  * drive.
42  * near_copies and far_copies must be at least one, and their product is at most
43  * raid_disks.
44  *
45  * If far_offset is true, then the far_copies are handled a bit differently.
46  * The copies are still in different stripes, but instead of be very far apart
47  * on disk, there are adjacent stripes.
48  */
49 
50 /*
51  * Number of guaranteed r10bios in case of extreme VM load:
52  */
53 #define	NR_RAID10_BIOS 256
54 
55 static void unplug_slaves(mddev_t *mddev);
56 
57 static void allow_barrier(conf_t *conf);
58 static void lower_barrier(conf_t *conf);
59 
60 static void * r10bio_pool_alloc(gfp_t gfp_flags, void *data)
61 {
62 	conf_t *conf = data;
63 	r10bio_t *r10_bio;
64 	int size = offsetof(struct r10bio_s, devs[conf->copies]);
65 
66 	/* allocate a r10bio with room for raid_disks entries in the bios array */
67 	r10_bio = kzalloc(size, gfp_flags);
68 	if (!r10_bio)
69 		unplug_slaves(conf->mddev);
70 
71 	return r10_bio;
72 }
73 
74 static void r10bio_pool_free(void *r10_bio, void *data)
75 {
76 	kfree(r10_bio);
77 }
78 
79 #define RESYNC_BLOCK_SIZE (64*1024)
80 //#define RESYNC_BLOCK_SIZE PAGE_SIZE
81 #define RESYNC_SECTORS (RESYNC_BLOCK_SIZE >> 9)
82 #define RESYNC_PAGES ((RESYNC_BLOCK_SIZE + PAGE_SIZE-1) / PAGE_SIZE)
83 #define RESYNC_WINDOW (2048*1024)
84 
85 /*
86  * When performing a resync, we need to read and compare, so
87  * we need as many pages are there are copies.
88  * When performing a recovery, we need 2 bios, one for read,
89  * one for write (we recover only one drive per r10buf)
90  *
91  */
92 static void * r10buf_pool_alloc(gfp_t gfp_flags, void *data)
93 {
94 	conf_t *conf = data;
95 	struct page *page;
96 	r10bio_t *r10_bio;
97 	struct bio *bio;
98 	int i, j;
99 	int nalloc;
100 
101 	r10_bio = r10bio_pool_alloc(gfp_flags, conf);
102 	if (!r10_bio) {
103 		unplug_slaves(conf->mddev);
104 		return NULL;
105 	}
106 
107 	if (test_bit(MD_RECOVERY_SYNC, &conf->mddev->recovery))
108 		nalloc = conf->copies; /* resync */
109 	else
110 		nalloc = 2; /* recovery */
111 
112 	/*
113 	 * Allocate bios.
114 	 */
115 	for (j = nalloc ; j-- ; ) {
116 		bio = bio_alloc(gfp_flags, RESYNC_PAGES);
117 		if (!bio)
118 			goto out_free_bio;
119 		r10_bio->devs[j].bio = bio;
120 	}
121 	/*
122 	 * Allocate RESYNC_PAGES data pages and attach them
123 	 * where needed.
124 	 */
125 	for (j = 0 ; j < nalloc; j++) {
126 		bio = r10_bio->devs[j].bio;
127 		for (i = 0; i < RESYNC_PAGES; i++) {
128 			page = alloc_page(gfp_flags);
129 			if (unlikely(!page))
130 				goto out_free_pages;
131 
132 			bio->bi_io_vec[i].bv_page = page;
133 		}
134 	}
135 
136 	return r10_bio;
137 
138 out_free_pages:
139 	for ( ; i > 0 ; i--)
140 		safe_put_page(bio->bi_io_vec[i-1].bv_page);
141 	while (j--)
142 		for (i = 0; i < RESYNC_PAGES ; i++)
143 			safe_put_page(r10_bio->devs[j].bio->bi_io_vec[i].bv_page);
144 	j = -1;
145 out_free_bio:
146 	while ( ++j < nalloc )
147 		bio_put(r10_bio->devs[j].bio);
148 	r10bio_pool_free(r10_bio, conf);
149 	return NULL;
150 }
151 
152 static void r10buf_pool_free(void *__r10_bio, void *data)
153 {
154 	int i;
155 	conf_t *conf = data;
156 	r10bio_t *r10bio = __r10_bio;
157 	int j;
158 
159 	for (j=0; j < conf->copies; j++) {
160 		struct bio *bio = r10bio->devs[j].bio;
161 		if (bio) {
162 			for (i = 0; i < RESYNC_PAGES; i++) {
163 				safe_put_page(bio->bi_io_vec[i].bv_page);
164 				bio->bi_io_vec[i].bv_page = NULL;
165 			}
166 			bio_put(bio);
167 		}
168 	}
169 	r10bio_pool_free(r10bio, conf);
170 }
171 
172 static void put_all_bios(conf_t *conf, r10bio_t *r10_bio)
173 {
174 	int i;
175 
176 	for (i = 0; i < conf->copies; i++) {
177 		struct bio **bio = & r10_bio->devs[i].bio;
178 		if (*bio && *bio != IO_BLOCKED)
179 			bio_put(*bio);
180 		*bio = NULL;
181 	}
182 }
183 
184 static void free_r10bio(r10bio_t *r10_bio)
185 {
186 	conf_t *conf = mddev_to_conf(r10_bio->mddev);
187 
188 	/*
189 	 * Wake up any possible resync thread that waits for the device
190 	 * to go idle.
191 	 */
192 	allow_barrier(conf);
193 
194 	put_all_bios(conf, r10_bio);
195 	mempool_free(r10_bio, conf->r10bio_pool);
196 }
197 
198 static void put_buf(r10bio_t *r10_bio)
199 {
200 	conf_t *conf = mddev_to_conf(r10_bio->mddev);
201 
202 	mempool_free(r10_bio, conf->r10buf_pool);
203 
204 	lower_barrier(conf);
205 }
206 
207 static void reschedule_retry(r10bio_t *r10_bio)
208 {
209 	unsigned long flags;
210 	mddev_t *mddev = r10_bio->mddev;
211 	conf_t *conf = mddev_to_conf(mddev);
212 
213 	spin_lock_irqsave(&conf->device_lock, flags);
214 	list_add(&r10_bio->retry_list, &conf->retry_list);
215 	conf->nr_queued ++;
216 	spin_unlock_irqrestore(&conf->device_lock, flags);
217 
218 	md_wakeup_thread(mddev->thread);
219 }
220 
221 /*
222  * raid_end_bio_io() is called when we have finished servicing a mirrored
223  * operation and are ready to return a success/failure code to the buffer
224  * cache layer.
225  */
226 static void raid_end_bio_io(r10bio_t *r10_bio)
227 {
228 	struct bio *bio = r10_bio->master_bio;
229 
230 	bio_endio(bio, bio->bi_size,
231 		test_bit(R10BIO_Uptodate, &r10_bio->state) ? 0 : -EIO);
232 	free_r10bio(r10_bio);
233 }
234 
235 /*
236  * Update disk head position estimator based on IRQ completion info.
237  */
238 static inline void update_head_pos(int slot, r10bio_t *r10_bio)
239 {
240 	conf_t *conf = mddev_to_conf(r10_bio->mddev);
241 
242 	conf->mirrors[r10_bio->devs[slot].devnum].head_position =
243 		r10_bio->devs[slot].addr + (r10_bio->sectors);
244 }
245 
246 static int raid10_end_read_request(struct bio *bio, unsigned int bytes_done, int error)
247 {
248 	int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags);
249 	r10bio_t * r10_bio = (r10bio_t *)(bio->bi_private);
250 	int slot, dev;
251 	conf_t *conf = mddev_to_conf(r10_bio->mddev);
252 
253 	if (bio->bi_size)
254 		return 1;
255 
256 	slot = r10_bio->read_slot;
257 	dev = r10_bio->devs[slot].devnum;
258 	/*
259 	 * this branch is our 'one mirror IO has finished' event handler:
260 	 */
261 	update_head_pos(slot, r10_bio);
262 
263 	if (uptodate) {
264 		/*
265 		 * Set R10BIO_Uptodate in our master bio, so that
266 		 * we will return a good error code to the higher
267 		 * levels even if IO on some other mirrored buffer fails.
268 		 *
269 		 * The 'master' represents the composite IO operation to
270 		 * user-side. So if something waits for IO, then it will
271 		 * wait for the 'master' bio.
272 		 */
273 		set_bit(R10BIO_Uptodate, &r10_bio->state);
274 		raid_end_bio_io(r10_bio);
275 	} else {
276 		/*
277 		 * oops, read error:
278 		 */
279 		char b[BDEVNAME_SIZE];
280 		if (printk_ratelimit())
281 			printk(KERN_ERR "raid10: %s: rescheduling sector %llu\n",
282 			       bdevname(conf->mirrors[dev].rdev->bdev,b), (unsigned long long)r10_bio->sector);
283 		reschedule_retry(r10_bio);
284 	}
285 
286 	rdev_dec_pending(conf->mirrors[dev].rdev, conf->mddev);
287 	return 0;
288 }
289 
290 static int raid10_end_write_request(struct bio *bio, unsigned int bytes_done, int error)
291 {
292 	int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags);
293 	r10bio_t * r10_bio = (r10bio_t *)(bio->bi_private);
294 	int slot, dev;
295 	conf_t *conf = mddev_to_conf(r10_bio->mddev);
296 
297 	if (bio->bi_size)
298 		return 1;
299 
300 	for (slot = 0; slot < conf->copies; slot++)
301 		if (r10_bio->devs[slot].bio == bio)
302 			break;
303 	dev = r10_bio->devs[slot].devnum;
304 
305 	/*
306 	 * this branch is our 'one mirror IO has finished' event handler:
307 	 */
308 	if (!uptodate) {
309 		md_error(r10_bio->mddev, conf->mirrors[dev].rdev);
310 		/* an I/O failed, we can't clear the bitmap */
311 		set_bit(R10BIO_Degraded, &r10_bio->state);
312 	} else
313 		/*
314 		 * Set R10BIO_Uptodate in our master bio, so that
315 		 * we will return a good error code for to the higher
316 		 * levels even if IO on some other mirrored buffer fails.
317 		 *
318 		 * The 'master' represents the composite IO operation to
319 		 * user-side. So if something waits for IO, then it will
320 		 * wait for the 'master' bio.
321 		 */
322 		set_bit(R10BIO_Uptodate, &r10_bio->state);
323 
324 	update_head_pos(slot, r10_bio);
325 
326 	/*
327 	 *
328 	 * Let's see if all mirrored write operations have finished
329 	 * already.
330 	 */
331 	if (atomic_dec_and_test(&r10_bio->remaining)) {
332 		/* clear the bitmap if all writes complete successfully */
333 		bitmap_endwrite(r10_bio->mddev->bitmap, r10_bio->sector,
334 				r10_bio->sectors,
335 				!test_bit(R10BIO_Degraded, &r10_bio->state),
336 				0);
337 		md_write_end(r10_bio->mddev);
338 		raid_end_bio_io(r10_bio);
339 	}
340 
341 	rdev_dec_pending(conf->mirrors[dev].rdev, conf->mddev);
342 	return 0;
343 }
344 
345 
346 /*
347  * RAID10 layout manager
348  * Aswell as the chunksize and raid_disks count, there are two
349  * parameters: near_copies and far_copies.
350  * near_copies * far_copies must be <= raid_disks.
351  * Normally one of these will be 1.
352  * If both are 1, we get raid0.
353  * If near_copies == raid_disks, we get raid1.
354  *
355  * Chunks are layed out in raid0 style with near_copies copies of the
356  * first chunk, followed by near_copies copies of the next chunk and
357  * so on.
358  * If far_copies > 1, then after 1/far_copies of the array has been assigned
359  * as described above, we start again with a device offset of near_copies.
360  * So we effectively have another copy of the whole array further down all
361  * the drives, but with blocks on different drives.
362  * With this layout, and block is never stored twice on the one device.
363  *
364  * raid10_find_phys finds the sector offset of a given virtual sector
365  * on each device that it is on.
366  *
367  * raid10_find_virt does the reverse mapping, from a device and a
368  * sector offset to a virtual address
369  */
370 
371 static void raid10_find_phys(conf_t *conf, r10bio_t *r10bio)
372 {
373 	int n,f;
374 	sector_t sector;
375 	sector_t chunk;
376 	sector_t stripe;
377 	int dev;
378 
379 	int slot = 0;
380 
381 	/* now calculate first sector/dev */
382 	chunk = r10bio->sector >> conf->chunk_shift;
383 	sector = r10bio->sector & conf->chunk_mask;
384 
385 	chunk *= conf->near_copies;
386 	stripe = chunk;
387 	dev = sector_div(stripe, conf->raid_disks);
388 	if (conf->far_offset)
389 		stripe *= conf->far_copies;
390 
391 	sector += stripe << conf->chunk_shift;
392 
393 	/* and calculate all the others */
394 	for (n=0; n < conf->near_copies; n++) {
395 		int d = dev;
396 		sector_t s = sector;
397 		r10bio->devs[slot].addr = sector;
398 		r10bio->devs[slot].devnum = d;
399 		slot++;
400 
401 		for (f = 1; f < conf->far_copies; f++) {
402 			d += conf->near_copies;
403 			if (d >= conf->raid_disks)
404 				d -= conf->raid_disks;
405 			s += conf->stride;
406 			r10bio->devs[slot].devnum = d;
407 			r10bio->devs[slot].addr = s;
408 			slot++;
409 		}
410 		dev++;
411 		if (dev >= conf->raid_disks) {
412 			dev = 0;
413 			sector += (conf->chunk_mask + 1);
414 		}
415 	}
416 	BUG_ON(slot != conf->copies);
417 }
418 
419 static sector_t raid10_find_virt(conf_t *conf, sector_t sector, int dev)
420 {
421 	sector_t offset, chunk, vchunk;
422 
423 	offset = sector & conf->chunk_mask;
424 	if (conf->far_offset) {
425 		int fc;
426 		chunk = sector >> conf->chunk_shift;
427 		fc = sector_div(chunk, conf->far_copies);
428 		dev -= fc * conf->near_copies;
429 		if (dev < 0)
430 			dev += conf->raid_disks;
431 	} else {
432 		while (sector >= conf->stride) {
433 			sector -= conf->stride;
434 			if (dev < conf->near_copies)
435 				dev += conf->raid_disks - conf->near_copies;
436 			else
437 				dev -= conf->near_copies;
438 		}
439 		chunk = sector >> conf->chunk_shift;
440 	}
441 	vchunk = chunk * conf->raid_disks + dev;
442 	sector_div(vchunk, conf->near_copies);
443 	return (vchunk << conf->chunk_shift) + offset;
444 }
445 
446 /**
447  *	raid10_mergeable_bvec -- tell bio layer if a two requests can be merged
448  *	@q: request queue
449  *	@bio: the buffer head that's been built up so far
450  *	@biovec: the request that could be merged to it.
451  *
452  *	Return amount of bytes we can accept at this offset
453  *      If near_copies == raid_disk, there are no striping issues,
454  *      but in that case, the function isn't called at all.
455  */
456 static int raid10_mergeable_bvec(request_queue_t *q, struct bio *bio,
457 				struct bio_vec *bio_vec)
458 {
459 	mddev_t *mddev = q->queuedata;
460 	sector_t sector = bio->bi_sector + get_start_sect(bio->bi_bdev);
461 	int max;
462 	unsigned int chunk_sectors = mddev->chunk_size >> 9;
463 	unsigned int bio_sectors = bio->bi_size >> 9;
464 
465 	max =  (chunk_sectors - ((sector & (chunk_sectors - 1)) + bio_sectors)) << 9;
466 	if (max < 0) max = 0; /* bio_add cannot handle a negative return */
467 	if (max <= bio_vec->bv_len && bio_sectors == 0)
468 		return bio_vec->bv_len;
469 	else
470 		return max;
471 }
472 
473 /*
474  * This routine returns the disk from which the requested read should
475  * be done. There is a per-array 'next expected sequential IO' sector
476  * number - if this matches on the next IO then we use the last disk.
477  * There is also a per-disk 'last know head position' sector that is
478  * maintained from IRQ contexts, both the normal and the resync IO
479  * completion handlers update this position correctly. If there is no
480  * perfect sequential match then we pick the disk whose head is closest.
481  *
482  * If there are 2 mirrors in the same 2 devices, performance degrades
483  * because position is mirror, not device based.
484  *
485  * The rdev for the device selected will have nr_pending incremented.
486  */
487 
488 /*
489  * FIXME: possibly should rethink readbalancing and do it differently
490  * depending on near_copies / far_copies geometry.
491  */
492 static int read_balance(conf_t *conf, r10bio_t *r10_bio)
493 {
494 	const unsigned long this_sector = r10_bio->sector;
495 	int disk, slot, nslot;
496 	const int sectors = r10_bio->sectors;
497 	sector_t new_distance, current_distance;
498 	mdk_rdev_t *rdev;
499 
500 	raid10_find_phys(conf, r10_bio);
501 	rcu_read_lock();
502 	/*
503 	 * Check if we can balance. We can balance on the whole
504 	 * device if no resync is going on (recovery is ok), or below
505 	 * the resync window. We take the first readable disk when
506 	 * above the resync window.
507 	 */
508 	if (conf->mddev->recovery_cp < MaxSector
509 	    && (this_sector + sectors >= conf->next_resync)) {
510 		/* make sure that disk is operational */
511 		slot = 0;
512 		disk = r10_bio->devs[slot].devnum;
513 
514 		while ((rdev = rcu_dereference(conf->mirrors[disk].rdev)) == NULL ||
515 		       r10_bio->devs[slot].bio == IO_BLOCKED ||
516 		       !test_bit(In_sync, &rdev->flags)) {
517 			slot++;
518 			if (slot == conf->copies) {
519 				slot = 0;
520 				disk = -1;
521 				break;
522 			}
523 			disk = r10_bio->devs[slot].devnum;
524 		}
525 		goto rb_out;
526 	}
527 
528 
529 	/* make sure the disk is operational */
530 	slot = 0;
531 	disk = r10_bio->devs[slot].devnum;
532 	while ((rdev=rcu_dereference(conf->mirrors[disk].rdev)) == NULL ||
533 	       r10_bio->devs[slot].bio == IO_BLOCKED ||
534 	       !test_bit(In_sync, &rdev->flags)) {
535 		slot ++;
536 		if (slot == conf->copies) {
537 			disk = -1;
538 			goto rb_out;
539 		}
540 		disk = r10_bio->devs[slot].devnum;
541 	}
542 
543 
544 	current_distance = abs(r10_bio->devs[slot].addr -
545 			       conf->mirrors[disk].head_position);
546 
547 	/* Find the disk whose head is closest */
548 
549 	for (nslot = slot; nslot < conf->copies; nslot++) {
550 		int ndisk = r10_bio->devs[nslot].devnum;
551 
552 
553 		if ((rdev=rcu_dereference(conf->mirrors[ndisk].rdev)) == NULL ||
554 		    r10_bio->devs[nslot].bio == IO_BLOCKED ||
555 		    !test_bit(In_sync, &rdev->flags))
556 			continue;
557 
558 		/* This optimisation is debatable, and completely destroys
559 		 * sequential read speed for 'far copies' arrays.  So only
560 		 * keep it for 'near' arrays, and review those later.
561 		 */
562 		if (conf->near_copies > 1 && !atomic_read(&rdev->nr_pending)) {
563 			disk = ndisk;
564 			slot = nslot;
565 			break;
566 		}
567 		new_distance = abs(r10_bio->devs[nslot].addr -
568 				   conf->mirrors[ndisk].head_position);
569 		if (new_distance < current_distance) {
570 			current_distance = new_distance;
571 			disk = ndisk;
572 			slot = nslot;
573 		}
574 	}
575 
576 rb_out:
577 	r10_bio->read_slot = slot;
578 /*	conf->next_seq_sect = this_sector + sectors;*/
579 
580 	if (disk >= 0 && (rdev=rcu_dereference(conf->mirrors[disk].rdev))!= NULL)
581 		atomic_inc(&conf->mirrors[disk].rdev->nr_pending);
582 	else
583 		disk = -1;
584 	rcu_read_unlock();
585 
586 	return disk;
587 }
588 
589 static void unplug_slaves(mddev_t *mddev)
590 {
591 	conf_t *conf = mddev_to_conf(mddev);
592 	int i;
593 
594 	rcu_read_lock();
595 	for (i=0; i<mddev->raid_disks; i++) {
596 		mdk_rdev_t *rdev = rcu_dereference(conf->mirrors[i].rdev);
597 		if (rdev && !test_bit(Faulty, &rdev->flags) && atomic_read(&rdev->nr_pending)) {
598 			request_queue_t *r_queue = bdev_get_queue(rdev->bdev);
599 
600 			atomic_inc(&rdev->nr_pending);
601 			rcu_read_unlock();
602 
603 			if (r_queue->unplug_fn)
604 				r_queue->unplug_fn(r_queue);
605 
606 			rdev_dec_pending(rdev, mddev);
607 			rcu_read_lock();
608 		}
609 	}
610 	rcu_read_unlock();
611 }
612 
613 static void raid10_unplug(request_queue_t *q)
614 {
615 	mddev_t *mddev = q->queuedata;
616 
617 	unplug_slaves(q->queuedata);
618 	md_wakeup_thread(mddev->thread);
619 }
620 
621 static int raid10_issue_flush(request_queue_t *q, struct gendisk *disk,
622 			     sector_t *error_sector)
623 {
624 	mddev_t *mddev = q->queuedata;
625 	conf_t *conf = mddev_to_conf(mddev);
626 	int i, ret = 0;
627 
628 	rcu_read_lock();
629 	for (i=0; i<mddev->raid_disks && ret == 0; i++) {
630 		mdk_rdev_t *rdev = rcu_dereference(conf->mirrors[i].rdev);
631 		if (rdev && !test_bit(Faulty, &rdev->flags)) {
632 			struct block_device *bdev = rdev->bdev;
633 			request_queue_t *r_queue = bdev_get_queue(bdev);
634 
635 			if (!r_queue->issue_flush_fn)
636 				ret = -EOPNOTSUPP;
637 			else {
638 				atomic_inc(&rdev->nr_pending);
639 				rcu_read_unlock();
640 				ret = r_queue->issue_flush_fn(r_queue, bdev->bd_disk,
641 							      error_sector);
642 				rdev_dec_pending(rdev, mddev);
643 				rcu_read_lock();
644 			}
645 		}
646 	}
647 	rcu_read_unlock();
648 	return ret;
649 }
650 
651 static int raid10_congested(void *data, int bits)
652 {
653 	mddev_t *mddev = data;
654 	conf_t *conf = mddev_to_conf(mddev);
655 	int i, ret = 0;
656 
657 	rcu_read_lock();
658 	for (i = 0; i < mddev->raid_disks && ret == 0; i++) {
659 		mdk_rdev_t *rdev = rcu_dereference(conf->mirrors[i].rdev);
660 		if (rdev && !test_bit(Faulty, &rdev->flags)) {
661 			request_queue_t *q = bdev_get_queue(rdev->bdev);
662 
663 			ret |= bdi_congested(&q->backing_dev_info, bits);
664 		}
665 	}
666 	rcu_read_unlock();
667 	return ret;
668 }
669 
670 
671 /* Barriers....
672  * Sometimes we need to suspend IO while we do something else,
673  * either some resync/recovery, or reconfigure the array.
674  * To do this we raise a 'barrier'.
675  * The 'barrier' is a counter that can be raised multiple times
676  * to count how many activities are happening which preclude
677  * normal IO.
678  * We can only raise the barrier if there is no pending IO.
679  * i.e. if nr_pending == 0.
680  * We choose only to raise the barrier if no-one is waiting for the
681  * barrier to go down.  This means that as soon as an IO request
682  * is ready, no other operations which require a barrier will start
683  * until the IO request has had a chance.
684  *
685  * So: regular IO calls 'wait_barrier'.  When that returns there
686  *    is no backgroup IO happening,  It must arrange to call
687  *    allow_barrier when it has finished its IO.
688  * backgroup IO calls must call raise_barrier.  Once that returns
689  *    there is no normal IO happeing.  It must arrange to call
690  *    lower_barrier when the particular background IO completes.
691  */
692 #define RESYNC_DEPTH 32
693 
694 static void raise_barrier(conf_t *conf, int force)
695 {
696 	BUG_ON(force && !conf->barrier);
697 	spin_lock_irq(&conf->resync_lock);
698 
699 	/* Wait until no block IO is waiting (unless 'force') */
700 	wait_event_lock_irq(conf->wait_barrier, force || !conf->nr_waiting,
701 			    conf->resync_lock,
702 			    raid10_unplug(conf->mddev->queue));
703 
704 	/* block any new IO from starting */
705 	conf->barrier++;
706 
707 	/* No wait for all pending IO to complete */
708 	wait_event_lock_irq(conf->wait_barrier,
709 			    !conf->nr_pending && conf->barrier < RESYNC_DEPTH,
710 			    conf->resync_lock,
711 			    raid10_unplug(conf->mddev->queue));
712 
713 	spin_unlock_irq(&conf->resync_lock);
714 }
715 
716 static void lower_barrier(conf_t *conf)
717 {
718 	unsigned long flags;
719 	spin_lock_irqsave(&conf->resync_lock, flags);
720 	conf->barrier--;
721 	spin_unlock_irqrestore(&conf->resync_lock, flags);
722 	wake_up(&conf->wait_barrier);
723 }
724 
725 static void wait_barrier(conf_t *conf)
726 {
727 	spin_lock_irq(&conf->resync_lock);
728 	if (conf->barrier) {
729 		conf->nr_waiting++;
730 		wait_event_lock_irq(conf->wait_barrier, !conf->barrier,
731 				    conf->resync_lock,
732 				    raid10_unplug(conf->mddev->queue));
733 		conf->nr_waiting--;
734 	}
735 	conf->nr_pending++;
736 	spin_unlock_irq(&conf->resync_lock);
737 }
738 
739 static void allow_barrier(conf_t *conf)
740 {
741 	unsigned long flags;
742 	spin_lock_irqsave(&conf->resync_lock, flags);
743 	conf->nr_pending--;
744 	spin_unlock_irqrestore(&conf->resync_lock, flags);
745 	wake_up(&conf->wait_barrier);
746 }
747 
748 static void freeze_array(conf_t *conf)
749 {
750 	/* stop syncio and normal IO and wait for everything to
751 	 * go quiet.
752 	 * We increment barrier and nr_waiting, and then
753 	 * wait until barrier+nr_pending match nr_queued+2
754 	 */
755 	spin_lock_irq(&conf->resync_lock);
756 	conf->barrier++;
757 	conf->nr_waiting++;
758 	wait_event_lock_irq(conf->wait_barrier,
759 			    conf->barrier+conf->nr_pending == conf->nr_queued+2,
760 			    conf->resync_lock,
761 			    raid10_unplug(conf->mddev->queue));
762 	spin_unlock_irq(&conf->resync_lock);
763 }
764 
765 static void unfreeze_array(conf_t *conf)
766 {
767 	/* reverse the effect of the freeze */
768 	spin_lock_irq(&conf->resync_lock);
769 	conf->barrier--;
770 	conf->nr_waiting--;
771 	wake_up(&conf->wait_barrier);
772 	spin_unlock_irq(&conf->resync_lock);
773 }
774 
775 static int make_request(request_queue_t *q, struct bio * bio)
776 {
777 	mddev_t *mddev = q->queuedata;
778 	conf_t *conf = mddev_to_conf(mddev);
779 	mirror_info_t *mirror;
780 	r10bio_t *r10_bio;
781 	struct bio *read_bio;
782 	int i;
783 	int chunk_sects = conf->chunk_mask + 1;
784 	const int rw = bio_data_dir(bio);
785 	const int do_sync = bio_sync(bio);
786 	struct bio_list bl;
787 	unsigned long flags;
788 
789 	if (unlikely(bio_barrier(bio))) {
790 		bio_endio(bio, bio->bi_size, -EOPNOTSUPP);
791 		return 0;
792 	}
793 
794 	/* If this request crosses a chunk boundary, we need to
795 	 * split it.  This will only happen for 1 PAGE (or less) requests.
796 	 */
797 	if (unlikely( (bio->bi_sector & conf->chunk_mask) + (bio->bi_size >> 9)
798 		      > chunk_sects &&
799 		    conf->near_copies < conf->raid_disks)) {
800 		struct bio_pair *bp;
801 		/* Sanity check -- queue functions should prevent this happening */
802 		if (bio->bi_vcnt != 1 ||
803 		    bio->bi_idx != 0)
804 			goto bad_map;
805 		/* This is a one page bio that upper layers
806 		 * refuse to split for us, so we need to split it.
807 		 */
808 		bp = bio_split(bio, bio_split_pool,
809 			       chunk_sects - (bio->bi_sector & (chunk_sects - 1)) );
810 		if (make_request(q, &bp->bio1))
811 			generic_make_request(&bp->bio1);
812 		if (make_request(q, &bp->bio2))
813 			generic_make_request(&bp->bio2);
814 
815 		bio_pair_release(bp);
816 		return 0;
817 	bad_map:
818 		printk("raid10_make_request bug: can't convert block across chunks"
819 		       " or bigger than %dk %llu %d\n", chunk_sects/2,
820 		       (unsigned long long)bio->bi_sector, bio->bi_size >> 10);
821 
822 		bio_io_error(bio, bio->bi_size);
823 		return 0;
824 	}
825 
826 	md_write_start(mddev, bio);
827 
828 	/*
829 	 * Register the new request and wait if the reconstruction
830 	 * thread has put up a bar for new requests.
831 	 * Continue immediately if no resync is active currently.
832 	 */
833 	wait_barrier(conf);
834 
835 	disk_stat_inc(mddev->gendisk, ios[rw]);
836 	disk_stat_add(mddev->gendisk, sectors[rw], bio_sectors(bio));
837 
838 	r10_bio = mempool_alloc(conf->r10bio_pool, GFP_NOIO);
839 
840 	r10_bio->master_bio = bio;
841 	r10_bio->sectors = bio->bi_size >> 9;
842 
843 	r10_bio->mddev = mddev;
844 	r10_bio->sector = bio->bi_sector;
845 	r10_bio->state = 0;
846 
847 	if (rw == READ) {
848 		/*
849 		 * read balancing logic:
850 		 */
851 		int disk = read_balance(conf, r10_bio);
852 		int slot = r10_bio->read_slot;
853 		if (disk < 0) {
854 			raid_end_bio_io(r10_bio);
855 			return 0;
856 		}
857 		mirror = conf->mirrors + disk;
858 
859 		read_bio = bio_clone(bio, GFP_NOIO);
860 
861 		r10_bio->devs[slot].bio = read_bio;
862 
863 		read_bio->bi_sector = r10_bio->devs[slot].addr +
864 			mirror->rdev->data_offset;
865 		read_bio->bi_bdev = mirror->rdev->bdev;
866 		read_bio->bi_end_io = raid10_end_read_request;
867 		read_bio->bi_rw = READ | do_sync;
868 		read_bio->bi_private = r10_bio;
869 
870 		generic_make_request(read_bio);
871 		return 0;
872 	}
873 
874 	/*
875 	 * WRITE:
876 	 */
877 	/* first select target devices under spinlock and
878 	 * inc refcount on their rdev.  Record them by setting
879 	 * bios[x] to bio
880 	 */
881 	raid10_find_phys(conf, r10_bio);
882 	rcu_read_lock();
883 	for (i = 0;  i < conf->copies; i++) {
884 		int d = r10_bio->devs[i].devnum;
885 		mdk_rdev_t *rdev = rcu_dereference(conf->mirrors[d].rdev);
886 		if (rdev &&
887 		    !test_bit(Faulty, &rdev->flags)) {
888 			atomic_inc(&rdev->nr_pending);
889 			r10_bio->devs[i].bio = bio;
890 		} else {
891 			r10_bio->devs[i].bio = NULL;
892 			set_bit(R10BIO_Degraded, &r10_bio->state);
893 		}
894 	}
895 	rcu_read_unlock();
896 
897 	atomic_set(&r10_bio->remaining, 0);
898 
899 	bio_list_init(&bl);
900 	for (i = 0; i < conf->copies; i++) {
901 		struct bio *mbio;
902 		int d = r10_bio->devs[i].devnum;
903 		if (!r10_bio->devs[i].bio)
904 			continue;
905 
906 		mbio = bio_clone(bio, GFP_NOIO);
907 		r10_bio->devs[i].bio = mbio;
908 
909 		mbio->bi_sector	= r10_bio->devs[i].addr+
910 			conf->mirrors[d].rdev->data_offset;
911 		mbio->bi_bdev = conf->mirrors[d].rdev->bdev;
912 		mbio->bi_end_io	= raid10_end_write_request;
913 		mbio->bi_rw = WRITE | do_sync;
914 		mbio->bi_private = r10_bio;
915 
916 		atomic_inc(&r10_bio->remaining);
917 		bio_list_add(&bl, mbio);
918 	}
919 
920 	bitmap_startwrite(mddev->bitmap, bio->bi_sector, r10_bio->sectors, 0);
921 	spin_lock_irqsave(&conf->device_lock, flags);
922 	bio_list_merge(&conf->pending_bio_list, &bl);
923 	blk_plug_device(mddev->queue);
924 	spin_unlock_irqrestore(&conf->device_lock, flags);
925 
926 	if (do_sync)
927 		md_wakeup_thread(mddev->thread);
928 
929 	return 0;
930 }
931 
932 static void status(struct seq_file *seq, mddev_t *mddev)
933 {
934 	conf_t *conf = mddev_to_conf(mddev);
935 	int i;
936 
937 	if (conf->near_copies < conf->raid_disks)
938 		seq_printf(seq, " %dK chunks", mddev->chunk_size/1024);
939 	if (conf->near_copies > 1)
940 		seq_printf(seq, " %d near-copies", conf->near_copies);
941 	if (conf->far_copies > 1) {
942 		if (conf->far_offset)
943 			seq_printf(seq, " %d offset-copies", conf->far_copies);
944 		else
945 			seq_printf(seq, " %d far-copies", conf->far_copies);
946 	}
947 	seq_printf(seq, " [%d/%d] [", conf->raid_disks,
948 					conf->raid_disks - mddev->degraded);
949 	for (i = 0; i < conf->raid_disks; i++)
950 		seq_printf(seq, "%s",
951 			      conf->mirrors[i].rdev &&
952 			      test_bit(In_sync, &conf->mirrors[i].rdev->flags) ? "U" : "_");
953 	seq_printf(seq, "]");
954 }
955 
956 static void error(mddev_t *mddev, mdk_rdev_t *rdev)
957 {
958 	char b[BDEVNAME_SIZE];
959 	conf_t *conf = mddev_to_conf(mddev);
960 
961 	/*
962 	 * If it is not operational, then we have already marked it as dead
963 	 * else if it is the last working disks, ignore the error, let the
964 	 * next level up know.
965 	 * else mark the drive as failed
966 	 */
967 	if (test_bit(In_sync, &rdev->flags)
968 	    && conf->raid_disks-mddev->degraded == 1)
969 		/*
970 		 * Don't fail the drive, just return an IO error.
971 		 * The test should really be more sophisticated than
972 		 * "working_disks == 1", but it isn't critical, and
973 		 * can wait until we do more sophisticated "is the drive
974 		 * really dead" tests...
975 		 */
976 		return;
977 	if (test_and_clear_bit(In_sync, &rdev->flags)) {
978 		unsigned long flags;
979 		spin_lock_irqsave(&conf->device_lock, flags);
980 		mddev->degraded++;
981 		spin_unlock_irqrestore(&conf->device_lock, flags);
982 		/*
983 		 * if recovery is running, make sure it aborts.
984 		 */
985 		set_bit(MD_RECOVERY_ERR, &mddev->recovery);
986 	}
987 	set_bit(Faulty, &rdev->flags);
988 	set_bit(MD_CHANGE_DEVS, &mddev->flags);
989 	printk(KERN_ALERT "raid10: Disk failure on %s, disabling device. \n"
990 		"	Operation continuing on %d devices\n",
991 		bdevname(rdev->bdev,b), conf->raid_disks - mddev->degraded);
992 }
993 
994 static void print_conf(conf_t *conf)
995 {
996 	int i;
997 	mirror_info_t *tmp;
998 
999 	printk("RAID10 conf printout:\n");
1000 	if (!conf) {
1001 		printk("(!conf)\n");
1002 		return;
1003 	}
1004 	printk(" --- wd:%d rd:%d\n", conf->raid_disks - conf->mddev->degraded,
1005 		conf->raid_disks);
1006 
1007 	for (i = 0; i < conf->raid_disks; i++) {
1008 		char b[BDEVNAME_SIZE];
1009 		tmp = conf->mirrors + i;
1010 		if (tmp->rdev)
1011 			printk(" disk %d, wo:%d, o:%d, dev:%s\n",
1012 				i, !test_bit(In_sync, &tmp->rdev->flags),
1013 			        !test_bit(Faulty, &tmp->rdev->flags),
1014 				bdevname(tmp->rdev->bdev,b));
1015 	}
1016 }
1017 
1018 static void close_sync(conf_t *conf)
1019 {
1020 	wait_barrier(conf);
1021 	allow_barrier(conf);
1022 
1023 	mempool_destroy(conf->r10buf_pool);
1024 	conf->r10buf_pool = NULL;
1025 }
1026 
1027 /* check if there are enough drives for
1028  * every block to appear on atleast one
1029  */
1030 static int enough(conf_t *conf)
1031 {
1032 	int first = 0;
1033 
1034 	do {
1035 		int n = conf->copies;
1036 		int cnt = 0;
1037 		while (n--) {
1038 			if (conf->mirrors[first].rdev)
1039 				cnt++;
1040 			first = (first+1) % conf->raid_disks;
1041 		}
1042 		if (cnt == 0)
1043 			return 0;
1044 	} while (first != 0);
1045 	return 1;
1046 }
1047 
1048 static int raid10_spare_active(mddev_t *mddev)
1049 {
1050 	int i;
1051 	conf_t *conf = mddev->private;
1052 	mirror_info_t *tmp;
1053 
1054 	/*
1055 	 * Find all non-in_sync disks within the RAID10 configuration
1056 	 * and mark them in_sync
1057 	 */
1058 	for (i = 0; i < conf->raid_disks; i++) {
1059 		tmp = conf->mirrors + i;
1060 		if (tmp->rdev
1061 		    && !test_bit(Faulty, &tmp->rdev->flags)
1062 		    && !test_and_set_bit(In_sync, &tmp->rdev->flags)) {
1063 			unsigned long flags;
1064 			spin_lock_irqsave(&conf->device_lock, flags);
1065 			mddev->degraded--;
1066 			spin_unlock_irqrestore(&conf->device_lock, flags);
1067 		}
1068 	}
1069 
1070 	print_conf(conf);
1071 	return 0;
1072 }
1073 
1074 
1075 static int raid10_add_disk(mddev_t *mddev, mdk_rdev_t *rdev)
1076 {
1077 	conf_t *conf = mddev->private;
1078 	int found = 0;
1079 	int mirror;
1080 	mirror_info_t *p;
1081 
1082 	if (mddev->recovery_cp < MaxSector)
1083 		/* only hot-add to in-sync arrays, as recovery is
1084 		 * very different from resync
1085 		 */
1086 		return 0;
1087 	if (!enough(conf))
1088 		return 0;
1089 
1090 	if (rdev->saved_raid_disk >= 0 &&
1091 	    conf->mirrors[rdev->saved_raid_disk].rdev == NULL)
1092 		mirror = rdev->saved_raid_disk;
1093 	else
1094 		mirror = 0;
1095 	for ( ; mirror < mddev->raid_disks; mirror++)
1096 		if ( !(p=conf->mirrors+mirror)->rdev) {
1097 
1098 			blk_queue_stack_limits(mddev->queue,
1099 					       rdev->bdev->bd_disk->queue);
1100 			/* as we don't honour merge_bvec_fn, we must never risk
1101 			 * violating it, so limit ->max_sector to one PAGE, as
1102 			 * a one page request is never in violation.
1103 			 */
1104 			if (rdev->bdev->bd_disk->queue->merge_bvec_fn &&
1105 			    mddev->queue->max_sectors > (PAGE_SIZE>>9))
1106 				mddev->queue->max_sectors = (PAGE_SIZE>>9);
1107 
1108 			p->head_position = 0;
1109 			rdev->raid_disk = mirror;
1110 			found = 1;
1111 			if (rdev->saved_raid_disk != mirror)
1112 				conf->fullsync = 1;
1113 			rcu_assign_pointer(p->rdev, rdev);
1114 			break;
1115 		}
1116 
1117 	print_conf(conf);
1118 	return found;
1119 }
1120 
1121 static int raid10_remove_disk(mddev_t *mddev, int number)
1122 {
1123 	conf_t *conf = mddev->private;
1124 	int err = 0;
1125 	mdk_rdev_t *rdev;
1126 	mirror_info_t *p = conf->mirrors+ number;
1127 
1128 	print_conf(conf);
1129 	rdev = p->rdev;
1130 	if (rdev) {
1131 		if (test_bit(In_sync, &rdev->flags) ||
1132 		    atomic_read(&rdev->nr_pending)) {
1133 			err = -EBUSY;
1134 			goto abort;
1135 		}
1136 		p->rdev = NULL;
1137 		synchronize_rcu();
1138 		if (atomic_read(&rdev->nr_pending)) {
1139 			/* lost the race, try later */
1140 			err = -EBUSY;
1141 			p->rdev = rdev;
1142 		}
1143 	}
1144 abort:
1145 
1146 	print_conf(conf);
1147 	return err;
1148 }
1149 
1150 
1151 static int end_sync_read(struct bio *bio, unsigned int bytes_done, int error)
1152 {
1153 	r10bio_t * r10_bio = (r10bio_t *)(bio->bi_private);
1154 	conf_t *conf = mddev_to_conf(r10_bio->mddev);
1155 	int i,d;
1156 
1157 	if (bio->bi_size)
1158 		return 1;
1159 
1160 	for (i=0; i<conf->copies; i++)
1161 		if (r10_bio->devs[i].bio == bio)
1162 			break;
1163 	BUG_ON(i == conf->copies);
1164 	update_head_pos(i, r10_bio);
1165 	d = r10_bio->devs[i].devnum;
1166 
1167 	if (test_bit(BIO_UPTODATE, &bio->bi_flags))
1168 		set_bit(R10BIO_Uptodate, &r10_bio->state);
1169 	else {
1170 		atomic_add(r10_bio->sectors,
1171 			   &conf->mirrors[d].rdev->corrected_errors);
1172 		if (!test_bit(MD_RECOVERY_SYNC, &conf->mddev->recovery))
1173 			md_error(r10_bio->mddev,
1174 				 conf->mirrors[d].rdev);
1175 	}
1176 
1177 	/* for reconstruct, we always reschedule after a read.
1178 	 * for resync, only after all reads
1179 	 */
1180 	if (test_bit(R10BIO_IsRecover, &r10_bio->state) ||
1181 	    atomic_dec_and_test(&r10_bio->remaining)) {
1182 		/* we have read all the blocks,
1183 		 * do the comparison in process context in raid10d
1184 		 */
1185 		reschedule_retry(r10_bio);
1186 	}
1187 	rdev_dec_pending(conf->mirrors[d].rdev, conf->mddev);
1188 	return 0;
1189 }
1190 
1191 static int end_sync_write(struct bio *bio, unsigned int bytes_done, int error)
1192 {
1193 	int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags);
1194 	r10bio_t * r10_bio = (r10bio_t *)(bio->bi_private);
1195 	mddev_t *mddev = r10_bio->mddev;
1196 	conf_t *conf = mddev_to_conf(mddev);
1197 	int i,d;
1198 
1199 	if (bio->bi_size)
1200 		return 1;
1201 
1202 	for (i = 0; i < conf->copies; i++)
1203 		if (r10_bio->devs[i].bio == bio)
1204 			break;
1205 	d = r10_bio->devs[i].devnum;
1206 
1207 	if (!uptodate)
1208 		md_error(mddev, conf->mirrors[d].rdev);
1209 	update_head_pos(i, r10_bio);
1210 
1211 	while (atomic_dec_and_test(&r10_bio->remaining)) {
1212 		if (r10_bio->master_bio == NULL) {
1213 			/* the primary of several recovery bios */
1214 			md_done_sync(mddev, r10_bio->sectors, 1);
1215 			put_buf(r10_bio);
1216 			break;
1217 		} else {
1218 			r10bio_t *r10_bio2 = (r10bio_t *)r10_bio->master_bio;
1219 			put_buf(r10_bio);
1220 			r10_bio = r10_bio2;
1221 		}
1222 	}
1223 	rdev_dec_pending(conf->mirrors[d].rdev, mddev);
1224 	return 0;
1225 }
1226 
1227 /*
1228  * Note: sync and recover and handled very differently for raid10
1229  * This code is for resync.
1230  * For resync, we read through virtual addresses and read all blocks.
1231  * If there is any error, we schedule a write.  The lowest numbered
1232  * drive is authoritative.
1233  * However requests come for physical address, so we need to map.
1234  * For every physical address there are raid_disks/copies virtual addresses,
1235  * which is always are least one, but is not necessarly an integer.
1236  * This means that a physical address can span multiple chunks, so we may
1237  * have to submit multiple io requests for a single sync request.
1238  */
1239 /*
1240  * We check if all blocks are in-sync and only write to blocks that
1241  * aren't in sync
1242  */
1243 static void sync_request_write(mddev_t *mddev, r10bio_t *r10_bio)
1244 {
1245 	conf_t *conf = mddev_to_conf(mddev);
1246 	int i, first;
1247 	struct bio *tbio, *fbio;
1248 
1249 	atomic_set(&r10_bio->remaining, 1);
1250 
1251 	/* find the first device with a block */
1252 	for (i=0; i<conf->copies; i++)
1253 		if (test_bit(BIO_UPTODATE, &r10_bio->devs[i].bio->bi_flags))
1254 			break;
1255 
1256 	if (i == conf->copies)
1257 		goto done;
1258 
1259 	first = i;
1260 	fbio = r10_bio->devs[i].bio;
1261 
1262 	/* now find blocks with errors */
1263 	for (i=0 ; i < conf->copies ; i++) {
1264 		int  j, d;
1265 		int vcnt = r10_bio->sectors >> (PAGE_SHIFT-9);
1266 
1267 		tbio = r10_bio->devs[i].bio;
1268 
1269 		if (tbio->bi_end_io != end_sync_read)
1270 			continue;
1271 		if (i == first)
1272 			continue;
1273 		if (test_bit(BIO_UPTODATE, &r10_bio->devs[i].bio->bi_flags)) {
1274 			/* We know that the bi_io_vec layout is the same for
1275 			 * both 'first' and 'i', so we just compare them.
1276 			 * All vec entries are PAGE_SIZE;
1277 			 */
1278 			for (j = 0; j < vcnt; j++)
1279 				if (memcmp(page_address(fbio->bi_io_vec[j].bv_page),
1280 					   page_address(tbio->bi_io_vec[j].bv_page),
1281 					   PAGE_SIZE))
1282 					break;
1283 			if (j == vcnt)
1284 				continue;
1285 			mddev->resync_mismatches += r10_bio->sectors;
1286 		}
1287 		if (test_bit(MD_RECOVERY_CHECK, &mddev->recovery))
1288 			/* Don't fix anything. */
1289 			continue;
1290 		/* Ok, we need to write this bio
1291 		 * First we need to fixup bv_offset, bv_len and
1292 		 * bi_vecs, as the read request might have corrupted these
1293 		 */
1294 		tbio->bi_vcnt = vcnt;
1295 		tbio->bi_size = r10_bio->sectors << 9;
1296 		tbio->bi_idx = 0;
1297 		tbio->bi_phys_segments = 0;
1298 		tbio->bi_hw_segments = 0;
1299 		tbio->bi_hw_front_size = 0;
1300 		tbio->bi_hw_back_size = 0;
1301 		tbio->bi_flags &= ~(BIO_POOL_MASK - 1);
1302 		tbio->bi_flags |= 1 << BIO_UPTODATE;
1303 		tbio->bi_next = NULL;
1304 		tbio->bi_rw = WRITE;
1305 		tbio->bi_private = r10_bio;
1306 		tbio->bi_sector = r10_bio->devs[i].addr;
1307 
1308 		for (j=0; j < vcnt ; j++) {
1309 			tbio->bi_io_vec[j].bv_offset = 0;
1310 			tbio->bi_io_vec[j].bv_len = PAGE_SIZE;
1311 
1312 			memcpy(page_address(tbio->bi_io_vec[j].bv_page),
1313 			       page_address(fbio->bi_io_vec[j].bv_page),
1314 			       PAGE_SIZE);
1315 		}
1316 		tbio->bi_end_io = end_sync_write;
1317 
1318 		d = r10_bio->devs[i].devnum;
1319 		atomic_inc(&conf->mirrors[d].rdev->nr_pending);
1320 		atomic_inc(&r10_bio->remaining);
1321 		md_sync_acct(conf->mirrors[d].rdev->bdev, tbio->bi_size >> 9);
1322 
1323 		tbio->bi_sector += conf->mirrors[d].rdev->data_offset;
1324 		tbio->bi_bdev = conf->mirrors[d].rdev->bdev;
1325 		generic_make_request(tbio);
1326 	}
1327 
1328 done:
1329 	if (atomic_dec_and_test(&r10_bio->remaining)) {
1330 		md_done_sync(mddev, r10_bio->sectors, 1);
1331 		put_buf(r10_bio);
1332 	}
1333 }
1334 
1335 /*
1336  * Now for the recovery code.
1337  * Recovery happens across physical sectors.
1338  * We recover all non-is_sync drives by finding the virtual address of
1339  * each, and then choose a working drive that also has that virt address.
1340  * There is a separate r10_bio for each non-in_sync drive.
1341  * Only the first two slots are in use. The first for reading,
1342  * The second for writing.
1343  *
1344  */
1345 
1346 static void recovery_request_write(mddev_t *mddev, r10bio_t *r10_bio)
1347 {
1348 	conf_t *conf = mddev_to_conf(mddev);
1349 	int i, d;
1350 	struct bio *bio, *wbio;
1351 
1352 
1353 	/* move the pages across to the second bio
1354 	 * and submit the write request
1355 	 */
1356 	bio = r10_bio->devs[0].bio;
1357 	wbio = r10_bio->devs[1].bio;
1358 	for (i=0; i < wbio->bi_vcnt; i++) {
1359 		struct page *p = bio->bi_io_vec[i].bv_page;
1360 		bio->bi_io_vec[i].bv_page = wbio->bi_io_vec[i].bv_page;
1361 		wbio->bi_io_vec[i].bv_page = p;
1362 	}
1363 	d = r10_bio->devs[1].devnum;
1364 
1365 	atomic_inc(&conf->mirrors[d].rdev->nr_pending);
1366 	md_sync_acct(conf->mirrors[d].rdev->bdev, wbio->bi_size >> 9);
1367 	if (test_bit(R10BIO_Uptodate, &r10_bio->state))
1368 		generic_make_request(wbio);
1369 	else
1370 		bio_endio(wbio, wbio->bi_size, -EIO);
1371 }
1372 
1373 
1374 /*
1375  * This is a kernel thread which:
1376  *
1377  *	1.	Retries failed read operations on working mirrors.
1378  *	2.	Updates the raid superblock when problems encounter.
1379  *	3.	Performs writes following reads for array synchronising.
1380  */
1381 
1382 static void fix_read_error(conf_t *conf, mddev_t *mddev, r10bio_t *r10_bio)
1383 {
1384 	int sect = 0; /* Offset from r10_bio->sector */
1385 	int sectors = r10_bio->sectors;
1386 	mdk_rdev_t*rdev;
1387 	while(sectors) {
1388 		int s = sectors;
1389 		int sl = r10_bio->read_slot;
1390 		int success = 0;
1391 		int start;
1392 
1393 		if (s > (PAGE_SIZE>>9))
1394 			s = PAGE_SIZE >> 9;
1395 
1396 		rcu_read_lock();
1397 		do {
1398 			int d = r10_bio->devs[sl].devnum;
1399 			rdev = rcu_dereference(conf->mirrors[d].rdev);
1400 			if (rdev &&
1401 			    test_bit(In_sync, &rdev->flags)) {
1402 				atomic_inc(&rdev->nr_pending);
1403 				rcu_read_unlock();
1404 				success = sync_page_io(rdev->bdev,
1405 						       r10_bio->devs[sl].addr +
1406 						       sect + rdev->data_offset,
1407 						       s<<9,
1408 						       conf->tmppage, READ);
1409 				rdev_dec_pending(rdev, mddev);
1410 				rcu_read_lock();
1411 				if (success)
1412 					break;
1413 			}
1414 			sl++;
1415 			if (sl == conf->copies)
1416 				sl = 0;
1417 		} while (!success && sl != r10_bio->read_slot);
1418 		rcu_read_unlock();
1419 
1420 		if (!success) {
1421 			/* Cannot read from anywhere -- bye bye array */
1422 			int dn = r10_bio->devs[r10_bio->read_slot].devnum;
1423 			md_error(mddev, conf->mirrors[dn].rdev);
1424 			break;
1425 		}
1426 
1427 		start = sl;
1428 		/* write it back and re-read */
1429 		rcu_read_lock();
1430 		while (sl != r10_bio->read_slot) {
1431 			int d;
1432 			if (sl==0)
1433 				sl = conf->copies;
1434 			sl--;
1435 			d = r10_bio->devs[sl].devnum;
1436 			rdev = rcu_dereference(conf->mirrors[d].rdev);
1437 			if (rdev &&
1438 			    test_bit(In_sync, &rdev->flags)) {
1439 				atomic_inc(&rdev->nr_pending);
1440 				rcu_read_unlock();
1441 				atomic_add(s, &rdev->corrected_errors);
1442 				if (sync_page_io(rdev->bdev,
1443 						 r10_bio->devs[sl].addr +
1444 						 sect + rdev->data_offset,
1445 						 s<<9, conf->tmppage, WRITE)
1446 				    == 0)
1447 					/* Well, this device is dead */
1448 					md_error(mddev, rdev);
1449 				rdev_dec_pending(rdev, mddev);
1450 				rcu_read_lock();
1451 			}
1452 		}
1453 		sl = start;
1454 		while (sl != r10_bio->read_slot) {
1455 			int d;
1456 			if (sl==0)
1457 				sl = conf->copies;
1458 			sl--;
1459 			d = r10_bio->devs[sl].devnum;
1460 			rdev = rcu_dereference(conf->mirrors[d].rdev);
1461 			if (rdev &&
1462 			    test_bit(In_sync, &rdev->flags)) {
1463 				char b[BDEVNAME_SIZE];
1464 				atomic_inc(&rdev->nr_pending);
1465 				rcu_read_unlock();
1466 				if (sync_page_io(rdev->bdev,
1467 						 r10_bio->devs[sl].addr +
1468 						 sect + rdev->data_offset,
1469 						 s<<9, conf->tmppage, READ) == 0)
1470 					/* Well, this device is dead */
1471 					md_error(mddev, rdev);
1472 				else
1473 					printk(KERN_INFO
1474 					       "raid10:%s: read error corrected"
1475 					       " (%d sectors at %llu on %s)\n",
1476 					       mdname(mddev), s,
1477 					       (unsigned long long)(sect+
1478 					            rdev->data_offset),
1479 					       bdevname(rdev->bdev, b));
1480 
1481 				rdev_dec_pending(rdev, mddev);
1482 				rcu_read_lock();
1483 			}
1484 		}
1485 		rcu_read_unlock();
1486 
1487 		sectors -= s;
1488 		sect += s;
1489 	}
1490 }
1491 
1492 static void raid10d(mddev_t *mddev)
1493 {
1494 	r10bio_t *r10_bio;
1495 	struct bio *bio;
1496 	unsigned long flags;
1497 	conf_t *conf = mddev_to_conf(mddev);
1498 	struct list_head *head = &conf->retry_list;
1499 	int unplug=0;
1500 	mdk_rdev_t *rdev;
1501 
1502 	md_check_recovery(mddev);
1503 
1504 	for (;;) {
1505 		char b[BDEVNAME_SIZE];
1506 		spin_lock_irqsave(&conf->device_lock, flags);
1507 
1508 		if (conf->pending_bio_list.head) {
1509 			bio = bio_list_get(&conf->pending_bio_list);
1510 			blk_remove_plug(mddev->queue);
1511 			spin_unlock_irqrestore(&conf->device_lock, flags);
1512 			/* flush any pending bitmap writes to disk before proceeding w/ I/O */
1513 			if (bitmap_unplug(mddev->bitmap) != 0)
1514 				printk("%s: bitmap file write failed!\n", mdname(mddev));
1515 
1516 			while (bio) { /* submit pending writes */
1517 				struct bio *next = bio->bi_next;
1518 				bio->bi_next = NULL;
1519 				generic_make_request(bio);
1520 				bio = next;
1521 			}
1522 			unplug = 1;
1523 
1524 			continue;
1525 		}
1526 
1527 		if (list_empty(head))
1528 			break;
1529 		r10_bio = list_entry(head->prev, r10bio_t, retry_list);
1530 		list_del(head->prev);
1531 		conf->nr_queued--;
1532 		spin_unlock_irqrestore(&conf->device_lock, flags);
1533 
1534 		mddev = r10_bio->mddev;
1535 		conf = mddev_to_conf(mddev);
1536 		if (test_bit(R10BIO_IsSync, &r10_bio->state)) {
1537 			sync_request_write(mddev, r10_bio);
1538 			unplug = 1;
1539 		} else 	if (test_bit(R10BIO_IsRecover, &r10_bio->state)) {
1540 			recovery_request_write(mddev, r10_bio);
1541 			unplug = 1;
1542 		} else {
1543 			int mirror;
1544 			/* we got a read error. Maybe the drive is bad.  Maybe just
1545 			 * the block and we can fix it.
1546 			 * We freeze all other IO, and try reading the block from
1547 			 * other devices.  When we find one, we re-write
1548 			 * and check it that fixes the read error.
1549 			 * This is all done synchronously while the array is
1550 			 * frozen.
1551 			 */
1552 			if (mddev->ro == 0) {
1553 				freeze_array(conf);
1554 				fix_read_error(conf, mddev, r10_bio);
1555 				unfreeze_array(conf);
1556 			}
1557 
1558 			bio = r10_bio->devs[r10_bio->read_slot].bio;
1559 			r10_bio->devs[r10_bio->read_slot].bio =
1560 				mddev->ro ? IO_BLOCKED : NULL;
1561 			bio_put(bio);
1562 			mirror = read_balance(conf, r10_bio);
1563 			if (mirror == -1) {
1564 				printk(KERN_ALERT "raid10: %s: unrecoverable I/O"
1565 				       " read error for block %llu\n",
1566 				       bdevname(bio->bi_bdev,b),
1567 				       (unsigned long long)r10_bio->sector);
1568 				raid_end_bio_io(r10_bio);
1569 			} else {
1570 				const int do_sync = bio_sync(r10_bio->master_bio);
1571 				rdev = conf->mirrors[mirror].rdev;
1572 				if (printk_ratelimit())
1573 					printk(KERN_ERR "raid10: %s: redirecting sector %llu to"
1574 					       " another mirror\n",
1575 					       bdevname(rdev->bdev,b),
1576 					       (unsigned long long)r10_bio->sector);
1577 				bio = bio_clone(r10_bio->master_bio, GFP_NOIO);
1578 				r10_bio->devs[r10_bio->read_slot].bio = bio;
1579 				bio->bi_sector = r10_bio->devs[r10_bio->read_slot].addr
1580 					+ rdev->data_offset;
1581 				bio->bi_bdev = rdev->bdev;
1582 				bio->bi_rw = READ | do_sync;
1583 				bio->bi_private = r10_bio;
1584 				bio->bi_end_io = raid10_end_read_request;
1585 				unplug = 1;
1586 				generic_make_request(bio);
1587 			}
1588 		}
1589 	}
1590 	spin_unlock_irqrestore(&conf->device_lock, flags);
1591 	if (unplug)
1592 		unplug_slaves(mddev);
1593 }
1594 
1595 
1596 static int init_resync(conf_t *conf)
1597 {
1598 	int buffs;
1599 
1600 	buffs = RESYNC_WINDOW / RESYNC_BLOCK_SIZE;
1601 	BUG_ON(conf->r10buf_pool);
1602 	conf->r10buf_pool = mempool_create(buffs, r10buf_pool_alloc, r10buf_pool_free, conf);
1603 	if (!conf->r10buf_pool)
1604 		return -ENOMEM;
1605 	conf->next_resync = 0;
1606 	return 0;
1607 }
1608 
1609 /*
1610  * perform a "sync" on one "block"
1611  *
1612  * We need to make sure that no normal I/O request - particularly write
1613  * requests - conflict with active sync requests.
1614  *
1615  * This is achieved by tracking pending requests and a 'barrier' concept
1616  * that can be installed to exclude normal IO requests.
1617  *
1618  * Resync and recovery are handled very differently.
1619  * We differentiate by looking at MD_RECOVERY_SYNC in mddev->recovery.
1620  *
1621  * For resync, we iterate over virtual addresses, read all copies,
1622  * and update if there are differences.  If only one copy is live,
1623  * skip it.
1624  * For recovery, we iterate over physical addresses, read a good
1625  * value for each non-in_sync drive, and over-write.
1626  *
1627  * So, for recovery we may have several outstanding complex requests for a
1628  * given address, one for each out-of-sync device.  We model this by allocating
1629  * a number of r10_bio structures, one for each out-of-sync device.
1630  * As we setup these structures, we collect all bio's together into a list
1631  * which we then process collectively to add pages, and then process again
1632  * to pass to generic_make_request.
1633  *
1634  * The r10_bio structures are linked using a borrowed master_bio pointer.
1635  * This link is counted in ->remaining.  When the r10_bio that points to NULL
1636  * has its remaining count decremented to 0, the whole complex operation
1637  * is complete.
1638  *
1639  */
1640 
1641 static sector_t sync_request(mddev_t *mddev, sector_t sector_nr, int *skipped, int go_faster)
1642 {
1643 	conf_t *conf = mddev_to_conf(mddev);
1644 	r10bio_t *r10_bio;
1645 	struct bio *biolist = NULL, *bio;
1646 	sector_t max_sector, nr_sectors;
1647 	int disk;
1648 	int i;
1649 	int max_sync;
1650 	int sync_blocks;
1651 
1652 	sector_t sectors_skipped = 0;
1653 	int chunks_skipped = 0;
1654 
1655 	if (!conf->r10buf_pool)
1656 		if (init_resync(conf))
1657 			return 0;
1658 
1659  skipped:
1660 	max_sector = mddev->size << 1;
1661 	if (test_bit(MD_RECOVERY_SYNC, &mddev->recovery))
1662 		max_sector = mddev->resync_max_sectors;
1663 	if (sector_nr >= max_sector) {
1664 		/* If we aborted, we need to abort the
1665 		 * sync on the 'current' bitmap chucks (there can
1666 		 * be several when recovering multiple devices).
1667 		 * as we may have started syncing it but not finished.
1668 		 * We can find the current address in
1669 		 * mddev->curr_resync, but for recovery,
1670 		 * we need to convert that to several
1671 		 * virtual addresses.
1672 		 */
1673 		if (mddev->curr_resync < max_sector) { /* aborted */
1674 			if (test_bit(MD_RECOVERY_SYNC, &mddev->recovery))
1675 				bitmap_end_sync(mddev->bitmap, mddev->curr_resync,
1676 						&sync_blocks, 1);
1677 			else for (i=0; i<conf->raid_disks; i++) {
1678 				sector_t sect =
1679 					raid10_find_virt(conf, mddev->curr_resync, i);
1680 				bitmap_end_sync(mddev->bitmap, sect,
1681 						&sync_blocks, 1);
1682 			}
1683 		} else /* completed sync */
1684 			conf->fullsync = 0;
1685 
1686 		bitmap_close_sync(mddev->bitmap);
1687 		close_sync(conf);
1688 		*skipped = 1;
1689 		return sectors_skipped;
1690 	}
1691 	if (chunks_skipped >= conf->raid_disks) {
1692 		/* if there has been nothing to do on any drive,
1693 		 * then there is nothing to do at all..
1694 		 */
1695 		*skipped = 1;
1696 		return (max_sector - sector_nr) + sectors_skipped;
1697 	}
1698 
1699 	/* make sure whole request will fit in a chunk - if chunks
1700 	 * are meaningful
1701 	 */
1702 	if (conf->near_copies < conf->raid_disks &&
1703 	    max_sector > (sector_nr | conf->chunk_mask))
1704 		max_sector = (sector_nr | conf->chunk_mask) + 1;
1705 	/*
1706 	 * If there is non-resync activity waiting for us then
1707 	 * put in a delay to throttle resync.
1708 	 */
1709 	if (!go_faster && conf->nr_waiting)
1710 		msleep_interruptible(1000);
1711 
1712 	/* Again, very different code for resync and recovery.
1713 	 * Both must result in an r10bio with a list of bios that
1714 	 * have bi_end_io, bi_sector, bi_bdev set,
1715 	 * and bi_private set to the r10bio.
1716 	 * For recovery, we may actually create several r10bios
1717 	 * with 2 bios in each, that correspond to the bios in the main one.
1718 	 * In this case, the subordinate r10bios link back through a
1719 	 * borrowed master_bio pointer, and the counter in the master
1720 	 * includes a ref from each subordinate.
1721 	 */
1722 	/* First, we decide what to do and set ->bi_end_io
1723 	 * To end_sync_read if we want to read, and
1724 	 * end_sync_write if we will want to write.
1725 	 */
1726 
1727 	max_sync = RESYNC_PAGES << (PAGE_SHIFT-9);
1728 	if (!test_bit(MD_RECOVERY_SYNC, &mddev->recovery)) {
1729 		/* recovery... the complicated one */
1730 		int i, j, k;
1731 		r10_bio = NULL;
1732 
1733 		for (i=0 ; i<conf->raid_disks; i++)
1734 			if (conf->mirrors[i].rdev &&
1735 			    !test_bit(In_sync, &conf->mirrors[i].rdev->flags)) {
1736 				int still_degraded = 0;
1737 				/* want to reconstruct this device */
1738 				r10bio_t *rb2 = r10_bio;
1739 				sector_t sect = raid10_find_virt(conf, sector_nr, i);
1740 				int must_sync;
1741 				/* Unless we are doing a full sync, we only need
1742 				 * to recover the block if it is set in the bitmap
1743 				 */
1744 				must_sync = bitmap_start_sync(mddev->bitmap, sect,
1745 							      &sync_blocks, 1);
1746 				if (sync_blocks < max_sync)
1747 					max_sync = sync_blocks;
1748 				if (!must_sync &&
1749 				    !conf->fullsync) {
1750 					/* yep, skip the sync_blocks here, but don't assume
1751 					 * that there will never be anything to do here
1752 					 */
1753 					chunks_skipped = -1;
1754 					continue;
1755 				}
1756 
1757 				r10_bio = mempool_alloc(conf->r10buf_pool, GFP_NOIO);
1758 				raise_barrier(conf, rb2 != NULL);
1759 				atomic_set(&r10_bio->remaining, 0);
1760 
1761 				r10_bio->master_bio = (struct bio*)rb2;
1762 				if (rb2)
1763 					atomic_inc(&rb2->remaining);
1764 				r10_bio->mddev = mddev;
1765 				set_bit(R10BIO_IsRecover, &r10_bio->state);
1766 				r10_bio->sector = sect;
1767 
1768 				raid10_find_phys(conf, r10_bio);
1769 				/* Need to check if this section will still be
1770 				 * degraded
1771 				 */
1772 				for (j=0; j<conf->copies;j++) {
1773 					int d = r10_bio->devs[j].devnum;
1774 					if (conf->mirrors[d].rdev == NULL ||
1775 					    test_bit(Faulty, &conf->mirrors[d].rdev->flags)) {
1776 						still_degraded = 1;
1777 						break;
1778 					}
1779 				}
1780 				must_sync = bitmap_start_sync(mddev->bitmap, sect,
1781 							      &sync_blocks, still_degraded);
1782 
1783 				for (j=0; j<conf->copies;j++) {
1784 					int d = r10_bio->devs[j].devnum;
1785 					if (conf->mirrors[d].rdev &&
1786 					    test_bit(In_sync, &conf->mirrors[d].rdev->flags)) {
1787 						/* This is where we read from */
1788 						bio = r10_bio->devs[0].bio;
1789 						bio->bi_next = biolist;
1790 						biolist = bio;
1791 						bio->bi_private = r10_bio;
1792 						bio->bi_end_io = end_sync_read;
1793 						bio->bi_rw = READ;
1794 						bio->bi_sector = r10_bio->devs[j].addr +
1795 							conf->mirrors[d].rdev->data_offset;
1796 						bio->bi_bdev = conf->mirrors[d].rdev->bdev;
1797 						atomic_inc(&conf->mirrors[d].rdev->nr_pending);
1798 						atomic_inc(&r10_bio->remaining);
1799 						/* and we write to 'i' */
1800 
1801 						for (k=0; k<conf->copies; k++)
1802 							if (r10_bio->devs[k].devnum == i)
1803 								break;
1804 						BUG_ON(k == conf->copies);
1805 						bio = r10_bio->devs[1].bio;
1806 						bio->bi_next = biolist;
1807 						biolist = bio;
1808 						bio->bi_private = r10_bio;
1809 						bio->bi_end_io = end_sync_write;
1810 						bio->bi_rw = WRITE;
1811 						bio->bi_sector = r10_bio->devs[k].addr +
1812 							conf->mirrors[i].rdev->data_offset;
1813 						bio->bi_bdev = conf->mirrors[i].rdev->bdev;
1814 
1815 						r10_bio->devs[0].devnum = d;
1816 						r10_bio->devs[1].devnum = i;
1817 
1818 						break;
1819 					}
1820 				}
1821 				if (j == conf->copies) {
1822 					/* Cannot recover, so abort the recovery */
1823 					put_buf(r10_bio);
1824 					r10_bio = rb2;
1825 					if (!test_and_set_bit(MD_RECOVERY_ERR, &mddev->recovery))
1826 						printk(KERN_INFO "raid10: %s: insufficient working devices for recovery.\n",
1827 						       mdname(mddev));
1828 					break;
1829 				}
1830 			}
1831 		if (biolist == NULL) {
1832 			while (r10_bio) {
1833 				r10bio_t *rb2 = r10_bio;
1834 				r10_bio = (r10bio_t*) rb2->master_bio;
1835 				rb2->master_bio = NULL;
1836 				put_buf(rb2);
1837 			}
1838 			goto giveup;
1839 		}
1840 	} else {
1841 		/* resync. Schedule a read for every block at this virt offset */
1842 		int count = 0;
1843 
1844 		if (!bitmap_start_sync(mddev->bitmap, sector_nr,
1845 				       &sync_blocks, mddev->degraded) &&
1846 		    !conf->fullsync && !test_bit(MD_RECOVERY_REQUESTED, &mddev->recovery)) {
1847 			/* We can skip this block */
1848 			*skipped = 1;
1849 			return sync_blocks + sectors_skipped;
1850 		}
1851 		if (sync_blocks < max_sync)
1852 			max_sync = sync_blocks;
1853 		r10_bio = mempool_alloc(conf->r10buf_pool, GFP_NOIO);
1854 
1855 		r10_bio->mddev = mddev;
1856 		atomic_set(&r10_bio->remaining, 0);
1857 		raise_barrier(conf, 0);
1858 		conf->next_resync = sector_nr;
1859 
1860 		r10_bio->master_bio = NULL;
1861 		r10_bio->sector = sector_nr;
1862 		set_bit(R10BIO_IsSync, &r10_bio->state);
1863 		raid10_find_phys(conf, r10_bio);
1864 		r10_bio->sectors = (sector_nr | conf->chunk_mask) - sector_nr +1;
1865 
1866 		for (i=0; i<conf->copies; i++) {
1867 			int d = r10_bio->devs[i].devnum;
1868 			bio = r10_bio->devs[i].bio;
1869 			bio->bi_end_io = NULL;
1870 			if (conf->mirrors[d].rdev == NULL ||
1871 			    test_bit(Faulty, &conf->mirrors[d].rdev->flags))
1872 				continue;
1873 			atomic_inc(&conf->mirrors[d].rdev->nr_pending);
1874 			atomic_inc(&r10_bio->remaining);
1875 			bio->bi_next = biolist;
1876 			biolist = bio;
1877 			bio->bi_private = r10_bio;
1878 			bio->bi_end_io = end_sync_read;
1879 			bio->bi_rw = READ;
1880 			bio->bi_sector = r10_bio->devs[i].addr +
1881 				conf->mirrors[d].rdev->data_offset;
1882 			bio->bi_bdev = conf->mirrors[d].rdev->bdev;
1883 			count++;
1884 		}
1885 
1886 		if (count < 2) {
1887 			for (i=0; i<conf->copies; i++) {
1888 				int d = r10_bio->devs[i].devnum;
1889 				if (r10_bio->devs[i].bio->bi_end_io)
1890 					rdev_dec_pending(conf->mirrors[d].rdev, mddev);
1891 			}
1892 			put_buf(r10_bio);
1893 			biolist = NULL;
1894 			goto giveup;
1895 		}
1896 	}
1897 
1898 	for (bio = biolist; bio ; bio=bio->bi_next) {
1899 
1900 		bio->bi_flags &= ~(BIO_POOL_MASK - 1);
1901 		if (bio->bi_end_io)
1902 			bio->bi_flags |= 1 << BIO_UPTODATE;
1903 		bio->bi_vcnt = 0;
1904 		bio->bi_idx = 0;
1905 		bio->bi_phys_segments = 0;
1906 		bio->bi_hw_segments = 0;
1907 		bio->bi_size = 0;
1908 	}
1909 
1910 	nr_sectors = 0;
1911 	if (sector_nr + max_sync < max_sector)
1912 		max_sector = sector_nr + max_sync;
1913 	do {
1914 		struct page *page;
1915 		int len = PAGE_SIZE;
1916 		disk = 0;
1917 		if (sector_nr + (len>>9) > max_sector)
1918 			len = (max_sector - sector_nr) << 9;
1919 		if (len == 0)
1920 			break;
1921 		for (bio= biolist ; bio ; bio=bio->bi_next) {
1922 			page = bio->bi_io_vec[bio->bi_vcnt].bv_page;
1923 			if (bio_add_page(bio, page, len, 0) == 0) {
1924 				/* stop here */
1925 				struct bio *bio2;
1926 				bio->bi_io_vec[bio->bi_vcnt].bv_page = page;
1927 				for (bio2 = biolist; bio2 && bio2 != bio; bio2 = bio2->bi_next) {
1928 					/* remove last page from this bio */
1929 					bio2->bi_vcnt--;
1930 					bio2->bi_size -= len;
1931 					bio2->bi_flags &= ~(1<< BIO_SEG_VALID);
1932 				}
1933 				goto bio_full;
1934 			}
1935 			disk = i;
1936 		}
1937 		nr_sectors += len>>9;
1938 		sector_nr += len>>9;
1939 	} while (biolist->bi_vcnt < RESYNC_PAGES);
1940  bio_full:
1941 	r10_bio->sectors = nr_sectors;
1942 
1943 	while (biolist) {
1944 		bio = biolist;
1945 		biolist = biolist->bi_next;
1946 
1947 		bio->bi_next = NULL;
1948 		r10_bio = bio->bi_private;
1949 		r10_bio->sectors = nr_sectors;
1950 
1951 		if (bio->bi_end_io == end_sync_read) {
1952 			md_sync_acct(bio->bi_bdev, nr_sectors);
1953 			generic_make_request(bio);
1954 		}
1955 	}
1956 
1957 	if (sectors_skipped)
1958 		/* pretend they weren't skipped, it makes
1959 		 * no important difference in this case
1960 		 */
1961 		md_done_sync(mddev, sectors_skipped, 1);
1962 
1963 	return sectors_skipped + nr_sectors;
1964  giveup:
1965 	/* There is nowhere to write, so all non-sync
1966 	 * drives must be failed, so try the next chunk...
1967 	 */
1968 	{
1969 	sector_t sec = max_sector - sector_nr;
1970 	sectors_skipped += sec;
1971 	chunks_skipped ++;
1972 	sector_nr = max_sector;
1973 	goto skipped;
1974 	}
1975 }
1976 
1977 static int run(mddev_t *mddev)
1978 {
1979 	conf_t *conf;
1980 	int i, disk_idx;
1981 	mirror_info_t *disk;
1982 	mdk_rdev_t *rdev;
1983 	struct list_head *tmp;
1984 	int nc, fc, fo;
1985 	sector_t stride, size;
1986 
1987 	if (mddev->chunk_size == 0) {
1988 		printk(KERN_ERR "md/raid10: non-zero chunk size required.\n");
1989 		return -EINVAL;
1990 	}
1991 
1992 	nc = mddev->layout & 255;
1993 	fc = (mddev->layout >> 8) & 255;
1994 	fo = mddev->layout & (1<<16);
1995 	if ((nc*fc) <2 || (nc*fc) > mddev->raid_disks ||
1996 	    (mddev->layout >> 17)) {
1997 		printk(KERN_ERR "raid10: %s: unsupported raid10 layout: 0x%8x\n",
1998 		       mdname(mddev), mddev->layout);
1999 		goto out;
2000 	}
2001 	/*
2002 	 * copy the already verified devices into our private RAID10
2003 	 * bookkeeping area. [whatever we allocate in run(),
2004 	 * should be freed in stop()]
2005 	 */
2006 	conf = kzalloc(sizeof(conf_t), GFP_KERNEL);
2007 	mddev->private = conf;
2008 	if (!conf) {
2009 		printk(KERN_ERR "raid10: couldn't allocate memory for %s\n",
2010 			mdname(mddev));
2011 		goto out;
2012 	}
2013 	conf->mirrors = kzalloc(sizeof(struct mirror_info)*mddev->raid_disks,
2014 				 GFP_KERNEL);
2015 	if (!conf->mirrors) {
2016 		printk(KERN_ERR "raid10: couldn't allocate memory for %s\n",
2017 		       mdname(mddev));
2018 		goto out_free_conf;
2019 	}
2020 
2021 	conf->tmppage = alloc_page(GFP_KERNEL);
2022 	if (!conf->tmppage)
2023 		goto out_free_conf;
2024 
2025 	conf->mddev = mddev;
2026 	conf->raid_disks = mddev->raid_disks;
2027 	conf->near_copies = nc;
2028 	conf->far_copies = fc;
2029 	conf->copies = nc*fc;
2030 	conf->far_offset = fo;
2031 	conf->chunk_mask = (sector_t)(mddev->chunk_size>>9)-1;
2032 	conf->chunk_shift = ffz(~mddev->chunk_size) - 9;
2033 	size = mddev->size >> (conf->chunk_shift-1);
2034 	sector_div(size, fc);
2035 	size = size * conf->raid_disks;
2036 	sector_div(size, nc);
2037 	/* 'size' is now the number of chunks in the array */
2038 	/* calculate "used chunks per device" in 'stride' */
2039 	stride = size * conf->copies;
2040 	sector_div(stride, conf->raid_disks);
2041 	mddev->size = stride  << (conf->chunk_shift-1);
2042 
2043 	if (fo)
2044 		stride = 1;
2045 	else
2046 		sector_div(stride, fc);
2047 	conf->stride = stride << conf->chunk_shift;
2048 
2049 	conf->r10bio_pool = mempool_create(NR_RAID10_BIOS, r10bio_pool_alloc,
2050 						r10bio_pool_free, conf);
2051 	if (!conf->r10bio_pool) {
2052 		printk(KERN_ERR "raid10: couldn't allocate memory for %s\n",
2053 			mdname(mddev));
2054 		goto out_free_conf;
2055 	}
2056 
2057 	ITERATE_RDEV(mddev, rdev, tmp) {
2058 		disk_idx = rdev->raid_disk;
2059 		if (disk_idx >= mddev->raid_disks
2060 		    || disk_idx < 0)
2061 			continue;
2062 		disk = conf->mirrors + disk_idx;
2063 
2064 		disk->rdev = rdev;
2065 
2066 		blk_queue_stack_limits(mddev->queue,
2067 				       rdev->bdev->bd_disk->queue);
2068 		/* as we don't honour merge_bvec_fn, we must never risk
2069 		 * violating it, so limit ->max_sector to one PAGE, as
2070 		 * a one page request is never in violation.
2071 		 */
2072 		if (rdev->bdev->bd_disk->queue->merge_bvec_fn &&
2073 		    mddev->queue->max_sectors > (PAGE_SIZE>>9))
2074 			mddev->queue->max_sectors = (PAGE_SIZE>>9);
2075 
2076 		disk->head_position = 0;
2077 	}
2078 	spin_lock_init(&conf->device_lock);
2079 	INIT_LIST_HEAD(&conf->retry_list);
2080 
2081 	spin_lock_init(&conf->resync_lock);
2082 	init_waitqueue_head(&conf->wait_barrier);
2083 
2084 	/* need to check that every block has at least one working mirror */
2085 	if (!enough(conf)) {
2086 		printk(KERN_ERR "raid10: not enough operational mirrors for %s\n",
2087 		       mdname(mddev));
2088 		goto out_free_conf;
2089 	}
2090 
2091 	mddev->degraded = 0;
2092 	for (i = 0; i < conf->raid_disks; i++) {
2093 
2094 		disk = conf->mirrors + i;
2095 
2096 		if (!disk->rdev ||
2097 		    !test_bit(In_sync, &disk->rdev->flags)) {
2098 			disk->head_position = 0;
2099 			mddev->degraded++;
2100 		}
2101 	}
2102 
2103 
2104 	mddev->thread = md_register_thread(raid10d, mddev, "%s_raid10");
2105 	if (!mddev->thread) {
2106 		printk(KERN_ERR
2107 		       "raid10: couldn't allocate thread for %s\n",
2108 		       mdname(mddev));
2109 		goto out_free_conf;
2110 	}
2111 
2112 	printk(KERN_INFO
2113 		"raid10: raid set %s active with %d out of %d devices\n",
2114 		mdname(mddev), mddev->raid_disks - mddev->degraded,
2115 		mddev->raid_disks);
2116 	/*
2117 	 * Ok, everything is just fine now
2118 	 */
2119 	mddev->array_size = size << (conf->chunk_shift-1);
2120 	mddev->resync_max_sectors = size << conf->chunk_shift;
2121 
2122 	mddev->queue->unplug_fn = raid10_unplug;
2123 	mddev->queue->issue_flush_fn = raid10_issue_flush;
2124 	mddev->queue->backing_dev_info.congested_fn = raid10_congested;
2125 	mddev->queue->backing_dev_info.congested_data = mddev;
2126 
2127 	/* Calculate max read-ahead size.
2128 	 * We need to readahead at least twice a whole stripe....
2129 	 * maybe...
2130 	 */
2131 	{
2132 		int stripe = conf->raid_disks * (mddev->chunk_size / PAGE_SIZE);
2133 		stripe /= conf->near_copies;
2134 		if (mddev->queue->backing_dev_info.ra_pages < 2* stripe)
2135 			mddev->queue->backing_dev_info.ra_pages = 2* stripe;
2136 	}
2137 
2138 	if (conf->near_copies < mddev->raid_disks)
2139 		blk_queue_merge_bvec(mddev->queue, raid10_mergeable_bvec);
2140 	return 0;
2141 
2142 out_free_conf:
2143 	if (conf->r10bio_pool)
2144 		mempool_destroy(conf->r10bio_pool);
2145 	safe_put_page(conf->tmppage);
2146 	kfree(conf->mirrors);
2147 	kfree(conf);
2148 	mddev->private = NULL;
2149 out:
2150 	return -EIO;
2151 }
2152 
2153 static int stop(mddev_t *mddev)
2154 {
2155 	conf_t *conf = mddev_to_conf(mddev);
2156 
2157 	md_unregister_thread(mddev->thread);
2158 	mddev->thread = NULL;
2159 	blk_sync_queue(mddev->queue); /* the unplug fn references 'conf'*/
2160 	if (conf->r10bio_pool)
2161 		mempool_destroy(conf->r10bio_pool);
2162 	kfree(conf->mirrors);
2163 	kfree(conf);
2164 	mddev->private = NULL;
2165 	return 0;
2166 }
2167 
2168 static void raid10_quiesce(mddev_t *mddev, int state)
2169 {
2170 	conf_t *conf = mddev_to_conf(mddev);
2171 
2172 	switch(state) {
2173 	case 1:
2174 		raise_barrier(conf, 0);
2175 		break;
2176 	case 0:
2177 		lower_barrier(conf);
2178 		break;
2179 	}
2180 	if (mddev->thread) {
2181 		if (mddev->bitmap)
2182 			mddev->thread->timeout = mddev->bitmap->daemon_sleep * HZ;
2183 		else
2184 			mddev->thread->timeout = MAX_SCHEDULE_TIMEOUT;
2185 		md_wakeup_thread(mddev->thread);
2186 	}
2187 }
2188 
2189 static struct mdk_personality raid10_personality =
2190 {
2191 	.name		= "raid10",
2192 	.level		= 10,
2193 	.owner		= THIS_MODULE,
2194 	.make_request	= make_request,
2195 	.run		= run,
2196 	.stop		= stop,
2197 	.status		= status,
2198 	.error_handler	= error,
2199 	.hot_add_disk	= raid10_add_disk,
2200 	.hot_remove_disk= raid10_remove_disk,
2201 	.spare_active	= raid10_spare_active,
2202 	.sync_request	= sync_request,
2203 	.quiesce	= raid10_quiesce,
2204 };
2205 
2206 static int __init raid_init(void)
2207 {
2208 	return register_md_personality(&raid10_personality);
2209 }
2210 
2211 static void raid_exit(void)
2212 {
2213 	unregister_md_personality(&raid10_personality);
2214 }
2215 
2216 module_init(raid_init);
2217 module_exit(raid_exit);
2218 MODULE_LICENSE("GPL");
2219 MODULE_ALIAS("md-personality-9"); /* RAID10 */
2220 MODULE_ALIAS("md-raid10");
2221 MODULE_ALIAS("md-level-10");
2222