1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (C) 2015 Shaohua Li <shli@fb.com>
4 * Copyright (C) 2016 Song Liu <songliubraving@fb.com>
5 */
6 #include <linux/kernel.h>
7 #include <linux/wait.h>
8 #include <linux/blkdev.h>
9 #include <linux/slab.h>
10 #include <linux/raid/md_p.h>
11 #include <linux/crc32c.h>
12 #include <linux/random.h>
13 #include <linux/kthread.h>
14 #include <linux/types.h>
15 #include "md.h"
16 #include "raid5.h"
17 #include "md-bitmap.h"
18 #include "raid5-log.h"
19
20 /*
21 * metadata/data stored in disk with 4k size unit (a block) regardless
22 * underneath hardware sector size. only works with PAGE_SIZE == 4096
23 */
24 #define BLOCK_SECTORS (8)
25 #define BLOCK_SECTOR_SHIFT (3)
26
27 /*
28 * log->max_free_space is min(1/4 disk size, 10G reclaimable space).
29 *
30 * In write through mode, the reclaim runs every log->max_free_space.
31 * This can prevent the recovery scans for too long
32 */
33 #define RECLAIM_MAX_FREE_SPACE (10 * 1024 * 1024 * 2) /* sector */
34 #define RECLAIM_MAX_FREE_SPACE_SHIFT (2)
35
36 /* wake up reclaim thread periodically */
37 #define R5C_RECLAIM_WAKEUP_INTERVAL (30 * HZ)
38 /* start flush with these full stripes */
39 #define R5C_FULL_STRIPE_FLUSH_BATCH(conf) (conf->max_nr_stripes / 4)
40 /* reclaim stripes in groups */
41 #define R5C_RECLAIM_STRIPE_GROUP (NR_STRIPE_HASH_LOCKS * 2)
42
43 /*
44 * We only need 2 bios per I/O unit to make progress, but ensure we
45 * have a few more available to not get too tight.
46 */
47 #define R5L_POOL_SIZE 4
48
49 static char *r5c_journal_mode_str[] = {"write-through",
50 "write-back"};
51 /*
52 * raid5 cache state machine
53 *
54 * With the RAID cache, each stripe works in two phases:
55 * - caching phase
56 * - writing-out phase
57 *
58 * These two phases are controlled by bit STRIPE_R5C_CACHING:
59 * if STRIPE_R5C_CACHING == 0, the stripe is in writing-out phase
60 * if STRIPE_R5C_CACHING == 1, the stripe is in caching phase
61 *
62 * When there is no journal, or the journal is in write-through mode,
63 * the stripe is always in writing-out phase.
64 *
65 * For write-back journal, the stripe is sent to caching phase on write
66 * (r5c_try_caching_write). r5c_make_stripe_write_out() kicks off
67 * the write-out phase by clearing STRIPE_R5C_CACHING.
68 *
69 * Stripes in caching phase do not write the raid disks. Instead, all
70 * writes are committed from the log device. Therefore, a stripe in
71 * caching phase handles writes as:
72 * - write to log device
73 * - return IO
74 *
75 * Stripes in writing-out phase handle writes as:
76 * - calculate parity
77 * - write pending data and parity to journal
78 * - write data and parity to raid disks
79 * - return IO for pending writes
80 */
81
82 struct r5l_log {
83 struct md_rdev *rdev;
84
85 u32 uuid_checksum;
86
87 sector_t device_size; /* log device size, round to
88 * BLOCK_SECTORS */
89 sector_t max_free_space; /* reclaim run if free space is at
90 * this size */
91
92 sector_t last_checkpoint; /* log tail. where recovery scan
93 * starts from */
94 u64 last_cp_seq; /* log tail sequence */
95
96 sector_t log_start; /* log head. where new data appends */
97 u64 seq; /* log head sequence */
98
99 sector_t next_checkpoint;
100
101 struct mutex io_mutex;
102 struct r5l_io_unit *current_io; /* current io_unit accepting new data */
103
104 spinlock_t io_list_lock;
105 struct list_head running_ios; /* io_units which are still running,
106 * and have not yet been completely
107 * written to the log */
108 struct list_head io_end_ios; /* io_units which have been completely
109 * written to the log but not yet written
110 * to the RAID */
111 struct list_head flushing_ios; /* io_units which are waiting for log
112 * cache flush */
113 struct list_head finished_ios; /* io_units which settle down in log disk */
114 struct bio flush_bio;
115
116 struct list_head no_mem_stripes; /* pending stripes, -ENOMEM */
117
118 struct kmem_cache *io_kc;
119 mempool_t io_pool;
120 struct bio_set bs;
121 mempool_t meta_pool;
122
123 struct md_thread __rcu *reclaim_thread;
124 unsigned long reclaim_target; /* number of space that need to be
125 * reclaimed. if it's 0, reclaim spaces
126 * used by io_units which are in
127 * IO_UNIT_STRIPE_END state (eg, reclaim
128 * doesn't wait for specific io_unit
129 * switching to IO_UNIT_STRIPE_END
130 * state) */
131 wait_queue_head_t iounit_wait;
132
133 struct list_head no_space_stripes; /* pending stripes, log has no space */
134 spinlock_t no_space_stripes_lock;
135
136 bool need_cache_flush;
137
138 /* for r5c_cache */
139 enum r5c_journal_mode r5c_journal_mode;
140
141 /* all stripes in r5cache, in the order of seq at sh->log_start */
142 struct list_head stripe_in_journal_list;
143
144 spinlock_t stripe_in_journal_lock;
145 atomic_t stripe_in_journal_count;
146
147 /* to submit async io_units, to fulfill ordering of flush */
148 struct work_struct deferred_io_work;
149 /* to disable write back during in degraded mode */
150 struct work_struct disable_writeback_work;
151
152 /* to for chunk_aligned_read in writeback mode, details below */
153 spinlock_t tree_lock;
154 struct radix_tree_root big_stripe_tree;
155 };
156
157 /*
158 * Enable chunk_aligned_read() with write back cache.
159 *
160 * Each chunk may contain more than one stripe (for example, a 256kB
161 * chunk contains 64 4kB-page, so this chunk contain 64 stripes). For
162 * chunk_aligned_read, these stripes are grouped into one "big_stripe".
163 * For each big_stripe, we count how many stripes of this big_stripe
164 * are in the write back cache. These data are tracked in a radix tree
165 * (big_stripe_tree). We use radix_tree item pointer as the counter.
166 * r5c_tree_index() is used to calculate keys for the radix tree.
167 *
168 * chunk_aligned_read() calls r5c_big_stripe_cached() to look up
169 * big_stripe of each chunk in the tree. If this big_stripe is in the
170 * tree, chunk_aligned_read() aborts. This look up is protected by
171 * rcu_read_lock().
172 *
173 * It is necessary to remember whether a stripe is counted in
174 * big_stripe_tree. Instead of adding new flag, we reuses existing flags:
175 * STRIPE_R5C_PARTIAL_STRIPE and STRIPE_R5C_FULL_STRIPE. If either of these
176 * two flags are set, the stripe is counted in big_stripe_tree. This
177 * requires moving set_bit(STRIPE_R5C_PARTIAL_STRIPE) to
178 * r5c_try_caching_write(); and moving clear_bit of
179 * STRIPE_R5C_PARTIAL_STRIPE and STRIPE_R5C_FULL_STRIPE to
180 * r5c_finish_stripe_write_out().
181 */
182
183 /*
184 * radix tree requests lowest 2 bits of data pointer to be 2b'00.
185 * So it is necessary to left shift the counter by 2 bits before using it
186 * as data pointer of the tree.
187 */
188 #define R5C_RADIX_COUNT_SHIFT 2
189
190 /*
191 * calculate key for big_stripe_tree
192 *
193 * sect: align_bi->bi_iter.bi_sector or sh->sector
194 */
r5c_tree_index(struct r5conf * conf,sector_t sect)195 static inline sector_t r5c_tree_index(struct r5conf *conf,
196 sector_t sect)
197 {
198 sector_div(sect, conf->chunk_sectors);
199 return sect;
200 }
201
202 /*
203 * an IO range starts from a meta data block and end at the next meta data
204 * block. The io unit's the meta data block tracks data/parity followed it. io
205 * unit is written to log disk with normal write, as we always flush log disk
206 * first and then start move data to raid disks, there is no requirement to
207 * write io unit with FLUSH/FUA
208 */
209 struct r5l_io_unit {
210 struct r5l_log *log;
211
212 struct page *meta_page; /* store meta block */
213 int meta_offset; /* current offset in meta_page */
214
215 struct bio *current_bio;/* current_bio accepting new data */
216
217 atomic_t pending_stripe;/* how many stripes not flushed to raid */
218 u64 seq; /* seq number of the metablock */
219 sector_t log_start; /* where the io_unit starts */
220 sector_t log_end; /* where the io_unit ends */
221 struct list_head log_sibling; /* log->running_ios */
222 struct list_head stripe_list; /* stripes added to the io_unit */
223
224 int state;
225 bool need_split_bio;
226 struct bio *split_bio;
227
228 unsigned int has_flush:1; /* include flush request */
229 unsigned int has_fua:1; /* include fua request */
230 unsigned int has_null_flush:1; /* include null flush request */
231 unsigned int has_flush_payload:1; /* include flush payload */
232 /*
233 * io isn't sent yet, flush/fua request can only be submitted till it's
234 * the first IO in running_ios list
235 */
236 unsigned int io_deferred:1;
237
238 struct bio_list flush_barriers; /* size == 0 flush bios */
239 };
240
241 /* r5l_io_unit state */
242 enum r5l_io_unit_state {
243 IO_UNIT_RUNNING = 0, /* accepting new IO */
244 IO_UNIT_IO_START = 1, /* io_unit bio start writing to log,
245 * don't accepting new bio */
246 IO_UNIT_IO_END = 2, /* io_unit bio finish writing to log */
247 IO_UNIT_STRIPE_END = 3, /* stripes data finished writing to raid */
248 };
249
r5c_is_writeback(struct r5l_log * log)250 bool r5c_is_writeback(struct r5l_log *log)
251 {
252 return (log != NULL &&
253 log->r5c_journal_mode == R5C_JOURNAL_MODE_WRITE_BACK);
254 }
255
r5l_ring_add(struct r5l_log * log,sector_t start,sector_t inc)256 static sector_t r5l_ring_add(struct r5l_log *log, sector_t start, sector_t inc)
257 {
258 start += inc;
259 if (start >= log->device_size)
260 start = start - log->device_size;
261 return start;
262 }
263
r5l_ring_distance(struct r5l_log * log,sector_t start,sector_t end)264 static sector_t r5l_ring_distance(struct r5l_log *log, sector_t start,
265 sector_t end)
266 {
267 if (end >= start)
268 return end - start;
269 else
270 return end + log->device_size - start;
271 }
272
r5l_has_free_space(struct r5l_log * log,sector_t size)273 static bool r5l_has_free_space(struct r5l_log *log, sector_t size)
274 {
275 sector_t used_size;
276
277 used_size = r5l_ring_distance(log, log->last_checkpoint,
278 log->log_start);
279
280 return log->device_size > used_size + size;
281 }
282
__r5l_set_io_unit_state(struct r5l_io_unit * io,enum r5l_io_unit_state state)283 static void __r5l_set_io_unit_state(struct r5l_io_unit *io,
284 enum r5l_io_unit_state state)
285 {
286 if (WARN_ON(io->state >= state))
287 return;
288 io->state = state;
289 }
290
291 static void
r5c_return_dev_pending_writes(struct r5conf * conf,struct r5dev * dev)292 r5c_return_dev_pending_writes(struct r5conf *conf, struct r5dev *dev)
293 {
294 struct bio *wbi, *wbi2;
295
296 wbi = dev->written;
297 dev->written = NULL;
298 while (wbi && wbi->bi_iter.bi_sector <
299 dev->sector + RAID5_STRIPE_SECTORS(conf)) {
300 wbi2 = r5_next_bio(conf, wbi, dev->sector);
301 md_write_end(conf->mddev);
302 bio_endio(wbi);
303 wbi = wbi2;
304 }
305 }
306
r5c_handle_cached_data_endio(struct r5conf * conf,struct stripe_head * sh,int disks)307 void r5c_handle_cached_data_endio(struct r5conf *conf,
308 struct stripe_head *sh, int disks)
309 {
310 int i;
311
312 for (i = sh->disks; i--; ) {
313 if (sh->dev[i].written) {
314 set_bit(R5_UPTODATE, &sh->dev[i].flags);
315 r5c_return_dev_pending_writes(conf, &sh->dev[i]);
316 }
317 }
318 }
319
320 void r5l_wake_reclaim(struct r5l_log *log, sector_t space);
321
322 /* Check whether we should flush some stripes to free up stripe cache */
r5c_check_stripe_cache_usage(struct r5conf * conf)323 void r5c_check_stripe_cache_usage(struct r5conf *conf)
324 {
325 int total_cached;
326 struct r5l_log *log = READ_ONCE(conf->log);
327
328 if (!r5c_is_writeback(log))
329 return;
330
331 total_cached = atomic_read(&conf->r5c_cached_partial_stripes) +
332 atomic_read(&conf->r5c_cached_full_stripes);
333
334 /*
335 * The following condition is true for either of the following:
336 * - stripe cache pressure high:
337 * total_cached > 3/4 min_nr_stripes ||
338 * empty_inactive_list_nr > 0
339 * - stripe cache pressure moderate:
340 * total_cached > 1/2 min_nr_stripes
341 */
342 if (total_cached > conf->min_nr_stripes * 1 / 2 ||
343 atomic_read(&conf->empty_inactive_list_nr) > 0)
344 r5l_wake_reclaim(log, 0);
345 }
346
347 /*
348 * flush cache when there are R5C_FULL_STRIPE_FLUSH_BATCH or more full
349 * stripes in the cache
350 */
r5c_check_cached_full_stripe(struct r5conf * conf)351 void r5c_check_cached_full_stripe(struct r5conf *conf)
352 {
353 struct r5l_log *log = READ_ONCE(conf->log);
354
355 if (!r5c_is_writeback(log))
356 return;
357
358 /*
359 * wake up reclaim for R5C_FULL_STRIPE_FLUSH_BATCH cached stripes
360 * or a full stripe (chunk size / 4k stripes).
361 */
362 if (atomic_read(&conf->r5c_cached_full_stripes) >=
363 min(R5C_FULL_STRIPE_FLUSH_BATCH(conf),
364 conf->chunk_sectors >> RAID5_STRIPE_SHIFT(conf)))
365 r5l_wake_reclaim(log, 0);
366 }
367
368 /*
369 * Total log space (in sectors) needed to flush all data in cache
370 *
371 * To avoid deadlock due to log space, it is necessary to reserve log
372 * space to flush critical stripes (stripes that occupying log space near
373 * last_checkpoint). This function helps check how much log space is
374 * required to flush all cached stripes.
375 *
376 * To reduce log space requirements, two mechanisms are used to give cache
377 * flush higher priorities:
378 * 1. In handle_stripe_dirtying() and schedule_reconstruction(),
379 * stripes ALREADY in journal can be flushed w/o pending writes;
380 * 2. In r5l_write_stripe() and r5c_cache_data(), stripes NOT in journal
381 * can be delayed (r5l_add_no_space_stripe).
382 *
383 * In cache flush, the stripe goes through 1 and then 2. For a stripe that
384 * already passed 1, flushing it requires at most (conf->max_degraded + 1)
385 * pages of journal space. For stripes that has not passed 1, flushing it
386 * requires (conf->raid_disks + 1) pages of journal space. There are at
387 * most (conf->group_cnt + 1) stripe that passed 1. So total journal space
388 * required to flush all cached stripes (in pages) is:
389 *
390 * (stripe_in_journal_count - group_cnt - 1) * (max_degraded + 1) +
391 * (group_cnt + 1) * (raid_disks + 1)
392 * or
393 * (stripe_in_journal_count) * (max_degraded + 1) +
394 * (group_cnt + 1) * (raid_disks - max_degraded)
395 */
r5c_log_required_to_flush_cache(struct r5conf * conf)396 static sector_t r5c_log_required_to_flush_cache(struct r5conf *conf)
397 {
398 struct r5l_log *log = READ_ONCE(conf->log);
399
400 if (!r5c_is_writeback(log))
401 return 0;
402
403 return BLOCK_SECTORS *
404 ((conf->max_degraded + 1) * atomic_read(&log->stripe_in_journal_count) +
405 (conf->raid_disks - conf->max_degraded) * (conf->group_cnt + 1));
406 }
407
408 /*
409 * evaluate log space usage and update R5C_LOG_TIGHT and R5C_LOG_CRITICAL
410 *
411 * R5C_LOG_TIGHT is set when free space on the log device is less than 3x of
412 * reclaim_required_space. R5C_LOG_CRITICAL is set when free space on the log
413 * device is less than 2x of reclaim_required_space.
414 */
r5c_update_log_state(struct r5l_log * log)415 static inline void r5c_update_log_state(struct r5l_log *log)
416 {
417 struct r5conf *conf = log->rdev->mddev->private;
418 sector_t free_space;
419 sector_t reclaim_space;
420 bool wake_reclaim = false;
421
422 if (!r5c_is_writeback(log))
423 return;
424
425 free_space = r5l_ring_distance(log, log->log_start,
426 log->last_checkpoint);
427 reclaim_space = r5c_log_required_to_flush_cache(conf);
428 if (free_space < 2 * reclaim_space)
429 set_bit(R5C_LOG_CRITICAL, &conf->cache_state);
430 else {
431 if (test_bit(R5C_LOG_CRITICAL, &conf->cache_state))
432 wake_reclaim = true;
433 clear_bit(R5C_LOG_CRITICAL, &conf->cache_state);
434 }
435 if (free_space < 3 * reclaim_space)
436 set_bit(R5C_LOG_TIGHT, &conf->cache_state);
437 else
438 clear_bit(R5C_LOG_TIGHT, &conf->cache_state);
439
440 if (wake_reclaim)
441 r5l_wake_reclaim(log, 0);
442 }
443
444 /*
445 * Put the stripe into writing-out phase by clearing STRIPE_R5C_CACHING.
446 * This function should only be called in write-back mode.
447 */
r5c_make_stripe_write_out(struct stripe_head * sh)448 void r5c_make_stripe_write_out(struct stripe_head *sh)
449 {
450 struct r5conf *conf = sh->raid_conf;
451 struct r5l_log *log = READ_ONCE(conf->log);
452
453 BUG_ON(!r5c_is_writeback(log));
454
455 WARN_ON(!test_bit(STRIPE_R5C_CACHING, &sh->state));
456 clear_bit(STRIPE_R5C_CACHING, &sh->state);
457
458 if (!test_and_set_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
459 atomic_inc(&conf->preread_active_stripes);
460 }
461
r5c_handle_data_cached(struct stripe_head * sh)462 static void r5c_handle_data_cached(struct stripe_head *sh)
463 {
464 int i;
465
466 for (i = sh->disks; i--; )
467 if (test_and_clear_bit(R5_Wantwrite, &sh->dev[i].flags)) {
468 set_bit(R5_InJournal, &sh->dev[i].flags);
469 clear_bit(R5_LOCKED, &sh->dev[i].flags);
470 }
471 clear_bit(STRIPE_LOG_TRAPPED, &sh->state);
472 }
473
474 /*
475 * this journal write must contain full parity,
476 * it may also contain some data pages
477 */
r5c_handle_parity_cached(struct stripe_head * sh)478 static void r5c_handle_parity_cached(struct stripe_head *sh)
479 {
480 int i;
481
482 for (i = sh->disks; i--; )
483 if (test_bit(R5_InJournal, &sh->dev[i].flags))
484 set_bit(R5_Wantwrite, &sh->dev[i].flags);
485 }
486
487 /*
488 * Setting proper flags after writing (or flushing) data and/or parity to the
489 * log device. This is called from r5l_log_endio() or r5l_log_flush_endio().
490 */
r5c_finish_cache_stripe(struct stripe_head * sh)491 static void r5c_finish_cache_stripe(struct stripe_head *sh)
492 {
493 struct r5l_log *log = READ_ONCE(sh->raid_conf->log);
494
495 if (log->r5c_journal_mode == R5C_JOURNAL_MODE_WRITE_THROUGH) {
496 BUG_ON(test_bit(STRIPE_R5C_CACHING, &sh->state));
497 /*
498 * Set R5_InJournal for parity dev[pd_idx]. This means
499 * all data AND parity in the journal. For RAID 6, it is
500 * NOT necessary to set the flag for dev[qd_idx], as the
501 * two parities are written out together.
502 */
503 set_bit(R5_InJournal, &sh->dev[sh->pd_idx].flags);
504 } else if (test_bit(STRIPE_R5C_CACHING, &sh->state)) {
505 r5c_handle_data_cached(sh);
506 } else {
507 r5c_handle_parity_cached(sh);
508 set_bit(R5_InJournal, &sh->dev[sh->pd_idx].flags);
509 }
510 }
511
r5l_io_run_stripes(struct r5l_io_unit * io)512 static void r5l_io_run_stripes(struct r5l_io_unit *io)
513 {
514 struct stripe_head *sh, *next;
515
516 list_for_each_entry_safe(sh, next, &io->stripe_list, log_list) {
517 list_del_init(&sh->log_list);
518
519 r5c_finish_cache_stripe(sh);
520
521 set_bit(STRIPE_HANDLE, &sh->state);
522 raid5_release_stripe(sh);
523 }
524 }
525
r5l_log_run_stripes(struct r5l_log * log)526 static void r5l_log_run_stripes(struct r5l_log *log)
527 {
528 struct r5l_io_unit *io, *next;
529
530 lockdep_assert_held(&log->io_list_lock);
531
532 list_for_each_entry_safe(io, next, &log->running_ios, log_sibling) {
533 /* don't change list order */
534 if (io->state < IO_UNIT_IO_END)
535 break;
536
537 list_move_tail(&io->log_sibling, &log->finished_ios);
538 r5l_io_run_stripes(io);
539 }
540 }
541
r5l_move_to_end_ios(struct r5l_log * log)542 static void r5l_move_to_end_ios(struct r5l_log *log)
543 {
544 struct r5l_io_unit *io, *next;
545
546 lockdep_assert_held(&log->io_list_lock);
547
548 list_for_each_entry_safe(io, next, &log->running_ios, log_sibling) {
549 /* don't change list order */
550 if (io->state < IO_UNIT_IO_END)
551 break;
552 list_move_tail(&io->log_sibling, &log->io_end_ios);
553 }
554 }
555
556 static void __r5l_stripe_write_finished(struct r5l_io_unit *io);
r5l_log_endio(struct bio * bio)557 static void r5l_log_endio(struct bio *bio)
558 {
559 struct r5l_io_unit *io = bio->bi_private;
560 struct r5l_io_unit *io_deferred;
561 struct r5l_log *log = io->log;
562 unsigned long flags;
563 bool has_null_flush;
564 bool has_flush_payload;
565
566 if (bio->bi_status)
567 md_error(log->rdev->mddev, log->rdev);
568
569 bio_put(bio);
570 mempool_free(io->meta_page, &log->meta_pool);
571
572 spin_lock_irqsave(&log->io_list_lock, flags);
573 __r5l_set_io_unit_state(io, IO_UNIT_IO_END);
574
575 /*
576 * if the io doesn't not have null_flush or flush payload,
577 * it is not safe to access it after releasing io_list_lock.
578 * Therefore, it is necessary to check the condition with
579 * the lock held.
580 */
581 has_null_flush = io->has_null_flush;
582 has_flush_payload = io->has_flush_payload;
583
584 if (log->need_cache_flush && !list_empty(&io->stripe_list))
585 r5l_move_to_end_ios(log);
586 else
587 r5l_log_run_stripes(log);
588 if (!list_empty(&log->running_ios)) {
589 /*
590 * FLUSH/FUA io_unit is deferred because of ordering, now we
591 * can dispatch it
592 */
593 io_deferred = list_first_entry(&log->running_ios,
594 struct r5l_io_unit, log_sibling);
595 if (io_deferred->io_deferred)
596 schedule_work(&log->deferred_io_work);
597 }
598
599 spin_unlock_irqrestore(&log->io_list_lock, flags);
600
601 if (log->need_cache_flush)
602 md_wakeup_thread(log->rdev->mddev->thread);
603
604 /* finish flush only io_unit and PAYLOAD_FLUSH only io_unit */
605 if (has_null_flush) {
606 struct bio *bi;
607
608 WARN_ON(bio_list_empty(&io->flush_barriers));
609 while ((bi = bio_list_pop(&io->flush_barriers)) != NULL) {
610 bio_endio(bi);
611 if (atomic_dec_and_test(&io->pending_stripe)) {
612 __r5l_stripe_write_finished(io);
613 return;
614 }
615 }
616 }
617 /* decrease pending_stripe for flush payload */
618 if (has_flush_payload)
619 if (atomic_dec_and_test(&io->pending_stripe))
620 __r5l_stripe_write_finished(io);
621 }
622
r5l_do_submit_io(struct r5l_log * log,struct r5l_io_unit * io)623 static void r5l_do_submit_io(struct r5l_log *log, struct r5l_io_unit *io)
624 {
625 unsigned long flags;
626
627 spin_lock_irqsave(&log->io_list_lock, flags);
628 __r5l_set_io_unit_state(io, IO_UNIT_IO_START);
629 spin_unlock_irqrestore(&log->io_list_lock, flags);
630
631 /*
632 * In case of journal device failures, submit_bio will get error
633 * and calls endio, then active stripes will continue write
634 * process. Therefore, it is not necessary to check Faulty bit
635 * of journal device here.
636 *
637 * We can't check split_bio after current_bio is submitted. If
638 * io->split_bio is null, after current_bio is submitted, current_bio
639 * might already be completed and the io_unit is freed. We submit
640 * split_bio first to avoid the issue.
641 */
642 if (io->split_bio) {
643 if (io->has_flush)
644 io->split_bio->bi_opf |= REQ_PREFLUSH;
645 if (io->has_fua)
646 io->split_bio->bi_opf |= REQ_FUA;
647 submit_bio(io->split_bio);
648 }
649
650 if (io->has_flush)
651 io->current_bio->bi_opf |= REQ_PREFLUSH;
652 if (io->has_fua)
653 io->current_bio->bi_opf |= REQ_FUA;
654 submit_bio(io->current_bio);
655 }
656
657 /* deferred io_unit will be dispatched here */
r5l_submit_io_async(struct work_struct * work)658 static void r5l_submit_io_async(struct work_struct *work)
659 {
660 struct r5l_log *log = container_of(work, struct r5l_log,
661 deferred_io_work);
662 struct r5l_io_unit *io = NULL;
663 unsigned long flags;
664
665 spin_lock_irqsave(&log->io_list_lock, flags);
666 if (!list_empty(&log->running_ios)) {
667 io = list_first_entry(&log->running_ios, struct r5l_io_unit,
668 log_sibling);
669 if (!io->io_deferred)
670 io = NULL;
671 else
672 io->io_deferred = 0;
673 }
674 spin_unlock_irqrestore(&log->io_list_lock, flags);
675 if (io)
676 r5l_do_submit_io(log, io);
677 }
678
r5c_disable_writeback_async(struct work_struct * work)679 static void r5c_disable_writeback_async(struct work_struct *work)
680 {
681 struct r5l_log *log = container_of(work, struct r5l_log,
682 disable_writeback_work);
683 struct mddev *mddev = log->rdev->mddev;
684 struct r5conf *conf = mddev->private;
685 int locked = 0;
686
687 if (log->r5c_journal_mode == R5C_JOURNAL_MODE_WRITE_THROUGH)
688 return;
689 pr_info("md/raid:%s: Disabling writeback cache for degraded array.\n",
690 mdname(mddev));
691
692 /* wait superblock change before suspend */
693 wait_event(mddev->sb_wait,
694 !READ_ONCE(conf->log) ||
695 (!test_bit(MD_SB_CHANGE_PENDING, &mddev->sb_flags) &&
696 (locked = mddev_trylock(mddev))));
697 if (locked) {
698 mddev_suspend(mddev);
699 log->r5c_journal_mode = R5C_JOURNAL_MODE_WRITE_THROUGH;
700 mddev_resume(mddev);
701 mddev_unlock(mddev);
702 }
703 }
704
r5l_submit_current_io(struct r5l_log * log)705 static void r5l_submit_current_io(struct r5l_log *log)
706 {
707 struct r5l_io_unit *io = log->current_io;
708 struct r5l_meta_block *block;
709 unsigned long flags;
710 u32 crc;
711 bool do_submit = true;
712
713 if (!io)
714 return;
715
716 block = page_address(io->meta_page);
717 block->meta_size = cpu_to_le32(io->meta_offset);
718 crc = crc32c_le(log->uuid_checksum, block, PAGE_SIZE);
719 block->checksum = cpu_to_le32(crc);
720
721 log->current_io = NULL;
722 spin_lock_irqsave(&log->io_list_lock, flags);
723 if (io->has_flush || io->has_fua) {
724 if (io != list_first_entry(&log->running_ios,
725 struct r5l_io_unit, log_sibling)) {
726 io->io_deferred = 1;
727 do_submit = false;
728 }
729 }
730 spin_unlock_irqrestore(&log->io_list_lock, flags);
731 if (do_submit)
732 r5l_do_submit_io(log, io);
733 }
734
r5l_bio_alloc(struct r5l_log * log)735 static struct bio *r5l_bio_alloc(struct r5l_log *log)
736 {
737 struct bio *bio = bio_alloc_bioset(log->rdev->bdev, BIO_MAX_VECS,
738 REQ_OP_WRITE, GFP_NOIO, &log->bs);
739
740 bio->bi_iter.bi_sector = log->rdev->data_offset + log->log_start;
741
742 return bio;
743 }
744
r5_reserve_log_entry(struct r5l_log * log,struct r5l_io_unit * io)745 static void r5_reserve_log_entry(struct r5l_log *log, struct r5l_io_unit *io)
746 {
747 log->log_start = r5l_ring_add(log, log->log_start, BLOCK_SECTORS);
748
749 r5c_update_log_state(log);
750 /*
751 * If we filled up the log device start from the beginning again,
752 * which will require a new bio.
753 *
754 * Note: for this to work properly the log size needs to me a multiple
755 * of BLOCK_SECTORS.
756 */
757 if (log->log_start == 0)
758 io->need_split_bio = true;
759
760 io->log_end = log->log_start;
761 }
762
r5l_new_meta(struct r5l_log * log)763 static struct r5l_io_unit *r5l_new_meta(struct r5l_log *log)
764 {
765 struct r5l_io_unit *io;
766 struct r5l_meta_block *block;
767
768 io = mempool_alloc(&log->io_pool, GFP_ATOMIC);
769 if (!io)
770 return NULL;
771 memset(io, 0, sizeof(*io));
772
773 io->log = log;
774 INIT_LIST_HEAD(&io->log_sibling);
775 INIT_LIST_HEAD(&io->stripe_list);
776 bio_list_init(&io->flush_barriers);
777 io->state = IO_UNIT_RUNNING;
778
779 io->meta_page = mempool_alloc(&log->meta_pool, GFP_NOIO);
780 block = page_address(io->meta_page);
781 clear_page(block);
782 block->magic = cpu_to_le32(R5LOG_MAGIC);
783 block->version = R5LOG_VERSION;
784 block->seq = cpu_to_le64(log->seq);
785 block->position = cpu_to_le64(log->log_start);
786
787 io->log_start = log->log_start;
788 io->meta_offset = sizeof(struct r5l_meta_block);
789 io->seq = log->seq++;
790
791 io->current_bio = r5l_bio_alloc(log);
792 io->current_bio->bi_end_io = r5l_log_endio;
793 io->current_bio->bi_private = io;
794 __bio_add_page(io->current_bio, io->meta_page, PAGE_SIZE, 0);
795
796 r5_reserve_log_entry(log, io);
797
798 spin_lock_irq(&log->io_list_lock);
799 list_add_tail(&io->log_sibling, &log->running_ios);
800 spin_unlock_irq(&log->io_list_lock);
801
802 return io;
803 }
804
r5l_get_meta(struct r5l_log * log,unsigned int payload_size)805 static int r5l_get_meta(struct r5l_log *log, unsigned int payload_size)
806 {
807 if (log->current_io &&
808 log->current_io->meta_offset + payload_size > PAGE_SIZE)
809 r5l_submit_current_io(log);
810
811 if (!log->current_io) {
812 log->current_io = r5l_new_meta(log);
813 if (!log->current_io)
814 return -ENOMEM;
815 }
816
817 return 0;
818 }
819
r5l_append_payload_meta(struct r5l_log * log,u16 type,sector_t location,u32 checksum1,u32 checksum2,bool checksum2_valid)820 static void r5l_append_payload_meta(struct r5l_log *log, u16 type,
821 sector_t location,
822 u32 checksum1, u32 checksum2,
823 bool checksum2_valid)
824 {
825 struct r5l_io_unit *io = log->current_io;
826 struct r5l_payload_data_parity *payload;
827
828 payload = page_address(io->meta_page) + io->meta_offset;
829 payload->header.type = cpu_to_le16(type);
830 payload->header.flags = cpu_to_le16(0);
831 payload->size = cpu_to_le32((1 + !!checksum2_valid) <<
832 (PAGE_SHIFT - 9));
833 payload->location = cpu_to_le64(location);
834 payload->checksum[0] = cpu_to_le32(checksum1);
835 if (checksum2_valid)
836 payload->checksum[1] = cpu_to_le32(checksum2);
837
838 io->meta_offset += sizeof(struct r5l_payload_data_parity) +
839 sizeof(__le32) * (1 + !!checksum2_valid);
840 }
841
r5l_append_payload_page(struct r5l_log * log,struct page * page)842 static void r5l_append_payload_page(struct r5l_log *log, struct page *page)
843 {
844 struct r5l_io_unit *io = log->current_io;
845
846 if (io->need_split_bio) {
847 BUG_ON(io->split_bio);
848 io->split_bio = io->current_bio;
849 io->current_bio = r5l_bio_alloc(log);
850 bio_chain(io->current_bio, io->split_bio);
851 io->need_split_bio = false;
852 }
853
854 if (!bio_add_page(io->current_bio, page, PAGE_SIZE, 0))
855 BUG();
856
857 r5_reserve_log_entry(log, io);
858 }
859
r5l_append_flush_payload(struct r5l_log * log,sector_t sect)860 static void r5l_append_flush_payload(struct r5l_log *log, sector_t sect)
861 {
862 struct mddev *mddev = log->rdev->mddev;
863 struct r5conf *conf = mddev->private;
864 struct r5l_io_unit *io;
865 struct r5l_payload_flush *payload;
866 int meta_size;
867
868 /*
869 * payload_flush requires extra writes to the journal.
870 * To avoid handling the extra IO in quiesce, just skip
871 * flush_payload
872 */
873 if (conf->quiesce)
874 return;
875
876 mutex_lock(&log->io_mutex);
877 meta_size = sizeof(struct r5l_payload_flush) + sizeof(__le64);
878
879 if (r5l_get_meta(log, meta_size)) {
880 mutex_unlock(&log->io_mutex);
881 return;
882 }
883
884 /* current implementation is one stripe per flush payload */
885 io = log->current_io;
886 payload = page_address(io->meta_page) + io->meta_offset;
887 payload->header.type = cpu_to_le16(R5LOG_PAYLOAD_FLUSH);
888 payload->header.flags = cpu_to_le16(0);
889 payload->size = cpu_to_le32(sizeof(__le64));
890 payload->flush_stripes[0] = cpu_to_le64(sect);
891 io->meta_offset += meta_size;
892 /* multiple flush payloads count as one pending_stripe */
893 if (!io->has_flush_payload) {
894 io->has_flush_payload = 1;
895 atomic_inc(&io->pending_stripe);
896 }
897 mutex_unlock(&log->io_mutex);
898 }
899
r5l_log_stripe(struct r5l_log * log,struct stripe_head * sh,int data_pages,int parity_pages)900 static int r5l_log_stripe(struct r5l_log *log, struct stripe_head *sh,
901 int data_pages, int parity_pages)
902 {
903 int i;
904 int meta_size;
905 int ret;
906 struct r5l_io_unit *io;
907
908 meta_size =
909 ((sizeof(struct r5l_payload_data_parity) + sizeof(__le32))
910 * data_pages) +
911 sizeof(struct r5l_payload_data_parity) +
912 sizeof(__le32) * parity_pages;
913
914 ret = r5l_get_meta(log, meta_size);
915 if (ret)
916 return ret;
917
918 io = log->current_io;
919
920 if (test_and_clear_bit(STRIPE_R5C_PREFLUSH, &sh->state))
921 io->has_flush = 1;
922
923 for (i = 0; i < sh->disks; i++) {
924 if (!test_bit(R5_Wantwrite, &sh->dev[i].flags) ||
925 test_bit(R5_InJournal, &sh->dev[i].flags))
926 continue;
927 if (i == sh->pd_idx || i == sh->qd_idx)
928 continue;
929 if (test_bit(R5_WantFUA, &sh->dev[i].flags) &&
930 log->r5c_journal_mode == R5C_JOURNAL_MODE_WRITE_BACK) {
931 io->has_fua = 1;
932 /*
933 * we need to flush journal to make sure recovery can
934 * reach the data with fua flag
935 */
936 io->has_flush = 1;
937 }
938 r5l_append_payload_meta(log, R5LOG_PAYLOAD_DATA,
939 raid5_compute_blocknr(sh, i, 0),
940 sh->dev[i].log_checksum, 0, false);
941 r5l_append_payload_page(log, sh->dev[i].page);
942 }
943
944 if (parity_pages == 2) {
945 r5l_append_payload_meta(log, R5LOG_PAYLOAD_PARITY,
946 sh->sector, sh->dev[sh->pd_idx].log_checksum,
947 sh->dev[sh->qd_idx].log_checksum, true);
948 r5l_append_payload_page(log, sh->dev[sh->pd_idx].page);
949 r5l_append_payload_page(log, sh->dev[sh->qd_idx].page);
950 } else if (parity_pages == 1) {
951 r5l_append_payload_meta(log, R5LOG_PAYLOAD_PARITY,
952 sh->sector, sh->dev[sh->pd_idx].log_checksum,
953 0, false);
954 r5l_append_payload_page(log, sh->dev[sh->pd_idx].page);
955 } else /* Just writing data, not parity, in caching phase */
956 BUG_ON(parity_pages != 0);
957
958 list_add_tail(&sh->log_list, &io->stripe_list);
959 atomic_inc(&io->pending_stripe);
960 sh->log_io = io;
961
962 if (log->r5c_journal_mode == R5C_JOURNAL_MODE_WRITE_THROUGH)
963 return 0;
964
965 if (sh->log_start == MaxSector) {
966 BUG_ON(!list_empty(&sh->r5c));
967 sh->log_start = io->log_start;
968 spin_lock_irq(&log->stripe_in_journal_lock);
969 list_add_tail(&sh->r5c,
970 &log->stripe_in_journal_list);
971 spin_unlock_irq(&log->stripe_in_journal_lock);
972 atomic_inc(&log->stripe_in_journal_count);
973 }
974 return 0;
975 }
976
977 /* add stripe to no_space_stripes, and then wake up reclaim */
r5l_add_no_space_stripe(struct r5l_log * log,struct stripe_head * sh)978 static inline void r5l_add_no_space_stripe(struct r5l_log *log,
979 struct stripe_head *sh)
980 {
981 spin_lock(&log->no_space_stripes_lock);
982 list_add_tail(&sh->log_list, &log->no_space_stripes);
983 spin_unlock(&log->no_space_stripes_lock);
984 }
985
986 /*
987 * running in raid5d, where reclaim could wait for raid5d too (when it flushes
988 * data from log to raid disks), so we shouldn't wait for reclaim here
989 */
r5l_write_stripe(struct r5l_log * log,struct stripe_head * sh)990 int r5l_write_stripe(struct r5l_log *log, struct stripe_head *sh)
991 {
992 struct r5conf *conf = sh->raid_conf;
993 int write_disks = 0;
994 int data_pages, parity_pages;
995 int reserve;
996 int i;
997 int ret = 0;
998 bool wake_reclaim = false;
999
1000 if (!log)
1001 return -EAGAIN;
1002 /* Don't support stripe batch */
1003 if (sh->log_io || !test_bit(R5_Wantwrite, &sh->dev[sh->pd_idx].flags) ||
1004 test_bit(STRIPE_SYNCING, &sh->state)) {
1005 /* the stripe is written to log, we start writing it to raid */
1006 clear_bit(STRIPE_LOG_TRAPPED, &sh->state);
1007 return -EAGAIN;
1008 }
1009
1010 WARN_ON(test_bit(STRIPE_R5C_CACHING, &sh->state));
1011
1012 for (i = 0; i < sh->disks; i++) {
1013 void *addr;
1014
1015 if (!test_bit(R5_Wantwrite, &sh->dev[i].flags) ||
1016 test_bit(R5_InJournal, &sh->dev[i].flags))
1017 continue;
1018
1019 write_disks++;
1020 /* checksum is already calculated in last run */
1021 if (test_bit(STRIPE_LOG_TRAPPED, &sh->state))
1022 continue;
1023 addr = kmap_atomic(sh->dev[i].page);
1024 sh->dev[i].log_checksum = crc32c_le(log->uuid_checksum,
1025 addr, PAGE_SIZE);
1026 kunmap_atomic(addr);
1027 }
1028 parity_pages = 1 + !!(sh->qd_idx >= 0);
1029 data_pages = write_disks - parity_pages;
1030
1031 set_bit(STRIPE_LOG_TRAPPED, &sh->state);
1032 /*
1033 * The stripe must enter state machine again to finish the write, so
1034 * don't delay.
1035 */
1036 clear_bit(STRIPE_DELAYED, &sh->state);
1037 atomic_inc(&sh->count);
1038
1039 mutex_lock(&log->io_mutex);
1040 /* meta + data */
1041 reserve = (1 + write_disks) << (PAGE_SHIFT - 9);
1042
1043 if (log->r5c_journal_mode == R5C_JOURNAL_MODE_WRITE_THROUGH) {
1044 if (!r5l_has_free_space(log, reserve)) {
1045 r5l_add_no_space_stripe(log, sh);
1046 wake_reclaim = true;
1047 } else {
1048 ret = r5l_log_stripe(log, sh, data_pages, parity_pages);
1049 if (ret) {
1050 spin_lock_irq(&log->io_list_lock);
1051 list_add_tail(&sh->log_list,
1052 &log->no_mem_stripes);
1053 spin_unlock_irq(&log->io_list_lock);
1054 }
1055 }
1056 } else { /* R5C_JOURNAL_MODE_WRITE_BACK */
1057 /*
1058 * log space critical, do not process stripes that are
1059 * not in cache yet (sh->log_start == MaxSector).
1060 */
1061 if (test_bit(R5C_LOG_CRITICAL, &conf->cache_state) &&
1062 sh->log_start == MaxSector) {
1063 r5l_add_no_space_stripe(log, sh);
1064 wake_reclaim = true;
1065 reserve = 0;
1066 } else if (!r5l_has_free_space(log, reserve)) {
1067 if (sh->log_start == log->last_checkpoint)
1068 BUG();
1069 else
1070 r5l_add_no_space_stripe(log, sh);
1071 } else {
1072 ret = r5l_log_stripe(log, sh, data_pages, parity_pages);
1073 if (ret) {
1074 spin_lock_irq(&log->io_list_lock);
1075 list_add_tail(&sh->log_list,
1076 &log->no_mem_stripes);
1077 spin_unlock_irq(&log->io_list_lock);
1078 }
1079 }
1080 }
1081
1082 mutex_unlock(&log->io_mutex);
1083 if (wake_reclaim)
1084 r5l_wake_reclaim(log, reserve);
1085 return 0;
1086 }
1087
r5l_write_stripe_run(struct r5l_log * log)1088 void r5l_write_stripe_run(struct r5l_log *log)
1089 {
1090 if (!log)
1091 return;
1092 mutex_lock(&log->io_mutex);
1093 r5l_submit_current_io(log);
1094 mutex_unlock(&log->io_mutex);
1095 }
1096
r5l_handle_flush_request(struct r5l_log * log,struct bio * bio)1097 int r5l_handle_flush_request(struct r5l_log *log, struct bio *bio)
1098 {
1099 if (log->r5c_journal_mode == R5C_JOURNAL_MODE_WRITE_THROUGH) {
1100 /*
1101 * in write through (journal only)
1102 * we flush log disk cache first, then write stripe data to
1103 * raid disks. So if bio is finished, the log disk cache is
1104 * flushed already. The recovery guarantees we can recovery
1105 * the bio from log disk, so we don't need to flush again
1106 */
1107 if (bio->bi_iter.bi_size == 0) {
1108 bio_endio(bio);
1109 return 0;
1110 }
1111 bio->bi_opf &= ~REQ_PREFLUSH;
1112 } else {
1113 /* write back (with cache) */
1114 if (bio->bi_iter.bi_size == 0) {
1115 mutex_lock(&log->io_mutex);
1116 r5l_get_meta(log, 0);
1117 bio_list_add(&log->current_io->flush_barriers, bio);
1118 log->current_io->has_flush = 1;
1119 log->current_io->has_null_flush = 1;
1120 atomic_inc(&log->current_io->pending_stripe);
1121 r5l_submit_current_io(log);
1122 mutex_unlock(&log->io_mutex);
1123 return 0;
1124 }
1125 }
1126 return -EAGAIN;
1127 }
1128
1129 /* This will run after log space is reclaimed */
r5l_run_no_space_stripes(struct r5l_log * log)1130 static void r5l_run_no_space_stripes(struct r5l_log *log)
1131 {
1132 struct stripe_head *sh;
1133
1134 spin_lock(&log->no_space_stripes_lock);
1135 while (!list_empty(&log->no_space_stripes)) {
1136 sh = list_first_entry(&log->no_space_stripes,
1137 struct stripe_head, log_list);
1138 list_del_init(&sh->log_list);
1139 set_bit(STRIPE_HANDLE, &sh->state);
1140 raid5_release_stripe(sh);
1141 }
1142 spin_unlock(&log->no_space_stripes_lock);
1143 }
1144
1145 /*
1146 * calculate new last_checkpoint
1147 * for write through mode, returns log->next_checkpoint
1148 * for write back, returns log_start of first sh in stripe_in_journal_list
1149 */
r5c_calculate_new_cp(struct r5conf * conf)1150 static sector_t r5c_calculate_new_cp(struct r5conf *conf)
1151 {
1152 struct stripe_head *sh;
1153 struct r5l_log *log = READ_ONCE(conf->log);
1154 sector_t new_cp;
1155 unsigned long flags;
1156
1157 if (log->r5c_journal_mode == R5C_JOURNAL_MODE_WRITE_THROUGH)
1158 return log->next_checkpoint;
1159
1160 spin_lock_irqsave(&log->stripe_in_journal_lock, flags);
1161 if (list_empty(&log->stripe_in_journal_list)) {
1162 /* all stripes flushed */
1163 spin_unlock_irqrestore(&log->stripe_in_journal_lock, flags);
1164 return log->next_checkpoint;
1165 }
1166 sh = list_first_entry(&log->stripe_in_journal_list,
1167 struct stripe_head, r5c);
1168 new_cp = sh->log_start;
1169 spin_unlock_irqrestore(&log->stripe_in_journal_lock, flags);
1170 return new_cp;
1171 }
1172
r5l_reclaimable_space(struct r5l_log * log)1173 static sector_t r5l_reclaimable_space(struct r5l_log *log)
1174 {
1175 struct r5conf *conf = log->rdev->mddev->private;
1176
1177 return r5l_ring_distance(log, log->last_checkpoint,
1178 r5c_calculate_new_cp(conf));
1179 }
1180
r5l_run_no_mem_stripe(struct r5l_log * log)1181 static void r5l_run_no_mem_stripe(struct r5l_log *log)
1182 {
1183 struct stripe_head *sh;
1184
1185 lockdep_assert_held(&log->io_list_lock);
1186
1187 if (!list_empty(&log->no_mem_stripes)) {
1188 sh = list_first_entry(&log->no_mem_stripes,
1189 struct stripe_head, log_list);
1190 list_del_init(&sh->log_list);
1191 set_bit(STRIPE_HANDLE, &sh->state);
1192 raid5_release_stripe(sh);
1193 }
1194 }
1195
r5l_complete_finished_ios(struct r5l_log * log)1196 static bool r5l_complete_finished_ios(struct r5l_log *log)
1197 {
1198 struct r5l_io_unit *io, *next;
1199 bool found = false;
1200
1201 lockdep_assert_held(&log->io_list_lock);
1202
1203 list_for_each_entry_safe(io, next, &log->finished_ios, log_sibling) {
1204 /* don't change list order */
1205 if (io->state < IO_UNIT_STRIPE_END)
1206 break;
1207
1208 log->next_checkpoint = io->log_start;
1209
1210 list_del(&io->log_sibling);
1211 mempool_free(io, &log->io_pool);
1212 r5l_run_no_mem_stripe(log);
1213
1214 found = true;
1215 }
1216
1217 return found;
1218 }
1219
__r5l_stripe_write_finished(struct r5l_io_unit * io)1220 static void __r5l_stripe_write_finished(struct r5l_io_unit *io)
1221 {
1222 struct r5l_log *log = io->log;
1223 struct r5conf *conf = log->rdev->mddev->private;
1224 unsigned long flags;
1225
1226 spin_lock_irqsave(&log->io_list_lock, flags);
1227 __r5l_set_io_unit_state(io, IO_UNIT_STRIPE_END);
1228
1229 if (!r5l_complete_finished_ios(log)) {
1230 spin_unlock_irqrestore(&log->io_list_lock, flags);
1231 return;
1232 }
1233
1234 if (r5l_reclaimable_space(log) > log->max_free_space ||
1235 test_bit(R5C_LOG_TIGHT, &conf->cache_state))
1236 r5l_wake_reclaim(log, 0);
1237
1238 spin_unlock_irqrestore(&log->io_list_lock, flags);
1239 wake_up(&log->iounit_wait);
1240 }
1241
r5l_stripe_write_finished(struct stripe_head * sh)1242 void r5l_stripe_write_finished(struct stripe_head *sh)
1243 {
1244 struct r5l_io_unit *io;
1245
1246 io = sh->log_io;
1247 sh->log_io = NULL;
1248
1249 if (io && atomic_dec_and_test(&io->pending_stripe))
1250 __r5l_stripe_write_finished(io);
1251 }
1252
r5l_log_flush_endio(struct bio * bio)1253 static void r5l_log_flush_endio(struct bio *bio)
1254 {
1255 struct r5l_log *log = container_of(bio, struct r5l_log,
1256 flush_bio);
1257 unsigned long flags;
1258 struct r5l_io_unit *io;
1259
1260 if (bio->bi_status)
1261 md_error(log->rdev->mddev, log->rdev);
1262 bio_uninit(bio);
1263
1264 spin_lock_irqsave(&log->io_list_lock, flags);
1265 list_for_each_entry(io, &log->flushing_ios, log_sibling)
1266 r5l_io_run_stripes(io);
1267 list_splice_tail_init(&log->flushing_ios, &log->finished_ios);
1268 spin_unlock_irqrestore(&log->io_list_lock, flags);
1269 }
1270
1271 /*
1272 * Starting dispatch IO to raid.
1273 * io_unit(meta) consists of a log. There is one situation we want to avoid. A
1274 * broken meta in the middle of a log causes recovery can't find meta at the
1275 * head of log. If operations require meta at the head persistent in log, we
1276 * must make sure meta before it persistent in log too. A case is:
1277 *
1278 * stripe data/parity is in log, we start write stripe to raid disks. stripe
1279 * data/parity must be persistent in log before we do the write to raid disks.
1280 *
1281 * The solution is we restrictly maintain io_unit list order. In this case, we
1282 * only write stripes of an io_unit to raid disks till the io_unit is the first
1283 * one whose data/parity is in log.
1284 */
r5l_flush_stripe_to_raid(struct r5l_log * log)1285 void r5l_flush_stripe_to_raid(struct r5l_log *log)
1286 {
1287 bool do_flush;
1288
1289 if (!log || !log->need_cache_flush)
1290 return;
1291
1292 spin_lock_irq(&log->io_list_lock);
1293 /* flush bio is running */
1294 if (!list_empty(&log->flushing_ios)) {
1295 spin_unlock_irq(&log->io_list_lock);
1296 return;
1297 }
1298 list_splice_tail_init(&log->io_end_ios, &log->flushing_ios);
1299 do_flush = !list_empty(&log->flushing_ios);
1300 spin_unlock_irq(&log->io_list_lock);
1301
1302 if (!do_flush)
1303 return;
1304 bio_init(&log->flush_bio, log->rdev->bdev, NULL, 0,
1305 REQ_OP_WRITE | REQ_PREFLUSH);
1306 log->flush_bio.bi_end_io = r5l_log_flush_endio;
1307 submit_bio(&log->flush_bio);
1308 }
1309
1310 static void r5l_write_super(struct r5l_log *log, sector_t cp);
r5l_write_super_and_discard_space(struct r5l_log * log,sector_t end)1311 static void r5l_write_super_and_discard_space(struct r5l_log *log,
1312 sector_t end)
1313 {
1314 struct block_device *bdev = log->rdev->bdev;
1315 struct mddev *mddev;
1316
1317 r5l_write_super(log, end);
1318
1319 if (!bdev_max_discard_sectors(bdev))
1320 return;
1321
1322 mddev = log->rdev->mddev;
1323 /*
1324 * Discard could zero data, so before discard we must make sure
1325 * superblock is updated to new log tail. Updating superblock (either
1326 * directly call md_update_sb() or depend on md thread) must hold
1327 * reconfig mutex. On the other hand, raid5_quiesce is called with
1328 * reconfig_mutex hold. The first step of raid5_quiesce() is waiting
1329 * for all IO finish, hence waiting for reclaim thread, while reclaim
1330 * thread is calling this function and waiting for reconfig mutex. So
1331 * there is a deadlock. We workaround this issue with a trylock.
1332 * FIXME: we could miss discard if we can't take reconfig mutex
1333 */
1334 set_mask_bits(&mddev->sb_flags, 0,
1335 BIT(MD_SB_CHANGE_DEVS) | BIT(MD_SB_CHANGE_PENDING));
1336 if (!mddev_trylock(mddev))
1337 return;
1338 md_update_sb(mddev, 1);
1339 mddev_unlock(mddev);
1340
1341 /* discard IO error really doesn't matter, ignore it */
1342 if (log->last_checkpoint < end) {
1343 blkdev_issue_discard(bdev,
1344 log->last_checkpoint + log->rdev->data_offset,
1345 end - log->last_checkpoint, GFP_NOIO);
1346 } else {
1347 blkdev_issue_discard(bdev,
1348 log->last_checkpoint + log->rdev->data_offset,
1349 log->device_size - log->last_checkpoint,
1350 GFP_NOIO);
1351 blkdev_issue_discard(bdev, log->rdev->data_offset, end,
1352 GFP_NOIO);
1353 }
1354 }
1355
1356 /*
1357 * r5c_flush_stripe moves stripe from cached list to handle_list. When called,
1358 * the stripe must be on r5c_cached_full_stripes or r5c_cached_partial_stripes.
1359 *
1360 * must hold conf->device_lock
1361 */
r5c_flush_stripe(struct r5conf * conf,struct stripe_head * sh)1362 static void r5c_flush_stripe(struct r5conf *conf, struct stripe_head *sh)
1363 {
1364 BUG_ON(list_empty(&sh->lru));
1365 BUG_ON(!test_bit(STRIPE_R5C_CACHING, &sh->state));
1366 BUG_ON(test_bit(STRIPE_HANDLE, &sh->state));
1367
1368 /*
1369 * The stripe is not ON_RELEASE_LIST, so it is safe to call
1370 * raid5_release_stripe() while holding conf->device_lock
1371 */
1372 BUG_ON(test_bit(STRIPE_ON_RELEASE_LIST, &sh->state));
1373 lockdep_assert_held(&conf->device_lock);
1374
1375 list_del_init(&sh->lru);
1376 atomic_inc(&sh->count);
1377
1378 set_bit(STRIPE_HANDLE, &sh->state);
1379 atomic_inc(&conf->active_stripes);
1380 r5c_make_stripe_write_out(sh);
1381
1382 if (test_bit(STRIPE_R5C_PARTIAL_STRIPE, &sh->state))
1383 atomic_inc(&conf->r5c_flushing_partial_stripes);
1384 else
1385 atomic_inc(&conf->r5c_flushing_full_stripes);
1386 raid5_release_stripe(sh);
1387 }
1388
1389 /*
1390 * if num == 0, flush all full stripes
1391 * if num > 0, flush all full stripes. If less than num full stripes are
1392 * flushed, flush some partial stripes until totally num stripes are
1393 * flushed or there is no more cached stripes.
1394 */
r5c_flush_cache(struct r5conf * conf,int num)1395 void r5c_flush_cache(struct r5conf *conf, int num)
1396 {
1397 int count;
1398 struct stripe_head *sh, *next;
1399
1400 lockdep_assert_held(&conf->device_lock);
1401 if (!READ_ONCE(conf->log))
1402 return;
1403
1404 count = 0;
1405 list_for_each_entry_safe(sh, next, &conf->r5c_full_stripe_list, lru) {
1406 r5c_flush_stripe(conf, sh);
1407 count++;
1408 }
1409
1410 if (count >= num)
1411 return;
1412 list_for_each_entry_safe(sh, next,
1413 &conf->r5c_partial_stripe_list, lru) {
1414 r5c_flush_stripe(conf, sh);
1415 if (++count >= num)
1416 break;
1417 }
1418 }
1419
r5c_do_reclaim(struct r5conf * conf)1420 static void r5c_do_reclaim(struct r5conf *conf)
1421 {
1422 struct r5l_log *log = READ_ONCE(conf->log);
1423 struct stripe_head *sh;
1424 int count = 0;
1425 unsigned long flags;
1426 int total_cached;
1427 int stripes_to_flush;
1428 int flushing_partial, flushing_full;
1429
1430 if (!r5c_is_writeback(log))
1431 return;
1432
1433 flushing_partial = atomic_read(&conf->r5c_flushing_partial_stripes);
1434 flushing_full = atomic_read(&conf->r5c_flushing_full_stripes);
1435 total_cached = atomic_read(&conf->r5c_cached_partial_stripes) +
1436 atomic_read(&conf->r5c_cached_full_stripes) -
1437 flushing_full - flushing_partial;
1438
1439 if (total_cached > conf->min_nr_stripes * 3 / 4 ||
1440 atomic_read(&conf->empty_inactive_list_nr) > 0)
1441 /*
1442 * if stripe cache pressure high, flush all full stripes and
1443 * some partial stripes
1444 */
1445 stripes_to_flush = R5C_RECLAIM_STRIPE_GROUP;
1446 else if (total_cached > conf->min_nr_stripes * 1 / 2 ||
1447 atomic_read(&conf->r5c_cached_full_stripes) - flushing_full >
1448 R5C_FULL_STRIPE_FLUSH_BATCH(conf))
1449 /*
1450 * if stripe cache pressure moderate, or if there is many full
1451 * stripes,flush all full stripes
1452 */
1453 stripes_to_flush = 0;
1454 else
1455 /* no need to flush */
1456 stripes_to_flush = -1;
1457
1458 if (stripes_to_flush >= 0) {
1459 spin_lock_irqsave(&conf->device_lock, flags);
1460 r5c_flush_cache(conf, stripes_to_flush);
1461 spin_unlock_irqrestore(&conf->device_lock, flags);
1462 }
1463
1464 /* if log space is tight, flush stripes on stripe_in_journal_list */
1465 if (test_bit(R5C_LOG_TIGHT, &conf->cache_state)) {
1466 spin_lock_irqsave(&log->stripe_in_journal_lock, flags);
1467 spin_lock(&conf->device_lock);
1468 list_for_each_entry(sh, &log->stripe_in_journal_list, r5c) {
1469 /*
1470 * stripes on stripe_in_journal_list could be in any
1471 * state of the stripe_cache state machine. In this
1472 * case, we only want to flush stripe on
1473 * r5c_cached_full/partial_stripes. The following
1474 * condition makes sure the stripe is on one of the
1475 * two lists.
1476 */
1477 if (!list_empty(&sh->lru) &&
1478 !test_bit(STRIPE_HANDLE, &sh->state) &&
1479 atomic_read(&sh->count) == 0) {
1480 r5c_flush_stripe(conf, sh);
1481 if (count++ >= R5C_RECLAIM_STRIPE_GROUP)
1482 break;
1483 }
1484 }
1485 spin_unlock(&conf->device_lock);
1486 spin_unlock_irqrestore(&log->stripe_in_journal_lock, flags);
1487 }
1488
1489 if (!test_bit(R5C_LOG_CRITICAL, &conf->cache_state))
1490 r5l_run_no_space_stripes(log);
1491
1492 md_wakeup_thread(conf->mddev->thread);
1493 }
1494
r5l_do_reclaim(struct r5l_log * log)1495 static void r5l_do_reclaim(struct r5l_log *log)
1496 {
1497 struct r5conf *conf = log->rdev->mddev->private;
1498 sector_t reclaim_target = xchg(&log->reclaim_target, 0);
1499 sector_t reclaimable;
1500 sector_t next_checkpoint;
1501 bool write_super;
1502
1503 spin_lock_irq(&log->io_list_lock);
1504 write_super = r5l_reclaimable_space(log) > log->max_free_space ||
1505 reclaim_target != 0 || !list_empty(&log->no_space_stripes);
1506 /*
1507 * move proper io_unit to reclaim list. We should not change the order.
1508 * reclaimable/unreclaimable io_unit can be mixed in the list, we
1509 * shouldn't reuse space of an unreclaimable io_unit
1510 */
1511 while (1) {
1512 reclaimable = r5l_reclaimable_space(log);
1513 if (reclaimable >= reclaim_target ||
1514 (list_empty(&log->running_ios) &&
1515 list_empty(&log->io_end_ios) &&
1516 list_empty(&log->flushing_ios) &&
1517 list_empty(&log->finished_ios)))
1518 break;
1519
1520 md_wakeup_thread(log->rdev->mddev->thread);
1521 wait_event_lock_irq(log->iounit_wait,
1522 r5l_reclaimable_space(log) > reclaimable,
1523 log->io_list_lock);
1524 }
1525
1526 next_checkpoint = r5c_calculate_new_cp(conf);
1527 spin_unlock_irq(&log->io_list_lock);
1528
1529 if (reclaimable == 0 || !write_super)
1530 return;
1531
1532 /*
1533 * write_super will flush cache of each raid disk. We must write super
1534 * here, because the log area might be reused soon and we don't want to
1535 * confuse recovery
1536 */
1537 r5l_write_super_and_discard_space(log, next_checkpoint);
1538
1539 mutex_lock(&log->io_mutex);
1540 log->last_checkpoint = next_checkpoint;
1541 r5c_update_log_state(log);
1542 mutex_unlock(&log->io_mutex);
1543
1544 r5l_run_no_space_stripes(log);
1545 }
1546
r5l_reclaim_thread(struct md_thread * thread)1547 static void r5l_reclaim_thread(struct md_thread *thread)
1548 {
1549 struct mddev *mddev = thread->mddev;
1550 struct r5conf *conf = mddev->private;
1551 struct r5l_log *log = READ_ONCE(conf->log);
1552
1553 if (!log)
1554 return;
1555 r5c_do_reclaim(conf);
1556 r5l_do_reclaim(log);
1557 }
1558
r5l_wake_reclaim(struct r5l_log * log,sector_t space)1559 void r5l_wake_reclaim(struct r5l_log *log, sector_t space)
1560 {
1561 unsigned long target;
1562 unsigned long new = (unsigned long)space; /* overflow in theory */
1563
1564 if (!log)
1565 return;
1566
1567 target = READ_ONCE(log->reclaim_target);
1568 do {
1569 if (new < target)
1570 return;
1571 } while (!try_cmpxchg(&log->reclaim_target, &target, new));
1572 md_wakeup_thread(log->reclaim_thread);
1573 }
1574
r5l_quiesce(struct r5l_log * log,int quiesce)1575 void r5l_quiesce(struct r5l_log *log, int quiesce)
1576 {
1577 struct mddev *mddev = log->rdev->mddev;
1578 struct md_thread *thread = rcu_dereference_protected(
1579 log->reclaim_thread, lockdep_is_held(&mddev->reconfig_mutex));
1580
1581 if (quiesce) {
1582 /* make sure r5l_write_super_and_discard_space exits */
1583 wake_up(&mddev->sb_wait);
1584 kthread_park(thread->tsk);
1585 r5l_wake_reclaim(log, MaxSector);
1586 r5l_do_reclaim(log);
1587 } else
1588 kthread_unpark(thread->tsk);
1589 }
1590
r5l_log_disk_error(struct r5conf * conf)1591 bool r5l_log_disk_error(struct r5conf *conf)
1592 {
1593 struct r5l_log *log = READ_ONCE(conf->log);
1594
1595 /* don't allow write if journal disk is missing */
1596 if (!log)
1597 return test_bit(MD_HAS_JOURNAL, &conf->mddev->flags);
1598 else
1599 return test_bit(Faulty, &log->rdev->flags);
1600 }
1601
1602 #define R5L_RECOVERY_PAGE_POOL_SIZE 256
1603
1604 struct r5l_recovery_ctx {
1605 struct page *meta_page; /* current meta */
1606 sector_t meta_total_blocks; /* total size of current meta and data */
1607 sector_t pos; /* recovery position */
1608 u64 seq; /* recovery position seq */
1609 int data_parity_stripes; /* number of data_parity stripes */
1610 int data_only_stripes; /* number of data_only stripes */
1611 struct list_head cached_list;
1612
1613 /*
1614 * read ahead page pool (ra_pool)
1615 * in recovery, log is read sequentially. It is not efficient to
1616 * read every page with sync_page_io(). The read ahead page pool
1617 * reads multiple pages with one IO, so further log read can
1618 * just copy data from the pool.
1619 */
1620 struct page *ra_pool[R5L_RECOVERY_PAGE_POOL_SIZE];
1621 struct bio_vec ra_bvec[R5L_RECOVERY_PAGE_POOL_SIZE];
1622 sector_t pool_offset; /* offset of first page in the pool */
1623 int total_pages; /* total allocated pages */
1624 int valid_pages; /* pages with valid data */
1625 };
1626
r5l_recovery_allocate_ra_pool(struct r5l_log * log,struct r5l_recovery_ctx * ctx)1627 static int r5l_recovery_allocate_ra_pool(struct r5l_log *log,
1628 struct r5l_recovery_ctx *ctx)
1629 {
1630 struct page *page;
1631
1632 ctx->valid_pages = 0;
1633 ctx->total_pages = 0;
1634 while (ctx->total_pages < R5L_RECOVERY_PAGE_POOL_SIZE) {
1635 page = alloc_page(GFP_KERNEL);
1636
1637 if (!page)
1638 break;
1639 ctx->ra_pool[ctx->total_pages] = page;
1640 ctx->total_pages += 1;
1641 }
1642
1643 if (ctx->total_pages == 0)
1644 return -ENOMEM;
1645
1646 ctx->pool_offset = 0;
1647 return 0;
1648 }
1649
r5l_recovery_free_ra_pool(struct r5l_log * log,struct r5l_recovery_ctx * ctx)1650 static void r5l_recovery_free_ra_pool(struct r5l_log *log,
1651 struct r5l_recovery_ctx *ctx)
1652 {
1653 int i;
1654
1655 for (i = 0; i < ctx->total_pages; ++i)
1656 put_page(ctx->ra_pool[i]);
1657 }
1658
1659 /*
1660 * fetch ctx->valid_pages pages from offset
1661 * In normal cases, ctx->valid_pages == ctx->total_pages after the call.
1662 * However, if the offset is close to the end of the journal device,
1663 * ctx->valid_pages could be smaller than ctx->total_pages
1664 */
r5l_recovery_fetch_ra_pool(struct r5l_log * log,struct r5l_recovery_ctx * ctx,sector_t offset)1665 static int r5l_recovery_fetch_ra_pool(struct r5l_log *log,
1666 struct r5l_recovery_ctx *ctx,
1667 sector_t offset)
1668 {
1669 struct bio bio;
1670 int ret;
1671
1672 bio_init(&bio, log->rdev->bdev, ctx->ra_bvec,
1673 R5L_RECOVERY_PAGE_POOL_SIZE, REQ_OP_READ);
1674 bio.bi_iter.bi_sector = log->rdev->data_offset + offset;
1675
1676 ctx->valid_pages = 0;
1677 ctx->pool_offset = offset;
1678
1679 while (ctx->valid_pages < ctx->total_pages) {
1680 __bio_add_page(&bio, ctx->ra_pool[ctx->valid_pages], PAGE_SIZE,
1681 0);
1682 ctx->valid_pages += 1;
1683
1684 offset = r5l_ring_add(log, offset, BLOCK_SECTORS);
1685
1686 if (offset == 0) /* reached end of the device */
1687 break;
1688 }
1689
1690 ret = submit_bio_wait(&bio);
1691 bio_uninit(&bio);
1692 return ret;
1693 }
1694
1695 /*
1696 * try read a page from the read ahead page pool, if the page is not in the
1697 * pool, call r5l_recovery_fetch_ra_pool
1698 */
r5l_recovery_read_page(struct r5l_log * log,struct r5l_recovery_ctx * ctx,struct page * page,sector_t offset)1699 static int r5l_recovery_read_page(struct r5l_log *log,
1700 struct r5l_recovery_ctx *ctx,
1701 struct page *page,
1702 sector_t offset)
1703 {
1704 int ret;
1705
1706 if (offset < ctx->pool_offset ||
1707 offset >= ctx->pool_offset + ctx->valid_pages * BLOCK_SECTORS) {
1708 ret = r5l_recovery_fetch_ra_pool(log, ctx, offset);
1709 if (ret)
1710 return ret;
1711 }
1712
1713 BUG_ON(offset < ctx->pool_offset ||
1714 offset >= ctx->pool_offset + ctx->valid_pages * BLOCK_SECTORS);
1715
1716 memcpy(page_address(page),
1717 page_address(ctx->ra_pool[(offset - ctx->pool_offset) >>
1718 BLOCK_SECTOR_SHIFT]),
1719 PAGE_SIZE);
1720 return 0;
1721 }
1722
r5l_recovery_read_meta_block(struct r5l_log * log,struct r5l_recovery_ctx * ctx)1723 static int r5l_recovery_read_meta_block(struct r5l_log *log,
1724 struct r5l_recovery_ctx *ctx)
1725 {
1726 struct page *page = ctx->meta_page;
1727 struct r5l_meta_block *mb;
1728 u32 crc, stored_crc;
1729 int ret;
1730
1731 ret = r5l_recovery_read_page(log, ctx, page, ctx->pos);
1732 if (ret != 0)
1733 return ret;
1734
1735 mb = page_address(page);
1736 stored_crc = le32_to_cpu(mb->checksum);
1737 mb->checksum = 0;
1738
1739 if (le32_to_cpu(mb->magic) != R5LOG_MAGIC ||
1740 le64_to_cpu(mb->seq) != ctx->seq ||
1741 mb->version != R5LOG_VERSION ||
1742 le64_to_cpu(mb->position) != ctx->pos)
1743 return -EINVAL;
1744
1745 crc = crc32c_le(log->uuid_checksum, mb, PAGE_SIZE);
1746 if (stored_crc != crc)
1747 return -EINVAL;
1748
1749 if (le32_to_cpu(mb->meta_size) > PAGE_SIZE)
1750 return -EINVAL;
1751
1752 ctx->meta_total_blocks = BLOCK_SECTORS;
1753
1754 return 0;
1755 }
1756
1757 static void
r5l_recovery_create_empty_meta_block(struct r5l_log * log,struct page * page,sector_t pos,u64 seq)1758 r5l_recovery_create_empty_meta_block(struct r5l_log *log,
1759 struct page *page,
1760 sector_t pos, u64 seq)
1761 {
1762 struct r5l_meta_block *mb;
1763
1764 mb = page_address(page);
1765 clear_page(mb);
1766 mb->magic = cpu_to_le32(R5LOG_MAGIC);
1767 mb->version = R5LOG_VERSION;
1768 mb->meta_size = cpu_to_le32(sizeof(struct r5l_meta_block));
1769 mb->seq = cpu_to_le64(seq);
1770 mb->position = cpu_to_le64(pos);
1771 }
1772
r5l_log_write_empty_meta_block(struct r5l_log * log,sector_t pos,u64 seq)1773 static int r5l_log_write_empty_meta_block(struct r5l_log *log, sector_t pos,
1774 u64 seq)
1775 {
1776 struct page *page;
1777 struct r5l_meta_block *mb;
1778
1779 page = alloc_page(GFP_KERNEL);
1780 if (!page)
1781 return -ENOMEM;
1782 r5l_recovery_create_empty_meta_block(log, page, pos, seq);
1783 mb = page_address(page);
1784 mb->checksum = cpu_to_le32(crc32c_le(log->uuid_checksum,
1785 mb, PAGE_SIZE));
1786 if (!sync_page_io(log->rdev, pos, PAGE_SIZE, page, REQ_OP_WRITE |
1787 REQ_SYNC | REQ_FUA, false)) {
1788 __free_page(page);
1789 return -EIO;
1790 }
1791 __free_page(page);
1792 return 0;
1793 }
1794
1795 /*
1796 * r5l_recovery_load_data and r5l_recovery_load_parity uses flag R5_Wantwrite
1797 * to mark valid (potentially not flushed) data in the journal.
1798 *
1799 * We already verified checksum in r5l_recovery_verify_data_checksum_for_mb,
1800 * so there should not be any mismatch here.
1801 */
r5l_recovery_load_data(struct r5l_log * log,struct stripe_head * sh,struct r5l_recovery_ctx * ctx,struct r5l_payload_data_parity * payload,sector_t log_offset)1802 static void r5l_recovery_load_data(struct r5l_log *log,
1803 struct stripe_head *sh,
1804 struct r5l_recovery_ctx *ctx,
1805 struct r5l_payload_data_parity *payload,
1806 sector_t log_offset)
1807 {
1808 struct mddev *mddev = log->rdev->mddev;
1809 struct r5conf *conf = mddev->private;
1810 int dd_idx;
1811
1812 raid5_compute_sector(conf,
1813 le64_to_cpu(payload->location), 0,
1814 &dd_idx, sh);
1815 r5l_recovery_read_page(log, ctx, sh->dev[dd_idx].page, log_offset);
1816 sh->dev[dd_idx].log_checksum =
1817 le32_to_cpu(payload->checksum[0]);
1818 ctx->meta_total_blocks += BLOCK_SECTORS;
1819
1820 set_bit(R5_Wantwrite, &sh->dev[dd_idx].flags);
1821 set_bit(STRIPE_R5C_CACHING, &sh->state);
1822 }
1823
r5l_recovery_load_parity(struct r5l_log * log,struct stripe_head * sh,struct r5l_recovery_ctx * ctx,struct r5l_payload_data_parity * payload,sector_t log_offset)1824 static void r5l_recovery_load_parity(struct r5l_log *log,
1825 struct stripe_head *sh,
1826 struct r5l_recovery_ctx *ctx,
1827 struct r5l_payload_data_parity *payload,
1828 sector_t log_offset)
1829 {
1830 struct mddev *mddev = log->rdev->mddev;
1831 struct r5conf *conf = mddev->private;
1832
1833 ctx->meta_total_blocks += BLOCK_SECTORS * conf->max_degraded;
1834 r5l_recovery_read_page(log, ctx, sh->dev[sh->pd_idx].page, log_offset);
1835 sh->dev[sh->pd_idx].log_checksum =
1836 le32_to_cpu(payload->checksum[0]);
1837 set_bit(R5_Wantwrite, &sh->dev[sh->pd_idx].flags);
1838
1839 if (sh->qd_idx >= 0) {
1840 r5l_recovery_read_page(
1841 log, ctx, sh->dev[sh->qd_idx].page,
1842 r5l_ring_add(log, log_offset, BLOCK_SECTORS));
1843 sh->dev[sh->qd_idx].log_checksum =
1844 le32_to_cpu(payload->checksum[1]);
1845 set_bit(R5_Wantwrite, &sh->dev[sh->qd_idx].flags);
1846 }
1847 clear_bit(STRIPE_R5C_CACHING, &sh->state);
1848 }
1849
r5l_recovery_reset_stripe(struct stripe_head * sh)1850 static void r5l_recovery_reset_stripe(struct stripe_head *sh)
1851 {
1852 int i;
1853
1854 sh->state = 0;
1855 sh->log_start = MaxSector;
1856 for (i = sh->disks; i--; )
1857 sh->dev[i].flags = 0;
1858 }
1859
1860 static void
r5l_recovery_replay_one_stripe(struct r5conf * conf,struct stripe_head * sh,struct r5l_recovery_ctx * ctx)1861 r5l_recovery_replay_one_stripe(struct r5conf *conf,
1862 struct stripe_head *sh,
1863 struct r5l_recovery_ctx *ctx)
1864 {
1865 struct md_rdev *rdev, *rrdev;
1866 int disk_index;
1867 int data_count = 0;
1868
1869 for (disk_index = 0; disk_index < sh->disks; disk_index++) {
1870 if (!test_bit(R5_Wantwrite, &sh->dev[disk_index].flags))
1871 continue;
1872 if (disk_index == sh->qd_idx || disk_index == sh->pd_idx)
1873 continue;
1874 data_count++;
1875 }
1876
1877 /*
1878 * stripes that only have parity must have been flushed
1879 * before the crash that we are now recovering from, so
1880 * there is nothing more to recovery.
1881 */
1882 if (data_count == 0)
1883 goto out;
1884
1885 for (disk_index = 0; disk_index < sh->disks; disk_index++) {
1886 if (!test_bit(R5_Wantwrite, &sh->dev[disk_index].flags))
1887 continue;
1888
1889 /* in case device is broken */
1890 rcu_read_lock();
1891 rdev = rcu_dereference(conf->disks[disk_index].rdev);
1892 if (rdev) {
1893 atomic_inc(&rdev->nr_pending);
1894 rcu_read_unlock();
1895 sync_page_io(rdev, sh->sector, PAGE_SIZE,
1896 sh->dev[disk_index].page, REQ_OP_WRITE,
1897 false);
1898 rdev_dec_pending(rdev, rdev->mddev);
1899 rcu_read_lock();
1900 }
1901 rrdev = rcu_dereference(conf->disks[disk_index].replacement);
1902 if (rrdev) {
1903 atomic_inc(&rrdev->nr_pending);
1904 rcu_read_unlock();
1905 sync_page_io(rrdev, sh->sector, PAGE_SIZE,
1906 sh->dev[disk_index].page, REQ_OP_WRITE,
1907 false);
1908 rdev_dec_pending(rrdev, rrdev->mddev);
1909 rcu_read_lock();
1910 }
1911 rcu_read_unlock();
1912 }
1913 ctx->data_parity_stripes++;
1914 out:
1915 r5l_recovery_reset_stripe(sh);
1916 }
1917
1918 static struct stripe_head *
r5c_recovery_alloc_stripe(struct r5conf * conf,sector_t stripe_sect,int noblock)1919 r5c_recovery_alloc_stripe(
1920 struct r5conf *conf,
1921 sector_t stripe_sect,
1922 int noblock)
1923 {
1924 struct stripe_head *sh;
1925
1926 sh = raid5_get_active_stripe(conf, NULL, stripe_sect,
1927 noblock ? R5_GAS_NOBLOCK : 0);
1928 if (!sh)
1929 return NULL; /* no more stripe available */
1930
1931 r5l_recovery_reset_stripe(sh);
1932
1933 return sh;
1934 }
1935
1936 static struct stripe_head *
r5c_recovery_lookup_stripe(struct list_head * list,sector_t sect)1937 r5c_recovery_lookup_stripe(struct list_head *list, sector_t sect)
1938 {
1939 struct stripe_head *sh;
1940
1941 list_for_each_entry(sh, list, lru)
1942 if (sh->sector == sect)
1943 return sh;
1944 return NULL;
1945 }
1946
1947 static void
r5c_recovery_drop_stripes(struct list_head * cached_stripe_list,struct r5l_recovery_ctx * ctx)1948 r5c_recovery_drop_stripes(struct list_head *cached_stripe_list,
1949 struct r5l_recovery_ctx *ctx)
1950 {
1951 struct stripe_head *sh, *next;
1952
1953 list_for_each_entry_safe(sh, next, cached_stripe_list, lru) {
1954 r5l_recovery_reset_stripe(sh);
1955 list_del_init(&sh->lru);
1956 raid5_release_stripe(sh);
1957 }
1958 }
1959
1960 static void
r5c_recovery_replay_stripes(struct list_head * cached_stripe_list,struct r5l_recovery_ctx * ctx)1961 r5c_recovery_replay_stripes(struct list_head *cached_stripe_list,
1962 struct r5l_recovery_ctx *ctx)
1963 {
1964 struct stripe_head *sh, *next;
1965
1966 list_for_each_entry_safe(sh, next, cached_stripe_list, lru)
1967 if (!test_bit(STRIPE_R5C_CACHING, &sh->state)) {
1968 r5l_recovery_replay_one_stripe(sh->raid_conf, sh, ctx);
1969 list_del_init(&sh->lru);
1970 raid5_release_stripe(sh);
1971 }
1972 }
1973
1974 /* if matches return 0; otherwise return -EINVAL */
1975 static int
r5l_recovery_verify_data_checksum(struct r5l_log * log,struct r5l_recovery_ctx * ctx,struct page * page,sector_t log_offset,__le32 log_checksum)1976 r5l_recovery_verify_data_checksum(struct r5l_log *log,
1977 struct r5l_recovery_ctx *ctx,
1978 struct page *page,
1979 sector_t log_offset, __le32 log_checksum)
1980 {
1981 void *addr;
1982 u32 checksum;
1983
1984 r5l_recovery_read_page(log, ctx, page, log_offset);
1985 addr = kmap_atomic(page);
1986 checksum = crc32c_le(log->uuid_checksum, addr, PAGE_SIZE);
1987 kunmap_atomic(addr);
1988 return (le32_to_cpu(log_checksum) == checksum) ? 0 : -EINVAL;
1989 }
1990
1991 /*
1992 * before loading data to stripe cache, we need verify checksum for all data,
1993 * if there is mismatch for any data page, we drop all data in the mata block
1994 */
1995 static int
r5l_recovery_verify_data_checksum_for_mb(struct r5l_log * log,struct r5l_recovery_ctx * ctx)1996 r5l_recovery_verify_data_checksum_for_mb(struct r5l_log *log,
1997 struct r5l_recovery_ctx *ctx)
1998 {
1999 struct mddev *mddev = log->rdev->mddev;
2000 struct r5conf *conf = mddev->private;
2001 struct r5l_meta_block *mb = page_address(ctx->meta_page);
2002 sector_t mb_offset = sizeof(struct r5l_meta_block);
2003 sector_t log_offset = r5l_ring_add(log, ctx->pos, BLOCK_SECTORS);
2004 struct page *page;
2005 struct r5l_payload_data_parity *payload;
2006 struct r5l_payload_flush *payload_flush;
2007
2008 page = alloc_page(GFP_KERNEL);
2009 if (!page)
2010 return -ENOMEM;
2011
2012 while (mb_offset < le32_to_cpu(mb->meta_size)) {
2013 payload = (void *)mb + mb_offset;
2014 payload_flush = (void *)mb + mb_offset;
2015
2016 if (le16_to_cpu(payload->header.type) == R5LOG_PAYLOAD_DATA) {
2017 if (r5l_recovery_verify_data_checksum(
2018 log, ctx, page, log_offset,
2019 payload->checksum[0]) < 0)
2020 goto mismatch;
2021 } else if (le16_to_cpu(payload->header.type) == R5LOG_PAYLOAD_PARITY) {
2022 if (r5l_recovery_verify_data_checksum(
2023 log, ctx, page, log_offset,
2024 payload->checksum[0]) < 0)
2025 goto mismatch;
2026 if (conf->max_degraded == 2 && /* q for RAID 6 */
2027 r5l_recovery_verify_data_checksum(
2028 log, ctx, page,
2029 r5l_ring_add(log, log_offset,
2030 BLOCK_SECTORS),
2031 payload->checksum[1]) < 0)
2032 goto mismatch;
2033 } else if (le16_to_cpu(payload->header.type) == R5LOG_PAYLOAD_FLUSH) {
2034 /* nothing to do for R5LOG_PAYLOAD_FLUSH here */
2035 } else /* not R5LOG_PAYLOAD_DATA/PARITY/FLUSH */
2036 goto mismatch;
2037
2038 if (le16_to_cpu(payload->header.type) == R5LOG_PAYLOAD_FLUSH) {
2039 mb_offset += sizeof(struct r5l_payload_flush) +
2040 le32_to_cpu(payload_flush->size);
2041 } else {
2042 /* DATA or PARITY payload */
2043 log_offset = r5l_ring_add(log, log_offset,
2044 le32_to_cpu(payload->size));
2045 mb_offset += sizeof(struct r5l_payload_data_parity) +
2046 sizeof(__le32) *
2047 (le32_to_cpu(payload->size) >> (PAGE_SHIFT - 9));
2048 }
2049
2050 }
2051
2052 put_page(page);
2053 return 0;
2054
2055 mismatch:
2056 put_page(page);
2057 return -EINVAL;
2058 }
2059
2060 /*
2061 * Analyze all data/parity pages in one meta block
2062 * Returns:
2063 * 0 for success
2064 * -EINVAL for unknown playload type
2065 * -EAGAIN for checksum mismatch of data page
2066 * -ENOMEM for run out of memory (alloc_page failed or run out of stripes)
2067 */
2068 static int
r5c_recovery_analyze_meta_block(struct r5l_log * log,struct r5l_recovery_ctx * ctx,struct list_head * cached_stripe_list)2069 r5c_recovery_analyze_meta_block(struct r5l_log *log,
2070 struct r5l_recovery_ctx *ctx,
2071 struct list_head *cached_stripe_list)
2072 {
2073 struct mddev *mddev = log->rdev->mddev;
2074 struct r5conf *conf = mddev->private;
2075 struct r5l_meta_block *mb;
2076 struct r5l_payload_data_parity *payload;
2077 struct r5l_payload_flush *payload_flush;
2078 int mb_offset;
2079 sector_t log_offset;
2080 sector_t stripe_sect;
2081 struct stripe_head *sh;
2082 int ret;
2083
2084 /*
2085 * for mismatch in data blocks, we will drop all data in this mb, but
2086 * we will still read next mb for other data with FLUSH flag, as
2087 * io_unit could finish out of order.
2088 */
2089 ret = r5l_recovery_verify_data_checksum_for_mb(log, ctx);
2090 if (ret == -EINVAL)
2091 return -EAGAIN;
2092 else if (ret)
2093 return ret; /* -ENOMEM duo to alloc_page() failed */
2094
2095 mb = page_address(ctx->meta_page);
2096 mb_offset = sizeof(struct r5l_meta_block);
2097 log_offset = r5l_ring_add(log, ctx->pos, BLOCK_SECTORS);
2098
2099 while (mb_offset < le32_to_cpu(mb->meta_size)) {
2100 int dd;
2101
2102 payload = (void *)mb + mb_offset;
2103 payload_flush = (void *)mb + mb_offset;
2104
2105 if (le16_to_cpu(payload->header.type) == R5LOG_PAYLOAD_FLUSH) {
2106 int i, count;
2107
2108 count = le32_to_cpu(payload_flush->size) / sizeof(__le64);
2109 for (i = 0; i < count; ++i) {
2110 stripe_sect = le64_to_cpu(payload_flush->flush_stripes[i]);
2111 sh = r5c_recovery_lookup_stripe(cached_stripe_list,
2112 stripe_sect);
2113 if (sh) {
2114 WARN_ON(test_bit(STRIPE_R5C_CACHING, &sh->state));
2115 r5l_recovery_reset_stripe(sh);
2116 list_del_init(&sh->lru);
2117 raid5_release_stripe(sh);
2118 }
2119 }
2120
2121 mb_offset += sizeof(struct r5l_payload_flush) +
2122 le32_to_cpu(payload_flush->size);
2123 continue;
2124 }
2125
2126 /* DATA or PARITY payload */
2127 stripe_sect = (le16_to_cpu(payload->header.type) == R5LOG_PAYLOAD_DATA) ?
2128 raid5_compute_sector(
2129 conf, le64_to_cpu(payload->location), 0, &dd,
2130 NULL)
2131 : le64_to_cpu(payload->location);
2132
2133 sh = r5c_recovery_lookup_stripe(cached_stripe_list,
2134 stripe_sect);
2135
2136 if (!sh) {
2137 sh = r5c_recovery_alloc_stripe(conf, stripe_sect, 1);
2138 /*
2139 * cannot get stripe from raid5_get_active_stripe
2140 * try replay some stripes
2141 */
2142 if (!sh) {
2143 r5c_recovery_replay_stripes(
2144 cached_stripe_list, ctx);
2145 sh = r5c_recovery_alloc_stripe(
2146 conf, stripe_sect, 1);
2147 }
2148 if (!sh) {
2149 int new_size = conf->min_nr_stripes * 2;
2150 pr_debug("md/raid:%s: Increasing stripe cache size to %d to recovery data on journal.\n",
2151 mdname(mddev),
2152 new_size);
2153 ret = raid5_set_cache_size(mddev, new_size);
2154 if (conf->min_nr_stripes <= new_size / 2) {
2155 pr_err("md/raid:%s: Cannot increase cache size, ret=%d, new_size=%d, min_nr_stripes=%d, max_nr_stripes=%d\n",
2156 mdname(mddev),
2157 ret,
2158 new_size,
2159 conf->min_nr_stripes,
2160 conf->max_nr_stripes);
2161 return -ENOMEM;
2162 }
2163 sh = r5c_recovery_alloc_stripe(
2164 conf, stripe_sect, 0);
2165 }
2166 if (!sh) {
2167 pr_err("md/raid:%s: Cannot get enough stripes due to memory pressure. Recovery failed.\n",
2168 mdname(mddev));
2169 return -ENOMEM;
2170 }
2171 list_add_tail(&sh->lru, cached_stripe_list);
2172 }
2173
2174 if (le16_to_cpu(payload->header.type) == R5LOG_PAYLOAD_DATA) {
2175 if (!test_bit(STRIPE_R5C_CACHING, &sh->state) &&
2176 test_bit(R5_Wantwrite, &sh->dev[sh->pd_idx].flags)) {
2177 r5l_recovery_replay_one_stripe(conf, sh, ctx);
2178 list_move_tail(&sh->lru, cached_stripe_list);
2179 }
2180 r5l_recovery_load_data(log, sh, ctx, payload,
2181 log_offset);
2182 } else if (le16_to_cpu(payload->header.type) == R5LOG_PAYLOAD_PARITY)
2183 r5l_recovery_load_parity(log, sh, ctx, payload,
2184 log_offset);
2185 else
2186 return -EINVAL;
2187
2188 log_offset = r5l_ring_add(log, log_offset,
2189 le32_to_cpu(payload->size));
2190
2191 mb_offset += sizeof(struct r5l_payload_data_parity) +
2192 sizeof(__le32) *
2193 (le32_to_cpu(payload->size) >> (PAGE_SHIFT - 9));
2194 }
2195
2196 return 0;
2197 }
2198
2199 /*
2200 * Load the stripe into cache. The stripe will be written out later by
2201 * the stripe cache state machine.
2202 */
r5c_recovery_load_one_stripe(struct r5l_log * log,struct stripe_head * sh)2203 static void r5c_recovery_load_one_stripe(struct r5l_log *log,
2204 struct stripe_head *sh)
2205 {
2206 struct r5dev *dev;
2207 int i;
2208
2209 for (i = sh->disks; i--; ) {
2210 dev = sh->dev + i;
2211 if (test_and_clear_bit(R5_Wantwrite, &dev->flags)) {
2212 set_bit(R5_InJournal, &dev->flags);
2213 set_bit(R5_UPTODATE, &dev->flags);
2214 }
2215 }
2216 }
2217
2218 /*
2219 * Scan through the log for all to-be-flushed data
2220 *
2221 * For stripes with data and parity, namely Data-Parity stripe
2222 * (STRIPE_R5C_CACHING == 0), we simply replay all the writes.
2223 *
2224 * For stripes with only data, namely Data-Only stripe
2225 * (STRIPE_R5C_CACHING == 1), we load them to stripe cache state machine.
2226 *
2227 * For a stripe, if we see data after parity, we should discard all previous
2228 * data and parity for this stripe, as these data are already flushed to
2229 * the array.
2230 *
2231 * At the end of the scan, we return the new journal_tail, which points to
2232 * first data-only stripe on the journal device, or next invalid meta block.
2233 */
r5c_recovery_flush_log(struct r5l_log * log,struct r5l_recovery_ctx * ctx)2234 static int r5c_recovery_flush_log(struct r5l_log *log,
2235 struct r5l_recovery_ctx *ctx)
2236 {
2237 struct stripe_head *sh;
2238 int ret = 0;
2239
2240 /* scan through the log */
2241 while (1) {
2242 if (r5l_recovery_read_meta_block(log, ctx))
2243 break;
2244
2245 ret = r5c_recovery_analyze_meta_block(log, ctx,
2246 &ctx->cached_list);
2247 /*
2248 * -EAGAIN means mismatch in data block, in this case, we still
2249 * try scan the next metablock
2250 */
2251 if (ret && ret != -EAGAIN)
2252 break; /* ret == -EINVAL or -ENOMEM */
2253 ctx->seq++;
2254 ctx->pos = r5l_ring_add(log, ctx->pos, ctx->meta_total_blocks);
2255 }
2256
2257 if (ret == -ENOMEM) {
2258 r5c_recovery_drop_stripes(&ctx->cached_list, ctx);
2259 return ret;
2260 }
2261
2262 /* replay data-parity stripes */
2263 r5c_recovery_replay_stripes(&ctx->cached_list, ctx);
2264
2265 /* load data-only stripes to stripe cache */
2266 list_for_each_entry(sh, &ctx->cached_list, lru) {
2267 WARN_ON(!test_bit(STRIPE_R5C_CACHING, &sh->state));
2268 r5c_recovery_load_one_stripe(log, sh);
2269 ctx->data_only_stripes++;
2270 }
2271
2272 return 0;
2273 }
2274
2275 /*
2276 * we did a recovery. Now ctx.pos points to an invalid meta block. New
2277 * log will start here. but we can't let superblock point to last valid
2278 * meta block. The log might looks like:
2279 * | meta 1| meta 2| meta 3|
2280 * meta 1 is valid, meta 2 is invalid. meta 3 could be valid. If
2281 * superblock points to meta 1, we write a new valid meta 2n. if crash
2282 * happens again, new recovery will start from meta 1. Since meta 2n is
2283 * valid now, recovery will think meta 3 is valid, which is wrong.
2284 * The solution is we create a new meta in meta2 with its seq == meta
2285 * 1's seq + 10000 and let superblock points to meta2. The same recovery
2286 * will not think meta 3 is a valid meta, because its seq doesn't match
2287 */
2288
2289 /*
2290 * Before recovery, the log looks like the following
2291 *
2292 * ---------------------------------------------
2293 * | valid log | invalid log |
2294 * ---------------------------------------------
2295 * ^
2296 * |- log->last_checkpoint
2297 * |- log->last_cp_seq
2298 *
2299 * Now we scan through the log until we see invalid entry
2300 *
2301 * ---------------------------------------------
2302 * | valid log | invalid log |
2303 * ---------------------------------------------
2304 * ^ ^
2305 * |- log->last_checkpoint |- ctx->pos
2306 * |- log->last_cp_seq |- ctx->seq
2307 *
2308 * From this point, we need to increase seq number by 10 to avoid
2309 * confusing next recovery.
2310 *
2311 * ---------------------------------------------
2312 * | valid log | invalid log |
2313 * ---------------------------------------------
2314 * ^ ^
2315 * |- log->last_checkpoint |- ctx->pos+1
2316 * |- log->last_cp_seq |- ctx->seq+10001
2317 *
2318 * However, it is not safe to start the state machine yet, because data only
2319 * parities are not yet secured in RAID. To save these data only parities, we
2320 * rewrite them from seq+11.
2321 *
2322 * -----------------------------------------------------------------
2323 * | valid log | data only stripes | invalid log |
2324 * -----------------------------------------------------------------
2325 * ^ ^
2326 * |- log->last_checkpoint |- ctx->pos+n
2327 * |- log->last_cp_seq |- ctx->seq+10000+n
2328 *
2329 * If failure happens again during this process, the recovery can safe start
2330 * again from log->last_checkpoint.
2331 *
2332 * Once data only stripes are rewritten to journal, we move log_tail
2333 *
2334 * -----------------------------------------------------------------
2335 * | old log | data only stripes | invalid log |
2336 * -----------------------------------------------------------------
2337 * ^ ^
2338 * |- log->last_checkpoint |- ctx->pos+n
2339 * |- log->last_cp_seq |- ctx->seq+10000+n
2340 *
2341 * Then we can safely start the state machine. If failure happens from this
2342 * point on, the recovery will start from new log->last_checkpoint.
2343 */
2344 static int
r5c_recovery_rewrite_data_only_stripes(struct r5l_log * log,struct r5l_recovery_ctx * ctx)2345 r5c_recovery_rewrite_data_only_stripes(struct r5l_log *log,
2346 struct r5l_recovery_ctx *ctx)
2347 {
2348 struct stripe_head *sh;
2349 struct mddev *mddev = log->rdev->mddev;
2350 struct page *page;
2351 sector_t next_checkpoint = MaxSector;
2352
2353 page = alloc_page(GFP_KERNEL);
2354 if (!page) {
2355 pr_err("md/raid:%s: cannot allocate memory to rewrite data only stripes\n",
2356 mdname(mddev));
2357 return -ENOMEM;
2358 }
2359
2360 WARN_ON(list_empty(&ctx->cached_list));
2361
2362 list_for_each_entry(sh, &ctx->cached_list, lru) {
2363 struct r5l_meta_block *mb;
2364 int i;
2365 int offset;
2366 sector_t write_pos;
2367
2368 WARN_ON(!test_bit(STRIPE_R5C_CACHING, &sh->state));
2369 r5l_recovery_create_empty_meta_block(log, page,
2370 ctx->pos, ctx->seq);
2371 mb = page_address(page);
2372 offset = le32_to_cpu(mb->meta_size);
2373 write_pos = r5l_ring_add(log, ctx->pos, BLOCK_SECTORS);
2374
2375 for (i = sh->disks; i--; ) {
2376 struct r5dev *dev = &sh->dev[i];
2377 struct r5l_payload_data_parity *payload;
2378 void *addr;
2379
2380 if (test_bit(R5_InJournal, &dev->flags)) {
2381 payload = (void *)mb + offset;
2382 payload->header.type = cpu_to_le16(
2383 R5LOG_PAYLOAD_DATA);
2384 payload->size = cpu_to_le32(BLOCK_SECTORS);
2385 payload->location = cpu_to_le64(
2386 raid5_compute_blocknr(sh, i, 0));
2387 addr = kmap_atomic(dev->page);
2388 payload->checksum[0] = cpu_to_le32(
2389 crc32c_le(log->uuid_checksum, addr,
2390 PAGE_SIZE));
2391 kunmap_atomic(addr);
2392 sync_page_io(log->rdev, write_pos, PAGE_SIZE,
2393 dev->page, REQ_OP_WRITE, false);
2394 write_pos = r5l_ring_add(log, write_pos,
2395 BLOCK_SECTORS);
2396 offset += sizeof(__le32) +
2397 sizeof(struct r5l_payload_data_parity);
2398
2399 }
2400 }
2401 mb->meta_size = cpu_to_le32(offset);
2402 mb->checksum = cpu_to_le32(crc32c_le(log->uuid_checksum,
2403 mb, PAGE_SIZE));
2404 sync_page_io(log->rdev, ctx->pos, PAGE_SIZE, page,
2405 REQ_OP_WRITE | REQ_SYNC | REQ_FUA, false);
2406 sh->log_start = ctx->pos;
2407 list_add_tail(&sh->r5c, &log->stripe_in_journal_list);
2408 atomic_inc(&log->stripe_in_journal_count);
2409 ctx->pos = write_pos;
2410 ctx->seq += 1;
2411 next_checkpoint = sh->log_start;
2412 }
2413 log->next_checkpoint = next_checkpoint;
2414 __free_page(page);
2415 return 0;
2416 }
2417
r5c_recovery_flush_data_only_stripes(struct r5l_log * log,struct r5l_recovery_ctx * ctx)2418 static void r5c_recovery_flush_data_only_stripes(struct r5l_log *log,
2419 struct r5l_recovery_ctx *ctx)
2420 {
2421 struct mddev *mddev = log->rdev->mddev;
2422 struct r5conf *conf = mddev->private;
2423 struct stripe_head *sh, *next;
2424 bool cleared_pending = false;
2425
2426 if (ctx->data_only_stripes == 0)
2427 return;
2428
2429 if (test_bit(MD_SB_CHANGE_PENDING, &mddev->sb_flags)) {
2430 cleared_pending = true;
2431 clear_bit(MD_SB_CHANGE_PENDING, &mddev->sb_flags);
2432 }
2433 log->r5c_journal_mode = R5C_JOURNAL_MODE_WRITE_BACK;
2434
2435 list_for_each_entry_safe(sh, next, &ctx->cached_list, lru) {
2436 r5c_make_stripe_write_out(sh);
2437 set_bit(STRIPE_HANDLE, &sh->state);
2438 list_del_init(&sh->lru);
2439 raid5_release_stripe(sh);
2440 }
2441
2442 /* reuse conf->wait_for_quiescent in recovery */
2443 wait_event(conf->wait_for_quiescent,
2444 atomic_read(&conf->active_stripes) == 0);
2445
2446 log->r5c_journal_mode = R5C_JOURNAL_MODE_WRITE_THROUGH;
2447 if (cleared_pending)
2448 set_bit(MD_SB_CHANGE_PENDING, &mddev->sb_flags);
2449 }
2450
r5l_recovery_log(struct r5l_log * log)2451 static int r5l_recovery_log(struct r5l_log *log)
2452 {
2453 struct mddev *mddev = log->rdev->mddev;
2454 struct r5l_recovery_ctx *ctx;
2455 int ret;
2456 sector_t pos;
2457
2458 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
2459 if (!ctx)
2460 return -ENOMEM;
2461
2462 ctx->pos = log->last_checkpoint;
2463 ctx->seq = log->last_cp_seq;
2464 INIT_LIST_HEAD(&ctx->cached_list);
2465 ctx->meta_page = alloc_page(GFP_KERNEL);
2466
2467 if (!ctx->meta_page) {
2468 ret = -ENOMEM;
2469 goto meta_page;
2470 }
2471
2472 if (r5l_recovery_allocate_ra_pool(log, ctx) != 0) {
2473 ret = -ENOMEM;
2474 goto ra_pool;
2475 }
2476
2477 ret = r5c_recovery_flush_log(log, ctx);
2478
2479 if (ret)
2480 goto error;
2481
2482 pos = ctx->pos;
2483 ctx->seq += 10000;
2484
2485 if ((ctx->data_only_stripes == 0) && (ctx->data_parity_stripes == 0))
2486 pr_info("md/raid:%s: starting from clean shutdown\n",
2487 mdname(mddev));
2488 else
2489 pr_info("md/raid:%s: recovering %d data-only stripes and %d data-parity stripes\n",
2490 mdname(mddev), ctx->data_only_stripes,
2491 ctx->data_parity_stripes);
2492
2493 if (ctx->data_only_stripes == 0) {
2494 log->next_checkpoint = ctx->pos;
2495 r5l_log_write_empty_meta_block(log, ctx->pos, ctx->seq++);
2496 ctx->pos = r5l_ring_add(log, ctx->pos, BLOCK_SECTORS);
2497 } else if (r5c_recovery_rewrite_data_only_stripes(log, ctx)) {
2498 pr_err("md/raid:%s: failed to rewrite stripes to journal\n",
2499 mdname(mddev));
2500 ret = -EIO;
2501 goto error;
2502 }
2503
2504 log->log_start = ctx->pos;
2505 log->seq = ctx->seq;
2506 log->last_checkpoint = pos;
2507 r5l_write_super(log, pos);
2508
2509 r5c_recovery_flush_data_only_stripes(log, ctx);
2510 ret = 0;
2511 error:
2512 r5l_recovery_free_ra_pool(log, ctx);
2513 ra_pool:
2514 __free_page(ctx->meta_page);
2515 meta_page:
2516 kfree(ctx);
2517 return ret;
2518 }
2519
r5l_write_super(struct r5l_log * log,sector_t cp)2520 static void r5l_write_super(struct r5l_log *log, sector_t cp)
2521 {
2522 struct mddev *mddev = log->rdev->mddev;
2523
2524 log->rdev->journal_tail = cp;
2525 set_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags);
2526 }
2527
r5c_journal_mode_show(struct mddev * mddev,char * page)2528 static ssize_t r5c_journal_mode_show(struct mddev *mddev, char *page)
2529 {
2530 struct r5conf *conf;
2531 int ret;
2532
2533 ret = mddev_lock(mddev);
2534 if (ret)
2535 return ret;
2536
2537 conf = mddev->private;
2538 if (!conf || !conf->log)
2539 goto out_unlock;
2540
2541 switch (conf->log->r5c_journal_mode) {
2542 case R5C_JOURNAL_MODE_WRITE_THROUGH:
2543 ret = snprintf(
2544 page, PAGE_SIZE, "[%s] %s\n",
2545 r5c_journal_mode_str[R5C_JOURNAL_MODE_WRITE_THROUGH],
2546 r5c_journal_mode_str[R5C_JOURNAL_MODE_WRITE_BACK]);
2547 break;
2548 case R5C_JOURNAL_MODE_WRITE_BACK:
2549 ret = snprintf(
2550 page, PAGE_SIZE, "%s [%s]\n",
2551 r5c_journal_mode_str[R5C_JOURNAL_MODE_WRITE_THROUGH],
2552 r5c_journal_mode_str[R5C_JOURNAL_MODE_WRITE_BACK]);
2553 break;
2554 default:
2555 ret = 0;
2556 }
2557
2558 out_unlock:
2559 mddev_unlock(mddev);
2560 return ret;
2561 }
2562
2563 /*
2564 * Set journal cache mode on @mddev (external API initially needed by dm-raid).
2565 *
2566 * @mode as defined in 'enum r5c_journal_mode'.
2567 *
2568 */
r5c_journal_mode_set(struct mddev * mddev,int mode)2569 int r5c_journal_mode_set(struct mddev *mddev, int mode)
2570 {
2571 struct r5conf *conf;
2572
2573 if (mode < R5C_JOURNAL_MODE_WRITE_THROUGH ||
2574 mode > R5C_JOURNAL_MODE_WRITE_BACK)
2575 return -EINVAL;
2576
2577 conf = mddev->private;
2578 if (!conf || !conf->log)
2579 return -ENODEV;
2580
2581 if (raid5_calc_degraded(conf) > 0 &&
2582 mode == R5C_JOURNAL_MODE_WRITE_BACK)
2583 return -EINVAL;
2584
2585 mddev_suspend(mddev);
2586 conf->log->r5c_journal_mode = mode;
2587 mddev_resume(mddev);
2588
2589 pr_debug("md/raid:%s: setting r5c cache mode to %d: %s\n",
2590 mdname(mddev), mode, r5c_journal_mode_str[mode]);
2591 return 0;
2592 }
2593 EXPORT_SYMBOL(r5c_journal_mode_set);
2594
r5c_journal_mode_store(struct mddev * mddev,const char * page,size_t length)2595 static ssize_t r5c_journal_mode_store(struct mddev *mddev,
2596 const char *page, size_t length)
2597 {
2598 int mode = ARRAY_SIZE(r5c_journal_mode_str);
2599 size_t len = length;
2600 int ret;
2601
2602 if (len < 2)
2603 return -EINVAL;
2604
2605 if (page[len - 1] == '\n')
2606 len--;
2607
2608 while (mode--)
2609 if (strlen(r5c_journal_mode_str[mode]) == len &&
2610 !strncmp(page, r5c_journal_mode_str[mode], len))
2611 break;
2612 ret = mddev_lock(mddev);
2613 if (ret)
2614 return ret;
2615 ret = r5c_journal_mode_set(mddev, mode);
2616 mddev_unlock(mddev);
2617 return ret ?: length;
2618 }
2619
2620 struct md_sysfs_entry
2621 r5c_journal_mode = __ATTR(journal_mode, 0644,
2622 r5c_journal_mode_show, r5c_journal_mode_store);
2623
2624 /*
2625 * Try handle write operation in caching phase. This function should only
2626 * be called in write-back mode.
2627 *
2628 * If all outstanding writes can be handled in caching phase, returns 0
2629 * If writes requires write-out phase, call r5c_make_stripe_write_out()
2630 * and returns -EAGAIN
2631 */
r5c_try_caching_write(struct r5conf * conf,struct stripe_head * sh,struct stripe_head_state * s,int disks)2632 int r5c_try_caching_write(struct r5conf *conf,
2633 struct stripe_head *sh,
2634 struct stripe_head_state *s,
2635 int disks)
2636 {
2637 struct r5l_log *log = READ_ONCE(conf->log);
2638 int i;
2639 struct r5dev *dev;
2640 int to_cache = 0;
2641 void __rcu **pslot;
2642 sector_t tree_index;
2643 int ret;
2644 uintptr_t refcount;
2645
2646 BUG_ON(!r5c_is_writeback(log));
2647
2648 if (!test_bit(STRIPE_R5C_CACHING, &sh->state)) {
2649 /*
2650 * There are two different scenarios here:
2651 * 1. The stripe has some data cached, and it is sent to
2652 * write-out phase for reclaim
2653 * 2. The stripe is clean, and this is the first write
2654 *
2655 * For 1, return -EAGAIN, so we continue with
2656 * handle_stripe_dirtying().
2657 *
2658 * For 2, set STRIPE_R5C_CACHING and continue with caching
2659 * write.
2660 */
2661
2662 /* case 1: anything injournal or anything in written */
2663 if (s->injournal > 0 || s->written > 0)
2664 return -EAGAIN;
2665 /* case 2 */
2666 set_bit(STRIPE_R5C_CACHING, &sh->state);
2667 }
2668
2669 /*
2670 * When run in degraded mode, array is set to write-through mode.
2671 * This check helps drain pending write safely in the transition to
2672 * write-through mode.
2673 *
2674 * When a stripe is syncing, the write is also handled in write
2675 * through mode.
2676 */
2677 if (s->failed || test_bit(STRIPE_SYNCING, &sh->state)) {
2678 r5c_make_stripe_write_out(sh);
2679 return -EAGAIN;
2680 }
2681
2682 for (i = disks; i--; ) {
2683 dev = &sh->dev[i];
2684 /* if non-overwrite, use writing-out phase */
2685 if (dev->towrite && !test_bit(R5_OVERWRITE, &dev->flags) &&
2686 !test_bit(R5_InJournal, &dev->flags)) {
2687 r5c_make_stripe_write_out(sh);
2688 return -EAGAIN;
2689 }
2690 }
2691
2692 /* if the stripe is not counted in big_stripe_tree, add it now */
2693 if (!test_bit(STRIPE_R5C_PARTIAL_STRIPE, &sh->state) &&
2694 !test_bit(STRIPE_R5C_FULL_STRIPE, &sh->state)) {
2695 tree_index = r5c_tree_index(conf, sh->sector);
2696 spin_lock(&log->tree_lock);
2697 pslot = radix_tree_lookup_slot(&log->big_stripe_tree,
2698 tree_index);
2699 if (pslot) {
2700 refcount = (uintptr_t)radix_tree_deref_slot_protected(
2701 pslot, &log->tree_lock) >>
2702 R5C_RADIX_COUNT_SHIFT;
2703 radix_tree_replace_slot(
2704 &log->big_stripe_tree, pslot,
2705 (void *)((refcount + 1) << R5C_RADIX_COUNT_SHIFT));
2706 } else {
2707 /*
2708 * this radix_tree_insert can fail safely, so no
2709 * need to call radix_tree_preload()
2710 */
2711 ret = radix_tree_insert(
2712 &log->big_stripe_tree, tree_index,
2713 (void *)(1 << R5C_RADIX_COUNT_SHIFT));
2714 if (ret) {
2715 spin_unlock(&log->tree_lock);
2716 r5c_make_stripe_write_out(sh);
2717 return -EAGAIN;
2718 }
2719 }
2720 spin_unlock(&log->tree_lock);
2721
2722 /*
2723 * set STRIPE_R5C_PARTIAL_STRIPE, this shows the stripe is
2724 * counted in the radix tree
2725 */
2726 set_bit(STRIPE_R5C_PARTIAL_STRIPE, &sh->state);
2727 atomic_inc(&conf->r5c_cached_partial_stripes);
2728 }
2729
2730 for (i = disks; i--; ) {
2731 dev = &sh->dev[i];
2732 if (dev->towrite) {
2733 set_bit(R5_Wantwrite, &dev->flags);
2734 set_bit(R5_Wantdrain, &dev->flags);
2735 set_bit(R5_LOCKED, &dev->flags);
2736 to_cache++;
2737 }
2738 }
2739
2740 if (to_cache) {
2741 set_bit(STRIPE_OP_BIODRAIN, &s->ops_request);
2742 /*
2743 * set STRIPE_LOG_TRAPPED, which triggers r5c_cache_data()
2744 * in ops_run_io(). STRIPE_LOG_TRAPPED will be cleared in
2745 * r5c_handle_data_cached()
2746 */
2747 set_bit(STRIPE_LOG_TRAPPED, &sh->state);
2748 }
2749
2750 return 0;
2751 }
2752
2753 /*
2754 * free extra pages (orig_page) we allocated for prexor
2755 */
r5c_release_extra_page(struct stripe_head * sh)2756 void r5c_release_extra_page(struct stripe_head *sh)
2757 {
2758 struct r5conf *conf = sh->raid_conf;
2759 int i;
2760 bool using_disk_info_extra_page;
2761
2762 using_disk_info_extra_page =
2763 sh->dev[0].orig_page == conf->disks[0].extra_page;
2764
2765 for (i = sh->disks; i--; )
2766 if (sh->dev[i].page != sh->dev[i].orig_page) {
2767 struct page *p = sh->dev[i].orig_page;
2768
2769 sh->dev[i].orig_page = sh->dev[i].page;
2770 clear_bit(R5_OrigPageUPTDODATE, &sh->dev[i].flags);
2771
2772 if (!using_disk_info_extra_page)
2773 put_page(p);
2774 }
2775
2776 if (using_disk_info_extra_page) {
2777 clear_bit(R5C_EXTRA_PAGE_IN_USE, &conf->cache_state);
2778 md_wakeup_thread(conf->mddev->thread);
2779 }
2780 }
2781
r5c_use_extra_page(struct stripe_head * sh)2782 void r5c_use_extra_page(struct stripe_head *sh)
2783 {
2784 struct r5conf *conf = sh->raid_conf;
2785 int i;
2786 struct r5dev *dev;
2787
2788 for (i = sh->disks; i--; ) {
2789 dev = &sh->dev[i];
2790 if (dev->orig_page != dev->page)
2791 put_page(dev->orig_page);
2792 dev->orig_page = conf->disks[i].extra_page;
2793 }
2794 }
2795
2796 /*
2797 * clean up the stripe (clear R5_InJournal for dev[pd_idx] etc.) after the
2798 * stripe is committed to RAID disks.
2799 */
r5c_finish_stripe_write_out(struct r5conf * conf,struct stripe_head * sh,struct stripe_head_state * s)2800 void r5c_finish_stripe_write_out(struct r5conf *conf,
2801 struct stripe_head *sh,
2802 struct stripe_head_state *s)
2803 {
2804 struct r5l_log *log = READ_ONCE(conf->log);
2805 int i;
2806 int do_wakeup = 0;
2807 sector_t tree_index;
2808 void __rcu **pslot;
2809 uintptr_t refcount;
2810
2811 if (!log || !test_bit(R5_InJournal, &sh->dev[sh->pd_idx].flags))
2812 return;
2813
2814 WARN_ON(test_bit(STRIPE_R5C_CACHING, &sh->state));
2815 clear_bit(R5_InJournal, &sh->dev[sh->pd_idx].flags);
2816
2817 if (log->r5c_journal_mode == R5C_JOURNAL_MODE_WRITE_THROUGH)
2818 return;
2819
2820 for (i = sh->disks; i--; ) {
2821 clear_bit(R5_InJournal, &sh->dev[i].flags);
2822 if (test_and_clear_bit(R5_Overlap, &sh->dev[i].flags))
2823 do_wakeup = 1;
2824 }
2825
2826 /*
2827 * analyse_stripe() runs before r5c_finish_stripe_write_out(),
2828 * We updated R5_InJournal, so we also update s->injournal.
2829 */
2830 s->injournal = 0;
2831
2832 if (test_and_clear_bit(STRIPE_FULL_WRITE, &sh->state))
2833 if (atomic_dec_and_test(&conf->pending_full_writes))
2834 md_wakeup_thread(conf->mddev->thread);
2835
2836 if (do_wakeup)
2837 wake_up(&conf->wait_for_overlap);
2838
2839 spin_lock_irq(&log->stripe_in_journal_lock);
2840 list_del_init(&sh->r5c);
2841 spin_unlock_irq(&log->stripe_in_journal_lock);
2842 sh->log_start = MaxSector;
2843
2844 atomic_dec(&log->stripe_in_journal_count);
2845 r5c_update_log_state(log);
2846
2847 /* stop counting this stripe in big_stripe_tree */
2848 if (test_bit(STRIPE_R5C_PARTIAL_STRIPE, &sh->state) ||
2849 test_bit(STRIPE_R5C_FULL_STRIPE, &sh->state)) {
2850 tree_index = r5c_tree_index(conf, sh->sector);
2851 spin_lock(&log->tree_lock);
2852 pslot = radix_tree_lookup_slot(&log->big_stripe_tree,
2853 tree_index);
2854 BUG_ON(pslot == NULL);
2855 refcount = (uintptr_t)radix_tree_deref_slot_protected(
2856 pslot, &log->tree_lock) >>
2857 R5C_RADIX_COUNT_SHIFT;
2858 if (refcount == 1)
2859 radix_tree_delete(&log->big_stripe_tree, tree_index);
2860 else
2861 radix_tree_replace_slot(
2862 &log->big_stripe_tree, pslot,
2863 (void *)((refcount - 1) << R5C_RADIX_COUNT_SHIFT));
2864 spin_unlock(&log->tree_lock);
2865 }
2866
2867 if (test_and_clear_bit(STRIPE_R5C_PARTIAL_STRIPE, &sh->state)) {
2868 BUG_ON(atomic_read(&conf->r5c_cached_partial_stripes) == 0);
2869 atomic_dec(&conf->r5c_flushing_partial_stripes);
2870 atomic_dec(&conf->r5c_cached_partial_stripes);
2871 }
2872
2873 if (test_and_clear_bit(STRIPE_R5C_FULL_STRIPE, &sh->state)) {
2874 BUG_ON(atomic_read(&conf->r5c_cached_full_stripes) == 0);
2875 atomic_dec(&conf->r5c_flushing_full_stripes);
2876 atomic_dec(&conf->r5c_cached_full_stripes);
2877 }
2878
2879 r5l_append_flush_payload(log, sh->sector);
2880 /* stripe is flused to raid disks, we can do resync now */
2881 if (test_bit(STRIPE_SYNC_REQUESTED, &sh->state))
2882 set_bit(STRIPE_HANDLE, &sh->state);
2883 }
2884
r5c_cache_data(struct r5l_log * log,struct stripe_head * sh)2885 int r5c_cache_data(struct r5l_log *log, struct stripe_head *sh)
2886 {
2887 struct r5conf *conf = sh->raid_conf;
2888 int pages = 0;
2889 int reserve;
2890 int i;
2891 int ret = 0;
2892
2893 BUG_ON(!log);
2894
2895 for (i = 0; i < sh->disks; i++) {
2896 void *addr;
2897
2898 if (!test_bit(R5_Wantwrite, &sh->dev[i].flags))
2899 continue;
2900 addr = kmap_atomic(sh->dev[i].page);
2901 sh->dev[i].log_checksum = crc32c_le(log->uuid_checksum,
2902 addr, PAGE_SIZE);
2903 kunmap_atomic(addr);
2904 pages++;
2905 }
2906 WARN_ON(pages == 0);
2907
2908 /*
2909 * The stripe must enter state machine again to call endio, so
2910 * don't delay.
2911 */
2912 clear_bit(STRIPE_DELAYED, &sh->state);
2913 atomic_inc(&sh->count);
2914
2915 mutex_lock(&log->io_mutex);
2916 /* meta + data */
2917 reserve = (1 + pages) << (PAGE_SHIFT - 9);
2918
2919 if (test_bit(R5C_LOG_CRITICAL, &conf->cache_state) &&
2920 sh->log_start == MaxSector)
2921 r5l_add_no_space_stripe(log, sh);
2922 else if (!r5l_has_free_space(log, reserve)) {
2923 if (sh->log_start == log->last_checkpoint)
2924 BUG();
2925 else
2926 r5l_add_no_space_stripe(log, sh);
2927 } else {
2928 ret = r5l_log_stripe(log, sh, pages, 0);
2929 if (ret) {
2930 spin_lock_irq(&log->io_list_lock);
2931 list_add_tail(&sh->log_list, &log->no_mem_stripes);
2932 spin_unlock_irq(&log->io_list_lock);
2933 }
2934 }
2935
2936 mutex_unlock(&log->io_mutex);
2937 return 0;
2938 }
2939
2940 /* check whether this big stripe is in write back cache. */
r5c_big_stripe_cached(struct r5conf * conf,sector_t sect)2941 bool r5c_big_stripe_cached(struct r5conf *conf, sector_t sect)
2942 {
2943 struct r5l_log *log = READ_ONCE(conf->log);
2944 sector_t tree_index;
2945 void *slot;
2946
2947 if (!log)
2948 return false;
2949
2950 WARN_ON_ONCE(!rcu_read_lock_held());
2951 tree_index = r5c_tree_index(conf, sect);
2952 slot = radix_tree_lookup(&log->big_stripe_tree, tree_index);
2953 return slot != NULL;
2954 }
2955
r5l_load_log(struct r5l_log * log)2956 static int r5l_load_log(struct r5l_log *log)
2957 {
2958 struct md_rdev *rdev = log->rdev;
2959 struct page *page;
2960 struct r5l_meta_block *mb;
2961 sector_t cp = log->rdev->journal_tail;
2962 u32 stored_crc, expected_crc;
2963 bool create_super = false;
2964 int ret = 0;
2965
2966 /* Make sure it's valid */
2967 if (cp >= rdev->sectors || round_down(cp, BLOCK_SECTORS) != cp)
2968 cp = 0;
2969 page = alloc_page(GFP_KERNEL);
2970 if (!page)
2971 return -ENOMEM;
2972
2973 if (!sync_page_io(rdev, cp, PAGE_SIZE, page, REQ_OP_READ, false)) {
2974 ret = -EIO;
2975 goto ioerr;
2976 }
2977 mb = page_address(page);
2978
2979 if (le32_to_cpu(mb->magic) != R5LOG_MAGIC ||
2980 mb->version != R5LOG_VERSION) {
2981 create_super = true;
2982 goto create;
2983 }
2984 stored_crc = le32_to_cpu(mb->checksum);
2985 mb->checksum = 0;
2986 expected_crc = crc32c_le(log->uuid_checksum, mb, PAGE_SIZE);
2987 if (stored_crc != expected_crc) {
2988 create_super = true;
2989 goto create;
2990 }
2991 if (le64_to_cpu(mb->position) != cp) {
2992 create_super = true;
2993 goto create;
2994 }
2995 create:
2996 if (create_super) {
2997 log->last_cp_seq = get_random_u32();
2998 cp = 0;
2999 r5l_log_write_empty_meta_block(log, cp, log->last_cp_seq);
3000 /*
3001 * Make sure super points to correct address. Log might have
3002 * data very soon. If super hasn't correct log tail address,
3003 * recovery can't find the log
3004 */
3005 r5l_write_super(log, cp);
3006 } else
3007 log->last_cp_seq = le64_to_cpu(mb->seq);
3008
3009 log->device_size = round_down(rdev->sectors, BLOCK_SECTORS);
3010 log->max_free_space = log->device_size >> RECLAIM_MAX_FREE_SPACE_SHIFT;
3011 if (log->max_free_space > RECLAIM_MAX_FREE_SPACE)
3012 log->max_free_space = RECLAIM_MAX_FREE_SPACE;
3013 log->last_checkpoint = cp;
3014
3015 __free_page(page);
3016
3017 if (create_super) {
3018 log->log_start = r5l_ring_add(log, cp, BLOCK_SECTORS);
3019 log->seq = log->last_cp_seq + 1;
3020 log->next_checkpoint = cp;
3021 } else
3022 ret = r5l_recovery_log(log);
3023
3024 r5c_update_log_state(log);
3025 return ret;
3026 ioerr:
3027 __free_page(page);
3028 return ret;
3029 }
3030
r5l_start(struct r5l_log * log)3031 int r5l_start(struct r5l_log *log)
3032 {
3033 int ret;
3034
3035 if (!log)
3036 return 0;
3037
3038 ret = r5l_load_log(log);
3039 if (ret) {
3040 struct mddev *mddev = log->rdev->mddev;
3041 struct r5conf *conf = mddev->private;
3042
3043 r5l_exit_log(conf);
3044 }
3045 return ret;
3046 }
3047
r5c_update_on_rdev_error(struct mddev * mddev,struct md_rdev * rdev)3048 void r5c_update_on_rdev_error(struct mddev *mddev, struct md_rdev *rdev)
3049 {
3050 struct r5conf *conf = mddev->private;
3051 struct r5l_log *log = READ_ONCE(conf->log);
3052
3053 if (!log)
3054 return;
3055
3056 if ((raid5_calc_degraded(conf) > 0 ||
3057 test_bit(Journal, &rdev->flags)) &&
3058 log->r5c_journal_mode == R5C_JOURNAL_MODE_WRITE_BACK)
3059 schedule_work(&log->disable_writeback_work);
3060 }
3061
r5l_init_log(struct r5conf * conf,struct md_rdev * rdev)3062 int r5l_init_log(struct r5conf *conf, struct md_rdev *rdev)
3063 {
3064 struct r5l_log *log;
3065 struct md_thread *thread;
3066 int ret;
3067
3068 pr_debug("md/raid:%s: using device %pg as journal\n",
3069 mdname(conf->mddev), rdev->bdev);
3070
3071 if (PAGE_SIZE != 4096)
3072 return -EINVAL;
3073
3074 /*
3075 * The PAGE_SIZE must be big enough to hold 1 r5l_meta_block and
3076 * raid_disks r5l_payload_data_parity.
3077 *
3078 * Write journal and cache does not work for very big array
3079 * (raid_disks > 203)
3080 */
3081 if (sizeof(struct r5l_meta_block) +
3082 ((sizeof(struct r5l_payload_data_parity) + sizeof(__le32)) *
3083 conf->raid_disks) > PAGE_SIZE) {
3084 pr_err("md/raid:%s: write journal/cache doesn't work for array with %d disks\n",
3085 mdname(conf->mddev), conf->raid_disks);
3086 return -EINVAL;
3087 }
3088
3089 log = kzalloc(sizeof(*log), GFP_KERNEL);
3090 if (!log)
3091 return -ENOMEM;
3092 log->rdev = rdev;
3093 log->need_cache_flush = bdev_write_cache(rdev->bdev);
3094 log->uuid_checksum = crc32c_le(~0, rdev->mddev->uuid,
3095 sizeof(rdev->mddev->uuid));
3096
3097 mutex_init(&log->io_mutex);
3098
3099 spin_lock_init(&log->io_list_lock);
3100 INIT_LIST_HEAD(&log->running_ios);
3101 INIT_LIST_HEAD(&log->io_end_ios);
3102 INIT_LIST_HEAD(&log->flushing_ios);
3103 INIT_LIST_HEAD(&log->finished_ios);
3104
3105 log->io_kc = KMEM_CACHE(r5l_io_unit, 0);
3106 if (!log->io_kc)
3107 goto io_kc;
3108
3109 ret = mempool_init_slab_pool(&log->io_pool, R5L_POOL_SIZE, log->io_kc);
3110 if (ret)
3111 goto io_pool;
3112
3113 ret = bioset_init(&log->bs, R5L_POOL_SIZE, 0, BIOSET_NEED_BVECS);
3114 if (ret)
3115 goto io_bs;
3116
3117 ret = mempool_init_page_pool(&log->meta_pool, R5L_POOL_SIZE, 0);
3118 if (ret)
3119 goto out_mempool;
3120
3121 spin_lock_init(&log->tree_lock);
3122 INIT_RADIX_TREE(&log->big_stripe_tree, GFP_NOWAIT | __GFP_NOWARN);
3123
3124 thread = md_register_thread(r5l_reclaim_thread, log->rdev->mddev,
3125 "reclaim");
3126 if (!thread)
3127 goto reclaim_thread;
3128
3129 thread->timeout = R5C_RECLAIM_WAKEUP_INTERVAL;
3130 rcu_assign_pointer(log->reclaim_thread, thread);
3131
3132 init_waitqueue_head(&log->iounit_wait);
3133
3134 INIT_LIST_HEAD(&log->no_mem_stripes);
3135
3136 INIT_LIST_HEAD(&log->no_space_stripes);
3137 spin_lock_init(&log->no_space_stripes_lock);
3138
3139 INIT_WORK(&log->deferred_io_work, r5l_submit_io_async);
3140 INIT_WORK(&log->disable_writeback_work, r5c_disable_writeback_async);
3141
3142 log->r5c_journal_mode = R5C_JOURNAL_MODE_WRITE_THROUGH;
3143 INIT_LIST_HEAD(&log->stripe_in_journal_list);
3144 spin_lock_init(&log->stripe_in_journal_lock);
3145 atomic_set(&log->stripe_in_journal_count, 0);
3146
3147 WRITE_ONCE(conf->log, log);
3148
3149 set_bit(MD_HAS_JOURNAL, &conf->mddev->flags);
3150 return 0;
3151
3152 reclaim_thread:
3153 mempool_exit(&log->meta_pool);
3154 out_mempool:
3155 bioset_exit(&log->bs);
3156 io_bs:
3157 mempool_exit(&log->io_pool);
3158 io_pool:
3159 kmem_cache_destroy(log->io_kc);
3160 io_kc:
3161 kfree(log);
3162 return -EINVAL;
3163 }
3164
r5l_exit_log(struct r5conf * conf)3165 void r5l_exit_log(struct r5conf *conf)
3166 {
3167 struct r5l_log *log = conf->log;
3168
3169 md_unregister_thread(conf->mddev, &log->reclaim_thread);
3170
3171 /*
3172 * 'reconfig_mutex' is held by caller, set 'confg->log' to NULL to
3173 * ensure disable_writeback_work wakes up and exits.
3174 */
3175 WRITE_ONCE(conf->log, NULL);
3176 wake_up(&conf->mddev->sb_wait);
3177 flush_work(&log->disable_writeback_work);
3178
3179 mempool_exit(&log->meta_pool);
3180 bioset_exit(&log->bs);
3181 mempool_exit(&log->io_pool);
3182 kmem_cache_destroy(log->io_kc);
3183 kfree(log);
3184 }
3185