Lines Matching refs:bitmap
35 static inline char *bmname(struct bitmap *bitmap) in bmname() argument
37 return bitmap->mddev ? mdname(bitmap->mddev) : "mdX"; in bmname()
50 static int md_bitmap_checkpage(struct bitmap_counts *bitmap, in md_bitmap_checkpage() argument
52 __releases(bitmap->lock) in md_bitmap_checkpage()
53 __acquires(bitmap->lock) in md_bitmap_checkpage()
57 WARN_ON_ONCE(page >= bitmap->pages); in md_bitmap_checkpage()
58 if (bitmap->bp[page].hijacked) /* it's hijacked, don't try to alloc */ in md_bitmap_checkpage()
61 if (bitmap->bp[page].map) /* page is already allocated, just return */ in md_bitmap_checkpage()
69 spin_unlock_irq(&bitmap->lock); in md_bitmap_checkpage()
84 spin_lock_irq(&bitmap->lock); in md_bitmap_checkpage()
93 if (!bitmap->bp[page].map) in md_bitmap_checkpage()
94 bitmap->bp[page].hijacked = 1; in md_bitmap_checkpage()
95 } else if (bitmap->bp[page].map || in md_bitmap_checkpage()
96 bitmap->bp[page].hijacked) { in md_bitmap_checkpage()
103 bitmap->bp[page].map = mappage; in md_bitmap_checkpage()
104 bitmap->missing_pages--; in md_bitmap_checkpage()
112 static void md_bitmap_checkfree(struct bitmap_counts *bitmap, unsigned long page) in md_bitmap_checkfree() argument
116 if (bitmap->bp[page].count) /* page is still busy */ in md_bitmap_checkfree()
121 if (bitmap->bp[page].hijacked) { /* page was hijacked, undo this now */ in md_bitmap_checkfree()
122 bitmap->bp[page].hijacked = 0; in md_bitmap_checkfree()
123 bitmap->bp[page].map = NULL; in md_bitmap_checkfree()
126 ptr = bitmap->bp[page].map; in md_bitmap_checkfree()
127 bitmap->bp[page].map = NULL; in md_bitmap_checkfree()
128 bitmap->missing_pages++; in md_bitmap_checkfree()
224 static int __write_sb_page(struct md_rdev *rdev, struct bitmap *bitmap, in __write_sb_page() argument
228 struct mddev *mddev = bitmap->mddev; in __write_sb_page()
229 struct bitmap_storage *store = &bitmap->storage; in __write_sb_page()
230 unsigned int bitmap_limit = (bitmap->storage.file_pages - pg_index) << in __write_sb_page()
280 static void write_sb_page(struct bitmap *bitmap, unsigned long pg_index, in write_sb_page() argument
283 struct mddev *mddev = bitmap->mddev; in write_sb_page()
289 if (__write_sb_page(rdev, bitmap, pg_index, page) < 0) { in write_sb_page()
290 set_bit(BITMAP_WRITE_ERROR, &bitmap->flags); in write_sb_page()
297 static void md_bitmap_file_kick(struct bitmap *bitmap);
300 static void write_file_page(struct bitmap *bitmap, struct page *page, int wait) in write_file_page() argument
305 atomic_inc(&bitmap->pending_writes); in write_file_page()
313 wait_event(bitmap->write_wait, in write_file_page()
314 atomic_read(&bitmap->pending_writes) == 0); in write_file_page()
319 struct bitmap *bitmap = bh->b_private; in end_bitmap_write() local
322 set_bit(BITMAP_WRITE_ERROR, &bitmap->flags); in end_bitmap_write()
323 if (atomic_dec_and_test(&bitmap->pending_writes)) in end_bitmap_write()
324 wake_up(&bitmap->write_wait); in end_bitmap_write()
352 struct bitmap *bitmap, unsigned long count, struct page *page) in read_file_page() argument
391 bh->b_private = bitmap; in read_file_page()
392 atomic_inc(&bitmap->pending_writes); in read_file_page()
401 wait_event(bitmap->write_wait, in read_file_page()
402 atomic_read(&bitmap->pending_writes)==0); in read_file_page()
403 if (test_bit(BITMAP_WRITE_ERROR, &bitmap->flags)) in read_file_page()
414 static void write_file_page(struct bitmap *bitmap, struct page *page, int wait) in write_file_page() argument
418 struct bitmap *bitmap, unsigned long count, struct page *page) in read_file_page() argument
435 static void filemap_write_page(struct bitmap *bitmap, unsigned long pg_index, in filemap_write_page() argument
438 struct bitmap_storage *store = &bitmap->storage; in filemap_write_page()
441 if (mddev_is_clustered(bitmap->mddev)) { in filemap_write_page()
447 write_file_page(bitmap, page, wait); in filemap_write_page()
449 write_sb_page(bitmap, pg_index, page, wait); in filemap_write_page()
457 static void md_bitmap_wait_writes(struct bitmap *bitmap) in md_bitmap_wait_writes() argument
459 if (bitmap->storage.file) in md_bitmap_wait_writes()
460 wait_event(bitmap->write_wait, in md_bitmap_wait_writes()
461 atomic_read(&bitmap->pending_writes)==0); in md_bitmap_wait_writes()
470 md_super_wait(bitmap->mddev); in md_bitmap_wait_writes()
475 void md_bitmap_update_sb(struct bitmap *bitmap) in md_bitmap_update_sb() argument
479 if (!bitmap || !bitmap->mddev) /* no bitmap for this array */ in md_bitmap_update_sb()
481 if (bitmap->mddev->bitmap_info.external) in md_bitmap_update_sb()
483 if (!bitmap->storage.sb_page) /* no superblock */ in md_bitmap_update_sb()
485 sb = kmap_atomic(bitmap->storage.sb_page); in md_bitmap_update_sb()
486 sb->events = cpu_to_le64(bitmap->mddev->events); in md_bitmap_update_sb()
487 if (bitmap->mddev->events < bitmap->events_cleared) in md_bitmap_update_sb()
489 bitmap->events_cleared = bitmap->mddev->events; in md_bitmap_update_sb()
490 sb->events_cleared = cpu_to_le64(bitmap->events_cleared); in md_bitmap_update_sb()
495 sb->state = cpu_to_le32(bitmap->flags & ~BIT(BITMAP_WRITE_ERROR)); in md_bitmap_update_sb()
497 sb->daemon_sleep = cpu_to_le32(bitmap->mddev->bitmap_info.daemon_sleep/HZ); in md_bitmap_update_sb()
498 sb->write_behind = cpu_to_le32(bitmap->mddev->bitmap_info.max_write_behind); in md_bitmap_update_sb()
500 sb->sync_size = cpu_to_le64(bitmap->mddev->resync_max_sectors); in md_bitmap_update_sb()
501 sb->chunksize = cpu_to_le32(bitmap->mddev->bitmap_info.chunksize); in md_bitmap_update_sb()
502 sb->nodes = cpu_to_le32(bitmap->mddev->bitmap_info.nodes); in md_bitmap_update_sb()
503 sb->sectors_reserved = cpu_to_le32(bitmap->mddev-> in md_bitmap_update_sb()
507 if (bitmap->storage.file) in md_bitmap_update_sb()
508 write_file_page(bitmap, bitmap->storage.sb_page, 1); in md_bitmap_update_sb()
510 write_sb_page(bitmap, bitmap->storage.sb_index, in md_bitmap_update_sb()
511 bitmap->storage.sb_page, 1); in md_bitmap_update_sb()
516 void md_bitmap_print_sb(struct bitmap *bitmap) in md_bitmap_print_sb() argument
520 if (!bitmap || !bitmap->storage.sb_page) in md_bitmap_print_sb()
522 sb = kmap_atomic(bitmap->storage.sb_page); in md_bitmap_print_sb()
523 pr_debug("%s: bitmap file superblock:\n", bmname(bitmap)); in md_bitmap_print_sb()
555 static int md_bitmap_new_disk_sb(struct bitmap *bitmap) in md_bitmap_new_disk_sb() argument
560 bitmap->storage.sb_page = alloc_page(GFP_KERNEL | __GFP_ZERO); in md_bitmap_new_disk_sb()
561 if (bitmap->storage.sb_page == NULL) in md_bitmap_new_disk_sb()
563 bitmap->storage.sb_index = 0; in md_bitmap_new_disk_sb()
565 sb = kmap_atomic(bitmap->storage.sb_page); in md_bitmap_new_disk_sb()
570 chunksize = bitmap->mddev->bitmap_info.chunksize; in md_bitmap_new_disk_sb()
579 daemon_sleep = bitmap->mddev->bitmap_info.daemon_sleep; in md_bitmap_new_disk_sb()
585 bitmap->mddev->bitmap_info.daemon_sleep = daemon_sleep; in md_bitmap_new_disk_sb()
591 write_behind = bitmap->mddev->bitmap_info.max_write_behind; in md_bitmap_new_disk_sb()
595 bitmap->mddev->bitmap_info.max_write_behind = write_behind; in md_bitmap_new_disk_sb()
598 sb->sync_size = cpu_to_le64(bitmap->mddev->resync_max_sectors); in md_bitmap_new_disk_sb()
600 memcpy(sb->uuid, bitmap->mddev->uuid, 16); in md_bitmap_new_disk_sb()
602 set_bit(BITMAP_STALE, &bitmap->flags); in md_bitmap_new_disk_sb()
603 sb->state = cpu_to_le32(bitmap->flags); in md_bitmap_new_disk_sb()
604 bitmap->events_cleared = bitmap->mddev->events; in md_bitmap_new_disk_sb()
605 sb->events_cleared = cpu_to_le64(bitmap->mddev->events); in md_bitmap_new_disk_sb()
606 bitmap->mddev->bitmap_info.nodes = 0; in md_bitmap_new_disk_sb()
614 static int md_bitmap_read_sb(struct bitmap *bitmap) in md_bitmap_read_sb() argument
626 if (!bitmap->storage.file && !bitmap->mddev->bitmap_info.offset) { in md_bitmap_read_sb()
630 set_bit(BITMAP_STALE, &bitmap->flags); in md_bitmap_read_sb()
638 bitmap->storage.sb_page = sb_page; in md_bitmap_read_sb()
642 if (bitmap->cluster_slot >= 0) { in md_bitmap_read_sb()
643 sector_t bm_blocks = bitmap->mddev->resync_max_sectors; in md_bitmap_read_sb()
646 (bitmap->mddev->bitmap_info.chunksize >> 9)); in md_bitmap_read_sb()
651 offset = bitmap->cluster_slot * (bm_blocks << 3); in md_bitmap_read_sb()
653 bitmap->cluster_slot, offset); in md_bitmap_read_sb()
656 if (bitmap->storage.file) { in md_bitmap_read_sb()
657 loff_t isize = i_size_read(bitmap->storage.file->f_mapping->host); in md_bitmap_read_sb()
660 err = read_file_page(bitmap->storage.file, 0, in md_bitmap_read_sb()
661 bitmap, bytes, sb_page); in md_bitmap_read_sb()
663 err = read_sb_page(bitmap->mddev, offset, sb_page, 0, in md_bitmap_read_sb()
693 bmname(bitmap), reason); in md_bitmap_read_sb()
703 strscpy(bitmap->mddev->bitmap_info.cluster_name, in md_bitmap_read_sb()
708 sb->sync_size = cpu_to_le64(bitmap->mddev->resync_max_sectors); in md_bitmap_read_sb()
710 if (bitmap->mddev->persistent) { in md_bitmap_read_sb()
715 if (memcmp(sb->uuid, bitmap->mddev->uuid, 16)) { in md_bitmap_read_sb()
717 bmname(bitmap)); in md_bitmap_read_sb()
721 if (!nodes && (events < bitmap->mddev->events)) { in md_bitmap_read_sb()
723 bmname(bitmap), events, in md_bitmap_read_sb()
724 (unsigned long long) bitmap->mddev->events); in md_bitmap_read_sb()
725 set_bit(BITMAP_STALE, &bitmap->flags); in md_bitmap_read_sb()
730 bitmap->flags |= le32_to_cpu(sb->state); in md_bitmap_read_sb()
732 set_bit(BITMAP_HOSTENDIAN, &bitmap->flags); in md_bitmap_read_sb()
733 bitmap->events_cleared = le64_to_cpu(sb->events_cleared); in md_bitmap_read_sb()
738 if (err == 0 && nodes && (bitmap->cluster_slot < 0)) { in md_bitmap_read_sb()
740 bitmap->mddev->bitmap_info.chunksize = chunksize; in md_bitmap_read_sb()
741 err = md_setup_cluster(bitmap->mddev, nodes); in md_bitmap_read_sb()
744 bmname(bitmap), err); in md_bitmap_read_sb()
747 bitmap->cluster_slot = md_cluster_ops->slot_number(bitmap->mddev); in md_bitmap_read_sb()
753 if (test_bit(BITMAP_STALE, &bitmap->flags)) in md_bitmap_read_sb()
754 bitmap->events_cleared = bitmap->mddev->events; in md_bitmap_read_sb()
755 bitmap->mddev->bitmap_info.chunksize = chunksize; in md_bitmap_read_sb()
756 bitmap->mddev->bitmap_info.daemon_sleep = daemon_sleep; in md_bitmap_read_sb()
757 bitmap->mddev->bitmap_info.max_write_behind = write_behind; in md_bitmap_read_sb()
758 bitmap->mddev->bitmap_info.nodes = nodes; in md_bitmap_read_sb()
759 if (bitmap->mddev->bitmap_info.space == 0 || in md_bitmap_read_sb()
760 bitmap->mddev->bitmap_info.space > sectors_reserved) in md_bitmap_read_sb()
761 bitmap->mddev->bitmap_info.space = sectors_reserved; in md_bitmap_read_sb()
763 md_bitmap_print_sb(bitmap); in md_bitmap_read_sb()
764 if (bitmap->cluster_slot < 0) in md_bitmap_read_sb()
765 md_cluster_stop(bitmap->mddev); in md_bitmap_read_sb()
893 static void md_bitmap_file_kick(struct bitmap *bitmap) in md_bitmap_file_kick() argument
895 if (!test_and_set_bit(BITMAP_STALE, &bitmap->flags)) { in md_bitmap_file_kick()
896 md_bitmap_update_sb(bitmap); in md_bitmap_file_kick()
898 if (bitmap->storage.file) { in md_bitmap_file_kick()
900 bmname(bitmap), bitmap->storage.file); in md_bitmap_file_kick()
904 bmname(bitmap)); in md_bitmap_file_kick()
915 static inline void set_page_attr(struct bitmap *bitmap, int pnum, in set_page_attr() argument
918 set_bit((pnum<<2) + attr, bitmap->storage.filemap_attr); in set_page_attr()
921 static inline void clear_page_attr(struct bitmap *bitmap, int pnum, in clear_page_attr() argument
924 clear_bit((pnum<<2) + attr, bitmap->storage.filemap_attr); in clear_page_attr()
927 static inline int test_page_attr(struct bitmap *bitmap, int pnum, in test_page_attr() argument
930 return test_bit((pnum<<2) + attr, bitmap->storage.filemap_attr); in test_page_attr()
933 static inline int test_and_clear_page_attr(struct bitmap *bitmap, int pnum, in test_and_clear_page_attr() argument
937 bitmap->storage.filemap_attr); in test_and_clear_page_attr()
946 static void md_bitmap_file_set_bit(struct bitmap *bitmap, sector_t block) in md_bitmap_file_set_bit() argument
951 unsigned long chunk = block >> bitmap->counts.chunkshift; in md_bitmap_file_set_bit()
952 struct bitmap_storage *store = &bitmap->storage; in md_bitmap_file_set_bit()
957 if (mddev_is_clustered(bitmap->mddev)) in md_bitmap_file_set_bit()
958 node_offset = bitmap->cluster_slot * store->file_pages; in md_bitmap_file_set_bit()
960 page = filemap_get_page(&bitmap->storage, chunk); in md_bitmap_file_set_bit()
963 bit = file_page_offset(&bitmap->storage, chunk); in md_bitmap_file_set_bit()
967 if (test_bit(BITMAP_HOSTENDIAN, &bitmap->flags)) in md_bitmap_file_set_bit()
974 set_page_attr(bitmap, index - node_offset, BITMAP_PAGE_DIRTY); in md_bitmap_file_set_bit()
977 static void md_bitmap_file_clear_bit(struct bitmap *bitmap, sector_t block) in md_bitmap_file_clear_bit() argument
982 unsigned long chunk = block >> bitmap->counts.chunkshift; in md_bitmap_file_clear_bit()
983 struct bitmap_storage *store = &bitmap->storage; in md_bitmap_file_clear_bit()
988 if (mddev_is_clustered(bitmap->mddev)) in md_bitmap_file_clear_bit()
989 node_offset = bitmap->cluster_slot * store->file_pages; in md_bitmap_file_clear_bit()
991 page = filemap_get_page(&bitmap->storage, chunk); in md_bitmap_file_clear_bit()
994 bit = file_page_offset(&bitmap->storage, chunk); in md_bitmap_file_clear_bit()
996 if (test_bit(BITMAP_HOSTENDIAN, &bitmap->flags)) in md_bitmap_file_clear_bit()
1001 if (!test_page_attr(bitmap, index - node_offset, BITMAP_PAGE_NEEDWRITE)) { in md_bitmap_file_clear_bit()
1002 set_page_attr(bitmap, index - node_offset, BITMAP_PAGE_PENDING); in md_bitmap_file_clear_bit()
1003 bitmap->allclean = 0; in md_bitmap_file_clear_bit()
1007 static int md_bitmap_file_test_bit(struct bitmap *bitmap, sector_t block) in md_bitmap_file_test_bit() argument
1012 unsigned long chunk = block >> bitmap->counts.chunkshift; in md_bitmap_file_test_bit()
1015 page = filemap_get_page(&bitmap->storage, chunk); in md_bitmap_file_test_bit()
1018 bit = file_page_offset(&bitmap->storage, chunk); in md_bitmap_file_test_bit()
1020 if (test_bit(BITMAP_HOSTENDIAN, &bitmap->flags)) in md_bitmap_file_test_bit()
1031 void md_bitmap_unplug(struct bitmap *bitmap) in md_bitmap_unplug() argument
1037 if (!md_bitmap_enabled(bitmap)) in md_bitmap_unplug()
1042 for (i = 0; i < bitmap->storage.file_pages; i++) { in md_bitmap_unplug()
1043 dirty = test_and_clear_page_attr(bitmap, i, BITMAP_PAGE_DIRTY); in md_bitmap_unplug()
1044 need_write = test_and_clear_page_attr(bitmap, i, in md_bitmap_unplug()
1048 md_bitmap_wait_writes(bitmap); in md_bitmap_unplug()
1049 if (bitmap->mddev->queue) in md_bitmap_unplug()
1050 blk_add_trace_msg(bitmap->mddev->queue, in md_bitmap_unplug()
1053 clear_page_attr(bitmap, i, BITMAP_PAGE_PENDING); in md_bitmap_unplug()
1054 filemap_write_page(bitmap, i, false); in md_bitmap_unplug()
1059 md_bitmap_wait_writes(bitmap); in md_bitmap_unplug()
1061 if (test_bit(BITMAP_WRITE_ERROR, &bitmap->flags)) in md_bitmap_unplug()
1062 md_bitmap_file_kick(bitmap); in md_bitmap_unplug()
1068 struct bitmap *bitmap; member
1077 md_bitmap_unplug(unplug_work->bitmap); in md_bitmap_unplug_fn()
1081 void md_bitmap_unplug_async(struct bitmap *bitmap) in md_bitmap_unplug_async() argument
1087 unplug_work.bitmap = bitmap; in md_bitmap_unplug_async()
1095 static void md_bitmap_set_memory_bits(struct bitmap *bitmap, sector_t offset, int needed);
1108 static int md_bitmap_init_from_disk(struct bitmap *bitmap, sector_t start) in md_bitmap_init_from_disk() argument
1110 bool outofdate = test_bit(BITMAP_STALE, &bitmap->flags); in md_bitmap_init_from_disk()
1111 struct mddev *mddev = bitmap->mddev; in md_bitmap_init_from_disk()
1112 unsigned long chunks = bitmap->counts.chunks; in md_bitmap_init_from_disk()
1113 struct bitmap_storage *store = &bitmap->storage; in md_bitmap_init_from_disk()
1126 int needed = ((sector_t)(i+1) << (bitmap->counts.chunkshift) in md_bitmap_init_from_disk()
1128 md_bitmap_set_memory_bits(bitmap, in md_bitmap_init_from_disk()
1129 (sector_t)i << bitmap->counts.chunkshift, in md_bitmap_init_from_disk()
1137 bmname(bitmap), in md_bitmap_init_from_disk()
1145 node_offset = bitmap->cluster_slot * (DIV_ROUND_UP(store->bytes, PAGE_SIZE)); in md_bitmap_init_from_disk()
1158 ret = read_file_page(file, i, bitmap, count, page); in md_bitmap_init_from_disk()
1168 bmname(bitmap)); in md_bitmap_init_from_disk()
1186 filemap_write_page(bitmap, i, true); in md_bitmap_init_from_disk()
1187 if (test_bit(BITMAP_WRITE_ERROR, &bitmap->flags)) { in md_bitmap_init_from_disk()
1195 struct page *page = filemap_get_page(&bitmap->storage, i); in md_bitmap_init_from_disk()
1196 unsigned long bit = file_page_offset(&bitmap->storage, i); in md_bitmap_init_from_disk()
1201 if (test_bit(BITMAP_HOSTENDIAN, &bitmap->flags)) in md_bitmap_init_from_disk()
1209 int needed = ((sector_t)(i+1) << bitmap->counts.chunkshift in md_bitmap_init_from_disk()
1211 md_bitmap_set_memory_bits(bitmap, in md_bitmap_init_from_disk()
1212 (sector_t)i << bitmap->counts.chunkshift, in md_bitmap_init_from_disk()
1219 bmname(bitmap), store->file_pages, in md_bitmap_init_from_disk()
1226 bmname(bitmap), ret); in md_bitmap_init_from_disk()
1230 void md_bitmap_write_all(struct bitmap *bitmap) in md_bitmap_write_all() argument
1237 if (!bitmap || !bitmap->storage.filemap) in md_bitmap_write_all()
1239 if (bitmap->storage.file) in md_bitmap_write_all()
1243 for (i = 0; i < bitmap->storage.file_pages; i++) in md_bitmap_write_all()
1244 set_page_attr(bitmap, i, in md_bitmap_write_all()
1246 bitmap->allclean = 0; in md_bitmap_write_all()
1249 static void md_bitmap_count_page(struct bitmap_counts *bitmap, in md_bitmap_count_page() argument
1252 sector_t chunk = offset >> bitmap->chunkshift; in md_bitmap_count_page()
1254 bitmap->bp[page].count += inc; in md_bitmap_count_page()
1255 md_bitmap_checkfree(bitmap, page); in md_bitmap_count_page()
1258 static void md_bitmap_set_pending(struct bitmap_counts *bitmap, sector_t offset) in md_bitmap_set_pending() argument
1260 sector_t chunk = offset >> bitmap->chunkshift; in md_bitmap_set_pending()
1262 struct bitmap_page *bp = &bitmap->bp[page]; in md_bitmap_set_pending()
1268 static bitmap_counter_t *md_bitmap_get_counter(struct bitmap_counts *bitmap,
1296 struct bitmap *bitmap; in md_bitmap_daemon_work() local
1306 bitmap = mddev->bitmap; in md_bitmap_daemon_work()
1307 if (bitmap == NULL) { in md_bitmap_daemon_work()
1311 if (time_before(jiffies, bitmap->daemon_lastrun in md_bitmap_daemon_work()
1315 bitmap->daemon_lastrun = jiffies; in md_bitmap_daemon_work()
1316 if (bitmap->allclean) { in md_bitmap_daemon_work()
1320 bitmap->allclean = 1; in md_bitmap_daemon_work()
1322 if (bitmap->mddev->queue) in md_bitmap_daemon_work()
1323 blk_add_trace_msg(bitmap->mddev->queue, in md_bitmap_daemon_work()
1330 for (j = 0; j < bitmap->storage.file_pages; j++) in md_bitmap_daemon_work()
1331 if (test_and_clear_page_attr(bitmap, j, in md_bitmap_daemon_work()
1333 set_page_attr(bitmap, j, in md_bitmap_daemon_work()
1336 if (bitmap->need_sync && in md_bitmap_daemon_work()
1341 bitmap->need_sync = 0; in md_bitmap_daemon_work()
1342 if (bitmap->storage.filemap) { in md_bitmap_daemon_work()
1343 sb = kmap_atomic(bitmap->storage.sb_page); in md_bitmap_daemon_work()
1345 cpu_to_le64(bitmap->events_cleared); in md_bitmap_daemon_work()
1347 set_page_attr(bitmap, 0, in md_bitmap_daemon_work()
1354 counts = &bitmap->counts; in md_bitmap_daemon_work()
1375 if (*bmc == 1 && !bitmap->need_sync) { in md_bitmap_daemon_work()
1379 md_bitmap_file_clear_bit(bitmap, block); in md_bitmap_daemon_work()
1383 bitmap->allclean = 0; in md_bitmap_daemon_work()
1388 md_bitmap_wait_writes(bitmap); in md_bitmap_daemon_work()
1398 j < bitmap->storage.file_pages in md_bitmap_daemon_work()
1399 && !test_bit(BITMAP_STALE, &bitmap->flags); in md_bitmap_daemon_work()
1401 if (test_page_attr(bitmap, j, in md_bitmap_daemon_work()
1405 if (bitmap->storage.filemap && in md_bitmap_daemon_work()
1406 test_and_clear_page_attr(bitmap, j, in md_bitmap_daemon_work()
1408 filemap_write_page(bitmap, j, false); in md_bitmap_daemon_work()
1412 if (bitmap->allclean == 0) in md_bitmap_daemon_work()
1417 static bitmap_counter_t *md_bitmap_get_counter(struct bitmap_counts *bitmap, in md_bitmap_get_counter() argument
1420 __releases(bitmap->lock) in md_bitmap_get_counter()
1421 __acquires(bitmap->lock) in md_bitmap_get_counter()
1427 sector_t chunk = offset >> bitmap->chunkshift; in md_bitmap_get_counter()
1430 sector_t csize = ((sector_t)1) << bitmap->chunkshift; in md_bitmap_get_counter()
1433 if (page >= bitmap->pages) { in md_bitmap_get_counter()
1442 err = md_bitmap_checkpage(bitmap, page, create, 0); in md_bitmap_get_counter()
1444 if (bitmap->bp[page].hijacked || in md_bitmap_get_counter()
1445 bitmap->bp[page].map == NULL) in md_bitmap_get_counter()
1446 csize = ((sector_t)1) << (bitmap->chunkshift + in md_bitmap_get_counter()
1456 if (bitmap->bp[page].hijacked) { /* hijacked pointer */ in md_bitmap_get_counter()
1461 &bitmap->bp[page].map)[hi]; in md_bitmap_get_counter()
1464 &(bitmap->bp[page].map[pageoff]); in md_bitmap_get_counter()
1467 int md_bitmap_startwrite(struct bitmap *bitmap, sector_t offset, unsigned long sectors, int behind) in md_bitmap_startwrite() argument
1469 if (!bitmap) in md_bitmap_startwrite()
1474 atomic_inc(&bitmap->behind_writes); in md_bitmap_startwrite()
1475 bw = atomic_read(&bitmap->behind_writes); in md_bitmap_startwrite()
1476 if (bw > bitmap->behind_writes_used) in md_bitmap_startwrite()
1477 bitmap->behind_writes_used = bw; in md_bitmap_startwrite()
1480 bw, bitmap->mddev->bitmap_info.max_write_behind); in md_bitmap_startwrite()
1487 spin_lock_irq(&bitmap->counts.lock); in md_bitmap_startwrite()
1488 bmc = md_bitmap_get_counter(&bitmap->counts, offset, &blocks, 1); in md_bitmap_startwrite()
1490 spin_unlock_irq(&bitmap->counts.lock); in md_bitmap_startwrite()
1500 prepare_to_wait(&bitmap->overflow_wait, &__wait, in md_bitmap_startwrite()
1502 spin_unlock_irq(&bitmap->counts.lock); in md_bitmap_startwrite()
1504 finish_wait(&bitmap->overflow_wait, &__wait); in md_bitmap_startwrite()
1510 md_bitmap_file_set_bit(bitmap, offset); in md_bitmap_startwrite()
1511 md_bitmap_count_page(&bitmap->counts, offset, 1); in md_bitmap_startwrite()
1519 spin_unlock_irq(&bitmap->counts.lock); in md_bitmap_startwrite()
1531 void md_bitmap_endwrite(struct bitmap *bitmap, sector_t offset, in md_bitmap_endwrite() argument
1534 if (!bitmap) in md_bitmap_endwrite()
1537 if (atomic_dec_and_test(&bitmap->behind_writes)) in md_bitmap_endwrite()
1538 wake_up(&bitmap->behind_wait); in md_bitmap_endwrite()
1540 atomic_read(&bitmap->behind_writes), in md_bitmap_endwrite()
1541 bitmap->mddev->bitmap_info.max_write_behind); in md_bitmap_endwrite()
1549 spin_lock_irqsave(&bitmap->counts.lock, flags); in md_bitmap_endwrite()
1550 bmc = md_bitmap_get_counter(&bitmap->counts, offset, &blocks, 0); in md_bitmap_endwrite()
1552 spin_unlock_irqrestore(&bitmap->counts.lock, flags); in md_bitmap_endwrite()
1556 if (success && !bitmap->mddev->degraded && in md_bitmap_endwrite()
1557 bitmap->events_cleared < bitmap->mddev->events) { in md_bitmap_endwrite()
1558 bitmap->events_cleared = bitmap->mddev->events; in md_bitmap_endwrite()
1559 bitmap->need_sync = 1; in md_bitmap_endwrite()
1560 sysfs_notify_dirent_safe(bitmap->sysfs_can_clear); in md_bitmap_endwrite()
1567 wake_up(&bitmap->overflow_wait); in md_bitmap_endwrite()
1571 md_bitmap_set_pending(&bitmap->counts, offset); in md_bitmap_endwrite()
1572 bitmap->allclean = 0; in md_bitmap_endwrite()
1574 spin_unlock_irqrestore(&bitmap->counts.lock, flags); in md_bitmap_endwrite()
1584 static int __bitmap_start_sync(struct bitmap *bitmap, sector_t offset, sector_t *blocks, in __bitmap_start_sync() argument
1589 if (bitmap == NULL) {/* FIXME or bitmap set as 'failed' */ in __bitmap_start_sync()
1593 spin_lock_irq(&bitmap->counts.lock); in __bitmap_start_sync()
1594 bmc = md_bitmap_get_counter(&bitmap->counts, offset, blocks, 0); in __bitmap_start_sync()
1608 spin_unlock_irq(&bitmap->counts.lock); in __bitmap_start_sync()
1612 int md_bitmap_start_sync(struct bitmap *bitmap, sector_t offset, sector_t *blocks, in md_bitmap_start_sync() argument
1627 rv |= __bitmap_start_sync(bitmap, offset, in md_bitmap_start_sync()
1636 void md_bitmap_end_sync(struct bitmap *bitmap, sector_t offset, sector_t *blocks, int aborted) in md_bitmap_end_sync() argument
1641 if (bitmap == NULL) { in md_bitmap_end_sync()
1645 spin_lock_irqsave(&bitmap->counts.lock, flags); in md_bitmap_end_sync()
1646 bmc = md_bitmap_get_counter(&bitmap->counts, offset, blocks, 0); in md_bitmap_end_sync()
1657 md_bitmap_set_pending(&bitmap->counts, offset); in md_bitmap_end_sync()
1658 bitmap->allclean = 0; in md_bitmap_end_sync()
1663 spin_unlock_irqrestore(&bitmap->counts.lock, flags); in md_bitmap_end_sync()
1667 void md_bitmap_close_sync(struct bitmap *bitmap) in md_bitmap_close_sync() argument
1675 if (!bitmap) in md_bitmap_close_sync()
1677 while (sector < bitmap->mddev->resync_max_sectors) { in md_bitmap_close_sync()
1678 md_bitmap_end_sync(bitmap, sector, &blocks, 0); in md_bitmap_close_sync()
1684 void md_bitmap_cond_end_sync(struct bitmap *bitmap, sector_t sector, bool force) in md_bitmap_cond_end_sync() argument
1689 if (!bitmap) in md_bitmap_cond_end_sync()
1692 bitmap->last_end_sync = jiffies; in md_bitmap_cond_end_sync()
1695 if (!force && time_before(jiffies, (bitmap->last_end_sync in md_bitmap_cond_end_sync()
1696 + bitmap->mddev->bitmap_info.daemon_sleep))) in md_bitmap_cond_end_sync()
1698 wait_event(bitmap->mddev->recovery_wait, in md_bitmap_cond_end_sync()
1699 atomic_read(&bitmap->mddev->recovery_active) == 0); in md_bitmap_cond_end_sync()
1701 bitmap->mddev->curr_resync_completed = sector; in md_bitmap_cond_end_sync()
1702 set_bit(MD_SB_CHANGE_CLEAN, &bitmap->mddev->sb_flags); in md_bitmap_cond_end_sync()
1703 sector &= ~((1ULL << bitmap->counts.chunkshift) - 1); in md_bitmap_cond_end_sync()
1705 while (s < sector && s < bitmap->mddev->resync_max_sectors) { in md_bitmap_cond_end_sync()
1706 md_bitmap_end_sync(bitmap, s, &blocks, 0); in md_bitmap_cond_end_sync()
1709 bitmap->last_end_sync = jiffies; in md_bitmap_cond_end_sync()
1710 sysfs_notify_dirent_safe(bitmap->mddev->sysfs_completed); in md_bitmap_cond_end_sync()
1718 struct bitmap *bitmap = mddev->bitmap; in md_bitmap_sync_with_cluster() local
1722 md_bitmap_end_sync(bitmap, sector, &blocks, 0); in md_bitmap_sync_with_cluster()
1728 md_bitmap_start_sync(bitmap, sector, &blocks, 0); in md_bitmap_sync_with_cluster()
1735 static void md_bitmap_set_memory_bits(struct bitmap *bitmap, sector_t offset, int needed) in md_bitmap_set_memory_bits() argument
1744 spin_lock_irq(&bitmap->counts.lock); in md_bitmap_set_memory_bits()
1745 bmc = md_bitmap_get_counter(&bitmap->counts, offset, &secs, 1); in md_bitmap_set_memory_bits()
1747 spin_unlock_irq(&bitmap->counts.lock); in md_bitmap_set_memory_bits()
1752 md_bitmap_count_page(&bitmap->counts, offset, 1); in md_bitmap_set_memory_bits()
1753 md_bitmap_set_pending(&bitmap->counts, offset); in md_bitmap_set_memory_bits()
1754 bitmap->allclean = 0; in md_bitmap_set_memory_bits()
1758 spin_unlock_irq(&bitmap->counts.lock); in md_bitmap_set_memory_bits()
1762 void md_bitmap_dirty_bits(struct bitmap *bitmap, unsigned long s, unsigned long e) in md_bitmap_dirty_bits() argument
1767 sector_t sec = (sector_t)chunk << bitmap->counts.chunkshift; in md_bitmap_dirty_bits()
1768 md_bitmap_set_memory_bits(bitmap, sec, 1); in md_bitmap_dirty_bits()
1769 md_bitmap_file_set_bit(bitmap, sec); in md_bitmap_dirty_bits()
1770 if (sec < bitmap->mddev->recovery_cp) in md_bitmap_dirty_bits()
1775 bitmap->mddev->recovery_cp = sec; in md_bitmap_dirty_bits()
1784 struct bitmap *bitmap = mddev->bitmap; in md_bitmap_flush() local
1787 if (!bitmap) /* there was no bitmap */ in md_bitmap_flush()
1794 bitmap->daemon_lastrun -= sleep; in md_bitmap_flush()
1796 bitmap->daemon_lastrun -= sleep; in md_bitmap_flush()
1798 bitmap->daemon_lastrun -= sleep; in md_bitmap_flush()
1802 md_bitmap_update_sb(bitmap); in md_bitmap_flush()
1808 void md_bitmap_free(struct bitmap *bitmap) in md_bitmap_free() argument
1813 if (!bitmap) /* there was no bitmap */ in md_bitmap_free()
1816 if (bitmap->sysfs_can_clear) in md_bitmap_free()
1817 sysfs_put(bitmap->sysfs_can_clear); in md_bitmap_free()
1819 if (mddev_is_clustered(bitmap->mddev) && bitmap->mddev->cluster_info && in md_bitmap_free()
1820 bitmap->cluster_slot == md_cluster_ops->slot_number(bitmap->mddev)) in md_bitmap_free()
1821 md_cluster_stop(bitmap->mddev); in md_bitmap_free()
1824 wait_event(bitmap->write_wait, in md_bitmap_free()
1825 atomic_read(&bitmap->pending_writes) == 0); in md_bitmap_free()
1828 md_bitmap_file_unmap(&bitmap->storage); in md_bitmap_free()
1830 bp = bitmap->counts.bp; in md_bitmap_free()
1831 pages = bitmap->counts.pages; in md_bitmap_free()
1840 kfree(bitmap); in md_bitmap_free()
1846 struct bitmap *bitmap = mddev->bitmap; in md_bitmap_wait_behind_writes() local
1849 if (bitmap && atomic_read(&bitmap->behind_writes) > 0) { in md_bitmap_wait_behind_writes()
1853 wait_event(bitmap->behind_wait, in md_bitmap_wait_behind_writes()
1854 atomic_read(&bitmap->behind_writes) == 0); in md_bitmap_wait_behind_writes()
1860 struct bitmap *bitmap = mddev->bitmap; in md_bitmap_destroy() local
1862 if (!bitmap) /* there was no bitmap */ in md_bitmap_destroy()
1871 mddev->bitmap = NULL; /* disconnect from the md device */ in md_bitmap_destroy()
1876 md_bitmap_free(bitmap); in md_bitmap_destroy()
1884 struct bitmap *md_bitmap_create(struct mddev *mddev, int slot) in md_bitmap_create()
1886 struct bitmap *bitmap; in md_bitmap_create() local
1902 bitmap = kzalloc(sizeof(*bitmap), GFP_KERNEL); in md_bitmap_create()
1903 if (!bitmap) in md_bitmap_create()
1906 spin_lock_init(&bitmap->counts.lock); in md_bitmap_create()
1907 atomic_set(&bitmap->pending_writes, 0); in md_bitmap_create()
1908 init_waitqueue_head(&bitmap->write_wait); in md_bitmap_create()
1909 init_waitqueue_head(&bitmap->overflow_wait); in md_bitmap_create()
1910 init_waitqueue_head(&bitmap->behind_wait); in md_bitmap_create()
1912 bitmap->mddev = mddev; in md_bitmap_create()
1913 bitmap->cluster_slot = slot; in md_bitmap_create()
1918 bitmap->sysfs_can_clear = sysfs_get_dirent(bm, "can_clear"); in md_bitmap_create()
1921 bitmap->sysfs_can_clear = NULL; in md_bitmap_create()
1923 bitmap->storage.file = file; in md_bitmap_create()
1939 err = md_bitmap_new_disk_sb(bitmap); in md_bitmap_create()
1941 err = md_bitmap_read_sb(bitmap); in md_bitmap_create()
1953 bitmap->daemon_lastrun = jiffies; in md_bitmap_create()
1954 err = md_bitmap_resize(bitmap, blocks, mddev->bitmap_info.chunksize, 1); in md_bitmap_create()
1959 bitmap->counts.pages, bmname(bitmap)); in md_bitmap_create()
1961 err = test_bit(BITMAP_WRITE_ERROR, &bitmap->flags) ? -EIO : 0; in md_bitmap_create()
1965 return bitmap; in md_bitmap_create()
1967 md_bitmap_free(bitmap); in md_bitmap_create()
1976 struct bitmap *bitmap = mddev->bitmap; in md_bitmap_load() local
1979 if (!bitmap) in md_bitmap_load()
1995 md_bitmap_start_sync(bitmap, sector, &blocks, 0); in md_bitmap_load()
1998 md_bitmap_close_sync(bitmap); in md_bitmap_load()
2001 || bitmap->events_cleared == mddev->events) in md_bitmap_load()
2007 err = md_bitmap_init_from_disk(bitmap, start); in md_bitmap_load()
2012 clear_bit(BITMAP_STALE, &bitmap->flags); in md_bitmap_load()
2015 set_bit(MD_RECOVERY_NEEDED, &bitmap->mddev->recovery); in md_bitmap_load()
2020 md_bitmap_update_sb(bitmap); in md_bitmap_load()
2022 if (test_bit(BITMAP_WRITE_ERROR, &bitmap->flags)) in md_bitmap_load()
2030 struct bitmap *get_bitmap_from_slot(struct mddev *mddev, int slot) in get_bitmap_from_slot()
2033 struct bitmap *bitmap; in get_bitmap_from_slot() local
2035 bitmap = md_bitmap_create(mddev, slot); in get_bitmap_from_slot()
2036 if (IS_ERR(bitmap)) { in get_bitmap_from_slot()
2037 rv = PTR_ERR(bitmap); in get_bitmap_from_slot()
2041 rv = md_bitmap_init_from_disk(bitmap, 0); in get_bitmap_from_slot()
2043 md_bitmap_free(bitmap); in get_bitmap_from_slot()
2047 return bitmap; in get_bitmap_from_slot()
2060 struct bitmap *bitmap; in md_bitmap_copy_from_slot() local
2062 bitmap = get_bitmap_from_slot(mddev, slot); in md_bitmap_copy_from_slot()
2063 if (IS_ERR(bitmap)) { in md_bitmap_copy_from_slot()
2068 counts = &bitmap->counts; in md_bitmap_copy_from_slot()
2071 if (md_bitmap_file_test_bit(bitmap, block)) { in md_bitmap_copy_from_slot()
2075 md_bitmap_file_clear_bit(bitmap, block); in md_bitmap_copy_from_slot()
2076 md_bitmap_set_memory_bits(mddev->bitmap, block, 1); in md_bitmap_copy_from_slot()
2077 md_bitmap_file_set_bit(mddev->bitmap, block); in md_bitmap_copy_from_slot()
2082 md_bitmap_update_sb(bitmap); in md_bitmap_copy_from_slot()
2085 for (i = 0; i < bitmap->storage.file_pages; i++) in md_bitmap_copy_from_slot()
2086 if (test_page_attr(bitmap, i, BITMAP_PAGE_PENDING)) in md_bitmap_copy_from_slot()
2087 set_page_attr(bitmap, i, BITMAP_PAGE_NEEDWRITE); in md_bitmap_copy_from_slot()
2088 md_bitmap_unplug(bitmap); in md_bitmap_copy_from_slot()
2090 md_bitmap_unplug(mddev->bitmap); in md_bitmap_copy_from_slot()
2093 md_bitmap_free(bitmap); in md_bitmap_copy_from_slot()
2100 void md_bitmap_status(struct seq_file *seq, struct bitmap *bitmap) in md_bitmap_status() argument
2105 if (!bitmap) in md_bitmap_status()
2108 counts = &bitmap->counts; in md_bitmap_status()
2110 chunk_kb = bitmap->mddev->bitmap_info.chunksize >> 10; in md_bitmap_status()
2117 chunk_kb ? chunk_kb : bitmap->mddev->bitmap_info.chunksize, in md_bitmap_status()
2119 if (bitmap->storage.file) { in md_bitmap_status()
2121 seq_file_path(seq, bitmap->storage.file, " \t\n"); in md_bitmap_status()
2127 int md_bitmap_resize(struct bitmap *bitmap, sector_t blocks, in md_bitmap_resize() argument
2150 if (bitmap->storage.file && !init) { in md_bitmap_resize()
2160 long space = bitmap->mddev->bitmap_info.space; in md_bitmap_resize()
2166 bytes = DIV_ROUND_UP(bitmap->counts.chunks, 8); in md_bitmap_resize()
2167 if (!bitmap->mddev->bitmap_info.external) in md_bitmap_resize()
2170 bitmap->mddev->bitmap_info.space = space; in md_bitmap_resize()
2172 chunkshift = bitmap->counts.chunkshift; in md_bitmap_resize()
2179 if (!bitmap->mddev->bitmap_info.external) in md_bitmap_resize()
2188 if (bitmap->mddev->bitmap_info.offset || bitmap->mddev->bitmap_info.file) in md_bitmap_resize()
2190 !bitmap->mddev->bitmap_info.external, in md_bitmap_resize()
2191 mddev_is_clustered(bitmap->mddev) in md_bitmap_resize()
2192 ? bitmap->cluster_slot : 0); in md_bitmap_resize()
2208 bitmap->mddev->pers->quiesce(bitmap->mddev, 1); in md_bitmap_resize()
2210 store.file = bitmap->storage.file; in md_bitmap_resize()
2211 bitmap->storage.file = NULL; in md_bitmap_resize()
2213 if (store.sb_page && bitmap->storage.sb_page) in md_bitmap_resize()
2215 page_address(bitmap->storage.sb_page), in md_bitmap_resize()
2217 spin_lock_irq(&bitmap->counts.lock); in md_bitmap_resize()
2218 md_bitmap_file_unmap(&bitmap->storage); in md_bitmap_resize()
2219 bitmap->storage = store; in md_bitmap_resize()
2221 old_counts = bitmap->counts; in md_bitmap_resize()
2222 bitmap->counts.bp = new_bp; in md_bitmap_resize()
2223 bitmap->counts.pages = pages; in md_bitmap_resize()
2224 bitmap->counts.missing_pages = pages; in md_bitmap_resize()
2225 bitmap->counts.chunkshift = chunkshift; in md_bitmap_resize()
2226 bitmap->counts.chunks = chunks; in md_bitmap_resize()
2227 bitmap->mddev->bitmap_info.chunksize = 1UL << (chunkshift + in md_bitmap_resize()
2234 if (mddev_is_clustered(bitmap->mddev)) { in md_bitmap_resize()
2237 ret = md_bitmap_checkpage(&bitmap->counts, page, 1, 1); in md_bitmap_resize()
2248 bitmap->counts.bp = old_counts.bp; in md_bitmap_resize()
2249 bitmap->counts.pages = old_counts.pages; in md_bitmap_resize()
2250 bitmap->counts.missing_pages = old_counts.pages; in md_bitmap_resize()
2251 bitmap->counts.chunkshift = old_counts.chunkshift; in md_bitmap_resize()
2252 bitmap->counts.chunks = old_counts.chunks; in md_bitmap_resize()
2253 bitmap->mddev->bitmap_info.chunksize = in md_bitmap_resize()
2259 bitmap->counts.bp[page].count += 1; in md_bitmap_resize()
2271 bmc_new = md_bitmap_get_counter(&bitmap->counts, block, &new_blocks, 1); in md_bitmap_resize()
2280 md_bitmap_file_set_bit(bitmap, block); in md_bitmap_resize()
2284 md_bitmap_count_page(&bitmap->counts, block, 1); in md_bitmap_resize()
2285 md_bitmap_set_pending(&bitmap->counts, block); in md_bitmap_resize()
2295 if (bitmap->counts.bp != old_counts.bp) { in md_bitmap_resize()
2307 bmc = md_bitmap_get_counter(&bitmap->counts, block, &new_blocks, 1); in md_bitmap_resize()
2314 md_bitmap_count_page(&bitmap->counts, block, 1); in md_bitmap_resize()
2315 md_bitmap_set_pending(&bitmap->counts, block); in md_bitmap_resize()
2320 for (i = 0; i < bitmap->storage.file_pages; i++) in md_bitmap_resize()
2321 set_page_attr(bitmap, i, BITMAP_PAGE_DIRTY); in md_bitmap_resize()
2323 spin_unlock_irq(&bitmap->counts.lock); in md_bitmap_resize()
2326 md_bitmap_unplug(bitmap); in md_bitmap_resize()
2327 bitmap->mddev->pers->quiesce(bitmap->mddev, 0); in md_bitmap_resize()
2368 if (mddev->bitmap || mddev->bitmap_info.file || in location_store()
2414 struct bitmap *bitmap; in location_store() local
2415 bitmap = md_bitmap_create(mddev, -1); in location_store()
2417 if (IS_ERR(bitmap)) in location_store()
2418 rv = PTR_ERR(bitmap); in location_store()
2420 mddev->bitmap = bitmap; in location_store()
2475 if (mddev->bitmap && in space_store()
2476 sectors < (mddev->bitmap->storage.bytes + 511) >> 9) in space_store()
2584 md_bitmap_update_sb(mddev->bitmap); in backlog_store()
2605 if (mddev->bitmap) in chunksize_store()
2633 if (mddev->bitmap || in metadata_store()
2654 if (mddev->bitmap) in can_clear_show()
2655 len = sprintf(page, "%s\n", (mddev->bitmap->need_sync ? in can_clear_show()
2665 if (mddev->bitmap == NULL) in can_clear_store()
2668 mddev->bitmap->need_sync = 1; in can_clear_store()
2672 mddev->bitmap->need_sync = 0; in can_clear_store()
2686 if (mddev->bitmap == NULL) in behind_writes_used_show()
2690 mddev->bitmap->behind_writes_used); in behind_writes_used_show()
2698 if (mddev->bitmap) in behind_writes_used_reset()
2699 mddev->bitmap->behind_writes_used = 0; in behind_writes_used_reset()