1 /* 2 drbd_bitmap.c 3 4 This file is part of DRBD by Philipp Reisner and Lars Ellenberg. 5 6 Copyright (C) 2004-2008, LINBIT Information Technologies GmbH. 7 Copyright (C) 2004-2008, Philipp Reisner <philipp.reisner@linbit.com>. 8 Copyright (C) 2004-2008, Lars Ellenberg <lars.ellenberg@linbit.com>. 9 10 drbd is free software; you can redistribute it and/or modify 11 it under the terms of the GNU General Public License as published by 12 the Free Software Foundation; either version 2, or (at your option) 13 any later version. 14 15 drbd is distributed in the hope that it will be useful, 16 but WITHOUT ANY WARRANTY; without even the implied warranty of 17 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 18 GNU General Public License for more details. 19 20 You should have received a copy of the GNU General Public License 21 along with drbd; see the file COPYING. If not, write to 22 the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA. 23 */ 24 25 #include <linux/bitops.h> 26 #include <linux/vmalloc.h> 27 #include <linux/string.h> 28 #include <linux/drbd.h> 29 #include <linux/slab.h> 30 #include <asm/kmap_types.h> 31 32 #include "drbd_int.h" 33 34 35 /* OPAQUE outside this file! 36 * interface defined in drbd_int.h 37 38 * convention: 39 * function name drbd_bm_... => used elsewhere, "public". 40 * function name bm_... => internal to implementation, "private". 41 */ 42 43 44 /* 45 * LIMITATIONS: 46 * We want to support >= peta byte of backend storage, while for now still using 47 * a granularity of one bit per 4KiB of storage. 48 * 1 << 50 bytes backend storage (1 PiB) 49 * 1 << (50 - 12) bits needed 50 * 38 --> we need u64 to index and count bits 51 * 1 << (38 - 3) bitmap bytes needed 52 * 35 --> we still need u64 to index and count bytes 53 * (that's 32 GiB of bitmap for 1 PiB storage) 54 * 1 << (35 - 2) 32bit longs needed 55 * 33 --> we'd even need u64 to index and count 32bit long words. 56 * 1 << (35 - 3) 64bit longs needed 57 * 32 --> we could get away with a 32bit unsigned int to index and count 58 * 64bit long words, but I rather stay with unsigned long for now. 59 * We probably should neither count nor point to bytes or long words 60 * directly, but either by bitnumber, or by page index and offset. 61 * 1 << (35 - 12) 62 * 22 --> we need that much 4KiB pages of bitmap. 63 * 1 << (22 + 3) --> on a 64bit arch, 64 * we need 32 MiB to store the array of page pointers. 65 * 66 * Because I'm lazy, and because the resulting patch was too large, too ugly 67 * and still incomplete, on 32bit we still "only" support 16 TiB (minus some), 68 * (1 << 32) bits * 4k storage. 69 * 70 71 * bitmap storage and IO: 72 * Bitmap is stored little endian on disk, and is kept little endian in 73 * core memory. Currently we still hold the full bitmap in core as long 74 * as we are "attached" to a local disk, which at 32 GiB for 1PiB storage 75 * seems excessive. 76 * 77 * We plan to reduce the amount of in-core bitmap pages by paging them in 78 * and out against their on-disk location as necessary, but need to make 79 * sure we don't cause too much meta data IO, and must not deadlock in 80 * tight memory situations. This needs some more work. 81 */ 82 83 /* 84 * NOTE 85 * Access to the *bm_pages is protected by bm_lock. 86 * It is safe to read the other members within the lock. 87 * 88 * drbd_bm_set_bits is called from bio_endio callbacks, 89 * We may be called with irq already disabled, 90 * so we need spin_lock_irqsave(). 91 * And we need the kmap_atomic. 92 */ 93 struct drbd_bitmap { 94 struct page **bm_pages; 95 spinlock_t bm_lock; 96 97 /* see LIMITATIONS: above */ 98 99 unsigned long bm_set; /* nr of set bits; THINK maybe atomic_t? */ 100 unsigned long bm_bits; 101 size_t bm_words; 102 size_t bm_number_of_pages; 103 sector_t bm_dev_capacity; 104 struct mutex bm_change; /* serializes resize operations */ 105 106 wait_queue_head_t bm_io_wait; /* used to serialize IO of single pages */ 107 108 enum bm_flag bm_flags; 109 110 /* debugging aid, in case we are still racy somewhere */ 111 char *bm_why; 112 struct task_struct *bm_task; 113 }; 114 115 #define bm_print_lock_info(m) __bm_print_lock_info(m, __func__) 116 static void __bm_print_lock_info(struct drbd_device *device, const char *func) 117 { 118 struct drbd_bitmap *b = device->bitmap; 119 if (!__ratelimit(&drbd_ratelimit_state)) 120 return; 121 drbd_err(device, "FIXME %s[%d] in %s, bitmap locked for '%s' by %s[%d]\n", 122 current->comm, task_pid_nr(current), 123 func, b->bm_why ?: "?", 124 b->bm_task->comm, task_pid_nr(b->bm_task)); 125 } 126 127 void drbd_bm_lock(struct drbd_device *device, char *why, enum bm_flag flags) 128 { 129 struct drbd_bitmap *b = device->bitmap; 130 int trylock_failed; 131 132 if (!b) { 133 drbd_err(device, "FIXME no bitmap in drbd_bm_lock!?\n"); 134 return; 135 } 136 137 trylock_failed = !mutex_trylock(&b->bm_change); 138 139 if (trylock_failed) { 140 drbd_warn(device, "%s[%d] going to '%s' but bitmap already locked for '%s' by %s[%d]\n", 141 current->comm, task_pid_nr(current), 142 why, b->bm_why ?: "?", 143 b->bm_task->comm, task_pid_nr(b->bm_task)); 144 mutex_lock(&b->bm_change); 145 } 146 if (BM_LOCKED_MASK & b->bm_flags) 147 drbd_err(device, "FIXME bitmap already locked in bm_lock\n"); 148 b->bm_flags |= flags & BM_LOCKED_MASK; 149 150 b->bm_why = why; 151 b->bm_task = current; 152 } 153 154 void drbd_bm_unlock(struct drbd_device *device) 155 { 156 struct drbd_bitmap *b = device->bitmap; 157 if (!b) { 158 drbd_err(device, "FIXME no bitmap in drbd_bm_unlock!?\n"); 159 return; 160 } 161 162 if (!(BM_LOCKED_MASK & device->bitmap->bm_flags)) 163 drbd_err(device, "FIXME bitmap not locked in bm_unlock\n"); 164 165 b->bm_flags &= ~BM_LOCKED_MASK; 166 b->bm_why = NULL; 167 b->bm_task = NULL; 168 mutex_unlock(&b->bm_change); 169 } 170 171 /* we store some "meta" info about our pages in page->private */ 172 /* at a granularity of 4k storage per bitmap bit: 173 * one peta byte storage: 1<<50 byte, 1<<38 * 4k storage blocks 174 * 1<<38 bits, 175 * 1<<23 4k bitmap pages. 176 * Use 24 bits as page index, covers 2 peta byte storage 177 * at a granularity of 4k per bit. 178 * Used to report the failed page idx on io error from the endio handlers. 179 */ 180 #define BM_PAGE_IDX_MASK ((1UL<<24)-1) 181 /* this page is currently read in, or written back */ 182 #define BM_PAGE_IO_LOCK 31 183 /* if there has been an IO error for this page */ 184 #define BM_PAGE_IO_ERROR 30 185 /* this is to be able to intelligently skip disk IO, 186 * set if bits have been set since last IO. */ 187 #define BM_PAGE_NEED_WRITEOUT 29 188 /* to mark for lazy writeout once syncer cleared all clearable bits, 189 * we if bits have been cleared since last IO. */ 190 #define BM_PAGE_LAZY_WRITEOUT 28 191 /* pages marked with this "HINT" will be considered for writeout 192 * on activity log transactions */ 193 #define BM_PAGE_HINT_WRITEOUT 27 194 195 /* store_page_idx uses non-atomic assignment. It is only used directly after 196 * allocating the page. All other bm_set_page_* and bm_clear_page_* need to 197 * use atomic bit manipulation, as set_out_of_sync (and therefore bitmap 198 * changes) may happen from various contexts, and wait_on_bit/wake_up_bit 199 * requires it all to be atomic as well. */ 200 static void bm_store_page_idx(struct page *page, unsigned long idx) 201 { 202 BUG_ON(0 != (idx & ~BM_PAGE_IDX_MASK)); 203 set_page_private(page, idx); 204 } 205 206 static unsigned long bm_page_to_idx(struct page *page) 207 { 208 return page_private(page) & BM_PAGE_IDX_MASK; 209 } 210 211 /* As is very unlikely that the same page is under IO from more than one 212 * context, we can get away with a bit per page and one wait queue per bitmap. 213 */ 214 static void bm_page_lock_io(struct drbd_device *device, int page_nr) 215 { 216 struct drbd_bitmap *b = device->bitmap; 217 void *addr = &page_private(b->bm_pages[page_nr]); 218 wait_event(b->bm_io_wait, !test_and_set_bit(BM_PAGE_IO_LOCK, addr)); 219 } 220 221 static void bm_page_unlock_io(struct drbd_device *device, int page_nr) 222 { 223 struct drbd_bitmap *b = device->bitmap; 224 void *addr = &page_private(b->bm_pages[page_nr]); 225 clear_bit_unlock(BM_PAGE_IO_LOCK, addr); 226 wake_up(&device->bitmap->bm_io_wait); 227 } 228 229 /* set _before_ submit_io, so it may be reset due to being changed 230 * while this page is in flight... will get submitted later again */ 231 static void bm_set_page_unchanged(struct page *page) 232 { 233 /* use cmpxchg? */ 234 clear_bit(BM_PAGE_NEED_WRITEOUT, &page_private(page)); 235 clear_bit(BM_PAGE_LAZY_WRITEOUT, &page_private(page)); 236 } 237 238 static void bm_set_page_need_writeout(struct page *page) 239 { 240 set_bit(BM_PAGE_NEED_WRITEOUT, &page_private(page)); 241 } 242 243 /** 244 * drbd_bm_mark_for_writeout() - mark a page with a "hint" to be considered for writeout 245 * @device: DRBD device. 246 * @page_nr: the bitmap page to mark with the "hint" flag 247 * 248 * From within an activity log transaction, we mark a few pages with these 249 * hints, then call drbd_bm_write_hinted(), which will only write out changed 250 * pages which are flagged with this mark. 251 */ 252 void drbd_bm_mark_for_writeout(struct drbd_device *device, int page_nr) 253 { 254 struct page *page; 255 if (page_nr >= device->bitmap->bm_number_of_pages) { 256 drbd_warn(device, "BAD: page_nr: %u, number_of_pages: %u\n", 257 page_nr, (int)device->bitmap->bm_number_of_pages); 258 return; 259 } 260 page = device->bitmap->bm_pages[page_nr]; 261 set_bit(BM_PAGE_HINT_WRITEOUT, &page_private(page)); 262 } 263 264 static int bm_test_page_unchanged(struct page *page) 265 { 266 volatile const unsigned long *addr = &page_private(page); 267 return (*addr & ((1UL<<BM_PAGE_NEED_WRITEOUT)|(1UL<<BM_PAGE_LAZY_WRITEOUT))) == 0; 268 } 269 270 static void bm_set_page_io_err(struct page *page) 271 { 272 set_bit(BM_PAGE_IO_ERROR, &page_private(page)); 273 } 274 275 static void bm_clear_page_io_err(struct page *page) 276 { 277 clear_bit(BM_PAGE_IO_ERROR, &page_private(page)); 278 } 279 280 static void bm_set_page_lazy_writeout(struct page *page) 281 { 282 set_bit(BM_PAGE_LAZY_WRITEOUT, &page_private(page)); 283 } 284 285 static int bm_test_page_lazy_writeout(struct page *page) 286 { 287 return test_bit(BM_PAGE_LAZY_WRITEOUT, &page_private(page)); 288 } 289 290 /* on a 32bit box, this would allow for exactly (2<<38) bits. */ 291 static unsigned int bm_word_to_page_idx(struct drbd_bitmap *b, unsigned long long_nr) 292 { 293 /* page_nr = (word*sizeof(long)) >> PAGE_SHIFT; */ 294 unsigned int page_nr = long_nr >> (PAGE_SHIFT - LN2_BPL + 3); 295 BUG_ON(page_nr >= b->bm_number_of_pages); 296 return page_nr; 297 } 298 299 static unsigned int bm_bit_to_page_idx(struct drbd_bitmap *b, u64 bitnr) 300 { 301 /* page_nr = (bitnr/8) >> PAGE_SHIFT; */ 302 unsigned int page_nr = bitnr >> (PAGE_SHIFT + 3); 303 BUG_ON(page_nr >= b->bm_number_of_pages); 304 return page_nr; 305 } 306 307 static unsigned long *__bm_map_pidx(struct drbd_bitmap *b, unsigned int idx) 308 { 309 struct page *page = b->bm_pages[idx]; 310 return (unsigned long *) kmap_atomic(page); 311 } 312 313 static unsigned long *bm_map_pidx(struct drbd_bitmap *b, unsigned int idx) 314 { 315 return __bm_map_pidx(b, idx); 316 } 317 318 static void __bm_unmap(unsigned long *p_addr) 319 { 320 kunmap_atomic(p_addr); 321 }; 322 323 static void bm_unmap(unsigned long *p_addr) 324 { 325 return __bm_unmap(p_addr); 326 } 327 328 /* long word offset of _bitmap_ sector */ 329 #define S2W(s) ((s)<<(BM_EXT_SHIFT-BM_BLOCK_SHIFT-LN2_BPL)) 330 /* word offset from start of bitmap to word number _in_page_ 331 * modulo longs per page 332 #define MLPP(X) ((X) % (PAGE_SIZE/sizeof(long)) 333 hm, well, Philipp thinks gcc might not optimize the % into & (... - 1) 334 so do it explicitly: 335 */ 336 #define MLPP(X) ((X) & ((PAGE_SIZE/sizeof(long))-1)) 337 338 /* Long words per page */ 339 #define LWPP (PAGE_SIZE/sizeof(long)) 340 341 /* 342 * actually most functions herein should take a struct drbd_bitmap*, not a 343 * struct drbd_device*, but for the debug macros I like to have the device around 344 * to be able to report device specific. 345 */ 346 347 348 static void bm_free_pages(struct page **pages, unsigned long number) 349 { 350 unsigned long i; 351 if (!pages) 352 return; 353 354 for (i = 0; i < number; i++) { 355 if (!pages[i]) { 356 printk(KERN_ALERT "drbd: bm_free_pages tried to free " 357 "a NULL pointer; i=%lu n=%lu\n", 358 i, number); 359 continue; 360 } 361 __free_page(pages[i]); 362 pages[i] = NULL; 363 } 364 } 365 366 static void bm_vk_free(void *ptr, int v) 367 { 368 if (v) 369 vfree(ptr); 370 else 371 kfree(ptr); 372 } 373 374 /* 375 * "have" and "want" are NUMBER OF PAGES. 376 */ 377 static struct page **bm_realloc_pages(struct drbd_bitmap *b, unsigned long want) 378 { 379 struct page **old_pages = b->bm_pages; 380 struct page **new_pages, *page; 381 unsigned int i, bytes, vmalloced = 0; 382 unsigned long have = b->bm_number_of_pages; 383 384 BUG_ON(have == 0 && old_pages != NULL); 385 BUG_ON(have != 0 && old_pages == NULL); 386 387 if (have == want) 388 return old_pages; 389 390 /* Trying kmalloc first, falling back to vmalloc. 391 * GFP_NOIO, as this is called while drbd IO is "suspended", 392 * and during resize or attach on diskless Primary, 393 * we must not block on IO to ourselves. 394 * Context is receiver thread or dmsetup. */ 395 bytes = sizeof(struct page *)*want; 396 new_pages = kzalloc(bytes, GFP_NOIO | __GFP_NOWARN); 397 if (!new_pages) { 398 new_pages = __vmalloc(bytes, 399 GFP_NOIO | __GFP_HIGHMEM | __GFP_ZERO, 400 PAGE_KERNEL); 401 if (!new_pages) 402 return NULL; 403 vmalloced = 1; 404 } 405 406 if (want >= have) { 407 for (i = 0; i < have; i++) 408 new_pages[i] = old_pages[i]; 409 for (; i < want; i++) { 410 page = alloc_page(GFP_NOIO | __GFP_HIGHMEM); 411 if (!page) { 412 bm_free_pages(new_pages + have, i - have); 413 bm_vk_free(new_pages, vmalloced); 414 return NULL; 415 } 416 /* we want to know which page it is 417 * from the endio handlers */ 418 bm_store_page_idx(page, i); 419 new_pages[i] = page; 420 } 421 } else { 422 for (i = 0; i < want; i++) 423 new_pages[i] = old_pages[i]; 424 /* NOT HERE, we are outside the spinlock! 425 bm_free_pages(old_pages + want, have - want); 426 */ 427 } 428 429 if (vmalloced) 430 b->bm_flags |= BM_P_VMALLOCED; 431 else 432 b->bm_flags &= ~BM_P_VMALLOCED; 433 434 return new_pages; 435 } 436 437 /* 438 * called on driver init only. TODO call when a device is created. 439 * allocates the drbd_bitmap, and stores it in device->bitmap. 440 */ 441 int drbd_bm_init(struct drbd_device *device) 442 { 443 struct drbd_bitmap *b = device->bitmap; 444 WARN_ON(b != NULL); 445 b = kzalloc(sizeof(struct drbd_bitmap), GFP_KERNEL); 446 if (!b) 447 return -ENOMEM; 448 spin_lock_init(&b->bm_lock); 449 mutex_init(&b->bm_change); 450 init_waitqueue_head(&b->bm_io_wait); 451 452 device->bitmap = b; 453 454 return 0; 455 } 456 457 sector_t drbd_bm_capacity(struct drbd_device *device) 458 { 459 if (!expect(device->bitmap)) 460 return 0; 461 return device->bitmap->bm_dev_capacity; 462 } 463 464 /* called on driver unload. TODO: call when a device is destroyed. 465 */ 466 void drbd_bm_cleanup(struct drbd_device *device) 467 { 468 if (!expect(device->bitmap)) 469 return; 470 bm_free_pages(device->bitmap->bm_pages, device->bitmap->bm_number_of_pages); 471 bm_vk_free(device->bitmap->bm_pages, (BM_P_VMALLOCED & device->bitmap->bm_flags)); 472 kfree(device->bitmap); 473 device->bitmap = NULL; 474 } 475 476 /* 477 * since (b->bm_bits % BITS_PER_LONG) != 0, 478 * this masks out the remaining bits. 479 * Returns the number of bits cleared. 480 */ 481 #define BITS_PER_PAGE (1UL << (PAGE_SHIFT + 3)) 482 #define BITS_PER_PAGE_MASK (BITS_PER_PAGE - 1) 483 #define BITS_PER_LONG_MASK (BITS_PER_LONG - 1) 484 static int bm_clear_surplus(struct drbd_bitmap *b) 485 { 486 unsigned long mask; 487 unsigned long *p_addr, *bm; 488 int tmp; 489 int cleared = 0; 490 491 /* number of bits modulo bits per page */ 492 tmp = (b->bm_bits & BITS_PER_PAGE_MASK); 493 /* mask the used bits of the word containing the last bit */ 494 mask = (1UL << (tmp & BITS_PER_LONG_MASK)) -1; 495 /* bitmap is always stored little endian, 496 * on disk and in core memory alike */ 497 mask = cpu_to_lel(mask); 498 499 p_addr = bm_map_pidx(b, b->bm_number_of_pages - 1); 500 bm = p_addr + (tmp/BITS_PER_LONG); 501 if (mask) { 502 /* If mask != 0, we are not exactly aligned, so bm now points 503 * to the long containing the last bit. 504 * If mask == 0, bm already points to the word immediately 505 * after the last (long word aligned) bit. */ 506 cleared = hweight_long(*bm & ~mask); 507 *bm &= mask; 508 bm++; 509 } 510 511 if (BITS_PER_LONG == 32 && ((bm - p_addr) & 1) == 1) { 512 /* on a 32bit arch, we may need to zero out 513 * a padding long to align with a 64bit remote */ 514 cleared += hweight_long(*bm); 515 *bm = 0; 516 } 517 bm_unmap(p_addr); 518 return cleared; 519 } 520 521 static void bm_set_surplus(struct drbd_bitmap *b) 522 { 523 unsigned long mask; 524 unsigned long *p_addr, *bm; 525 int tmp; 526 527 /* number of bits modulo bits per page */ 528 tmp = (b->bm_bits & BITS_PER_PAGE_MASK); 529 /* mask the used bits of the word containing the last bit */ 530 mask = (1UL << (tmp & BITS_PER_LONG_MASK)) -1; 531 /* bitmap is always stored little endian, 532 * on disk and in core memory alike */ 533 mask = cpu_to_lel(mask); 534 535 p_addr = bm_map_pidx(b, b->bm_number_of_pages - 1); 536 bm = p_addr + (tmp/BITS_PER_LONG); 537 if (mask) { 538 /* If mask != 0, we are not exactly aligned, so bm now points 539 * to the long containing the last bit. 540 * If mask == 0, bm already points to the word immediately 541 * after the last (long word aligned) bit. */ 542 *bm |= ~mask; 543 bm++; 544 } 545 546 if (BITS_PER_LONG == 32 && ((bm - p_addr) & 1) == 1) { 547 /* on a 32bit arch, we may need to zero out 548 * a padding long to align with a 64bit remote */ 549 *bm = ~0UL; 550 } 551 bm_unmap(p_addr); 552 } 553 554 /* you better not modify the bitmap while this is running, 555 * or its results will be stale */ 556 static unsigned long bm_count_bits(struct drbd_bitmap *b) 557 { 558 unsigned long *p_addr; 559 unsigned long bits = 0; 560 unsigned long mask = (1UL << (b->bm_bits & BITS_PER_LONG_MASK)) -1; 561 int idx, i, last_word; 562 563 /* all but last page */ 564 for (idx = 0; idx < b->bm_number_of_pages - 1; idx++) { 565 p_addr = __bm_map_pidx(b, idx); 566 for (i = 0; i < LWPP; i++) 567 bits += hweight_long(p_addr[i]); 568 __bm_unmap(p_addr); 569 cond_resched(); 570 } 571 /* last (or only) page */ 572 last_word = ((b->bm_bits - 1) & BITS_PER_PAGE_MASK) >> LN2_BPL; 573 p_addr = __bm_map_pidx(b, idx); 574 for (i = 0; i < last_word; i++) 575 bits += hweight_long(p_addr[i]); 576 p_addr[last_word] &= cpu_to_lel(mask); 577 bits += hweight_long(p_addr[last_word]); 578 /* 32bit arch, may have an unused padding long */ 579 if (BITS_PER_LONG == 32 && (last_word & 1) == 0) 580 p_addr[last_word+1] = 0; 581 __bm_unmap(p_addr); 582 return bits; 583 } 584 585 /* offset and len in long words.*/ 586 static void bm_memset(struct drbd_bitmap *b, size_t offset, int c, size_t len) 587 { 588 unsigned long *p_addr, *bm; 589 unsigned int idx; 590 size_t do_now, end; 591 592 end = offset + len; 593 594 if (end > b->bm_words) { 595 printk(KERN_ALERT "drbd: bm_memset end > bm_words\n"); 596 return; 597 } 598 599 while (offset < end) { 600 do_now = min_t(size_t, ALIGN(offset + 1, LWPP), end) - offset; 601 idx = bm_word_to_page_idx(b, offset); 602 p_addr = bm_map_pidx(b, idx); 603 bm = p_addr + MLPP(offset); 604 if (bm+do_now > p_addr + LWPP) { 605 printk(KERN_ALERT "drbd: BUG BUG BUG! p_addr:%p bm:%p do_now:%d\n", 606 p_addr, bm, (int)do_now); 607 } else 608 memset(bm, c, do_now * sizeof(long)); 609 bm_unmap(p_addr); 610 bm_set_page_need_writeout(b->bm_pages[idx]); 611 offset += do_now; 612 } 613 } 614 615 /* For the layout, see comment above drbd_md_set_sector_offsets(). */ 616 static u64 drbd_md_on_disk_bits(struct drbd_backing_dev *ldev) 617 { 618 u64 bitmap_sectors; 619 if (ldev->md.al_offset == 8) 620 bitmap_sectors = ldev->md.md_size_sect - ldev->md.bm_offset; 621 else 622 bitmap_sectors = ldev->md.al_offset - ldev->md.bm_offset; 623 return bitmap_sectors << (9 + 3); 624 } 625 626 /* 627 * make sure the bitmap has enough room for the attached storage, 628 * if necessary, resize. 629 * called whenever we may have changed the device size. 630 * returns -ENOMEM if we could not allocate enough memory, 0 on success. 631 * In case this is actually a resize, we copy the old bitmap into the new one. 632 * Otherwise, the bitmap is initialized to all bits set. 633 */ 634 int drbd_bm_resize(struct drbd_device *device, sector_t capacity, int set_new_bits) 635 { 636 struct drbd_bitmap *b = device->bitmap; 637 unsigned long bits, words, owords, obits; 638 unsigned long want, have, onpages; /* number of pages */ 639 struct page **npages, **opages = NULL; 640 int err = 0, growing; 641 int opages_vmalloced; 642 643 if (!expect(b)) 644 return -ENOMEM; 645 646 drbd_bm_lock(device, "resize", BM_LOCKED_MASK); 647 648 drbd_info(device, "drbd_bm_resize called with capacity == %llu\n", 649 (unsigned long long)capacity); 650 651 if (capacity == b->bm_dev_capacity) 652 goto out; 653 654 opages_vmalloced = (BM_P_VMALLOCED & b->bm_flags); 655 656 if (capacity == 0) { 657 spin_lock_irq(&b->bm_lock); 658 opages = b->bm_pages; 659 onpages = b->bm_number_of_pages; 660 owords = b->bm_words; 661 b->bm_pages = NULL; 662 b->bm_number_of_pages = 663 b->bm_set = 664 b->bm_bits = 665 b->bm_words = 666 b->bm_dev_capacity = 0; 667 spin_unlock_irq(&b->bm_lock); 668 bm_free_pages(opages, onpages); 669 bm_vk_free(opages, opages_vmalloced); 670 goto out; 671 } 672 bits = BM_SECT_TO_BIT(ALIGN(capacity, BM_SECT_PER_BIT)); 673 674 /* if we would use 675 words = ALIGN(bits,BITS_PER_LONG) >> LN2_BPL; 676 a 32bit host could present the wrong number of words 677 to a 64bit host. 678 */ 679 words = ALIGN(bits, 64) >> LN2_BPL; 680 681 if (get_ldev(device)) { 682 u64 bits_on_disk = drbd_md_on_disk_bits(device->ldev); 683 put_ldev(device); 684 if (bits > bits_on_disk) { 685 drbd_info(device, "bits = %lu\n", bits); 686 drbd_info(device, "bits_on_disk = %llu\n", bits_on_disk); 687 err = -ENOSPC; 688 goto out; 689 } 690 } 691 692 want = ALIGN(words*sizeof(long), PAGE_SIZE) >> PAGE_SHIFT; 693 have = b->bm_number_of_pages; 694 if (want == have) { 695 D_ASSERT(device, b->bm_pages != NULL); 696 npages = b->bm_pages; 697 } else { 698 if (drbd_insert_fault(device, DRBD_FAULT_BM_ALLOC)) 699 npages = NULL; 700 else 701 npages = bm_realloc_pages(b, want); 702 } 703 704 if (!npages) { 705 err = -ENOMEM; 706 goto out; 707 } 708 709 spin_lock_irq(&b->bm_lock); 710 opages = b->bm_pages; 711 owords = b->bm_words; 712 obits = b->bm_bits; 713 714 growing = bits > obits; 715 if (opages && growing && set_new_bits) 716 bm_set_surplus(b); 717 718 b->bm_pages = npages; 719 b->bm_number_of_pages = want; 720 b->bm_bits = bits; 721 b->bm_words = words; 722 b->bm_dev_capacity = capacity; 723 724 if (growing) { 725 if (set_new_bits) { 726 bm_memset(b, owords, 0xff, words-owords); 727 b->bm_set += bits - obits; 728 } else 729 bm_memset(b, owords, 0x00, words-owords); 730 731 } 732 733 if (want < have) { 734 /* implicit: (opages != NULL) && (opages != npages) */ 735 bm_free_pages(opages + want, have - want); 736 } 737 738 (void)bm_clear_surplus(b); 739 740 spin_unlock_irq(&b->bm_lock); 741 if (opages != npages) 742 bm_vk_free(opages, opages_vmalloced); 743 if (!growing) 744 b->bm_set = bm_count_bits(b); 745 drbd_info(device, "resync bitmap: bits=%lu words=%lu pages=%lu\n", bits, words, want); 746 747 out: 748 drbd_bm_unlock(device); 749 return err; 750 } 751 752 /* inherently racy: 753 * if not protected by other means, return value may be out of date when 754 * leaving this function... 755 * we still need to lock it, since it is important that this returns 756 * bm_set == 0 precisely. 757 * 758 * maybe bm_set should be atomic_t ? 759 */ 760 unsigned long _drbd_bm_total_weight(struct drbd_device *device) 761 { 762 struct drbd_bitmap *b = device->bitmap; 763 unsigned long s; 764 unsigned long flags; 765 766 if (!expect(b)) 767 return 0; 768 if (!expect(b->bm_pages)) 769 return 0; 770 771 spin_lock_irqsave(&b->bm_lock, flags); 772 s = b->bm_set; 773 spin_unlock_irqrestore(&b->bm_lock, flags); 774 775 return s; 776 } 777 778 unsigned long drbd_bm_total_weight(struct drbd_device *device) 779 { 780 unsigned long s; 781 /* if I don't have a disk, I don't know about out-of-sync status */ 782 if (!get_ldev_if_state(device, D_NEGOTIATING)) 783 return 0; 784 s = _drbd_bm_total_weight(device); 785 put_ldev(device); 786 return s; 787 } 788 789 size_t drbd_bm_words(struct drbd_device *device) 790 { 791 struct drbd_bitmap *b = device->bitmap; 792 if (!expect(b)) 793 return 0; 794 if (!expect(b->bm_pages)) 795 return 0; 796 797 return b->bm_words; 798 } 799 800 unsigned long drbd_bm_bits(struct drbd_device *device) 801 { 802 struct drbd_bitmap *b = device->bitmap; 803 if (!expect(b)) 804 return 0; 805 806 return b->bm_bits; 807 } 808 809 /* merge number words from buffer into the bitmap starting at offset. 810 * buffer[i] is expected to be little endian unsigned long. 811 * bitmap must be locked by drbd_bm_lock. 812 * currently only used from receive_bitmap. 813 */ 814 void drbd_bm_merge_lel(struct drbd_device *device, size_t offset, size_t number, 815 unsigned long *buffer) 816 { 817 struct drbd_bitmap *b = device->bitmap; 818 unsigned long *p_addr, *bm; 819 unsigned long word, bits; 820 unsigned int idx; 821 size_t end, do_now; 822 823 end = offset + number; 824 825 if (!expect(b)) 826 return; 827 if (!expect(b->bm_pages)) 828 return; 829 if (number == 0) 830 return; 831 WARN_ON(offset >= b->bm_words); 832 WARN_ON(end > b->bm_words); 833 834 spin_lock_irq(&b->bm_lock); 835 while (offset < end) { 836 do_now = min_t(size_t, ALIGN(offset+1, LWPP), end) - offset; 837 idx = bm_word_to_page_idx(b, offset); 838 p_addr = bm_map_pidx(b, idx); 839 bm = p_addr + MLPP(offset); 840 offset += do_now; 841 while (do_now--) { 842 bits = hweight_long(*bm); 843 word = *bm | *buffer++; 844 *bm++ = word; 845 b->bm_set += hweight_long(word) - bits; 846 } 847 bm_unmap(p_addr); 848 bm_set_page_need_writeout(b->bm_pages[idx]); 849 } 850 /* with 32bit <-> 64bit cross-platform connect 851 * this is only correct for current usage, 852 * where we _know_ that we are 64 bit aligned, 853 * and know that this function is used in this way, too... 854 */ 855 if (end == b->bm_words) 856 b->bm_set -= bm_clear_surplus(b); 857 spin_unlock_irq(&b->bm_lock); 858 } 859 860 /* copy number words from the bitmap starting at offset into the buffer. 861 * buffer[i] will be little endian unsigned long. 862 */ 863 void drbd_bm_get_lel(struct drbd_device *device, size_t offset, size_t number, 864 unsigned long *buffer) 865 { 866 struct drbd_bitmap *b = device->bitmap; 867 unsigned long *p_addr, *bm; 868 size_t end, do_now; 869 870 end = offset + number; 871 872 if (!expect(b)) 873 return; 874 if (!expect(b->bm_pages)) 875 return; 876 877 spin_lock_irq(&b->bm_lock); 878 if ((offset >= b->bm_words) || 879 (end > b->bm_words) || 880 (number <= 0)) 881 drbd_err(device, "offset=%lu number=%lu bm_words=%lu\n", 882 (unsigned long) offset, 883 (unsigned long) number, 884 (unsigned long) b->bm_words); 885 else { 886 while (offset < end) { 887 do_now = min_t(size_t, ALIGN(offset+1, LWPP), end) - offset; 888 p_addr = bm_map_pidx(b, bm_word_to_page_idx(b, offset)); 889 bm = p_addr + MLPP(offset); 890 offset += do_now; 891 while (do_now--) 892 *buffer++ = *bm++; 893 bm_unmap(p_addr); 894 } 895 } 896 spin_unlock_irq(&b->bm_lock); 897 } 898 899 /* set all bits in the bitmap */ 900 void drbd_bm_set_all(struct drbd_device *device) 901 { 902 struct drbd_bitmap *b = device->bitmap; 903 if (!expect(b)) 904 return; 905 if (!expect(b->bm_pages)) 906 return; 907 908 spin_lock_irq(&b->bm_lock); 909 bm_memset(b, 0, 0xff, b->bm_words); 910 (void)bm_clear_surplus(b); 911 b->bm_set = b->bm_bits; 912 spin_unlock_irq(&b->bm_lock); 913 } 914 915 /* clear all bits in the bitmap */ 916 void drbd_bm_clear_all(struct drbd_device *device) 917 { 918 struct drbd_bitmap *b = device->bitmap; 919 if (!expect(b)) 920 return; 921 if (!expect(b->bm_pages)) 922 return; 923 924 spin_lock_irq(&b->bm_lock); 925 bm_memset(b, 0, 0, b->bm_words); 926 b->bm_set = 0; 927 spin_unlock_irq(&b->bm_lock); 928 } 929 930 struct bm_aio_ctx { 931 struct drbd_device *device; 932 atomic_t in_flight; 933 unsigned int done; 934 unsigned flags; 935 #define BM_AIO_COPY_PAGES 1 936 #define BM_AIO_WRITE_HINTED 2 937 #define BM_WRITE_ALL_PAGES 4 938 int error; 939 struct kref kref; 940 }; 941 942 static void bm_aio_ctx_destroy(struct kref *kref) 943 { 944 struct bm_aio_ctx *ctx = container_of(kref, struct bm_aio_ctx, kref); 945 946 put_ldev(ctx->device); 947 kfree(ctx); 948 } 949 950 /* bv_page may be a copy, or may be the original */ 951 static void bm_async_io_complete(struct bio *bio, int error) 952 { 953 struct bm_aio_ctx *ctx = bio->bi_private; 954 struct drbd_device *device = ctx->device; 955 struct drbd_bitmap *b = device->bitmap; 956 unsigned int idx = bm_page_to_idx(bio->bi_io_vec[0].bv_page); 957 int uptodate = bio_flagged(bio, BIO_UPTODATE); 958 959 960 /* strange behavior of some lower level drivers... 961 * fail the request by clearing the uptodate flag, 962 * but do not return any error?! 963 * do we want to WARN() on this? */ 964 if (!error && !uptodate) 965 error = -EIO; 966 967 if ((ctx->flags & BM_AIO_COPY_PAGES) == 0 && 968 !bm_test_page_unchanged(b->bm_pages[idx])) 969 drbd_warn(device, "bitmap page idx %u changed during IO!\n", idx); 970 971 if (error) { 972 /* ctx error will hold the completed-last non-zero error code, 973 * in case error codes differ. */ 974 ctx->error = error; 975 bm_set_page_io_err(b->bm_pages[idx]); 976 /* Not identical to on disk version of it. 977 * Is BM_PAGE_IO_ERROR enough? */ 978 if (__ratelimit(&drbd_ratelimit_state)) 979 drbd_err(device, "IO ERROR %d on bitmap page idx %u\n", 980 error, idx); 981 } else { 982 bm_clear_page_io_err(b->bm_pages[idx]); 983 dynamic_drbd_dbg(device, "bitmap page idx %u completed\n", idx); 984 } 985 986 bm_page_unlock_io(device, idx); 987 988 if (ctx->flags & BM_AIO_COPY_PAGES) 989 mempool_free(bio->bi_io_vec[0].bv_page, drbd_md_io_page_pool); 990 991 bio_put(bio); 992 993 if (atomic_dec_and_test(&ctx->in_flight)) { 994 ctx->done = 1; 995 wake_up(&device->misc_wait); 996 kref_put(&ctx->kref, &bm_aio_ctx_destroy); 997 } 998 } 999 1000 static void bm_page_io_async(struct bm_aio_ctx *ctx, int page_nr, int rw) __must_hold(local) 1001 { 1002 struct bio *bio = bio_alloc_drbd(GFP_NOIO); 1003 struct drbd_device *device = ctx->device; 1004 struct drbd_bitmap *b = device->bitmap; 1005 struct page *page; 1006 unsigned int len; 1007 1008 sector_t on_disk_sector = 1009 device->ldev->md.md_offset + device->ldev->md.bm_offset; 1010 on_disk_sector += ((sector_t)page_nr) << (PAGE_SHIFT-9); 1011 1012 /* this might happen with very small 1013 * flexible external meta data device, 1014 * or with PAGE_SIZE > 4k */ 1015 len = min_t(unsigned int, PAGE_SIZE, 1016 (drbd_md_last_sector(device->ldev) - on_disk_sector + 1)<<9); 1017 1018 /* serialize IO on this page */ 1019 bm_page_lock_io(device, page_nr); 1020 /* before memcpy and submit, 1021 * so it can be redirtied any time */ 1022 bm_set_page_unchanged(b->bm_pages[page_nr]); 1023 1024 if (ctx->flags & BM_AIO_COPY_PAGES) { 1025 page = mempool_alloc(drbd_md_io_page_pool, __GFP_HIGHMEM|__GFP_WAIT); 1026 copy_highpage(page, b->bm_pages[page_nr]); 1027 bm_store_page_idx(page, page_nr); 1028 } else 1029 page = b->bm_pages[page_nr]; 1030 bio->bi_bdev = device->ldev->md_bdev; 1031 bio->bi_iter.bi_sector = on_disk_sector; 1032 /* bio_add_page of a single page to an empty bio will always succeed, 1033 * according to api. Do we want to assert that? */ 1034 bio_add_page(bio, page, len, 0); 1035 bio->bi_private = ctx; 1036 bio->bi_end_io = bm_async_io_complete; 1037 1038 if (drbd_insert_fault(device, (rw & WRITE) ? DRBD_FAULT_MD_WR : DRBD_FAULT_MD_RD)) { 1039 bio->bi_rw |= rw; 1040 bio_endio(bio, -EIO); 1041 } else { 1042 submit_bio(rw, bio); 1043 /* this should not count as user activity and cause the 1044 * resync to throttle -- see drbd_rs_should_slow_down(). */ 1045 atomic_add(len >> 9, &device->rs_sect_ev); 1046 } 1047 } 1048 1049 /* 1050 * bm_rw: read/write the whole bitmap from/to its on disk location. 1051 */ 1052 static int bm_rw(struct drbd_device *device, int rw, unsigned flags, unsigned lazy_writeout_upper_idx) __must_hold(local) 1053 { 1054 struct bm_aio_ctx *ctx; 1055 struct drbd_bitmap *b = device->bitmap; 1056 int num_pages, i, count = 0; 1057 unsigned long now; 1058 char ppb[10]; 1059 int err = 0; 1060 1061 /* 1062 * We are protected against bitmap disappearing/resizing by holding an 1063 * ldev reference (caller must have called get_ldev()). 1064 * For read/write, we are protected against changes to the bitmap by 1065 * the bitmap lock (see drbd_bitmap_io). 1066 * For lazy writeout, we don't care for ongoing changes to the bitmap, 1067 * as we submit copies of pages anyways. 1068 */ 1069 1070 ctx = kmalloc(sizeof(struct bm_aio_ctx), GFP_NOIO); 1071 if (!ctx) 1072 return -ENOMEM; 1073 1074 *ctx = (struct bm_aio_ctx) { 1075 .device = device, 1076 .in_flight = ATOMIC_INIT(1), 1077 .done = 0, 1078 .flags = flags, 1079 .error = 0, 1080 .kref = { ATOMIC_INIT(2) }, 1081 }; 1082 1083 if (!get_ldev_if_state(device, D_ATTACHING)) { /* put is in bm_aio_ctx_destroy() */ 1084 drbd_err(device, "ASSERT FAILED: get_ldev_if_state() == 1 in bm_rw()\n"); 1085 kfree(ctx); 1086 return -ENODEV; 1087 } 1088 1089 if (!ctx->flags) 1090 WARN_ON(!(BM_LOCKED_MASK & b->bm_flags)); 1091 1092 num_pages = b->bm_number_of_pages; 1093 1094 now = jiffies; 1095 1096 /* let the layers below us try to merge these bios... */ 1097 for (i = 0; i < num_pages; i++) { 1098 /* ignore completely unchanged pages */ 1099 if (lazy_writeout_upper_idx && i == lazy_writeout_upper_idx) 1100 break; 1101 if (rw & WRITE) { 1102 if ((flags & BM_AIO_WRITE_HINTED) && 1103 !test_and_clear_bit(BM_PAGE_HINT_WRITEOUT, 1104 &page_private(b->bm_pages[i]))) 1105 continue; 1106 1107 if (!(flags & BM_WRITE_ALL_PAGES) && 1108 bm_test_page_unchanged(b->bm_pages[i])) { 1109 dynamic_drbd_dbg(device, "skipped bm write for idx %u\n", i); 1110 continue; 1111 } 1112 /* during lazy writeout, 1113 * ignore those pages not marked for lazy writeout. */ 1114 if (lazy_writeout_upper_idx && 1115 !bm_test_page_lazy_writeout(b->bm_pages[i])) { 1116 dynamic_drbd_dbg(device, "skipped bm lazy write for idx %u\n", i); 1117 continue; 1118 } 1119 } 1120 atomic_inc(&ctx->in_flight); 1121 bm_page_io_async(ctx, i, rw); 1122 ++count; 1123 cond_resched(); 1124 } 1125 1126 /* 1127 * We initialize ctx->in_flight to one to make sure bm_async_io_complete 1128 * will not set ctx->done early, and decrement / test it here. If there 1129 * are still some bios in flight, we need to wait for them here. 1130 * If all IO is done already (or nothing had been submitted), there is 1131 * no need to wait. Still, we need to put the kref associated with the 1132 * "in_flight reached zero, all done" event. 1133 */ 1134 if (!atomic_dec_and_test(&ctx->in_flight)) 1135 wait_until_done_or_force_detached(device, device->ldev, &ctx->done); 1136 else 1137 kref_put(&ctx->kref, &bm_aio_ctx_destroy); 1138 1139 /* summary for global bitmap IO */ 1140 if (flags == 0) 1141 drbd_info(device, "bitmap %s of %u pages took %lu jiffies\n", 1142 rw == WRITE ? "WRITE" : "READ", 1143 count, jiffies - now); 1144 1145 if (ctx->error) { 1146 drbd_alert(device, "we had at least one MD IO ERROR during bitmap IO\n"); 1147 drbd_chk_io_error(device, 1, DRBD_META_IO_ERROR); 1148 err = -EIO; /* ctx->error ? */ 1149 } 1150 1151 if (atomic_read(&ctx->in_flight)) 1152 err = -EIO; /* Disk timeout/force-detach during IO... */ 1153 1154 now = jiffies; 1155 if (rw == WRITE) { 1156 drbd_md_flush(device); 1157 } else /* rw == READ */ { 1158 b->bm_set = bm_count_bits(b); 1159 drbd_info(device, "recounting of set bits took additional %lu jiffies\n", 1160 jiffies - now); 1161 } 1162 now = b->bm_set; 1163 1164 if (flags == 0) 1165 drbd_info(device, "%s (%lu bits) marked out-of-sync by on disk bit-map.\n", 1166 ppsize(ppb, now << (BM_BLOCK_SHIFT-10)), now); 1167 1168 kref_put(&ctx->kref, &bm_aio_ctx_destroy); 1169 return err; 1170 } 1171 1172 /** 1173 * drbd_bm_read() - Read the whole bitmap from its on disk location. 1174 * @device: DRBD device. 1175 */ 1176 int drbd_bm_read(struct drbd_device *device) __must_hold(local) 1177 { 1178 return bm_rw(device, READ, 0, 0); 1179 } 1180 1181 /** 1182 * drbd_bm_write() - Write the whole bitmap to its on disk location. 1183 * @device: DRBD device. 1184 * 1185 * Will only write pages that have changed since last IO. 1186 */ 1187 int drbd_bm_write(struct drbd_device *device) __must_hold(local) 1188 { 1189 return bm_rw(device, WRITE, 0, 0); 1190 } 1191 1192 /** 1193 * drbd_bm_write_all() - Write the whole bitmap to its on disk location. 1194 * @device: DRBD device. 1195 * 1196 * Will write all pages. 1197 */ 1198 int drbd_bm_write_all(struct drbd_device *device) __must_hold(local) 1199 { 1200 return bm_rw(device, WRITE, BM_WRITE_ALL_PAGES, 0); 1201 } 1202 1203 /** 1204 * drbd_bm_write_copy_pages() - Write the whole bitmap to its on disk location. 1205 * @device: DRBD device. 1206 * 1207 * Will only write pages that have changed since last IO. 1208 * In contrast to drbd_bm_write(), this will copy the bitmap pages 1209 * to temporary writeout pages. It is intended to trigger a full write-out 1210 * while still allowing the bitmap to change, for example if a resync or online 1211 * verify is aborted due to a failed peer disk, while local IO continues, or 1212 * pending resync acks are still being processed. 1213 */ 1214 int drbd_bm_write_copy_pages(struct drbd_device *device) __must_hold(local) 1215 { 1216 return bm_rw(device, WRITE, BM_AIO_COPY_PAGES, 0); 1217 } 1218 1219 /** 1220 * drbd_bm_write_hinted() - Write bitmap pages with "hint" marks, if they have changed. 1221 * @device: DRBD device. 1222 */ 1223 int drbd_bm_write_hinted(struct drbd_device *device) __must_hold(local) 1224 { 1225 return bm_rw(device, WRITE, BM_AIO_WRITE_HINTED | BM_AIO_COPY_PAGES, 0); 1226 } 1227 1228 /** 1229 * drbd_bm_write_page() - Writes a PAGE_SIZE aligned piece of bitmap 1230 * @device: DRBD device. 1231 * @idx: bitmap page index 1232 * 1233 * We don't want to special case on logical_block_size of the backend device, 1234 * so we submit PAGE_SIZE aligned pieces. 1235 * Note that on "most" systems, PAGE_SIZE is 4k. 1236 * 1237 * In case this becomes an issue on systems with larger PAGE_SIZE, 1238 * we may want to change this again to write 4k aligned 4k pieces. 1239 */ 1240 int drbd_bm_write_page(struct drbd_device *device, unsigned int idx) __must_hold(local) 1241 { 1242 struct bm_aio_ctx *ctx; 1243 int err; 1244 1245 if (bm_test_page_unchanged(device->bitmap->bm_pages[idx])) { 1246 dynamic_drbd_dbg(device, "skipped bm page write for idx %u\n", idx); 1247 return 0; 1248 } 1249 1250 ctx = kmalloc(sizeof(struct bm_aio_ctx), GFP_NOIO); 1251 if (!ctx) 1252 return -ENOMEM; 1253 1254 *ctx = (struct bm_aio_ctx) { 1255 .device = device, 1256 .in_flight = ATOMIC_INIT(1), 1257 .done = 0, 1258 .flags = BM_AIO_COPY_PAGES, 1259 .error = 0, 1260 .kref = { ATOMIC_INIT(2) }, 1261 }; 1262 1263 if (!get_ldev_if_state(device, D_ATTACHING)) { /* put is in bm_aio_ctx_destroy() */ 1264 drbd_err(device, "ASSERT FAILED: get_ldev_if_state() == 1 in drbd_bm_write_page()\n"); 1265 kfree(ctx); 1266 return -ENODEV; 1267 } 1268 1269 bm_page_io_async(ctx, idx, WRITE_SYNC); 1270 wait_until_done_or_force_detached(device, device->ldev, &ctx->done); 1271 1272 if (ctx->error) 1273 drbd_chk_io_error(device, 1, DRBD_META_IO_ERROR); 1274 /* that causes us to detach, so the in memory bitmap will be 1275 * gone in a moment as well. */ 1276 1277 device->bm_writ_cnt++; 1278 err = atomic_read(&ctx->in_flight) ? -EIO : ctx->error; 1279 kref_put(&ctx->kref, &bm_aio_ctx_destroy); 1280 return err; 1281 } 1282 1283 /* NOTE 1284 * find_first_bit returns int, we return unsigned long. 1285 * For this to work on 32bit arch with bitnumbers > (1<<32), 1286 * we'd need to return u64, and get a whole lot of other places 1287 * fixed where we still use unsigned long. 1288 * 1289 * this returns a bit number, NOT a sector! 1290 */ 1291 static unsigned long __bm_find_next(struct drbd_device *device, unsigned long bm_fo, 1292 const int find_zero_bit) 1293 { 1294 struct drbd_bitmap *b = device->bitmap; 1295 unsigned long *p_addr; 1296 unsigned long bit_offset; 1297 unsigned i; 1298 1299 1300 if (bm_fo > b->bm_bits) { 1301 drbd_err(device, "bm_fo=%lu bm_bits=%lu\n", bm_fo, b->bm_bits); 1302 bm_fo = DRBD_END_OF_BITMAP; 1303 } else { 1304 while (bm_fo < b->bm_bits) { 1305 /* bit offset of the first bit in the page */ 1306 bit_offset = bm_fo & ~BITS_PER_PAGE_MASK; 1307 p_addr = __bm_map_pidx(b, bm_bit_to_page_idx(b, bm_fo)); 1308 1309 if (find_zero_bit) 1310 i = find_next_zero_bit_le(p_addr, 1311 PAGE_SIZE*8, bm_fo & BITS_PER_PAGE_MASK); 1312 else 1313 i = find_next_bit_le(p_addr, 1314 PAGE_SIZE*8, bm_fo & BITS_PER_PAGE_MASK); 1315 1316 __bm_unmap(p_addr); 1317 if (i < PAGE_SIZE*8) { 1318 bm_fo = bit_offset + i; 1319 if (bm_fo >= b->bm_bits) 1320 break; 1321 goto found; 1322 } 1323 bm_fo = bit_offset + PAGE_SIZE*8; 1324 } 1325 bm_fo = DRBD_END_OF_BITMAP; 1326 } 1327 found: 1328 return bm_fo; 1329 } 1330 1331 static unsigned long bm_find_next(struct drbd_device *device, 1332 unsigned long bm_fo, const int find_zero_bit) 1333 { 1334 struct drbd_bitmap *b = device->bitmap; 1335 unsigned long i = DRBD_END_OF_BITMAP; 1336 1337 if (!expect(b)) 1338 return i; 1339 if (!expect(b->bm_pages)) 1340 return i; 1341 1342 spin_lock_irq(&b->bm_lock); 1343 if (BM_DONT_TEST & b->bm_flags) 1344 bm_print_lock_info(device); 1345 1346 i = __bm_find_next(device, bm_fo, find_zero_bit); 1347 1348 spin_unlock_irq(&b->bm_lock); 1349 return i; 1350 } 1351 1352 unsigned long drbd_bm_find_next(struct drbd_device *device, unsigned long bm_fo) 1353 { 1354 return bm_find_next(device, bm_fo, 0); 1355 } 1356 1357 #if 0 1358 /* not yet needed for anything. */ 1359 unsigned long drbd_bm_find_next_zero(struct drbd_device *device, unsigned long bm_fo) 1360 { 1361 return bm_find_next(device, bm_fo, 1); 1362 } 1363 #endif 1364 1365 /* does not spin_lock_irqsave. 1366 * you must take drbd_bm_lock() first */ 1367 unsigned long _drbd_bm_find_next(struct drbd_device *device, unsigned long bm_fo) 1368 { 1369 /* WARN_ON(!(BM_DONT_SET & device->b->bm_flags)); */ 1370 return __bm_find_next(device, bm_fo, 0); 1371 } 1372 1373 unsigned long _drbd_bm_find_next_zero(struct drbd_device *device, unsigned long bm_fo) 1374 { 1375 /* WARN_ON(!(BM_DONT_SET & device->b->bm_flags)); */ 1376 return __bm_find_next(device, bm_fo, 1); 1377 } 1378 1379 /* returns number of bits actually changed. 1380 * for val != 0, we change 0 -> 1, return code positive 1381 * for val == 0, we change 1 -> 0, return code negative 1382 * wants bitnr, not sector. 1383 * expected to be called for only a few bits (e - s about BITS_PER_LONG). 1384 * Must hold bitmap lock already. */ 1385 static int __bm_change_bits_to(struct drbd_device *device, const unsigned long s, 1386 unsigned long e, int val) 1387 { 1388 struct drbd_bitmap *b = device->bitmap; 1389 unsigned long *p_addr = NULL; 1390 unsigned long bitnr; 1391 unsigned int last_page_nr = -1U; 1392 int c = 0; 1393 int changed_total = 0; 1394 1395 if (e >= b->bm_bits) { 1396 drbd_err(device, "ASSERT FAILED: bit_s=%lu bit_e=%lu bm_bits=%lu\n", 1397 s, e, b->bm_bits); 1398 e = b->bm_bits ? b->bm_bits -1 : 0; 1399 } 1400 for (bitnr = s; bitnr <= e; bitnr++) { 1401 unsigned int page_nr = bm_bit_to_page_idx(b, bitnr); 1402 if (page_nr != last_page_nr) { 1403 if (p_addr) 1404 __bm_unmap(p_addr); 1405 if (c < 0) 1406 bm_set_page_lazy_writeout(b->bm_pages[last_page_nr]); 1407 else if (c > 0) 1408 bm_set_page_need_writeout(b->bm_pages[last_page_nr]); 1409 changed_total += c; 1410 c = 0; 1411 p_addr = __bm_map_pidx(b, page_nr); 1412 last_page_nr = page_nr; 1413 } 1414 if (val) 1415 c += (0 == __test_and_set_bit_le(bitnr & BITS_PER_PAGE_MASK, p_addr)); 1416 else 1417 c -= (0 != __test_and_clear_bit_le(bitnr & BITS_PER_PAGE_MASK, p_addr)); 1418 } 1419 if (p_addr) 1420 __bm_unmap(p_addr); 1421 if (c < 0) 1422 bm_set_page_lazy_writeout(b->bm_pages[last_page_nr]); 1423 else if (c > 0) 1424 bm_set_page_need_writeout(b->bm_pages[last_page_nr]); 1425 changed_total += c; 1426 b->bm_set += changed_total; 1427 return changed_total; 1428 } 1429 1430 /* returns number of bits actually changed. 1431 * for val != 0, we change 0 -> 1, return code positive 1432 * for val == 0, we change 1 -> 0, return code negative 1433 * wants bitnr, not sector */ 1434 static int bm_change_bits_to(struct drbd_device *device, const unsigned long s, 1435 const unsigned long e, int val) 1436 { 1437 unsigned long flags; 1438 struct drbd_bitmap *b = device->bitmap; 1439 int c = 0; 1440 1441 if (!expect(b)) 1442 return 1; 1443 if (!expect(b->bm_pages)) 1444 return 0; 1445 1446 spin_lock_irqsave(&b->bm_lock, flags); 1447 if ((val ? BM_DONT_SET : BM_DONT_CLEAR) & b->bm_flags) 1448 bm_print_lock_info(device); 1449 1450 c = __bm_change_bits_to(device, s, e, val); 1451 1452 spin_unlock_irqrestore(&b->bm_lock, flags); 1453 return c; 1454 } 1455 1456 /* returns number of bits changed 0 -> 1 */ 1457 int drbd_bm_set_bits(struct drbd_device *device, const unsigned long s, const unsigned long e) 1458 { 1459 return bm_change_bits_to(device, s, e, 1); 1460 } 1461 1462 /* returns number of bits changed 1 -> 0 */ 1463 int drbd_bm_clear_bits(struct drbd_device *device, const unsigned long s, const unsigned long e) 1464 { 1465 return -bm_change_bits_to(device, s, e, 0); 1466 } 1467 1468 /* sets all bits in full words, 1469 * from first_word up to, but not including, last_word */ 1470 static inline void bm_set_full_words_within_one_page(struct drbd_bitmap *b, 1471 int page_nr, int first_word, int last_word) 1472 { 1473 int i; 1474 int bits; 1475 int changed = 0; 1476 unsigned long *paddr = kmap_atomic(b->bm_pages[page_nr]); 1477 for (i = first_word; i < last_word; i++) { 1478 bits = hweight_long(paddr[i]); 1479 paddr[i] = ~0UL; 1480 changed += BITS_PER_LONG - bits; 1481 } 1482 kunmap_atomic(paddr); 1483 if (changed) { 1484 /* We only need lazy writeout, the information is still in the 1485 * remote bitmap as well, and is reconstructed during the next 1486 * bitmap exchange, if lost locally due to a crash. */ 1487 bm_set_page_lazy_writeout(b->bm_pages[page_nr]); 1488 b->bm_set += changed; 1489 } 1490 } 1491 1492 /* Same thing as drbd_bm_set_bits, 1493 * but more efficient for a large bit range. 1494 * You must first drbd_bm_lock(). 1495 * Can be called to set the whole bitmap in one go. 1496 * Sets bits from s to e _inclusive_. */ 1497 void _drbd_bm_set_bits(struct drbd_device *device, const unsigned long s, const unsigned long e) 1498 { 1499 /* First set_bit from the first bit (s) 1500 * up to the next long boundary (sl), 1501 * then assign full words up to the last long boundary (el), 1502 * then set_bit up to and including the last bit (e). 1503 * 1504 * Do not use memset, because we must account for changes, 1505 * so we need to loop over the words with hweight() anyways. 1506 */ 1507 struct drbd_bitmap *b = device->bitmap; 1508 unsigned long sl = ALIGN(s,BITS_PER_LONG); 1509 unsigned long el = (e+1) & ~((unsigned long)BITS_PER_LONG-1); 1510 int first_page; 1511 int last_page; 1512 int page_nr; 1513 int first_word; 1514 int last_word; 1515 1516 if (e - s <= 3*BITS_PER_LONG) { 1517 /* don't bother; el and sl may even be wrong. */ 1518 spin_lock_irq(&b->bm_lock); 1519 __bm_change_bits_to(device, s, e, 1); 1520 spin_unlock_irq(&b->bm_lock); 1521 return; 1522 } 1523 1524 /* difference is large enough that we can trust sl and el */ 1525 1526 spin_lock_irq(&b->bm_lock); 1527 1528 /* bits filling the current long */ 1529 if (sl) 1530 __bm_change_bits_to(device, s, sl-1, 1); 1531 1532 first_page = sl >> (3 + PAGE_SHIFT); 1533 last_page = el >> (3 + PAGE_SHIFT); 1534 1535 /* MLPP: modulo longs per page */ 1536 /* LWPP: long words per page */ 1537 first_word = MLPP(sl >> LN2_BPL); 1538 last_word = LWPP; 1539 1540 /* first and full pages, unless first page == last page */ 1541 for (page_nr = first_page; page_nr < last_page; page_nr++) { 1542 bm_set_full_words_within_one_page(device->bitmap, page_nr, first_word, last_word); 1543 spin_unlock_irq(&b->bm_lock); 1544 cond_resched(); 1545 first_word = 0; 1546 spin_lock_irq(&b->bm_lock); 1547 } 1548 /* last page (respectively only page, for first page == last page) */ 1549 last_word = MLPP(el >> LN2_BPL); 1550 1551 /* consider bitmap->bm_bits = 32768, bitmap->bm_number_of_pages = 1. (or multiples). 1552 * ==> e = 32767, el = 32768, last_page = 2, 1553 * and now last_word = 0. 1554 * We do not want to touch last_page in this case, 1555 * as we did not allocate it, it is not present in bitmap->bm_pages. 1556 */ 1557 if (last_word) 1558 bm_set_full_words_within_one_page(device->bitmap, last_page, first_word, last_word); 1559 1560 /* possibly trailing bits. 1561 * example: (e & 63) == 63, el will be e+1. 1562 * if that even was the very last bit, 1563 * it would trigger an assert in __bm_change_bits_to() 1564 */ 1565 if (el <= e) 1566 __bm_change_bits_to(device, el, e, 1); 1567 spin_unlock_irq(&b->bm_lock); 1568 } 1569 1570 /* returns bit state 1571 * wants bitnr, NOT sector. 1572 * inherently racy... area needs to be locked by means of {al,rs}_lru 1573 * 1 ... bit set 1574 * 0 ... bit not set 1575 * -1 ... first out of bounds access, stop testing for bits! 1576 */ 1577 int drbd_bm_test_bit(struct drbd_device *device, const unsigned long bitnr) 1578 { 1579 unsigned long flags; 1580 struct drbd_bitmap *b = device->bitmap; 1581 unsigned long *p_addr; 1582 int i; 1583 1584 if (!expect(b)) 1585 return 0; 1586 if (!expect(b->bm_pages)) 1587 return 0; 1588 1589 spin_lock_irqsave(&b->bm_lock, flags); 1590 if (BM_DONT_TEST & b->bm_flags) 1591 bm_print_lock_info(device); 1592 if (bitnr < b->bm_bits) { 1593 p_addr = bm_map_pidx(b, bm_bit_to_page_idx(b, bitnr)); 1594 i = test_bit_le(bitnr & BITS_PER_PAGE_MASK, p_addr) ? 1 : 0; 1595 bm_unmap(p_addr); 1596 } else if (bitnr == b->bm_bits) { 1597 i = -1; 1598 } else { /* (bitnr > b->bm_bits) */ 1599 drbd_err(device, "bitnr=%lu > bm_bits=%lu\n", bitnr, b->bm_bits); 1600 i = 0; 1601 } 1602 1603 spin_unlock_irqrestore(&b->bm_lock, flags); 1604 return i; 1605 } 1606 1607 /* returns number of bits set in the range [s, e] */ 1608 int drbd_bm_count_bits(struct drbd_device *device, const unsigned long s, const unsigned long e) 1609 { 1610 unsigned long flags; 1611 struct drbd_bitmap *b = device->bitmap; 1612 unsigned long *p_addr = NULL; 1613 unsigned long bitnr; 1614 unsigned int page_nr = -1U; 1615 int c = 0; 1616 1617 /* If this is called without a bitmap, that is a bug. But just to be 1618 * robust in case we screwed up elsewhere, in that case pretend there 1619 * was one dirty bit in the requested area, so we won't try to do a 1620 * local read there (no bitmap probably implies no disk) */ 1621 if (!expect(b)) 1622 return 1; 1623 if (!expect(b->bm_pages)) 1624 return 1; 1625 1626 spin_lock_irqsave(&b->bm_lock, flags); 1627 if (BM_DONT_TEST & b->bm_flags) 1628 bm_print_lock_info(device); 1629 for (bitnr = s; bitnr <= e; bitnr++) { 1630 unsigned int idx = bm_bit_to_page_idx(b, bitnr); 1631 if (page_nr != idx) { 1632 page_nr = idx; 1633 if (p_addr) 1634 bm_unmap(p_addr); 1635 p_addr = bm_map_pidx(b, idx); 1636 } 1637 if (expect(bitnr < b->bm_bits)) 1638 c += (0 != test_bit_le(bitnr - (page_nr << (PAGE_SHIFT+3)), p_addr)); 1639 else 1640 drbd_err(device, "bitnr=%lu bm_bits=%lu\n", bitnr, b->bm_bits); 1641 } 1642 if (p_addr) 1643 bm_unmap(p_addr); 1644 spin_unlock_irqrestore(&b->bm_lock, flags); 1645 return c; 1646 } 1647 1648 1649 /* inherently racy... 1650 * return value may be already out-of-date when this function returns. 1651 * but the general usage is that this is only use during a cstate when bits are 1652 * only cleared, not set, and typically only care for the case when the return 1653 * value is zero, or we already "locked" this "bitmap extent" by other means. 1654 * 1655 * enr is bm-extent number, since we chose to name one sector (512 bytes) 1656 * worth of the bitmap a "bitmap extent". 1657 * 1658 * TODO 1659 * I think since we use it like a reference count, we should use the real 1660 * reference count of some bitmap extent element from some lru instead... 1661 * 1662 */ 1663 int drbd_bm_e_weight(struct drbd_device *device, unsigned long enr) 1664 { 1665 struct drbd_bitmap *b = device->bitmap; 1666 int count, s, e; 1667 unsigned long flags; 1668 unsigned long *p_addr, *bm; 1669 1670 if (!expect(b)) 1671 return 0; 1672 if (!expect(b->bm_pages)) 1673 return 0; 1674 1675 spin_lock_irqsave(&b->bm_lock, flags); 1676 if (BM_DONT_TEST & b->bm_flags) 1677 bm_print_lock_info(device); 1678 1679 s = S2W(enr); 1680 e = min((size_t)S2W(enr+1), b->bm_words); 1681 count = 0; 1682 if (s < b->bm_words) { 1683 int n = e-s; 1684 p_addr = bm_map_pidx(b, bm_word_to_page_idx(b, s)); 1685 bm = p_addr + MLPP(s); 1686 while (n--) 1687 count += hweight_long(*bm++); 1688 bm_unmap(p_addr); 1689 } else { 1690 drbd_err(device, "start offset (%d) too large in drbd_bm_e_weight\n", s); 1691 } 1692 spin_unlock_irqrestore(&b->bm_lock, flags); 1693 return count; 1694 } 1695