1b411b363SPhilipp Reisner /* 2b411b363SPhilipp Reisner drbd_worker.c 3b411b363SPhilipp Reisner 4b411b363SPhilipp Reisner This file is part of DRBD by Philipp Reisner and Lars Ellenberg. 5b411b363SPhilipp Reisner 6b411b363SPhilipp Reisner Copyright (C) 2001-2008, LINBIT Information Technologies GmbH. 7b411b363SPhilipp Reisner Copyright (C) 1999-2008, Philipp Reisner <philipp.reisner@linbit.com>. 8b411b363SPhilipp Reisner Copyright (C) 2002-2008, Lars Ellenberg <lars.ellenberg@linbit.com>. 9b411b363SPhilipp Reisner 10b411b363SPhilipp Reisner drbd is free software; you can redistribute it and/or modify 11b411b363SPhilipp Reisner it under the terms of the GNU General Public License as published by 12b411b363SPhilipp Reisner the Free Software Foundation; either version 2, or (at your option) 13b411b363SPhilipp Reisner any later version. 14b411b363SPhilipp Reisner 15b411b363SPhilipp Reisner drbd is distributed in the hope that it will be useful, 16b411b363SPhilipp Reisner but WITHOUT ANY WARRANTY; without even the implied warranty of 17b411b363SPhilipp Reisner MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 18b411b363SPhilipp Reisner GNU General Public License for more details. 19b411b363SPhilipp Reisner 20b411b363SPhilipp Reisner You should have received a copy of the GNU General Public License 21b411b363SPhilipp Reisner along with drbd; see the file COPYING. If not, write to 22b411b363SPhilipp Reisner the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA. 23b411b363SPhilipp Reisner 24b411b363SPhilipp Reisner */ 25b411b363SPhilipp Reisner 26b411b363SPhilipp Reisner #include <linux/module.h> 27b411b363SPhilipp Reisner #include <linux/drbd.h> 28b411b363SPhilipp Reisner #include <linux/sched.h> 29b411b363SPhilipp Reisner #include <linux/wait.h> 30b411b363SPhilipp Reisner #include <linux/mm.h> 31b411b363SPhilipp Reisner #include <linux/memcontrol.h> 32b411b363SPhilipp Reisner #include <linux/mm_inline.h> 33b411b363SPhilipp Reisner #include <linux/slab.h> 34b411b363SPhilipp Reisner #include <linux/random.h> 35b411b363SPhilipp Reisner #include <linux/string.h> 36b411b363SPhilipp Reisner #include <linux/scatterlist.h> 37b411b363SPhilipp Reisner 38b411b363SPhilipp Reisner #include "drbd_int.h" 39b411b363SPhilipp Reisner #include "drbd_req.h" 40b411b363SPhilipp Reisner 4100d56944SPhilipp Reisner static int w_make_ov_request(struct drbd_work *w, int cancel); 42b411b363SPhilipp Reisner 43b411b363SPhilipp Reisner 44c5a91619SAndreas Gruenbacher /* endio handlers: 45c5a91619SAndreas Gruenbacher * drbd_md_io_complete (defined here) 46fcefa62eSAndreas Gruenbacher * drbd_request_endio (defined here) 47fcefa62eSAndreas Gruenbacher * drbd_peer_request_endio (defined here) 48c5a91619SAndreas Gruenbacher * bm_async_io_complete (defined in drbd_bitmap.c) 49c5a91619SAndreas Gruenbacher * 50b411b363SPhilipp Reisner * For all these callbacks, note the following: 51b411b363SPhilipp Reisner * The callbacks will be called in irq context by the IDE drivers, 52b411b363SPhilipp Reisner * and in Softirqs/Tasklets/BH context by the SCSI drivers. 53b411b363SPhilipp Reisner * Try to get the locking right :) 54b411b363SPhilipp Reisner * 55b411b363SPhilipp Reisner */ 56b411b363SPhilipp Reisner 57b411b363SPhilipp Reisner 58b411b363SPhilipp Reisner /* About the global_state_lock 59b411b363SPhilipp Reisner Each state transition on an device holds a read lock. In case we have 6095f8efd0SAndreas Gruenbacher to evaluate the resync after dependencies, we grab a write lock, because 61b411b363SPhilipp Reisner we need stable states on all devices for that. */ 62b411b363SPhilipp Reisner rwlock_t global_state_lock; 63b411b363SPhilipp Reisner 64b411b363SPhilipp Reisner /* used for synchronous meta data and bitmap IO 65b411b363SPhilipp Reisner * submitted by drbd_md_sync_page_io() 66b411b363SPhilipp Reisner */ 67b411b363SPhilipp Reisner void drbd_md_io_complete(struct bio *bio, int error) 68b411b363SPhilipp Reisner { 69b411b363SPhilipp Reisner struct drbd_md_io *md_io; 70cdfda633SPhilipp Reisner struct drbd_conf *mdev; 71b411b363SPhilipp Reisner 72b411b363SPhilipp Reisner md_io = (struct drbd_md_io *)bio->bi_private; 73cdfda633SPhilipp Reisner mdev = container_of(md_io, struct drbd_conf, md_io); 74cdfda633SPhilipp Reisner 75b411b363SPhilipp Reisner md_io->error = error; 76b411b363SPhilipp Reisner 770cfac5ddSPhilipp Reisner /* We grabbed an extra reference in _drbd_md_sync_page_io() to be able 780cfac5ddSPhilipp Reisner * to timeout on the lower level device, and eventually detach from it. 790cfac5ddSPhilipp Reisner * If this io completion runs after that timeout expired, this 800cfac5ddSPhilipp Reisner * drbd_md_put_buffer() may allow us to finally try and re-attach. 810cfac5ddSPhilipp Reisner * During normal operation, this only puts that extra reference 820cfac5ddSPhilipp Reisner * down to 1 again. 830cfac5ddSPhilipp Reisner * Make sure we first drop the reference, and only then signal 840cfac5ddSPhilipp Reisner * completion, or we may (in drbd_al_read_log()) cycle so fast into the 850cfac5ddSPhilipp Reisner * next drbd_md_sync_page_io(), that we trigger the 860cfac5ddSPhilipp Reisner * ASSERT(atomic_read(&mdev->md_io_in_use) == 1) there. 870cfac5ddSPhilipp Reisner */ 880cfac5ddSPhilipp Reisner drbd_md_put_buffer(mdev); 89cdfda633SPhilipp Reisner md_io->done = 1; 90cdfda633SPhilipp Reisner wake_up(&mdev->misc_wait); 91cdfda633SPhilipp Reisner bio_put(bio); 92cdfda633SPhilipp Reisner put_ldev(mdev); 93b411b363SPhilipp Reisner } 94b411b363SPhilipp Reisner 95b411b363SPhilipp Reisner /* reads on behalf of the partner, 96b411b363SPhilipp Reisner * "submitted" by the receiver 97b411b363SPhilipp Reisner */ 98db830c46SAndreas Gruenbacher void drbd_endio_read_sec_final(struct drbd_peer_request *peer_req) __releases(local) 99b411b363SPhilipp Reisner { 100b411b363SPhilipp Reisner unsigned long flags = 0; 101a21e9298SPhilipp Reisner struct drbd_conf *mdev = peer_req->w.mdev; 102b411b363SPhilipp Reisner 10387eeee41SPhilipp Reisner spin_lock_irqsave(&mdev->tconn->req_lock, flags); 104db830c46SAndreas Gruenbacher mdev->read_cnt += peer_req->i.size >> 9; 105db830c46SAndreas Gruenbacher list_del(&peer_req->w.list); 106b411b363SPhilipp Reisner if (list_empty(&mdev->read_ee)) 107b411b363SPhilipp Reisner wake_up(&mdev->ee_wait); 108db830c46SAndreas Gruenbacher if (test_bit(__EE_WAS_ERROR, &peer_req->flags)) 1090c849666SLars Ellenberg __drbd_chk_io_error(mdev, DRBD_IO_ERROR); 11087eeee41SPhilipp Reisner spin_unlock_irqrestore(&mdev->tconn->req_lock, flags); 111b411b363SPhilipp Reisner 112d5b27b01SLars Ellenberg drbd_queue_work(&mdev->tconn->sender_work, &peer_req->w); 113b411b363SPhilipp Reisner put_ldev(mdev); 114b411b363SPhilipp Reisner } 115b411b363SPhilipp Reisner 116b411b363SPhilipp Reisner /* writes on behalf of the partner, or resync writes, 11745bb912bSLars Ellenberg * "submitted" by the receiver, final stage. */ 118db830c46SAndreas Gruenbacher static void drbd_endio_write_sec_final(struct drbd_peer_request *peer_req) __releases(local) 119b411b363SPhilipp Reisner { 120b411b363SPhilipp Reisner unsigned long flags = 0; 121a21e9298SPhilipp Reisner struct drbd_conf *mdev = peer_req->w.mdev; 122181286adSLars Ellenberg struct drbd_interval i; 123b411b363SPhilipp Reisner int do_wake; 124579b57edSAndreas Gruenbacher u64 block_id; 125b411b363SPhilipp Reisner int do_al_complete_io; 126b411b363SPhilipp Reisner 127db830c46SAndreas Gruenbacher /* after we moved peer_req to done_ee, 128b411b363SPhilipp Reisner * we may no longer access it, 129b411b363SPhilipp Reisner * it may be freed/reused already! 130b411b363SPhilipp Reisner * (as soon as we release the req_lock) */ 131181286adSLars Ellenberg i = peer_req->i; 132db830c46SAndreas Gruenbacher do_al_complete_io = peer_req->flags & EE_CALL_AL_COMPLETE_IO; 133db830c46SAndreas Gruenbacher block_id = peer_req->block_id; 134b411b363SPhilipp Reisner 13587eeee41SPhilipp Reisner spin_lock_irqsave(&mdev->tconn->req_lock, flags); 136db830c46SAndreas Gruenbacher mdev->writ_cnt += peer_req->i.size >> 9; 137db830c46SAndreas Gruenbacher list_del(&peer_req->w.list); /* has been on active_ee or sync_ee */ 138db830c46SAndreas Gruenbacher list_add_tail(&peer_req->w.list, &mdev->done_ee); 139b411b363SPhilipp Reisner 140bb3bfe96SAndreas Gruenbacher /* 1415e472264SAndreas Gruenbacher * Do not remove from the write_requests tree here: we did not send the 142bb3bfe96SAndreas Gruenbacher * Ack yet and did not wake possibly waiting conflicting requests. 143bb3bfe96SAndreas Gruenbacher * Removed from the tree from "drbd_process_done_ee" within the 144bb3bfe96SAndreas Gruenbacher * appropriate w.cb (e_end_block/e_end_resync_block) or from 145bb3bfe96SAndreas Gruenbacher * _drbd_clear_done_ee. 146bb3bfe96SAndreas Gruenbacher */ 147b411b363SPhilipp Reisner 148579b57edSAndreas Gruenbacher do_wake = list_empty(block_id == ID_SYNCER ? &mdev->sync_ee : &mdev->active_ee); 149b411b363SPhilipp Reisner 150db830c46SAndreas Gruenbacher if (test_bit(__EE_WAS_ERROR, &peer_req->flags)) 1510c849666SLars Ellenberg __drbd_chk_io_error(mdev, DRBD_IO_ERROR); 15287eeee41SPhilipp Reisner spin_unlock_irqrestore(&mdev->tconn->req_lock, flags); 153b411b363SPhilipp Reisner 154579b57edSAndreas Gruenbacher if (block_id == ID_SYNCER) 155181286adSLars Ellenberg drbd_rs_complete_io(mdev, i.sector); 156b411b363SPhilipp Reisner 157b411b363SPhilipp Reisner if (do_wake) 158b411b363SPhilipp Reisner wake_up(&mdev->ee_wait); 159b411b363SPhilipp Reisner 160b411b363SPhilipp Reisner if (do_al_complete_io) 161181286adSLars Ellenberg drbd_al_complete_io(mdev, &i); 162b411b363SPhilipp Reisner 1630625ac19SPhilipp Reisner wake_asender(mdev->tconn); 164b411b363SPhilipp Reisner put_ldev(mdev); 16545bb912bSLars Ellenberg } 166b411b363SPhilipp Reisner 16745bb912bSLars Ellenberg /* writes on behalf of the partner, or resync writes, 16845bb912bSLars Ellenberg * "submitted" by the receiver. 16945bb912bSLars Ellenberg */ 170fcefa62eSAndreas Gruenbacher void drbd_peer_request_endio(struct bio *bio, int error) 17145bb912bSLars Ellenberg { 172db830c46SAndreas Gruenbacher struct drbd_peer_request *peer_req = bio->bi_private; 173a21e9298SPhilipp Reisner struct drbd_conf *mdev = peer_req->w.mdev; 17445bb912bSLars Ellenberg int uptodate = bio_flagged(bio, BIO_UPTODATE); 17545bb912bSLars Ellenberg int is_write = bio_data_dir(bio) == WRITE; 17645bb912bSLars Ellenberg 17707194272SLars Ellenberg if (error && __ratelimit(&drbd_ratelimit_state)) 17845bb912bSLars Ellenberg dev_warn(DEV, "%s: error=%d s=%llus\n", 17945bb912bSLars Ellenberg is_write ? "write" : "read", error, 180db830c46SAndreas Gruenbacher (unsigned long long)peer_req->i.sector); 18145bb912bSLars Ellenberg if (!error && !uptodate) { 18207194272SLars Ellenberg if (__ratelimit(&drbd_ratelimit_state)) 18345bb912bSLars Ellenberg dev_warn(DEV, "%s: setting error to -EIO s=%llus\n", 18445bb912bSLars Ellenberg is_write ? "write" : "read", 185db830c46SAndreas Gruenbacher (unsigned long long)peer_req->i.sector); 18645bb912bSLars Ellenberg /* strange behavior of some lower level drivers... 18745bb912bSLars Ellenberg * fail the request by clearing the uptodate flag, 18845bb912bSLars Ellenberg * but do not return any error?! */ 18945bb912bSLars Ellenberg error = -EIO; 19045bb912bSLars Ellenberg } 19145bb912bSLars Ellenberg 19245bb912bSLars Ellenberg if (error) 193db830c46SAndreas Gruenbacher set_bit(__EE_WAS_ERROR, &peer_req->flags); 19445bb912bSLars Ellenberg 19545bb912bSLars Ellenberg bio_put(bio); /* no need for the bio anymore */ 196db830c46SAndreas Gruenbacher if (atomic_dec_and_test(&peer_req->pending_bios)) { 19745bb912bSLars Ellenberg if (is_write) 198db830c46SAndreas Gruenbacher drbd_endio_write_sec_final(peer_req); 19945bb912bSLars Ellenberg else 200db830c46SAndreas Gruenbacher drbd_endio_read_sec_final(peer_req); 20145bb912bSLars Ellenberg } 202b411b363SPhilipp Reisner } 203b411b363SPhilipp Reisner 204b411b363SPhilipp Reisner /* read, readA or write requests on R_PRIMARY coming from drbd_make_request 205b411b363SPhilipp Reisner */ 206fcefa62eSAndreas Gruenbacher void drbd_request_endio(struct bio *bio, int error) 207b411b363SPhilipp Reisner { 208a115413dSLars Ellenberg unsigned long flags; 209b411b363SPhilipp Reisner struct drbd_request *req = bio->bi_private; 210a21e9298SPhilipp Reisner struct drbd_conf *mdev = req->w.mdev; 211a115413dSLars Ellenberg struct bio_and_error m; 212b411b363SPhilipp Reisner enum drbd_req_event what; 213b411b363SPhilipp Reisner int uptodate = bio_flagged(bio, BIO_UPTODATE); 214b411b363SPhilipp Reisner 215b411b363SPhilipp Reisner if (!error && !uptodate) { 216b411b363SPhilipp Reisner dev_warn(DEV, "p %s: setting error to -EIO\n", 217b411b363SPhilipp Reisner bio_data_dir(bio) == WRITE ? "write" : "read"); 218b411b363SPhilipp Reisner /* strange behavior of some lower level drivers... 219b411b363SPhilipp Reisner * fail the request by clearing the uptodate flag, 220b411b363SPhilipp Reisner * but do not return any error?! */ 221b411b363SPhilipp Reisner error = -EIO; 222b411b363SPhilipp Reisner } 223b411b363SPhilipp Reisner 224b411b363SPhilipp Reisner /* to avoid recursion in __req_mod */ 225b411b363SPhilipp Reisner if (unlikely(error)) { 226b411b363SPhilipp Reisner what = (bio_data_dir(bio) == WRITE) 2278554df1cSAndreas Gruenbacher ? WRITE_COMPLETED_WITH_ERROR 2285c3c7e64SLars Ellenberg : (bio_rw(bio) == READ) 2298554df1cSAndreas Gruenbacher ? READ_COMPLETED_WITH_ERROR 2308554df1cSAndreas Gruenbacher : READ_AHEAD_COMPLETED_WITH_ERROR; 231b411b363SPhilipp Reisner } else 2328554df1cSAndreas Gruenbacher what = COMPLETED_OK; 233b411b363SPhilipp Reisner 234b411b363SPhilipp Reisner bio_put(req->private_bio); 235b411b363SPhilipp Reisner req->private_bio = ERR_PTR(error); 236b411b363SPhilipp Reisner 237a115413dSLars Ellenberg /* not req_mod(), we need irqsave here! */ 23887eeee41SPhilipp Reisner spin_lock_irqsave(&mdev->tconn->req_lock, flags); 239a115413dSLars Ellenberg __req_mod(req, what, &m); 24087eeee41SPhilipp Reisner spin_unlock_irqrestore(&mdev->tconn->req_lock, flags); 2412415308eSLars Ellenberg put_ldev(mdev); 242a115413dSLars Ellenberg 243a115413dSLars Ellenberg if (m.bio) 244a115413dSLars Ellenberg complete_master_bio(mdev, &m); 245b411b363SPhilipp Reisner } 246b411b363SPhilipp Reisner 247f6ffca9fSAndreas Gruenbacher void drbd_csum_ee(struct drbd_conf *mdev, struct crypto_hash *tfm, 248db830c46SAndreas Gruenbacher struct drbd_peer_request *peer_req, void *digest) 24945bb912bSLars Ellenberg { 25045bb912bSLars Ellenberg struct hash_desc desc; 25145bb912bSLars Ellenberg struct scatterlist sg; 252db830c46SAndreas Gruenbacher struct page *page = peer_req->pages; 25345bb912bSLars Ellenberg struct page *tmp; 25445bb912bSLars Ellenberg unsigned len; 25545bb912bSLars Ellenberg 25645bb912bSLars Ellenberg desc.tfm = tfm; 25745bb912bSLars Ellenberg desc.flags = 0; 25845bb912bSLars Ellenberg 25945bb912bSLars Ellenberg sg_init_table(&sg, 1); 26045bb912bSLars Ellenberg crypto_hash_init(&desc); 26145bb912bSLars Ellenberg 26245bb912bSLars Ellenberg while ((tmp = page_chain_next(page))) { 26345bb912bSLars Ellenberg /* all but the last page will be fully used */ 26445bb912bSLars Ellenberg sg_set_page(&sg, page, PAGE_SIZE, 0); 26545bb912bSLars Ellenberg crypto_hash_update(&desc, &sg, sg.length); 26645bb912bSLars Ellenberg page = tmp; 26745bb912bSLars Ellenberg } 26845bb912bSLars Ellenberg /* and now the last, possibly only partially used page */ 269db830c46SAndreas Gruenbacher len = peer_req->i.size & (PAGE_SIZE - 1); 27045bb912bSLars Ellenberg sg_set_page(&sg, page, len ?: PAGE_SIZE, 0); 27145bb912bSLars Ellenberg crypto_hash_update(&desc, &sg, sg.length); 27245bb912bSLars Ellenberg crypto_hash_final(&desc, digest); 27345bb912bSLars Ellenberg } 27445bb912bSLars Ellenberg 27545bb912bSLars Ellenberg void drbd_csum_bio(struct drbd_conf *mdev, struct crypto_hash *tfm, struct bio *bio, void *digest) 276b411b363SPhilipp Reisner { 277b411b363SPhilipp Reisner struct hash_desc desc; 278b411b363SPhilipp Reisner struct scatterlist sg; 279b411b363SPhilipp Reisner struct bio_vec *bvec; 280b411b363SPhilipp Reisner int i; 281b411b363SPhilipp Reisner 282b411b363SPhilipp Reisner desc.tfm = tfm; 283b411b363SPhilipp Reisner desc.flags = 0; 284b411b363SPhilipp Reisner 285b411b363SPhilipp Reisner sg_init_table(&sg, 1); 286b411b363SPhilipp Reisner crypto_hash_init(&desc); 287b411b363SPhilipp Reisner 2884b8514eeSLars Ellenberg bio_for_each_segment(bvec, bio, i) { 289b411b363SPhilipp Reisner sg_set_page(&sg, bvec->bv_page, bvec->bv_len, bvec->bv_offset); 290b411b363SPhilipp Reisner crypto_hash_update(&desc, &sg, sg.length); 291b411b363SPhilipp Reisner } 292b411b363SPhilipp Reisner crypto_hash_final(&desc, digest); 293b411b363SPhilipp Reisner } 294b411b363SPhilipp Reisner 2959676c760SLars Ellenberg /* MAYBE merge common code with w_e_end_ov_req */ 29699920dc5SAndreas Gruenbacher static int w_e_send_csum(struct drbd_work *w, int cancel) 297b411b363SPhilipp Reisner { 29800d56944SPhilipp Reisner struct drbd_peer_request *peer_req = container_of(w, struct drbd_peer_request, w); 29900d56944SPhilipp Reisner struct drbd_conf *mdev = w->mdev; 300b411b363SPhilipp Reisner int digest_size; 301b411b363SPhilipp Reisner void *digest; 30299920dc5SAndreas Gruenbacher int err = 0; 303b411b363SPhilipp Reisner 30453ea4331SLars Ellenberg if (unlikely(cancel)) 30553ea4331SLars Ellenberg goto out; 306b411b363SPhilipp Reisner 3079676c760SLars Ellenberg if (unlikely((peer_req->flags & EE_WAS_ERROR) != 0)) 30853ea4331SLars Ellenberg goto out; 30953ea4331SLars Ellenberg 310f399002eSLars Ellenberg digest_size = crypto_hash_digestsize(mdev->tconn->csums_tfm); 311b411b363SPhilipp Reisner digest = kmalloc(digest_size, GFP_NOIO); 312b411b363SPhilipp Reisner if (digest) { 313db830c46SAndreas Gruenbacher sector_t sector = peer_req->i.sector; 314db830c46SAndreas Gruenbacher unsigned int size = peer_req->i.size; 315f399002eSLars Ellenberg drbd_csum_ee(mdev, mdev->tconn->csums_tfm, peer_req, digest); 3169676c760SLars Ellenberg /* Free peer_req and pages before send. 31753ea4331SLars Ellenberg * In case we block on congestion, we could otherwise run into 31853ea4331SLars Ellenberg * some distributed deadlock, if the other side blocks on 31953ea4331SLars Ellenberg * congestion as well, because our receiver blocks in 320c37c8ecfSAndreas Gruenbacher * drbd_alloc_pages due to pp_in_use > max_buffers. */ 3213967deb1SAndreas Gruenbacher drbd_free_peer_req(mdev, peer_req); 322db830c46SAndreas Gruenbacher peer_req = NULL; 323b411b363SPhilipp Reisner inc_rs_pending(mdev); 32499920dc5SAndreas Gruenbacher err = drbd_send_drequest_csum(mdev, sector, size, 32553ea4331SLars Ellenberg digest, digest_size, 326b411b363SPhilipp Reisner P_CSUM_RS_REQUEST); 327b411b363SPhilipp Reisner kfree(digest); 328b411b363SPhilipp Reisner } else { 329b411b363SPhilipp Reisner dev_err(DEV, "kmalloc() of digest failed.\n"); 33099920dc5SAndreas Gruenbacher err = -ENOMEM; 331b411b363SPhilipp Reisner } 332b411b363SPhilipp Reisner 33353ea4331SLars Ellenberg out: 334db830c46SAndreas Gruenbacher if (peer_req) 3353967deb1SAndreas Gruenbacher drbd_free_peer_req(mdev, peer_req); 336b411b363SPhilipp Reisner 33799920dc5SAndreas Gruenbacher if (unlikely(err)) 338b411b363SPhilipp Reisner dev_err(DEV, "drbd_send_drequest(..., csum) failed\n"); 33999920dc5SAndreas Gruenbacher return err; 340b411b363SPhilipp Reisner } 341b411b363SPhilipp Reisner 342b411b363SPhilipp Reisner #define GFP_TRY (__GFP_HIGHMEM | __GFP_NOWARN) 343b411b363SPhilipp Reisner 344b411b363SPhilipp Reisner static int read_for_csum(struct drbd_conf *mdev, sector_t sector, int size) 345b411b363SPhilipp Reisner { 346db830c46SAndreas Gruenbacher struct drbd_peer_request *peer_req; 347b411b363SPhilipp Reisner 348b411b363SPhilipp Reisner if (!get_ldev(mdev)) 34980a40e43SLars Ellenberg return -EIO; 350b411b363SPhilipp Reisner 351e3555d85SPhilipp Reisner if (drbd_rs_should_slow_down(mdev, sector)) 3520f0601f4SLars Ellenberg goto defer; 3530f0601f4SLars Ellenberg 354b411b363SPhilipp Reisner /* GFP_TRY, because if there is no memory available right now, this may 355b411b363SPhilipp Reisner * be rescheduled for later. It is "only" background resync, after all. */ 3560db55363SAndreas Gruenbacher peer_req = drbd_alloc_peer_req(mdev, ID_SYNCER /* unused */, sector, 3570db55363SAndreas Gruenbacher size, GFP_TRY); 358db830c46SAndreas Gruenbacher if (!peer_req) 35980a40e43SLars Ellenberg goto defer; 360b411b363SPhilipp Reisner 361db830c46SAndreas Gruenbacher peer_req->w.cb = w_e_send_csum; 36287eeee41SPhilipp Reisner spin_lock_irq(&mdev->tconn->req_lock); 363db830c46SAndreas Gruenbacher list_add(&peer_req->w.list, &mdev->read_ee); 36487eeee41SPhilipp Reisner spin_unlock_irq(&mdev->tconn->req_lock); 365b411b363SPhilipp Reisner 3660f0601f4SLars Ellenberg atomic_add(size >> 9, &mdev->rs_sect_ev); 367fbe29decSAndreas Gruenbacher if (drbd_submit_peer_request(mdev, peer_req, READ, DRBD_FAULT_RS_RD) == 0) 36880a40e43SLars Ellenberg return 0; 36945bb912bSLars Ellenberg 37010f6d992SLars Ellenberg /* If it failed because of ENOMEM, retry should help. If it failed 37110f6d992SLars Ellenberg * because bio_add_page failed (probably broken lower level driver), 37210f6d992SLars Ellenberg * retry may or may not help. 37310f6d992SLars Ellenberg * If it does not, you may need to force disconnect. */ 37487eeee41SPhilipp Reisner spin_lock_irq(&mdev->tconn->req_lock); 375db830c46SAndreas Gruenbacher list_del(&peer_req->w.list); 37687eeee41SPhilipp Reisner spin_unlock_irq(&mdev->tconn->req_lock); 37722cc37a9SLars Ellenberg 3783967deb1SAndreas Gruenbacher drbd_free_peer_req(mdev, peer_req); 37980a40e43SLars Ellenberg defer: 38045bb912bSLars Ellenberg put_ldev(mdev); 38180a40e43SLars Ellenberg return -EAGAIN; 382b411b363SPhilipp Reisner } 383b411b363SPhilipp Reisner 38499920dc5SAndreas Gruenbacher int w_resync_timer(struct drbd_work *w, int cancel) 385794abb75SPhilipp Reisner { 38600d56944SPhilipp Reisner struct drbd_conf *mdev = w->mdev; 387794abb75SPhilipp Reisner switch (mdev->state.conn) { 388794abb75SPhilipp Reisner case C_VERIFY_S: 38900d56944SPhilipp Reisner w_make_ov_request(w, cancel); 390794abb75SPhilipp Reisner break; 391794abb75SPhilipp Reisner case C_SYNC_TARGET: 39200d56944SPhilipp Reisner w_make_resync_request(w, cancel); 393794abb75SPhilipp Reisner break; 394794abb75SPhilipp Reisner } 395794abb75SPhilipp Reisner 39699920dc5SAndreas Gruenbacher return 0; 397794abb75SPhilipp Reisner } 398794abb75SPhilipp Reisner 399b411b363SPhilipp Reisner void resync_timer_fn(unsigned long data) 400b411b363SPhilipp Reisner { 401b411b363SPhilipp Reisner struct drbd_conf *mdev = (struct drbd_conf *) data; 402b411b363SPhilipp Reisner 403794abb75SPhilipp Reisner if (list_empty(&mdev->resync_work.list)) 404d5b27b01SLars Ellenberg drbd_queue_work(&mdev->tconn->sender_work, &mdev->resync_work); 405b411b363SPhilipp Reisner } 406b411b363SPhilipp Reisner 407778f271dSPhilipp Reisner static void fifo_set(struct fifo_buffer *fb, int value) 408778f271dSPhilipp Reisner { 409778f271dSPhilipp Reisner int i; 410778f271dSPhilipp Reisner 411778f271dSPhilipp Reisner for (i = 0; i < fb->size; i++) 412f10f2623SPhilipp Reisner fb->values[i] = value; 413778f271dSPhilipp Reisner } 414778f271dSPhilipp Reisner 415778f271dSPhilipp Reisner static int fifo_push(struct fifo_buffer *fb, int value) 416778f271dSPhilipp Reisner { 417778f271dSPhilipp Reisner int ov; 418778f271dSPhilipp Reisner 419778f271dSPhilipp Reisner ov = fb->values[fb->head_index]; 420778f271dSPhilipp Reisner fb->values[fb->head_index++] = value; 421778f271dSPhilipp Reisner 422778f271dSPhilipp Reisner if (fb->head_index >= fb->size) 423778f271dSPhilipp Reisner fb->head_index = 0; 424778f271dSPhilipp Reisner 425778f271dSPhilipp Reisner return ov; 426778f271dSPhilipp Reisner } 427778f271dSPhilipp Reisner 428778f271dSPhilipp Reisner static void fifo_add_val(struct fifo_buffer *fb, int value) 429778f271dSPhilipp Reisner { 430778f271dSPhilipp Reisner int i; 431778f271dSPhilipp Reisner 432778f271dSPhilipp Reisner for (i = 0; i < fb->size; i++) 433778f271dSPhilipp Reisner fb->values[i] += value; 434778f271dSPhilipp Reisner } 435778f271dSPhilipp Reisner 4369958c857SPhilipp Reisner struct fifo_buffer *fifo_alloc(int fifo_size) 4379958c857SPhilipp Reisner { 4389958c857SPhilipp Reisner struct fifo_buffer *fb; 4399958c857SPhilipp Reisner 4409958c857SPhilipp Reisner fb = kzalloc(sizeof(struct fifo_buffer) + sizeof(int) * fifo_size, GFP_KERNEL); 4419958c857SPhilipp Reisner if (!fb) 4429958c857SPhilipp Reisner return NULL; 4439958c857SPhilipp Reisner 4449958c857SPhilipp Reisner fb->head_index = 0; 4459958c857SPhilipp Reisner fb->size = fifo_size; 4469958c857SPhilipp Reisner fb->total = 0; 4479958c857SPhilipp Reisner 4489958c857SPhilipp Reisner return fb; 4499958c857SPhilipp Reisner } 4509958c857SPhilipp Reisner 4519d77a5feSPhilipp Reisner static int drbd_rs_controller(struct drbd_conf *mdev) 452778f271dSPhilipp Reisner { 453daeda1ccSPhilipp Reisner struct disk_conf *dc; 454778f271dSPhilipp Reisner unsigned int sect_in; /* Number of sectors that came in since the last turn */ 455778f271dSPhilipp Reisner unsigned int want; /* The number of sectors we want in the proxy */ 456778f271dSPhilipp Reisner int req_sect; /* Number of sectors to request in this turn */ 457778f271dSPhilipp Reisner int correction; /* Number of sectors more we need in the proxy*/ 458778f271dSPhilipp Reisner int cps; /* correction per invocation of drbd_rs_controller() */ 459778f271dSPhilipp Reisner int steps; /* Number of time steps to plan ahead */ 460778f271dSPhilipp Reisner int curr_corr; 461778f271dSPhilipp Reisner int max_sect; 462813472ceSPhilipp Reisner struct fifo_buffer *plan; 463778f271dSPhilipp Reisner 464778f271dSPhilipp Reisner sect_in = atomic_xchg(&mdev->rs_sect_in, 0); /* Number of sectors that came in */ 465778f271dSPhilipp Reisner mdev->rs_in_flight -= sect_in; 466778f271dSPhilipp Reisner 467daeda1ccSPhilipp Reisner dc = rcu_dereference(mdev->ldev->disk_conf); 468813472ceSPhilipp Reisner plan = rcu_dereference(mdev->rs_plan_s); 469778f271dSPhilipp Reisner 470813472ceSPhilipp Reisner steps = plan->size; /* (dc->c_plan_ahead * 10 * SLEEP_TIME) / HZ; */ 471778f271dSPhilipp Reisner 472778f271dSPhilipp Reisner if (mdev->rs_in_flight + sect_in == 0) { /* At start of resync */ 473daeda1ccSPhilipp Reisner want = ((dc->resync_rate * 2 * SLEEP_TIME) / HZ) * steps; 474778f271dSPhilipp Reisner } else { /* normal path */ 475daeda1ccSPhilipp Reisner want = dc->c_fill_target ? dc->c_fill_target : 476daeda1ccSPhilipp Reisner sect_in * dc->c_delay_target * HZ / (SLEEP_TIME * 10); 477778f271dSPhilipp Reisner } 478778f271dSPhilipp Reisner 479813472ceSPhilipp Reisner correction = want - mdev->rs_in_flight - plan->total; 480778f271dSPhilipp Reisner 481778f271dSPhilipp Reisner /* Plan ahead */ 482778f271dSPhilipp Reisner cps = correction / steps; 483813472ceSPhilipp Reisner fifo_add_val(plan, cps); 484813472ceSPhilipp Reisner plan->total += cps * steps; 485778f271dSPhilipp Reisner 486778f271dSPhilipp Reisner /* What we do in this step */ 487813472ceSPhilipp Reisner curr_corr = fifo_push(plan, 0); 488813472ceSPhilipp Reisner plan->total -= curr_corr; 489778f271dSPhilipp Reisner 490778f271dSPhilipp Reisner req_sect = sect_in + curr_corr; 491778f271dSPhilipp Reisner if (req_sect < 0) 492778f271dSPhilipp Reisner req_sect = 0; 493778f271dSPhilipp Reisner 494daeda1ccSPhilipp Reisner max_sect = (dc->c_max_rate * 2 * SLEEP_TIME) / HZ; 495778f271dSPhilipp Reisner if (req_sect > max_sect) 496778f271dSPhilipp Reisner req_sect = max_sect; 497778f271dSPhilipp Reisner 498778f271dSPhilipp Reisner /* 499778f271dSPhilipp Reisner dev_warn(DEV, "si=%u if=%d wa=%u co=%d st=%d cps=%d pl=%d cc=%d rs=%d\n", 500778f271dSPhilipp Reisner sect_in, mdev->rs_in_flight, want, correction, 501778f271dSPhilipp Reisner steps, cps, mdev->rs_planed, curr_corr, req_sect); 502778f271dSPhilipp Reisner */ 503778f271dSPhilipp Reisner 504778f271dSPhilipp Reisner return req_sect; 505778f271dSPhilipp Reisner } 506778f271dSPhilipp Reisner 5079d77a5feSPhilipp Reisner static int drbd_rs_number_requests(struct drbd_conf *mdev) 508e65f440dSLars Ellenberg { 509e65f440dSLars Ellenberg int number; 510813472ceSPhilipp Reisner 511813472ceSPhilipp Reisner rcu_read_lock(); 512813472ceSPhilipp Reisner if (rcu_dereference(mdev->rs_plan_s)->size) { 513e65f440dSLars Ellenberg number = drbd_rs_controller(mdev) >> (BM_BLOCK_SHIFT - 9); 514e65f440dSLars Ellenberg mdev->c_sync_rate = number * HZ * (BM_BLOCK_SIZE / 1024) / SLEEP_TIME; 515e65f440dSLars Ellenberg } else { 516daeda1ccSPhilipp Reisner mdev->c_sync_rate = rcu_dereference(mdev->ldev->disk_conf)->resync_rate; 517e65f440dSLars Ellenberg number = SLEEP_TIME * mdev->c_sync_rate / ((BM_BLOCK_SIZE / 1024) * HZ); 518e65f440dSLars Ellenberg } 519813472ceSPhilipp Reisner rcu_read_unlock(); 520e65f440dSLars Ellenberg 521e65f440dSLars Ellenberg /* ignore the amount of pending requests, the resync controller should 522e65f440dSLars Ellenberg * throttle down to incoming reply rate soon enough anyways. */ 523e65f440dSLars Ellenberg return number; 524e65f440dSLars Ellenberg } 525e65f440dSLars Ellenberg 52699920dc5SAndreas Gruenbacher int w_make_resync_request(struct drbd_work *w, int cancel) 527b411b363SPhilipp Reisner { 52800d56944SPhilipp Reisner struct drbd_conf *mdev = w->mdev; 529b411b363SPhilipp Reisner unsigned long bit; 530b411b363SPhilipp Reisner sector_t sector; 531b411b363SPhilipp Reisner const sector_t capacity = drbd_get_capacity(mdev->this_bdev); 5321816a2b4SLars Ellenberg int max_bio_size; 533e65f440dSLars Ellenberg int number, rollback_i, size; 534b411b363SPhilipp Reisner int align, queued, sndbuf; 5350f0601f4SLars Ellenberg int i = 0; 536b411b363SPhilipp Reisner 537b411b363SPhilipp Reisner if (unlikely(cancel)) 53899920dc5SAndreas Gruenbacher return 0; 539b411b363SPhilipp Reisner 540af85e8e8SLars Ellenberg if (mdev->rs_total == 0) { 541af85e8e8SLars Ellenberg /* empty resync? */ 542af85e8e8SLars Ellenberg drbd_resync_finished(mdev); 54399920dc5SAndreas Gruenbacher return 0; 544af85e8e8SLars Ellenberg } 545af85e8e8SLars Ellenberg 546b411b363SPhilipp Reisner if (!get_ldev(mdev)) { 547b411b363SPhilipp Reisner /* Since we only need to access mdev->rsync a 548b411b363SPhilipp Reisner get_ldev_if_state(mdev,D_FAILED) would be sufficient, but 549b411b363SPhilipp Reisner to continue resync with a broken disk makes no sense at 550b411b363SPhilipp Reisner all */ 551b411b363SPhilipp Reisner dev_err(DEV, "Disk broke down during resync!\n"); 55299920dc5SAndreas Gruenbacher return 0; 553b411b363SPhilipp Reisner } 554b411b363SPhilipp Reisner 5550cfdd247SPhilipp Reisner max_bio_size = queue_max_hw_sectors(mdev->rq_queue) << 9; 556e65f440dSLars Ellenberg number = drbd_rs_number_requests(mdev); 557e65f440dSLars Ellenberg if (number == 0) 5580f0601f4SLars Ellenberg goto requeue; 559b411b363SPhilipp Reisner 560b411b363SPhilipp Reisner for (i = 0; i < number; i++) { 561b411b363SPhilipp Reisner /* Stop generating RS requests, when half of the send buffer is filled */ 562e42325a5SPhilipp Reisner mutex_lock(&mdev->tconn->data.mutex); 563e42325a5SPhilipp Reisner if (mdev->tconn->data.socket) { 564e42325a5SPhilipp Reisner queued = mdev->tconn->data.socket->sk->sk_wmem_queued; 565e42325a5SPhilipp Reisner sndbuf = mdev->tconn->data.socket->sk->sk_sndbuf; 566b411b363SPhilipp Reisner } else { 567b411b363SPhilipp Reisner queued = 1; 568b411b363SPhilipp Reisner sndbuf = 0; 569b411b363SPhilipp Reisner } 570e42325a5SPhilipp Reisner mutex_unlock(&mdev->tconn->data.mutex); 571b411b363SPhilipp Reisner if (queued > sndbuf / 2) 572b411b363SPhilipp Reisner goto requeue; 573b411b363SPhilipp Reisner 574b411b363SPhilipp Reisner next_sector: 575b411b363SPhilipp Reisner size = BM_BLOCK_SIZE; 576b411b363SPhilipp Reisner bit = drbd_bm_find_next(mdev, mdev->bm_resync_fo); 577b411b363SPhilipp Reisner 5784b0715f0SLars Ellenberg if (bit == DRBD_END_OF_BITMAP) { 579b411b363SPhilipp Reisner mdev->bm_resync_fo = drbd_bm_bits(mdev); 580b411b363SPhilipp Reisner put_ldev(mdev); 58199920dc5SAndreas Gruenbacher return 0; 582b411b363SPhilipp Reisner } 583b411b363SPhilipp Reisner 584b411b363SPhilipp Reisner sector = BM_BIT_TO_SECT(bit); 585b411b363SPhilipp Reisner 586e3555d85SPhilipp Reisner if (drbd_rs_should_slow_down(mdev, sector) || 587e3555d85SPhilipp Reisner drbd_try_rs_begin_io(mdev, sector)) { 588b411b363SPhilipp Reisner mdev->bm_resync_fo = bit; 589b411b363SPhilipp Reisner goto requeue; 590b411b363SPhilipp Reisner } 591b411b363SPhilipp Reisner mdev->bm_resync_fo = bit + 1; 592b411b363SPhilipp Reisner 593b411b363SPhilipp Reisner if (unlikely(drbd_bm_test_bit(mdev, bit) == 0)) { 594b411b363SPhilipp Reisner drbd_rs_complete_io(mdev, sector); 595b411b363SPhilipp Reisner goto next_sector; 596b411b363SPhilipp Reisner } 597b411b363SPhilipp Reisner 5981816a2b4SLars Ellenberg #if DRBD_MAX_BIO_SIZE > BM_BLOCK_SIZE 599b411b363SPhilipp Reisner /* try to find some adjacent bits. 600b411b363SPhilipp Reisner * we stop if we have already the maximum req size. 601b411b363SPhilipp Reisner * 602b411b363SPhilipp Reisner * Additionally always align bigger requests, in order to 603b411b363SPhilipp Reisner * be prepared for all stripe sizes of software RAIDs. 604b411b363SPhilipp Reisner */ 605b411b363SPhilipp Reisner align = 1; 606d207450cSPhilipp Reisner rollback_i = i; 607b411b363SPhilipp Reisner for (;;) { 6081816a2b4SLars Ellenberg if (size + BM_BLOCK_SIZE > max_bio_size) 609b411b363SPhilipp Reisner break; 610b411b363SPhilipp Reisner 611b411b363SPhilipp Reisner /* Be always aligned */ 612b411b363SPhilipp Reisner if (sector & ((1<<(align+3))-1)) 613b411b363SPhilipp Reisner break; 614b411b363SPhilipp Reisner 615b411b363SPhilipp Reisner /* do not cross extent boundaries */ 616b411b363SPhilipp Reisner if (((bit+1) & BM_BLOCKS_PER_BM_EXT_MASK) == 0) 617b411b363SPhilipp Reisner break; 618b411b363SPhilipp Reisner /* now, is it actually dirty, after all? 619b411b363SPhilipp Reisner * caution, drbd_bm_test_bit is tri-state for some 620b411b363SPhilipp Reisner * obscure reason; ( b == 0 ) would get the out-of-band 621b411b363SPhilipp Reisner * only accidentally right because of the "oddly sized" 622b411b363SPhilipp Reisner * adjustment below */ 623b411b363SPhilipp Reisner if (drbd_bm_test_bit(mdev, bit+1) != 1) 624b411b363SPhilipp Reisner break; 625b411b363SPhilipp Reisner bit++; 626b411b363SPhilipp Reisner size += BM_BLOCK_SIZE; 627b411b363SPhilipp Reisner if ((BM_BLOCK_SIZE << align) <= size) 628b411b363SPhilipp Reisner align++; 629b411b363SPhilipp Reisner i++; 630b411b363SPhilipp Reisner } 631b411b363SPhilipp Reisner /* if we merged some, 632b411b363SPhilipp Reisner * reset the offset to start the next drbd_bm_find_next from */ 633b411b363SPhilipp Reisner if (size > BM_BLOCK_SIZE) 634b411b363SPhilipp Reisner mdev->bm_resync_fo = bit + 1; 635b411b363SPhilipp Reisner #endif 636b411b363SPhilipp Reisner 637b411b363SPhilipp Reisner /* adjust very last sectors, in case we are oddly sized */ 638b411b363SPhilipp Reisner if (sector + (size>>9) > capacity) 639b411b363SPhilipp Reisner size = (capacity-sector)<<9; 640f399002eSLars Ellenberg if (mdev->tconn->agreed_pro_version >= 89 && mdev->tconn->csums_tfm) { 641b411b363SPhilipp Reisner switch (read_for_csum(mdev, sector, size)) { 64280a40e43SLars Ellenberg case -EIO: /* Disk failure */ 643b411b363SPhilipp Reisner put_ldev(mdev); 64499920dc5SAndreas Gruenbacher return -EIO; 64580a40e43SLars Ellenberg case -EAGAIN: /* allocation failed, or ldev busy */ 646b411b363SPhilipp Reisner drbd_rs_complete_io(mdev, sector); 647b411b363SPhilipp Reisner mdev->bm_resync_fo = BM_SECT_TO_BIT(sector); 648d207450cSPhilipp Reisner i = rollback_i; 649b411b363SPhilipp Reisner goto requeue; 65080a40e43SLars Ellenberg case 0: 65180a40e43SLars Ellenberg /* everything ok */ 65280a40e43SLars Ellenberg break; 65380a40e43SLars Ellenberg default: 65480a40e43SLars Ellenberg BUG(); 655b411b363SPhilipp Reisner } 656b411b363SPhilipp Reisner } else { 65799920dc5SAndreas Gruenbacher int err; 65899920dc5SAndreas Gruenbacher 659b411b363SPhilipp Reisner inc_rs_pending(mdev); 66099920dc5SAndreas Gruenbacher err = drbd_send_drequest(mdev, P_RS_DATA_REQUEST, 66199920dc5SAndreas Gruenbacher sector, size, ID_SYNCER); 66299920dc5SAndreas Gruenbacher if (err) { 663b411b363SPhilipp Reisner dev_err(DEV, "drbd_send_drequest() failed, aborting...\n"); 664b411b363SPhilipp Reisner dec_rs_pending(mdev); 665b411b363SPhilipp Reisner put_ldev(mdev); 66699920dc5SAndreas Gruenbacher return err; 667b411b363SPhilipp Reisner } 668b411b363SPhilipp Reisner } 669b411b363SPhilipp Reisner } 670b411b363SPhilipp Reisner 671b411b363SPhilipp Reisner if (mdev->bm_resync_fo >= drbd_bm_bits(mdev)) { 672b411b363SPhilipp Reisner /* last syncer _request_ was sent, 673b411b363SPhilipp Reisner * but the P_RS_DATA_REPLY not yet received. sync will end (and 674b411b363SPhilipp Reisner * next sync group will resume), as soon as we receive the last 675b411b363SPhilipp Reisner * resync data block, and the last bit is cleared. 676b411b363SPhilipp Reisner * until then resync "work" is "inactive" ... 677b411b363SPhilipp Reisner */ 678b411b363SPhilipp Reisner put_ldev(mdev); 67999920dc5SAndreas Gruenbacher return 0; 680b411b363SPhilipp Reisner } 681b411b363SPhilipp Reisner 682b411b363SPhilipp Reisner requeue: 683778f271dSPhilipp Reisner mdev->rs_in_flight += (i << (BM_BLOCK_SHIFT - 9)); 684b411b363SPhilipp Reisner mod_timer(&mdev->resync_timer, jiffies + SLEEP_TIME); 685b411b363SPhilipp Reisner put_ldev(mdev); 68699920dc5SAndreas Gruenbacher return 0; 687b411b363SPhilipp Reisner } 688b411b363SPhilipp Reisner 68900d56944SPhilipp Reisner static int w_make_ov_request(struct drbd_work *w, int cancel) 690b411b363SPhilipp Reisner { 69100d56944SPhilipp Reisner struct drbd_conf *mdev = w->mdev; 692b411b363SPhilipp Reisner int number, i, size; 693b411b363SPhilipp Reisner sector_t sector; 694b411b363SPhilipp Reisner const sector_t capacity = drbd_get_capacity(mdev->this_bdev); 69558ffa580SLars Ellenberg bool stop_sector_reached = false; 696b411b363SPhilipp Reisner 697b411b363SPhilipp Reisner if (unlikely(cancel)) 698b411b363SPhilipp Reisner return 1; 699b411b363SPhilipp Reisner 7002649f080SLars Ellenberg number = drbd_rs_number_requests(mdev); 701b411b363SPhilipp Reisner 702b411b363SPhilipp Reisner sector = mdev->ov_position; 703b411b363SPhilipp Reisner for (i = 0; i < number; i++) { 70458ffa580SLars Ellenberg if (sector >= capacity) 705b411b363SPhilipp Reisner return 1; 70658ffa580SLars Ellenberg 70758ffa580SLars Ellenberg /* We check for "finished" only in the reply path: 70858ffa580SLars Ellenberg * w_e_end_ov_reply(). 70958ffa580SLars Ellenberg * We need to send at least one request out. */ 71058ffa580SLars Ellenberg stop_sector_reached = i > 0 71158ffa580SLars Ellenberg && verify_can_do_stop_sector(mdev) 71258ffa580SLars Ellenberg && sector >= mdev->ov_stop_sector; 71358ffa580SLars Ellenberg if (stop_sector_reached) 71458ffa580SLars Ellenberg break; 715b411b363SPhilipp Reisner 716b411b363SPhilipp Reisner size = BM_BLOCK_SIZE; 717b411b363SPhilipp Reisner 718e3555d85SPhilipp Reisner if (drbd_rs_should_slow_down(mdev, sector) || 719e3555d85SPhilipp Reisner drbd_try_rs_begin_io(mdev, sector)) { 720b411b363SPhilipp Reisner mdev->ov_position = sector; 721b411b363SPhilipp Reisner goto requeue; 722b411b363SPhilipp Reisner } 723b411b363SPhilipp Reisner 724b411b363SPhilipp Reisner if (sector + (size>>9) > capacity) 725b411b363SPhilipp Reisner size = (capacity-sector)<<9; 726b411b363SPhilipp Reisner 727b411b363SPhilipp Reisner inc_rs_pending(mdev); 7285b9f499cSAndreas Gruenbacher if (drbd_send_ov_request(mdev, sector, size)) { 729b411b363SPhilipp Reisner dec_rs_pending(mdev); 730b411b363SPhilipp Reisner return 0; 731b411b363SPhilipp Reisner } 732b411b363SPhilipp Reisner sector += BM_SECT_PER_BIT; 733b411b363SPhilipp Reisner } 734b411b363SPhilipp Reisner mdev->ov_position = sector; 735b411b363SPhilipp Reisner 736b411b363SPhilipp Reisner requeue: 7372649f080SLars Ellenberg mdev->rs_in_flight += (i << (BM_BLOCK_SHIFT - 9)); 73858ffa580SLars Ellenberg if (i == 0 || !stop_sector_reached) 739b411b363SPhilipp Reisner mod_timer(&mdev->resync_timer, jiffies + SLEEP_TIME); 740b411b363SPhilipp Reisner return 1; 741b411b363SPhilipp Reisner } 742b411b363SPhilipp Reisner 74399920dc5SAndreas Gruenbacher int w_ov_finished(struct drbd_work *w, int cancel) 744b411b363SPhilipp Reisner { 74500d56944SPhilipp Reisner struct drbd_conf *mdev = w->mdev; 746b411b363SPhilipp Reisner kfree(w); 7478f7bed77SAndreas Gruenbacher ov_out_of_sync_print(mdev); 748b411b363SPhilipp Reisner drbd_resync_finished(mdev); 749b411b363SPhilipp Reisner 75099920dc5SAndreas Gruenbacher return 0; 751b411b363SPhilipp Reisner } 752b411b363SPhilipp Reisner 75399920dc5SAndreas Gruenbacher static int w_resync_finished(struct drbd_work *w, int cancel) 754b411b363SPhilipp Reisner { 75500d56944SPhilipp Reisner struct drbd_conf *mdev = w->mdev; 756b411b363SPhilipp Reisner kfree(w); 757b411b363SPhilipp Reisner 758b411b363SPhilipp Reisner drbd_resync_finished(mdev); 759b411b363SPhilipp Reisner 76099920dc5SAndreas Gruenbacher return 0; 761b411b363SPhilipp Reisner } 762b411b363SPhilipp Reisner 763af85e8e8SLars Ellenberg static void ping_peer(struct drbd_conf *mdev) 764af85e8e8SLars Ellenberg { 7652a67d8b9SPhilipp Reisner struct drbd_tconn *tconn = mdev->tconn; 7662a67d8b9SPhilipp Reisner 7672a67d8b9SPhilipp Reisner clear_bit(GOT_PING_ACK, &tconn->flags); 7682a67d8b9SPhilipp Reisner request_ping(tconn); 7692a67d8b9SPhilipp Reisner wait_event(tconn->ping_wait, 7702a67d8b9SPhilipp Reisner test_bit(GOT_PING_ACK, &tconn->flags) || mdev->state.conn < C_CONNECTED); 771af85e8e8SLars Ellenberg } 772af85e8e8SLars Ellenberg 773b411b363SPhilipp Reisner int drbd_resync_finished(struct drbd_conf *mdev) 774b411b363SPhilipp Reisner { 775b411b363SPhilipp Reisner unsigned long db, dt, dbdt; 776b411b363SPhilipp Reisner unsigned long n_oos; 777b411b363SPhilipp Reisner union drbd_state os, ns; 778b411b363SPhilipp Reisner struct drbd_work *w; 779b411b363SPhilipp Reisner char *khelper_cmd = NULL; 78026525618SLars Ellenberg int verify_done = 0; 781b411b363SPhilipp Reisner 782b411b363SPhilipp Reisner /* Remove all elements from the resync LRU. Since future actions 783b411b363SPhilipp Reisner * might set bits in the (main) bitmap, then the entries in the 784b411b363SPhilipp Reisner * resync LRU would be wrong. */ 785b411b363SPhilipp Reisner if (drbd_rs_del_all(mdev)) { 786b411b363SPhilipp Reisner /* In case this is not possible now, most probably because 787b411b363SPhilipp Reisner * there are P_RS_DATA_REPLY Packets lingering on the worker's 788b411b363SPhilipp Reisner * queue (or even the read operations for those packets 789b411b363SPhilipp Reisner * is not finished by now). Retry in 100ms. */ 790b411b363SPhilipp Reisner 79120ee6390SPhilipp Reisner schedule_timeout_interruptible(HZ / 10); 792b411b363SPhilipp Reisner w = kmalloc(sizeof(struct drbd_work), GFP_ATOMIC); 793b411b363SPhilipp Reisner if (w) { 794b411b363SPhilipp Reisner w->cb = w_resync_finished; 7959b743da9SPhilipp Reisner w->mdev = mdev; 796d5b27b01SLars Ellenberg drbd_queue_work(&mdev->tconn->sender_work, w); 797b411b363SPhilipp Reisner return 1; 798b411b363SPhilipp Reisner } 799b411b363SPhilipp Reisner dev_err(DEV, "Warn failed to drbd_rs_del_all() and to kmalloc(w).\n"); 800b411b363SPhilipp Reisner } 801b411b363SPhilipp Reisner 802b411b363SPhilipp Reisner dt = (jiffies - mdev->rs_start - mdev->rs_paused) / HZ; 803b411b363SPhilipp Reisner if (dt <= 0) 804b411b363SPhilipp Reisner dt = 1; 80558ffa580SLars Ellenberg 806b411b363SPhilipp Reisner db = mdev->rs_total; 80758ffa580SLars Ellenberg /* adjust for verify start and stop sectors, respective reached position */ 80858ffa580SLars Ellenberg if (mdev->state.conn == C_VERIFY_S || mdev->state.conn == C_VERIFY_T) 80958ffa580SLars Ellenberg db -= mdev->ov_left; 81058ffa580SLars Ellenberg 811b411b363SPhilipp Reisner dbdt = Bit2KB(db/dt); 812b411b363SPhilipp Reisner mdev->rs_paused /= HZ; 813b411b363SPhilipp Reisner 814b411b363SPhilipp Reisner if (!get_ldev(mdev)) 815b411b363SPhilipp Reisner goto out; 816b411b363SPhilipp Reisner 817af85e8e8SLars Ellenberg ping_peer(mdev); 818af85e8e8SLars Ellenberg 81987eeee41SPhilipp Reisner spin_lock_irq(&mdev->tconn->req_lock); 82078bae59bSPhilipp Reisner os = drbd_read_state(mdev); 821b411b363SPhilipp Reisner 82226525618SLars Ellenberg verify_done = (os.conn == C_VERIFY_S || os.conn == C_VERIFY_T); 82326525618SLars Ellenberg 824b411b363SPhilipp Reisner /* This protects us against multiple calls (that can happen in the presence 825b411b363SPhilipp Reisner of application IO), and against connectivity loss just before we arrive here. */ 826b411b363SPhilipp Reisner if (os.conn <= C_CONNECTED) 827b411b363SPhilipp Reisner goto out_unlock; 828b411b363SPhilipp Reisner 829b411b363SPhilipp Reisner ns = os; 830b411b363SPhilipp Reisner ns.conn = C_CONNECTED; 831b411b363SPhilipp Reisner 832b411b363SPhilipp Reisner dev_info(DEV, "%s done (total %lu sec; paused %lu sec; %lu K/sec)\n", 83326525618SLars Ellenberg verify_done ? "Online verify" : "Resync", 834b411b363SPhilipp Reisner dt + mdev->rs_paused, mdev->rs_paused, dbdt); 835b411b363SPhilipp Reisner 836b411b363SPhilipp Reisner n_oos = drbd_bm_total_weight(mdev); 837b411b363SPhilipp Reisner 838b411b363SPhilipp Reisner if (os.conn == C_VERIFY_S || os.conn == C_VERIFY_T) { 839b411b363SPhilipp Reisner if (n_oos) { 840b411b363SPhilipp Reisner dev_alert(DEV, "Online verify found %lu %dk block out of sync!\n", 841b411b363SPhilipp Reisner n_oos, Bit2KB(1)); 842b411b363SPhilipp Reisner khelper_cmd = "out-of-sync"; 843b411b363SPhilipp Reisner } 844b411b363SPhilipp Reisner } else { 845b411b363SPhilipp Reisner D_ASSERT((n_oos - mdev->rs_failed) == 0); 846b411b363SPhilipp Reisner 847b411b363SPhilipp Reisner if (os.conn == C_SYNC_TARGET || os.conn == C_PAUSED_SYNC_T) 848b411b363SPhilipp Reisner khelper_cmd = "after-resync-target"; 849b411b363SPhilipp Reisner 850f399002eSLars Ellenberg if (mdev->tconn->csums_tfm && mdev->rs_total) { 851b411b363SPhilipp Reisner const unsigned long s = mdev->rs_same_csum; 852b411b363SPhilipp Reisner const unsigned long t = mdev->rs_total; 853b411b363SPhilipp Reisner const int ratio = 854b411b363SPhilipp Reisner (t == 0) ? 0 : 855b411b363SPhilipp Reisner (t < 100000) ? ((s*100)/t) : (s/(t/100)); 856b411b363SPhilipp Reisner dev_info(DEV, "%u %% had equal checksums, eliminated: %luK; " 857b411b363SPhilipp Reisner "transferred %luK total %luK\n", 858b411b363SPhilipp Reisner ratio, 859b411b363SPhilipp Reisner Bit2KB(mdev->rs_same_csum), 860b411b363SPhilipp Reisner Bit2KB(mdev->rs_total - mdev->rs_same_csum), 861b411b363SPhilipp Reisner Bit2KB(mdev->rs_total)); 862b411b363SPhilipp Reisner } 863b411b363SPhilipp Reisner } 864b411b363SPhilipp Reisner 865b411b363SPhilipp Reisner if (mdev->rs_failed) { 866b411b363SPhilipp Reisner dev_info(DEV, " %lu failed blocks\n", mdev->rs_failed); 867b411b363SPhilipp Reisner 868b411b363SPhilipp Reisner if (os.conn == C_SYNC_TARGET || os.conn == C_PAUSED_SYNC_T) { 869b411b363SPhilipp Reisner ns.disk = D_INCONSISTENT; 870b411b363SPhilipp Reisner ns.pdsk = D_UP_TO_DATE; 871b411b363SPhilipp Reisner } else { 872b411b363SPhilipp Reisner ns.disk = D_UP_TO_DATE; 873b411b363SPhilipp Reisner ns.pdsk = D_INCONSISTENT; 874b411b363SPhilipp Reisner } 875b411b363SPhilipp Reisner } else { 876b411b363SPhilipp Reisner ns.disk = D_UP_TO_DATE; 877b411b363SPhilipp Reisner ns.pdsk = D_UP_TO_DATE; 878b411b363SPhilipp Reisner 879b411b363SPhilipp Reisner if (os.conn == C_SYNC_TARGET || os.conn == C_PAUSED_SYNC_T) { 880b411b363SPhilipp Reisner if (mdev->p_uuid) { 881b411b363SPhilipp Reisner int i; 882b411b363SPhilipp Reisner for (i = UI_BITMAP ; i <= UI_HISTORY_END ; i++) 883b411b363SPhilipp Reisner _drbd_uuid_set(mdev, i, mdev->p_uuid[i]); 884b411b363SPhilipp Reisner drbd_uuid_set(mdev, UI_BITMAP, mdev->ldev->md.uuid[UI_CURRENT]); 885b411b363SPhilipp Reisner _drbd_uuid_set(mdev, UI_CURRENT, mdev->p_uuid[UI_CURRENT]); 886b411b363SPhilipp Reisner } else { 887b411b363SPhilipp Reisner dev_err(DEV, "mdev->p_uuid is NULL! BUG\n"); 888b411b363SPhilipp Reisner } 889b411b363SPhilipp Reisner } 890b411b363SPhilipp Reisner 89162b0da3aSLars Ellenberg if (!(os.conn == C_VERIFY_S || os.conn == C_VERIFY_T)) { 89262b0da3aSLars Ellenberg /* for verify runs, we don't update uuids here, 89362b0da3aSLars Ellenberg * so there would be nothing to report. */ 894b411b363SPhilipp Reisner drbd_uuid_set_bm(mdev, 0UL); 89562b0da3aSLars Ellenberg drbd_print_uuids(mdev, "updated UUIDs"); 896b411b363SPhilipp Reisner if (mdev->p_uuid) { 897b411b363SPhilipp Reisner /* Now the two UUID sets are equal, update what we 898b411b363SPhilipp Reisner * know of the peer. */ 899b411b363SPhilipp Reisner int i; 900b411b363SPhilipp Reisner for (i = UI_CURRENT ; i <= UI_HISTORY_END ; i++) 901b411b363SPhilipp Reisner mdev->p_uuid[i] = mdev->ldev->md.uuid[i]; 902b411b363SPhilipp Reisner } 903b411b363SPhilipp Reisner } 90462b0da3aSLars Ellenberg } 905b411b363SPhilipp Reisner 906b411b363SPhilipp Reisner _drbd_set_state(mdev, ns, CS_VERBOSE, NULL); 907b411b363SPhilipp Reisner out_unlock: 90887eeee41SPhilipp Reisner spin_unlock_irq(&mdev->tconn->req_lock); 909b411b363SPhilipp Reisner put_ldev(mdev); 910b411b363SPhilipp Reisner out: 911b411b363SPhilipp Reisner mdev->rs_total = 0; 912b411b363SPhilipp Reisner mdev->rs_failed = 0; 913b411b363SPhilipp Reisner mdev->rs_paused = 0; 91458ffa580SLars Ellenberg 91558ffa580SLars Ellenberg /* reset start sector, if we reached end of device */ 91658ffa580SLars Ellenberg if (verify_done && mdev->ov_left == 0) 917b411b363SPhilipp Reisner mdev->ov_start_sector = 0; 918b411b363SPhilipp Reisner 91913d42685SLars Ellenberg drbd_md_sync(mdev); 92013d42685SLars Ellenberg 921b411b363SPhilipp Reisner if (khelper_cmd) 922b411b363SPhilipp Reisner drbd_khelper(mdev, khelper_cmd); 923b411b363SPhilipp Reisner 924b411b363SPhilipp Reisner return 1; 925b411b363SPhilipp Reisner } 926b411b363SPhilipp Reisner 927b411b363SPhilipp Reisner /* helper */ 928db830c46SAndreas Gruenbacher static void move_to_net_ee_or_free(struct drbd_conf *mdev, struct drbd_peer_request *peer_req) 929b411b363SPhilipp Reisner { 930045417f7SAndreas Gruenbacher if (drbd_peer_req_has_active_page(peer_req)) { 931b411b363SPhilipp Reisner /* This might happen if sendpage() has not finished */ 932db830c46SAndreas Gruenbacher int i = (peer_req->i.size + PAGE_SIZE -1) >> PAGE_SHIFT; 933435f0740SLars Ellenberg atomic_add(i, &mdev->pp_in_use_by_net); 934435f0740SLars Ellenberg atomic_sub(i, &mdev->pp_in_use); 93587eeee41SPhilipp Reisner spin_lock_irq(&mdev->tconn->req_lock); 936db830c46SAndreas Gruenbacher list_add_tail(&peer_req->w.list, &mdev->net_ee); 93787eeee41SPhilipp Reisner spin_unlock_irq(&mdev->tconn->req_lock); 938435f0740SLars Ellenberg wake_up(&drbd_pp_wait); 939b411b363SPhilipp Reisner } else 9403967deb1SAndreas Gruenbacher drbd_free_peer_req(mdev, peer_req); 941b411b363SPhilipp Reisner } 942b411b363SPhilipp Reisner 943b411b363SPhilipp Reisner /** 944b411b363SPhilipp Reisner * w_e_end_data_req() - Worker callback, to send a P_DATA_REPLY packet in response to a P_DATA_REQUEST 945b411b363SPhilipp Reisner * @mdev: DRBD device. 946b411b363SPhilipp Reisner * @w: work object. 947b411b363SPhilipp Reisner * @cancel: The connection will be closed anyways 948b411b363SPhilipp Reisner */ 94999920dc5SAndreas Gruenbacher int w_e_end_data_req(struct drbd_work *w, int cancel) 950b411b363SPhilipp Reisner { 951db830c46SAndreas Gruenbacher struct drbd_peer_request *peer_req = container_of(w, struct drbd_peer_request, w); 95200d56944SPhilipp Reisner struct drbd_conf *mdev = w->mdev; 95399920dc5SAndreas Gruenbacher int err; 954b411b363SPhilipp Reisner 955b411b363SPhilipp Reisner if (unlikely(cancel)) { 9563967deb1SAndreas Gruenbacher drbd_free_peer_req(mdev, peer_req); 957b411b363SPhilipp Reisner dec_unacked(mdev); 95899920dc5SAndreas Gruenbacher return 0; 959b411b363SPhilipp Reisner } 960b411b363SPhilipp Reisner 961db830c46SAndreas Gruenbacher if (likely((peer_req->flags & EE_WAS_ERROR) == 0)) { 96299920dc5SAndreas Gruenbacher err = drbd_send_block(mdev, P_DATA_REPLY, peer_req); 963b411b363SPhilipp Reisner } else { 964b411b363SPhilipp Reisner if (__ratelimit(&drbd_ratelimit_state)) 965b411b363SPhilipp Reisner dev_err(DEV, "Sending NegDReply. sector=%llus.\n", 966db830c46SAndreas Gruenbacher (unsigned long long)peer_req->i.sector); 967b411b363SPhilipp Reisner 96899920dc5SAndreas Gruenbacher err = drbd_send_ack(mdev, P_NEG_DREPLY, peer_req); 969b411b363SPhilipp Reisner } 970b411b363SPhilipp Reisner 971b411b363SPhilipp Reisner dec_unacked(mdev); 972b411b363SPhilipp Reisner 973db830c46SAndreas Gruenbacher move_to_net_ee_or_free(mdev, peer_req); 974b411b363SPhilipp Reisner 97599920dc5SAndreas Gruenbacher if (unlikely(err)) 976b411b363SPhilipp Reisner dev_err(DEV, "drbd_send_block() failed\n"); 97799920dc5SAndreas Gruenbacher return err; 978b411b363SPhilipp Reisner } 979b411b363SPhilipp Reisner 980b411b363SPhilipp Reisner /** 981a209b4aeSAndreas Gruenbacher * w_e_end_rsdata_req() - Worker callback to send a P_RS_DATA_REPLY packet in response to a P_RS_DATA_REQUEST 982b411b363SPhilipp Reisner * @mdev: DRBD device. 983b411b363SPhilipp Reisner * @w: work object. 984b411b363SPhilipp Reisner * @cancel: The connection will be closed anyways 985b411b363SPhilipp Reisner */ 98699920dc5SAndreas Gruenbacher int w_e_end_rsdata_req(struct drbd_work *w, int cancel) 987b411b363SPhilipp Reisner { 988db830c46SAndreas Gruenbacher struct drbd_peer_request *peer_req = container_of(w, struct drbd_peer_request, w); 98900d56944SPhilipp Reisner struct drbd_conf *mdev = w->mdev; 99099920dc5SAndreas Gruenbacher int err; 991b411b363SPhilipp Reisner 992b411b363SPhilipp Reisner if (unlikely(cancel)) { 9933967deb1SAndreas Gruenbacher drbd_free_peer_req(mdev, peer_req); 994b411b363SPhilipp Reisner dec_unacked(mdev); 99599920dc5SAndreas Gruenbacher return 0; 996b411b363SPhilipp Reisner } 997b411b363SPhilipp Reisner 998b411b363SPhilipp Reisner if (get_ldev_if_state(mdev, D_FAILED)) { 999db830c46SAndreas Gruenbacher drbd_rs_complete_io(mdev, peer_req->i.sector); 1000b411b363SPhilipp Reisner put_ldev(mdev); 1001b411b363SPhilipp Reisner } 1002b411b363SPhilipp Reisner 1003d612d309SPhilipp Reisner if (mdev->state.conn == C_AHEAD) { 100499920dc5SAndreas Gruenbacher err = drbd_send_ack(mdev, P_RS_CANCEL, peer_req); 1005db830c46SAndreas Gruenbacher } else if (likely((peer_req->flags & EE_WAS_ERROR) == 0)) { 1006b411b363SPhilipp Reisner if (likely(mdev->state.pdsk >= D_INCONSISTENT)) { 1007b411b363SPhilipp Reisner inc_rs_pending(mdev); 100899920dc5SAndreas Gruenbacher err = drbd_send_block(mdev, P_RS_DATA_REPLY, peer_req); 1009b411b363SPhilipp Reisner } else { 1010b411b363SPhilipp Reisner if (__ratelimit(&drbd_ratelimit_state)) 1011b411b363SPhilipp Reisner dev_err(DEV, "Not sending RSDataReply, " 1012b411b363SPhilipp Reisner "partner DISKLESS!\n"); 101399920dc5SAndreas Gruenbacher err = 0; 1014b411b363SPhilipp Reisner } 1015b411b363SPhilipp Reisner } else { 1016b411b363SPhilipp Reisner if (__ratelimit(&drbd_ratelimit_state)) 1017b411b363SPhilipp Reisner dev_err(DEV, "Sending NegRSDReply. sector %llus.\n", 1018db830c46SAndreas Gruenbacher (unsigned long long)peer_req->i.sector); 1019b411b363SPhilipp Reisner 102099920dc5SAndreas Gruenbacher err = drbd_send_ack(mdev, P_NEG_RS_DREPLY, peer_req); 1021b411b363SPhilipp Reisner 1022b411b363SPhilipp Reisner /* update resync data with failure */ 1023db830c46SAndreas Gruenbacher drbd_rs_failed_io(mdev, peer_req->i.sector, peer_req->i.size); 1024b411b363SPhilipp Reisner } 1025b411b363SPhilipp Reisner 1026b411b363SPhilipp Reisner dec_unacked(mdev); 1027b411b363SPhilipp Reisner 1028db830c46SAndreas Gruenbacher move_to_net_ee_or_free(mdev, peer_req); 1029b411b363SPhilipp Reisner 103099920dc5SAndreas Gruenbacher if (unlikely(err)) 1031b411b363SPhilipp Reisner dev_err(DEV, "drbd_send_block() failed\n"); 103299920dc5SAndreas Gruenbacher return err; 1033b411b363SPhilipp Reisner } 1034b411b363SPhilipp Reisner 103599920dc5SAndreas Gruenbacher int w_e_end_csum_rs_req(struct drbd_work *w, int cancel) 1036b411b363SPhilipp Reisner { 1037db830c46SAndreas Gruenbacher struct drbd_peer_request *peer_req = container_of(w, struct drbd_peer_request, w); 103800d56944SPhilipp Reisner struct drbd_conf *mdev = w->mdev; 1039b411b363SPhilipp Reisner struct digest_info *di; 1040b411b363SPhilipp Reisner int digest_size; 1041b411b363SPhilipp Reisner void *digest = NULL; 104299920dc5SAndreas Gruenbacher int err, eq = 0; 1043b411b363SPhilipp Reisner 1044b411b363SPhilipp Reisner if (unlikely(cancel)) { 10453967deb1SAndreas Gruenbacher drbd_free_peer_req(mdev, peer_req); 1046b411b363SPhilipp Reisner dec_unacked(mdev); 104799920dc5SAndreas Gruenbacher return 0; 1048b411b363SPhilipp Reisner } 1049b411b363SPhilipp Reisner 10501d53f09eSLars Ellenberg if (get_ldev(mdev)) { 1051db830c46SAndreas Gruenbacher drbd_rs_complete_io(mdev, peer_req->i.sector); 10521d53f09eSLars Ellenberg put_ldev(mdev); 10531d53f09eSLars Ellenberg } 1054b411b363SPhilipp Reisner 1055db830c46SAndreas Gruenbacher di = peer_req->digest; 1056b411b363SPhilipp Reisner 1057db830c46SAndreas Gruenbacher if (likely((peer_req->flags & EE_WAS_ERROR) == 0)) { 1058b411b363SPhilipp Reisner /* quick hack to try to avoid a race against reconfiguration. 1059b411b363SPhilipp Reisner * a real fix would be much more involved, 1060b411b363SPhilipp Reisner * introducing more locking mechanisms */ 1061f399002eSLars Ellenberg if (mdev->tconn->csums_tfm) { 1062f399002eSLars Ellenberg digest_size = crypto_hash_digestsize(mdev->tconn->csums_tfm); 1063b411b363SPhilipp Reisner D_ASSERT(digest_size == di->digest_size); 1064b411b363SPhilipp Reisner digest = kmalloc(digest_size, GFP_NOIO); 1065b411b363SPhilipp Reisner } 1066b411b363SPhilipp Reisner if (digest) { 1067f399002eSLars Ellenberg drbd_csum_ee(mdev, mdev->tconn->csums_tfm, peer_req, digest); 1068b411b363SPhilipp Reisner eq = !memcmp(digest, di->digest, digest_size); 1069b411b363SPhilipp Reisner kfree(digest); 1070b411b363SPhilipp Reisner } 1071b411b363SPhilipp Reisner 1072b411b363SPhilipp Reisner if (eq) { 1073db830c46SAndreas Gruenbacher drbd_set_in_sync(mdev, peer_req->i.sector, peer_req->i.size); 1074676396d5SLars Ellenberg /* rs_same_csums unit is BM_BLOCK_SIZE */ 1075db830c46SAndreas Gruenbacher mdev->rs_same_csum += peer_req->i.size >> BM_BLOCK_SHIFT; 107699920dc5SAndreas Gruenbacher err = drbd_send_ack(mdev, P_RS_IS_IN_SYNC, peer_req); 1077b411b363SPhilipp Reisner } else { 1078b411b363SPhilipp Reisner inc_rs_pending(mdev); 1079db830c46SAndreas Gruenbacher peer_req->block_id = ID_SYNCER; /* By setting block_id, digest pointer becomes invalid! */ 1080db830c46SAndreas Gruenbacher peer_req->flags &= ~EE_HAS_DIGEST; /* This peer request no longer has a digest pointer */ 1081204bba99SPhilipp Reisner kfree(di); 108299920dc5SAndreas Gruenbacher err = drbd_send_block(mdev, P_RS_DATA_REPLY, peer_req); 1083b411b363SPhilipp Reisner } 1084b411b363SPhilipp Reisner } else { 108599920dc5SAndreas Gruenbacher err = drbd_send_ack(mdev, P_NEG_RS_DREPLY, peer_req); 1086b411b363SPhilipp Reisner if (__ratelimit(&drbd_ratelimit_state)) 1087b411b363SPhilipp Reisner dev_err(DEV, "Sending NegDReply. I guess it gets messy.\n"); 1088b411b363SPhilipp Reisner } 1089b411b363SPhilipp Reisner 1090b411b363SPhilipp Reisner dec_unacked(mdev); 1091db830c46SAndreas Gruenbacher move_to_net_ee_or_free(mdev, peer_req); 1092b411b363SPhilipp Reisner 109399920dc5SAndreas Gruenbacher if (unlikely(err)) 1094b411b363SPhilipp Reisner dev_err(DEV, "drbd_send_block/ack() failed\n"); 109599920dc5SAndreas Gruenbacher return err; 1096b411b363SPhilipp Reisner } 1097b411b363SPhilipp Reisner 109899920dc5SAndreas Gruenbacher int w_e_end_ov_req(struct drbd_work *w, int cancel) 1099b411b363SPhilipp Reisner { 1100db830c46SAndreas Gruenbacher struct drbd_peer_request *peer_req = container_of(w, struct drbd_peer_request, w); 110100d56944SPhilipp Reisner struct drbd_conf *mdev = w->mdev; 1102db830c46SAndreas Gruenbacher sector_t sector = peer_req->i.sector; 1103db830c46SAndreas Gruenbacher unsigned int size = peer_req->i.size; 1104b411b363SPhilipp Reisner int digest_size; 1105b411b363SPhilipp Reisner void *digest; 110699920dc5SAndreas Gruenbacher int err = 0; 1107b411b363SPhilipp Reisner 1108b411b363SPhilipp Reisner if (unlikely(cancel)) 1109b411b363SPhilipp Reisner goto out; 1110b411b363SPhilipp Reisner 1111f399002eSLars Ellenberg digest_size = crypto_hash_digestsize(mdev->tconn->verify_tfm); 1112b411b363SPhilipp Reisner digest = kmalloc(digest_size, GFP_NOIO); 11138f21420eSPhilipp Reisner if (!digest) { 111499920dc5SAndreas Gruenbacher err = 1; /* terminate the connection in case the allocation failed */ 11158f21420eSPhilipp Reisner goto out; 11168f21420eSPhilipp Reisner } 11178f21420eSPhilipp Reisner 1118db830c46SAndreas Gruenbacher if (likely(!(peer_req->flags & EE_WAS_ERROR))) 1119f399002eSLars Ellenberg drbd_csum_ee(mdev, mdev->tconn->verify_tfm, peer_req, digest); 11208f21420eSPhilipp Reisner else 11218f21420eSPhilipp Reisner memset(digest, 0, digest_size); 11228f21420eSPhilipp Reisner 112353ea4331SLars Ellenberg /* Free e and pages before send. 112453ea4331SLars Ellenberg * In case we block on congestion, we could otherwise run into 112553ea4331SLars Ellenberg * some distributed deadlock, if the other side blocks on 112653ea4331SLars Ellenberg * congestion as well, because our receiver blocks in 1127c37c8ecfSAndreas Gruenbacher * drbd_alloc_pages due to pp_in_use > max_buffers. */ 11283967deb1SAndreas Gruenbacher drbd_free_peer_req(mdev, peer_req); 1129db830c46SAndreas Gruenbacher peer_req = NULL; 1130b411b363SPhilipp Reisner inc_rs_pending(mdev); 113199920dc5SAndreas Gruenbacher err = drbd_send_drequest_csum(mdev, sector, size, digest, digest_size, P_OV_REPLY); 113299920dc5SAndreas Gruenbacher if (err) 1133b411b363SPhilipp Reisner dec_rs_pending(mdev); 1134b411b363SPhilipp Reisner kfree(digest); 1135b411b363SPhilipp Reisner 1136b411b363SPhilipp Reisner out: 1137db830c46SAndreas Gruenbacher if (peer_req) 11383967deb1SAndreas Gruenbacher drbd_free_peer_req(mdev, peer_req); 1139b411b363SPhilipp Reisner dec_unacked(mdev); 114099920dc5SAndreas Gruenbacher return err; 1141b411b363SPhilipp Reisner } 1142b411b363SPhilipp Reisner 11438f7bed77SAndreas Gruenbacher void drbd_ov_out_of_sync_found(struct drbd_conf *mdev, sector_t sector, int size) 1144b411b363SPhilipp Reisner { 1145b411b363SPhilipp Reisner if (mdev->ov_last_oos_start + mdev->ov_last_oos_size == sector) { 1146b411b363SPhilipp Reisner mdev->ov_last_oos_size += size>>9; 1147b411b363SPhilipp Reisner } else { 1148b411b363SPhilipp Reisner mdev->ov_last_oos_start = sector; 1149b411b363SPhilipp Reisner mdev->ov_last_oos_size = size>>9; 1150b411b363SPhilipp Reisner } 1151b411b363SPhilipp Reisner drbd_set_out_of_sync(mdev, sector, size); 1152b411b363SPhilipp Reisner } 1153b411b363SPhilipp Reisner 115499920dc5SAndreas Gruenbacher int w_e_end_ov_reply(struct drbd_work *w, int cancel) 1155b411b363SPhilipp Reisner { 1156db830c46SAndreas Gruenbacher struct drbd_peer_request *peer_req = container_of(w, struct drbd_peer_request, w); 115700d56944SPhilipp Reisner struct drbd_conf *mdev = w->mdev; 1158b411b363SPhilipp Reisner struct digest_info *di; 1159b411b363SPhilipp Reisner void *digest; 1160db830c46SAndreas Gruenbacher sector_t sector = peer_req->i.sector; 1161db830c46SAndreas Gruenbacher unsigned int size = peer_req->i.size; 116253ea4331SLars Ellenberg int digest_size; 116399920dc5SAndreas Gruenbacher int err, eq = 0; 116458ffa580SLars Ellenberg bool stop_sector_reached = false; 1165b411b363SPhilipp Reisner 1166b411b363SPhilipp Reisner if (unlikely(cancel)) { 11673967deb1SAndreas Gruenbacher drbd_free_peer_req(mdev, peer_req); 1168b411b363SPhilipp Reisner dec_unacked(mdev); 116999920dc5SAndreas Gruenbacher return 0; 1170b411b363SPhilipp Reisner } 1171b411b363SPhilipp Reisner 1172b411b363SPhilipp Reisner /* after "cancel", because after drbd_disconnect/drbd_rs_cancel_all 1173b411b363SPhilipp Reisner * the resync lru has been cleaned up already */ 11741d53f09eSLars Ellenberg if (get_ldev(mdev)) { 1175db830c46SAndreas Gruenbacher drbd_rs_complete_io(mdev, peer_req->i.sector); 11761d53f09eSLars Ellenberg put_ldev(mdev); 11771d53f09eSLars Ellenberg } 1178b411b363SPhilipp Reisner 1179db830c46SAndreas Gruenbacher di = peer_req->digest; 1180b411b363SPhilipp Reisner 1181db830c46SAndreas Gruenbacher if (likely((peer_req->flags & EE_WAS_ERROR) == 0)) { 1182f399002eSLars Ellenberg digest_size = crypto_hash_digestsize(mdev->tconn->verify_tfm); 1183b411b363SPhilipp Reisner digest = kmalloc(digest_size, GFP_NOIO); 1184b411b363SPhilipp Reisner if (digest) { 1185f399002eSLars Ellenberg drbd_csum_ee(mdev, mdev->tconn->verify_tfm, peer_req, digest); 1186b411b363SPhilipp Reisner 1187b411b363SPhilipp Reisner D_ASSERT(digest_size == di->digest_size); 1188b411b363SPhilipp Reisner eq = !memcmp(digest, di->digest, digest_size); 1189b411b363SPhilipp Reisner kfree(digest); 1190b411b363SPhilipp Reisner } 1191b411b363SPhilipp Reisner } 1192b411b363SPhilipp Reisner 11939676c760SLars Ellenberg /* Free peer_req and pages before send. 119453ea4331SLars Ellenberg * In case we block on congestion, we could otherwise run into 119553ea4331SLars Ellenberg * some distributed deadlock, if the other side blocks on 119653ea4331SLars Ellenberg * congestion as well, because our receiver blocks in 1197c37c8ecfSAndreas Gruenbacher * drbd_alloc_pages due to pp_in_use > max_buffers. */ 11983967deb1SAndreas Gruenbacher drbd_free_peer_req(mdev, peer_req); 1199b411b363SPhilipp Reisner if (!eq) 12008f7bed77SAndreas Gruenbacher drbd_ov_out_of_sync_found(mdev, sector, size); 1201b411b363SPhilipp Reisner else 12028f7bed77SAndreas Gruenbacher ov_out_of_sync_print(mdev); 1203b411b363SPhilipp Reisner 120499920dc5SAndreas Gruenbacher err = drbd_send_ack_ex(mdev, P_OV_RESULT, sector, size, 1205b411b363SPhilipp Reisner eq ? ID_IN_SYNC : ID_OUT_OF_SYNC); 1206b411b363SPhilipp Reisner 120753ea4331SLars Ellenberg dec_unacked(mdev); 1208b411b363SPhilipp Reisner 1209ea5442afSLars Ellenberg --mdev->ov_left; 1210ea5442afSLars Ellenberg 1211ea5442afSLars Ellenberg /* let's advance progress step marks only for every other megabyte */ 1212ea5442afSLars Ellenberg if ((mdev->ov_left & 0x200) == 0x200) 1213ea5442afSLars Ellenberg drbd_advance_rs_marks(mdev, mdev->ov_left); 1214ea5442afSLars Ellenberg 121558ffa580SLars Ellenberg stop_sector_reached = verify_can_do_stop_sector(mdev) && 121658ffa580SLars Ellenberg (sector + (size>>9)) >= mdev->ov_stop_sector; 121758ffa580SLars Ellenberg 121858ffa580SLars Ellenberg if (mdev->ov_left == 0 || stop_sector_reached) { 12198f7bed77SAndreas Gruenbacher ov_out_of_sync_print(mdev); 1220b411b363SPhilipp Reisner drbd_resync_finished(mdev); 1221b411b363SPhilipp Reisner } 1222b411b363SPhilipp Reisner 122399920dc5SAndreas Gruenbacher return err; 1224b411b363SPhilipp Reisner } 1225b411b363SPhilipp Reisner 122699920dc5SAndreas Gruenbacher int w_prev_work_done(struct drbd_work *w, int cancel) 1227b411b363SPhilipp Reisner { 1228b411b363SPhilipp Reisner struct drbd_wq_barrier *b = container_of(w, struct drbd_wq_barrier, w); 122900d56944SPhilipp Reisner 1230b411b363SPhilipp Reisner complete(&b->done); 123199920dc5SAndreas Gruenbacher return 0; 1232b411b363SPhilipp Reisner } 1233b411b363SPhilipp Reisner 1234b6dd1a89SLars Ellenberg /* FIXME 1235b6dd1a89SLars Ellenberg * We need to track the number of pending barrier acks, 1236b6dd1a89SLars Ellenberg * and to be able to wait for them. 1237b6dd1a89SLars Ellenberg * See also comment in drbd_adm_attach before drbd_suspend_io. 1238b6dd1a89SLars Ellenberg */ 1239b6dd1a89SLars Ellenberg int drbd_send_barrier(struct drbd_tconn *tconn) 1240b411b363SPhilipp Reisner { 12419f5bdc33SAndreas Gruenbacher struct p_barrier *p; 1242b6dd1a89SLars Ellenberg struct drbd_socket *sock; 1243b411b363SPhilipp Reisner 1244b6dd1a89SLars Ellenberg sock = &tconn->data; 1245b6dd1a89SLars Ellenberg p = conn_prepare_command(tconn, sock); 12469f5bdc33SAndreas Gruenbacher if (!p) 12479f5bdc33SAndreas Gruenbacher return -EIO; 1248b6dd1a89SLars Ellenberg p->barrier = tconn->send.current_epoch_nr; 1249b6dd1a89SLars Ellenberg p->pad = 0; 1250b6dd1a89SLars Ellenberg tconn->send.current_epoch_writes = 0; 1251b6dd1a89SLars Ellenberg 1252b6dd1a89SLars Ellenberg return conn_send_command(tconn, sock, P_BARRIER, sizeof(*p), NULL, 0); 1253b411b363SPhilipp Reisner } 1254b411b363SPhilipp Reisner 125599920dc5SAndreas Gruenbacher int w_send_write_hint(struct drbd_work *w, int cancel) 1256b411b363SPhilipp Reisner { 125700d56944SPhilipp Reisner struct drbd_conf *mdev = w->mdev; 12589f5bdc33SAndreas Gruenbacher struct drbd_socket *sock; 12599f5bdc33SAndreas Gruenbacher 1260b411b363SPhilipp Reisner if (cancel) 126199920dc5SAndreas Gruenbacher return 0; 12629f5bdc33SAndreas Gruenbacher sock = &mdev->tconn->data; 12639f5bdc33SAndreas Gruenbacher if (!drbd_prepare_command(mdev, sock)) 12649f5bdc33SAndreas Gruenbacher return -EIO; 1265e658983aSAndreas Gruenbacher return drbd_send_command(mdev, sock, P_UNPLUG_REMOTE, 0, NULL, 0); 1266b411b363SPhilipp Reisner } 1267b411b363SPhilipp Reisner 12684eb9b3cbSLars Ellenberg static void re_init_if_first_write(struct drbd_tconn *tconn, unsigned int epoch) 12694eb9b3cbSLars Ellenberg { 12704eb9b3cbSLars Ellenberg if (!tconn->send.seen_any_write_yet) { 12714eb9b3cbSLars Ellenberg tconn->send.seen_any_write_yet = true; 12724eb9b3cbSLars Ellenberg tconn->send.current_epoch_nr = epoch; 12734eb9b3cbSLars Ellenberg tconn->send.current_epoch_writes = 0; 12744eb9b3cbSLars Ellenberg } 12754eb9b3cbSLars Ellenberg } 12764eb9b3cbSLars Ellenberg 12774eb9b3cbSLars Ellenberg static void maybe_send_barrier(struct drbd_tconn *tconn, unsigned int epoch) 12784eb9b3cbSLars Ellenberg { 12794eb9b3cbSLars Ellenberg /* re-init if first write on this connection */ 12804eb9b3cbSLars Ellenberg if (!tconn->send.seen_any_write_yet) 12814eb9b3cbSLars Ellenberg return; 12824eb9b3cbSLars Ellenberg if (tconn->send.current_epoch_nr != epoch) { 12834eb9b3cbSLars Ellenberg if (tconn->send.current_epoch_writes) 12844eb9b3cbSLars Ellenberg drbd_send_barrier(tconn); 12854eb9b3cbSLars Ellenberg tconn->send.current_epoch_nr = epoch; 12864eb9b3cbSLars Ellenberg } 12874eb9b3cbSLars Ellenberg } 12884eb9b3cbSLars Ellenberg 12898f7bed77SAndreas Gruenbacher int w_send_out_of_sync(struct drbd_work *w, int cancel) 129073a01a18SPhilipp Reisner { 129173a01a18SPhilipp Reisner struct drbd_request *req = container_of(w, struct drbd_request, w); 129200d56944SPhilipp Reisner struct drbd_conf *mdev = w->mdev; 1293b6dd1a89SLars Ellenberg struct drbd_tconn *tconn = mdev->tconn; 129499920dc5SAndreas Gruenbacher int err; 129573a01a18SPhilipp Reisner 129673a01a18SPhilipp Reisner if (unlikely(cancel)) { 12978554df1cSAndreas Gruenbacher req_mod(req, SEND_CANCELED); 129899920dc5SAndreas Gruenbacher return 0; 129973a01a18SPhilipp Reisner } 130073a01a18SPhilipp Reisner 1301b6dd1a89SLars Ellenberg /* this time, no tconn->send.current_epoch_writes++; 1302b6dd1a89SLars Ellenberg * If it was sent, it was the closing barrier for the last 1303b6dd1a89SLars Ellenberg * replicated epoch, before we went into AHEAD mode. 1304b6dd1a89SLars Ellenberg * No more barriers will be sent, until we leave AHEAD mode again. */ 13054eb9b3cbSLars Ellenberg maybe_send_barrier(tconn, req->epoch); 1306b6dd1a89SLars Ellenberg 13078f7bed77SAndreas Gruenbacher err = drbd_send_out_of_sync(mdev, req); 13088554df1cSAndreas Gruenbacher req_mod(req, OOS_HANDED_TO_NETWORK); 130973a01a18SPhilipp Reisner 131099920dc5SAndreas Gruenbacher return err; 131173a01a18SPhilipp Reisner } 131273a01a18SPhilipp Reisner 1313b411b363SPhilipp Reisner /** 1314b411b363SPhilipp Reisner * w_send_dblock() - Worker callback to send a P_DATA packet in order to mirror a write request 1315b411b363SPhilipp Reisner * @mdev: DRBD device. 1316b411b363SPhilipp Reisner * @w: work object. 1317b411b363SPhilipp Reisner * @cancel: The connection will be closed anyways 1318b411b363SPhilipp Reisner */ 131999920dc5SAndreas Gruenbacher int w_send_dblock(struct drbd_work *w, int cancel) 1320b411b363SPhilipp Reisner { 1321b411b363SPhilipp Reisner struct drbd_request *req = container_of(w, struct drbd_request, w); 132200d56944SPhilipp Reisner struct drbd_conf *mdev = w->mdev; 1323b6dd1a89SLars Ellenberg struct drbd_tconn *tconn = mdev->tconn; 132499920dc5SAndreas Gruenbacher int err; 1325b411b363SPhilipp Reisner 1326b411b363SPhilipp Reisner if (unlikely(cancel)) { 13278554df1cSAndreas Gruenbacher req_mod(req, SEND_CANCELED); 132899920dc5SAndreas Gruenbacher return 0; 1329b411b363SPhilipp Reisner } 1330b411b363SPhilipp Reisner 13314eb9b3cbSLars Ellenberg re_init_if_first_write(tconn, req->epoch); 13324eb9b3cbSLars Ellenberg maybe_send_barrier(tconn, req->epoch); 1333b6dd1a89SLars Ellenberg tconn->send.current_epoch_writes++; 1334b6dd1a89SLars Ellenberg 133599920dc5SAndreas Gruenbacher err = drbd_send_dblock(mdev, req); 133699920dc5SAndreas Gruenbacher req_mod(req, err ? SEND_FAILED : HANDED_OVER_TO_NETWORK); 1337b411b363SPhilipp Reisner 133899920dc5SAndreas Gruenbacher return err; 1339b411b363SPhilipp Reisner } 1340b411b363SPhilipp Reisner 1341b411b363SPhilipp Reisner /** 1342b411b363SPhilipp Reisner * w_send_read_req() - Worker callback to send a read request (P_DATA_REQUEST) packet 1343b411b363SPhilipp Reisner * @mdev: DRBD device. 1344b411b363SPhilipp Reisner * @w: work object. 1345b411b363SPhilipp Reisner * @cancel: The connection will be closed anyways 1346b411b363SPhilipp Reisner */ 134799920dc5SAndreas Gruenbacher int w_send_read_req(struct drbd_work *w, int cancel) 1348b411b363SPhilipp Reisner { 1349b411b363SPhilipp Reisner struct drbd_request *req = container_of(w, struct drbd_request, w); 135000d56944SPhilipp Reisner struct drbd_conf *mdev = w->mdev; 1351b6dd1a89SLars Ellenberg struct drbd_tconn *tconn = mdev->tconn; 135299920dc5SAndreas Gruenbacher int err; 1353b411b363SPhilipp Reisner 1354b411b363SPhilipp Reisner if (unlikely(cancel)) { 13558554df1cSAndreas Gruenbacher req_mod(req, SEND_CANCELED); 135699920dc5SAndreas Gruenbacher return 0; 1357b411b363SPhilipp Reisner } 1358b411b363SPhilipp Reisner 1359b6dd1a89SLars Ellenberg /* Even read requests may close a write epoch, 1360b6dd1a89SLars Ellenberg * if there was any yet. */ 13614eb9b3cbSLars Ellenberg maybe_send_barrier(tconn, req->epoch); 1362b6dd1a89SLars Ellenberg 136399920dc5SAndreas Gruenbacher err = drbd_send_drequest(mdev, P_DATA_REQUEST, req->i.sector, req->i.size, 1364b411b363SPhilipp Reisner (unsigned long)req); 1365b411b363SPhilipp Reisner 136699920dc5SAndreas Gruenbacher req_mod(req, err ? SEND_FAILED : HANDED_OVER_TO_NETWORK); 1367b411b363SPhilipp Reisner 136899920dc5SAndreas Gruenbacher return err; 1369b411b363SPhilipp Reisner } 1370b411b363SPhilipp Reisner 137199920dc5SAndreas Gruenbacher int w_restart_disk_io(struct drbd_work *w, int cancel) 1372265be2d0SPhilipp Reisner { 1373265be2d0SPhilipp Reisner struct drbd_request *req = container_of(w, struct drbd_request, w); 137400d56944SPhilipp Reisner struct drbd_conf *mdev = w->mdev; 1375265be2d0SPhilipp Reisner 13760778286aSPhilipp Reisner if (bio_data_dir(req->master_bio) == WRITE && req->rq_state & RQ_IN_ACT_LOG) 1377181286adSLars Ellenberg drbd_al_begin_io(mdev, &req->i); 1378265be2d0SPhilipp Reisner 1379265be2d0SPhilipp Reisner drbd_req_make_private_bio(req, req->master_bio); 1380265be2d0SPhilipp Reisner req->private_bio->bi_bdev = mdev->ldev->backing_bdev; 1381265be2d0SPhilipp Reisner generic_make_request(req->private_bio); 1382265be2d0SPhilipp Reisner 138399920dc5SAndreas Gruenbacher return 0; 1384265be2d0SPhilipp Reisner } 1385265be2d0SPhilipp Reisner 1386b411b363SPhilipp Reisner static int _drbd_may_sync_now(struct drbd_conf *mdev) 1387b411b363SPhilipp Reisner { 1388b411b363SPhilipp Reisner struct drbd_conf *odev = mdev; 138995f8efd0SAndreas Gruenbacher int resync_after; 1390b411b363SPhilipp Reisner 1391b411b363SPhilipp Reisner while (1) { 1392438c8374SPhilipp Reisner if (!odev->ldev) 1393438c8374SPhilipp Reisner return 1; 1394daeda1ccSPhilipp Reisner rcu_read_lock(); 139595f8efd0SAndreas Gruenbacher resync_after = rcu_dereference(odev->ldev->disk_conf)->resync_after; 1396daeda1ccSPhilipp Reisner rcu_read_unlock(); 139795f8efd0SAndreas Gruenbacher if (resync_after == -1) 1398b411b363SPhilipp Reisner return 1; 139995f8efd0SAndreas Gruenbacher odev = minor_to_mdev(resync_after); 1400841ce241SAndreas Gruenbacher if (!expect(odev)) 1401841ce241SAndreas Gruenbacher return 1; 1402b411b363SPhilipp Reisner if ((odev->state.conn >= C_SYNC_SOURCE && 1403b411b363SPhilipp Reisner odev->state.conn <= C_PAUSED_SYNC_T) || 1404b411b363SPhilipp Reisner odev->state.aftr_isp || odev->state.peer_isp || 1405b411b363SPhilipp Reisner odev->state.user_isp) 1406b411b363SPhilipp Reisner return 0; 1407b411b363SPhilipp Reisner } 1408b411b363SPhilipp Reisner } 1409b411b363SPhilipp Reisner 1410b411b363SPhilipp Reisner /** 1411b411b363SPhilipp Reisner * _drbd_pause_after() - Pause resync on all devices that may not resync now 1412b411b363SPhilipp Reisner * @mdev: DRBD device. 1413b411b363SPhilipp Reisner * 1414b411b363SPhilipp Reisner * Called from process context only (admin command and after_state_ch). 1415b411b363SPhilipp Reisner */ 1416b411b363SPhilipp Reisner static int _drbd_pause_after(struct drbd_conf *mdev) 1417b411b363SPhilipp Reisner { 1418b411b363SPhilipp Reisner struct drbd_conf *odev; 1419b411b363SPhilipp Reisner int i, rv = 0; 1420b411b363SPhilipp Reisner 1421695d08faSPhilipp Reisner rcu_read_lock(); 142281a5d60eSPhilipp Reisner idr_for_each_entry(&minors, odev, i) { 1423b411b363SPhilipp Reisner if (odev->state.conn == C_STANDALONE && odev->state.disk == D_DISKLESS) 1424b411b363SPhilipp Reisner continue; 1425b411b363SPhilipp Reisner if (!_drbd_may_sync_now(odev)) 1426b411b363SPhilipp Reisner rv |= (__drbd_set_state(_NS(odev, aftr_isp, 1), CS_HARD, NULL) 1427b411b363SPhilipp Reisner != SS_NOTHING_TO_DO); 1428b411b363SPhilipp Reisner } 1429695d08faSPhilipp Reisner rcu_read_unlock(); 1430b411b363SPhilipp Reisner 1431b411b363SPhilipp Reisner return rv; 1432b411b363SPhilipp Reisner } 1433b411b363SPhilipp Reisner 1434b411b363SPhilipp Reisner /** 1435b411b363SPhilipp Reisner * _drbd_resume_next() - Resume resync on all devices that may resync now 1436b411b363SPhilipp Reisner * @mdev: DRBD device. 1437b411b363SPhilipp Reisner * 1438b411b363SPhilipp Reisner * Called from process context only (admin command and worker). 1439b411b363SPhilipp Reisner */ 1440b411b363SPhilipp Reisner static int _drbd_resume_next(struct drbd_conf *mdev) 1441b411b363SPhilipp Reisner { 1442b411b363SPhilipp Reisner struct drbd_conf *odev; 1443b411b363SPhilipp Reisner int i, rv = 0; 1444b411b363SPhilipp Reisner 1445695d08faSPhilipp Reisner rcu_read_lock(); 144681a5d60eSPhilipp Reisner idr_for_each_entry(&minors, odev, i) { 1447b411b363SPhilipp Reisner if (odev->state.conn == C_STANDALONE && odev->state.disk == D_DISKLESS) 1448b411b363SPhilipp Reisner continue; 1449b411b363SPhilipp Reisner if (odev->state.aftr_isp) { 1450b411b363SPhilipp Reisner if (_drbd_may_sync_now(odev)) 1451b411b363SPhilipp Reisner rv |= (__drbd_set_state(_NS(odev, aftr_isp, 0), 1452b411b363SPhilipp Reisner CS_HARD, NULL) 1453b411b363SPhilipp Reisner != SS_NOTHING_TO_DO) ; 1454b411b363SPhilipp Reisner } 1455b411b363SPhilipp Reisner } 1456695d08faSPhilipp Reisner rcu_read_unlock(); 1457b411b363SPhilipp Reisner return rv; 1458b411b363SPhilipp Reisner } 1459b411b363SPhilipp Reisner 1460b411b363SPhilipp Reisner void resume_next_sg(struct drbd_conf *mdev) 1461b411b363SPhilipp Reisner { 1462b411b363SPhilipp Reisner write_lock_irq(&global_state_lock); 1463b411b363SPhilipp Reisner _drbd_resume_next(mdev); 1464b411b363SPhilipp Reisner write_unlock_irq(&global_state_lock); 1465b411b363SPhilipp Reisner } 1466b411b363SPhilipp Reisner 1467b411b363SPhilipp Reisner void suspend_other_sg(struct drbd_conf *mdev) 1468b411b363SPhilipp Reisner { 1469b411b363SPhilipp Reisner write_lock_irq(&global_state_lock); 1470b411b363SPhilipp Reisner _drbd_pause_after(mdev); 1471b411b363SPhilipp Reisner write_unlock_irq(&global_state_lock); 1472b411b363SPhilipp Reisner } 1473b411b363SPhilipp Reisner 1474dc97b708SPhilipp Reisner /* caller must hold global_state_lock */ 147595f8efd0SAndreas Gruenbacher enum drbd_ret_code drbd_resync_after_valid(struct drbd_conf *mdev, int o_minor) 1476b411b363SPhilipp Reisner { 1477b411b363SPhilipp Reisner struct drbd_conf *odev; 147895f8efd0SAndreas Gruenbacher int resync_after; 1479b411b363SPhilipp Reisner 1480b411b363SPhilipp Reisner if (o_minor == -1) 1481b411b363SPhilipp Reisner return NO_ERROR; 1482b411b363SPhilipp Reisner if (o_minor < -1 || minor_to_mdev(o_minor) == NULL) 148395f8efd0SAndreas Gruenbacher return ERR_RESYNC_AFTER; 1484b411b363SPhilipp Reisner 1485b411b363SPhilipp Reisner /* check for loops */ 1486b411b363SPhilipp Reisner odev = minor_to_mdev(o_minor); 1487b411b363SPhilipp Reisner while (1) { 1488b411b363SPhilipp Reisner if (odev == mdev) 148995f8efd0SAndreas Gruenbacher return ERR_RESYNC_AFTER_CYCLE; 1490b411b363SPhilipp Reisner 1491daeda1ccSPhilipp Reisner rcu_read_lock(); 149295f8efd0SAndreas Gruenbacher resync_after = rcu_dereference(odev->ldev->disk_conf)->resync_after; 1493daeda1ccSPhilipp Reisner rcu_read_unlock(); 1494b411b363SPhilipp Reisner /* dependency chain ends here, no cycles. */ 149595f8efd0SAndreas Gruenbacher if (resync_after == -1) 1496b411b363SPhilipp Reisner return NO_ERROR; 1497b411b363SPhilipp Reisner 1498b411b363SPhilipp Reisner /* follow the dependency chain */ 149995f8efd0SAndreas Gruenbacher odev = minor_to_mdev(resync_after); 1500b411b363SPhilipp Reisner } 1501b411b363SPhilipp Reisner } 1502b411b363SPhilipp Reisner 1503dc97b708SPhilipp Reisner /* caller must hold global_state_lock */ 150495f8efd0SAndreas Gruenbacher void drbd_resync_after_changed(struct drbd_conf *mdev) 1505b411b363SPhilipp Reisner { 1506b411b363SPhilipp Reisner int changes; 1507b411b363SPhilipp Reisner 1508b411b363SPhilipp Reisner do { 1509b411b363SPhilipp Reisner changes = _drbd_pause_after(mdev); 1510b411b363SPhilipp Reisner changes |= _drbd_resume_next(mdev); 1511b411b363SPhilipp Reisner } while (changes); 1512b411b363SPhilipp Reisner } 1513b411b363SPhilipp Reisner 15149bd28d3cSLars Ellenberg void drbd_rs_controller_reset(struct drbd_conf *mdev) 15159bd28d3cSLars Ellenberg { 1516813472ceSPhilipp Reisner struct fifo_buffer *plan; 1517813472ceSPhilipp Reisner 15189bd28d3cSLars Ellenberg atomic_set(&mdev->rs_sect_in, 0); 15199bd28d3cSLars Ellenberg atomic_set(&mdev->rs_sect_ev, 0); 15209bd28d3cSLars Ellenberg mdev->rs_in_flight = 0; 1521813472ceSPhilipp Reisner 1522813472ceSPhilipp Reisner /* Updating the RCU protected object in place is necessary since 1523813472ceSPhilipp Reisner this function gets called from atomic context. 1524813472ceSPhilipp Reisner It is valid since all other updates also lead to an completely 1525813472ceSPhilipp Reisner empty fifo */ 1526813472ceSPhilipp Reisner rcu_read_lock(); 1527813472ceSPhilipp Reisner plan = rcu_dereference(mdev->rs_plan_s); 1528813472ceSPhilipp Reisner plan->total = 0; 1529813472ceSPhilipp Reisner fifo_set(plan, 0); 1530813472ceSPhilipp Reisner rcu_read_unlock(); 15319bd28d3cSLars Ellenberg } 15329bd28d3cSLars Ellenberg 15331f04af33SPhilipp Reisner void start_resync_timer_fn(unsigned long data) 15341f04af33SPhilipp Reisner { 15351f04af33SPhilipp Reisner struct drbd_conf *mdev = (struct drbd_conf *) data; 15361f04af33SPhilipp Reisner 1537d5b27b01SLars Ellenberg drbd_queue_work(&mdev->tconn->sender_work, &mdev->start_resync_work); 15381f04af33SPhilipp Reisner } 15391f04af33SPhilipp Reisner 154099920dc5SAndreas Gruenbacher int w_start_resync(struct drbd_work *w, int cancel) 15411f04af33SPhilipp Reisner { 154200d56944SPhilipp Reisner struct drbd_conf *mdev = w->mdev; 154300d56944SPhilipp Reisner 15441f04af33SPhilipp Reisner if (atomic_read(&mdev->unacked_cnt) || atomic_read(&mdev->rs_pending_cnt)) { 15451f04af33SPhilipp Reisner dev_warn(DEV, "w_start_resync later...\n"); 15461f04af33SPhilipp Reisner mdev->start_resync_timer.expires = jiffies + HZ/10; 15471f04af33SPhilipp Reisner add_timer(&mdev->start_resync_timer); 154899920dc5SAndreas Gruenbacher return 0; 15491f04af33SPhilipp Reisner } 15501f04af33SPhilipp Reisner 15511f04af33SPhilipp Reisner drbd_start_resync(mdev, C_SYNC_SOURCE); 155236baf611SPhilipp Reisner clear_bit(AHEAD_TO_SYNC_SOURCE, &mdev->flags); 155399920dc5SAndreas Gruenbacher return 0; 15541f04af33SPhilipp Reisner } 15551f04af33SPhilipp Reisner 1556b411b363SPhilipp Reisner /** 1557b411b363SPhilipp Reisner * drbd_start_resync() - Start the resync process 1558b411b363SPhilipp Reisner * @mdev: DRBD device. 1559b411b363SPhilipp Reisner * @side: Either C_SYNC_SOURCE or C_SYNC_TARGET 1560b411b363SPhilipp Reisner * 1561b411b363SPhilipp Reisner * This function might bring you directly into one of the 1562b411b363SPhilipp Reisner * C_PAUSED_SYNC_* states. 1563b411b363SPhilipp Reisner */ 1564b411b363SPhilipp Reisner void drbd_start_resync(struct drbd_conf *mdev, enum drbd_conns side) 1565b411b363SPhilipp Reisner { 1566b411b363SPhilipp Reisner union drbd_state ns; 1567b411b363SPhilipp Reisner int r; 1568b411b363SPhilipp Reisner 1569c4752ef1SPhilipp Reisner if (mdev->state.conn >= C_SYNC_SOURCE && mdev->state.conn < C_AHEAD) { 1570b411b363SPhilipp Reisner dev_err(DEV, "Resync already running!\n"); 1571b411b363SPhilipp Reisner return; 1572b411b363SPhilipp Reisner } 1573b411b363SPhilipp Reisner 1574e64a3294SPhilipp Reisner if (!test_bit(B_RS_H_DONE, &mdev->flags)) { 1575b411b363SPhilipp Reisner if (side == C_SYNC_TARGET) { 1576b411b363SPhilipp Reisner /* Since application IO was locked out during C_WF_BITMAP_T and 1577b411b363SPhilipp Reisner C_WF_SYNC_UUID we are still unmodified. Before going to C_SYNC_TARGET 1578b411b363SPhilipp Reisner we check that we might make the data inconsistent. */ 1579b411b363SPhilipp Reisner r = drbd_khelper(mdev, "before-resync-target"); 1580b411b363SPhilipp Reisner r = (r >> 8) & 0xff; 1581b411b363SPhilipp Reisner if (r > 0) { 1582b411b363SPhilipp Reisner dev_info(DEV, "before-resync-target handler returned %d, " 1583b411b363SPhilipp Reisner "dropping connection.\n", r); 158438fa9988SPhilipp Reisner conn_request_state(mdev->tconn, NS(conn, C_DISCONNECTING), CS_HARD); 1585b411b363SPhilipp Reisner return; 1586b411b363SPhilipp Reisner } 158709b9e797SPhilipp Reisner } else /* C_SYNC_SOURCE */ { 158809b9e797SPhilipp Reisner r = drbd_khelper(mdev, "before-resync-source"); 158909b9e797SPhilipp Reisner r = (r >> 8) & 0xff; 159009b9e797SPhilipp Reisner if (r > 0) { 159109b9e797SPhilipp Reisner if (r == 3) { 159209b9e797SPhilipp Reisner dev_info(DEV, "before-resync-source handler returned %d, " 159309b9e797SPhilipp Reisner "ignoring. Old userland tools?", r); 159409b9e797SPhilipp Reisner } else { 159509b9e797SPhilipp Reisner dev_info(DEV, "before-resync-source handler returned %d, " 159609b9e797SPhilipp Reisner "dropping connection.\n", r); 159738fa9988SPhilipp Reisner conn_request_state(mdev->tconn, NS(conn, C_DISCONNECTING), CS_HARD); 159809b9e797SPhilipp Reisner return; 159909b9e797SPhilipp Reisner } 160009b9e797SPhilipp Reisner } 1601b411b363SPhilipp Reisner } 1602e64a3294SPhilipp Reisner } 1603b411b363SPhilipp Reisner 1604e64a3294SPhilipp Reisner if (current == mdev->tconn->worker.task) { 1605dad20554SPhilipp Reisner /* The worker should not sleep waiting for state_mutex, 1606e64a3294SPhilipp Reisner that can take long */ 16078410da8fSPhilipp Reisner if (!mutex_trylock(mdev->state_mutex)) { 1608e64a3294SPhilipp Reisner set_bit(B_RS_H_DONE, &mdev->flags); 1609e64a3294SPhilipp Reisner mdev->start_resync_timer.expires = jiffies + HZ/5; 1610e64a3294SPhilipp Reisner add_timer(&mdev->start_resync_timer); 1611e64a3294SPhilipp Reisner return; 1612e64a3294SPhilipp Reisner } 1613e64a3294SPhilipp Reisner } else { 16148410da8fSPhilipp Reisner mutex_lock(mdev->state_mutex); 1615e64a3294SPhilipp Reisner } 1616e64a3294SPhilipp Reisner clear_bit(B_RS_H_DONE, &mdev->flags); 1617b411b363SPhilipp Reisner 16180cfac5ddSPhilipp Reisner write_lock_irq(&global_state_lock); 1619b411b363SPhilipp Reisner if (!get_ldev_if_state(mdev, D_NEGOTIATING)) { 16200cfac5ddSPhilipp Reisner write_unlock_irq(&global_state_lock); 16218410da8fSPhilipp Reisner mutex_unlock(mdev->state_mutex); 1622b411b363SPhilipp Reisner return; 1623b411b363SPhilipp Reisner } 1624b411b363SPhilipp Reisner 162578bae59bSPhilipp Reisner ns = drbd_read_state(mdev); 1626b411b363SPhilipp Reisner 1627b411b363SPhilipp Reisner ns.aftr_isp = !_drbd_may_sync_now(mdev); 1628b411b363SPhilipp Reisner 1629b411b363SPhilipp Reisner ns.conn = side; 1630b411b363SPhilipp Reisner 1631b411b363SPhilipp Reisner if (side == C_SYNC_TARGET) 1632b411b363SPhilipp Reisner ns.disk = D_INCONSISTENT; 1633b411b363SPhilipp Reisner else /* side == C_SYNC_SOURCE */ 1634b411b363SPhilipp Reisner ns.pdsk = D_INCONSISTENT; 1635b411b363SPhilipp Reisner 1636b411b363SPhilipp Reisner r = __drbd_set_state(mdev, ns, CS_VERBOSE, NULL); 163778bae59bSPhilipp Reisner ns = drbd_read_state(mdev); 1638b411b363SPhilipp Reisner 1639b411b363SPhilipp Reisner if (ns.conn < C_CONNECTED) 1640b411b363SPhilipp Reisner r = SS_UNKNOWN_ERROR; 1641b411b363SPhilipp Reisner 1642b411b363SPhilipp Reisner if (r == SS_SUCCESS) { 16431d7734a0SLars Ellenberg unsigned long tw = drbd_bm_total_weight(mdev); 16441d7734a0SLars Ellenberg unsigned long now = jiffies; 16451d7734a0SLars Ellenberg int i; 16461d7734a0SLars Ellenberg 1647b411b363SPhilipp Reisner mdev->rs_failed = 0; 1648b411b363SPhilipp Reisner mdev->rs_paused = 0; 1649b411b363SPhilipp Reisner mdev->rs_same_csum = 0; 16500f0601f4SLars Ellenberg mdev->rs_last_events = 0; 16510f0601f4SLars Ellenberg mdev->rs_last_sect_ev = 0; 16521d7734a0SLars Ellenberg mdev->rs_total = tw; 16531d7734a0SLars Ellenberg mdev->rs_start = now; 16541d7734a0SLars Ellenberg for (i = 0; i < DRBD_SYNC_MARKS; i++) { 16551d7734a0SLars Ellenberg mdev->rs_mark_left[i] = tw; 16561d7734a0SLars Ellenberg mdev->rs_mark_time[i] = now; 16571d7734a0SLars Ellenberg } 1658b411b363SPhilipp Reisner _drbd_pause_after(mdev); 1659b411b363SPhilipp Reisner } 1660b411b363SPhilipp Reisner write_unlock_irq(&global_state_lock); 16615a22db89SLars Ellenberg 16626c922ed5SLars Ellenberg if (r == SS_SUCCESS) { 16636c922ed5SLars Ellenberg dev_info(DEV, "Began resync as %s (will sync %lu KB [%lu bits set]).\n", 16646c922ed5SLars Ellenberg drbd_conn_str(ns.conn), 16656c922ed5SLars Ellenberg (unsigned long) mdev->rs_total << (BM_BLOCK_SHIFT-10), 16666c922ed5SLars Ellenberg (unsigned long) mdev->rs_total); 16675a22db89SLars Ellenberg if (side == C_SYNC_TARGET) 16685a22db89SLars Ellenberg mdev->bm_resync_fo = 0; 16695a22db89SLars Ellenberg 16705a22db89SLars Ellenberg /* Since protocol 96, we must serialize drbd_gen_and_send_sync_uuid 16715a22db89SLars Ellenberg * with w_send_oos, or the sync target will get confused as to 16725a22db89SLars Ellenberg * how much bits to resync. We cannot do that always, because for an 16735a22db89SLars Ellenberg * empty resync and protocol < 95, we need to do it here, as we call 16745a22db89SLars Ellenberg * drbd_resync_finished from here in that case. 16755a22db89SLars Ellenberg * We drbd_gen_and_send_sync_uuid here for protocol < 96, 16765a22db89SLars Ellenberg * and from after_state_ch otherwise. */ 167731890f4aSPhilipp Reisner if (side == C_SYNC_SOURCE && mdev->tconn->agreed_pro_version < 96) 16785a22db89SLars Ellenberg drbd_gen_and_send_sync_uuid(mdev); 1679b411b363SPhilipp Reisner 168031890f4aSPhilipp Reisner if (mdev->tconn->agreed_pro_version < 95 && mdev->rs_total == 0) { 1681af85e8e8SLars Ellenberg /* This still has a race (about when exactly the peers 1682af85e8e8SLars Ellenberg * detect connection loss) that can lead to a full sync 1683af85e8e8SLars Ellenberg * on next handshake. In 8.3.9 we fixed this with explicit 1684af85e8e8SLars Ellenberg * resync-finished notifications, but the fix 1685af85e8e8SLars Ellenberg * introduces a protocol change. Sleeping for some 1686af85e8e8SLars Ellenberg * time longer than the ping interval + timeout on the 1687af85e8e8SLars Ellenberg * SyncSource, to give the SyncTarget the chance to 1688af85e8e8SLars Ellenberg * detect connection loss, then waiting for a ping 1689af85e8e8SLars Ellenberg * response (implicit in drbd_resync_finished) reduces 1690af85e8e8SLars Ellenberg * the race considerably, but does not solve it. */ 169144ed167dSPhilipp Reisner if (side == C_SYNC_SOURCE) { 169244ed167dSPhilipp Reisner struct net_conf *nc; 169344ed167dSPhilipp Reisner int timeo; 169444ed167dSPhilipp Reisner 169544ed167dSPhilipp Reisner rcu_read_lock(); 169644ed167dSPhilipp Reisner nc = rcu_dereference(mdev->tconn->net_conf); 169744ed167dSPhilipp Reisner timeo = nc->ping_int * HZ + nc->ping_timeo * HZ / 9; 169844ed167dSPhilipp Reisner rcu_read_unlock(); 169944ed167dSPhilipp Reisner schedule_timeout_interruptible(timeo); 170044ed167dSPhilipp Reisner } 1701b411b363SPhilipp Reisner drbd_resync_finished(mdev); 1702b411b363SPhilipp Reisner } 1703b411b363SPhilipp Reisner 17049bd28d3cSLars Ellenberg drbd_rs_controller_reset(mdev); 1705b411b363SPhilipp Reisner /* ns.conn may already be != mdev->state.conn, 1706b411b363SPhilipp Reisner * we may have been paused in between, or become paused until 1707b411b363SPhilipp Reisner * the timer triggers. 1708b411b363SPhilipp Reisner * No matter, that is handled in resync_timer_fn() */ 1709b411b363SPhilipp Reisner if (ns.conn == C_SYNC_TARGET) 1710b411b363SPhilipp Reisner mod_timer(&mdev->resync_timer, jiffies); 1711b411b363SPhilipp Reisner 1712b411b363SPhilipp Reisner drbd_md_sync(mdev); 1713b411b363SPhilipp Reisner } 17145a22db89SLars Ellenberg put_ldev(mdev); 17158410da8fSPhilipp Reisner mutex_unlock(mdev->state_mutex); 1716b411b363SPhilipp Reisner } 1717b411b363SPhilipp Reisner 1718b6dd1a89SLars Ellenberg /* If the resource already closed the current epoch, but we did not 1719b6dd1a89SLars Ellenberg * (because we have not yet seen new requests), we should send the 1720b6dd1a89SLars Ellenberg * corresponding barrier now. Must be checked within the same spinlock 1721b6dd1a89SLars Ellenberg * that is used to check for new requests. */ 1722b6dd1a89SLars Ellenberg bool need_to_send_barrier(struct drbd_tconn *connection) 1723b6dd1a89SLars Ellenberg { 1724b6dd1a89SLars Ellenberg if (!connection->send.seen_any_write_yet) 1725b6dd1a89SLars Ellenberg return false; 1726b6dd1a89SLars Ellenberg 1727b6dd1a89SLars Ellenberg /* Skip barriers that do not contain any writes. 1728b6dd1a89SLars Ellenberg * This may happen during AHEAD mode. */ 1729b6dd1a89SLars Ellenberg if (!connection->send.current_epoch_writes) 1730b6dd1a89SLars Ellenberg return false; 1731b6dd1a89SLars Ellenberg 1732b6dd1a89SLars Ellenberg /* ->req_lock is held when requests are queued on 1733b6dd1a89SLars Ellenberg * connection->sender_work, and put into ->transfer_log. 1734b6dd1a89SLars Ellenberg * It is also held when ->current_tle_nr is increased. 1735b6dd1a89SLars Ellenberg * So either there are already new requests queued, 1736b6dd1a89SLars Ellenberg * and corresponding barriers will be send there. 1737b6dd1a89SLars Ellenberg * Or nothing new is queued yet, so the difference will be 1. 1738b6dd1a89SLars Ellenberg */ 1739b6dd1a89SLars Ellenberg if (atomic_read(&connection->current_tle_nr) != 1740b6dd1a89SLars Ellenberg connection->send.current_epoch_nr + 1) 1741b6dd1a89SLars Ellenberg return false; 1742b6dd1a89SLars Ellenberg 1743b6dd1a89SLars Ellenberg return true; 1744b6dd1a89SLars Ellenberg } 1745b6dd1a89SLars Ellenberg 17468c0785a5SLars Ellenberg bool dequeue_work_batch(struct drbd_work_queue *queue, struct list_head *work_list) 17478c0785a5SLars Ellenberg { 17488c0785a5SLars Ellenberg spin_lock_irq(&queue->q_lock); 17498c0785a5SLars Ellenberg list_splice_init(&queue->q, work_list); 17508c0785a5SLars Ellenberg spin_unlock_irq(&queue->q_lock); 17518c0785a5SLars Ellenberg return !list_empty(work_list); 17528c0785a5SLars Ellenberg } 17538c0785a5SLars Ellenberg 17548c0785a5SLars Ellenberg bool dequeue_work_item(struct drbd_work_queue *queue, struct list_head *work_list) 17558c0785a5SLars Ellenberg { 17568c0785a5SLars Ellenberg spin_lock_irq(&queue->q_lock); 17578c0785a5SLars Ellenberg if (!list_empty(&queue->q)) 17588c0785a5SLars Ellenberg list_move(queue->q.next, work_list); 17598c0785a5SLars Ellenberg spin_unlock_irq(&queue->q_lock); 17608c0785a5SLars Ellenberg return !list_empty(work_list); 17618c0785a5SLars Ellenberg } 17628c0785a5SLars Ellenberg 1763b6dd1a89SLars Ellenberg void wait_for_work(struct drbd_tconn *connection, struct list_head *work_list) 1764b6dd1a89SLars Ellenberg { 1765b6dd1a89SLars Ellenberg DEFINE_WAIT(wait); 1766b6dd1a89SLars Ellenberg struct net_conf *nc; 1767b6dd1a89SLars Ellenberg int uncork, cork; 1768b6dd1a89SLars Ellenberg 1769b6dd1a89SLars Ellenberg dequeue_work_item(&connection->sender_work, work_list); 1770b6dd1a89SLars Ellenberg if (!list_empty(work_list)) 1771b6dd1a89SLars Ellenberg return; 1772b6dd1a89SLars Ellenberg 1773b6dd1a89SLars Ellenberg /* Still nothing to do? 1774b6dd1a89SLars Ellenberg * Maybe we still need to close the current epoch, 1775b6dd1a89SLars Ellenberg * even if no new requests are queued yet. 1776b6dd1a89SLars Ellenberg * 1777b6dd1a89SLars Ellenberg * Also, poke TCP, just in case. 1778b6dd1a89SLars Ellenberg * Then wait for new work (or signal). */ 1779b6dd1a89SLars Ellenberg rcu_read_lock(); 1780b6dd1a89SLars Ellenberg nc = rcu_dereference(connection->net_conf); 1781b6dd1a89SLars Ellenberg uncork = nc ? nc->tcp_cork : 0; 1782b6dd1a89SLars Ellenberg rcu_read_unlock(); 1783b6dd1a89SLars Ellenberg if (uncork) { 1784b6dd1a89SLars Ellenberg mutex_lock(&connection->data.mutex); 1785b6dd1a89SLars Ellenberg if (connection->data.socket) 1786b6dd1a89SLars Ellenberg drbd_tcp_uncork(connection->data.socket); 1787b6dd1a89SLars Ellenberg mutex_unlock(&connection->data.mutex); 1788b6dd1a89SLars Ellenberg } 1789b6dd1a89SLars Ellenberg 1790b6dd1a89SLars Ellenberg for (;;) { 1791b6dd1a89SLars Ellenberg int send_barrier; 1792b6dd1a89SLars Ellenberg prepare_to_wait(&connection->sender_work.q_wait, &wait, TASK_INTERRUPTIBLE); 1793b6dd1a89SLars Ellenberg spin_lock_irq(&connection->req_lock); 1794b6dd1a89SLars Ellenberg spin_lock(&connection->sender_work.q_lock); /* FIXME get rid of this one? */ 1795b6dd1a89SLars Ellenberg list_splice_init(&connection->sender_work.q, work_list); 1796b6dd1a89SLars Ellenberg spin_unlock(&connection->sender_work.q_lock); /* FIXME get rid of this one? */ 1797b6dd1a89SLars Ellenberg if (!list_empty(work_list) || signal_pending(current)) { 1798b6dd1a89SLars Ellenberg spin_unlock_irq(&connection->req_lock); 1799b6dd1a89SLars Ellenberg break; 1800b6dd1a89SLars Ellenberg } 1801b6dd1a89SLars Ellenberg send_barrier = need_to_send_barrier(connection); 1802b6dd1a89SLars Ellenberg spin_unlock_irq(&connection->req_lock); 1803b6dd1a89SLars Ellenberg if (send_barrier) { 1804b6dd1a89SLars Ellenberg drbd_send_barrier(connection); 1805b6dd1a89SLars Ellenberg connection->send.current_epoch_nr++; 1806b6dd1a89SLars Ellenberg } 1807b6dd1a89SLars Ellenberg schedule(); 1808b6dd1a89SLars Ellenberg /* may be woken up for other things but new work, too, 1809b6dd1a89SLars Ellenberg * e.g. if the current epoch got closed. 1810b6dd1a89SLars Ellenberg * In which case we send the barrier above. */ 1811b6dd1a89SLars Ellenberg } 1812b6dd1a89SLars Ellenberg finish_wait(&connection->sender_work.q_wait, &wait); 1813b6dd1a89SLars Ellenberg 1814b6dd1a89SLars Ellenberg /* someone may have changed the config while we have been waiting above. */ 1815b6dd1a89SLars Ellenberg rcu_read_lock(); 1816b6dd1a89SLars Ellenberg nc = rcu_dereference(connection->net_conf); 1817b6dd1a89SLars Ellenberg cork = nc ? nc->tcp_cork : 0; 1818b6dd1a89SLars Ellenberg rcu_read_unlock(); 1819b6dd1a89SLars Ellenberg mutex_lock(&connection->data.mutex); 1820b6dd1a89SLars Ellenberg if (connection->data.socket) { 1821b6dd1a89SLars Ellenberg if (cork) 1822b6dd1a89SLars Ellenberg drbd_tcp_cork(connection->data.socket); 1823b6dd1a89SLars Ellenberg else if (!uncork) 1824b6dd1a89SLars Ellenberg drbd_tcp_uncork(connection->data.socket); 1825b6dd1a89SLars Ellenberg } 1826b6dd1a89SLars Ellenberg mutex_unlock(&connection->data.mutex); 1827b6dd1a89SLars Ellenberg } 1828b6dd1a89SLars Ellenberg 1829b411b363SPhilipp Reisner int drbd_worker(struct drbd_thread *thi) 1830b411b363SPhilipp Reisner { 1831392c8801SPhilipp Reisner struct drbd_tconn *tconn = thi->tconn; 1832b411b363SPhilipp Reisner struct drbd_work *w = NULL; 18330e29d163SPhilipp Reisner struct drbd_conf *mdev; 1834b411b363SPhilipp Reisner LIST_HEAD(work_list); 18358c0785a5SLars Ellenberg int vnr; 1836b411b363SPhilipp Reisner 1837e77a0a5cSAndreas Gruenbacher while (get_t_state(thi) == RUNNING) { 183880822284SPhilipp Reisner drbd_thread_current_set_cpu(thi); 1839b411b363SPhilipp Reisner 18408c0785a5SLars Ellenberg /* as long as we use drbd_queue_work_front(), 18418c0785a5SLars Ellenberg * we may only dequeue single work items here, not batches. */ 18428c0785a5SLars Ellenberg if (list_empty(&work_list)) 1843b6dd1a89SLars Ellenberg wait_for_work(tconn, &work_list); 1844b411b363SPhilipp Reisner 18458c0785a5SLars Ellenberg if (signal_pending(current)) { 1846b411b363SPhilipp Reisner flush_signals(current); 184719393e10SPhilipp Reisner if (get_t_state(thi) == RUNNING) { 184819393e10SPhilipp Reisner conn_warn(tconn, "Worker got an unexpected signal\n"); 1849b411b363SPhilipp Reisner continue; 185019393e10SPhilipp Reisner } 1851b411b363SPhilipp Reisner break; 1852b411b363SPhilipp Reisner } 1853b411b363SPhilipp Reisner 1854e77a0a5cSAndreas Gruenbacher if (get_t_state(thi) != RUNNING) 1855b411b363SPhilipp Reisner break; 1856b411b363SPhilipp Reisner 18578c0785a5SLars Ellenberg while (!list_empty(&work_list)) { 18588c0785a5SLars Ellenberg w = list_first_entry(&work_list, struct drbd_work, list); 1859b411b363SPhilipp Reisner list_del_init(&w->list); 18608c0785a5SLars Ellenberg if (w->cb(w, tconn->cstate < C_WF_REPORT_PARAMS) == 0) 18618c0785a5SLars Ellenberg continue; 1862bbeb641cSPhilipp Reisner if (tconn->cstate >= C_WF_REPORT_PARAMS) 1863bbeb641cSPhilipp Reisner conn_request_state(tconn, NS(conn, C_NETWORK_FAILURE), CS_HARD); 1864b411b363SPhilipp Reisner } 1865b411b363SPhilipp Reisner } 1866b411b363SPhilipp Reisner 18678c0785a5SLars Ellenberg do { 1868b411b363SPhilipp Reisner while (!list_empty(&work_list)) { 18698c0785a5SLars Ellenberg w = list_first_entry(&work_list, struct drbd_work, list); 1870b411b363SPhilipp Reisner list_del_init(&w->list); 187100d56944SPhilipp Reisner w->cb(w, 1); 1872b411b363SPhilipp Reisner } 1873d5b27b01SLars Ellenberg dequeue_work_batch(&tconn->sender_work, &work_list); 18748c0785a5SLars Ellenberg } while (!list_empty(&work_list)); 1875b411b363SPhilipp Reisner 1876c141ebdaSPhilipp Reisner rcu_read_lock(); 1877f399002eSLars Ellenberg idr_for_each_entry(&tconn->volumes, mdev, vnr) { 18780e29d163SPhilipp Reisner D_ASSERT(mdev->state.disk == D_DISKLESS && mdev->state.conn == C_STANDALONE); 1879c141ebdaSPhilipp Reisner kref_get(&mdev->kref); 1880c141ebdaSPhilipp Reisner rcu_read_unlock(); 18810e29d163SPhilipp Reisner drbd_mdev_cleanup(mdev); 1882c141ebdaSPhilipp Reisner kref_put(&mdev->kref, &drbd_minor_destroy); 1883c141ebdaSPhilipp Reisner rcu_read_lock(); 18840e29d163SPhilipp Reisner } 1885c141ebdaSPhilipp Reisner rcu_read_unlock(); 1886b411b363SPhilipp Reisner 1887b411b363SPhilipp Reisner return 0; 1888b411b363SPhilipp Reisner } 1889