1c6ae4c04SThomas Gleixner // SPDX-License-Identifier: GPL-2.0-or-later 2b411b363SPhilipp Reisner /* 3b411b363SPhilipp Reisner drbd_worker.c 4b411b363SPhilipp Reisner 5b411b363SPhilipp Reisner This file is part of DRBD by Philipp Reisner and Lars Ellenberg. 6b411b363SPhilipp Reisner 7b411b363SPhilipp Reisner Copyright (C) 2001-2008, LINBIT Information Technologies GmbH. 8b411b363SPhilipp Reisner Copyright (C) 1999-2008, Philipp Reisner <philipp.reisner@linbit.com>. 9b411b363SPhilipp Reisner Copyright (C) 2002-2008, Lars Ellenberg <lars.ellenberg@linbit.com>. 10b411b363SPhilipp Reisner 11b411b363SPhilipp Reisner 12b411b363SPhilipp Reisner */ 13b411b363SPhilipp Reisner 14b411b363SPhilipp Reisner #include <linux/module.h> 15b411b363SPhilipp Reisner #include <linux/drbd.h> 16174cd4b1SIngo Molnar #include <linux/sched/signal.h> 17b411b363SPhilipp Reisner #include <linux/wait.h> 18b411b363SPhilipp Reisner #include <linux/mm.h> 19b411b363SPhilipp Reisner #include <linux/memcontrol.h> 20b411b363SPhilipp Reisner #include <linux/mm_inline.h> 21b411b363SPhilipp Reisner #include <linux/slab.h> 22b411b363SPhilipp Reisner #include <linux/random.h> 23b411b363SPhilipp Reisner #include <linux/string.h> 24b411b363SPhilipp Reisner #include <linux/scatterlist.h> 25c6a564ffSChristoph Hellwig #include <linux/part_stat.h> 26b411b363SPhilipp Reisner 27b411b363SPhilipp Reisner #include "drbd_int.h" 28a3603a6eSAndreas Gruenbacher #include "drbd_protocol.h" 29b411b363SPhilipp Reisner #include "drbd_req.h" 30b411b363SPhilipp Reisner 31d448a2e1SAndreas Gruenbacher static int make_ov_request(struct drbd_device *, int); 32d448a2e1SAndreas Gruenbacher static int make_resync_request(struct drbd_device *, int); 33b411b363SPhilipp Reisner 34c5a91619SAndreas Gruenbacher /* endio handlers: 35ed15b795SAndreas Gruenbacher * drbd_md_endio (defined here) 36fcefa62eSAndreas Gruenbacher * drbd_request_endio (defined here) 37fcefa62eSAndreas Gruenbacher * drbd_peer_request_endio (defined here) 38ed15b795SAndreas Gruenbacher * drbd_bm_endio (defined in drbd_bitmap.c) 39c5a91619SAndreas Gruenbacher * 40b411b363SPhilipp Reisner * For all these callbacks, note the following: 41b411b363SPhilipp Reisner * The callbacks will be called in irq context by the IDE drivers, 42b411b363SPhilipp Reisner * and in Softirqs/Tasklets/BH context by the SCSI drivers. 43b411b363SPhilipp Reisner * Try to get the locking right :) 44b411b363SPhilipp Reisner * 45b411b363SPhilipp Reisner */ 46b411b363SPhilipp Reisner 47b411b363SPhilipp Reisner /* used for synchronous meta data and bitmap IO 48b411b363SPhilipp Reisner * submitted by drbd_md_sync_page_io() 49b411b363SPhilipp Reisner */ 504246a0b6SChristoph Hellwig void drbd_md_endio(struct bio *bio) 51b411b363SPhilipp Reisner { 52b30ab791SAndreas Gruenbacher struct drbd_device *device; 53b411b363SPhilipp Reisner 54e37d2438SLars Ellenberg device = bio->bi_private; 554e4cbee9SChristoph Hellwig device->md_io.error = blk_status_to_errno(bio->bi_status); 56b411b363SPhilipp Reisner 577c752ed3SLars Ellenberg /* special case: drbd_md_read() during drbd_adm_attach() */ 587c752ed3SLars Ellenberg if (device->ldev) 597c752ed3SLars Ellenberg put_ldev(device); 607c752ed3SLars Ellenberg bio_put(bio); 617c752ed3SLars Ellenberg 620cfac5ddSPhilipp Reisner /* We grabbed an extra reference in _drbd_md_sync_page_io() to be able 630cfac5ddSPhilipp Reisner * to timeout on the lower level device, and eventually detach from it. 640cfac5ddSPhilipp Reisner * If this io completion runs after that timeout expired, this 650cfac5ddSPhilipp Reisner * drbd_md_put_buffer() may allow us to finally try and re-attach. 660cfac5ddSPhilipp Reisner * During normal operation, this only puts that extra reference 670cfac5ddSPhilipp Reisner * down to 1 again. 680cfac5ddSPhilipp Reisner * Make sure we first drop the reference, and only then signal 690cfac5ddSPhilipp Reisner * completion, or we may (in drbd_al_read_log()) cycle so fast into the 700cfac5ddSPhilipp Reisner * next drbd_md_sync_page_io(), that we trigger the 71b30ab791SAndreas Gruenbacher * ASSERT(atomic_read(&device->md_io_in_use) == 1) there. 720cfac5ddSPhilipp Reisner */ 73b30ab791SAndreas Gruenbacher drbd_md_put_buffer(device); 74e37d2438SLars Ellenberg device->md_io.done = 1; 75b30ab791SAndreas Gruenbacher wake_up(&device->misc_wait); 76b411b363SPhilipp Reisner } 77b411b363SPhilipp Reisner 78b411b363SPhilipp Reisner /* reads on behalf of the partner, 79b411b363SPhilipp Reisner * "submitted" by the receiver 80b411b363SPhilipp Reisner */ 81a186e478SRashika Kheria static void drbd_endio_read_sec_final(struct drbd_peer_request *peer_req) __releases(local) 82b411b363SPhilipp Reisner { 83b411b363SPhilipp Reisner unsigned long flags = 0; 846780139cSAndreas Gruenbacher struct drbd_peer_device *peer_device = peer_req->peer_device; 856780139cSAndreas Gruenbacher struct drbd_device *device = peer_device->device; 86b411b363SPhilipp Reisner 870500813fSAndreas Gruenbacher spin_lock_irqsave(&device->resource->req_lock, flags); 88b30ab791SAndreas Gruenbacher device->read_cnt += peer_req->i.size >> 9; 89a8cd15baSAndreas Gruenbacher list_del(&peer_req->w.list); 90b30ab791SAndreas Gruenbacher if (list_empty(&device->read_ee)) 91b30ab791SAndreas Gruenbacher wake_up(&device->ee_wait); 92db830c46SAndreas Gruenbacher if (test_bit(__EE_WAS_ERROR, &peer_req->flags)) 93b30ab791SAndreas Gruenbacher __drbd_chk_io_error(device, DRBD_READ_ERROR); 940500813fSAndreas Gruenbacher spin_unlock_irqrestore(&device->resource->req_lock, flags); 95b411b363SPhilipp Reisner 966780139cSAndreas Gruenbacher drbd_queue_work(&peer_device->connection->sender_work, &peer_req->w); 97b30ab791SAndreas Gruenbacher put_ldev(device); 98b411b363SPhilipp Reisner } 99b411b363SPhilipp Reisner 100b411b363SPhilipp Reisner /* writes on behalf of the partner, or resync writes, 10145bb912bSLars Ellenberg * "submitted" by the receiver, final stage. */ 102a0fb3c47SLars Ellenberg void drbd_endio_write_sec_final(struct drbd_peer_request *peer_req) __releases(local) 103b411b363SPhilipp Reisner { 104b411b363SPhilipp Reisner unsigned long flags = 0; 1056780139cSAndreas Gruenbacher struct drbd_peer_device *peer_device = peer_req->peer_device; 1066780139cSAndreas Gruenbacher struct drbd_device *device = peer_device->device; 107668700b4SPhilipp Reisner struct drbd_connection *connection = peer_device->connection; 108181286adSLars Ellenberg struct drbd_interval i; 109b411b363SPhilipp Reisner int do_wake; 110579b57edSAndreas Gruenbacher u64 block_id; 111b411b363SPhilipp Reisner int do_al_complete_io; 112b411b363SPhilipp Reisner 113db830c46SAndreas Gruenbacher /* after we moved peer_req to done_ee, 114b411b363SPhilipp Reisner * we may no longer access it, 115b411b363SPhilipp Reisner * it may be freed/reused already! 116b411b363SPhilipp Reisner * (as soon as we release the req_lock) */ 117181286adSLars Ellenberg i = peer_req->i; 118db830c46SAndreas Gruenbacher do_al_complete_io = peer_req->flags & EE_CALL_AL_COMPLETE_IO; 119db830c46SAndreas Gruenbacher block_id = peer_req->block_id; 12021ae5d7fSLars Ellenberg peer_req->flags &= ~EE_CALL_AL_COMPLETE_IO; 121b411b363SPhilipp Reisner 122e1fbc4caSLars Ellenberg if (peer_req->flags & EE_WAS_ERROR) { 123e1fbc4caSLars Ellenberg /* In protocol != C, we usually do not send write acks. 124e1fbc4caSLars Ellenberg * In case of a write error, send the neg ack anyways. */ 125e1fbc4caSLars Ellenberg if (!__test_and_set_bit(__EE_SEND_WRITE_ACK, &peer_req->flags)) 126e1fbc4caSLars Ellenberg inc_unacked(device); 127e1fbc4caSLars Ellenberg drbd_set_out_of_sync(device, peer_req->i.sector, peer_req->i.size); 128e1fbc4caSLars Ellenberg } 129e1fbc4caSLars Ellenberg 1300500813fSAndreas Gruenbacher spin_lock_irqsave(&device->resource->req_lock, flags); 131b30ab791SAndreas Gruenbacher device->writ_cnt += peer_req->i.size >> 9; 132a8cd15baSAndreas Gruenbacher list_move_tail(&peer_req->w.list, &device->done_ee); 133b411b363SPhilipp Reisner 134bb3bfe96SAndreas Gruenbacher /* 1355e472264SAndreas Gruenbacher * Do not remove from the write_requests tree here: we did not send the 136bb3bfe96SAndreas Gruenbacher * Ack yet and did not wake possibly waiting conflicting requests. 137bb3bfe96SAndreas Gruenbacher * Removed from the tree from "drbd_process_done_ee" within the 13884b8c06bSAndreas Gruenbacher * appropriate dw.cb (e_end_block/e_end_resync_block) or from 139bb3bfe96SAndreas Gruenbacher * _drbd_clear_done_ee. 140bb3bfe96SAndreas Gruenbacher */ 141b411b363SPhilipp Reisner 142b30ab791SAndreas Gruenbacher do_wake = list_empty(block_id == ID_SYNCER ? &device->sync_ee : &device->active_ee); 143b411b363SPhilipp Reisner 1449305455aSBart Van Assche /* FIXME do we want to detach for failed REQ_OP_DISCARD? 145f31e583aSLars Ellenberg * ((peer_req->flags & (EE_WAS_ERROR|EE_TRIM)) == EE_WAS_ERROR) */ 146a0fb3c47SLars Ellenberg if (peer_req->flags & EE_WAS_ERROR) 147b30ab791SAndreas Gruenbacher __drbd_chk_io_error(device, DRBD_WRITE_ERROR); 148668700b4SPhilipp Reisner 149668700b4SPhilipp Reisner if (connection->cstate >= C_WF_REPORT_PARAMS) { 150668700b4SPhilipp Reisner kref_get(&device->kref); /* put is in drbd_send_acks_wf() */ 151668700b4SPhilipp Reisner if (!queue_work(connection->ack_sender, &peer_device->send_acks_work)) 152668700b4SPhilipp Reisner kref_put(&device->kref, drbd_destroy_device); 153668700b4SPhilipp Reisner } 1540500813fSAndreas Gruenbacher spin_unlock_irqrestore(&device->resource->req_lock, flags); 155b411b363SPhilipp Reisner 156579b57edSAndreas Gruenbacher if (block_id == ID_SYNCER) 157b30ab791SAndreas Gruenbacher drbd_rs_complete_io(device, i.sector); 158b411b363SPhilipp Reisner 159b411b363SPhilipp Reisner if (do_wake) 160b30ab791SAndreas Gruenbacher wake_up(&device->ee_wait); 161b411b363SPhilipp Reisner 162b411b363SPhilipp Reisner if (do_al_complete_io) 163b30ab791SAndreas Gruenbacher drbd_al_complete_io(device, &i); 164b411b363SPhilipp Reisner 165b30ab791SAndreas Gruenbacher put_ldev(device); 16645bb912bSLars Ellenberg } 167b411b363SPhilipp Reisner 16845bb912bSLars Ellenberg /* writes on behalf of the partner, or resync writes, 16945bb912bSLars Ellenberg * "submitted" by the receiver. 17045bb912bSLars Ellenberg */ 1714246a0b6SChristoph Hellwig void drbd_peer_request_endio(struct bio *bio) 17245bb912bSLars Ellenberg { 173db830c46SAndreas Gruenbacher struct drbd_peer_request *peer_req = bio->bi_private; 174a8cd15baSAndreas Gruenbacher struct drbd_device *device = peer_req->peer_device->device; 1757e5fec31SFabian Frederick bool is_write = bio_data_dir(bio) == WRITE; 17645c21793SChristoph Hellwig bool is_discard = bio_op(bio) == REQ_OP_WRITE_ZEROES || 17745c21793SChristoph Hellwig bio_op(bio) == REQ_OP_DISCARD; 17845bb912bSLars Ellenberg 1794e4cbee9SChristoph Hellwig if (bio->bi_status && __ratelimit(&drbd_ratelimit_state)) 180d0180171SAndreas Gruenbacher drbd_warn(device, "%s: error=%d s=%llus\n", 181a0fb3c47SLars Ellenberg is_write ? (is_discard ? "discard" : "write") 1824e4cbee9SChristoph Hellwig : "read", bio->bi_status, 183db830c46SAndreas Gruenbacher (unsigned long long)peer_req->i.sector); 18445bb912bSLars Ellenberg 1854e4cbee9SChristoph Hellwig if (bio->bi_status) 186db830c46SAndreas Gruenbacher set_bit(__EE_WAS_ERROR, &peer_req->flags); 18745bb912bSLars Ellenberg 18845bb912bSLars Ellenberg bio_put(bio); /* no need for the bio anymore */ 189db830c46SAndreas Gruenbacher if (atomic_dec_and_test(&peer_req->pending_bios)) { 19045bb912bSLars Ellenberg if (is_write) 191db830c46SAndreas Gruenbacher drbd_endio_write_sec_final(peer_req); 19245bb912bSLars Ellenberg else 193db830c46SAndreas Gruenbacher drbd_endio_read_sec_final(peer_req); 19445bb912bSLars Ellenberg } 195b411b363SPhilipp Reisner } 196b411b363SPhilipp Reisner 1971ffa7bfaSBaoyou Xie static void 1981ffa7bfaSBaoyou Xie drbd_panic_after_delayed_completion_of_aborted_request(struct drbd_device *device) 199142207f7SLars Ellenberg { 200142207f7SLars Ellenberg panic("drbd%u %s/%u potential random memory corruption caused by delayed completion of aborted local request\n", 201142207f7SLars Ellenberg device->minor, device->resource->name, device->vnr); 202142207f7SLars Ellenberg } 203142207f7SLars Ellenberg 204b411b363SPhilipp Reisner /* read, readA or write requests on R_PRIMARY coming from drbd_make_request 205b411b363SPhilipp Reisner */ 2064246a0b6SChristoph Hellwig void drbd_request_endio(struct bio *bio) 207b411b363SPhilipp Reisner { 208a115413dSLars Ellenberg unsigned long flags; 209b411b363SPhilipp Reisner struct drbd_request *req = bio->bi_private; 21084b8c06bSAndreas Gruenbacher struct drbd_device *device = req->device; 211a115413dSLars Ellenberg struct bio_and_error m; 212b411b363SPhilipp Reisner enum drbd_req_event what; 2131b6dd252SPhilipp Reisner 2141b6dd252SPhilipp Reisner /* If this request was aborted locally before, 2151b6dd252SPhilipp Reisner * but now was completed "successfully", 2161b6dd252SPhilipp Reisner * chances are that this caused arbitrary data corruption. 2171b6dd252SPhilipp Reisner * 2181b6dd252SPhilipp Reisner * "aborting" requests, or force-detaching the disk, is intended for 2191b6dd252SPhilipp Reisner * completely blocked/hung local backing devices which do no longer 2201b6dd252SPhilipp Reisner * complete requests at all, not even do error completions. In this 2211b6dd252SPhilipp Reisner * situation, usually a hard-reset and failover is the only way out. 2221b6dd252SPhilipp Reisner * 2231b6dd252SPhilipp Reisner * By "aborting", basically faking a local error-completion, 2241b6dd252SPhilipp Reisner * we allow for a more graceful swichover by cleanly migrating services. 2251b6dd252SPhilipp Reisner * Still the affected node has to be rebooted "soon". 2261b6dd252SPhilipp Reisner * 2271b6dd252SPhilipp Reisner * By completing these requests, we allow the upper layers to re-use 2281b6dd252SPhilipp Reisner * the associated data pages. 2291b6dd252SPhilipp Reisner * 2301b6dd252SPhilipp Reisner * If later the local backing device "recovers", and now DMAs some data 2311b6dd252SPhilipp Reisner * from disk into the original request pages, in the best case it will 2321b6dd252SPhilipp Reisner * just put random data into unused pages; but typically it will corrupt 2331b6dd252SPhilipp Reisner * meanwhile completely unrelated data, causing all sorts of damage. 2341b6dd252SPhilipp Reisner * 2351b6dd252SPhilipp Reisner * Which means delayed successful completion, 2361b6dd252SPhilipp Reisner * especially for READ requests, 2371b6dd252SPhilipp Reisner * is a reason to panic(). 2381b6dd252SPhilipp Reisner * 2391b6dd252SPhilipp Reisner * We assume that a delayed *error* completion is OK, 2401b6dd252SPhilipp Reisner * though we still will complain noisily about it. 2411b6dd252SPhilipp Reisner */ 2421b6dd252SPhilipp Reisner if (unlikely(req->rq_state & RQ_LOCAL_ABORTED)) { 2431b6dd252SPhilipp Reisner if (__ratelimit(&drbd_ratelimit_state)) 244d0180171SAndreas Gruenbacher drbd_emerg(device, "delayed completion of aborted local request; disk-timeout may be too aggressive\n"); 2451b6dd252SPhilipp Reisner 2464e4cbee9SChristoph Hellwig if (!bio->bi_status) 247142207f7SLars Ellenberg drbd_panic_after_delayed_completion_of_aborted_request(device); 2481b6dd252SPhilipp Reisner } 2491b6dd252SPhilipp Reisner 250b411b363SPhilipp Reisner /* to avoid recursion in __req_mod */ 2514e4cbee9SChristoph Hellwig if (unlikely(bio->bi_status)) { 25270246286SChristoph Hellwig switch (bio_op(bio)) { 25345c21793SChristoph Hellwig case REQ_OP_WRITE_ZEROES: 25470246286SChristoph Hellwig case REQ_OP_DISCARD: 2554e4cbee9SChristoph Hellwig if (bio->bi_status == BLK_STS_NOTSUPP) 25670246286SChristoph Hellwig what = DISCARD_COMPLETED_NOTSUPP; 2572f632aebSLars Ellenberg else 25870246286SChristoph Hellwig what = DISCARD_COMPLETED_WITH_ERROR; 25970246286SChristoph Hellwig break; 26070246286SChristoph Hellwig case REQ_OP_READ: 2611eff9d32SJens Axboe if (bio->bi_opf & REQ_RAHEAD) 26270246286SChristoph Hellwig what = READ_AHEAD_COMPLETED_WITH_ERROR; 26370246286SChristoph Hellwig else 26470246286SChristoph Hellwig what = READ_COMPLETED_WITH_ERROR; 26570246286SChristoph Hellwig break; 26670246286SChristoph Hellwig default: 26770246286SChristoph Hellwig what = WRITE_COMPLETED_WITH_ERROR; 26870246286SChristoph Hellwig break; 26970246286SChristoph Hellwig } 27070246286SChristoph Hellwig } else { 2718554df1cSAndreas Gruenbacher what = COMPLETED_OK; 27270246286SChristoph Hellwig } 273b411b363SPhilipp Reisner 2744e4cbee9SChristoph Hellwig req->private_bio = ERR_PTR(blk_status_to_errno(bio->bi_status)); 27564dafbc9SLars Ellenberg bio_put(bio); 276b411b363SPhilipp Reisner 277a115413dSLars Ellenberg /* not req_mod(), we need irqsave here! */ 2780500813fSAndreas Gruenbacher spin_lock_irqsave(&device->resource->req_lock, flags); 279a115413dSLars Ellenberg __req_mod(req, what, &m); 2800500813fSAndreas Gruenbacher spin_unlock_irqrestore(&device->resource->req_lock, flags); 281b30ab791SAndreas Gruenbacher put_ldev(device); 282a115413dSLars Ellenberg 283a115413dSLars Ellenberg if (m.bio) 284b30ab791SAndreas Gruenbacher complete_master_bio(device, &m); 285b411b363SPhilipp Reisner } 286b411b363SPhilipp Reisner 2873d0e6375SKees Cook void drbd_csum_ee(struct crypto_shash *tfm, struct drbd_peer_request *peer_req, void *digest) 28845bb912bSLars Ellenberg { 2893d0e6375SKees Cook SHASH_DESC_ON_STACK(desc, tfm); 290db830c46SAndreas Gruenbacher struct page *page = peer_req->pages; 29145bb912bSLars Ellenberg struct page *tmp; 29245bb912bSLars Ellenberg unsigned len; 2933d0e6375SKees Cook void *src; 29445bb912bSLars Ellenberg 2953d0e6375SKees Cook desc->tfm = tfm; 29645bb912bSLars Ellenberg 2973d0e6375SKees Cook crypto_shash_init(desc); 29845bb912bSLars Ellenberg 2993d0e6375SKees Cook src = kmap_atomic(page); 30045bb912bSLars Ellenberg while ((tmp = page_chain_next(page))) { 30145bb912bSLars Ellenberg /* all but the last page will be fully used */ 3023d0e6375SKees Cook crypto_shash_update(desc, src, PAGE_SIZE); 3033d0e6375SKees Cook kunmap_atomic(src); 30445bb912bSLars Ellenberg page = tmp; 3053d0e6375SKees Cook src = kmap_atomic(page); 30645bb912bSLars Ellenberg } 30745bb912bSLars Ellenberg /* and now the last, possibly only partially used page */ 308db830c46SAndreas Gruenbacher len = peer_req->i.size & (PAGE_SIZE - 1); 3093d0e6375SKees Cook crypto_shash_update(desc, src, len ?: PAGE_SIZE); 3103d0e6375SKees Cook kunmap_atomic(src); 3113d0e6375SKees Cook 3123d0e6375SKees Cook crypto_shash_final(desc, digest); 3133d0e6375SKees Cook shash_desc_zero(desc); 31445bb912bSLars Ellenberg } 31545bb912bSLars Ellenberg 3163d0e6375SKees Cook void drbd_csum_bio(struct crypto_shash *tfm, struct bio *bio, void *digest) 317b411b363SPhilipp Reisner { 3183d0e6375SKees Cook SHASH_DESC_ON_STACK(desc, tfm); 3197988613bSKent Overstreet struct bio_vec bvec; 3207988613bSKent Overstreet struct bvec_iter iter; 321b411b363SPhilipp Reisner 3223d0e6375SKees Cook desc->tfm = tfm; 323b411b363SPhilipp Reisner 3243d0e6375SKees Cook crypto_shash_init(desc); 325b411b363SPhilipp Reisner 3267988613bSKent Overstreet bio_for_each_segment(bvec, bio, iter) { 3273d0e6375SKees Cook u8 *src; 3283d0e6375SKees Cook 329*47227850SChristoph Hellwig src = bvec_kmap_local(&bvec); 330*47227850SChristoph Hellwig crypto_shash_update(desc, src, bvec.bv_len); 331*47227850SChristoph Hellwig kunmap_local(src); 3323d0e6375SKees Cook 3339104d31aSLars Ellenberg /* REQ_OP_WRITE_SAME has only one segment, 3349104d31aSLars Ellenberg * checksum the payload only once. */ 3359104d31aSLars Ellenberg if (bio_op(bio) == REQ_OP_WRITE_SAME) 3369104d31aSLars Ellenberg break; 337b411b363SPhilipp Reisner } 3383d0e6375SKees Cook crypto_shash_final(desc, digest); 3393d0e6375SKees Cook shash_desc_zero(desc); 340b411b363SPhilipp Reisner } 341b411b363SPhilipp Reisner 3429676c760SLars Ellenberg /* MAYBE merge common code with w_e_end_ov_req */ 34399920dc5SAndreas Gruenbacher static int w_e_send_csum(struct drbd_work *w, int cancel) 344b411b363SPhilipp Reisner { 345a8cd15baSAndreas Gruenbacher struct drbd_peer_request *peer_req = container_of(w, struct drbd_peer_request, w); 3466780139cSAndreas Gruenbacher struct drbd_peer_device *peer_device = peer_req->peer_device; 3476780139cSAndreas Gruenbacher struct drbd_device *device = peer_device->device; 348b411b363SPhilipp Reisner int digest_size; 349b411b363SPhilipp Reisner void *digest; 35099920dc5SAndreas Gruenbacher int err = 0; 351b411b363SPhilipp Reisner 35253ea4331SLars Ellenberg if (unlikely(cancel)) 35353ea4331SLars Ellenberg goto out; 354b411b363SPhilipp Reisner 3559676c760SLars Ellenberg if (unlikely((peer_req->flags & EE_WAS_ERROR) != 0)) 35653ea4331SLars Ellenberg goto out; 35753ea4331SLars Ellenberg 3583d0e6375SKees Cook digest_size = crypto_shash_digestsize(peer_device->connection->csums_tfm); 359b411b363SPhilipp Reisner digest = kmalloc(digest_size, GFP_NOIO); 360b411b363SPhilipp Reisner if (digest) { 361db830c46SAndreas Gruenbacher sector_t sector = peer_req->i.sector; 362db830c46SAndreas Gruenbacher unsigned int size = peer_req->i.size; 3636780139cSAndreas Gruenbacher drbd_csum_ee(peer_device->connection->csums_tfm, peer_req, digest); 3649676c760SLars Ellenberg /* Free peer_req and pages before send. 36553ea4331SLars Ellenberg * In case we block on congestion, we could otherwise run into 36653ea4331SLars Ellenberg * some distributed deadlock, if the other side blocks on 36753ea4331SLars Ellenberg * congestion as well, because our receiver blocks in 368c37c8ecfSAndreas Gruenbacher * drbd_alloc_pages due to pp_in_use > max_buffers. */ 369b30ab791SAndreas Gruenbacher drbd_free_peer_req(device, peer_req); 370db830c46SAndreas Gruenbacher peer_req = NULL; 371b30ab791SAndreas Gruenbacher inc_rs_pending(device); 3726780139cSAndreas Gruenbacher err = drbd_send_drequest_csum(peer_device, sector, size, 37353ea4331SLars Ellenberg digest, digest_size, 374b411b363SPhilipp Reisner P_CSUM_RS_REQUEST); 375b411b363SPhilipp Reisner kfree(digest); 376b411b363SPhilipp Reisner } else { 377d0180171SAndreas Gruenbacher drbd_err(device, "kmalloc() of digest failed.\n"); 37899920dc5SAndreas Gruenbacher err = -ENOMEM; 379b411b363SPhilipp Reisner } 380b411b363SPhilipp Reisner 38153ea4331SLars Ellenberg out: 382db830c46SAndreas Gruenbacher if (peer_req) 383b30ab791SAndreas Gruenbacher drbd_free_peer_req(device, peer_req); 384b411b363SPhilipp Reisner 38599920dc5SAndreas Gruenbacher if (unlikely(err)) 386d0180171SAndreas Gruenbacher drbd_err(device, "drbd_send_drequest(..., csum) failed\n"); 38799920dc5SAndreas Gruenbacher return err; 388b411b363SPhilipp Reisner } 389b411b363SPhilipp Reisner 390b411b363SPhilipp Reisner #define GFP_TRY (__GFP_HIGHMEM | __GFP_NOWARN) 391b411b363SPhilipp Reisner 39269a22773SAndreas Gruenbacher static int read_for_csum(struct drbd_peer_device *peer_device, sector_t sector, int size) 393b411b363SPhilipp Reisner { 39469a22773SAndreas Gruenbacher struct drbd_device *device = peer_device->device; 395db830c46SAndreas Gruenbacher struct drbd_peer_request *peer_req; 396b411b363SPhilipp Reisner 397b30ab791SAndreas Gruenbacher if (!get_ldev(device)) 39880a40e43SLars Ellenberg return -EIO; 399b411b363SPhilipp Reisner 400b411b363SPhilipp Reisner /* GFP_TRY, because if there is no memory available right now, this may 401b411b363SPhilipp Reisner * be rescheduled for later. It is "only" background resync, after all. */ 40269a22773SAndreas Gruenbacher peer_req = drbd_alloc_peer_req(peer_device, ID_SYNCER /* unused */, sector, 4039104d31aSLars Ellenberg size, size, GFP_TRY); 404db830c46SAndreas Gruenbacher if (!peer_req) 40580a40e43SLars Ellenberg goto defer; 406b411b363SPhilipp Reisner 407a8cd15baSAndreas Gruenbacher peer_req->w.cb = w_e_send_csum; 4080500813fSAndreas Gruenbacher spin_lock_irq(&device->resource->req_lock); 409b9ed7080SLars Ellenberg list_add_tail(&peer_req->w.list, &device->read_ee); 4100500813fSAndreas Gruenbacher spin_unlock_irq(&device->resource->req_lock); 411b411b363SPhilipp Reisner 412b30ab791SAndreas Gruenbacher atomic_add(size >> 9, &device->rs_sect_ev); 413bb3cc85eSMike Christie if (drbd_submit_peer_request(device, peer_req, REQ_OP_READ, 0, 414bb3cc85eSMike Christie DRBD_FAULT_RS_RD) == 0) 41580a40e43SLars Ellenberg return 0; 41645bb912bSLars Ellenberg 41710f6d992SLars Ellenberg /* If it failed because of ENOMEM, retry should help. If it failed 41810f6d992SLars Ellenberg * because bio_add_page failed (probably broken lower level driver), 41910f6d992SLars Ellenberg * retry may or may not help. 42010f6d992SLars Ellenberg * If it does not, you may need to force disconnect. */ 4210500813fSAndreas Gruenbacher spin_lock_irq(&device->resource->req_lock); 422a8cd15baSAndreas Gruenbacher list_del(&peer_req->w.list); 4230500813fSAndreas Gruenbacher spin_unlock_irq(&device->resource->req_lock); 42422cc37a9SLars Ellenberg 425b30ab791SAndreas Gruenbacher drbd_free_peer_req(device, peer_req); 42680a40e43SLars Ellenberg defer: 427b30ab791SAndreas Gruenbacher put_ldev(device); 42880a40e43SLars Ellenberg return -EAGAIN; 429b411b363SPhilipp Reisner } 430b411b363SPhilipp Reisner 43199920dc5SAndreas Gruenbacher int w_resync_timer(struct drbd_work *w, int cancel) 432794abb75SPhilipp Reisner { 43384b8c06bSAndreas Gruenbacher struct drbd_device *device = 43484b8c06bSAndreas Gruenbacher container_of(w, struct drbd_device, resync_work); 43584b8c06bSAndreas Gruenbacher 436b30ab791SAndreas Gruenbacher switch (device->state.conn) { 437794abb75SPhilipp Reisner case C_VERIFY_S: 438d448a2e1SAndreas Gruenbacher make_ov_request(device, cancel); 439794abb75SPhilipp Reisner break; 440794abb75SPhilipp Reisner case C_SYNC_TARGET: 441d448a2e1SAndreas Gruenbacher make_resync_request(device, cancel); 442794abb75SPhilipp Reisner break; 443794abb75SPhilipp Reisner } 444794abb75SPhilipp Reisner 44599920dc5SAndreas Gruenbacher return 0; 446794abb75SPhilipp Reisner } 447794abb75SPhilipp Reisner 4482bccef39SKees Cook void resync_timer_fn(struct timer_list *t) 449b411b363SPhilipp Reisner { 4502bccef39SKees Cook struct drbd_device *device = from_timer(device, t, resync_timer); 451b411b363SPhilipp Reisner 45215e26f6aSLars Ellenberg drbd_queue_work_if_unqueued( 45315e26f6aSLars Ellenberg &first_peer_device(device)->connection->sender_work, 45484b8c06bSAndreas Gruenbacher &device->resync_work); 455b411b363SPhilipp Reisner } 456b411b363SPhilipp Reisner 457778f271dSPhilipp Reisner static void fifo_set(struct fifo_buffer *fb, int value) 458778f271dSPhilipp Reisner { 459778f271dSPhilipp Reisner int i; 460778f271dSPhilipp Reisner 461778f271dSPhilipp Reisner for (i = 0; i < fb->size; i++) 462f10f2623SPhilipp Reisner fb->values[i] = value; 463778f271dSPhilipp Reisner } 464778f271dSPhilipp Reisner 465778f271dSPhilipp Reisner static int fifo_push(struct fifo_buffer *fb, int value) 466778f271dSPhilipp Reisner { 467778f271dSPhilipp Reisner int ov; 468778f271dSPhilipp Reisner 469778f271dSPhilipp Reisner ov = fb->values[fb->head_index]; 470778f271dSPhilipp Reisner fb->values[fb->head_index++] = value; 471778f271dSPhilipp Reisner 472778f271dSPhilipp Reisner if (fb->head_index >= fb->size) 473778f271dSPhilipp Reisner fb->head_index = 0; 474778f271dSPhilipp Reisner 475778f271dSPhilipp Reisner return ov; 476778f271dSPhilipp Reisner } 477778f271dSPhilipp Reisner 478778f271dSPhilipp Reisner static void fifo_add_val(struct fifo_buffer *fb, int value) 479778f271dSPhilipp Reisner { 480778f271dSPhilipp Reisner int i; 481778f271dSPhilipp Reisner 482778f271dSPhilipp Reisner for (i = 0; i < fb->size; i++) 483778f271dSPhilipp Reisner fb->values[i] += value; 484778f271dSPhilipp Reisner } 485778f271dSPhilipp Reisner 4866a365874SStephen Kitt struct fifo_buffer *fifo_alloc(unsigned int fifo_size) 4879958c857SPhilipp Reisner { 4889958c857SPhilipp Reisner struct fifo_buffer *fb; 4899958c857SPhilipp Reisner 4906a365874SStephen Kitt fb = kzalloc(struct_size(fb, values, fifo_size), GFP_NOIO); 4919958c857SPhilipp Reisner if (!fb) 4929958c857SPhilipp Reisner return NULL; 4939958c857SPhilipp Reisner 4949958c857SPhilipp Reisner fb->head_index = 0; 4959958c857SPhilipp Reisner fb->size = fifo_size; 4969958c857SPhilipp Reisner fb->total = 0; 4979958c857SPhilipp Reisner 4989958c857SPhilipp Reisner return fb; 4999958c857SPhilipp Reisner } 5009958c857SPhilipp Reisner 5010e49d7b0SLars Ellenberg static int drbd_rs_controller(struct drbd_device *device, unsigned int sect_in) 502778f271dSPhilipp Reisner { 503daeda1ccSPhilipp Reisner struct disk_conf *dc; 5047f34f614SLars Ellenberg unsigned int want; /* The number of sectors we want in-flight */ 505778f271dSPhilipp Reisner int req_sect; /* Number of sectors to request in this turn */ 5067f34f614SLars Ellenberg int correction; /* Number of sectors more we need in-flight */ 507778f271dSPhilipp Reisner int cps; /* correction per invocation of drbd_rs_controller() */ 508778f271dSPhilipp Reisner int steps; /* Number of time steps to plan ahead */ 509778f271dSPhilipp Reisner int curr_corr; 510778f271dSPhilipp Reisner int max_sect; 511813472ceSPhilipp Reisner struct fifo_buffer *plan; 512778f271dSPhilipp Reisner 513b30ab791SAndreas Gruenbacher dc = rcu_dereference(device->ldev->disk_conf); 514b30ab791SAndreas Gruenbacher plan = rcu_dereference(device->rs_plan_s); 515778f271dSPhilipp Reisner 516813472ceSPhilipp Reisner steps = plan->size; /* (dc->c_plan_ahead * 10 * SLEEP_TIME) / HZ; */ 517778f271dSPhilipp Reisner 518b30ab791SAndreas Gruenbacher if (device->rs_in_flight + sect_in == 0) { /* At start of resync */ 519daeda1ccSPhilipp Reisner want = ((dc->resync_rate * 2 * SLEEP_TIME) / HZ) * steps; 520778f271dSPhilipp Reisner } else { /* normal path */ 521daeda1ccSPhilipp Reisner want = dc->c_fill_target ? dc->c_fill_target : 522daeda1ccSPhilipp Reisner sect_in * dc->c_delay_target * HZ / (SLEEP_TIME * 10); 523778f271dSPhilipp Reisner } 524778f271dSPhilipp Reisner 525b30ab791SAndreas Gruenbacher correction = want - device->rs_in_flight - plan->total; 526778f271dSPhilipp Reisner 527778f271dSPhilipp Reisner /* Plan ahead */ 528778f271dSPhilipp Reisner cps = correction / steps; 529813472ceSPhilipp Reisner fifo_add_val(plan, cps); 530813472ceSPhilipp Reisner plan->total += cps * steps; 531778f271dSPhilipp Reisner 532778f271dSPhilipp Reisner /* What we do in this step */ 533813472ceSPhilipp Reisner curr_corr = fifo_push(plan, 0); 534813472ceSPhilipp Reisner plan->total -= curr_corr; 535778f271dSPhilipp Reisner 536778f271dSPhilipp Reisner req_sect = sect_in + curr_corr; 537778f271dSPhilipp Reisner if (req_sect < 0) 538778f271dSPhilipp Reisner req_sect = 0; 539778f271dSPhilipp Reisner 540daeda1ccSPhilipp Reisner max_sect = (dc->c_max_rate * 2 * SLEEP_TIME) / HZ; 541778f271dSPhilipp Reisner if (req_sect > max_sect) 542778f271dSPhilipp Reisner req_sect = max_sect; 543778f271dSPhilipp Reisner 544778f271dSPhilipp Reisner /* 545d0180171SAndreas Gruenbacher drbd_warn(device, "si=%u if=%d wa=%u co=%d st=%d cps=%d pl=%d cc=%d rs=%d\n", 546b30ab791SAndreas Gruenbacher sect_in, device->rs_in_flight, want, correction, 547b30ab791SAndreas Gruenbacher steps, cps, device->rs_planed, curr_corr, req_sect); 548778f271dSPhilipp Reisner */ 549778f271dSPhilipp Reisner 550778f271dSPhilipp Reisner return req_sect; 551778f271dSPhilipp Reisner } 552778f271dSPhilipp Reisner 553b30ab791SAndreas Gruenbacher static int drbd_rs_number_requests(struct drbd_device *device) 554e65f440dSLars Ellenberg { 5550e49d7b0SLars Ellenberg unsigned int sect_in; /* Number of sectors that came in since the last turn */ 5560e49d7b0SLars Ellenberg int number, mxb; 5570e49d7b0SLars Ellenberg 5580e49d7b0SLars Ellenberg sect_in = atomic_xchg(&device->rs_sect_in, 0); 5590e49d7b0SLars Ellenberg device->rs_in_flight -= sect_in; 560813472ceSPhilipp Reisner 561813472ceSPhilipp Reisner rcu_read_lock(); 5620e49d7b0SLars Ellenberg mxb = drbd_get_max_buffers(device) / 2; 563b30ab791SAndreas Gruenbacher if (rcu_dereference(device->rs_plan_s)->size) { 5640e49d7b0SLars Ellenberg number = drbd_rs_controller(device, sect_in) >> (BM_BLOCK_SHIFT - 9); 565b30ab791SAndreas Gruenbacher device->c_sync_rate = number * HZ * (BM_BLOCK_SIZE / 1024) / SLEEP_TIME; 566e65f440dSLars Ellenberg } else { 567b30ab791SAndreas Gruenbacher device->c_sync_rate = rcu_dereference(device->ldev->disk_conf)->resync_rate; 568b30ab791SAndreas Gruenbacher number = SLEEP_TIME * device->c_sync_rate / ((BM_BLOCK_SIZE / 1024) * HZ); 569e65f440dSLars Ellenberg } 570813472ceSPhilipp Reisner rcu_read_unlock(); 571e65f440dSLars Ellenberg 5720e49d7b0SLars Ellenberg /* Don't have more than "max-buffers"/2 in-flight. 5730e49d7b0SLars Ellenberg * Otherwise we may cause the remote site to stall on drbd_alloc_pages(), 5740e49d7b0SLars Ellenberg * potentially causing a distributed deadlock on congestion during 5750e49d7b0SLars Ellenberg * online-verify or (checksum-based) resync, if max-buffers, 5760e49d7b0SLars Ellenberg * socket buffer sizes and resync rate settings are mis-configured. */ 5777f34f614SLars Ellenberg 5787f34f614SLars Ellenberg /* note that "number" is in units of "BM_BLOCK_SIZE" (which is 4k), 5797f34f614SLars Ellenberg * mxb (as used here, and in drbd_alloc_pages on the peer) is 5807f34f614SLars Ellenberg * "number of pages" (typically also 4k), 5817f34f614SLars Ellenberg * but "rs_in_flight" is in "sectors" (512 Byte). */ 5827f34f614SLars Ellenberg if (mxb - device->rs_in_flight/8 < number) 5837f34f614SLars Ellenberg number = mxb - device->rs_in_flight/8; 5840e49d7b0SLars Ellenberg 585e65f440dSLars Ellenberg return number; 586e65f440dSLars Ellenberg } 587e65f440dSLars Ellenberg 58844a4d551SLars Ellenberg static int make_resync_request(struct drbd_device *const device, int cancel) 589b411b363SPhilipp Reisner { 59044a4d551SLars Ellenberg struct drbd_peer_device *const peer_device = first_peer_device(device); 59144a4d551SLars Ellenberg struct drbd_connection *const connection = peer_device ? peer_device->connection : NULL; 592b411b363SPhilipp Reisner unsigned long bit; 593b411b363SPhilipp Reisner sector_t sector; 594155bd9d1SChristoph Hellwig const sector_t capacity = get_capacity(device->vdisk); 5951816a2b4SLars Ellenberg int max_bio_size; 596e65f440dSLars Ellenberg int number, rollback_i, size; 597506afb62SLars Ellenberg int align, requeue = 0; 5980f0601f4SLars Ellenberg int i = 0; 59992d94ae6SPhilipp Reisner int discard_granularity = 0; 600b411b363SPhilipp Reisner 601b411b363SPhilipp Reisner if (unlikely(cancel)) 60299920dc5SAndreas Gruenbacher return 0; 603b411b363SPhilipp Reisner 604b30ab791SAndreas Gruenbacher if (device->rs_total == 0) { 605af85e8e8SLars Ellenberg /* empty resync? */ 606b30ab791SAndreas Gruenbacher drbd_resync_finished(device); 60799920dc5SAndreas Gruenbacher return 0; 608af85e8e8SLars Ellenberg } 609af85e8e8SLars Ellenberg 610b30ab791SAndreas Gruenbacher if (!get_ldev(device)) { 611b30ab791SAndreas Gruenbacher /* Since we only need to access device->rsync a 612b30ab791SAndreas Gruenbacher get_ldev_if_state(device,D_FAILED) would be sufficient, but 613b411b363SPhilipp Reisner to continue resync with a broken disk makes no sense at 614b411b363SPhilipp Reisner all */ 615d0180171SAndreas Gruenbacher drbd_err(device, "Disk broke down during resync!\n"); 61699920dc5SAndreas Gruenbacher return 0; 617b411b363SPhilipp Reisner } 618b411b363SPhilipp Reisner 6199104d31aSLars Ellenberg if (connection->agreed_features & DRBD_FF_THIN_RESYNC) { 62092d94ae6SPhilipp Reisner rcu_read_lock(); 62192d94ae6SPhilipp Reisner discard_granularity = rcu_dereference(device->ldev->disk_conf)->rs_discard_granularity; 62292d94ae6SPhilipp Reisner rcu_read_unlock(); 62392d94ae6SPhilipp Reisner } 62492d94ae6SPhilipp Reisner 625b30ab791SAndreas Gruenbacher max_bio_size = queue_max_hw_sectors(device->rq_queue) << 9; 626b30ab791SAndreas Gruenbacher number = drbd_rs_number_requests(device); 6270e49d7b0SLars Ellenberg if (number <= 0) 6280f0601f4SLars Ellenberg goto requeue; 629b411b363SPhilipp Reisner 630b411b363SPhilipp Reisner for (i = 0; i < number; i++) { 631506afb62SLars Ellenberg /* Stop generating RS requests when half of the send buffer is filled, 632506afb62SLars Ellenberg * but notify TCP that we'd like to have more space. */ 63344a4d551SLars Ellenberg mutex_lock(&connection->data.mutex); 63444a4d551SLars Ellenberg if (connection->data.socket) { 635506afb62SLars Ellenberg struct sock *sk = connection->data.socket->sk; 636506afb62SLars Ellenberg int queued = sk->sk_wmem_queued; 637506afb62SLars Ellenberg int sndbuf = sk->sk_sndbuf; 638506afb62SLars Ellenberg if (queued > sndbuf / 2) { 639506afb62SLars Ellenberg requeue = 1; 640506afb62SLars Ellenberg if (sk->sk_socket) 641506afb62SLars Ellenberg set_bit(SOCK_NOSPACE, &sk->sk_socket->flags); 642b411b363SPhilipp Reisner } 643506afb62SLars Ellenberg } else 644506afb62SLars Ellenberg requeue = 1; 64544a4d551SLars Ellenberg mutex_unlock(&connection->data.mutex); 646506afb62SLars Ellenberg if (requeue) 647b411b363SPhilipp Reisner goto requeue; 648b411b363SPhilipp Reisner 649b411b363SPhilipp Reisner next_sector: 650b411b363SPhilipp Reisner size = BM_BLOCK_SIZE; 651b30ab791SAndreas Gruenbacher bit = drbd_bm_find_next(device, device->bm_resync_fo); 652b411b363SPhilipp Reisner 6534b0715f0SLars Ellenberg if (bit == DRBD_END_OF_BITMAP) { 654b30ab791SAndreas Gruenbacher device->bm_resync_fo = drbd_bm_bits(device); 655b30ab791SAndreas Gruenbacher put_ldev(device); 65699920dc5SAndreas Gruenbacher return 0; 657b411b363SPhilipp Reisner } 658b411b363SPhilipp Reisner 659b411b363SPhilipp Reisner sector = BM_BIT_TO_SECT(bit); 660b411b363SPhilipp Reisner 661ad3fee79SLars Ellenberg if (drbd_try_rs_begin_io(device, sector)) { 662b30ab791SAndreas Gruenbacher device->bm_resync_fo = bit; 663b411b363SPhilipp Reisner goto requeue; 664b411b363SPhilipp Reisner } 665b30ab791SAndreas Gruenbacher device->bm_resync_fo = bit + 1; 666b411b363SPhilipp Reisner 667b30ab791SAndreas Gruenbacher if (unlikely(drbd_bm_test_bit(device, bit) == 0)) { 668b30ab791SAndreas Gruenbacher drbd_rs_complete_io(device, sector); 669b411b363SPhilipp Reisner goto next_sector; 670b411b363SPhilipp Reisner } 671b411b363SPhilipp Reisner 6721816a2b4SLars Ellenberg #if DRBD_MAX_BIO_SIZE > BM_BLOCK_SIZE 673b411b363SPhilipp Reisner /* try to find some adjacent bits. 674b411b363SPhilipp Reisner * we stop if we have already the maximum req size. 675b411b363SPhilipp Reisner * 676b411b363SPhilipp Reisner * Additionally always align bigger requests, in order to 677b411b363SPhilipp Reisner * be prepared for all stripe sizes of software RAIDs. 678b411b363SPhilipp Reisner */ 679b411b363SPhilipp Reisner align = 1; 680d207450cSPhilipp Reisner rollback_i = i; 6816377b923SLars Ellenberg while (i < number) { 6821816a2b4SLars Ellenberg if (size + BM_BLOCK_SIZE > max_bio_size) 683b411b363SPhilipp Reisner break; 684b411b363SPhilipp Reisner 685b411b363SPhilipp Reisner /* Be always aligned */ 686b411b363SPhilipp Reisner if (sector & ((1<<(align+3))-1)) 687b411b363SPhilipp Reisner break; 688b411b363SPhilipp Reisner 68992d94ae6SPhilipp Reisner if (discard_granularity && size == discard_granularity) 69092d94ae6SPhilipp Reisner break; 69192d94ae6SPhilipp Reisner 692b411b363SPhilipp Reisner /* do not cross extent boundaries */ 693b411b363SPhilipp Reisner if (((bit+1) & BM_BLOCKS_PER_BM_EXT_MASK) == 0) 694b411b363SPhilipp Reisner break; 695b411b363SPhilipp Reisner /* now, is it actually dirty, after all? 696b411b363SPhilipp Reisner * caution, drbd_bm_test_bit is tri-state for some 697b411b363SPhilipp Reisner * obscure reason; ( b == 0 ) would get the out-of-band 698b411b363SPhilipp Reisner * only accidentally right because of the "oddly sized" 699b411b363SPhilipp Reisner * adjustment below */ 700b30ab791SAndreas Gruenbacher if (drbd_bm_test_bit(device, bit+1) != 1) 701b411b363SPhilipp Reisner break; 702b411b363SPhilipp Reisner bit++; 703b411b363SPhilipp Reisner size += BM_BLOCK_SIZE; 704b411b363SPhilipp Reisner if ((BM_BLOCK_SIZE << align) <= size) 705b411b363SPhilipp Reisner align++; 706b411b363SPhilipp Reisner i++; 707b411b363SPhilipp Reisner } 708b411b363SPhilipp Reisner /* if we merged some, 709b411b363SPhilipp Reisner * reset the offset to start the next drbd_bm_find_next from */ 710b411b363SPhilipp Reisner if (size > BM_BLOCK_SIZE) 711b30ab791SAndreas Gruenbacher device->bm_resync_fo = bit + 1; 712b411b363SPhilipp Reisner #endif 713b411b363SPhilipp Reisner 714b411b363SPhilipp Reisner /* adjust very last sectors, in case we are oddly sized */ 715b411b363SPhilipp Reisner if (sector + (size>>9) > capacity) 716b411b363SPhilipp Reisner size = (capacity-sector)<<9; 717aaaba345SLars Ellenberg 718aaaba345SLars Ellenberg if (device->use_csums) { 71944a4d551SLars Ellenberg switch (read_for_csum(peer_device, sector, size)) { 72080a40e43SLars Ellenberg case -EIO: /* Disk failure */ 721b30ab791SAndreas Gruenbacher put_ldev(device); 72299920dc5SAndreas Gruenbacher return -EIO; 72380a40e43SLars Ellenberg case -EAGAIN: /* allocation failed, or ldev busy */ 724b30ab791SAndreas Gruenbacher drbd_rs_complete_io(device, sector); 725b30ab791SAndreas Gruenbacher device->bm_resync_fo = BM_SECT_TO_BIT(sector); 726d207450cSPhilipp Reisner i = rollback_i; 727b411b363SPhilipp Reisner goto requeue; 72880a40e43SLars Ellenberg case 0: 72980a40e43SLars Ellenberg /* everything ok */ 73080a40e43SLars Ellenberg break; 73180a40e43SLars Ellenberg default: 73280a40e43SLars Ellenberg BUG(); 733b411b363SPhilipp Reisner } 734b411b363SPhilipp Reisner } else { 73599920dc5SAndreas Gruenbacher int err; 73699920dc5SAndreas Gruenbacher 737b30ab791SAndreas Gruenbacher inc_rs_pending(device); 73892d94ae6SPhilipp Reisner err = drbd_send_drequest(peer_device, 73992d94ae6SPhilipp Reisner size == discard_granularity ? P_RS_THIN_REQ : P_RS_DATA_REQUEST, 74099920dc5SAndreas Gruenbacher sector, size, ID_SYNCER); 74199920dc5SAndreas Gruenbacher if (err) { 742d0180171SAndreas Gruenbacher drbd_err(device, "drbd_send_drequest() failed, aborting...\n"); 743b30ab791SAndreas Gruenbacher dec_rs_pending(device); 744b30ab791SAndreas Gruenbacher put_ldev(device); 74599920dc5SAndreas Gruenbacher return err; 746b411b363SPhilipp Reisner } 747b411b363SPhilipp Reisner } 748b411b363SPhilipp Reisner } 749b411b363SPhilipp Reisner 750b30ab791SAndreas Gruenbacher if (device->bm_resync_fo >= drbd_bm_bits(device)) { 751b411b363SPhilipp Reisner /* last syncer _request_ was sent, 752b411b363SPhilipp Reisner * but the P_RS_DATA_REPLY not yet received. sync will end (and 753b411b363SPhilipp Reisner * next sync group will resume), as soon as we receive the last 754b411b363SPhilipp Reisner * resync data block, and the last bit is cleared. 755b411b363SPhilipp Reisner * until then resync "work" is "inactive" ... 756b411b363SPhilipp Reisner */ 757b30ab791SAndreas Gruenbacher put_ldev(device); 75899920dc5SAndreas Gruenbacher return 0; 759b411b363SPhilipp Reisner } 760b411b363SPhilipp Reisner 761b411b363SPhilipp Reisner requeue: 762b30ab791SAndreas Gruenbacher device->rs_in_flight += (i << (BM_BLOCK_SHIFT - 9)); 763b30ab791SAndreas Gruenbacher mod_timer(&device->resync_timer, jiffies + SLEEP_TIME); 764b30ab791SAndreas Gruenbacher put_ldev(device); 76599920dc5SAndreas Gruenbacher return 0; 766b411b363SPhilipp Reisner } 767b411b363SPhilipp Reisner 768d448a2e1SAndreas Gruenbacher static int make_ov_request(struct drbd_device *device, int cancel) 769b411b363SPhilipp Reisner { 770b411b363SPhilipp Reisner int number, i, size; 771b411b363SPhilipp Reisner sector_t sector; 772155bd9d1SChristoph Hellwig const sector_t capacity = get_capacity(device->vdisk); 77358ffa580SLars Ellenberg bool stop_sector_reached = false; 774b411b363SPhilipp Reisner 775b411b363SPhilipp Reisner if (unlikely(cancel)) 776b411b363SPhilipp Reisner return 1; 777b411b363SPhilipp Reisner 778b30ab791SAndreas Gruenbacher number = drbd_rs_number_requests(device); 779b411b363SPhilipp Reisner 780b30ab791SAndreas Gruenbacher sector = device->ov_position; 781b411b363SPhilipp Reisner for (i = 0; i < number; i++) { 78258ffa580SLars Ellenberg if (sector >= capacity) 783b411b363SPhilipp Reisner return 1; 78458ffa580SLars Ellenberg 78558ffa580SLars Ellenberg /* We check for "finished" only in the reply path: 78658ffa580SLars Ellenberg * w_e_end_ov_reply(). 78758ffa580SLars Ellenberg * We need to send at least one request out. */ 78858ffa580SLars Ellenberg stop_sector_reached = i > 0 789b30ab791SAndreas Gruenbacher && verify_can_do_stop_sector(device) 790b30ab791SAndreas Gruenbacher && sector >= device->ov_stop_sector; 79158ffa580SLars Ellenberg if (stop_sector_reached) 79258ffa580SLars Ellenberg break; 793b411b363SPhilipp Reisner 794b411b363SPhilipp Reisner size = BM_BLOCK_SIZE; 795b411b363SPhilipp Reisner 796ad3fee79SLars Ellenberg if (drbd_try_rs_begin_io(device, sector)) { 797b30ab791SAndreas Gruenbacher device->ov_position = sector; 798b411b363SPhilipp Reisner goto requeue; 799b411b363SPhilipp Reisner } 800b411b363SPhilipp Reisner 801b411b363SPhilipp Reisner if (sector + (size>>9) > capacity) 802b411b363SPhilipp Reisner size = (capacity-sector)<<9; 803b411b363SPhilipp Reisner 804b30ab791SAndreas Gruenbacher inc_rs_pending(device); 80569a22773SAndreas Gruenbacher if (drbd_send_ov_request(first_peer_device(device), sector, size)) { 806b30ab791SAndreas Gruenbacher dec_rs_pending(device); 807b411b363SPhilipp Reisner return 0; 808b411b363SPhilipp Reisner } 809b411b363SPhilipp Reisner sector += BM_SECT_PER_BIT; 810b411b363SPhilipp Reisner } 811b30ab791SAndreas Gruenbacher device->ov_position = sector; 812b411b363SPhilipp Reisner 813b411b363SPhilipp Reisner requeue: 814b30ab791SAndreas Gruenbacher device->rs_in_flight += (i << (BM_BLOCK_SHIFT - 9)); 81558ffa580SLars Ellenberg if (i == 0 || !stop_sector_reached) 816b30ab791SAndreas Gruenbacher mod_timer(&device->resync_timer, jiffies + SLEEP_TIME); 817b411b363SPhilipp Reisner return 1; 818b411b363SPhilipp Reisner } 819b411b363SPhilipp Reisner 82099920dc5SAndreas Gruenbacher int w_ov_finished(struct drbd_work *w, int cancel) 821b411b363SPhilipp Reisner { 82284b8c06bSAndreas Gruenbacher struct drbd_device_work *dw = 82384b8c06bSAndreas Gruenbacher container_of(w, struct drbd_device_work, w); 82484b8c06bSAndreas Gruenbacher struct drbd_device *device = dw->device; 82584b8c06bSAndreas Gruenbacher kfree(dw); 826b30ab791SAndreas Gruenbacher ov_out_of_sync_print(device); 827b30ab791SAndreas Gruenbacher drbd_resync_finished(device); 828b411b363SPhilipp Reisner 82999920dc5SAndreas Gruenbacher return 0; 830b411b363SPhilipp Reisner } 831b411b363SPhilipp Reisner 83299920dc5SAndreas Gruenbacher static int w_resync_finished(struct drbd_work *w, int cancel) 833b411b363SPhilipp Reisner { 83484b8c06bSAndreas Gruenbacher struct drbd_device_work *dw = 83584b8c06bSAndreas Gruenbacher container_of(w, struct drbd_device_work, w); 83684b8c06bSAndreas Gruenbacher struct drbd_device *device = dw->device; 83784b8c06bSAndreas Gruenbacher kfree(dw); 838b411b363SPhilipp Reisner 839b30ab791SAndreas Gruenbacher drbd_resync_finished(device); 840b411b363SPhilipp Reisner 84199920dc5SAndreas Gruenbacher return 0; 842b411b363SPhilipp Reisner } 843b411b363SPhilipp Reisner 844b30ab791SAndreas Gruenbacher static void ping_peer(struct drbd_device *device) 845af85e8e8SLars Ellenberg { 846a6b32bc3SAndreas Gruenbacher struct drbd_connection *connection = first_peer_device(device)->connection; 8472a67d8b9SPhilipp Reisner 848bde89a9eSAndreas Gruenbacher clear_bit(GOT_PING_ACK, &connection->flags); 849bde89a9eSAndreas Gruenbacher request_ping(connection); 850bde89a9eSAndreas Gruenbacher wait_event(connection->ping_wait, 851bde89a9eSAndreas Gruenbacher test_bit(GOT_PING_ACK, &connection->flags) || device->state.conn < C_CONNECTED); 852af85e8e8SLars Ellenberg } 853af85e8e8SLars Ellenberg 854b30ab791SAndreas Gruenbacher int drbd_resync_finished(struct drbd_device *device) 855b411b363SPhilipp Reisner { 85626a96110SLars Ellenberg struct drbd_connection *connection = first_peer_device(device)->connection; 857b411b363SPhilipp Reisner unsigned long db, dt, dbdt; 858b411b363SPhilipp Reisner unsigned long n_oos; 859b411b363SPhilipp Reisner union drbd_state os, ns; 86084b8c06bSAndreas Gruenbacher struct drbd_device_work *dw; 861b411b363SPhilipp Reisner char *khelper_cmd = NULL; 86226525618SLars Ellenberg int verify_done = 0; 863b411b363SPhilipp Reisner 864b411b363SPhilipp Reisner /* Remove all elements from the resync LRU. Since future actions 865b411b363SPhilipp Reisner * might set bits in the (main) bitmap, then the entries in the 866b411b363SPhilipp Reisner * resync LRU would be wrong. */ 867b30ab791SAndreas Gruenbacher if (drbd_rs_del_all(device)) { 868b411b363SPhilipp Reisner /* In case this is not possible now, most probably because 869b411b363SPhilipp Reisner * there are P_RS_DATA_REPLY Packets lingering on the worker's 870b411b363SPhilipp Reisner * queue (or even the read operations for those packets 871b411b363SPhilipp Reisner * is not finished by now). Retry in 100ms. */ 872b411b363SPhilipp Reisner 87320ee6390SPhilipp Reisner schedule_timeout_interruptible(HZ / 10); 87484b8c06bSAndreas Gruenbacher dw = kmalloc(sizeof(struct drbd_device_work), GFP_ATOMIC); 87584b8c06bSAndreas Gruenbacher if (dw) { 87684b8c06bSAndreas Gruenbacher dw->w.cb = w_resync_finished; 87784b8c06bSAndreas Gruenbacher dw->device = device; 87826a96110SLars Ellenberg drbd_queue_work(&connection->sender_work, &dw->w); 879b411b363SPhilipp Reisner return 1; 880b411b363SPhilipp Reisner } 88184b8c06bSAndreas Gruenbacher drbd_err(device, "Warn failed to drbd_rs_del_all() and to kmalloc(dw).\n"); 882b411b363SPhilipp Reisner } 883b411b363SPhilipp Reisner 884b30ab791SAndreas Gruenbacher dt = (jiffies - device->rs_start - device->rs_paused) / HZ; 885b411b363SPhilipp Reisner if (dt <= 0) 886b411b363SPhilipp Reisner dt = 1; 88758ffa580SLars Ellenberg 888b30ab791SAndreas Gruenbacher db = device->rs_total; 88958ffa580SLars Ellenberg /* adjust for verify start and stop sectors, respective reached position */ 890b30ab791SAndreas Gruenbacher if (device->state.conn == C_VERIFY_S || device->state.conn == C_VERIFY_T) 891b30ab791SAndreas Gruenbacher db -= device->ov_left; 89258ffa580SLars Ellenberg 893b411b363SPhilipp Reisner dbdt = Bit2KB(db/dt); 894b30ab791SAndreas Gruenbacher device->rs_paused /= HZ; 895b411b363SPhilipp Reisner 896b30ab791SAndreas Gruenbacher if (!get_ldev(device)) 897b411b363SPhilipp Reisner goto out; 898b411b363SPhilipp Reisner 899b30ab791SAndreas Gruenbacher ping_peer(device); 900af85e8e8SLars Ellenberg 9010500813fSAndreas Gruenbacher spin_lock_irq(&device->resource->req_lock); 902b30ab791SAndreas Gruenbacher os = drbd_read_state(device); 903b411b363SPhilipp Reisner 90426525618SLars Ellenberg verify_done = (os.conn == C_VERIFY_S || os.conn == C_VERIFY_T); 90526525618SLars Ellenberg 906b411b363SPhilipp Reisner /* This protects us against multiple calls (that can happen in the presence 907b411b363SPhilipp Reisner of application IO), and against connectivity loss just before we arrive here. */ 908b411b363SPhilipp Reisner if (os.conn <= C_CONNECTED) 909b411b363SPhilipp Reisner goto out_unlock; 910b411b363SPhilipp Reisner 911b411b363SPhilipp Reisner ns = os; 912b411b363SPhilipp Reisner ns.conn = C_CONNECTED; 913b411b363SPhilipp Reisner 914d0180171SAndreas Gruenbacher drbd_info(device, "%s done (total %lu sec; paused %lu sec; %lu K/sec)\n", 91526525618SLars Ellenberg verify_done ? "Online verify" : "Resync", 916b30ab791SAndreas Gruenbacher dt + device->rs_paused, device->rs_paused, dbdt); 917b411b363SPhilipp Reisner 918b30ab791SAndreas Gruenbacher n_oos = drbd_bm_total_weight(device); 919b411b363SPhilipp Reisner 920b411b363SPhilipp Reisner if (os.conn == C_VERIFY_S || os.conn == C_VERIFY_T) { 921b411b363SPhilipp Reisner if (n_oos) { 922d0180171SAndreas Gruenbacher drbd_alert(device, "Online verify found %lu %dk block out of sync!\n", 923b411b363SPhilipp Reisner n_oos, Bit2KB(1)); 924b411b363SPhilipp Reisner khelper_cmd = "out-of-sync"; 925b411b363SPhilipp Reisner } 926b411b363SPhilipp Reisner } else { 9270b0ba1efSAndreas Gruenbacher D_ASSERT(device, (n_oos - device->rs_failed) == 0); 928b411b363SPhilipp Reisner 929b411b363SPhilipp Reisner if (os.conn == C_SYNC_TARGET || os.conn == C_PAUSED_SYNC_T) 930b411b363SPhilipp Reisner khelper_cmd = "after-resync-target"; 931b411b363SPhilipp Reisner 932aaaba345SLars Ellenberg if (device->use_csums && device->rs_total) { 933b30ab791SAndreas Gruenbacher const unsigned long s = device->rs_same_csum; 934b30ab791SAndreas Gruenbacher const unsigned long t = device->rs_total; 935b411b363SPhilipp Reisner const int ratio = 936b411b363SPhilipp Reisner (t == 0) ? 0 : 937b411b363SPhilipp Reisner (t < 100000) ? ((s*100)/t) : (s/(t/100)); 938d0180171SAndreas Gruenbacher drbd_info(device, "%u %% had equal checksums, eliminated: %luK; " 939b411b363SPhilipp Reisner "transferred %luK total %luK\n", 940b411b363SPhilipp Reisner ratio, 941b30ab791SAndreas Gruenbacher Bit2KB(device->rs_same_csum), 942b30ab791SAndreas Gruenbacher Bit2KB(device->rs_total - device->rs_same_csum), 943b30ab791SAndreas Gruenbacher Bit2KB(device->rs_total)); 944b411b363SPhilipp Reisner } 945b411b363SPhilipp Reisner } 946b411b363SPhilipp Reisner 947b30ab791SAndreas Gruenbacher if (device->rs_failed) { 948d0180171SAndreas Gruenbacher drbd_info(device, " %lu failed blocks\n", device->rs_failed); 949b411b363SPhilipp Reisner 950b411b363SPhilipp Reisner if (os.conn == C_SYNC_TARGET || os.conn == C_PAUSED_SYNC_T) { 951b411b363SPhilipp Reisner ns.disk = D_INCONSISTENT; 952b411b363SPhilipp Reisner ns.pdsk = D_UP_TO_DATE; 953b411b363SPhilipp Reisner } else { 954b411b363SPhilipp Reisner ns.disk = D_UP_TO_DATE; 955b411b363SPhilipp Reisner ns.pdsk = D_INCONSISTENT; 956b411b363SPhilipp Reisner } 957b411b363SPhilipp Reisner } else { 958b411b363SPhilipp Reisner ns.disk = D_UP_TO_DATE; 959b411b363SPhilipp Reisner ns.pdsk = D_UP_TO_DATE; 960b411b363SPhilipp Reisner 961b411b363SPhilipp Reisner if (os.conn == C_SYNC_TARGET || os.conn == C_PAUSED_SYNC_T) { 962b30ab791SAndreas Gruenbacher if (device->p_uuid) { 963b411b363SPhilipp Reisner int i; 964b411b363SPhilipp Reisner for (i = UI_BITMAP ; i <= UI_HISTORY_END ; i++) 965b30ab791SAndreas Gruenbacher _drbd_uuid_set(device, i, device->p_uuid[i]); 966b30ab791SAndreas Gruenbacher drbd_uuid_set(device, UI_BITMAP, device->ldev->md.uuid[UI_CURRENT]); 967b30ab791SAndreas Gruenbacher _drbd_uuid_set(device, UI_CURRENT, device->p_uuid[UI_CURRENT]); 968b411b363SPhilipp Reisner } else { 969d0180171SAndreas Gruenbacher drbd_err(device, "device->p_uuid is NULL! BUG\n"); 970b411b363SPhilipp Reisner } 971b411b363SPhilipp Reisner } 972b411b363SPhilipp Reisner 97362b0da3aSLars Ellenberg if (!(os.conn == C_VERIFY_S || os.conn == C_VERIFY_T)) { 97462b0da3aSLars Ellenberg /* for verify runs, we don't update uuids here, 97562b0da3aSLars Ellenberg * so there would be nothing to report. */ 976b30ab791SAndreas Gruenbacher drbd_uuid_set_bm(device, 0UL); 977b30ab791SAndreas Gruenbacher drbd_print_uuids(device, "updated UUIDs"); 978b30ab791SAndreas Gruenbacher if (device->p_uuid) { 979b411b363SPhilipp Reisner /* Now the two UUID sets are equal, update what we 980b411b363SPhilipp Reisner * know of the peer. */ 981b411b363SPhilipp Reisner int i; 982b411b363SPhilipp Reisner for (i = UI_CURRENT ; i <= UI_HISTORY_END ; i++) 983b30ab791SAndreas Gruenbacher device->p_uuid[i] = device->ldev->md.uuid[i]; 984b411b363SPhilipp Reisner } 985b411b363SPhilipp Reisner } 98662b0da3aSLars Ellenberg } 987b411b363SPhilipp Reisner 988b30ab791SAndreas Gruenbacher _drbd_set_state(device, ns, CS_VERBOSE, NULL); 989b411b363SPhilipp Reisner out_unlock: 9900500813fSAndreas Gruenbacher spin_unlock_irq(&device->resource->req_lock); 99126a96110SLars Ellenberg 99226a96110SLars Ellenberg /* If we have been sync source, and have an effective fencing-policy, 99326a96110SLars Ellenberg * once *all* volumes are back in sync, call "unfence". */ 99426a96110SLars Ellenberg if (os.conn == C_SYNC_SOURCE) { 99526a96110SLars Ellenberg enum drbd_disk_state disk_state = D_MASK; 99626a96110SLars Ellenberg enum drbd_disk_state pdsk_state = D_MASK; 99726a96110SLars Ellenberg enum drbd_fencing_p fp = FP_DONT_CARE; 99826a96110SLars Ellenberg 99926a96110SLars Ellenberg rcu_read_lock(); 100026a96110SLars Ellenberg fp = rcu_dereference(device->ldev->disk_conf)->fencing; 100126a96110SLars Ellenberg if (fp != FP_DONT_CARE) { 100226a96110SLars Ellenberg struct drbd_peer_device *peer_device; 100326a96110SLars Ellenberg int vnr; 100426a96110SLars Ellenberg idr_for_each_entry(&connection->peer_devices, peer_device, vnr) { 100526a96110SLars Ellenberg struct drbd_device *device = peer_device->device; 100626a96110SLars Ellenberg disk_state = min_t(enum drbd_disk_state, disk_state, device->state.disk); 100726a96110SLars Ellenberg pdsk_state = min_t(enum drbd_disk_state, pdsk_state, device->state.pdsk); 100826a96110SLars Ellenberg } 100926a96110SLars Ellenberg } 101026a96110SLars Ellenberg rcu_read_unlock(); 101126a96110SLars Ellenberg if (disk_state == D_UP_TO_DATE && pdsk_state == D_UP_TO_DATE) 101226a96110SLars Ellenberg conn_khelper(connection, "unfence-peer"); 101326a96110SLars Ellenberg } 101426a96110SLars Ellenberg 1015b30ab791SAndreas Gruenbacher put_ldev(device); 1016b411b363SPhilipp Reisner out: 1017b30ab791SAndreas Gruenbacher device->rs_total = 0; 1018b30ab791SAndreas Gruenbacher device->rs_failed = 0; 1019b30ab791SAndreas Gruenbacher device->rs_paused = 0; 102058ffa580SLars Ellenberg 102158ffa580SLars Ellenberg /* reset start sector, if we reached end of device */ 1022b30ab791SAndreas Gruenbacher if (verify_done && device->ov_left == 0) 1023b30ab791SAndreas Gruenbacher device->ov_start_sector = 0; 1024b411b363SPhilipp Reisner 1025b30ab791SAndreas Gruenbacher drbd_md_sync(device); 102613d42685SLars Ellenberg 1027b411b363SPhilipp Reisner if (khelper_cmd) 1028b30ab791SAndreas Gruenbacher drbd_khelper(device, khelper_cmd); 1029b411b363SPhilipp Reisner 1030b411b363SPhilipp Reisner return 1; 1031b411b363SPhilipp Reisner } 1032b411b363SPhilipp Reisner 1033b411b363SPhilipp Reisner /* helper */ 1034b30ab791SAndreas Gruenbacher static void move_to_net_ee_or_free(struct drbd_device *device, struct drbd_peer_request *peer_req) 1035b411b363SPhilipp Reisner { 1036045417f7SAndreas Gruenbacher if (drbd_peer_req_has_active_page(peer_req)) { 1037b411b363SPhilipp Reisner /* This might happen if sendpage() has not finished */ 1038db830c46SAndreas Gruenbacher int i = (peer_req->i.size + PAGE_SIZE -1) >> PAGE_SHIFT; 1039b30ab791SAndreas Gruenbacher atomic_add(i, &device->pp_in_use_by_net); 1040b30ab791SAndreas Gruenbacher atomic_sub(i, &device->pp_in_use); 10410500813fSAndreas Gruenbacher spin_lock_irq(&device->resource->req_lock); 1042a8cd15baSAndreas Gruenbacher list_add_tail(&peer_req->w.list, &device->net_ee); 10430500813fSAndreas Gruenbacher spin_unlock_irq(&device->resource->req_lock); 1044435f0740SLars Ellenberg wake_up(&drbd_pp_wait); 1045b411b363SPhilipp Reisner } else 1046b30ab791SAndreas Gruenbacher drbd_free_peer_req(device, peer_req); 1047b411b363SPhilipp Reisner } 1048b411b363SPhilipp Reisner 1049b411b363SPhilipp Reisner /** 1050b411b363SPhilipp Reisner * w_e_end_data_req() - Worker callback, to send a P_DATA_REPLY packet in response to a P_DATA_REQUEST 1051b411b363SPhilipp Reisner * @w: work object. 1052b411b363SPhilipp Reisner * @cancel: The connection will be closed anyways 1053b411b363SPhilipp Reisner */ 105499920dc5SAndreas Gruenbacher int w_e_end_data_req(struct drbd_work *w, int cancel) 1055b411b363SPhilipp Reisner { 1056a8cd15baSAndreas Gruenbacher struct drbd_peer_request *peer_req = container_of(w, struct drbd_peer_request, w); 10576780139cSAndreas Gruenbacher struct drbd_peer_device *peer_device = peer_req->peer_device; 10586780139cSAndreas Gruenbacher struct drbd_device *device = peer_device->device; 105999920dc5SAndreas Gruenbacher int err; 1060b411b363SPhilipp Reisner 1061b411b363SPhilipp Reisner if (unlikely(cancel)) { 1062b30ab791SAndreas Gruenbacher drbd_free_peer_req(device, peer_req); 1063b30ab791SAndreas Gruenbacher dec_unacked(device); 106499920dc5SAndreas Gruenbacher return 0; 1065b411b363SPhilipp Reisner } 1066b411b363SPhilipp Reisner 1067db830c46SAndreas Gruenbacher if (likely((peer_req->flags & EE_WAS_ERROR) == 0)) { 10686780139cSAndreas Gruenbacher err = drbd_send_block(peer_device, P_DATA_REPLY, peer_req); 1069b411b363SPhilipp Reisner } else { 1070b411b363SPhilipp Reisner if (__ratelimit(&drbd_ratelimit_state)) 1071d0180171SAndreas Gruenbacher drbd_err(device, "Sending NegDReply. sector=%llus.\n", 1072db830c46SAndreas Gruenbacher (unsigned long long)peer_req->i.sector); 1073b411b363SPhilipp Reisner 10746780139cSAndreas Gruenbacher err = drbd_send_ack(peer_device, P_NEG_DREPLY, peer_req); 1075b411b363SPhilipp Reisner } 1076b411b363SPhilipp Reisner 1077b30ab791SAndreas Gruenbacher dec_unacked(device); 1078b411b363SPhilipp Reisner 1079b30ab791SAndreas Gruenbacher move_to_net_ee_or_free(device, peer_req); 1080b411b363SPhilipp Reisner 108199920dc5SAndreas Gruenbacher if (unlikely(err)) 1082d0180171SAndreas Gruenbacher drbd_err(device, "drbd_send_block() failed\n"); 108399920dc5SAndreas Gruenbacher return err; 1084b411b363SPhilipp Reisner } 1085b411b363SPhilipp Reisner 1086700ca8c0SPhilipp Reisner static bool all_zero(struct drbd_peer_request *peer_req) 1087700ca8c0SPhilipp Reisner { 1088700ca8c0SPhilipp Reisner struct page *page = peer_req->pages; 1089700ca8c0SPhilipp Reisner unsigned int len = peer_req->i.size; 1090700ca8c0SPhilipp Reisner 1091700ca8c0SPhilipp Reisner page_chain_for_each(page) { 1092700ca8c0SPhilipp Reisner unsigned int l = min_t(unsigned int, len, PAGE_SIZE); 1093700ca8c0SPhilipp Reisner unsigned int i, words = l / sizeof(long); 1094700ca8c0SPhilipp Reisner unsigned long *d; 1095700ca8c0SPhilipp Reisner 1096700ca8c0SPhilipp Reisner d = kmap_atomic(page); 1097700ca8c0SPhilipp Reisner for (i = 0; i < words; i++) { 1098700ca8c0SPhilipp Reisner if (d[i]) { 1099700ca8c0SPhilipp Reisner kunmap_atomic(d); 1100700ca8c0SPhilipp Reisner return false; 1101700ca8c0SPhilipp Reisner } 1102700ca8c0SPhilipp Reisner } 1103700ca8c0SPhilipp Reisner kunmap_atomic(d); 1104700ca8c0SPhilipp Reisner len -= l; 1105700ca8c0SPhilipp Reisner } 1106700ca8c0SPhilipp Reisner 1107700ca8c0SPhilipp Reisner return true; 1108700ca8c0SPhilipp Reisner } 1109700ca8c0SPhilipp Reisner 1110b411b363SPhilipp Reisner /** 1111a209b4aeSAndreas Gruenbacher * w_e_end_rsdata_req() - Worker callback to send a P_RS_DATA_REPLY packet in response to a P_RS_DATA_REQUEST 1112b411b363SPhilipp Reisner * @w: work object. 1113b411b363SPhilipp Reisner * @cancel: The connection will be closed anyways 1114b411b363SPhilipp Reisner */ 111599920dc5SAndreas Gruenbacher int w_e_end_rsdata_req(struct drbd_work *w, int cancel) 1116b411b363SPhilipp Reisner { 1117a8cd15baSAndreas Gruenbacher struct drbd_peer_request *peer_req = container_of(w, struct drbd_peer_request, w); 11186780139cSAndreas Gruenbacher struct drbd_peer_device *peer_device = peer_req->peer_device; 11196780139cSAndreas Gruenbacher struct drbd_device *device = peer_device->device; 112099920dc5SAndreas Gruenbacher int err; 1121b411b363SPhilipp Reisner 1122b411b363SPhilipp Reisner if (unlikely(cancel)) { 1123b30ab791SAndreas Gruenbacher drbd_free_peer_req(device, peer_req); 1124b30ab791SAndreas Gruenbacher dec_unacked(device); 112599920dc5SAndreas Gruenbacher return 0; 1126b411b363SPhilipp Reisner } 1127b411b363SPhilipp Reisner 1128b30ab791SAndreas Gruenbacher if (get_ldev_if_state(device, D_FAILED)) { 1129b30ab791SAndreas Gruenbacher drbd_rs_complete_io(device, peer_req->i.sector); 1130b30ab791SAndreas Gruenbacher put_ldev(device); 1131b411b363SPhilipp Reisner } 1132b411b363SPhilipp Reisner 1133b30ab791SAndreas Gruenbacher if (device->state.conn == C_AHEAD) { 11346780139cSAndreas Gruenbacher err = drbd_send_ack(peer_device, P_RS_CANCEL, peer_req); 1135db830c46SAndreas Gruenbacher } else if (likely((peer_req->flags & EE_WAS_ERROR) == 0)) { 1136b30ab791SAndreas Gruenbacher if (likely(device->state.pdsk >= D_INCONSISTENT)) { 1137b30ab791SAndreas Gruenbacher inc_rs_pending(device); 1138700ca8c0SPhilipp Reisner if (peer_req->flags & EE_RS_THIN_REQ && all_zero(peer_req)) 1139700ca8c0SPhilipp Reisner err = drbd_send_rs_deallocated(peer_device, peer_req); 1140700ca8c0SPhilipp Reisner else 11416780139cSAndreas Gruenbacher err = drbd_send_block(peer_device, P_RS_DATA_REPLY, peer_req); 1142b411b363SPhilipp Reisner } else { 1143b411b363SPhilipp Reisner if (__ratelimit(&drbd_ratelimit_state)) 1144d0180171SAndreas Gruenbacher drbd_err(device, "Not sending RSDataReply, " 1145b411b363SPhilipp Reisner "partner DISKLESS!\n"); 114699920dc5SAndreas Gruenbacher err = 0; 1147b411b363SPhilipp Reisner } 1148b411b363SPhilipp Reisner } else { 1149b411b363SPhilipp Reisner if (__ratelimit(&drbd_ratelimit_state)) 1150d0180171SAndreas Gruenbacher drbd_err(device, "Sending NegRSDReply. sector %llus.\n", 1151db830c46SAndreas Gruenbacher (unsigned long long)peer_req->i.sector); 1152b411b363SPhilipp Reisner 11536780139cSAndreas Gruenbacher err = drbd_send_ack(peer_device, P_NEG_RS_DREPLY, peer_req); 1154b411b363SPhilipp Reisner 1155b411b363SPhilipp Reisner /* update resync data with failure */ 1156b30ab791SAndreas Gruenbacher drbd_rs_failed_io(device, peer_req->i.sector, peer_req->i.size); 1157b411b363SPhilipp Reisner } 1158b411b363SPhilipp Reisner 1159b30ab791SAndreas Gruenbacher dec_unacked(device); 1160b411b363SPhilipp Reisner 1161b30ab791SAndreas Gruenbacher move_to_net_ee_or_free(device, peer_req); 1162b411b363SPhilipp Reisner 116399920dc5SAndreas Gruenbacher if (unlikely(err)) 1164d0180171SAndreas Gruenbacher drbd_err(device, "drbd_send_block() failed\n"); 116599920dc5SAndreas Gruenbacher return err; 1166b411b363SPhilipp Reisner } 1167b411b363SPhilipp Reisner 116899920dc5SAndreas Gruenbacher int w_e_end_csum_rs_req(struct drbd_work *w, int cancel) 1169b411b363SPhilipp Reisner { 1170a8cd15baSAndreas Gruenbacher struct drbd_peer_request *peer_req = container_of(w, struct drbd_peer_request, w); 11716780139cSAndreas Gruenbacher struct drbd_peer_device *peer_device = peer_req->peer_device; 11726780139cSAndreas Gruenbacher struct drbd_device *device = peer_device->device; 1173b411b363SPhilipp Reisner struct digest_info *di; 1174b411b363SPhilipp Reisner int digest_size; 1175b411b363SPhilipp Reisner void *digest = NULL; 117699920dc5SAndreas Gruenbacher int err, eq = 0; 1177b411b363SPhilipp Reisner 1178b411b363SPhilipp Reisner if (unlikely(cancel)) { 1179b30ab791SAndreas Gruenbacher drbd_free_peer_req(device, peer_req); 1180b30ab791SAndreas Gruenbacher dec_unacked(device); 118199920dc5SAndreas Gruenbacher return 0; 1182b411b363SPhilipp Reisner } 1183b411b363SPhilipp Reisner 1184b30ab791SAndreas Gruenbacher if (get_ldev(device)) { 1185b30ab791SAndreas Gruenbacher drbd_rs_complete_io(device, peer_req->i.sector); 1186b30ab791SAndreas Gruenbacher put_ldev(device); 11871d53f09eSLars Ellenberg } 1188b411b363SPhilipp Reisner 1189db830c46SAndreas Gruenbacher di = peer_req->digest; 1190b411b363SPhilipp Reisner 1191db830c46SAndreas Gruenbacher if (likely((peer_req->flags & EE_WAS_ERROR) == 0)) { 1192b411b363SPhilipp Reisner /* quick hack to try to avoid a race against reconfiguration. 1193b411b363SPhilipp Reisner * a real fix would be much more involved, 1194b411b363SPhilipp Reisner * introducing more locking mechanisms */ 11956780139cSAndreas Gruenbacher if (peer_device->connection->csums_tfm) { 11963d0e6375SKees Cook digest_size = crypto_shash_digestsize(peer_device->connection->csums_tfm); 11970b0ba1efSAndreas Gruenbacher D_ASSERT(device, digest_size == di->digest_size); 1198b411b363SPhilipp Reisner digest = kmalloc(digest_size, GFP_NOIO); 1199b411b363SPhilipp Reisner } 1200b411b363SPhilipp Reisner if (digest) { 12016780139cSAndreas Gruenbacher drbd_csum_ee(peer_device->connection->csums_tfm, peer_req, digest); 1202b411b363SPhilipp Reisner eq = !memcmp(digest, di->digest, digest_size); 1203b411b363SPhilipp Reisner kfree(digest); 1204b411b363SPhilipp Reisner } 1205b411b363SPhilipp Reisner 1206b411b363SPhilipp Reisner if (eq) { 1207b30ab791SAndreas Gruenbacher drbd_set_in_sync(device, peer_req->i.sector, peer_req->i.size); 1208676396d5SLars Ellenberg /* rs_same_csums unit is BM_BLOCK_SIZE */ 1209b30ab791SAndreas Gruenbacher device->rs_same_csum += peer_req->i.size >> BM_BLOCK_SHIFT; 12106780139cSAndreas Gruenbacher err = drbd_send_ack(peer_device, P_RS_IS_IN_SYNC, peer_req); 1211b411b363SPhilipp Reisner } else { 1212b30ab791SAndreas Gruenbacher inc_rs_pending(device); 1213db830c46SAndreas Gruenbacher peer_req->block_id = ID_SYNCER; /* By setting block_id, digest pointer becomes invalid! */ 1214db830c46SAndreas Gruenbacher peer_req->flags &= ~EE_HAS_DIGEST; /* This peer request no longer has a digest pointer */ 1215204bba99SPhilipp Reisner kfree(di); 12166780139cSAndreas Gruenbacher err = drbd_send_block(peer_device, P_RS_DATA_REPLY, peer_req); 1217b411b363SPhilipp Reisner } 1218b411b363SPhilipp Reisner } else { 12196780139cSAndreas Gruenbacher err = drbd_send_ack(peer_device, P_NEG_RS_DREPLY, peer_req); 1220b411b363SPhilipp Reisner if (__ratelimit(&drbd_ratelimit_state)) 1221d0180171SAndreas Gruenbacher drbd_err(device, "Sending NegDReply. I guess it gets messy.\n"); 1222b411b363SPhilipp Reisner } 1223b411b363SPhilipp Reisner 1224b30ab791SAndreas Gruenbacher dec_unacked(device); 1225b30ab791SAndreas Gruenbacher move_to_net_ee_or_free(device, peer_req); 1226b411b363SPhilipp Reisner 122799920dc5SAndreas Gruenbacher if (unlikely(err)) 1228d0180171SAndreas Gruenbacher drbd_err(device, "drbd_send_block/ack() failed\n"); 122999920dc5SAndreas Gruenbacher return err; 1230b411b363SPhilipp Reisner } 1231b411b363SPhilipp Reisner 123299920dc5SAndreas Gruenbacher int w_e_end_ov_req(struct drbd_work *w, int cancel) 1233b411b363SPhilipp Reisner { 1234a8cd15baSAndreas Gruenbacher struct drbd_peer_request *peer_req = container_of(w, struct drbd_peer_request, w); 12356780139cSAndreas Gruenbacher struct drbd_peer_device *peer_device = peer_req->peer_device; 12366780139cSAndreas Gruenbacher struct drbd_device *device = peer_device->device; 1237db830c46SAndreas Gruenbacher sector_t sector = peer_req->i.sector; 1238db830c46SAndreas Gruenbacher unsigned int size = peer_req->i.size; 1239b411b363SPhilipp Reisner int digest_size; 1240b411b363SPhilipp Reisner void *digest; 124199920dc5SAndreas Gruenbacher int err = 0; 1242b411b363SPhilipp Reisner 1243b411b363SPhilipp Reisner if (unlikely(cancel)) 1244b411b363SPhilipp Reisner goto out; 1245b411b363SPhilipp Reisner 12463d0e6375SKees Cook digest_size = crypto_shash_digestsize(peer_device->connection->verify_tfm); 1247b411b363SPhilipp Reisner digest = kmalloc(digest_size, GFP_NOIO); 12488f21420eSPhilipp Reisner if (!digest) { 124999920dc5SAndreas Gruenbacher err = 1; /* terminate the connection in case the allocation failed */ 12508f21420eSPhilipp Reisner goto out; 12518f21420eSPhilipp Reisner } 12528f21420eSPhilipp Reisner 1253db830c46SAndreas Gruenbacher if (likely(!(peer_req->flags & EE_WAS_ERROR))) 12546780139cSAndreas Gruenbacher drbd_csum_ee(peer_device->connection->verify_tfm, peer_req, digest); 12558f21420eSPhilipp Reisner else 12568f21420eSPhilipp Reisner memset(digest, 0, digest_size); 12578f21420eSPhilipp Reisner 125853ea4331SLars Ellenberg /* Free e and pages before send. 125953ea4331SLars Ellenberg * In case we block on congestion, we could otherwise run into 126053ea4331SLars Ellenberg * some distributed deadlock, if the other side blocks on 126153ea4331SLars Ellenberg * congestion as well, because our receiver blocks in 1262c37c8ecfSAndreas Gruenbacher * drbd_alloc_pages due to pp_in_use > max_buffers. */ 1263b30ab791SAndreas Gruenbacher drbd_free_peer_req(device, peer_req); 1264db830c46SAndreas Gruenbacher peer_req = NULL; 1265b30ab791SAndreas Gruenbacher inc_rs_pending(device); 12666780139cSAndreas Gruenbacher err = drbd_send_drequest_csum(peer_device, sector, size, digest, digest_size, P_OV_REPLY); 126799920dc5SAndreas Gruenbacher if (err) 1268b30ab791SAndreas Gruenbacher dec_rs_pending(device); 1269b411b363SPhilipp Reisner kfree(digest); 1270b411b363SPhilipp Reisner 1271b411b363SPhilipp Reisner out: 1272db830c46SAndreas Gruenbacher if (peer_req) 1273b30ab791SAndreas Gruenbacher drbd_free_peer_req(device, peer_req); 1274b30ab791SAndreas Gruenbacher dec_unacked(device); 127599920dc5SAndreas Gruenbacher return err; 1276b411b363SPhilipp Reisner } 1277b411b363SPhilipp Reisner 1278b30ab791SAndreas Gruenbacher void drbd_ov_out_of_sync_found(struct drbd_device *device, sector_t sector, int size) 1279b411b363SPhilipp Reisner { 1280b30ab791SAndreas Gruenbacher if (device->ov_last_oos_start + device->ov_last_oos_size == sector) { 1281b30ab791SAndreas Gruenbacher device->ov_last_oos_size += size>>9; 1282b411b363SPhilipp Reisner } else { 1283b30ab791SAndreas Gruenbacher device->ov_last_oos_start = sector; 1284b30ab791SAndreas Gruenbacher device->ov_last_oos_size = size>>9; 1285b411b363SPhilipp Reisner } 1286b30ab791SAndreas Gruenbacher drbd_set_out_of_sync(device, sector, size); 1287b411b363SPhilipp Reisner } 1288b411b363SPhilipp Reisner 128999920dc5SAndreas Gruenbacher int w_e_end_ov_reply(struct drbd_work *w, int cancel) 1290b411b363SPhilipp Reisner { 1291a8cd15baSAndreas Gruenbacher struct drbd_peer_request *peer_req = container_of(w, struct drbd_peer_request, w); 12926780139cSAndreas Gruenbacher struct drbd_peer_device *peer_device = peer_req->peer_device; 12936780139cSAndreas Gruenbacher struct drbd_device *device = peer_device->device; 1294b411b363SPhilipp Reisner struct digest_info *di; 1295b411b363SPhilipp Reisner void *digest; 1296db830c46SAndreas Gruenbacher sector_t sector = peer_req->i.sector; 1297db830c46SAndreas Gruenbacher unsigned int size = peer_req->i.size; 129853ea4331SLars Ellenberg int digest_size; 129999920dc5SAndreas Gruenbacher int err, eq = 0; 130058ffa580SLars Ellenberg bool stop_sector_reached = false; 1301b411b363SPhilipp Reisner 1302b411b363SPhilipp Reisner if (unlikely(cancel)) { 1303b30ab791SAndreas Gruenbacher drbd_free_peer_req(device, peer_req); 1304b30ab791SAndreas Gruenbacher dec_unacked(device); 130599920dc5SAndreas Gruenbacher return 0; 1306b411b363SPhilipp Reisner } 1307b411b363SPhilipp Reisner 1308b411b363SPhilipp Reisner /* after "cancel", because after drbd_disconnect/drbd_rs_cancel_all 1309b411b363SPhilipp Reisner * the resync lru has been cleaned up already */ 1310b30ab791SAndreas Gruenbacher if (get_ldev(device)) { 1311b30ab791SAndreas Gruenbacher drbd_rs_complete_io(device, peer_req->i.sector); 1312b30ab791SAndreas Gruenbacher put_ldev(device); 13131d53f09eSLars Ellenberg } 1314b411b363SPhilipp Reisner 1315db830c46SAndreas Gruenbacher di = peer_req->digest; 1316b411b363SPhilipp Reisner 1317db830c46SAndreas Gruenbacher if (likely((peer_req->flags & EE_WAS_ERROR) == 0)) { 13183d0e6375SKees Cook digest_size = crypto_shash_digestsize(peer_device->connection->verify_tfm); 1319b411b363SPhilipp Reisner digest = kmalloc(digest_size, GFP_NOIO); 1320b411b363SPhilipp Reisner if (digest) { 13216780139cSAndreas Gruenbacher drbd_csum_ee(peer_device->connection->verify_tfm, peer_req, digest); 1322b411b363SPhilipp Reisner 13230b0ba1efSAndreas Gruenbacher D_ASSERT(device, digest_size == di->digest_size); 1324b411b363SPhilipp Reisner eq = !memcmp(digest, di->digest, digest_size); 1325b411b363SPhilipp Reisner kfree(digest); 1326b411b363SPhilipp Reisner } 1327b411b363SPhilipp Reisner } 1328b411b363SPhilipp Reisner 13299676c760SLars Ellenberg /* Free peer_req and pages before send. 133053ea4331SLars Ellenberg * In case we block on congestion, we could otherwise run into 133153ea4331SLars Ellenberg * some distributed deadlock, if the other side blocks on 133253ea4331SLars Ellenberg * congestion as well, because our receiver blocks in 1333c37c8ecfSAndreas Gruenbacher * drbd_alloc_pages due to pp_in_use > max_buffers. */ 1334b30ab791SAndreas Gruenbacher drbd_free_peer_req(device, peer_req); 1335b411b363SPhilipp Reisner if (!eq) 1336b30ab791SAndreas Gruenbacher drbd_ov_out_of_sync_found(device, sector, size); 1337b411b363SPhilipp Reisner else 1338b30ab791SAndreas Gruenbacher ov_out_of_sync_print(device); 1339b411b363SPhilipp Reisner 13406780139cSAndreas Gruenbacher err = drbd_send_ack_ex(peer_device, P_OV_RESULT, sector, size, 1341b411b363SPhilipp Reisner eq ? ID_IN_SYNC : ID_OUT_OF_SYNC); 1342b411b363SPhilipp Reisner 1343b30ab791SAndreas Gruenbacher dec_unacked(device); 1344b411b363SPhilipp Reisner 1345b30ab791SAndreas Gruenbacher --device->ov_left; 1346ea5442afSLars Ellenberg 1347ea5442afSLars Ellenberg /* let's advance progress step marks only for every other megabyte */ 1348b30ab791SAndreas Gruenbacher if ((device->ov_left & 0x200) == 0x200) 1349b30ab791SAndreas Gruenbacher drbd_advance_rs_marks(device, device->ov_left); 1350ea5442afSLars Ellenberg 1351b30ab791SAndreas Gruenbacher stop_sector_reached = verify_can_do_stop_sector(device) && 1352b30ab791SAndreas Gruenbacher (sector + (size>>9)) >= device->ov_stop_sector; 135358ffa580SLars Ellenberg 1354b30ab791SAndreas Gruenbacher if (device->ov_left == 0 || stop_sector_reached) { 1355b30ab791SAndreas Gruenbacher ov_out_of_sync_print(device); 1356b30ab791SAndreas Gruenbacher drbd_resync_finished(device); 1357b411b363SPhilipp Reisner } 1358b411b363SPhilipp Reisner 135999920dc5SAndreas Gruenbacher return err; 1360b411b363SPhilipp Reisner } 1361b411b363SPhilipp Reisner 1362b6dd1a89SLars Ellenberg /* FIXME 1363b6dd1a89SLars Ellenberg * We need to track the number of pending barrier acks, 1364b6dd1a89SLars Ellenberg * and to be able to wait for them. 1365b6dd1a89SLars Ellenberg * See also comment in drbd_adm_attach before drbd_suspend_io. 1366b6dd1a89SLars Ellenberg */ 1367bde89a9eSAndreas Gruenbacher static int drbd_send_barrier(struct drbd_connection *connection) 1368b411b363SPhilipp Reisner { 13699f5bdc33SAndreas Gruenbacher struct p_barrier *p; 1370b6dd1a89SLars Ellenberg struct drbd_socket *sock; 1371b411b363SPhilipp Reisner 1372bde89a9eSAndreas Gruenbacher sock = &connection->data; 1373bde89a9eSAndreas Gruenbacher p = conn_prepare_command(connection, sock); 13749f5bdc33SAndreas Gruenbacher if (!p) 13759f5bdc33SAndreas Gruenbacher return -EIO; 1376bde89a9eSAndreas Gruenbacher p->barrier = connection->send.current_epoch_nr; 1377b6dd1a89SLars Ellenberg p->pad = 0; 1378bde89a9eSAndreas Gruenbacher connection->send.current_epoch_writes = 0; 137984d34f2fSLars Ellenberg connection->send.last_sent_barrier_jif = jiffies; 1380b6dd1a89SLars Ellenberg 1381bde89a9eSAndreas Gruenbacher return conn_send_command(connection, sock, P_BARRIER, sizeof(*p), NULL, 0); 1382b411b363SPhilipp Reisner } 1383b411b363SPhilipp Reisner 1384c51a0ef3SLars Ellenberg static int pd_send_unplug_remote(struct drbd_peer_device *pd) 1385c51a0ef3SLars Ellenberg { 1386c51a0ef3SLars Ellenberg struct drbd_socket *sock = &pd->connection->data; 1387c51a0ef3SLars Ellenberg if (!drbd_prepare_command(pd, sock)) 1388c51a0ef3SLars Ellenberg return -EIO; 1389c51a0ef3SLars Ellenberg return drbd_send_command(pd, sock, P_UNPLUG_REMOTE, 0, NULL, 0); 1390c51a0ef3SLars Ellenberg } 1391c51a0ef3SLars Ellenberg 139299920dc5SAndreas Gruenbacher int w_send_write_hint(struct drbd_work *w, int cancel) 1393b411b363SPhilipp Reisner { 139484b8c06bSAndreas Gruenbacher struct drbd_device *device = 139584b8c06bSAndreas Gruenbacher container_of(w, struct drbd_device, unplug_work); 13969f5bdc33SAndreas Gruenbacher 1397b411b363SPhilipp Reisner if (cancel) 139899920dc5SAndreas Gruenbacher return 0; 1399c51a0ef3SLars Ellenberg return pd_send_unplug_remote(first_peer_device(device)); 1400b411b363SPhilipp Reisner } 1401b411b363SPhilipp Reisner 1402bde89a9eSAndreas Gruenbacher static void re_init_if_first_write(struct drbd_connection *connection, unsigned int epoch) 14034eb9b3cbSLars Ellenberg { 1404bde89a9eSAndreas Gruenbacher if (!connection->send.seen_any_write_yet) { 1405bde89a9eSAndreas Gruenbacher connection->send.seen_any_write_yet = true; 1406bde89a9eSAndreas Gruenbacher connection->send.current_epoch_nr = epoch; 1407bde89a9eSAndreas Gruenbacher connection->send.current_epoch_writes = 0; 140884d34f2fSLars Ellenberg connection->send.last_sent_barrier_jif = jiffies; 14094eb9b3cbSLars Ellenberg } 14104eb9b3cbSLars Ellenberg } 14114eb9b3cbSLars Ellenberg 1412bde89a9eSAndreas Gruenbacher static void maybe_send_barrier(struct drbd_connection *connection, unsigned int epoch) 14134eb9b3cbSLars Ellenberg { 14144eb9b3cbSLars Ellenberg /* re-init if first write on this connection */ 1415bde89a9eSAndreas Gruenbacher if (!connection->send.seen_any_write_yet) 14164eb9b3cbSLars Ellenberg return; 1417bde89a9eSAndreas Gruenbacher if (connection->send.current_epoch_nr != epoch) { 1418bde89a9eSAndreas Gruenbacher if (connection->send.current_epoch_writes) 1419bde89a9eSAndreas Gruenbacher drbd_send_barrier(connection); 1420bde89a9eSAndreas Gruenbacher connection->send.current_epoch_nr = epoch; 14214eb9b3cbSLars Ellenberg } 14224eb9b3cbSLars Ellenberg } 14234eb9b3cbSLars Ellenberg 14248f7bed77SAndreas Gruenbacher int w_send_out_of_sync(struct drbd_work *w, int cancel) 142573a01a18SPhilipp Reisner { 142673a01a18SPhilipp Reisner struct drbd_request *req = container_of(w, struct drbd_request, w); 142784b8c06bSAndreas Gruenbacher struct drbd_device *device = req->device; 142844a4d551SLars Ellenberg struct drbd_peer_device *const peer_device = first_peer_device(device); 142944a4d551SLars Ellenberg struct drbd_connection *const connection = peer_device->connection; 143099920dc5SAndreas Gruenbacher int err; 143173a01a18SPhilipp Reisner 143273a01a18SPhilipp Reisner if (unlikely(cancel)) { 14338554df1cSAndreas Gruenbacher req_mod(req, SEND_CANCELED); 143499920dc5SAndreas Gruenbacher return 0; 143573a01a18SPhilipp Reisner } 1436e5f891b2SLars Ellenberg req->pre_send_jif = jiffies; 143773a01a18SPhilipp Reisner 1438bde89a9eSAndreas Gruenbacher /* this time, no connection->send.current_epoch_writes++; 1439b6dd1a89SLars Ellenberg * If it was sent, it was the closing barrier for the last 1440b6dd1a89SLars Ellenberg * replicated epoch, before we went into AHEAD mode. 1441b6dd1a89SLars Ellenberg * No more barriers will be sent, until we leave AHEAD mode again. */ 1442bde89a9eSAndreas Gruenbacher maybe_send_barrier(connection, req->epoch); 1443b6dd1a89SLars Ellenberg 144444a4d551SLars Ellenberg err = drbd_send_out_of_sync(peer_device, req); 14458554df1cSAndreas Gruenbacher req_mod(req, OOS_HANDED_TO_NETWORK); 144673a01a18SPhilipp Reisner 144799920dc5SAndreas Gruenbacher return err; 144873a01a18SPhilipp Reisner } 144973a01a18SPhilipp Reisner 1450b411b363SPhilipp Reisner /** 1451b411b363SPhilipp Reisner * w_send_dblock() - Worker callback to send a P_DATA packet in order to mirror a write request 1452b411b363SPhilipp Reisner * @w: work object. 1453b411b363SPhilipp Reisner * @cancel: The connection will be closed anyways 1454b411b363SPhilipp Reisner */ 145599920dc5SAndreas Gruenbacher int w_send_dblock(struct drbd_work *w, int cancel) 1456b411b363SPhilipp Reisner { 1457b411b363SPhilipp Reisner struct drbd_request *req = container_of(w, struct drbd_request, w); 145884b8c06bSAndreas Gruenbacher struct drbd_device *device = req->device; 145944a4d551SLars Ellenberg struct drbd_peer_device *const peer_device = first_peer_device(device); 146044a4d551SLars Ellenberg struct drbd_connection *connection = peer_device->connection; 1461c51a0ef3SLars Ellenberg bool do_send_unplug = req->rq_state & RQ_UNPLUG; 146299920dc5SAndreas Gruenbacher int err; 1463b411b363SPhilipp Reisner 1464b411b363SPhilipp Reisner if (unlikely(cancel)) { 14658554df1cSAndreas Gruenbacher req_mod(req, SEND_CANCELED); 146699920dc5SAndreas Gruenbacher return 0; 1467b411b363SPhilipp Reisner } 1468e5f891b2SLars Ellenberg req->pre_send_jif = jiffies; 1469b411b363SPhilipp Reisner 1470bde89a9eSAndreas Gruenbacher re_init_if_first_write(connection, req->epoch); 1471bde89a9eSAndreas Gruenbacher maybe_send_barrier(connection, req->epoch); 1472bde89a9eSAndreas Gruenbacher connection->send.current_epoch_writes++; 1473b6dd1a89SLars Ellenberg 147444a4d551SLars Ellenberg err = drbd_send_dblock(peer_device, req); 147599920dc5SAndreas Gruenbacher req_mod(req, err ? SEND_FAILED : HANDED_OVER_TO_NETWORK); 1476b411b363SPhilipp Reisner 1477c51a0ef3SLars Ellenberg if (do_send_unplug && !err) 1478c51a0ef3SLars Ellenberg pd_send_unplug_remote(peer_device); 1479c51a0ef3SLars Ellenberg 148099920dc5SAndreas Gruenbacher return err; 1481b411b363SPhilipp Reisner } 1482b411b363SPhilipp Reisner 1483b411b363SPhilipp Reisner /** 1484b411b363SPhilipp Reisner * w_send_read_req() - Worker callback to send a read request (P_DATA_REQUEST) packet 1485b411b363SPhilipp Reisner * @w: work object. 1486b411b363SPhilipp Reisner * @cancel: The connection will be closed anyways 1487b411b363SPhilipp Reisner */ 148899920dc5SAndreas Gruenbacher int w_send_read_req(struct drbd_work *w, int cancel) 1489b411b363SPhilipp Reisner { 1490b411b363SPhilipp Reisner struct drbd_request *req = container_of(w, struct drbd_request, w); 149184b8c06bSAndreas Gruenbacher struct drbd_device *device = req->device; 149244a4d551SLars Ellenberg struct drbd_peer_device *const peer_device = first_peer_device(device); 149344a4d551SLars Ellenberg struct drbd_connection *connection = peer_device->connection; 1494c51a0ef3SLars Ellenberg bool do_send_unplug = req->rq_state & RQ_UNPLUG; 149599920dc5SAndreas Gruenbacher int err; 1496b411b363SPhilipp Reisner 1497b411b363SPhilipp Reisner if (unlikely(cancel)) { 14988554df1cSAndreas Gruenbacher req_mod(req, SEND_CANCELED); 149999920dc5SAndreas Gruenbacher return 0; 1500b411b363SPhilipp Reisner } 1501e5f891b2SLars Ellenberg req->pre_send_jif = jiffies; 1502b411b363SPhilipp Reisner 1503b6dd1a89SLars Ellenberg /* Even read requests may close a write epoch, 1504b6dd1a89SLars Ellenberg * if there was any yet. */ 1505bde89a9eSAndreas Gruenbacher maybe_send_barrier(connection, req->epoch); 1506b6dd1a89SLars Ellenberg 150744a4d551SLars Ellenberg err = drbd_send_drequest(peer_device, P_DATA_REQUEST, req->i.sector, req->i.size, 1508b411b363SPhilipp Reisner (unsigned long)req); 1509b411b363SPhilipp Reisner 151099920dc5SAndreas Gruenbacher req_mod(req, err ? SEND_FAILED : HANDED_OVER_TO_NETWORK); 1511b411b363SPhilipp Reisner 1512c51a0ef3SLars Ellenberg if (do_send_unplug && !err) 1513c51a0ef3SLars Ellenberg pd_send_unplug_remote(peer_device); 1514c51a0ef3SLars Ellenberg 151599920dc5SAndreas Gruenbacher return err; 1516b411b363SPhilipp Reisner } 1517b411b363SPhilipp Reisner 151899920dc5SAndreas Gruenbacher int w_restart_disk_io(struct drbd_work *w, int cancel) 1519265be2d0SPhilipp Reisner { 1520265be2d0SPhilipp Reisner struct drbd_request *req = container_of(w, struct drbd_request, w); 152184b8c06bSAndreas Gruenbacher struct drbd_device *device = req->device; 1522265be2d0SPhilipp Reisner 15230778286aSPhilipp Reisner if (bio_data_dir(req->master_bio) == WRITE && req->rq_state & RQ_IN_ACT_LOG) 15244dd726f0SLars Ellenberg drbd_al_begin_io(device, &req->i); 1525265be2d0SPhilipp Reisner 1526abfc426dSChristoph Hellwig req->private_bio = bio_alloc_clone(device->ldev->backing_bdev, 1527abfc426dSChristoph Hellwig req->master_bio, GFP_NOIO, 1528ae7153f1SChristoph Hellwig &drbd_io_bio_set); 1529ae7153f1SChristoph Hellwig req->private_bio->bi_private = req; 1530ae7153f1SChristoph Hellwig req->private_bio->bi_end_io = drbd_request_endio; 1531ed00aabdSChristoph Hellwig submit_bio_noacct(req->private_bio); 1532265be2d0SPhilipp Reisner 153399920dc5SAndreas Gruenbacher return 0; 1534265be2d0SPhilipp Reisner } 1535265be2d0SPhilipp Reisner 1536b30ab791SAndreas Gruenbacher static int _drbd_may_sync_now(struct drbd_device *device) 1537b411b363SPhilipp Reisner { 1538b30ab791SAndreas Gruenbacher struct drbd_device *odev = device; 153995f8efd0SAndreas Gruenbacher int resync_after; 1540b411b363SPhilipp Reisner 1541b411b363SPhilipp Reisner while (1) { 1542a3f8f7dcSLars Ellenberg if (!odev->ldev || odev->state.disk == D_DISKLESS) 1543438c8374SPhilipp Reisner return 1; 1544daeda1ccSPhilipp Reisner rcu_read_lock(); 154595f8efd0SAndreas Gruenbacher resync_after = rcu_dereference(odev->ldev->disk_conf)->resync_after; 1546daeda1ccSPhilipp Reisner rcu_read_unlock(); 154795f8efd0SAndreas Gruenbacher if (resync_after == -1) 1548b411b363SPhilipp Reisner return 1; 1549b30ab791SAndreas Gruenbacher odev = minor_to_device(resync_after); 1550a3f8f7dcSLars Ellenberg if (!odev) 1551841ce241SAndreas Gruenbacher return 1; 1552b411b363SPhilipp Reisner if ((odev->state.conn >= C_SYNC_SOURCE && 1553b411b363SPhilipp Reisner odev->state.conn <= C_PAUSED_SYNC_T) || 1554b411b363SPhilipp Reisner odev->state.aftr_isp || odev->state.peer_isp || 1555b411b363SPhilipp Reisner odev->state.user_isp) 1556b411b363SPhilipp Reisner return 0; 1557b411b363SPhilipp Reisner } 1558b411b363SPhilipp Reisner } 1559b411b363SPhilipp Reisner 1560b411b363SPhilipp Reisner /** 156128bc3b8cSAndreas Gruenbacher * drbd_pause_after() - Pause resync on all devices that may not resync now 1562b30ab791SAndreas Gruenbacher * @device: DRBD device. 1563b411b363SPhilipp Reisner * 1564b411b363SPhilipp Reisner * Called from process context only (admin command and after_state_ch). 1565b411b363SPhilipp Reisner */ 156628bc3b8cSAndreas Gruenbacher static bool drbd_pause_after(struct drbd_device *device) 1567b411b363SPhilipp Reisner { 156828bc3b8cSAndreas Gruenbacher bool changed = false; 156954761697SAndreas Gruenbacher struct drbd_device *odev; 157028bc3b8cSAndreas Gruenbacher int i; 1571b411b363SPhilipp Reisner 1572695d08faSPhilipp Reisner rcu_read_lock(); 157305a10ec7SAndreas Gruenbacher idr_for_each_entry(&drbd_devices, odev, i) { 1574b411b363SPhilipp Reisner if (odev->state.conn == C_STANDALONE && odev->state.disk == D_DISKLESS) 1575b411b363SPhilipp Reisner continue; 157628bc3b8cSAndreas Gruenbacher if (!_drbd_may_sync_now(odev) && 157728bc3b8cSAndreas Gruenbacher _drbd_set_state(_NS(odev, aftr_isp, 1), 157828bc3b8cSAndreas Gruenbacher CS_HARD, NULL) != SS_NOTHING_TO_DO) 157928bc3b8cSAndreas Gruenbacher changed = true; 1580b411b363SPhilipp Reisner } 1581695d08faSPhilipp Reisner rcu_read_unlock(); 1582b411b363SPhilipp Reisner 158328bc3b8cSAndreas Gruenbacher return changed; 1584b411b363SPhilipp Reisner } 1585b411b363SPhilipp Reisner 1586b411b363SPhilipp Reisner /** 158728bc3b8cSAndreas Gruenbacher * drbd_resume_next() - Resume resync on all devices that may resync now 1588b30ab791SAndreas Gruenbacher * @device: DRBD device. 1589b411b363SPhilipp Reisner * 1590b411b363SPhilipp Reisner * Called from process context only (admin command and worker). 1591b411b363SPhilipp Reisner */ 159228bc3b8cSAndreas Gruenbacher static bool drbd_resume_next(struct drbd_device *device) 1593b411b363SPhilipp Reisner { 159428bc3b8cSAndreas Gruenbacher bool changed = false; 159554761697SAndreas Gruenbacher struct drbd_device *odev; 159628bc3b8cSAndreas Gruenbacher int i; 1597b411b363SPhilipp Reisner 1598695d08faSPhilipp Reisner rcu_read_lock(); 159905a10ec7SAndreas Gruenbacher idr_for_each_entry(&drbd_devices, odev, i) { 1600b411b363SPhilipp Reisner if (odev->state.conn == C_STANDALONE && odev->state.disk == D_DISKLESS) 1601b411b363SPhilipp Reisner continue; 1602b411b363SPhilipp Reisner if (odev->state.aftr_isp) { 160328bc3b8cSAndreas Gruenbacher if (_drbd_may_sync_now(odev) && 160428bc3b8cSAndreas Gruenbacher _drbd_set_state(_NS(odev, aftr_isp, 0), 160528bc3b8cSAndreas Gruenbacher CS_HARD, NULL) != SS_NOTHING_TO_DO) 160628bc3b8cSAndreas Gruenbacher changed = true; 1607b411b363SPhilipp Reisner } 1608b411b363SPhilipp Reisner } 1609695d08faSPhilipp Reisner rcu_read_unlock(); 161028bc3b8cSAndreas Gruenbacher return changed; 1611b411b363SPhilipp Reisner } 1612b411b363SPhilipp Reisner 1613b30ab791SAndreas Gruenbacher void resume_next_sg(struct drbd_device *device) 1614b411b363SPhilipp Reisner { 161528bc3b8cSAndreas Gruenbacher lock_all_resources(); 161628bc3b8cSAndreas Gruenbacher drbd_resume_next(device); 161728bc3b8cSAndreas Gruenbacher unlock_all_resources(); 1618b411b363SPhilipp Reisner } 1619b411b363SPhilipp Reisner 1620b30ab791SAndreas Gruenbacher void suspend_other_sg(struct drbd_device *device) 1621b411b363SPhilipp Reisner { 162228bc3b8cSAndreas Gruenbacher lock_all_resources(); 162328bc3b8cSAndreas Gruenbacher drbd_pause_after(device); 162428bc3b8cSAndreas Gruenbacher unlock_all_resources(); 1625b411b363SPhilipp Reisner } 1626b411b363SPhilipp Reisner 162728bc3b8cSAndreas Gruenbacher /* caller must lock_all_resources() */ 1628b30ab791SAndreas Gruenbacher enum drbd_ret_code drbd_resync_after_valid(struct drbd_device *device, int o_minor) 1629b411b363SPhilipp Reisner { 163054761697SAndreas Gruenbacher struct drbd_device *odev; 163195f8efd0SAndreas Gruenbacher int resync_after; 1632b411b363SPhilipp Reisner 1633b411b363SPhilipp Reisner if (o_minor == -1) 1634b411b363SPhilipp Reisner return NO_ERROR; 1635a3f8f7dcSLars Ellenberg if (o_minor < -1 || o_minor > MINORMASK) 163695f8efd0SAndreas Gruenbacher return ERR_RESYNC_AFTER; 1637b411b363SPhilipp Reisner 1638b411b363SPhilipp Reisner /* check for loops */ 1639b30ab791SAndreas Gruenbacher odev = minor_to_device(o_minor); 1640b411b363SPhilipp Reisner while (1) { 1641b30ab791SAndreas Gruenbacher if (odev == device) 164295f8efd0SAndreas Gruenbacher return ERR_RESYNC_AFTER_CYCLE; 1643b411b363SPhilipp Reisner 1644a3f8f7dcSLars Ellenberg /* You are free to depend on diskless, non-existing, 1645a3f8f7dcSLars Ellenberg * or not yet/no longer existing minors. 1646a3f8f7dcSLars Ellenberg * We only reject dependency loops. 1647a3f8f7dcSLars Ellenberg * We cannot follow the dependency chain beyond a detached or 1648a3f8f7dcSLars Ellenberg * missing minor. 1649a3f8f7dcSLars Ellenberg */ 1650a3f8f7dcSLars Ellenberg if (!odev || !odev->ldev || odev->state.disk == D_DISKLESS) 1651a3f8f7dcSLars Ellenberg return NO_ERROR; 1652a3f8f7dcSLars Ellenberg 1653daeda1ccSPhilipp Reisner rcu_read_lock(); 165495f8efd0SAndreas Gruenbacher resync_after = rcu_dereference(odev->ldev->disk_conf)->resync_after; 1655daeda1ccSPhilipp Reisner rcu_read_unlock(); 1656b411b363SPhilipp Reisner /* dependency chain ends here, no cycles. */ 165795f8efd0SAndreas Gruenbacher if (resync_after == -1) 1658b411b363SPhilipp Reisner return NO_ERROR; 1659b411b363SPhilipp Reisner 1660b411b363SPhilipp Reisner /* follow the dependency chain */ 1661b30ab791SAndreas Gruenbacher odev = minor_to_device(resync_after); 1662b411b363SPhilipp Reisner } 1663b411b363SPhilipp Reisner } 1664b411b363SPhilipp Reisner 166528bc3b8cSAndreas Gruenbacher /* caller must lock_all_resources() */ 1666b30ab791SAndreas Gruenbacher void drbd_resync_after_changed(struct drbd_device *device) 1667b411b363SPhilipp Reisner { 166828bc3b8cSAndreas Gruenbacher int changed; 1669b411b363SPhilipp Reisner 1670b411b363SPhilipp Reisner do { 167128bc3b8cSAndreas Gruenbacher changed = drbd_pause_after(device); 167228bc3b8cSAndreas Gruenbacher changed |= drbd_resume_next(device); 167328bc3b8cSAndreas Gruenbacher } while (changed); 1674b411b363SPhilipp Reisner } 1675b411b363SPhilipp Reisner 1676b30ab791SAndreas Gruenbacher void drbd_rs_controller_reset(struct drbd_device *device) 16779bd28d3cSLars Ellenberg { 16788c40c7c4SChristoph Hellwig struct gendisk *disk = device->ldev->backing_bdev->bd_disk; 1679813472ceSPhilipp Reisner struct fifo_buffer *plan; 1680813472ceSPhilipp Reisner 1681b30ab791SAndreas Gruenbacher atomic_set(&device->rs_sect_in, 0); 1682b30ab791SAndreas Gruenbacher atomic_set(&device->rs_sect_ev, 0); 1683b30ab791SAndreas Gruenbacher device->rs_in_flight = 0; 1684cb8432d6SChristoph Hellwig device->rs_last_events = 16858446fe92SChristoph Hellwig (int)part_stat_read_accum(disk->part0, sectors); 1686813472ceSPhilipp Reisner 1687813472ceSPhilipp Reisner /* Updating the RCU protected object in place is necessary since 1688813472ceSPhilipp Reisner this function gets called from atomic context. 1689813472ceSPhilipp Reisner It is valid since all other updates also lead to an completely 1690813472ceSPhilipp Reisner empty fifo */ 1691813472ceSPhilipp Reisner rcu_read_lock(); 1692b30ab791SAndreas Gruenbacher plan = rcu_dereference(device->rs_plan_s); 1693813472ceSPhilipp Reisner plan->total = 0; 1694813472ceSPhilipp Reisner fifo_set(plan, 0); 1695813472ceSPhilipp Reisner rcu_read_unlock(); 16969bd28d3cSLars Ellenberg } 16979bd28d3cSLars Ellenberg 16982bccef39SKees Cook void start_resync_timer_fn(struct timer_list *t) 16991f04af33SPhilipp Reisner { 17002bccef39SKees Cook struct drbd_device *device = from_timer(device, t, start_resync_timer); 1701ac0acb9eSLars Ellenberg drbd_device_post_work(device, RS_START); 17021f04af33SPhilipp Reisner } 17031f04af33SPhilipp Reisner 1704ac0acb9eSLars Ellenberg static void do_start_resync(struct drbd_device *device) 17051f04af33SPhilipp Reisner { 1706b30ab791SAndreas Gruenbacher if (atomic_read(&device->unacked_cnt) || atomic_read(&device->rs_pending_cnt)) { 1707ac0acb9eSLars Ellenberg drbd_warn(device, "postponing start_resync ...\n"); 1708b30ab791SAndreas Gruenbacher device->start_resync_timer.expires = jiffies + HZ/10; 1709b30ab791SAndreas Gruenbacher add_timer(&device->start_resync_timer); 1710ac0acb9eSLars Ellenberg return; 17111f04af33SPhilipp Reisner } 17121f04af33SPhilipp Reisner 1713b30ab791SAndreas Gruenbacher drbd_start_resync(device, C_SYNC_SOURCE); 1714b30ab791SAndreas Gruenbacher clear_bit(AHEAD_TO_SYNC_SOURCE, &device->flags); 17151f04af33SPhilipp Reisner } 17161f04af33SPhilipp Reisner 1717aaaba345SLars Ellenberg static bool use_checksum_based_resync(struct drbd_connection *connection, struct drbd_device *device) 1718aaaba345SLars Ellenberg { 1719aaaba345SLars Ellenberg bool csums_after_crash_only; 1720aaaba345SLars Ellenberg rcu_read_lock(); 1721aaaba345SLars Ellenberg csums_after_crash_only = rcu_dereference(connection->net_conf)->csums_after_crash_only; 1722aaaba345SLars Ellenberg rcu_read_unlock(); 1723aaaba345SLars Ellenberg return connection->agreed_pro_version >= 89 && /* supported? */ 1724aaaba345SLars Ellenberg connection->csums_tfm && /* configured? */ 17257e5fec31SFabian Frederick (csums_after_crash_only == false /* use for each resync? */ 1726aaaba345SLars Ellenberg || test_bit(CRASHED_PRIMARY, &device->flags)); /* or only after Primary crash? */ 1727aaaba345SLars Ellenberg } 1728aaaba345SLars Ellenberg 1729b411b363SPhilipp Reisner /** 1730b411b363SPhilipp Reisner * drbd_start_resync() - Start the resync process 1731b30ab791SAndreas Gruenbacher * @device: DRBD device. 1732b411b363SPhilipp Reisner * @side: Either C_SYNC_SOURCE or C_SYNC_TARGET 1733b411b363SPhilipp Reisner * 1734b411b363SPhilipp Reisner * This function might bring you directly into one of the 1735b411b363SPhilipp Reisner * C_PAUSED_SYNC_* states. 1736b411b363SPhilipp Reisner */ 1737b30ab791SAndreas Gruenbacher void drbd_start_resync(struct drbd_device *device, enum drbd_conns side) 1738b411b363SPhilipp Reisner { 173944a4d551SLars Ellenberg struct drbd_peer_device *peer_device = first_peer_device(device); 174044a4d551SLars Ellenberg struct drbd_connection *connection = peer_device ? peer_device->connection : NULL; 1741b411b363SPhilipp Reisner union drbd_state ns; 1742b411b363SPhilipp Reisner int r; 1743b411b363SPhilipp Reisner 1744b30ab791SAndreas Gruenbacher if (device->state.conn >= C_SYNC_SOURCE && device->state.conn < C_AHEAD) { 1745d0180171SAndreas Gruenbacher drbd_err(device, "Resync already running!\n"); 1746b411b363SPhilipp Reisner return; 1747b411b363SPhilipp Reisner } 1748b411b363SPhilipp Reisner 1749d3d2948fSRoland Kammerer if (!connection) { 1750d3d2948fSRoland Kammerer drbd_err(device, "No connection to peer, aborting!\n"); 1751d3d2948fSRoland Kammerer return; 1752d3d2948fSRoland Kammerer } 1753d3d2948fSRoland Kammerer 1754b30ab791SAndreas Gruenbacher if (!test_bit(B_RS_H_DONE, &device->flags)) { 1755b411b363SPhilipp Reisner if (side == C_SYNC_TARGET) { 1756b411b363SPhilipp Reisner /* Since application IO was locked out during C_WF_BITMAP_T and 1757b411b363SPhilipp Reisner C_WF_SYNC_UUID we are still unmodified. Before going to C_SYNC_TARGET 1758b411b363SPhilipp Reisner we check that we might make the data inconsistent. */ 1759b30ab791SAndreas Gruenbacher r = drbd_khelper(device, "before-resync-target"); 1760b411b363SPhilipp Reisner r = (r >> 8) & 0xff; 1761b411b363SPhilipp Reisner if (r > 0) { 1762d0180171SAndreas Gruenbacher drbd_info(device, "before-resync-target handler returned %d, " 1763b411b363SPhilipp Reisner "dropping connection.\n", r); 176444a4d551SLars Ellenberg conn_request_state(connection, NS(conn, C_DISCONNECTING), CS_HARD); 1765b411b363SPhilipp Reisner return; 1766b411b363SPhilipp Reisner } 176709b9e797SPhilipp Reisner } else /* C_SYNC_SOURCE */ { 1768b30ab791SAndreas Gruenbacher r = drbd_khelper(device, "before-resync-source"); 176909b9e797SPhilipp Reisner r = (r >> 8) & 0xff; 177009b9e797SPhilipp Reisner if (r > 0) { 177109b9e797SPhilipp Reisner if (r == 3) { 1772d0180171SAndreas Gruenbacher drbd_info(device, "before-resync-source handler returned %d, " 177309b9e797SPhilipp Reisner "ignoring. Old userland tools?", r); 177409b9e797SPhilipp Reisner } else { 1775d0180171SAndreas Gruenbacher drbd_info(device, "before-resync-source handler returned %d, " 177609b9e797SPhilipp Reisner "dropping connection.\n", r); 177744a4d551SLars Ellenberg conn_request_state(connection, 1778a6b32bc3SAndreas Gruenbacher NS(conn, C_DISCONNECTING), CS_HARD); 177909b9e797SPhilipp Reisner return; 178009b9e797SPhilipp Reisner } 178109b9e797SPhilipp Reisner } 1782b411b363SPhilipp Reisner } 1783e64a3294SPhilipp Reisner } 1784b411b363SPhilipp Reisner 178544a4d551SLars Ellenberg if (current == connection->worker.task) { 1786dad20554SPhilipp Reisner /* The worker should not sleep waiting for state_mutex, 1787e64a3294SPhilipp Reisner that can take long */ 1788b30ab791SAndreas Gruenbacher if (!mutex_trylock(device->state_mutex)) { 1789b30ab791SAndreas Gruenbacher set_bit(B_RS_H_DONE, &device->flags); 1790b30ab791SAndreas Gruenbacher device->start_resync_timer.expires = jiffies + HZ/5; 1791b30ab791SAndreas Gruenbacher add_timer(&device->start_resync_timer); 1792e64a3294SPhilipp Reisner return; 1793e64a3294SPhilipp Reisner } 1794e64a3294SPhilipp Reisner } else { 1795b30ab791SAndreas Gruenbacher mutex_lock(device->state_mutex); 1796e64a3294SPhilipp Reisner } 1797b411b363SPhilipp Reisner 179828bc3b8cSAndreas Gruenbacher lock_all_resources(); 179928bc3b8cSAndreas Gruenbacher clear_bit(B_RS_H_DONE, &device->flags); 1800a700471bSPhilipp Reisner /* Did some connection breakage or IO error race with us? */ 1801b30ab791SAndreas Gruenbacher if (device->state.conn < C_CONNECTED 1802b30ab791SAndreas Gruenbacher || !get_ldev_if_state(device, D_NEGOTIATING)) { 180328bc3b8cSAndreas Gruenbacher unlock_all_resources(); 180428bc3b8cSAndreas Gruenbacher goto out; 1805b411b363SPhilipp Reisner } 1806b411b363SPhilipp Reisner 1807b30ab791SAndreas Gruenbacher ns = drbd_read_state(device); 1808b411b363SPhilipp Reisner 1809b30ab791SAndreas Gruenbacher ns.aftr_isp = !_drbd_may_sync_now(device); 1810b411b363SPhilipp Reisner 1811b411b363SPhilipp Reisner ns.conn = side; 1812b411b363SPhilipp Reisner 1813b411b363SPhilipp Reisner if (side == C_SYNC_TARGET) 1814b411b363SPhilipp Reisner ns.disk = D_INCONSISTENT; 1815b411b363SPhilipp Reisner else /* side == C_SYNC_SOURCE */ 1816b411b363SPhilipp Reisner ns.pdsk = D_INCONSISTENT; 1817b411b363SPhilipp Reisner 181828bc3b8cSAndreas Gruenbacher r = _drbd_set_state(device, ns, CS_VERBOSE, NULL); 1819b30ab791SAndreas Gruenbacher ns = drbd_read_state(device); 1820b411b363SPhilipp Reisner 1821b411b363SPhilipp Reisner if (ns.conn < C_CONNECTED) 1822b411b363SPhilipp Reisner r = SS_UNKNOWN_ERROR; 1823b411b363SPhilipp Reisner 1824b411b363SPhilipp Reisner if (r == SS_SUCCESS) { 1825b30ab791SAndreas Gruenbacher unsigned long tw = drbd_bm_total_weight(device); 18261d7734a0SLars Ellenberg unsigned long now = jiffies; 18271d7734a0SLars Ellenberg int i; 18281d7734a0SLars Ellenberg 1829b30ab791SAndreas Gruenbacher device->rs_failed = 0; 1830b30ab791SAndreas Gruenbacher device->rs_paused = 0; 1831b30ab791SAndreas Gruenbacher device->rs_same_csum = 0; 1832b30ab791SAndreas Gruenbacher device->rs_last_sect_ev = 0; 1833b30ab791SAndreas Gruenbacher device->rs_total = tw; 1834b30ab791SAndreas Gruenbacher device->rs_start = now; 18351d7734a0SLars Ellenberg for (i = 0; i < DRBD_SYNC_MARKS; i++) { 1836b30ab791SAndreas Gruenbacher device->rs_mark_left[i] = tw; 1837b30ab791SAndreas Gruenbacher device->rs_mark_time[i] = now; 18381d7734a0SLars Ellenberg } 183928bc3b8cSAndreas Gruenbacher drbd_pause_after(device); 18405ab7d2c0SLars Ellenberg /* Forget potentially stale cached per resync extent bit-counts. 18415ab7d2c0SLars Ellenberg * Open coded drbd_rs_cancel_all(device), we already have IRQs 18425ab7d2c0SLars Ellenberg * disabled, and know the disk state is ok. */ 18435ab7d2c0SLars Ellenberg spin_lock(&device->al_lock); 18445ab7d2c0SLars Ellenberg lc_reset(device->resync); 18455ab7d2c0SLars Ellenberg device->resync_locked = 0; 18465ab7d2c0SLars Ellenberg device->resync_wenr = LC_FREE; 18475ab7d2c0SLars Ellenberg spin_unlock(&device->al_lock); 1848b411b363SPhilipp Reisner } 184928bc3b8cSAndreas Gruenbacher unlock_all_resources(); 18505a22db89SLars Ellenberg 18516c922ed5SLars Ellenberg if (r == SS_SUCCESS) { 18525ab7d2c0SLars Ellenberg wake_up(&device->al_wait); /* for lc_reset() above */ 1853328e0f12SPhilipp Reisner /* reset rs_last_bcast when a resync or verify is started, 1854328e0f12SPhilipp Reisner * to deal with potential jiffies wrap. */ 1855b30ab791SAndreas Gruenbacher device->rs_last_bcast = jiffies - HZ; 1856328e0f12SPhilipp Reisner 1857d0180171SAndreas Gruenbacher drbd_info(device, "Began resync as %s (will sync %lu KB [%lu bits set]).\n", 18586c922ed5SLars Ellenberg drbd_conn_str(ns.conn), 1859b30ab791SAndreas Gruenbacher (unsigned long) device->rs_total << (BM_BLOCK_SHIFT-10), 1860b30ab791SAndreas Gruenbacher (unsigned long) device->rs_total); 1861aaaba345SLars Ellenberg if (side == C_SYNC_TARGET) { 1862b30ab791SAndreas Gruenbacher device->bm_resync_fo = 0; 1863aaaba345SLars Ellenberg device->use_csums = use_checksum_based_resync(connection, device); 1864aaaba345SLars Ellenberg } else { 18657e5fec31SFabian Frederick device->use_csums = false; 1866aaaba345SLars Ellenberg } 18675a22db89SLars Ellenberg 18685a22db89SLars Ellenberg /* Since protocol 96, we must serialize drbd_gen_and_send_sync_uuid 18695a22db89SLars Ellenberg * with w_send_oos, or the sync target will get confused as to 18705a22db89SLars Ellenberg * how much bits to resync. We cannot do that always, because for an 18715a22db89SLars Ellenberg * empty resync and protocol < 95, we need to do it here, as we call 18725a22db89SLars Ellenberg * drbd_resync_finished from here in that case. 18735a22db89SLars Ellenberg * We drbd_gen_and_send_sync_uuid here for protocol < 96, 18745a22db89SLars Ellenberg * and from after_state_ch otherwise. */ 187544a4d551SLars Ellenberg if (side == C_SYNC_SOURCE && connection->agreed_pro_version < 96) 187644a4d551SLars Ellenberg drbd_gen_and_send_sync_uuid(peer_device); 1877b411b363SPhilipp Reisner 187844a4d551SLars Ellenberg if (connection->agreed_pro_version < 95 && device->rs_total == 0) { 1879af85e8e8SLars Ellenberg /* This still has a race (about when exactly the peers 1880af85e8e8SLars Ellenberg * detect connection loss) that can lead to a full sync 1881af85e8e8SLars Ellenberg * on next handshake. In 8.3.9 we fixed this with explicit 1882af85e8e8SLars Ellenberg * resync-finished notifications, but the fix 1883af85e8e8SLars Ellenberg * introduces a protocol change. Sleeping for some 1884af85e8e8SLars Ellenberg * time longer than the ping interval + timeout on the 1885af85e8e8SLars Ellenberg * SyncSource, to give the SyncTarget the chance to 1886af85e8e8SLars Ellenberg * detect connection loss, then waiting for a ping 1887af85e8e8SLars Ellenberg * response (implicit in drbd_resync_finished) reduces 1888af85e8e8SLars Ellenberg * the race considerably, but does not solve it. */ 188944ed167dSPhilipp Reisner if (side == C_SYNC_SOURCE) { 189044ed167dSPhilipp Reisner struct net_conf *nc; 189144ed167dSPhilipp Reisner int timeo; 189244ed167dSPhilipp Reisner 189344ed167dSPhilipp Reisner rcu_read_lock(); 189444a4d551SLars Ellenberg nc = rcu_dereference(connection->net_conf); 189544ed167dSPhilipp Reisner timeo = nc->ping_int * HZ + nc->ping_timeo * HZ / 9; 189644ed167dSPhilipp Reisner rcu_read_unlock(); 189744ed167dSPhilipp Reisner schedule_timeout_interruptible(timeo); 189844ed167dSPhilipp Reisner } 1899b30ab791SAndreas Gruenbacher drbd_resync_finished(device); 1900b411b363SPhilipp Reisner } 1901b411b363SPhilipp Reisner 1902b30ab791SAndreas Gruenbacher drbd_rs_controller_reset(device); 1903b30ab791SAndreas Gruenbacher /* ns.conn may already be != device->state.conn, 1904b411b363SPhilipp Reisner * we may have been paused in between, or become paused until 1905b411b363SPhilipp Reisner * the timer triggers. 1906b411b363SPhilipp Reisner * No matter, that is handled in resync_timer_fn() */ 1907b411b363SPhilipp Reisner if (ns.conn == C_SYNC_TARGET) 1908b30ab791SAndreas Gruenbacher mod_timer(&device->resync_timer, jiffies); 1909b411b363SPhilipp Reisner 1910b30ab791SAndreas Gruenbacher drbd_md_sync(device); 1911b411b363SPhilipp Reisner } 1912b30ab791SAndreas Gruenbacher put_ldev(device); 191328bc3b8cSAndreas Gruenbacher out: 1914b30ab791SAndreas Gruenbacher mutex_unlock(device->state_mutex); 1915b411b363SPhilipp Reisner } 1916b411b363SPhilipp Reisner 1917e334f550SLars Ellenberg static void update_on_disk_bitmap(struct drbd_device *device, bool resync_done) 1918c7a58db4SLars Ellenberg { 1919c7a58db4SLars Ellenberg struct sib_info sib = { .sib_reason = SIB_SYNC_PROGRESS, }; 1920c7a58db4SLars Ellenberg device->rs_last_bcast = jiffies; 1921c7a58db4SLars Ellenberg 1922c7a58db4SLars Ellenberg if (!get_ldev(device)) 1923c7a58db4SLars Ellenberg return; 1924c7a58db4SLars Ellenberg 1925c7a58db4SLars Ellenberg drbd_bm_write_lazy(device, 0); 19265ab7d2c0SLars Ellenberg if (resync_done && is_sync_state(device->state.conn)) 1927c7a58db4SLars Ellenberg drbd_resync_finished(device); 19285ab7d2c0SLars Ellenberg 1929c7a58db4SLars Ellenberg drbd_bcast_event(device, &sib); 1930c7a58db4SLars Ellenberg /* update timestamp, in case it took a while to write out stuff */ 1931c7a58db4SLars Ellenberg device->rs_last_bcast = jiffies; 1932c7a58db4SLars Ellenberg put_ldev(device); 1933c7a58db4SLars Ellenberg } 1934c7a58db4SLars Ellenberg 1935e334f550SLars Ellenberg static void drbd_ldev_destroy(struct drbd_device *device) 1936e334f550SLars Ellenberg { 1937e334f550SLars Ellenberg lc_destroy(device->resync); 1938e334f550SLars Ellenberg device->resync = NULL; 1939e334f550SLars Ellenberg lc_destroy(device->act_log); 1940e334f550SLars Ellenberg device->act_log = NULL; 1941d1b80853SAndreas Gruenbacher 1942d1b80853SAndreas Gruenbacher __acquire(local); 194363a7c8adSLars Ellenberg drbd_backing_dev_free(device, device->ldev); 1944d1b80853SAndreas Gruenbacher device->ldev = NULL; 1945d1b80853SAndreas Gruenbacher __release(local); 1946d1b80853SAndreas Gruenbacher 1947e334f550SLars Ellenberg clear_bit(GOING_DISKLESS, &device->flags); 1948e334f550SLars Ellenberg wake_up(&device->misc_wait); 1949e334f550SLars Ellenberg } 1950e334f550SLars Ellenberg 1951e334f550SLars Ellenberg static void go_diskless(struct drbd_device *device) 1952e334f550SLars Ellenberg { 1953e334f550SLars Ellenberg D_ASSERT(device, device->state.disk == D_FAILED); 1954e334f550SLars Ellenberg /* we cannot assert local_cnt == 0 here, as get_ldev_if_state will 1955e334f550SLars Ellenberg * inc/dec it frequently. Once we are D_DISKLESS, no one will touch 1956e334f550SLars Ellenberg * the protected members anymore, though, so once put_ldev reaches zero 1957e334f550SLars Ellenberg * again, it will be safe to free them. */ 1958e334f550SLars Ellenberg 1959e334f550SLars Ellenberg /* Try to write changed bitmap pages, read errors may have just 1960e334f550SLars Ellenberg * set some bits outside the area covered by the activity log. 1961e334f550SLars Ellenberg * 1962e334f550SLars Ellenberg * If we have an IO error during the bitmap writeout, 1963e334f550SLars Ellenberg * we will want a full sync next time, just in case. 1964e334f550SLars Ellenberg * (Do we want a specific meta data flag for this?) 1965e334f550SLars Ellenberg * 1966e334f550SLars Ellenberg * If that does not make it to stable storage either, 1967e334f550SLars Ellenberg * we cannot do anything about that anymore. 1968e334f550SLars Ellenberg * 1969e334f550SLars Ellenberg * We still need to check if both bitmap and ldev are present, we may 1970e334f550SLars Ellenberg * end up here after a failed attach, before ldev was even assigned. 1971e334f550SLars Ellenberg */ 1972e334f550SLars Ellenberg if (device->bitmap && device->ldev) { 1973e334f550SLars Ellenberg /* An interrupted resync or similar is allowed to recounts bits 1974e334f550SLars Ellenberg * while we detach. 1975e334f550SLars Ellenberg * Any modifications would not be expected anymore, though. 1976e334f550SLars Ellenberg */ 1977e334f550SLars Ellenberg if (drbd_bitmap_io_from_worker(device, drbd_bm_write, 1978e334f550SLars Ellenberg "detach", BM_LOCKED_TEST_ALLOWED)) { 1979e334f550SLars Ellenberg if (test_bit(WAS_READ_ERROR, &device->flags)) { 1980e334f550SLars Ellenberg drbd_md_set_flag(device, MDF_FULL_SYNC); 1981e334f550SLars Ellenberg drbd_md_sync(device); 1982e334f550SLars Ellenberg } 1983e334f550SLars Ellenberg } 1984e334f550SLars Ellenberg } 1985e334f550SLars Ellenberg 1986e334f550SLars Ellenberg drbd_force_state(device, NS(disk, D_DISKLESS)); 1987e334f550SLars Ellenberg } 1988e334f550SLars Ellenberg 1989ac0acb9eSLars Ellenberg static int do_md_sync(struct drbd_device *device) 1990ac0acb9eSLars Ellenberg { 1991ac0acb9eSLars Ellenberg drbd_warn(device, "md_sync_timer expired! Worker calls drbd_md_sync().\n"); 1992ac0acb9eSLars Ellenberg drbd_md_sync(device); 1993ac0acb9eSLars Ellenberg return 0; 1994ac0acb9eSLars Ellenberg } 1995ac0acb9eSLars Ellenberg 1996944410e9SLars Ellenberg /* only called from drbd_worker thread, no locking */ 1997944410e9SLars Ellenberg void __update_timing_details( 1998944410e9SLars Ellenberg struct drbd_thread_timing_details *tdp, 1999944410e9SLars Ellenberg unsigned int *cb_nr, 2000944410e9SLars Ellenberg void *cb, 2001944410e9SLars Ellenberg const char *fn, const unsigned int line) 2002944410e9SLars Ellenberg { 2003944410e9SLars Ellenberg unsigned int i = *cb_nr % DRBD_THREAD_DETAILS_HIST; 2004944410e9SLars Ellenberg struct drbd_thread_timing_details *td = tdp + i; 2005944410e9SLars Ellenberg 2006944410e9SLars Ellenberg td->start_jif = jiffies; 2007944410e9SLars Ellenberg td->cb_addr = cb; 2008944410e9SLars Ellenberg td->caller_fn = fn; 2009944410e9SLars Ellenberg td->line = line; 2010944410e9SLars Ellenberg td->cb_nr = *cb_nr; 2011944410e9SLars Ellenberg 2012944410e9SLars Ellenberg i = (i+1) % DRBD_THREAD_DETAILS_HIST; 2013944410e9SLars Ellenberg td = tdp + i; 2014944410e9SLars Ellenberg memset(td, 0, sizeof(*td)); 2015944410e9SLars Ellenberg 2016944410e9SLars Ellenberg ++(*cb_nr); 2017944410e9SLars Ellenberg } 2018944410e9SLars Ellenberg 2019e334f550SLars Ellenberg static void do_device_work(struct drbd_device *device, const unsigned long todo) 2020e334f550SLars Ellenberg { 2021b47a06d1SAndreas Gruenbacher if (test_bit(MD_SYNC, &todo)) 2022ac0acb9eSLars Ellenberg do_md_sync(device); 2023b47a06d1SAndreas Gruenbacher if (test_bit(RS_DONE, &todo) || 2024b47a06d1SAndreas Gruenbacher test_bit(RS_PROGRESS, &todo)) 2025b47a06d1SAndreas Gruenbacher update_on_disk_bitmap(device, test_bit(RS_DONE, &todo)); 2026b47a06d1SAndreas Gruenbacher if (test_bit(GO_DISKLESS, &todo)) 2027e334f550SLars Ellenberg go_diskless(device); 2028b47a06d1SAndreas Gruenbacher if (test_bit(DESTROY_DISK, &todo)) 2029e334f550SLars Ellenberg drbd_ldev_destroy(device); 2030b47a06d1SAndreas Gruenbacher if (test_bit(RS_START, &todo)) 2031ac0acb9eSLars Ellenberg do_start_resync(device); 2032e334f550SLars Ellenberg } 2033e334f550SLars Ellenberg 2034e334f550SLars Ellenberg #define DRBD_DEVICE_WORK_MASK \ 2035e334f550SLars Ellenberg ((1UL << GO_DISKLESS) \ 2036e334f550SLars Ellenberg |(1UL << DESTROY_DISK) \ 2037ac0acb9eSLars Ellenberg |(1UL << MD_SYNC) \ 2038ac0acb9eSLars Ellenberg |(1UL << RS_START) \ 2039e334f550SLars Ellenberg |(1UL << RS_PROGRESS) \ 2040e334f550SLars Ellenberg |(1UL << RS_DONE) \ 2041e334f550SLars Ellenberg ) 2042e334f550SLars Ellenberg 2043e334f550SLars Ellenberg static unsigned long get_work_bits(unsigned long *flags) 2044e334f550SLars Ellenberg { 2045e334f550SLars Ellenberg unsigned long old, new; 2046e334f550SLars Ellenberg do { 2047e334f550SLars Ellenberg old = *flags; 2048e334f550SLars Ellenberg new = old & ~DRBD_DEVICE_WORK_MASK; 2049e334f550SLars Ellenberg } while (cmpxchg(flags, old, new) != old); 2050e334f550SLars Ellenberg return old & DRBD_DEVICE_WORK_MASK; 2051e334f550SLars Ellenberg } 2052e334f550SLars Ellenberg 2053e334f550SLars Ellenberg static void do_unqueued_work(struct drbd_connection *connection) 2054c7a58db4SLars Ellenberg { 2055c7a58db4SLars Ellenberg struct drbd_peer_device *peer_device; 2056c7a58db4SLars Ellenberg int vnr; 2057c7a58db4SLars Ellenberg 2058c7a58db4SLars Ellenberg rcu_read_lock(); 2059c7a58db4SLars Ellenberg idr_for_each_entry(&connection->peer_devices, peer_device, vnr) { 2060c7a58db4SLars Ellenberg struct drbd_device *device = peer_device->device; 2061e334f550SLars Ellenberg unsigned long todo = get_work_bits(&device->flags); 2062e334f550SLars Ellenberg if (!todo) 2063c7a58db4SLars Ellenberg continue; 20645ab7d2c0SLars Ellenberg 2065c7a58db4SLars Ellenberg kref_get(&device->kref); 2066c7a58db4SLars Ellenberg rcu_read_unlock(); 2067e334f550SLars Ellenberg do_device_work(device, todo); 2068c7a58db4SLars Ellenberg kref_put(&device->kref, drbd_destroy_device); 2069c7a58db4SLars Ellenberg rcu_read_lock(); 2070c7a58db4SLars Ellenberg } 2071c7a58db4SLars Ellenberg rcu_read_unlock(); 2072c7a58db4SLars Ellenberg } 2073c7a58db4SLars Ellenberg 2074a186e478SRashika Kheria static bool dequeue_work_batch(struct drbd_work_queue *queue, struct list_head *work_list) 20758c0785a5SLars Ellenberg { 20768c0785a5SLars Ellenberg spin_lock_irq(&queue->q_lock); 207715e26f6aSLars Ellenberg list_splice_tail_init(&queue->q, work_list); 20788c0785a5SLars Ellenberg spin_unlock_irq(&queue->q_lock); 20798c0785a5SLars Ellenberg return !list_empty(work_list); 20808c0785a5SLars Ellenberg } 20818c0785a5SLars Ellenberg 2082bde89a9eSAndreas Gruenbacher static void wait_for_work(struct drbd_connection *connection, struct list_head *work_list) 2083b6dd1a89SLars Ellenberg { 2084b6dd1a89SLars Ellenberg DEFINE_WAIT(wait); 2085b6dd1a89SLars Ellenberg struct net_conf *nc; 2086b6dd1a89SLars Ellenberg int uncork, cork; 2087b6dd1a89SLars Ellenberg 2088abde9cc6SLars Ellenberg dequeue_work_batch(&connection->sender_work, work_list); 2089b6dd1a89SLars Ellenberg if (!list_empty(work_list)) 2090b6dd1a89SLars Ellenberg return; 2091b6dd1a89SLars Ellenberg 2092b6dd1a89SLars Ellenberg /* Still nothing to do? 2093b6dd1a89SLars Ellenberg * Maybe we still need to close the current epoch, 2094b6dd1a89SLars Ellenberg * even if no new requests are queued yet. 2095b6dd1a89SLars Ellenberg * 2096b6dd1a89SLars Ellenberg * Also, poke TCP, just in case. 2097b6dd1a89SLars Ellenberg * Then wait for new work (or signal). */ 2098b6dd1a89SLars Ellenberg rcu_read_lock(); 2099b6dd1a89SLars Ellenberg nc = rcu_dereference(connection->net_conf); 2100b6dd1a89SLars Ellenberg uncork = nc ? nc->tcp_cork : 0; 2101b6dd1a89SLars Ellenberg rcu_read_unlock(); 2102b6dd1a89SLars Ellenberg if (uncork) { 2103b6dd1a89SLars Ellenberg mutex_lock(&connection->data.mutex); 2104b6dd1a89SLars Ellenberg if (connection->data.socket) 2105db10538aSChristoph Hellwig tcp_sock_set_cork(connection->data.socket->sk, false); 2106b6dd1a89SLars Ellenberg mutex_unlock(&connection->data.mutex); 2107b6dd1a89SLars Ellenberg } 2108b6dd1a89SLars Ellenberg 2109b6dd1a89SLars Ellenberg for (;;) { 2110b6dd1a89SLars Ellenberg int send_barrier; 2111b6dd1a89SLars Ellenberg prepare_to_wait(&connection->sender_work.q_wait, &wait, TASK_INTERRUPTIBLE); 21120500813fSAndreas Gruenbacher spin_lock_irq(&connection->resource->req_lock); 2113b6dd1a89SLars Ellenberg spin_lock(&connection->sender_work.q_lock); /* FIXME get rid of this one? */ 2114bc317a9eSLars Ellenberg if (!list_empty(&connection->sender_work.q)) 21154dd726f0SLars Ellenberg list_splice_tail_init(&connection->sender_work.q, work_list); 2116b6dd1a89SLars Ellenberg spin_unlock(&connection->sender_work.q_lock); /* FIXME get rid of this one? */ 2117b6dd1a89SLars Ellenberg if (!list_empty(work_list) || signal_pending(current)) { 21180500813fSAndreas Gruenbacher spin_unlock_irq(&connection->resource->req_lock); 2119b6dd1a89SLars Ellenberg break; 2120b6dd1a89SLars Ellenberg } 2121f9c78128SLars Ellenberg 2122f9c78128SLars Ellenberg /* We found nothing new to do, no to-be-communicated request, 2123f9c78128SLars Ellenberg * no other work item. We may still need to close the last 2124f9c78128SLars Ellenberg * epoch. Next incoming request epoch will be connection -> 2125f9c78128SLars Ellenberg * current transfer log epoch number. If that is different 2126f9c78128SLars Ellenberg * from the epoch of the last request we communicated, it is 2127f9c78128SLars Ellenberg * safe to send the epoch separating barrier now. 2128f9c78128SLars Ellenberg */ 2129f9c78128SLars Ellenberg send_barrier = 2130f9c78128SLars Ellenberg atomic_read(&connection->current_tle_nr) != 2131f9c78128SLars Ellenberg connection->send.current_epoch_nr; 21320500813fSAndreas Gruenbacher spin_unlock_irq(&connection->resource->req_lock); 2133f9c78128SLars Ellenberg 2134f9c78128SLars Ellenberg if (send_barrier) 2135f9c78128SLars Ellenberg maybe_send_barrier(connection, 2136f9c78128SLars Ellenberg connection->send.current_epoch_nr + 1); 21375ab7d2c0SLars Ellenberg 2138e334f550SLars Ellenberg if (test_bit(DEVICE_WORK_PENDING, &connection->flags)) 21395ab7d2c0SLars Ellenberg break; 21405ab7d2c0SLars Ellenberg 2141a80ca1aeSLars Ellenberg /* drbd_send() may have called flush_signals() */ 2142a80ca1aeSLars Ellenberg if (get_t_state(&connection->worker) != RUNNING) 2143a80ca1aeSLars Ellenberg break; 21445ab7d2c0SLars Ellenberg 2145b6dd1a89SLars Ellenberg schedule(); 2146b6dd1a89SLars Ellenberg /* may be woken up for other things but new work, too, 2147b6dd1a89SLars Ellenberg * e.g. if the current epoch got closed. 2148b6dd1a89SLars Ellenberg * In which case we send the barrier above. */ 2149b6dd1a89SLars Ellenberg } 2150b6dd1a89SLars Ellenberg finish_wait(&connection->sender_work.q_wait, &wait); 2151b6dd1a89SLars Ellenberg 2152b6dd1a89SLars Ellenberg /* someone may have changed the config while we have been waiting above. */ 2153b6dd1a89SLars Ellenberg rcu_read_lock(); 2154b6dd1a89SLars Ellenberg nc = rcu_dereference(connection->net_conf); 2155b6dd1a89SLars Ellenberg cork = nc ? nc->tcp_cork : 0; 2156b6dd1a89SLars Ellenberg rcu_read_unlock(); 2157b6dd1a89SLars Ellenberg mutex_lock(&connection->data.mutex); 2158b6dd1a89SLars Ellenberg if (connection->data.socket) { 2159b6dd1a89SLars Ellenberg if (cork) 2160db10538aSChristoph Hellwig tcp_sock_set_cork(connection->data.socket->sk, true); 2161b6dd1a89SLars Ellenberg else if (!uncork) 2162db10538aSChristoph Hellwig tcp_sock_set_cork(connection->data.socket->sk, false); 2163b6dd1a89SLars Ellenberg } 2164b6dd1a89SLars Ellenberg mutex_unlock(&connection->data.mutex); 2165b6dd1a89SLars Ellenberg } 2166b6dd1a89SLars Ellenberg 2167b411b363SPhilipp Reisner int drbd_worker(struct drbd_thread *thi) 2168b411b363SPhilipp Reisner { 2169bde89a9eSAndreas Gruenbacher struct drbd_connection *connection = thi->connection; 21706db7e50aSAndreas Gruenbacher struct drbd_work *w = NULL; 2171c06ece6bSAndreas Gruenbacher struct drbd_peer_device *peer_device; 2172b411b363SPhilipp Reisner LIST_HEAD(work_list); 21738c0785a5SLars Ellenberg int vnr; 2174b411b363SPhilipp Reisner 2175e77a0a5cSAndreas Gruenbacher while (get_t_state(thi) == RUNNING) { 217680822284SPhilipp Reisner drbd_thread_current_set_cpu(thi); 2177b411b363SPhilipp Reisner 2178944410e9SLars Ellenberg if (list_empty(&work_list)) { 2179944410e9SLars Ellenberg update_worker_timing_details(connection, wait_for_work); 2180bde89a9eSAndreas Gruenbacher wait_for_work(connection, &work_list); 2181944410e9SLars Ellenberg } 2182b411b363SPhilipp Reisner 2183944410e9SLars Ellenberg if (test_and_clear_bit(DEVICE_WORK_PENDING, &connection->flags)) { 2184944410e9SLars Ellenberg update_worker_timing_details(connection, do_unqueued_work); 2185e334f550SLars Ellenberg do_unqueued_work(connection); 2186944410e9SLars Ellenberg } 21875ab7d2c0SLars Ellenberg 21888c0785a5SLars Ellenberg if (signal_pending(current)) { 2189b411b363SPhilipp Reisner flush_signals(current); 219019393e10SPhilipp Reisner if (get_t_state(thi) == RUNNING) { 21911ec861ebSAndreas Gruenbacher drbd_warn(connection, "Worker got an unexpected signal\n"); 2192b411b363SPhilipp Reisner continue; 219319393e10SPhilipp Reisner } 2194b411b363SPhilipp Reisner break; 2195b411b363SPhilipp Reisner } 2196b411b363SPhilipp Reisner 2197e77a0a5cSAndreas Gruenbacher if (get_t_state(thi) != RUNNING) 2198b411b363SPhilipp Reisner break; 2199b411b363SPhilipp Reisner 2200729e8b87SLars Ellenberg if (!list_empty(&work_list)) { 22016db7e50aSAndreas Gruenbacher w = list_first_entry(&work_list, struct drbd_work, list); 22026db7e50aSAndreas Gruenbacher list_del_init(&w->list); 2203944410e9SLars Ellenberg update_worker_timing_details(connection, w->cb); 22046db7e50aSAndreas Gruenbacher if (w->cb(w, connection->cstate < C_WF_REPORT_PARAMS) == 0) 22058c0785a5SLars Ellenberg continue; 2206bde89a9eSAndreas Gruenbacher if (connection->cstate >= C_WF_REPORT_PARAMS) 2207bde89a9eSAndreas Gruenbacher conn_request_state(connection, NS(conn, C_NETWORK_FAILURE), CS_HARD); 2208b411b363SPhilipp Reisner } 2209b411b363SPhilipp Reisner } 2210b411b363SPhilipp Reisner 22118c0785a5SLars Ellenberg do { 2212944410e9SLars Ellenberg if (test_and_clear_bit(DEVICE_WORK_PENDING, &connection->flags)) { 2213944410e9SLars Ellenberg update_worker_timing_details(connection, do_unqueued_work); 2214e334f550SLars Ellenberg do_unqueued_work(connection); 2215944410e9SLars Ellenberg } 2216729e8b87SLars Ellenberg if (!list_empty(&work_list)) { 22176db7e50aSAndreas Gruenbacher w = list_first_entry(&work_list, struct drbd_work, list); 22186db7e50aSAndreas Gruenbacher list_del_init(&w->list); 2219944410e9SLars Ellenberg update_worker_timing_details(connection, w->cb); 22206db7e50aSAndreas Gruenbacher w->cb(w, 1); 2221729e8b87SLars Ellenberg } else 2222bde89a9eSAndreas Gruenbacher dequeue_work_batch(&connection->sender_work, &work_list); 2223e334f550SLars Ellenberg } while (!list_empty(&work_list) || test_bit(DEVICE_WORK_PENDING, &connection->flags)); 2224b411b363SPhilipp Reisner 2225c141ebdaSPhilipp Reisner rcu_read_lock(); 2226c06ece6bSAndreas Gruenbacher idr_for_each_entry(&connection->peer_devices, peer_device, vnr) { 2227c06ece6bSAndreas Gruenbacher struct drbd_device *device = peer_device->device; 22280b0ba1efSAndreas Gruenbacher D_ASSERT(device, device->state.disk == D_DISKLESS && device->state.conn == C_STANDALONE); 2229b30ab791SAndreas Gruenbacher kref_get(&device->kref); 2230c141ebdaSPhilipp Reisner rcu_read_unlock(); 2231b30ab791SAndreas Gruenbacher drbd_device_cleanup(device); 223205a10ec7SAndreas Gruenbacher kref_put(&device->kref, drbd_destroy_device); 2233c141ebdaSPhilipp Reisner rcu_read_lock(); 22340e29d163SPhilipp Reisner } 2235c141ebdaSPhilipp Reisner rcu_read_unlock(); 2236b411b363SPhilipp Reisner 2237b411b363SPhilipp Reisner return 0; 2238b411b363SPhilipp Reisner } 2239