1b411b363SPhilipp Reisner /* 2b411b363SPhilipp Reisner drbd_worker.c 3b411b363SPhilipp Reisner 4b411b363SPhilipp Reisner This file is part of DRBD by Philipp Reisner and Lars Ellenberg. 5b411b363SPhilipp Reisner 6b411b363SPhilipp Reisner Copyright (C) 2001-2008, LINBIT Information Technologies GmbH. 7b411b363SPhilipp Reisner Copyright (C) 1999-2008, Philipp Reisner <philipp.reisner@linbit.com>. 8b411b363SPhilipp Reisner Copyright (C) 2002-2008, Lars Ellenberg <lars.ellenberg@linbit.com>. 9b411b363SPhilipp Reisner 10b411b363SPhilipp Reisner drbd is free software; you can redistribute it and/or modify 11b411b363SPhilipp Reisner it under the terms of the GNU General Public License as published by 12b411b363SPhilipp Reisner the Free Software Foundation; either version 2, or (at your option) 13b411b363SPhilipp Reisner any later version. 14b411b363SPhilipp Reisner 15b411b363SPhilipp Reisner drbd is distributed in the hope that it will be useful, 16b411b363SPhilipp Reisner but WITHOUT ANY WARRANTY; without even the implied warranty of 17b411b363SPhilipp Reisner MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 18b411b363SPhilipp Reisner GNU General Public License for more details. 19b411b363SPhilipp Reisner 20b411b363SPhilipp Reisner You should have received a copy of the GNU General Public License 21b411b363SPhilipp Reisner along with drbd; see the file COPYING. If not, write to 22b411b363SPhilipp Reisner the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA. 23b411b363SPhilipp Reisner 24b411b363SPhilipp Reisner */ 25b411b363SPhilipp Reisner 26b411b363SPhilipp Reisner #include <linux/module.h> 27b411b363SPhilipp Reisner #include <linux/drbd.h> 28174cd4b1SIngo Molnar #include <linux/sched/signal.h> 29b411b363SPhilipp Reisner #include <linux/wait.h> 30b411b363SPhilipp Reisner #include <linux/mm.h> 31b411b363SPhilipp Reisner #include <linux/memcontrol.h> 32b411b363SPhilipp Reisner #include <linux/mm_inline.h> 33b411b363SPhilipp Reisner #include <linux/slab.h> 34b411b363SPhilipp Reisner #include <linux/random.h> 35b411b363SPhilipp Reisner #include <linux/string.h> 36b411b363SPhilipp Reisner #include <linux/scatterlist.h> 37b411b363SPhilipp Reisner 38b411b363SPhilipp Reisner #include "drbd_int.h" 39a3603a6eSAndreas Gruenbacher #include "drbd_protocol.h" 40b411b363SPhilipp Reisner #include "drbd_req.h" 41b411b363SPhilipp Reisner 42d448a2e1SAndreas Gruenbacher static int make_ov_request(struct drbd_device *, int); 43d448a2e1SAndreas Gruenbacher static int make_resync_request(struct drbd_device *, int); 44b411b363SPhilipp Reisner 45c5a91619SAndreas Gruenbacher /* endio handlers: 46ed15b795SAndreas Gruenbacher * drbd_md_endio (defined here) 47fcefa62eSAndreas Gruenbacher * drbd_request_endio (defined here) 48fcefa62eSAndreas Gruenbacher * drbd_peer_request_endio (defined here) 49ed15b795SAndreas Gruenbacher * drbd_bm_endio (defined in drbd_bitmap.c) 50c5a91619SAndreas Gruenbacher * 51b411b363SPhilipp Reisner * For all these callbacks, note the following: 52b411b363SPhilipp Reisner * The callbacks will be called in irq context by the IDE drivers, 53b411b363SPhilipp Reisner * and in Softirqs/Tasklets/BH context by the SCSI drivers. 54b411b363SPhilipp Reisner * Try to get the locking right :) 55b411b363SPhilipp Reisner * 56b411b363SPhilipp Reisner */ 57b411b363SPhilipp Reisner 58b411b363SPhilipp Reisner /* used for synchronous meta data and bitmap IO 59b411b363SPhilipp Reisner * submitted by drbd_md_sync_page_io() 60b411b363SPhilipp Reisner */ 614246a0b6SChristoph Hellwig void drbd_md_endio(struct bio *bio) 62b411b363SPhilipp Reisner { 63b30ab791SAndreas Gruenbacher struct drbd_device *device; 64b411b363SPhilipp Reisner 65e37d2438SLars Ellenberg device = bio->bi_private; 664e4cbee9SChristoph Hellwig device->md_io.error = blk_status_to_errno(bio->bi_status); 67b411b363SPhilipp Reisner 680cfac5ddSPhilipp Reisner /* We grabbed an extra reference in _drbd_md_sync_page_io() to be able 690cfac5ddSPhilipp Reisner * to timeout on the lower level device, and eventually detach from it. 700cfac5ddSPhilipp Reisner * If this io completion runs after that timeout expired, this 710cfac5ddSPhilipp Reisner * drbd_md_put_buffer() may allow us to finally try and re-attach. 720cfac5ddSPhilipp Reisner * During normal operation, this only puts that extra reference 730cfac5ddSPhilipp Reisner * down to 1 again. 740cfac5ddSPhilipp Reisner * Make sure we first drop the reference, and only then signal 750cfac5ddSPhilipp Reisner * completion, or we may (in drbd_al_read_log()) cycle so fast into the 760cfac5ddSPhilipp Reisner * next drbd_md_sync_page_io(), that we trigger the 77b30ab791SAndreas Gruenbacher * ASSERT(atomic_read(&device->md_io_in_use) == 1) there. 780cfac5ddSPhilipp Reisner */ 79b30ab791SAndreas Gruenbacher drbd_md_put_buffer(device); 80e37d2438SLars Ellenberg device->md_io.done = 1; 81b30ab791SAndreas Gruenbacher wake_up(&device->misc_wait); 82cdfda633SPhilipp Reisner bio_put(bio); 83b30ab791SAndreas Gruenbacher if (device->ldev) /* special case: drbd_md_read() during drbd_adm_attach() */ 84b30ab791SAndreas Gruenbacher put_ldev(device); 85b411b363SPhilipp Reisner } 86b411b363SPhilipp Reisner 87b411b363SPhilipp Reisner /* reads on behalf of the partner, 88b411b363SPhilipp Reisner * "submitted" by the receiver 89b411b363SPhilipp Reisner */ 90a186e478SRashika Kheria static void drbd_endio_read_sec_final(struct drbd_peer_request *peer_req) __releases(local) 91b411b363SPhilipp Reisner { 92b411b363SPhilipp Reisner unsigned long flags = 0; 936780139cSAndreas Gruenbacher struct drbd_peer_device *peer_device = peer_req->peer_device; 946780139cSAndreas Gruenbacher struct drbd_device *device = peer_device->device; 95b411b363SPhilipp Reisner 960500813fSAndreas Gruenbacher spin_lock_irqsave(&device->resource->req_lock, flags); 97b30ab791SAndreas Gruenbacher device->read_cnt += peer_req->i.size >> 9; 98a8cd15baSAndreas Gruenbacher list_del(&peer_req->w.list); 99b30ab791SAndreas Gruenbacher if (list_empty(&device->read_ee)) 100b30ab791SAndreas Gruenbacher wake_up(&device->ee_wait); 101db830c46SAndreas Gruenbacher if (test_bit(__EE_WAS_ERROR, &peer_req->flags)) 102b30ab791SAndreas Gruenbacher __drbd_chk_io_error(device, DRBD_READ_ERROR); 1030500813fSAndreas Gruenbacher spin_unlock_irqrestore(&device->resource->req_lock, flags); 104b411b363SPhilipp Reisner 1056780139cSAndreas Gruenbacher drbd_queue_work(&peer_device->connection->sender_work, &peer_req->w); 106b30ab791SAndreas Gruenbacher put_ldev(device); 107b411b363SPhilipp Reisner } 108b411b363SPhilipp Reisner 109b411b363SPhilipp Reisner /* writes on behalf of the partner, or resync writes, 11045bb912bSLars Ellenberg * "submitted" by the receiver, final stage. */ 111a0fb3c47SLars Ellenberg void drbd_endio_write_sec_final(struct drbd_peer_request *peer_req) __releases(local) 112b411b363SPhilipp Reisner { 113b411b363SPhilipp Reisner unsigned long flags = 0; 1146780139cSAndreas Gruenbacher struct drbd_peer_device *peer_device = peer_req->peer_device; 1156780139cSAndreas Gruenbacher struct drbd_device *device = peer_device->device; 116668700b4SPhilipp Reisner struct drbd_connection *connection = peer_device->connection; 117181286adSLars Ellenberg struct drbd_interval i; 118b411b363SPhilipp Reisner int do_wake; 119579b57edSAndreas Gruenbacher u64 block_id; 120b411b363SPhilipp Reisner int do_al_complete_io; 121b411b363SPhilipp Reisner 122db830c46SAndreas Gruenbacher /* after we moved peer_req to done_ee, 123b411b363SPhilipp Reisner * we may no longer access it, 124b411b363SPhilipp Reisner * it may be freed/reused already! 125b411b363SPhilipp Reisner * (as soon as we release the req_lock) */ 126181286adSLars Ellenberg i = peer_req->i; 127db830c46SAndreas Gruenbacher do_al_complete_io = peer_req->flags & EE_CALL_AL_COMPLETE_IO; 128db830c46SAndreas Gruenbacher block_id = peer_req->block_id; 12921ae5d7fSLars Ellenberg peer_req->flags &= ~EE_CALL_AL_COMPLETE_IO; 130b411b363SPhilipp Reisner 1310500813fSAndreas Gruenbacher spin_lock_irqsave(&device->resource->req_lock, flags); 132b30ab791SAndreas Gruenbacher device->writ_cnt += peer_req->i.size >> 9; 133a8cd15baSAndreas Gruenbacher list_move_tail(&peer_req->w.list, &device->done_ee); 134b411b363SPhilipp Reisner 135bb3bfe96SAndreas Gruenbacher /* 1365e472264SAndreas Gruenbacher * Do not remove from the write_requests tree here: we did not send the 137bb3bfe96SAndreas Gruenbacher * Ack yet and did not wake possibly waiting conflicting requests. 138bb3bfe96SAndreas Gruenbacher * Removed from the tree from "drbd_process_done_ee" within the 13984b8c06bSAndreas Gruenbacher * appropriate dw.cb (e_end_block/e_end_resync_block) or from 140bb3bfe96SAndreas Gruenbacher * _drbd_clear_done_ee. 141bb3bfe96SAndreas Gruenbacher */ 142b411b363SPhilipp Reisner 143b30ab791SAndreas Gruenbacher do_wake = list_empty(block_id == ID_SYNCER ? &device->sync_ee : &device->active_ee); 144b411b363SPhilipp Reisner 145a0fb3c47SLars Ellenberg /* FIXME do we want to detach for failed REQ_DISCARD? 146a0fb3c47SLars Ellenberg * ((peer_req->flags & (EE_WAS_ERROR|EE_IS_TRIM)) == EE_WAS_ERROR) */ 147a0fb3c47SLars Ellenberg if (peer_req->flags & EE_WAS_ERROR) 148b30ab791SAndreas Gruenbacher __drbd_chk_io_error(device, DRBD_WRITE_ERROR); 149668700b4SPhilipp Reisner 150668700b4SPhilipp Reisner if (connection->cstate >= C_WF_REPORT_PARAMS) { 151668700b4SPhilipp Reisner kref_get(&device->kref); /* put is in drbd_send_acks_wf() */ 152668700b4SPhilipp Reisner if (!queue_work(connection->ack_sender, &peer_device->send_acks_work)) 153668700b4SPhilipp Reisner kref_put(&device->kref, drbd_destroy_device); 154668700b4SPhilipp Reisner } 1550500813fSAndreas Gruenbacher spin_unlock_irqrestore(&device->resource->req_lock, flags); 156b411b363SPhilipp Reisner 157579b57edSAndreas Gruenbacher if (block_id == ID_SYNCER) 158b30ab791SAndreas Gruenbacher drbd_rs_complete_io(device, i.sector); 159b411b363SPhilipp Reisner 160b411b363SPhilipp Reisner if (do_wake) 161b30ab791SAndreas Gruenbacher wake_up(&device->ee_wait); 162b411b363SPhilipp Reisner 163b411b363SPhilipp Reisner if (do_al_complete_io) 164b30ab791SAndreas Gruenbacher drbd_al_complete_io(device, &i); 165b411b363SPhilipp Reisner 166b30ab791SAndreas Gruenbacher put_ldev(device); 16745bb912bSLars Ellenberg } 168b411b363SPhilipp Reisner 16945bb912bSLars Ellenberg /* writes on behalf of the partner, or resync writes, 17045bb912bSLars Ellenberg * "submitted" by the receiver. 17145bb912bSLars Ellenberg */ 1724246a0b6SChristoph Hellwig void drbd_peer_request_endio(struct bio *bio) 17345bb912bSLars Ellenberg { 174db830c46SAndreas Gruenbacher struct drbd_peer_request *peer_req = bio->bi_private; 175a8cd15baSAndreas Gruenbacher struct drbd_device *device = peer_req->peer_device->device; 1767e5fec31SFabian Frederick bool is_write = bio_data_dir(bio) == WRITE; 17745c21793SChristoph Hellwig bool is_discard = bio_op(bio) == REQ_OP_WRITE_ZEROES || 17845c21793SChristoph Hellwig bio_op(bio) == REQ_OP_DISCARD; 17945bb912bSLars Ellenberg 1804e4cbee9SChristoph Hellwig if (bio->bi_status && __ratelimit(&drbd_ratelimit_state)) 181d0180171SAndreas Gruenbacher drbd_warn(device, "%s: error=%d s=%llus\n", 182a0fb3c47SLars Ellenberg is_write ? (is_discard ? "discard" : "write") 1834e4cbee9SChristoph Hellwig : "read", bio->bi_status, 184db830c46SAndreas Gruenbacher (unsigned long long)peer_req->i.sector); 18545bb912bSLars Ellenberg 1864e4cbee9SChristoph Hellwig if (bio->bi_status) 187db830c46SAndreas Gruenbacher set_bit(__EE_WAS_ERROR, &peer_req->flags); 18845bb912bSLars Ellenberg 18945bb912bSLars Ellenberg bio_put(bio); /* no need for the bio anymore */ 190db830c46SAndreas Gruenbacher if (atomic_dec_and_test(&peer_req->pending_bios)) { 19145bb912bSLars Ellenberg if (is_write) 192db830c46SAndreas Gruenbacher drbd_endio_write_sec_final(peer_req); 19345bb912bSLars Ellenberg else 194db830c46SAndreas Gruenbacher drbd_endio_read_sec_final(peer_req); 19545bb912bSLars Ellenberg } 196b411b363SPhilipp Reisner } 197b411b363SPhilipp Reisner 198142207f7SLars Ellenberg void drbd_panic_after_delayed_completion_of_aborted_request(struct drbd_device *device) 199142207f7SLars Ellenberg { 200142207f7SLars Ellenberg panic("drbd%u %s/%u potential random memory corruption caused by delayed completion of aborted local request\n", 201142207f7SLars Ellenberg device->minor, device->resource->name, device->vnr); 202142207f7SLars Ellenberg } 203142207f7SLars Ellenberg 204b411b363SPhilipp Reisner /* read, readA or write requests on R_PRIMARY coming from drbd_make_request 205b411b363SPhilipp Reisner */ 2064246a0b6SChristoph Hellwig void drbd_request_endio(struct bio *bio) 207b411b363SPhilipp Reisner { 208a115413dSLars Ellenberg unsigned long flags; 209b411b363SPhilipp Reisner struct drbd_request *req = bio->bi_private; 21084b8c06bSAndreas Gruenbacher struct drbd_device *device = req->device; 211a115413dSLars Ellenberg struct bio_and_error m; 212b411b363SPhilipp Reisner enum drbd_req_event what; 2131b6dd252SPhilipp Reisner 2141b6dd252SPhilipp Reisner /* If this request was aborted locally before, 2151b6dd252SPhilipp Reisner * but now was completed "successfully", 2161b6dd252SPhilipp Reisner * chances are that this caused arbitrary data corruption. 2171b6dd252SPhilipp Reisner * 2181b6dd252SPhilipp Reisner * "aborting" requests, or force-detaching the disk, is intended for 2191b6dd252SPhilipp Reisner * completely blocked/hung local backing devices which do no longer 2201b6dd252SPhilipp Reisner * complete requests at all, not even do error completions. In this 2211b6dd252SPhilipp Reisner * situation, usually a hard-reset and failover is the only way out. 2221b6dd252SPhilipp Reisner * 2231b6dd252SPhilipp Reisner * By "aborting", basically faking a local error-completion, 2241b6dd252SPhilipp Reisner * we allow for a more graceful swichover by cleanly migrating services. 2251b6dd252SPhilipp Reisner * Still the affected node has to be rebooted "soon". 2261b6dd252SPhilipp Reisner * 2271b6dd252SPhilipp Reisner * By completing these requests, we allow the upper layers to re-use 2281b6dd252SPhilipp Reisner * the associated data pages. 2291b6dd252SPhilipp Reisner * 2301b6dd252SPhilipp Reisner * If later the local backing device "recovers", and now DMAs some data 2311b6dd252SPhilipp Reisner * from disk into the original request pages, in the best case it will 2321b6dd252SPhilipp Reisner * just put random data into unused pages; but typically it will corrupt 2331b6dd252SPhilipp Reisner * meanwhile completely unrelated data, causing all sorts of damage. 2341b6dd252SPhilipp Reisner * 2351b6dd252SPhilipp Reisner * Which means delayed successful completion, 2361b6dd252SPhilipp Reisner * especially for READ requests, 2371b6dd252SPhilipp Reisner * is a reason to panic(). 2381b6dd252SPhilipp Reisner * 2391b6dd252SPhilipp Reisner * We assume that a delayed *error* completion is OK, 2401b6dd252SPhilipp Reisner * though we still will complain noisily about it. 2411b6dd252SPhilipp Reisner */ 2421b6dd252SPhilipp Reisner if (unlikely(req->rq_state & RQ_LOCAL_ABORTED)) { 2431b6dd252SPhilipp Reisner if (__ratelimit(&drbd_ratelimit_state)) 244d0180171SAndreas Gruenbacher drbd_emerg(device, "delayed completion of aborted local request; disk-timeout may be too aggressive\n"); 2451b6dd252SPhilipp Reisner 2464e4cbee9SChristoph Hellwig if (!bio->bi_status) 247142207f7SLars Ellenberg drbd_panic_after_delayed_completion_of_aborted_request(device); 2481b6dd252SPhilipp Reisner } 2491b6dd252SPhilipp Reisner 250b411b363SPhilipp Reisner /* to avoid recursion in __req_mod */ 2514e4cbee9SChristoph Hellwig if (unlikely(bio->bi_status)) { 25270246286SChristoph Hellwig switch (bio_op(bio)) { 25345c21793SChristoph Hellwig case REQ_OP_WRITE_ZEROES: 25470246286SChristoph Hellwig case REQ_OP_DISCARD: 2554e4cbee9SChristoph Hellwig if (bio->bi_status == BLK_STS_NOTSUPP) 25670246286SChristoph Hellwig what = DISCARD_COMPLETED_NOTSUPP; 2572f632aebSLars Ellenberg else 25870246286SChristoph Hellwig what = DISCARD_COMPLETED_WITH_ERROR; 25970246286SChristoph Hellwig break; 26070246286SChristoph Hellwig case REQ_OP_READ: 2611eff9d32SJens Axboe if (bio->bi_opf & REQ_RAHEAD) 26270246286SChristoph Hellwig what = READ_AHEAD_COMPLETED_WITH_ERROR; 26370246286SChristoph Hellwig else 26470246286SChristoph Hellwig what = READ_COMPLETED_WITH_ERROR; 26570246286SChristoph Hellwig break; 26670246286SChristoph Hellwig default: 26770246286SChristoph Hellwig what = WRITE_COMPLETED_WITH_ERROR; 26870246286SChristoph Hellwig break; 26970246286SChristoph Hellwig } 27070246286SChristoph Hellwig } else { 2718554df1cSAndreas Gruenbacher what = COMPLETED_OK; 27270246286SChristoph Hellwig } 273b411b363SPhilipp Reisner 274b411b363SPhilipp Reisner bio_put(req->private_bio); 2754e4cbee9SChristoph Hellwig req->private_bio = ERR_PTR(blk_status_to_errno(bio->bi_status)); 276b411b363SPhilipp Reisner 277a115413dSLars Ellenberg /* not req_mod(), we need irqsave here! */ 2780500813fSAndreas Gruenbacher spin_lock_irqsave(&device->resource->req_lock, flags); 279a115413dSLars Ellenberg __req_mod(req, what, &m); 2800500813fSAndreas Gruenbacher spin_unlock_irqrestore(&device->resource->req_lock, flags); 281b30ab791SAndreas Gruenbacher put_ldev(device); 282a115413dSLars Ellenberg 283a115413dSLars Ellenberg if (m.bio) 284b30ab791SAndreas Gruenbacher complete_master_bio(device, &m); 285b411b363SPhilipp Reisner } 286b411b363SPhilipp Reisner 2879534d671SHerbert Xu void drbd_csum_ee(struct crypto_ahash *tfm, struct drbd_peer_request *peer_req, void *digest) 28845bb912bSLars Ellenberg { 2899534d671SHerbert Xu AHASH_REQUEST_ON_STACK(req, tfm); 29045bb912bSLars Ellenberg struct scatterlist sg; 291db830c46SAndreas Gruenbacher struct page *page = peer_req->pages; 29245bb912bSLars Ellenberg struct page *tmp; 29345bb912bSLars Ellenberg unsigned len; 29445bb912bSLars Ellenberg 2959534d671SHerbert Xu ahash_request_set_tfm(req, tfm); 2969534d671SHerbert Xu ahash_request_set_callback(req, 0, NULL, NULL); 29745bb912bSLars Ellenberg 29845bb912bSLars Ellenberg sg_init_table(&sg, 1); 2999534d671SHerbert Xu crypto_ahash_init(req); 30045bb912bSLars Ellenberg 30145bb912bSLars Ellenberg while ((tmp = page_chain_next(page))) { 30245bb912bSLars Ellenberg /* all but the last page will be fully used */ 30345bb912bSLars Ellenberg sg_set_page(&sg, page, PAGE_SIZE, 0); 3049534d671SHerbert Xu ahash_request_set_crypt(req, &sg, NULL, sg.length); 3059534d671SHerbert Xu crypto_ahash_update(req); 30645bb912bSLars Ellenberg page = tmp; 30745bb912bSLars Ellenberg } 30845bb912bSLars Ellenberg /* and now the last, possibly only partially used page */ 309db830c46SAndreas Gruenbacher len = peer_req->i.size & (PAGE_SIZE - 1); 31045bb912bSLars Ellenberg sg_set_page(&sg, page, len ?: PAGE_SIZE, 0); 3119534d671SHerbert Xu ahash_request_set_crypt(req, &sg, digest, sg.length); 3129534d671SHerbert Xu crypto_ahash_finup(req); 3139534d671SHerbert Xu ahash_request_zero(req); 31445bb912bSLars Ellenberg } 31545bb912bSLars Ellenberg 3169534d671SHerbert Xu void drbd_csum_bio(struct crypto_ahash *tfm, struct bio *bio, void *digest) 317b411b363SPhilipp Reisner { 3189534d671SHerbert Xu AHASH_REQUEST_ON_STACK(req, tfm); 319b411b363SPhilipp Reisner struct scatterlist sg; 3207988613bSKent Overstreet struct bio_vec bvec; 3217988613bSKent Overstreet struct bvec_iter iter; 322b411b363SPhilipp Reisner 3239534d671SHerbert Xu ahash_request_set_tfm(req, tfm); 3249534d671SHerbert Xu ahash_request_set_callback(req, 0, NULL, NULL); 325b411b363SPhilipp Reisner 326b411b363SPhilipp Reisner sg_init_table(&sg, 1); 3279534d671SHerbert Xu crypto_ahash_init(req); 328b411b363SPhilipp Reisner 3297988613bSKent Overstreet bio_for_each_segment(bvec, bio, iter) { 3307988613bSKent Overstreet sg_set_page(&sg, bvec.bv_page, bvec.bv_len, bvec.bv_offset); 3319534d671SHerbert Xu ahash_request_set_crypt(req, &sg, NULL, sg.length); 3329534d671SHerbert Xu crypto_ahash_update(req); 3339104d31aSLars Ellenberg /* REQ_OP_WRITE_SAME has only one segment, 3349104d31aSLars Ellenberg * checksum the payload only once. */ 3359104d31aSLars Ellenberg if (bio_op(bio) == REQ_OP_WRITE_SAME) 3369104d31aSLars Ellenberg break; 337b411b363SPhilipp Reisner } 3389534d671SHerbert Xu ahash_request_set_crypt(req, NULL, digest, 0); 3399534d671SHerbert Xu crypto_ahash_final(req); 3409534d671SHerbert Xu ahash_request_zero(req); 341b411b363SPhilipp Reisner } 342b411b363SPhilipp Reisner 3439676c760SLars Ellenberg /* MAYBE merge common code with w_e_end_ov_req */ 34499920dc5SAndreas Gruenbacher static int w_e_send_csum(struct drbd_work *w, int cancel) 345b411b363SPhilipp Reisner { 346a8cd15baSAndreas Gruenbacher struct drbd_peer_request *peer_req = container_of(w, struct drbd_peer_request, w); 3476780139cSAndreas Gruenbacher struct drbd_peer_device *peer_device = peer_req->peer_device; 3486780139cSAndreas Gruenbacher struct drbd_device *device = peer_device->device; 349b411b363SPhilipp Reisner int digest_size; 350b411b363SPhilipp Reisner void *digest; 35199920dc5SAndreas Gruenbacher int err = 0; 352b411b363SPhilipp Reisner 35353ea4331SLars Ellenberg if (unlikely(cancel)) 35453ea4331SLars Ellenberg goto out; 355b411b363SPhilipp Reisner 3569676c760SLars Ellenberg if (unlikely((peer_req->flags & EE_WAS_ERROR) != 0)) 35753ea4331SLars Ellenberg goto out; 35853ea4331SLars Ellenberg 3599534d671SHerbert Xu digest_size = crypto_ahash_digestsize(peer_device->connection->csums_tfm); 360b411b363SPhilipp Reisner digest = kmalloc(digest_size, GFP_NOIO); 361b411b363SPhilipp Reisner if (digest) { 362db830c46SAndreas Gruenbacher sector_t sector = peer_req->i.sector; 363db830c46SAndreas Gruenbacher unsigned int size = peer_req->i.size; 3646780139cSAndreas Gruenbacher drbd_csum_ee(peer_device->connection->csums_tfm, peer_req, digest); 3659676c760SLars Ellenberg /* Free peer_req and pages before send. 36653ea4331SLars Ellenberg * In case we block on congestion, we could otherwise run into 36753ea4331SLars Ellenberg * some distributed deadlock, if the other side blocks on 36853ea4331SLars Ellenberg * congestion as well, because our receiver blocks in 369c37c8ecfSAndreas Gruenbacher * drbd_alloc_pages due to pp_in_use > max_buffers. */ 370b30ab791SAndreas Gruenbacher drbd_free_peer_req(device, peer_req); 371db830c46SAndreas Gruenbacher peer_req = NULL; 372b30ab791SAndreas Gruenbacher inc_rs_pending(device); 3736780139cSAndreas Gruenbacher err = drbd_send_drequest_csum(peer_device, sector, size, 37453ea4331SLars Ellenberg digest, digest_size, 375b411b363SPhilipp Reisner P_CSUM_RS_REQUEST); 376b411b363SPhilipp Reisner kfree(digest); 377b411b363SPhilipp Reisner } else { 378d0180171SAndreas Gruenbacher drbd_err(device, "kmalloc() of digest failed.\n"); 37999920dc5SAndreas Gruenbacher err = -ENOMEM; 380b411b363SPhilipp Reisner } 381b411b363SPhilipp Reisner 38253ea4331SLars Ellenberg out: 383db830c46SAndreas Gruenbacher if (peer_req) 384b30ab791SAndreas Gruenbacher drbd_free_peer_req(device, peer_req); 385b411b363SPhilipp Reisner 38699920dc5SAndreas Gruenbacher if (unlikely(err)) 387d0180171SAndreas Gruenbacher drbd_err(device, "drbd_send_drequest(..., csum) failed\n"); 38899920dc5SAndreas Gruenbacher return err; 389b411b363SPhilipp Reisner } 390b411b363SPhilipp Reisner 391b411b363SPhilipp Reisner #define GFP_TRY (__GFP_HIGHMEM | __GFP_NOWARN) 392b411b363SPhilipp Reisner 39369a22773SAndreas Gruenbacher static int read_for_csum(struct drbd_peer_device *peer_device, sector_t sector, int size) 394b411b363SPhilipp Reisner { 39569a22773SAndreas Gruenbacher struct drbd_device *device = peer_device->device; 396db830c46SAndreas Gruenbacher struct drbd_peer_request *peer_req; 397b411b363SPhilipp Reisner 398b30ab791SAndreas Gruenbacher if (!get_ldev(device)) 39980a40e43SLars Ellenberg return -EIO; 400b411b363SPhilipp Reisner 401b411b363SPhilipp Reisner /* GFP_TRY, because if there is no memory available right now, this may 402b411b363SPhilipp Reisner * be rescheduled for later. It is "only" background resync, after all. */ 40369a22773SAndreas Gruenbacher peer_req = drbd_alloc_peer_req(peer_device, ID_SYNCER /* unused */, sector, 4049104d31aSLars Ellenberg size, size, GFP_TRY); 405db830c46SAndreas Gruenbacher if (!peer_req) 40680a40e43SLars Ellenberg goto defer; 407b411b363SPhilipp Reisner 408a8cd15baSAndreas Gruenbacher peer_req->w.cb = w_e_send_csum; 4090500813fSAndreas Gruenbacher spin_lock_irq(&device->resource->req_lock); 410b9ed7080SLars Ellenberg list_add_tail(&peer_req->w.list, &device->read_ee); 4110500813fSAndreas Gruenbacher spin_unlock_irq(&device->resource->req_lock); 412b411b363SPhilipp Reisner 413b30ab791SAndreas Gruenbacher atomic_add(size >> 9, &device->rs_sect_ev); 414bb3cc85eSMike Christie if (drbd_submit_peer_request(device, peer_req, REQ_OP_READ, 0, 415bb3cc85eSMike Christie DRBD_FAULT_RS_RD) == 0) 41680a40e43SLars Ellenberg return 0; 41745bb912bSLars Ellenberg 41810f6d992SLars Ellenberg /* If it failed because of ENOMEM, retry should help. If it failed 41910f6d992SLars Ellenberg * because bio_add_page failed (probably broken lower level driver), 42010f6d992SLars Ellenberg * retry may or may not help. 42110f6d992SLars Ellenberg * If it does not, you may need to force disconnect. */ 4220500813fSAndreas Gruenbacher spin_lock_irq(&device->resource->req_lock); 423a8cd15baSAndreas Gruenbacher list_del(&peer_req->w.list); 4240500813fSAndreas Gruenbacher spin_unlock_irq(&device->resource->req_lock); 42522cc37a9SLars Ellenberg 426b30ab791SAndreas Gruenbacher drbd_free_peer_req(device, peer_req); 42780a40e43SLars Ellenberg defer: 428b30ab791SAndreas Gruenbacher put_ldev(device); 42980a40e43SLars Ellenberg return -EAGAIN; 430b411b363SPhilipp Reisner } 431b411b363SPhilipp Reisner 43299920dc5SAndreas Gruenbacher int w_resync_timer(struct drbd_work *w, int cancel) 433794abb75SPhilipp Reisner { 43484b8c06bSAndreas Gruenbacher struct drbd_device *device = 43584b8c06bSAndreas Gruenbacher container_of(w, struct drbd_device, resync_work); 43684b8c06bSAndreas Gruenbacher 437b30ab791SAndreas Gruenbacher switch (device->state.conn) { 438794abb75SPhilipp Reisner case C_VERIFY_S: 439d448a2e1SAndreas Gruenbacher make_ov_request(device, cancel); 440794abb75SPhilipp Reisner break; 441794abb75SPhilipp Reisner case C_SYNC_TARGET: 442d448a2e1SAndreas Gruenbacher make_resync_request(device, cancel); 443794abb75SPhilipp Reisner break; 444794abb75SPhilipp Reisner } 445794abb75SPhilipp Reisner 44699920dc5SAndreas Gruenbacher return 0; 447794abb75SPhilipp Reisner } 448794abb75SPhilipp Reisner 449b411b363SPhilipp Reisner void resync_timer_fn(unsigned long data) 450b411b363SPhilipp Reisner { 451b30ab791SAndreas Gruenbacher struct drbd_device *device = (struct drbd_device *) data; 452b411b363SPhilipp Reisner 45315e26f6aSLars Ellenberg drbd_queue_work_if_unqueued( 45415e26f6aSLars Ellenberg &first_peer_device(device)->connection->sender_work, 45584b8c06bSAndreas Gruenbacher &device->resync_work); 456b411b363SPhilipp Reisner } 457b411b363SPhilipp Reisner 458778f271dSPhilipp Reisner static void fifo_set(struct fifo_buffer *fb, int value) 459778f271dSPhilipp Reisner { 460778f271dSPhilipp Reisner int i; 461778f271dSPhilipp Reisner 462778f271dSPhilipp Reisner for (i = 0; i < fb->size; i++) 463f10f2623SPhilipp Reisner fb->values[i] = value; 464778f271dSPhilipp Reisner } 465778f271dSPhilipp Reisner 466778f271dSPhilipp Reisner static int fifo_push(struct fifo_buffer *fb, int value) 467778f271dSPhilipp Reisner { 468778f271dSPhilipp Reisner int ov; 469778f271dSPhilipp Reisner 470778f271dSPhilipp Reisner ov = fb->values[fb->head_index]; 471778f271dSPhilipp Reisner fb->values[fb->head_index++] = value; 472778f271dSPhilipp Reisner 473778f271dSPhilipp Reisner if (fb->head_index >= fb->size) 474778f271dSPhilipp Reisner fb->head_index = 0; 475778f271dSPhilipp Reisner 476778f271dSPhilipp Reisner return ov; 477778f271dSPhilipp Reisner } 478778f271dSPhilipp Reisner 479778f271dSPhilipp Reisner static void fifo_add_val(struct fifo_buffer *fb, int value) 480778f271dSPhilipp Reisner { 481778f271dSPhilipp Reisner int i; 482778f271dSPhilipp Reisner 483778f271dSPhilipp Reisner for (i = 0; i < fb->size; i++) 484778f271dSPhilipp Reisner fb->values[i] += value; 485778f271dSPhilipp Reisner } 486778f271dSPhilipp Reisner 4879958c857SPhilipp Reisner struct fifo_buffer *fifo_alloc(int fifo_size) 4889958c857SPhilipp Reisner { 4899958c857SPhilipp Reisner struct fifo_buffer *fb; 4909958c857SPhilipp Reisner 4918747d30aSLars Ellenberg fb = kzalloc(sizeof(struct fifo_buffer) + sizeof(int) * fifo_size, GFP_NOIO); 4929958c857SPhilipp Reisner if (!fb) 4939958c857SPhilipp Reisner return NULL; 4949958c857SPhilipp Reisner 4959958c857SPhilipp Reisner fb->head_index = 0; 4969958c857SPhilipp Reisner fb->size = fifo_size; 4979958c857SPhilipp Reisner fb->total = 0; 4989958c857SPhilipp Reisner 4999958c857SPhilipp Reisner return fb; 5009958c857SPhilipp Reisner } 5019958c857SPhilipp Reisner 5020e49d7b0SLars Ellenberg static int drbd_rs_controller(struct drbd_device *device, unsigned int sect_in) 503778f271dSPhilipp Reisner { 504daeda1ccSPhilipp Reisner struct disk_conf *dc; 5057f34f614SLars Ellenberg unsigned int want; /* The number of sectors we want in-flight */ 506778f271dSPhilipp Reisner int req_sect; /* Number of sectors to request in this turn */ 5077f34f614SLars Ellenberg int correction; /* Number of sectors more we need in-flight */ 508778f271dSPhilipp Reisner int cps; /* correction per invocation of drbd_rs_controller() */ 509778f271dSPhilipp Reisner int steps; /* Number of time steps to plan ahead */ 510778f271dSPhilipp Reisner int curr_corr; 511778f271dSPhilipp Reisner int max_sect; 512813472ceSPhilipp Reisner struct fifo_buffer *plan; 513778f271dSPhilipp Reisner 514b30ab791SAndreas Gruenbacher dc = rcu_dereference(device->ldev->disk_conf); 515b30ab791SAndreas Gruenbacher plan = rcu_dereference(device->rs_plan_s); 516778f271dSPhilipp Reisner 517813472ceSPhilipp Reisner steps = plan->size; /* (dc->c_plan_ahead * 10 * SLEEP_TIME) / HZ; */ 518778f271dSPhilipp Reisner 519b30ab791SAndreas Gruenbacher if (device->rs_in_flight + sect_in == 0) { /* At start of resync */ 520daeda1ccSPhilipp Reisner want = ((dc->resync_rate * 2 * SLEEP_TIME) / HZ) * steps; 521778f271dSPhilipp Reisner } else { /* normal path */ 522daeda1ccSPhilipp Reisner want = dc->c_fill_target ? dc->c_fill_target : 523daeda1ccSPhilipp Reisner sect_in * dc->c_delay_target * HZ / (SLEEP_TIME * 10); 524778f271dSPhilipp Reisner } 525778f271dSPhilipp Reisner 526b30ab791SAndreas Gruenbacher correction = want - device->rs_in_flight - plan->total; 527778f271dSPhilipp Reisner 528778f271dSPhilipp Reisner /* Plan ahead */ 529778f271dSPhilipp Reisner cps = correction / steps; 530813472ceSPhilipp Reisner fifo_add_val(plan, cps); 531813472ceSPhilipp Reisner plan->total += cps * steps; 532778f271dSPhilipp Reisner 533778f271dSPhilipp Reisner /* What we do in this step */ 534813472ceSPhilipp Reisner curr_corr = fifo_push(plan, 0); 535813472ceSPhilipp Reisner plan->total -= curr_corr; 536778f271dSPhilipp Reisner 537778f271dSPhilipp Reisner req_sect = sect_in + curr_corr; 538778f271dSPhilipp Reisner if (req_sect < 0) 539778f271dSPhilipp Reisner req_sect = 0; 540778f271dSPhilipp Reisner 541daeda1ccSPhilipp Reisner max_sect = (dc->c_max_rate * 2 * SLEEP_TIME) / HZ; 542778f271dSPhilipp Reisner if (req_sect > max_sect) 543778f271dSPhilipp Reisner req_sect = max_sect; 544778f271dSPhilipp Reisner 545778f271dSPhilipp Reisner /* 546d0180171SAndreas Gruenbacher drbd_warn(device, "si=%u if=%d wa=%u co=%d st=%d cps=%d pl=%d cc=%d rs=%d\n", 547b30ab791SAndreas Gruenbacher sect_in, device->rs_in_flight, want, correction, 548b30ab791SAndreas Gruenbacher steps, cps, device->rs_planed, curr_corr, req_sect); 549778f271dSPhilipp Reisner */ 550778f271dSPhilipp Reisner 551778f271dSPhilipp Reisner return req_sect; 552778f271dSPhilipp Reisner } 553778f271dSPhilipp Reisner 554b30ab791SAndreas Gruenbacher static int drbd_rs_number_requests(struct drbd_device *device) 555e65f440dSLars Ellenberg { 5560e49d7b0SLars Ellenberg unsigned int sect_in; /* Number of sectors that came in since the last turn */ 5570e49d7b0SLars Ellenberg int number, mxb; 5580e49d7b0SLars Ellenberg 5590e49d7b0SLars Ellenberg sect_in = atomic_xchg(&device->rs_sect_in, 0); 5600e49d7b0SLars Ellenberg device->rs_in_flight -= sect_in; 561813472ceSPhilipp Reisner 562813472ceSPhilipp Reisner rcu_read_lock(); 5630e49d7b0SLars Ellenberg mxb = drbd_get_max_buffers(device) / 2; 564b30ab791SAndreas Gruenbacher if (rcu_dereference(device->rs_plan_s)->size) { 5650e49d7b0SLars Ellenberg number = drbd_rs_controller(device, sect_in) >> (BM_BLOCK_SHIFT - 9); 566b30ab791SAndreas Gruenbacher device->c_sync_rate = number * HZ * (BM_BLOCK_SIZE / 1024) / SLEEP_TIME; 567e65f440dSLars Ellenberg } else { 568b30ab791SAndreas Gruenbacher device->c_sync_rate = rcu_dereference(device->ldev->disk_conf)->resync_rate; 569b30ab791SAndreas Gruenbacher number = SLEEP_TIME * device->c_sync_rate / ((BM_BLOCK_SIZE / 1024) * HZ); 570e65f440dSLars Ellenberg } 571813472ceSPhilipp Reisner rcu_read_unlock(); 572e65f440dSLars Ellenberg 5730e49d7b0SLars Ellenberg /* Don't have more than "max-buffers"/2 in-flight. 5740e49d7b0SLars Ellenberg * Otherwise we may cause the remote site to stall on drbd_alloc_pages(), 5750e49d7b0SLars Ellenberg * potentially causing a distributed deadlock on congestion during 5760e49d7b0SLars Ellenberg * online-verify or (checksum-based) resync, if max-buffers, 5770e49d7b0SLars Ellenberg * socket buffer sizes and resync rate settings are mis-configured. */ 5787f34f614SLars Ellenberg 5797f34f614SLars Ellenberg /* note that "number" is in units of "BM_BLOCK_SIZE" (which is 4k), 5807f34f614SLars Ellenberg * mxb (as used here, and in drbd_alloc_pages on the peer) is 5817f34f614SLars Ellenberg * "number of pages" (typically also 4k), 5827f34f614SLars Ellenberg * but "rs_in_flight" is in "sectors" (512 Byte). */ 5837f34f614SLars Ellenberg if (mxb - device->rs_in_flight/8 < number) 5847f34f614SLars Ellenberg number = mxb - device->rs_in_flight/8; 5850e49d7b0SLars Ellenberg 586e65f440dSLars Ellenberg return number; 587e65f440dSLars Ellenberg } 588e65f440dSLars Ellenberg 58944a4d551SLars Ellenberg static int make_resync_request(struct drbd_device *const device, int cancel) 590b411b363SPhilipp Reisner { 59144a4d551SLars Ellenberg struct drbd_peer_device *const peer_device = first_peer_device(device); 59244a4d551SLars Ellenberg struct drbd_connection *const connection = peer_device ? peer_device->connection : NULL; 593b411b363SPhilipp Reisner unsigned long bit; 594b411b363SPhilipp Reisner sector_t sector; 595b30ab791SAndreas Gruenbacher const sector_t capacity = drbd_get_capacity(device->this_bdev); 5961816a2b4SLars Ellenberg int max_bio_size; 597e65f440dSLars Ellenberg int number, rollback_i, size; 598506afb62SLars Ellenberg int align, requeue = 0; 5990f0601f4SLars Ellenberg int i = 0; 60092d94ae6SPhilipp Reisner int discard_granularity = 0; 601b411b363SPhilipp Reisner 602b411b363SPhilipp Reisner if (unlikely(cancel)) 60399920dc5SAndreas Gruenbacher return 0; 604b411b363SPhilipp Reisner 605b30ab791SAndreas Gruenbacher if (device->rs_total == 0) { 606af85e8e8SLars Ellenberg /* empty resync? */ 607b30ab791SAndreas Gruenbacher drbd_resync_finished(device); 60899920dc5SAndreas Gruenbacher return 0; 609af85e8e8SLars Ellenberg } 610af85e8e8SLars Ellenberg 611b30ab791SAndreas Gruenbacher if (!get_ldev(device)) { 612b30ab791SAndreas Gruenbacher /* Since we only need to access device->rsync a 613b30ab791SAndreas Gruenbacher get_ldev_if_state(device,D_FAILED) would be sufficient, but 614b411b363SPhilipp Reisner to continue resync with a broken disk makes no sense at 615b411b363SPhilipp Reisner all */ 616d0180171SAndreas Gruenbacher drbd_err(device, "Disk broke down during resync!\n"); 61799920dc5SAndreas Gruenbacher return 0; 618b411b363SPhilipp Reisner } 619b411b363SPhilipp Reisner 6209104d31aSLars Ellenberg if (connection->agreed_features & DRBD_FF_THIN_RESYNC) { 62192d94ae6SPhilipp Reisner rcu_read_lock(); 62292d94ae6SPhilipp Reisner discard_granularity = rcu_dereference(device->ldev->disk_conf)->rs_discard_granularity; 62392d94ae6SPhilipp Reisner rcu_read_unlock(); 62492d94ae6SPhilipp Reisner } 62592d94ae6SPhilipp Reisner 626b30ab791SAndreas Gruenbacher max_bio_size = queue_max_hw_sectors(device->rq_queue) << 9; 627b30ab791SAndreas Gruenbacher number = drbd_rs_number_requests(device); 6280e49d7b0SLars Ellenberg if (number <= 0) 6290f0601f4SLars Ellenberg goto requeue; 630b411b363SPhilipp Reisner 631b411b363SPhilipp Reisner for (i = 0; i < number; i++) { 632506afb62SLars Ellenberg /* Stop generating RS requests when half of the send buffer is filled, 633506afb62SLars Ellenberg * but notify TCP that we'd like to have more space. */ 63444a4d551SLars Ellenberg mutex_lock(&connection->data.mutex); 63544a4d551SLars Ellenberg if (connection->data.socket) { 636506afb62SLars Ellenberg struct sock *sk = connection->data.socket->sk; 637506afb62SLars Ellenberg int queued = sk->sk_wmem_queued; 638506afb62SLars Ellenberg int sndbuf = sk->sk_sndbuf; 639506afb62SLars Ellenberg if (queued > sndbuf / 2) { 640506afb62SLars Ellenberg requeue = 1; 641506afb62SLars Ellenberg if (sk->sk_socket) 642506afb62SLars Ellenberg set_bit(SOCK_NOSPACE, &sk->sk_socket->flags); 643b411b363SPhilipp Reisner } 644506afb62SLars Ellenberg } else 645506afb62SLars Ellenberg requeue = 1; 64644a4d551SLars Ellenberg mutex_unlock(&connection->data.mutex); 647506afb62SLars Ellenberg if (requeue) 648b411b363SPhilipp Reisner goto requeue; 649b411b363SPhilipp Reisner 650b411b363SPhilipp Reisner next_sector: 651b411b363SPhilipp Reisner size = BM_BLOCK_SIZE; 652b30ab791SAndreas Gruenbacher bit = drbd_bm_find_next(device, device->bm_resync_fo); 653b411b363SPhilipp Reisner 6544b0715f0SLars Ellenberg if (bit == DRBD_END_OF_BITMAP) { 655b30ab791SAndreas Gruenbacher device->bm_resync_fo = drbd_bm_bits(device); 656b30ab791SAndreas Gruenbacher put_ldev(device); 65799920dc5SAndreas Gruenbacher return 0; 658b411b363SPhilipp Reisner } 659b411b363SPhilipp Reisner 660b411b363SPhilipp Reisner sector = BM_BIT_TO_SECT(bit); 661b411b363SPhilipp Reisner 662ad3fee79SLars Ellenberg if (drbd_try_rs_begin_io(device, sector)) { 663b30ab791SAndreas Gruenbacher device->bm_resync_fo = bit; 664b411b363SPhilipp Reisner goto requeue; 665b411b363SPhilipp Reisner } 666b30ab791SAndreas Gruenbacher device->bm_resync_fo = bit + 1; 667b411b363SPhilipp Reisner 668b30ab791SAndreas Gruenbacher if (unlikely(drbd_bm_test_bit(device, bit) == 0)) { 669b30ab791SAndreas Gruenbacher drbd_rs_complete_io(device, sector); 670b411b363SPhilipp Reisner goto next_sector; 671b411b363SPhilipp Reisner } 672b411b363SPhilipp Reisner 6731816a2b4SLars Ellenberg #if DRBD_MAX_BIO_SIZE > BM_BLOCK_SIZE 674b411b363SPhilipp Reisner /* try to find some adjacent bits. 675b411b363SPhilipp Reisner * we stop if we have already the maximum req size. 676b411b363SPhilipp Reisner * 677b411b363SPhilipp Reisner * Additionally always align bigger requests, in order to 678b411b363SPhilipp Reisner * be prepared for all stripe sizes of software RAIDs. 679b411b363SPhilipp Reisner */ 680b411b363SPhilipp Reisner align = 1; 681d207450cSPhilipp Reisner rollback_i = i; 6826377b923SLars Ellenberg while (i < number) { 6831816a2b4SLars Ellenberg if (size + BM_BLOCK_SIZE > max_bio_size) 684b411b363SPhilipp Reisner break; 685b411b363SPhilipp Reisner 686b411b363SPhilipp Reisner /* Be always aligned */ 687b411b363SPhilipp Reisner if (sector & ((1<<(align+3))-1)) 688b411b363SPhilipp Reisner break; 689b411b363SPhilipp Reisner 69092d94ae6SPhilipp Reisner if (discard_granularity && size == discard_granularity) 69192d94ae6SPhilipp Reisner break; 69292d94ae6SPhilipp Reisner 693b411b363SPhilipp Reisner /* do not cross extent boundaries */ 694b411b363SPhilipp Reisner if (((bit+1) & BM_BLOCKS_PER_BM_EXT_MASK) == 0) 695b411b363SPhilipp Reisner break; 696b411b363SPhilipp Reisner /* now, is it actually dirty, after all? 697b411b363SPhilipp Reisner * caution, drbd_bm_test_bit is tri-state for some 698b411b363SPhilipp Reisner * obscure reason; ( b == 0 ) would get the out-of-band 699b411b363SPhilipp Reisner * only accidentally right because of the "oddly sized" 700b411b363SPhilipp Reisner * adjustment below */ 701b30ab791SAndreas Gruenbacher if (drbd_bm_test_bit(device, bit+1) != 1) 702b411b363SPhilipp Reisner break; 703b411b363SPhilipp Reisner bit++; 704b411b363SPhilipp Reisner size += BM_BLOCK_SIZE; 705b411b363SPhilipp Reisner if ((BM_BLOCK_SIZE << align) <= size) 706b411b363SPhilipp Reisner align++; 707b411b363SPhilipp Reisner i++; 708b411b363SPhilipp Reisner } 709b411b363SPhilipp Reisner /* if we merged some, 710b411b363SPhilipp Reisner * reset the offset to start the next drbd_bm_find_next from */ 711b411b363SPhilipp Reisner if (size > BM_BLOCK_SIZE) 712b30ab791SAndreas Gruenbacher device->bm_resync_fo = bit + 1; 713b411b363SPhilipp Reisner #endif 714b411b363SPhilipp Reisner 715b411b363SPhilipp Reisner /* adjust very last sectors, in case we are oddly sized */ 716b411b363SPhilipp Reisner if (sector + (size>>9) > capacity) 717b411b363SPhilipp Reisner size = (capacity-sector)<<9; 718aaaba345SLars Ellenberg 719aaaba345SLars Ellenberg if (device->use_csums) { 72044a4d551SLars Ellenberg switch (read_for_csum(peer_device, sector, size)) { 72180a40e43SLars Ellenberg case -EIO: /* Disk failure */ 722b30ab791SAndreas Gruenbacher put_ldev(device); 72399920dc5SAndreas Gruenbacher return -EIO; 72480a40e43SLars Ellenberg case -EAGAIN: /* allocation failed, or ldev busy */ 725b30ab791SAndreas Gruenbacher drbd_rs_complete_io(device, sector); 726b30ab791SAndreas Gruenbacher device->bm_resync_fo = BM_SECT_TO_BIT(sector); 727d207450cSPhilipp Reisner i = rollback_i; 728b411b363SPhilipp Reisner goto requeue; 72980a40e43SLars Ellenberg case 0: 73080a40e43SLars Ellenberg /* everything ok */ 73180a40e43SLars Ellenberg break; 73280a40e43SLars Ellenberg default: 73380a40e43SLars Ellenberg BUG(); 734b411b363SPhilipp Reisner } 735b411b363SPhilipp Reisner } else { 73699920dc5SAndreas Gruenbacher int err; 73799920dc5SAndreas Gruenbacher 738b30ab791SAndreas Gruenbacher inc_rs_pending(device); 73992d94ae6SPhilipp Reisner err = drbd_send_drequest(peer_device, 74092d94ae6SPhilipp Reisner size == discard_granularity ? P_RS_THIN_REQ : P_RS_DATA_REQUEST, 74199920dc5SAndreas Gruenbacher sector, size, ID_SYNCER); 74299920dc5SAndreas Gruenbacher if (err) { 743d0180171SAndreas Gruenbacher drbd_err(device, "drbd_send_drequest() failed, aborting...\n"); 744b30ab791SAndreas Gruenbacher dec_rs_pending(device); 745b30ab791SAndreas Gruenbacher put_ldev(device); 74699920dc5SAndreas Gruenbacher return err; 747b411b363SPhilipp Reisner } 748b411b363SPhilipp Reisner } 749b411b363SPhilipp Reisner } 750b411b363SPhilipp Reisner 751b30ab791SAndreas Gruenbacher if (device->bm_resync_fo >= drbd_bm_bits(device)) { 752b411b363SPhilipp Reisner /* last syncer _request_ was sent, 753b411b363SPhilipp Reisner * but the P_RS_DATA_REPLY not yet received. sync will end (and 754b411b363SPhilipp Reisner * next sync group will resume), as soon as we receive the last 755b411b363SPhilipp Reisner * resync data block, and the last bit is cleared. 756b411b363SPhilipp Reisner * until then resync "work" is "inactive" ... 757b411b363SPhilipp Reisner */ 758b30ab791SAndreas Gruenbacher put_ldev(device); 75999920dc5SAndreas Gruenbacher return 0; 760b411b363SPhilipp Reisner } 761b411b363SPhilipp Reisner 762b411b363SPhilipp Reisner requeue: 763b30ab791SAndreas Gruenbacher device->rs_in_flight += (i << (BM_BLOCK_SHIFT - 9)); 764b30ab791SAndreas Gruenbacher mod_timer(&device->resync_timer, jiffies + SLEEP_TIME); 765b30ab791SAndreas Gruenbacher put_ldev(device); 76699920dc5SAndreas Gruenbacher return 0; 767b411b363SPhilipp Reisner } 768b411b363SPhilipp Reisner 769d448a2e1SAndreas Gruenbacher static int make_ov_request(struct drbd_device *device, int cancel) 770b411b363SPhilipp Reisner { 771b411b363SPhilipp Reisner int number, i, size; 772b411b363SPhilipp Reisner sector_t sector; 773b30ab791SAndreas Gruenbacher const sector_t capacity = drbd_get_capacity(device->this_bdev); 77458ffa580SLars Ellenberg bool stop_sector_reached = false; 775b411b363SPhilipp Reisner 776b411b363SPhilipp Reisner if (unlikely(cancel)) 777b411b363SPhilipp Reisner return 1; 778b411b363SPhilipp Reisner 779b30ab791SAndreas Gruenbacher number = drbd_rs_number_requests(device); 780b411b363SPhilipp Reisner 781b30ab791SAndreas Gruenbacher sector = device->ov_position; 782b411b363SPhilipp Reisner for (i = 0; i < number; i++) { 78358ffa580SLars Ellenberg if (sector >= capacity) 784b411b363SPhilipp Reisner return 1; 78558ffa580SLars Ellenberg 78658ffa580SLars Ellenberg /* We check for "finished" only in the reply path: 78758ffa580SLars Ellenberg * w_e_end_ov_reply(). 78858ffa580SLars Ellenberg * We need to send at least one request out. */ 78958ffa580SLars Ellenberg stop_sector_reached = i > 0 790b30ab791SAndreas Gruenbacher && verify_can_do_stop_sector(device) 791b30ab791SAndreas Gruenbacher && sector >= device->ov_stop_sector; 79258ffa580SLars Ellenberg if (stop_sector_reached) 79358ffa580SLars Ellenberg break; 794b411b363SPhilipp Reisner 795b411b363SPhilipp Reisner size = BM_BLOCK_SIZE; 796b411b363SPhilipp Reisner 797ad3fee79SLars Ellenberg if (drbd_try_rs_begin_io(device, sector)) { 798b30ab791SAndreas Gruenbacher device->ov_position = sector; 799b411b363SPhilipp Reisner goto requeue; 800b411b363SPhilipp Reisner } 801b411b363SPhilipp Reisner 802b411b363SPhilipp Reisner if (sector + (size>>9) > capacity) 803b411b363SPhilipp Reisner size = (capacity-sector)<<9; 804b411b363SPhilipp Reisner 805b30ab791SAndreas Gruenbacher inc_rs_pending(device); 80669a22773SAndreas Gruenbacher if (drbd_send_ov_request(first_peer_device(device), sector, size)) { 807b30ab791SAndreas Gruenbacher dec_rs_pending(device); 808b411b363SPhilipp Reisner return 0; 809b411b363SPhilipp Reisner } 810b411b363SPhilipp Reisner sector += BM_SECT_PER_BIT; 811b411b363SPhilipp Reisner } 812b30ab791SAndreas Gruenbacher device->ov_position = sector; 813b411b363SPhilipp Reisner 814b411b363SPhilipp Reisner requeue: 815b30ab791SAndreas Gruenbacher device->rs_in_flight += (i << (BM_BLOCK_SHIFT - 9)); 81658ffa580SLars Ellenberg if (i == 0 || !stop_sector_reached) 817b30ab791SAndreas Gruenbacher mod_timer(&device->resync_timer, jiffies + SLEEP_TIME); 818b411b363SPhilipp Reisner return 1; 819b411b363SPhilipp Reisner } 820b411b363SPhilipp Reisner 82199920dc5SAndreas Gruenbacher int w_ov_finished(struct drbd_work *w, int cancel) 822b411b363SPhilipp Reisner { 82384b8c06bSAndreas Gruenbacher struct drbd_device_work *dw = 82484b8c06bSAndreas Gruenbacher container_of(w, struct drbd_device_work, w); 82584b8c06bSAndreas Gruenbacher struct drbd_device *device = dw->device; 82684b8c06bSAndreas Gruenbacher kfree(dw); 827b30ab791SAndreas Gruenbacher ov_out_of_sync_print(device); 828b30ab791SAndreas Gruenbacher drbd_resync_finished(device); 829b411b363SPhilipp Reisner 83099920dc5SAndreas Gruenbacher return 0; 831b411b363SPhilipp Reisner } 832b411b363SPhilipp Reisner 83399920dc5SAndreas Gruenbacher static int w_resync_finished(struct drbd_work *w, int cancel) 834b411b363SPhilipp Reisner { 83584b8c06bSAndreas Gruenbacher struct drbd_device_work *dw = 83684b8c06bSAndreas Gruenbacher container_of(w, struct drbd_device_work, w); 83784b8c06bSAndreas Gruenbacher struct drbd_device *device = dw->device; 83884b8c06bSAndreas Gruenbacher kfree(dw); 839b411b363SPhilipp Reisner 840b30ab791SAndreas Gruenbacher drbd_resync_finished(device); 841b411b363SPhilipp Reisner 84299920dc5SAndreas Gruenbacher return 0; 843b411b363SPhilipp Reisner } 844b411b363SPhilipp Reisner 845b30ab791SAndreas Gruenbacher static void ping_peer(struct drbd_device *device) 846af85e8e8SLars Ellenberg { 847a6b32bc3SAndreas Gruenbacher struct drbd_connection *connection = first_peer_device(device)->connection; 8482a67d8b9SPhilipp Reisner 849bde89a9eSAndreas Gruenbacher clear_bit(GOT_PING_ACK, &connection->flags); 850bde89a9eSAndreas Gruenbacher request_ping(connection); 851bde89a9eSAndreas Gruenbacher wait_event(connection->ping_wait, 852bde89a9eSAndreas Gruenbacher test_bit(GOT_PING_ACK, &connection->flags) || device->state.conn < C_CONNECTED); 853af85e8e8SLars Ellenberg } 854af85e8e8SLars Ellenberg 855b30ab791SAndreas Gruenbacher int drbd_resync_finished(struct drbd_device *device) 856b411b363SPhilipp Reisner { 85726a96110SLars Ellenberg struct drbd_connection *connection = first_peer_device(device)->connection; 858b411b363SPhilipp Reisner unsigned long db, dt, dbdt; 859b411b363SPhilipp Reisner unsigned long n_oos; 860b411b363SPhilipp Reisner union drbd_state os, ns; 86184b8c06bSAndreas Gruenbacher struct drbd_device_work *dw; 862b411b363SPhilipp Reisner char *khelper_cmd = NULL; 86326525618SLars Ellenberg int verify_done = 0; 864b411b363SPhilipp Reisner 865b411b363SPhilipp Reisner /* Remove all elements from the resync LRU. Since future actions 866b411b363SPhilipp Reisner * might set bits in the (main) bitmap, then the entries in the 867b411b363SPhilipp Reisner * resync LRU would be wrong. */ 868b30ab791SAndreas Gruenbacher if (drbd_rs_del_all(device)) { 869b411b363SPhilipp Reisner /* In case this is not possible now, most probably because 870b411b363SPhilipp Reisner * there are P_RS_DATA_REPLY Packets lingering on the worker's 871b411b363SPhilipp Reisner * queue (or even the read operations for those packets 872b411b363SPhilipp Reisner * is not finished by now). Retry in 100ms. */ 873b411b363SPhilipp Reisner 87420ee6390SPhilipp Reisner schedule_timeout_interruptible(HZ / 10); 87584b8c06bSAndreas Gruenbacher dw = kmalloc(sizeof(struct drbd_device_work), GFP_ATOMIC); 87684b8c06bSAndreas Gruenbacher if (dw) { 87784b8c06bSAndreas Gruenbacher dw->w.cb = w_resync_finished; 87884b8c06bSAndreas Gruenbacher dw->device = device; 87926a96110SLars Ellenberg drbd_queue_work(&connection->sender_work, &dw->w); 880b411b363SPhilipp Reisner return 1; 881b411b363SPhilipp Reisner } 88284b8c06bSAndreas Gruenbacher drbd_err(device, "Warn failed to drbd_rs_del_all() and to kmalloc(dw).\n"); 883b411b363SPhilipp Reisner } 884b411b363SPhilipp Reisner 885b30ab791SAndreas Gruenbacher dt = (jiffies - device->rs_start - device->rs_paused) / HZ; 886b411b363SPhilipp Reisner if (dt <= 0) 887b411b363SPhilipp Reisner dt = 1; 88858ffa580SLars Ellenberg 889b30ab791SAndreas Gruenbacher db = device->rs_total; 89058ffa580SLars Ellenberg /* adjust for verify start and stop sectors, respective reached position */ 891b30ab791SAndreas Gruenbacher if (device->state.conn == C_VERIFY_S || device->state.conn == C_VERIFY_T) 892b30ab791SAndreas Gruenbacher db -= device->ov_left; 89358ffa580SLars Ellenberg 894b411b363SPhilipp Reisner dbdt = Bit2KB(db/dt); 895b30ab791SAndreas Gruenbacher device->rs_paused /= HZ; 896b411b363SPhilipp Reisner 897b30ab791SAndreas Gruenbacher if (!get_ldev(device)) 898b411b363SPhilipp Reisner goto out; 899b411b363SPhilipp Reisner 900b30ab791SAndreas Gruenbacher ping_peer(device); 901af85e8e8SLars Ellenberg 9020500813fSAndreas Gruenbacher spin_lock_irq(&device->resource->req_lock); 903b30ab791SAndreas Gruenbacher os = drbd_read_state(device); 904b411b363SPhilipp Reisner 90526525618SLars Ellenberg verify_done = (os.conn == C_VERIFY_S || os.conn == C_VERIFY_T); 90626525618SLars Ellenberg 907b411b363SPhilipp Reisner /* This protects us against multiple calls (that can happen in the presence 908b411b363SPhilipp Reisner of application IO), and against connectivity loss just before we arrive here. */ 909b411b363SPhilipp Reisner if (os.conn <= C_CONNECTED) 910b411b363SPhilipp Reisner goto out_unlock; 911b411b363SPhilipp Reisner 912b411b363SPhilipp Reisner ns = os; 913b411b363SPhilipp Reisner ns.conn = C_CONNECTED; 914b411b363SPhilipp Reisner 915d0180171SAndreas Gruenbacher drbd_info(device, "%s done (total %lu sec; paused %lu sec; %lu K/sec)\n", 91626525618SLars Ellenberg verify_done ? "Online verify" : "Resync", 917b30ab791SAndreas Gruenbacher dt + device->rs_paused, device->rs_paused, dbdt); 918b411b363SPhilipp Reisner 919b30ab791SAndreas Gruenbacher n_oos = drbd_bm_total_weight(device); 920b411b363SPhilipp Reisner 921b411b363SPhilipp Reisner if (os.conn == C_VERIFY_S || os.conn == C_VERIFY_T) { 922b411b363SPhilipp Reisner if (n_oos) { 923d0180171SAndreas Gruenbacher drbd_alert(device, "Online verify found %lu %dk block out of sync!\n", 924b411b363SPhilipp Reisner n_oos, Bit2KB(1)); 925b411b363SPhilipp Reisner khelper_cmd = "out-of-sync"; 926b411b363SPhilipp Reisner } 927b411b363SPhilipp Reisner } else { 9280b0ba1efSAndreas Gruenbacher D_ASSERT(device, (n_oos - device->rs_failed) == 0); 929b411b363SPhilipp Reisner 930b411b363SPhilipp Reisner if (os.conn == C_SYNC_TARGET || os.conn == C_PAUSED_SYNC_T) 931b411b363SPhilipp Reisner khelper_cmd = "after-resync-target"; 932b411b363SPhilipp Reisner 933aaaba345SLars Ellenberg if (device->use_csums && device->rs_total) { 934b30ab791SAndreas Gruenbacher const unsigned long s = device->rs_same_csum; 935b30ab791SAndreas Gruenbacher const unsigned long t = device->rs_total; 936b411b363SPhilipp Reisner const int ratio = 937b411b363SPhilipp Reisner (t == 0) ? 0 : 938b411b363SPhilipp Reisner (t < 100000) ? ((s*100)/t) : (s/(t/100)); 939d0180171SAndreas Gruenbacher drbd_info(device, "%u %% had equal checksums, eliminated: %luK; " 940b411b363SPhilipp Reisner "transferred %luK total %luK\n", 941b411b363SPhilipp Reisner ratio, 942b30ab791SAndreas Gruenbacher Bit2KB(device->rs_same_csum), 943b30ab791SAndreas Gruenbacher Bit2KB(device->rs_total - device->rs_same_csum), 944b30ab791SAndreas Gruenbacher Bit2KB(device->rs_total)); 945b411b363SPhilipp Reisner } 946b411b363SPhilipp Reisner } 947b411b363SPhilipp Reisner 948b30ab791SAndreas Gruenbacher if (device->rs_failed) { 949d0180171SAndreas Gruenbacher drbd_info(device, " %lu failed blocks\n", device->rs_failed); 950b411b363SPhilipp Reisner 951b411b363SPhilipp Reisner if (os.conn == C_SYNC_TARGET || os.conn == C_PAUSED_SYNC_T) { 952b411b363SPhilipp Reisner ns.disk = D_INCONSISTENT; 953b411b363SPhilipp Reisner ns.pdsk = D_UP_TO_DATE; 954b411b363SPhilipp Reisner } else { 955b411b363SPhilipp Reisner ns.disk = D_UP_TO_DATE; 956b411b363SPhilipp Reisner ns.pdsk = D_INCONSISTENT; 957b411b363SPhilipp Reisner } 958b411b363SPhilipp Reisner } else { 959b411b363SPhilipp Reisner ns.disk = D_UP_TO_DATE; 960b411b363SPhilipp Reisner ns.pdsk = D_UP_TO_DATE; 961b411b363SPhilipp Reisner 962b411b363SPhilipp Reisner if (os.conn == C_SYNC_TARGET || os.conn == C_PAUSED_SYNC_T) { 963b30ab791SAndreas Gruenbacher if (device->p_uuid) { 964b411b363SPhilipp Reisner int i; 965b411b363SPhilipp Reisner for (i = UI_BITMAP ; i <= UI_HISTORY_END ; i++) 966b30ab791SAndreas Gruenbacher _drbd_uuid_set(device, i, device->p_uuid[i]); 967b30ab791SAndreas Gruenbacher drbd_uuid_set(device, UI_BITMAP, device->ldev->md.uuid[UI_CURRENT]); 968b30ab791SAndreas Gruenbacher _drbd_uuid_set(device, UI_CURRENT, device->p_uuid[UI_CURRENT]); 969b411b363SPhilipp Reisner } else { 970d0180171SAndreas Gruenbacher drbd_err(device, "device->p_uuid is NULL! BUG\n"); 971b411b363SPhilipp Reisner } 972b411b363SPhilipp Reisner } 973b411b363SPhilipp Reisner 97462b0da3aSLars Ellenberg if (!(os.conn == C_VERIFY_S || os.conn == C_VERIFY_T)) { 97562b0da3aSLars Ellenberg /* for verify runs, we don't update uuids here, 97662b0da3aSLars Ellenberg * so there would be nothing to report. */ 977b30ab791SAndreas Gruenbacher drbd_uuid_set_bm(device, 0UL); 978b30ab791SAndreas Gruenbacher drbd_print_uuids(device, "updated UUIDs"); 979b30ab791SAndreas Gruenbacher if (device->p_uuid) { 980b411b363SPhilipp Reisner /* Now the two UUID sets are equal, update what we 981b411b363SPhilipp Reisner * know of the peer. */ 982b411b363SPhilipp Reisner int i; 983b411b363SPhilipp Reisner for (i = UI_CURRENT ; i <= UI_HISTORY_END ; i++) 984b30ab791SAndreas Gruenbacher device->p_uuid[i] = device->ldev->md.uuid[i]; 985b411b363SPhilipp Reisner } 986b411b363SPhilipp Reisner } 98762b0da3aSLars Ellenberg } 988b411b363SPhilipp Reisner 989b30ab791SAndreas Gruenbacher _drbd_set_state(device, ns, CS_VERBOSE, NULL); 990b411b363SPhilipp Reisner out_unlock: 9910500813fSAndreas Gruenbacher spin_unlock_irq(&device->resource->req_lock); 99226a96110SLars Ellenberg 99326a96110SLars Ellenberg /* If we have been sync source, and have an effective fencing-policy, 99426a96110SLars Ellenberg * once *all* volumes are back in sync, call "unfence". */ 99526a96110SLars Ellenberg if (os.conn == C_SYNC_SOURCE) { 99626a96110SLars Ellenberg enum drbd_disk_state disk_state = D_MASK; 99726a96110SLars Ellenberg enum drbd_disk_state pdsk_state = D_MASK; 99826a96110SLars Ellenberg enum drbd_fencing_p fp = FP_DONT_CARE; 99926a96110SLars Ellenberg 100026a96110SLars Ellenberg rcu_read_lock(); 100126a96110SLars Ellenberg fp = rcu_dereference(device->ldev->disk_conf)->fencing; 100226a96110SLars Ellenberg if (fp != FP_DONT_CARE) { 100326a96110SLars Ellenberg struct drbd_peer_device *peer_device; 100426a96110SLars Ellenberg int vnr; 100526a96110SLars Ellenberg idr_for_each_entry(&connection->peer_devices, peer_device, vnr) { 100626a96110SLars Ellenberg struct drbd_device *device = peer_device->device; 100726a96110SLars Ellenberg disk_state = min_t(enum drbd_disk_state, disk_state, device->state.disk); 100826a96110SLars Ellenberg pdsk_state = min_t(enum drbd_disk_state, pdsk_state, device->state.pdsk); 100926a96110SLars Ellenberg } 101026a96110SLars Ellenberg } 101126a96110SLars Ellenberg rcu_read_unlock(); 101226a96110SLars Ellenberg if (disk_state == D_UP_TO_DATE && pdsk_state == D_UP_TO_DATE) 101326a96110SLars Ellenberg conn_khelper(connection, "unfence-peer"); 101426a96110SLars Ellenberg } 101526a96110SLars Ellenberg 1016b30ab791SAndreas Gruenbacher put_ldev(device); 1017b411b363SPhilipp Reisner out: 1018b30ab791SAndreas Gruenbacher device->rs_total = 0; 1019b30ab791SAndreas Gruenbacher device->rs_failed = 0; 1020b30ab791SAndreas Gruenbacher device->rs_paused = 0; 102158ffa580SLars Ellenberg 102258ffa580SLars Ellenberg /* reset start sector, if we reached end of device */ 1023b30ab791SAndreas Gruenbacher if (verify_done && device->ov_left == 0) 1024b30ab791SAndreas Gruenbacher device->ov_start_sector = 0; 1025b411b363SPhilipp Reisner 1026b30ab791SAndreas Gruenbacher drbd_md_sync(device); 102713d42685SLars Ellenberg 1028b411b363SPhilipp Reisner if (khelper_cmd) 1029b30ab791SAndreas Gruenbacher drbd_khelper(device, khelper_cmd); 1030b411b363SPhilipp Reisner 1031b411b363SPhilipp Reisner return 1; 1032b411b363SPhilipp Reisner } 1033b411b363SPhilipp Reisner 1034b411b363SPhilipp Reisner /* helper */ 1035b30ab791SAndreas Gruenbacher static void move_to_net_ee_or_free(struct drbd_device *device, struct drbd_peer_request *peer_req) 1036b411b363SPhilipp Reisner { 1037045417f7SAndreas Gruenbacher if (drbd_peer_req_has_active_page(peer_req)) { 1038b411b363SPhilipp Reisner /* This might happen if sendpage() has not finished */ 1039db830c46SAndreas Gruenbacher int i = (peer_req->i.size + PAGE_SIZE -1) >> PAGE_SHIFT; 1040b30ab791SAndreas Gruenbacher atomic_add(i, &device->pp_in_use_by_net); 1041b30ab791SAndreas Gruenbacher atomic_sub(i, &device->pp_in_use); 10420500813fSAndreas Gruenbacher spin_lock_irq(&device->resource->req_lock); 1043a8cd15baSAndreas Gruenbacher list_add_tail(&peer_req->w.list, &device->net_ee); 10440500813fSAndreas Gruenbacher spin_unlock_irq(&device->resource->req_lock); 1045435f0740SLars Ellenberg wake_up(&drbd_pp_wait); 1046b411b363SPhilipp Reisner } else 1047b30ab791SAndreas Gruenbacher drbd_free_peer_req(device, peer_req); 1048b411b363SPhilipp Reisner } 1049b411b363SPhilipp Reisner 1050b411b363SPhilipp Reisner /** 1051b411b363SPhilipp Reisner * w_e_end_data_req() - Worker callback, to send a P_DATA_REPLY packet in response to a P_DATA_REQUEST 1052b411b363SPhilipp Reisner * @w: work object. 1053b411b363SPhilipp Reisner * @cancel: The connection will be closed anyways 1054b411b363SPhilipp Reisner */ 105599920dc5SAndreas Gruenbacher int w_e_end_data_req(struct drbd_work *w, int cancel) 1056b411b363SPhilipp Reisner { 1057a8cd15baSAndreas Gruenbacher struct drbd_peer_request *peer_req = container_of(w, struct drbd_peer_request, w); 10586780139cSAndreas Gruenbacher struct drbd_peer_device *peer_device = peer_req->peer_device; 10596780139cSAndreas Gruenbacher struct drbd_device *device = peer_device->device; 106099920dc5SAndreas Gruenbacher int err; 1061b411b363SPhilipp Reisner 1062b411b363SPhilipp Reisner if (unlikely(cancel)) { 1063b30ab791SAndreas Gruenbacher drbd_free_peer_req(device, peer_req); 1064b30ab791SAndreas Gruenbacher dec_unacked(device); 106599920dc5SAndreas Gruenbacher return 0; 1066b411b363SPhilipp Reisner } 1067b411b363SPhilipp Reisner 1068db830c46SAndreas Gruenbacher if (likely((peer_req->flags & EE_WAS_ERROR) == 0)) { 10696780139cSAndreas Gruenbacher err = drbd_send_block(peer_device, P_DATA_REPLY, peer_req); 1070b411b363SPhilipp Reisner } else { 1071b411b363SPhilipp Reisner if (__ratelimit(&drbd_ratelimit_state)) 1072d0180171SAndreas Gruenbacher drbd_err(device, "Sending NegDReply. sector=%llus.\n", 1073db830c46SAndreas Gruenbacher (unsigned long long)peer_req->i.sector); 1074b411b363SPhilipp Reisner 10756780139cSAndreas Gruenbacher err = drbd_send_ack(peer_device, P_NEG_DREPLY, peer_req); 1076b411b363SPhilipp Reisner } 1077b411b363SPhilipp Reisner 1078b30ab791SAndreas Gruenbacher dec_unacked(device); 1079b411b363SPhilipp Reisner 1080b30ab791SAndreas Gruenbacher move_to_net_ee_or_free(device, peer_req); 1081b411b363SPhilipp Reisner 108299920dc5SAndreas Gruenbacher if (unlikely(err)) 1083d0180171SAndreas Gruenbacher drbd_err(device, "drbd_send_block() failed\n"); 108499920dc5SAndreas Gruenbacher return err; 1085b411b363SPhilipp Reisner } 1086b411b363SPhilipp Reisner 1087700ca8c0SPhilipp Reisner static bool all_zero(struct drbd_peer_request *peer_req) 1088700ca8c0SPhilipp Reisner { 1089700ca8c0SPhilipp Reisner struct page *page = peer_req->pages; 1090700ca8c0SPhilipp Reisner unsigned int len = peer_req->i.size; 1091700ca8c0SPhilipp Reisner 1092700ca8c0SPhilipp Reisner page_chain_for_each(page) { 1093700ca8c0SPhilipp Reisner unsigned int l = min_t(unsigned int, len, PAGE_SIZE); 1094700ca8c0SPhilipp Reisner unsigned int i, words = l / sizeof(long); 1095700ca8c0SPhilipp Reisner unsigned long *d; 1096700ca8c0SPhilipp Reisner 1097700ca8c0SPhilipp Reisner d = kmap_atomic(page); 1098700ca8c0SPhilipp Reisner for (i = 0; i < words; i++) { 1099700ca8c0SPhilipp Reisner if (d[i]) { 1100700ca8c0SPhilipp Reisner kunmap_atomic(d); 1101700ca8c0SPhilipp Reisner return false; 1102700ca8c0SPhilipp Reisner } 1103700ca8c0SPhilipp Reisner } 1104700ca8c0SPhilipp Reisner kunmap_atomic(d); 1105700ca8c0SPhilipp Reisner len -= l; 1106700ca8c0SPhilipp Reisner } 1107700ca8c0SPhilipp Reisner 1108700ca8c0SPhilipp Reisner return true; 1109700ca8c0SPhilipp Reisner } 1110700ca8c0SPhilipp Reisner 1111b411b363SPhilipp Reisner /** 1112a209b4aeSAndreas Gruenbacher * w_e_end_rsdata_req() - Worker callback to send a P_RS_DATA_REPLY packet in response to a P_RS_DATA_REQUEST 1113b411b363SPhilipp Reisner * @w: work object. 1114b411b363SPhilipp Reisner * @cancel: The connection will be closed anyways 1115b411b363SPhilipp Reisner */ 111699920dc5SAndreas Gruenbacher int w_e_end_rsdata_req(struct drbd_work *w, int cancel) 1117b411b363SPhilipp Reisner { 1118a8cd15baSAndreas Gruenbacher struct drbd_peer_request *peer_req = container_of(w, struct drbd_peer_request, w); 11196780139cSAndreas Gruenbacher struct drbd_peer_device *peer_device = peer_req->peer_device; 11206780139cSAndreas Gruenbacher struct drbd_device *device = peer_device->device; 112199920dc5SAndreas Gruenbacher int err; 1122b411b363SPhilipp Reisner 1123b411b363SPhilipp Reisner if (unlikely(cancel)) { 1124b30ab791SAndreas Gruenbacher drbd_free_peer_req(device, peer_req); 1125b30ab791SAndreas Gruenbacher dec_unacked(device); 112699920dc5SAndreas Gruenbacher return 0; 1127b411b363SPhilipp Reisner } 1128b411b363SPhilipp Reisner 1129b30ab791SAndreas Gruenbacher if (get_ldev_if_state(device, D_FAILED)) { 1130b30ab791SAndreas Gruenbacher drbd_rs_complete_io(device, peer_req->i.sector); 1131b30ab791SAndreas Gruenbacher put_ldev(device); 1132b411b363SPhilipp Reisner } 1133b411b363SPhilipp Reisner 1134b30ab791SAndreas Gruenbacher if (device->state.conn == C_AHEAD) { 11356780139cSAndreas Gruenbacher err = drbd_send_ack(peer_device, P_RS_CANCEL, peer_req); 1136db830c46SAndreas Gruenbacher } else if (likely((peer_req->flags & EE_WAS_ERROR) == 0)) { 1137b30ab791SAndreas Gruenbacher if (likely(device->state.pdsk >= D_INCONSISTENT)) { 1138b30ab791SAndreas Gruenbacher inc_rs_pending(device); 1139700ca8c0SPhilipp Reisner if (peer_req->flags & EE_RS_THIN_REQ && all_zero(peer_req)) 1140700ca8c0SPhilipp Reisner err = drbd_send_rs_deallocated(peer_device, peer_req); 1141700ca8c0SPhilipp Reisner else 11426780139cSAndreas Gruenbacher err = drbd_send_block(peer_device, P_RS_DATA_REPLY, peer_req); 1143b411b363SPhilipp Reisner } else { 1144b411b363SPhilipp Reisner if (__ratelimit(&drbd_ratelimit_state)) 1145d0180171SAndreas Gruenbacher drbd_err(device, "Not sending RSDataReply, " 1146b411b363SPhilipp Reisner "partner DISKLESS!\n"); 114799920dc5SAndreas Gruenbacher err = 0; 1148b411b363SPhilipp Reisner } 1149b411b363SPhilipp Reisner } else { 1150b411b363SPhilipp Reisner if (__ratelimit(&drbd_ratelimit_state)) 1151d0180171SAndreas Gruenbacher drbd_err(device, "Sending NegRSDReply. sector %llus.\n", 1152db830c46SAndreas Gruenbacher (unsigned long long)peer_req->i.sector); 1153b411b363SPhilipp Reisner 11546780139cSAndreas Gruenbacher err = drbd_send_ack(peer_device, P_NEG_RS_DREPLY, peer_req); 1155b411b363SPhilipp Reisner 1156b411b363SPhilipp Reisner /* update resync data with failure */ 1157b30ab791SAndreas Gruenbacher drbd_rs_failed_io(device, peer_req->i.sector, peer_req->i.size); 1158b411b363SPhilipp Reisner } 1159b411b363SPhilipp Reisner 1160b30ab791SAndreas Gruenbacher dec_unacked(device); 1161b411b363SPhilipp Reisner 1162b30ab791SAndreas Gruenbacher move_to_net_ee_or_free(device, peer_req); 1163b411b363SPhilipp Reisner 116499920dc5SAndreas Gruenbacher if (unlikely(err)) 1165d0180171SAndreas Gruenbacher drbd_err(device, "drbd_send_block() failed\n"); 116699920dc5SAndreas Gruenbacher return err; 1167b411b363SPhilipp Reisner } 1168b411b363SPhilipp Reisner 116999920dc5SAndreas Gruenbacher int w_e_end_csum_rs_req(struct drbd_work *w, int cancel) 1170b411b363SPhilipp Reisner { 1171a8cd15baSAndreas Gruenbacher struct drbd_peer_request *peer_req = container_of(w, struct drbd_peer_request, w); 11726780139cSAndreas Gruenbacher struct drbd_peer_device *peer_device = peer_req->peer_device; 11736780139cSAndreas Gruenbacher struct drbd_device *device = peer_device->device; 1174b411b363SPhilipp Reisner struct digest_info *di; 1175b411b363SPhilipp Reisner int digest_size; 1176b411b363SPhilipp Reisner void *digest = NULL; 117799920dc5SAndreas Gruenbacher int err, eq = 0; 1178b411b363SPhilipp Reisner 1179b411b363SPhilipp Reisner if (unlikely(cancel)) { 1180b30ab791SAndreas Gruenbacher drbd_free_peer_req(device, peer_req); 1181b30ab791SAndreas Gruenbacher dec_unacked(device); 118299920dc5SAndreas Gruenbacher return 0; 1183b411b363SPhilipp Reisner } 1184b411b363SPhilipp Reisner 1185b30ab791SAndreas Gruenbacher if (get_ldev(device)) { 1186b30ab791SAndreas Gruenbacher drbd_rs_complete_io(device, peer_req->i.sector); 1187b30ab791SAndreas Gruenbacher put_ldev(device); 11881d53f09eSLars Ellenberg } 1189b411b363SPhilipp Reisner 1190db830c46SAndreas Gruenbacher di = peer_req->digest; 1191b411b363SPhilipp Reisner 1192db830c46SAndreas Gruenbacher if (likely((peer_req->flags & EE_WAS_ERROR) == 0)) { 1193b411b363SPhilipp Reisner /* quick hack to try to avoid a race against reconfiguration. 1194b411b363SPhilipp Reisner * a real fix would be much more involved, 1195b411b363SPhilipp Reisner * introducing more locking mechanisms */ 11966780139cSAndreas Gruenbacher if (peer_device->connection->csums_tfm) { 11979534d671SHerbert Xu digest_size = crypto_ahash_digestsize(peer_device->connection->csums_tfm); 11980b0ba1efSAndreas Gruenbacher D_ASSERT(device, digest_size == di->digest_size); 1199b411b363SPhilipp Reisner digest = kmalloc(digest_size, GFP_NOIO); 1200b411b363SPhilipp Reisner } 1201b411b363SPhilipp Reisner if (digest) { 12026780139cSAndreas Gruenbacher drbd_csum_ee(peer_device->connection->csums_tfm, peer_req, digest); 1203b411b363SPhilipp Reisner eq = !memcmp(digest, di->digest, digest_size); 1204b411b363SPhilipp Reisner kfree(digest); 1205b411b363SPhilipp Reisner } 1206b411b363SPhilipp Reisner 1207b411b363SPhilipp Reisner if (eq) { 1208b30ab791SAndreas Gruenbacher drbd_set_in_sync(device, peer_req->i.sector, peer_req->i.size); 1209676396d5SLars Ellenberg /* rs_same_csums unit is BM_BLOCK_SIZE */ 1210b30ab791SAndreas Gruenbacher device->rs_same_csum += peer_req->i.size >> BM_BLOCK_SHIFT; 12116780139cSAndreas Gruenbacher err = drbd_send_ack(peer_device, P_RS_IS_IN_SYNC, peer_req); 1212b411b363SPhilipp Reisner } else { 1213b30ab791SAndreas Gruenbacher inc_rs_pending(device); 1214db830c46SAndreas Gruenbacher peer_req->block_id = ID_SYNCER; /* By setting block_id, digest pointer becomes invalid! */ 1215db830c46SAndreas Gruenbacher peer_req->flags &= ~EE_HAS_DIGEST; /* This peer request no longer has a digest pointer */ 1216204bba99SPhilipp Reisner kfree(di); 12176780139cSAndreas Gruenbacher err = drbd_send_block(peer_device, P_RS_DATA_REPLY, peer_req); 1218b411b363SPhilipp Reisner } 1219b411b363SPhilipp Reisner } else { 12206780139cSAndreas Gruenbacher err = drbd_send_ack(peer_device, P_NEG_RS_DREPLY, peer_req); 1221b411b363SPhilipp Reisner if (__ratelimit(&drbd_ratelimit_state)) 1222d0180171SAndreas Gruenbacher drbd_err(device, "Sending NegDReply. I guess it gets messy.\n"); 1223b411b363SPhilipp Reisner } 1224b411b363SPhilipp Reisner 1225b30ab791SAndreas Gruenbacher dec_unacked(device); 1226b30ab791SAndreas Gruenbacher move_to_net_ee_or_free(device, peer_req); 1227b411b363SPhilipp Reisner 122899920dc5SAndreas Gruenbacher if (unlikely(err)) 1229d0180171SAndreas Gruenbacher drbd_err(device, "drbd_send_block/ack() failed\n"); 123099920dc5SAndreas Gruenbacher return err; 1231b411b363SPhilipp Reisner } 1232b411b363SPhilipp Reisner 123399920dc5SAndreas Gruenbacher int w_e_end_ov_req(struct drbd_work *w, int cancel) 1234b411b363SPhilipp Reisner { 1235a8cd15baSAndreas Gruenbacher struct drbd_peer_request *peer_req = container_of(w, struct drbd_peer_request, w); 12366780139cSAndreas Gruenbacher struct drbd_peer_device *peer_device = peer_req->peer_device; 12376780139cSAndreas Gruenbacher struct drbd_device *device = peer_device->device; 1238db830c46SAndreas Gruenbacher sector_t sector = peer_req->i.sector; 1239db830c46SAndreas Gruenbacher unsigned int size = peer_req->i.size; 1240b411b363SPhilipp Reisner int digest_size; 1241b411b363SPhilipp Reisner void *digest; 124299920dc5SAndreas Gruenbacher int err = 0; 1243b411b363SPhilipp Reisner 1244b411b363SPhilipp Reisner if (unlikely(cancel)) 1245b411b363SPhilipp Reisner goto out; 1246b411b363SPhilipp Reisner 12479534d671SHerbert Xu digest_size = crypto_ahash_digestsize(peer_device->connection->verify_tfm); 1248b411b363SPhilipp Reisner digest = kmalloc(digest_size, GFP_NOIO); 12498f21420eSPhilipp Reisner if (!digest) { 125099920dc5SAndreas Gruenbacher err = 1; /* terminate the connection in case the allocation failed */ 12518f21420eSPhilipp Reisner goto out; 12528f21420eSPhilipp Reisner } 12538f21420eSPhilipp Reisner 1254db830c46SAndreas Gruenbacher if (likely(!(peer_req->flags & EE_WAS_ERROR))) 12556780139cSAndreas Gruenbacher drbd_csum_ee(peer_device->connection->verify_tfm, peer_req, digest); 12568f21420eSPhilipp Reisner else 12578f21420eSPhilipp Reisner memset(digest, 0, digest_size); 12588f21420eSPhilipp Reisner 125953ea4331SLars Ellenberg /* Free e and pages before send. 126053ea4331SLars Ellenberg * In case we block on congestion, we could otherwise run into 126153ea4331SLars Ellenberg * some distributed deadlock, if the other side blocks on 126253ea4331SLars Ellenberg * congestion as well, because our receiver blocks in 1263c37c8ecfSAndreas Gruenbacher * drbd_alloc_pages due to pp_in_use > max_buffers. */ 1264b30ab791SAndreas Gruenbacher drbd_free_peer_req(device, peer_req); 1265db830c46SAndreas Gruenbacher peer_req = NULL; 1266b30ab791SAndreas Gruenbacher inc_rs_pending(device); 12676780139cSAndreas Gruenbacher err = drbd_send_drequest_csum(peer_device, sector, size, digest, digest_size, P_OV_REPLY); 126899920dc5SAndreas Gruenbacher if (err) 1269b30ab791SAndreas Gruenbacher dec_rs_pending(device); 1270b411b363SPhilipp Reisner kfree(digest); 1271b411b363SPhilipp Reisner 1272b411b363SPhilipp Reisner out: 1273db830c46SAndreas Gruenbacher if (peer_req) 1274b30ab791SAndreas Gruenbacher drbd_free_peer_req(device, peer_req); 1275b30ab791SAndreas Gruenbacher dec_unacked(device); 127699920dc5SAndreas Gruenbacher return err; 1277b411b363SPhilipp Reisner } 1278b411b363SPhilipp Reisner 1279b30ab791SAndreas Gruenbacher void drbd_ov_out_of_sync_found(struct drbd_device *device, sector_t sector, int size) 1280b411b363SPhilipp Reisner { 1281b30ab791SAndreas Gruenbacher if (device->ov_last_oos_start + device->ov_last_oos_size == sector) { 1282b30ab791SAndreas Gruenbacher device->ov_last_oos_size += size>>9; 1283b411b363SPhilipp Reisner } else { 1284b30ab791SAndreas Gruenbacher device->ov_last_oos_start = sector; 1285b30ab791SAndreas Gruenbacher device->ov_last_oos_size = size>>9; 1286b411b363SPhilipp Reisner } 1287b30ab791SAndreas Gruenbacher drbd_set_out_of_sync(device, sector, size); 1288b411b363SPhilipp Reisner } 1289b411b363SPhilipp Reisner 129099920dc5SAndreas Gruenbacher int w_e_end_ov_reply(struct drbd_work *w, int cancel) 1291b411b363SPhilipp Reisner { 1292a8cd15baSAndreas Gruenbacher struct drbd_peer_request *peer_req = container_of(w, struct drbd_peer_request, w); 12936780139cSAndreas Gruenbacher struct drbd_peer_device *peer_device = peer_req->peer_device; 12946780139cSAndreas Gruenbacher struct drbd_device *device = peer_device->device; 1295b411b363SPhilipp Reisner struct digest_info *di; 1296b411b363SPhilipp Reisner void *digest; 1297db830c46SAndreas Gruenbacher sector_t sector = peer_req->i.sector; 1298db830c46SAndreas Gruenbacher unsigned int size = peer_req->i.size; 129953ea4331SLars Ellenberg int digest_size; 130099920dc5SAndreas Gruenbacher int err, eq = 0; 130158ffa580SLars Ellenberg bool stop_sector_reached = false; 1302b411b363SPhilipp Reisner 1303b411b363SPhilipp Reisner if (unlikely(cancel)) { 1304b30ab791SAndreas Gruenbacher drbd_free_peer_req(device, peer_req); 1305b30ab791SAndreas Gruenbacher dec_unacked(device); 130699920dc5SAndreas Gruenbacher return 0; 1307b411b363SPhilipp Reisner } 1308b411b363SPhilipp Reisner 1309b411b363SPhilipp Reisner /* after "cancel", because after drbd_disconnect/drbd_rs_cancel_all 1310b411b363SPhilipp Reisner * the resync lru has been cleaned up already */ 1311b30ab791SAndreas Gruenbacher if (get_ldev(device)) { 1312b30ab791SAndreas Gruenbacher drbd_rs_complete_io(device, peer_req->i.sector); 1313b30ab791SAndreas Gruenbacher put_ldev(device); 13141d53f09eSLars Ellenberg } 1315b411b363SPhilipp Reisner 1316db830c46SAndreas Gruenbacher di = peer_req->digest; 1317b411b363SPhilipp Reisner 1318db830c46SAndreas Gruenbacher if (likely((peer_req->flags & EE_WAS_ERROR) == 0)) { 13199534d671SHerbert Xu digest_size = crypto_ahash_digestsize(peer_device->connection->verify_tfm); 1320b411b363SPhilipp Reisner digest = kmalloc(digest_size, GFP_NOIO); 1321b411b363SPhilipp Reisner if (digest) { 13226780139cSAndreas Gruenbacher drbd_csum_ee(peer_device->connection->verify_tfm, peer_req, digest); 1323b411b363SPhilipp Reisner 13240b0ba1efSAndreas Gruenbacher D_ASSERT(device, digest_size == di->digest_size); 1325b411b363SPhilipp Reisner eq = !memcmp(digest, di->digest, digest_size); 1326b411b363SPhilipp Reisner kfree(digest); 1327b411b363SPhilipp Reisner } 1328b411b363SPhilipp Reisner } 1329b411b363SPhilipp Reisner 13309676c760SLars Ellenberg /* Free peer_req and pages before send. 133153ea4331SLars Ellenberg * In case we block on congestion, we could otherwise run into 133253ea4331SLars Ellenberg * some distributed deadlock, if the other side blocks on 133353ea4331SLars Ellenberg * congestion as well, because our receiver blocks in 1334c37c8ecfSAndreas Gruenbacher * drbd_alloc_pages due to pp_in_use > max_buffers. */ 1335b30ab791SAndreas Gruenbacher drbd_free_peer_req(device, peer_req); 1336b411b363SPhilipp Reisner if (!eq) 1337b30ab791SAndreas Gruenbacher drbd_ov_out_of_sync_found(device, sector, size); 1338b411b363SPhilipp Reisner else 1339b30ab791SAndreas Gruenbacher ov_out_of_sync_print(device); 1340b411b363SPhilipp Reisner 13416780139cSAndreas Gruenbacher err = drbd_send_ack_ex(peer_device, P_OV_RESULT, sector, size, 1342b411b363SPhilipp Reisner eq ? ID_IN_SYNC : ID_OUT_OF_SYNC); 1343b411b363SPhilipp Reisner 1344b30ab791SAndreas Gruenbacher dec_unacked(device); 1345b411b363SPhilipp Reisner 1346b30ab791SAndreas Gruenbacher --device->ov_left; 1347ea5442afSLars Ellenberg 1348ea5442afSLars Ellenberg /* let's advance progress step marks only for every other megabyte */ 1349b30ab791SAndreas Gruenbacher if ((device->ov_left & 0x200) == 0x200) 1350b30ab791SAndreas Gruenbacher drbd_advance_rs_marks(device, device->ov_left); 1351ea5442afSLars Ellenberg 1352b30ab791SAndreas Gruenbacher stop_sector_reached = verify_can_do_stop_sector(device) && 1353b30ab791SAndreas Gruenbacher (sector + (size>>9)) >= device->ov_stop_sector; 135458ffa580SLars Ellenberg 1355b30ab791SAndreas Gruenbacher if (device->ov_left == 0 || stop_sector_reached) { 1356b30ab791SAndreas Gruenbacher ov_out_of_sync_print(device); 1357b30ab791SAndreas Gruenbacher drbd_resync_finished(device); 1358b411b363SPhilipp Reisner } 1359b411b363SPhilipp Reisner 136099920dc5SAndreas Gruenbacher return err; 1361b411b363SPhilipp Reisner } 1362b411b363SPhilipp Reisner 1363b6dd1a89SLars Ellenberg /* FIXME 1364b6dd1a89SLars Ellenberg * We need to track the number of pending barrier acks, 1365b6dd1a89SLars Ellenberg * and to be able to wait for them. 1366b6dd1a89SLars Ellenberg * See also comment in drbd_adm_attach before drbd_suspend_io. 1367b6dd1a89SLars Ellenberg */ 1368bde89a9eSAndreas Gruenbacher static int drbd_send_barrier(struct drbd_connection *connection) 1369b411b363SPhilipp Reisner { 13709f5bdc33SAndreas Gruenbacher struct p_barrier *p; 1371b6dd1a89SLars Ellenberg struct drbd_socket *sock; 1372b411b363SPhilipp Reisner 1373bde89a9eSAndreas Gruenbacher sock = &connection->data; 1374bde89a9eSAndreas Gruenbacher p = conn_prepare_command(connection, sock); 13759f5bdc33SAndreas Gruenbacher if (!p) 13769f5bdc33SAndreas Gruenbacher return -EIO; 1377bde89a9eSAndreas Gruenbacher p->barrier = connection->send.current_epoch_nr; 1378b6dd1a89SLars Ellenberg p->pad = 0; 1379bde89a9eSAndreas Gruenbacher connection->send.current_epoch_writes = 0; 138084d34f2fSLars Ellenberg connection->send.last_sent_barrier_jif = jiffies; 1381b6dd1a89SLars Ellenberg 1382bde89a9eSAndreas Gruenbacher return conn_send_command(connection, sock, P_BARRIER, sizeof(*p), NULL, 0); 1383b411b363SPhilipp Reisner } 1384b411b363SPhilipp Reisner 138599920dc5SAndreas Gruenbacher int w_send_write_hint(struct drbd_work *w, int cancel) 1386b411b363SPhilipp Reisner { 138784b8c06bSAndreas Gruenbacher struct drbd_device *device = 138884b8c06bSAndreas Gruenbacher container_of(w, struct drbd_device, unplug_work); 13899f5bdc33SAndreas Gruenbacher struct drbd_socket *sock; 13909f5bdc33SAndreas Gruenbacher 1391b411b363SPhilipp Reisner if (cancel) 139299920dc5SAndreas Gruenbacher return 0; 1393a6b32bc3SAndreas Gruenbacher sock = &first_peer_device(device)->connection->data; 139469a22773SAndreas Gruenbacher if (!drbd_prepare_command(first_peer_device(device), sock)) 13959f5bdc33SAndreas Gruenbacher return -EIO; 139669a22773SAndreas Gruenbacher return drbd_send_command(first_peer_device(device), sock, P_UNPLUG_REMOTE, 0, NULL, 0); 1397b411b363SPhilipp Reisner } 1398b411b363SPhilipp Reisner 1399bde89a9eSAndreas Gruenbacher static void re_init_if_first_write(struct drbd_connection *connection, unsigned int epoch) 14004eb9b3cbSLars Ellenberg { 1401bde89a9eSAndreas Gruenbacher if (!connection->send.seen_any_write_yet) { 1402bde89a9eSAndreas Gruenbacher connection->send.seen_any_write_yet = true; 1403bde89a9eSAndreas Gruenbacher connection->send.current_epoch_nr = epoch; 1404bde89a9eSAndreas Gruenbacher connection->send.current_epoch_writes = 0; 140584d34f2fSLars Ellenberg connection->send.last_sent_barrier_jif = jiffies; 14064eb9b3cbSLars Ellenberg } 14074eb9b3cbSLars Ellenberg } 14084eb9b3cbSLars Ellenberg 1409bde89a9eSAndreas Gruenbacher static void maybe_send_barrier(struct drbd_connection *connection, unsigned int epoch) 14104eb9b3cbSLars Ellenberg { 14114eb9b3cbSLars Ellenberg /* re-init if first write on this connection */ 1412bde89a9eSAndreas Gruenbacher if (!connection->send.seen_any_write_yet) 14134eb9b3cbSLars Ellenberg return; 1414bde89a9eSAndreas Gruenbacher if (connection->send.current_epoch_nr != epoch) { 1415bde89a9eSAndreas Gruenbacher if (connection->send.current_epoch_writes) 1416bde89a9eSAndreas Gruenbacher drbd_send_barrier(connection); 1417bde89a9eSAndreas Gruenbacher connection->send.current_epoch_nr = epoch; 14184eb9b3cbSLars Ellenberg } 14194eb9b3cbSLars Ellenberg } 14204eb9b3cbSLars Ellenberg 14218f7bed77SAndreas Gruenbacher int w_send_out_of_sync(struct drbd_work *w, int cancel) 142273a01a18SPhilipp Reisner { 142373a01a18SPhilipp Reisner struct drbd_request *req = container_of(w, struct drbd_request, w); 142484b8c06bSAndreas Gruenbacher struct drbd_device *device = req->device; 142544a4d551SLars Ellenberg struct drbd_peer_device *const peer_device = first_peer_device(device); 142644a4d551SLars Ellenberg struct drbd_connection *const connection = peer_device->connection; 142799920dc5SAndreas Gruenbacher int err; 142873a01a18SPhilipp Reisner 142973a01a18SPhilipp Reisner if (unlikely(cancel)) { 14308554df1cSAndreas Gruenbacher req_mod(req, SEND_CANCELED); 143199920dc5SAndreas Gruenbacher return 0; 143273a01a18SPhilipp Reisner } 1433e5f891b2SLars Ellenberg req->pre_send_jif = jiffies; 143473a01a18SPhilipp Reisner 1435bde89a9eSAndreas Gruenbacher /* this time, no connection->send.current_epoch_writes++; 1436b6dd1a89SLars Ellenberg * If it was sent, it was the closing barrier for the last 1437b6dd1a89SLars Ellenberg * replicated epoch, before we went into AHEAD mode. 1438b6dd1a89SLars Ellenberg * No more barriers will be sent, until we leave AHEAD mode again. */ 1439bde89a9eSAndreas Gruenbacher maybe_send_barrier(connection, req->epoch); 1440b6dd1a89SLars Ellenberg 144144a4d551SLars Ellenberg err = drbd_send_out_of_sync(peer_device, req); 14428554df1cSAndreas Gruenbacher req_mod(req, OOS_HANDED_TO_NETWORK); 144373a01a18SPhilipp Reisner 144499920dc5SAndreas Gruenbacher return err; 144573a01a18SPhilipp Reisner } 144673a01a18SPhilipp Reisner 1447b411b363SPhilipp Reisner /** 1448b411b363SPhilipp Reisner * w_send_dblock() - Worker callback to send a P_DATA packet in order to mirror a write request 1449b411b363SPhilipp Reisner * @w: work object. 1450b411b363SPhilipp Reisner * @cancel: The connection will be closed anyways 1451b411b363SPhilipp Reisner */ 145299920dc5SAndreas Gruenbacher int w_send_dblock(struct drbd_work *w, int cancel) 1453b411b363SPhilipp Reisner { 1454b411b363SPhilipp Reisner struct drbd_request *req = container_of(w, struct drbd_request, w); 145584b8c06bSAndreas Gruenbacher struct drbd_device *device = req->device; 145644a4d551SLars Ellenberg struct drbd_peer_device *const peer_device = first_peer_device(device); 145744a4d551SLars Ellenberg struct drbd_connection *connection = peer_device->connection; 145899920dc5SAndreas Gruenbacher int err; 1459b411b363SPhilipp Reisner 1460b411b363SPhilipp Reisner if (unlikely(cancel)) { 14618554df1cSAndreas Gruenbacher req_mod(req, SEND_CANCELED); 146299920dc5SAndreas Gruenbacher return 0; 1463b411b363SPhilipp Reisner } 1464e5f891b2SLars Ellenberg req->pre_send_jif = jiffies; 1465b411b363SPhilipp Reisner 1466bde89a9eSAndreas Gruenbacher re_init_if_first_write(connection, req->epoch); 1467bde89a9eSAndreas Gruenbacher maybe_send_barrier(connection, req->epoch); 1468bde89a9eSAndreas Gruenbacher connection->send.current_epoch_writes++; 1469b6dd1a89SLars Ellenberg 147044a4d551SLars Ellenberg err = drbd_send_dblock(peer_device, req); 147199920dc5SAndreas Gruenbacher req_mod(req, err ? SEND_FAILED : HANDED_OVER_TO_NETWORK); 1472b411b363SPhilipp Reisner 147399920dc5SAndreas Gruenbacher return err; 1474b411b363SPhilipp Reisner } 1475b411b363SPhilipp Reisner 1476b411b363SPhilipp Reisner /** 1477b411b363SPhilipp Reisner * w_send_read_req() - Worker callback to send a read request (P_DATA_REQUEST) packet 1478b411b363SPhilipp Reisner * @w: work object. 1479b411b363SPhilipp Reisner * @cancel: The connection will be closed anyways 1480b411b363SPhilipp Reisner */ 148199920dc5SAndreas Gruenbacher int w_send_read_req(struct drbd_work *w, int cancel) 1482b411b363SPhilipp Reisner { 1483b411b363SPhilipp Reisner struct drbd_request *req = container_of(w, struct drbd_request, w); 148484b8c06bSAndreas Gruenbacher struct drbd_device *device = req->device; 148544a4d551SLars Ellenberg struct drbd_peer_device *const peer_device = first_peer_device(device); 148644a4d551SLars Ellenberg struct drbd_connection *connection = peer_device->connection; 148799920dc5SAndreas Gruenbacher int err; 1488b411b363SPhilipp Reisner 1489b411b363SPhilipp Reisner if (unlikely(cancel)) { 14908554df1cSAndreas Gruenbacher req_mod(req, SEND_CANCELED); 149199920dc5SAndreas Gruenbacher return 0; 1492b411b363SPhilipp Reisner } 1493e5f891b2SLars Ellenberg req->pre_send_jif = jiffies; 1494b411b363SPhilipp Reisner 1495b6dd1a89SLars Ellenberg /* Even read requests may close a write epoch, 1496b6dd1a89SLars Ellenberg * if there was any yet. */ 1497bde89a9eSAndreas Gruenbacher maybe_send_barrier(connection, req->epoch); 1498b6dd1a89SLars Ellenberg 149944a4d551SLars Ellenberg err = drbd_send_drequest(peer_device, P_DATA_REQUEST, req->i.sector, req->i.size, 1500b411b363SPhilipp Reisner (unsigned long)req); 1501b411b363SPhilipp Reisner 150299920dc5SAndreas Gruenbacher req_mod(req, err ? SEND_FAILED : HANDED_OVER_TO_NETWORK); 1503b411b363SPhilipp Reisner 150499920dc5SAndreas Gruenbacher return err; 1505b411b363SPhilipp Reisner } 1506b411b363SPhilipp Reisner 150799920dc5SAndreas Gruenbacher int w_restart_disk_io(struct drbd_work *w, int cancel) 1508265be2d0SPhilipp Reisner { 1509265be2d0SPhilipp Reisner struct drbd_request *req = container_of(w, struct drbd_request, w); 151084b8c06bSAndreas Gruenbacher struct drbd_device *device = req->device; 1511265be2d0SPhilipp Reisner 15120778286aSPhilipp Reisner if (bio_data_dir(req->master_bio) == WRITE && req->rq_state & RQ_IN_ACT_LOG) 15134dd726f0SLars Ellenberg drbd_al_begin_io(device, &req->i); 1514265be2d0SPhilipp Reisner 1515265be2d0SPhilipp Reisner drbd_req_make_private_bio(req, req->master_bio); 1516b30ab791SAndreas Gruenbacher req->private_bio->bi_bdev = device->ldev->backing_bdev; 1517265be2d0SPhilipp Reisner generic_make_request(req->private_bio); 1518265be2d0SPhilipp Reisner 151999920dc5SAndreas Gruenbacher return 0; 1520265be2d0SPhilipp Reisner } 1521265be2d0SPhilipp Reisner 1522b30ab791SAndreas Gruenbacher static int _drbd_may_sync_now(struct drbd_device *device) 1523b411b363SPhilipp Reisner { 1524b30ab791SAndreas Gruenbacher struct drbd_device *odev = device; 152595f8efd0SAndreas Gruenbacher int resync_after; 1526b411b363SPhilipp Reisner 1527b411b363SPhilipp Reisner while (1) { 1528a3f8f7dcSLars Ellenberg if (!odev->ldev || odev->state.disk == D_DISKLESS) 1529438c8374SPhilipp Reisner return 1; 1530daeda1ccSPhilipp Reisner rcu_read_lock(); 153195f8efd0SAndreas Gruenbacher resync_after = rcu_dereference(odev->ldev->disk_conf)->resync_after; 1532daeda1ccSPhilipp Reisner rcu_read_unlock(); 153395f8efd0SAndreas Gruenbacher if (resync_after == -1) 1534b411b363SPhilipp Reisner return 1; 1535b30ab791SAndreas Gruenbacher odev = minor_to_device(resync_after); 1536a3f8f7dcSLars Ellenberg if (!odev) 1537841ce241SAndreas Gruenbacher return 1; 1538b411b363SPhilipp Reisner if ((odev->state.conn >= C_SYNC_SOURCE && 1539b411b363SPhilipp Reisner odev->state.conn <= C_PAUSED_SYNC_T) || 1540b411b363SPhilipp Reisner odev->state.aftr_isp || odev->state.peer_isp || 1541b411b363SPhilipp Reisner odev->state.user_isp) 1542b411b363SPhilipp Reisner return 0; 1543b411b363SPhilipp Reisner } 1544b411b363SPhilipp Reisner } 1545b411b363SPhilipp Reisner 1546b411b363SPhilipp Reisner /** 154728bc3b8cSAndreas Gruenbacher * drbd_pause_after() - Pause resync on all devices that may not resync now 1548b30ab791SAndreas Gruenbacher * @device: DRBD device. 1549b411b363SPhilipp Reisner * 1550b411b363SPhilipp Reisner * Called from process context only (admin command and after_state_ch). 1551b411b363SPhilipp Reisner */ 155228bc3b8cSAndreas Gruenbacher static bool drbd_pause_after(struct drbd_device *device) 1553b411b363SPhilipp Reisner { 155428bc3b8cSAndreas Gruenbacher bool changed = false; 155554761697SAndreas Gruenbacher struct drbd_device *odev; 155628bc3b8cSAndreas Gruenbacher int i; 1557b411b363SPhilipp Reisner 1558695d08faSPhilipp Reisner rcu_read_lock(); 155905a10ec7SAndreas Gruenbacher idr_for_each_entry(&drbd_devices, odev, i) { 1560b411b363SPhilipp Reisner if (odev->state.conn == C_STANDALONE && odev->state.disk == D_DISKLESS) 1561b411b363SPhilipp Reisner continue; 156228bc3b8cSAndreas Gruenbacher if (!_drbd_may_sync_now(odev) && 156328bc3b8cSAndreas Gruenbacher _drbd_set_state(_NS(odev, aftr_isp, 1), 156428bc3b8cSAndreas Gruenbacher CS_HARD, NULL) != SS_NOTHING_TO_DO) 156528bc3b8cSAndreas Gruenbacher changed = true; 1566b411b363SPhilipp Reisner } 1567695d08faSPhilipp Reisner rcu_read_unlock(); 1568b411b363SPhilipp Reisner 156928bc3b8cSAndreas Gruenbacher return changed; 1570b411b363SPhilipp Reisner } 1571b411b363SPhilipp Reisner 1572b411b363SPhilipp Reisner /** 157328bc3b8cSAndreas Gruenbacher * drbd_resume_next() - Resume resync on all devices that may resync now 1574b30ab791SAndreas Gruenbacher * @device: DRBD device. 1575b411b363SPhilipp Reisner * 1576b411b363SPhilipp Reisner * Called from process context only (admin command and worker). 1577b411b363SPhilipp Reisner */ 157828bc3b8cSAndreas Gruenbacher static bool drbd_resume_next(struct drbd_device *device) 1579b411b363SPhilipp Reisner { 158028bc3b8cSAndreas Gruenbacher bool changed = false; 158154761697SAndreas Gruenbacher struct drbd_device *odev; 158228bc3b8cSAndreas Gruenbacher int i; 1583b411b363SPhilipp Reisner 1584695d08faSPhilipp Reisner rcu_read_lock(); 158505a10ec7SAndreas Gruenbacher idr_for_each_entry(&drbd_devices, odev, i) { 1586b411b363SPhilipp Reisner if (odev->state.conn == C_STANDALONE && odev->state.disk == D_DISKLESS) 1587b411b363SPhilipp Reisner continue; 1588b411b363SPhilipp Reisner if (odev->state.aftr_isp) { 158928bc3b8cSAndreas Gruenbacher if (_drbd_may_sync_now(odev) && 159028bc3b8cSAndreas Gruenbacher _drbd_set_state(_NS(odev, aftr_isp, 0), 159128bc3b8cSAndreas Gruenbacher CS_HARD, NULL) != SS_NOTHING_TO_DO) 159228bc3b8cSAndreas Gruenbacher changed = true; 1593b411b363SPhilipp Reisner } 1594b411b363SPhilipp Reisner } 1595695d08faSPhilipp Reisner rcu_read_unlock(); 159628bc3b8cSAndreas Gruenbacher return changed; 1597b411b363SPhilipp Reisner } 1598b411b363SPhilipp Reisner 1599b30ab791SAndreas Gruenbacher void resume_next_sg(struct drbd_device *device) 1600b411b363SPhilipp Reisner { 160128bc3b8cSAndreas Gruenbacher lock_all_resources(); 160228bc3b8cSAndreas Gruenbacher drbd_resume_next(device); 160328bc3b8cSAndreas Gruenbacher unlock_all_resources(); 1604b411b363SPhilipp Reisner } 1605b411b363SPhilipp Reisner 1606b30ab791SAndreas Gruenbacher void suspend_other_sg(struct drbd_device *device) 1607b411b363SPhilipp Reisner { 160828bc3b8cSAndreas Gruenbacher lock_all_resources(); 160928bc3b8cSAndreas Gruenbacher drbd_pause_after(device); 161028bc3b8cSAndreas Gruenbacher unlock_all_resources(); 1611b411b363SPhilipp Reisner } 1612b411b363SPhilipp Reisner 161328bc3b8cSAndreas Gruenbacher /* caller must lock_all_resources() */ 1614b30ab791SAndreas Gruenbacher enum drbd_ret_code drbd_resync_after_valid(struct drbd_device *device, int o_minor) 1615b411b363SPhilipp Reisner { 161654761697SAndreas Gruenbacher struct drbd_device *odev; 161795f8efd0SAndreas Gruenbacher int resync_after; 1618b411b363SPhilipp Reisner 1619b411b363SPhilipp Reisner if (o_minor == -1) 1620b411b363SPhilipp Reisner return NO_ERROR; 1621a3f8f7dcSLars Ellenberg if (o_minor < -1 || o_minor > MINORMASK) 162295f8efd0SAndreas Gruenbacher return ERR_RESYNC_AFTER; 1623b411b363SPhilipp Reisner 1624b411b363SPhilipp Reisner /* check for loops */ 1625b30ab791SAndreas Gruenbacher odev = minor_to_device(o_minor); 1626b411b363SPhilipp Reisner while (1) { 1627b30ab791SAndreas Gruenbacher if (odev == device) 162895f8efd0SAndreas Gruenbacher return ERR_RESYNC_AFTER_CYCLE; 1629b411b363SPhilipp Reisner 1630a3f8f7dcSLars Ellenberg /* You are free to depend on diskless, non-existing, 1631a3f8f7dcSLars Ellenberg * or not yet/no longer existing minors. 1632a3f8f7dcSLars Ellenberg * We only reject dependency loops. 1633a3f8f7dcSLars Ellenberg * We cannot follow the dependency chain beyond a detached or 1634a3f8f7dcSLars Ellenberg * missing minor. 1635a3f8f7dcSLars Ellenberg */ 1636a3f8f7dcSLars Ellenberg if (!odev || !odev->ldev || odev->state.disk == D_DISKLESS) 1637a3f8f7dcSLars Ellenberg return NO_ERROR; 1638a3f8f7dcSLars Ellenberg 1639daeda1ccSPhilipp Reisner rcu_read_lock(); 164095f8efd0SAndreas Gruenbacher resync_after = rcu_dereference(odev->ldev->disk_conf)->resync_after; 1641daeda1ccSPhilipp Reisner rcu_read_unlock(); 1642b411b363SPhilipp Reisner /* dependency chain ends here, no cycles. */ 164395f8efd0SAndreas Gruenbacher if (resync_after == -1) 1644b411b363SPhilipp Reisner return NO_ERROR; 1645b411b363SPhilipp Reisner 1646b411b363SPhilipp Reisner /* follow the dependency chain */ 1647b30ab791SAndreas Gruenbacher odev = minor_to_device(resync_after); 1648b411b363SPhilipp Reisner } 1649b411b363SPhilipp Reisner } 1650b411b363SPhilipp Reisner 165128bc3b8cSAndreas Gruenbacher /* caller must lock_all_resources() */ 1652b30ab791SAndreas Gruenbacher void drbd_resync_after_changed(struct drbd_device *device) 1653b411b363SPhilipp Reisner { 165428bc3b8cSAndreas Gruenbacher int changed; 1655b411b363SPhilipp Reisner 1656b411b363SPhilipp Reisner do { 165728bc3b8cSAndreas Gruenbacher changed = drbd_pause_after(device); 165828bc3b8cSAndreas Gruenbacher changed |= drbd_resume_next(device); 165928bc3b8cSAndreas Gruenbacher } while (changed); 1660b411b363SPhilipp Reisner } 1661b411b363SPhilipp Reisner 1662b30ab791SAndreas Gruenbacher void drbd_rs_controller_reset(struct drbd_device *device) 16639bd28d3cSLars Ellenberg { 1664ff8bd88bSLars Ellenberg struct gendisk *disk = device->ldev->backing_bdev->bd_contains->bd_disk; 1665813472ceSPhilipp Reisner struct fifo_buffer *plan; 1666813472ceSPhilipp Reisner 1667b30ab791SAndreas Gruenbacher atomic_set(&device->rs_sect_in, 0); 1668b30ab791SAndreas Gruenbacher atomic_set(&device->rs_sect_ev, 0); 1669b30ab791SAndreas Gruenbacher device->rs_in_flight = 0; 1670ff8bd88bSLars Ellenberg device->rs_last_events = 1671ff8bd88bSLars Ellenberg (int)part_stat_read(&disk->part0, sectors[0]) + 1672ff8bd88bSLars Ellenberg (int)part_stat_read(&disk->part0, sectors[1]); 1673813472ceSPhilipp Reisner 1674813472ceSPhilipp Reisner /* Updating the RCU protected object in place is necessary since 1675813472ceSPhilipp Reisner this function gets called from atomic context. 1676813472ceSPhilipp Reisner It is valid since all other updates also lead to an completely 1677813472ceSPhilipp Reisner empty fifo */ 1678813472ceSPhilipp Reisner rcu_read_lock(); 1679b30ab791SAndreas Gruenbacher plan = rcu_dereference(device->rs_plan_s); 1680813472ceSPhilipp Reisner plan->total = 0; 1681813472ceSPhilipp Reisner fifo_set(plan, 0); 1682813472ceSPhilipp Reisner rcu_read_unlock(); 16839bd28d3cSLars Ellenberg } 16849bd28d3cSLars Ellenberg 16851f04af33SPhilipp Reisner void start_resync_timer_fn(unsigned long data) 16861f04af33SPhilipp Reisner { 1687b30ab791SAndreas Gruenbacher struct drbd_device *device = (struct drbd_device *) data; 1688ac0acb9eSLars Ellenberg drbd_device_post_work(device, RS_START); 16891f04af33SPhilipp Reisner } 16901f04af33SPhilipp Reisner 1691ac0acb9eSLars Ellenberg static void do_start_resync(struct drbd_device *device) 16921f04af33SPhilipp Reisner { 1693b30ab791SAndreas Gruenbacher if (atomic_read(&device->unacked_cnt) || atomic_read(&device->rs_pending_cnt)) { 1694ac0acb9eSLars Ellenberg drbd_warn(device, "postponing start_resync ...\n"); 1695b30ab791SAndreas Gruenbacher device->start_resync_timer.expires = jiffies + HZ/10; 1696b30ab791SAndreas Gruenbacher add_timer(&device->start_resync_timer); 1697ac0acb9eSLars Ellenberg return; 16981f04af33SPhilipp Reisner } 16991f04af33SPhilipp Reisner 1700b30ab791SAndreas Gruenbacher drbd_start_resync(device, C_SYNC_SOURCE); 1701b30ab791SAndreas Gruenbacher clear_bit(AHEAD_TO_SYNC_SOURCE, &device->flags); 17021f04af33SPhilipp Reisner } 17031f04af33SPhilipp Reisner 1704aaaba345SLars Ellenberg static bool use_checksum_based_resync(struct drbd_connection *connection, struct drbd_device *device) 1705aaaba345SLars Ellenberg { 1706aaaba345SLars Ellenberg bool csums_after_crash_only; 1707aaaba345SLars Ellenberg rcu_read_lock(); 1708aaaba345SLars Ellenberg csums_after_crash_only = rcu_dereference(connection->net_conf)->csums_after_crash_only; 1709aaaba345SLars Ellenberg rcu_read_unlock(); 1710aaaba345SLars Ellenberg return connection->agreed_pro_version >= 89 && /* supported? */ 1711aaaba345SLars Ellenberg connection->csums_tfm && /* configured? */ 17127e5fec31SFabian Frederick (csums_after_crash_only == false /* use for each resync? */ 1713aaaba345SLars Ellenberg || test_bit(CRASHED_PRIMARY, &device->flags)); /* or only after Primary crash? */ 1714aaaba345SLars Ellenberg } 1715aaaba345SLars Ellenberg 1716b411b363SPhilipp Reisner /** 1717b411b363SPhilipp Reisner * drbd_start_resync() - Start the resync process 1718b30ab791SAndreas Gruenbacher * @device: DRBD device. 1719b411b363SPhilipp Reisner * @side: Either C_SYNC_SOURCE or C_SYNC_TARGET 1720b411b363SPhilipp Reisner * 1721b411b363SPhilipp Reisner * This function might bring you directly into one of the 1722b411b363SPhilipp Reisner * C_PAUSED_SYNC_* states. 1723b411b363SPhilipp Reisner */ 1724b30ab791SAndreas Gruenbacher void drbd_start_resync(struct drbd_device *device, enum drbd_conns side) 1725b411b363SPhilipp Reisner { 172644a4d551SLars Ellenberg struct drbd_peer_device *peer_device = first_peer_device(device); 172744a4d551SLars Ellenberg struct drbd_connection *connection = peer_device ? peer_device->connection : NULL; 1728b411b363SPhilipp Reisner union drbd_state ns; 1729b411b363SPhilipp Reisner int r; 1730b411b363SPhilipp Reisner 1731b30ab791SAndreas Gruenbacher if (device->state.conn >= C_SYNC_SOURCE && device->state.conn < C_AHEAD) { 1732d0180171SAndreas Gruenbacher drbd_err(device, "Resync already running!\n"); 1733b411b363SPhilipp Reisner return; 1734b411b363SPhilipp Reisner } 1735b411b363SPhilipp Reisner 1736b30ab791SAndreas Gruenbacher if (!test_bit(B_RS_H_DONE, &device->flags)) { 1737b411b363SPhilipp Reisner if (side == C_SYNC_TARGET) { 1738b411b363SPhilipp Reisner /* Since application IO was locked out during C_WF_BITMAP_T and 1739b411b363SPhilipp Reisner C_WF_SYNC_UUID we are still unmodified. Before going to C_SYNC_TARGET 1740b411b363SPhilipp Reisner we check that we might make the data inconsistent. */ 1741b30ab791SAndreas Gruenbacher r = drbd_khelper(device, "before-resync-target"); 1742b411b363SPhilipp Reisner r = (r >> 8) & 0xff; 1743b411b363SPhilipp Reisner if (r > 0) { 1744d0180171SAndreas Gruenbacher drbd_info(device, "before-resync-target handler returned %d, " 1745b411b363SPhilipp Reisner "dropping connection.\n", r); 174644a4d551SLars Ellenberg conn_request_state(connection, NS(conn, C_DISCONNECTING), CS_HARD); 1747b411b363SPhilipp Reisner return; 1748b411b363SPhilipp Reisner } 174909b9e797SPhilipp Reisner } else /* C_SYNC_SOURCE */ { 1750b30ab791SAndreas Gruenbacher r = drbd_khelper(device, "before-resync-source"); 175109b9e797SPhilipp Reisner r = (r >> 8) & 0xff; 175209b9e797SPhilipp Reisner if (r > 0) { 175309b9e797SPhilipp Reisner if (r == 3) { 1754d0180171SAndreas Gruenbacher drbd_info(device, "before-resync-source handler returned %d, " 175509b9e797SPhilipp Reisner "ignoring. Old userland tools?", r); 175609b9e797SPhilipp Reisner } else { 1757d0180171SAndreas Gruenbacher drbd_info(device, "before-resync-source handler returned %d, " 175809b9e797SPhilipp Reisner "dropping connection.\n", r); 175944a4d551SLars Ellenberg conn_request_state(connection, 1760a6b32bc3SAndreas Gruenbacher NS(conn, C_DISCONNECTING), CS_HARD); 176109b9e797SPhilipp Reisner return; 176209b9e797SPhilipp Reisner } 176309b9e797SPhilipp Reisner } 1764b411b363SPhilipp Reisner } 1765e64a3294SPhilipp Reisner } 1766b411b363SPhilipp Reisner 176744a4d551SLars Ellenberg if (current == connection->worker.task) { 1768dad20554SPhilipp Reisner /* The worker should not sleep waiting for state_mutex, 1769e64a3294SPhilipp Reisner that can take long */ 1770b30ab791SAndreas Gruenbacher if (!mutex_trylock(device->state_mutex)) { 1771b30ab791SAndreas Gruenbacher set_bit(B_RS_H_DONE, &device->flags); 1772b30ab791SAndreas Gruenbacher device->start_resync_timer.expires = jiffies + HZ/5; 1773b30ab791SAndreas Gruenbacher add_timer(&device->start_resync_timer); 1774e64a3294SPhilipp Reisner return; 1775e64a3294SPhilipp Reisner } 1776e64a3294SPhilipp Reisner } else { 1777b30ab791SAndreas Gruenbacher mutex_lock(device->state_mutex); 1778e64a3294SPhilipp Reisner } 1779b411b363SPhilipp Reisner 178028bc3b8cSAndreas Gruenbacher lock_all_resources(); 178128bc3b8cSAndreas Gruenbacher clear_bit(B_RS_H_DONE, &device->flags); 1782a700471bSPhilipp Reisner /* Did some connection breakage or IO error race with us? */ 1783b30ab791SAndreas Gruenbacher if (device->state.conn < C_CONNECTED 1784b30ab791SAndreas Gruenbacher || !get_ldev_if_state(device, D_NEGOTIATING)) { 178528bc3b8cSAndreas Gruenbacher unlock_all_resources(); 178628bc3b8cSAndreas Gruenbacher goto out; 1787b411b363SPhilipp Reisner } 1788b411b363SPhilipp Reisner 1789b30ab791SAndreas Gruenbacher ns = drbd_read_state(device); 1790b411b363SPhilipp Reisner 1791b30ab791SAndreas Gruenbacher ns.aftr_isp = !_drbd_may_sync_now(device); 1792b411b363SPhilipp Reisner 1793b411b363SPhilipp Reisner ns.conn = side; 1794b411b363SPhilipp Reisner 1795b411b363SPhilipp Reisner if (side == C_SYNC_TARGET) 1796b411b363SPhilipp Reisner ns.disk = D_INCONSISTENT; 1797b411b363SPhilipp Reisner else /* side == C_SYNC_SOURCE */ 1798b411b363SPhilipp Reisner ns.pdsk = D_INCONSISTENT; 1799b411b363SPhilipp Reisner 180028bc3b8cSAndreas Gruenbacher r = _drbd_set_state(device, ns, CS_VERBOSE, NULL); 1801b30ab791SAndreas Gruenbacher ns = drbd_read_state(device); 1802b411b363SPhilipp Reisner 1803b411b363SPhilipp Reisner if (ns.conn < C_CONNECTED) 1804b411b363SPhilipp Reisner r = SS_UNKNOWN_ERROR; 1805b411b363SPhilipp Reisner 1806b411b363SPhilipp Reisner if (r == SS_SUCCESS) { 1807b30ab791SAndreas Gruenbacher unsigned long tw = drbd_bm_total_weight(device); 18081d7734a0SLars Ellenberg unsigned long now = jiffies; 18091d7734a0SLars Ellenberg int i; 18101d7734a0SLars Ellenberg 1811b30ab791SAndreas Gruenbacher device->rs_failed = 0; 1812b30ab791SAndreas Gruenbacher device->rs_paused = 0; 1813b30ab791SAndreas Gruenbacher device->rs_same_csum = 0; 1814b30ab791SAndreas Gruenbacher device->rs_last_sect_ev = 0; 1815b30ab791SAndreas Gruenbacher device->rs_total = tw; 1816b30ab791SAndreas Gruenbacher device->rs_start = now; 18171d7734a0SLars Ellenberg for (i = 0; i < DRBD_SYNC_MARKS; i++) { 1818b30ab791SAndreas Gruenbacher device->rs_mark_left[i] = tw; 1819b30ab791SAndreas Gruenbacher device->rs_mark_time[i] = now; 18201d7734a0SLars Ellenberg } 182128bc3b8cSAndreas Gruenbacher drbd_pause_after(device); 18225ab7d2c0SLars Ellenberg /* Forget potentially stale cached per resync extent bit-counts. 18235ab7d2c0SLars Ellenberg * Open coded drbd_rs_cancel_all(device), we already have IRQs 18245ab7d2c0SLars Ellenberg * disabled, and know the disk state is ok. */ 18255ab7d2c0SLars Ellenberg spin_lock(&device->al_lock); 18265ab7d2c0SLars Ellenberg lc_reset(device->resync); 18275ab7d2c0SLars Ellenberg device->resync_locked = 0; 18285ab7d2c0SLars Ellenberg device->resync_wenr = LC_FREE; 18295ab7d2c0SLars Ellenberg spin_unlock(&device->al_lock); 1830b411b363SPhilipp Reisner } 183128bc3b8cSAndreas Gruenbacher unlock_all_resources(); 18325a22db89SLars Ellenberg 18336c922ed5SLars Ellenberg if (r == SS_SUCCESS) { 18345ab7d2c0SLars Ellenberg wake_up(&device->al_wait); /* for lc_reset() above */ 1835328e0f12SPhilipp Reisner /* reset rs_last_bcast when a resync or verify is started, 1836328e0f12SPhilipp Reisner * to deal with potential jiffies wrap. */ 1837b30ab791SAndreas Gruenbacher device->rs_last_bcast = jiffies - HZ; 1838328e0f12SPhilipp Reisner 1839d0180171SAndreas Gruenbacher drbd_info(device, "Began resync as %s (will sync %lu KB [%lu bits set]).\n", 18406c922ed5SLars Ellenberg drbd_conn_str(ns.conn), 1841b30ab791SAndreas Gruenbacher (unsigned long) device->rs_total << (BM_BLOCK_SHIFT-10), 1842b30ab791SAndreas Gruenbacher (unsigned long) device->rs_total); 1843aaaba345SLars Ellenberg if (side == C_SYNC_TARGET) { 1844b30ab791SAndreas Gruenbacher device->bm_resync_fo = 0; 1845aaaba345SLars Ellenberg device->use_csums = use_checksum_based_resync(connection, device); 1846aaaba345SLars Ellenberg } else { 18477e5fec31SFabian Frederick device->use_csums = false; 1848aaaba345SLars Ellenberg } 18495a22db89SLars Ellenberg 18505a22db89SLars Ellenberg /* Since protocol 96, we must serialize drbd_gen_and_send_sync_uuid 18515a22db89SLars Ellenberg * with w_send_oos, or the sync target will get confused as to 18525a22db89SLars Ellenberg * how much bits to resync. We cannot do that always, because for an 18535a22db89SLars Ellenberg * empty resync and protocol < 95, we need to do it here, as we call 18545a22db89SLars Ellenberg * drbd_resync_finished from here in that case. 18555a22db89SLars Ellenberg * We drbd_gen_and_send_sync_uuid here for protocol < 96, 18565a22db89SLars Ellenberg * and from after_state_ch otherwise. */ 185744a4d551SLars Ellenberg if (side == C_SYNC_SOURCE && connection->agreed_pro_version < 96) 185844a4d551SLars Ellenberg drbd_gen_and_send_sync_uuid(peer_device); 1859b411b363SPhilipp Reisner 186044a4d551SLars Ellenberg if (connection->agreed_pro_version < 95 && device->rs_total == 0) { 1861af85e8e8SLars Ellenberg /* This still has a race (about when exactly the peers 1862af85e8e8SLars Ellenberg * detect connection loss) that can lead to a full sync 1863af85e8e8SLars Ellenberg * on next handshake. In 8.3.9 we fixed this with explicit 1864af85e8e8SLars Ellenberg * resync-finished notifications, but the fix 1865af85e8e8SLars Ellenberg * introduces a protocol change. Sleeping for some 1866af85e8e8SLars Ellenberg * time longer than the ping interval + timeout on the 1867af85e8e8SLars Ellenberg * SyncSource, to give the SyncTarget the chance to 1868af85e8e8SLars Ellenberg * detect connection loss, then waiting for a ping 1869af85e8e8SLars Ellenberg * response (implicit in drbd_resync_finished) reduces 1870af85e8e8SLars Ellenberg * the race considerably, but does not solve it. */ 187144ed167dSPhilipp Reisner if (side == C_SYNC_SOURCE) { 187244ed167dSPhilipp Reisner struct net_conf *nc; 187344ed167dSPhilipp Reisner int timeo; 187444ed167dSPhilipp Reisner 187544ed167dSPhilipp Reisner rcu_read_lock(); 187644a4d551SLars Ellenberg nc = rcu_dereference(connection->net_conf); 187744ed167dSPhilipp Reisner timeo = nc->ping_int * HZ + nc->ping_timeo * HZ / 9; 187844ed167dSPhilipp Reisner rcu_read_unlock(); 187944ed167dSPhilipp Reisner schedule_timeout_interruptible(timeo); 188044ed167dSPhilipp Reisner } 1881b30ab791SAndreas Gruenbacher drbd_resync_finished(device); 1882b411b363SPhilipp Reisner } 1883b411b363SPhilipp Reisner 1884b30ab791SAndreas Gruenbacher drbd_rs_controller_reset(device); 1885b30ab791SAndreas Gruenbacher /* ns.conn may already be != device->state.conn, 1886b411b363SPhilipp Reisner * we may have been paused in between, or become paused until 1887b411b363SPhilipp Reisner * the timer triggers. 1888b411b363SPhilipp Reisner * No matter, that is handled in resync_timer_fn() */ 1889b411b363SPhilipp Reisner if (ns.conn == C_SYNC_TARGET) 1890b30ab791SAndreas Gruenbacher mod_timer(&device->resync_timer, jiffies); 1891b411b363SPhilipp Reisner 1892b30ab791SAndreas Gruenbacher drbd_md_sync(device); 1893b411b363SPhilipp Reisner } 1894b30ab791SAndreas Gruenbacher put_ldev(device); 189528bc3b8cSAndreas Gruenbacher out: 1896b30ab791SAndreas Gruenbacher mutex_unlock(device->state_mutex); 1897b411b363SPhilipp Reisner } 1898b411b363SPhilipp Reisner 1899e334f550SLars Ellenberg static void update_on_disk_bitmap(struct drbd_device *device, bool resync_done) 1900c7a58db4SLars Ellenberg { 1901c7a58db4SLars Ellenberg struct sib_info sib = { .sib_reason = SIB_SYNC_PROGRESS, }; 1902c7a58db4SLars Ellenberg device->rs_last_bcast = jiffies; 1903c7a58db4SLars Ellenberg 1904c7a58db4SLars Ellenberg if (!get_ldev(device)) 1905c7a58db4SLars Ellenberg return; 1906c7a58db4SLars Ellenberg 1907c7a58db4SLars Ellenberg drbd_bm_write_lazy(device, 0); 19085ab7d2c0SLars Ellenberg if (resync_done && is_sync_state(device->state.conn)) 1909c7a58db4SLars Ellenberg drbd_resync_finished(device); 19105ab7d2c0SLars Ellenberg 1911c7a58db4SLars Ellenberg drbd_bcast_event(device, &sib); 1912c7a58db4SLars Ellenberg /* update timestamp, in case it took a while to write out stuff */ 1913c7a58db4SLars Ellenberg device->rs_last_bcast = jiffies; 1914c7a58db4SLars Ellenberg put_ldev(device); 1915c7a58db4SLars Ellenberg } 1916c7a58db4SLars Ellenberg 1917e334f550SLars Ellenberg static void drbd_ldev_destroy(struct drbd_device *device) 1918e334f550SLars Ellenberg { 1919e334f550SLars Ellenberg lc_destroy(device->resync); 1920e334f550SLars Ellenberg device->resync = NULL; 1921e334f550SLars Ellenberg lc_destroy(device->act_log); 1922e334f550SLars Ellenberg device->act_log = NULL; 1923d1b80853SAndreas Gruenbacher 1924d1b80853SAndreas Gruenbacher __acquire(local); 192563a7c8adSLars Ellenberg drbd_backing_dev_free(device, device->ldev); 1926d1b80853SAndreas Gruenbacher device->ldev = NULL; 1927d1b80853SAndreas Gruenbacher __release(local); 1928d1b80853SAndreas Gruenbacher 1929e334f550SLars Ellenberg clear_bit(GOING_DISKLESS, &device->flags); 1930e334f550SLars Ellenberg wake_up(&device->misc_wait); 1931e334f550SLars Ellenberg } 1932e334f550SLars Ellenberg 1933e334f550SLars Ellenberg static void go_diskless(struct drbd_device *device) 1934e334f550SLars Ellenberg { 1935e334f550SLars Ellenberg D_ASSERT(device, device->state.disk == D_FAILED); 1936e334f550SLars Ellenberg /* we cannot assert local_cnt == 0 here, as get_ldev_if_state will 1937e334f550SLars Ellenberg * inc/dec it frequently. Once we are D_DISKLESS, no one will touch 1938e334f550SLars Ellenberg * the protected members anymore, though, so once put_ldev reaches zero 1939e334f550SLars Ellenberg * again, it will be safe to free them. */ 1940e334f550SLars Ellenberg 1941e334f550SLars Ellenberg /* Try to write changed bitmap pages, read errors may have just 1942e334f550SLars Ellenberg * set some bits outside the area covered by the activity log. 1943e334f550SLars Ellenberg * 1944e334f550SLars Ellenberg * If we have an IO error during the bitmap writeout, 1945e334f550SLars Ellenberg * we will want a full sync next time, just in case. 1946e334f550SLars Ellenberg * (Do we want a specific meta data flag for this?) 1947e334f550SLars Ellenberg * 1948e334f550SLars Ellenberg * If that does not make it to stable storage either, 1949e334f550SLars Ellenberg * we cannot do anything about that anymore. 1950e334f550SLars Ellenberg * 1951e334f550SLars Ellenberg * We still need to check if both bitmap and ldev are present, we may 1952e334f550SLars Ellenberg * end up here after a failed attach, before ldev was even assigned. 1953e334f550SLars Ellenberg */ 1954e334f550SLars Ellenberg if (device->bitmap && device->ldev) { 1955e334f550SLars Ellenberg /* An interrupted resync or similar is allowed to recounts bits 1956e334f550SLars Ellenberg * while we detach. 1957e334f550SLars Ellenberg * Any modifications would not be expected anymore, though. 1958e334f550SLars Ellenberg */ 1959e334f550SLars Ellenberg if (drbd_bitmap_io_from_worker(device, drbd_bm_write, 1960e334f550SLars Ellenberg "detach", BM_LOCKED_TEST_ALLOWED)) { 1961e334f550SLars Ellenberg if (test_bit(WAS_READ_ERROR, &device->flags)) { 1962e334f550SLars Ellenberg drbd_md_set_flag(device, MDF_FULL_SYNC); 1963e334f550SLars Ellenberg drbd_md_sync(device); 1964e334f550SLars Ellenberg } 1965e334f550SLars Ellenberg } 1966e334f550SLars Ellenberg } 1967e334f550SLars Ellenberg 1968e334f550SLars Ellenberg drbd_force_state(device, NS(disk, D_DISKLESS)); 1969e334f550SLars Ellenberg } 1970e334f550SLars Ellenberg 1971ac0acb9eSLars Ellenberg static int do_md_sync(struct drbd_device *device) 1972ac0acb9eSLars Ellenberg { 1973ac0acb9eSLars Ellenberg drbd_warn(device, "md_sync_timer expired! Worker calls drbd_md_sync().\n"); 1974ac0acb9eSLars Ellenberg drbd_md_sync(device); 1975ac0acb9eSLars Ellenberg return 0; 1976ac0acb9eSLars Ellenberg } 1977ac0acb9eSLars Ellenberg 1978944410e9SLars Ellenberg /* only called from drbd_worker thread, no locking */ 1979944410e9SLars Ellenberg void __update_timing_details( 1980944410e9SLars Ellenberg struct drbd_thread_timing_details *tdp, 1981944410e9SLars Ellenberg unsigned int *cb_nr, 1982944410e9SLars Ellenberg void *cb, 1983944410e9SLars Ellenberg const char *fn, const unsigned int line) 1984944410e9SLars Ellenberg { 1985944410e9SLars Ellenberg unsigned int i = *cb_nr % DRBD_THREAD_DETAILS_HIST; 1986944410e9SLars Ellenberg struct drbd_thread_timing_details *td = tdp + i; 1987944410e9SLars Ellenberg 1988944410e9SLars Ellenberg td->start_jif = jiffies; 1989944410e9SLars Ellenberg td->cb_addr = cb; 1990944410e9SLars Ellenberg td->caller_fn = fn; 1991944410e9SLars Ellenberg td->line = line; 1992944410e9SLars Ellenberg td->cb_nr = *cb_nr; 1993944410e9SLars Ellenberg 1994944410e9SLars Ellenberg i = (i+1) % DRBD_THREAD_DETAILS_HIST; 1995944410e9SLars Ellenberg td = tdp + i; 1996944410e9SLars Ellenberg memset(td, 0, sizeof(*td)); 1997944410e9SLars Ellenberg 1998944410e9SLars Ellenberg ++(*cb_nr); 1999944410e9SLars Ellenberg } 2000944410e9SLars Ellenberg 2001e334f550SLars Ellenberg static void do_device_work(struct drbd_device *device, const unsigned long todo) 2002e334f550SLars Ellenberg { 2003b47a06d1SAndreas Gruenbacher if (test_bit(MD_SYNC, &todo)) 2004ac0acb9eSLars Ellenberg do_md_sync(device); 2005b47a06d1SAndreas Gruenbacher if (test_bit(RS_DONE, &todo) || 2006b47a06d1SAndreas Gruenbacher test_bit(RS_PROGRESS, &todo)) 2007b47a06d1SAndreas Gruenbacher update_on_disk_bitmap(device, test_bit(RS_DONE, &todo)); 2008b47a06d1SAndreas Gruenbacher if (test_bit(GO_DISKLESS, &todo)) 2009e334f550SLars Ellenberg go_diskless(device); 2010b47a06d1SAndreas Gruenbacher if (test_bit(DESTROY_DISK, &todo)) 2011e334f550SLars Ellenberg drbd_ldev_destroy(device); 2012b47a06d1SAndreas Gruenbacher if (test_bit(RS_START, &todo)) 2013ac0acb9eSLars Ellenberg do_start_resync(device); 2014e334f550SLars Ellenberg } 2015e334f550SLars Ellenberg 2016e334f550SLars Ellenberg #define DRBD_DEVICE_WORK_MASK \ 2017e334f550SLars Ellenberg ((1UL << GO_DISKLESS) \ 2018e334f550SLars Ellenberg |(1UL << DESTROY_DISK) \ 2019ac0acb9eSLars Ellenberg |(1UL << MD_SYNC) \ 2020ac0acb9eSLars Ellenberg |(1UL << RS_START) \ 2021e334f550SLars Ellenberg |(1UL << RS_PROGRESS) \ 2022e334f550SLars Ellenberg |(1UL << RS_DONE) \ 2023e334f550SLars Ellenberg ) 2024e334f550SLars Ellenberg 2025e334f550SLars Ellenberg static unsigned long get_work_bits(unsigned long *flags) 2026e334f550SLars Ellenberg { 2027e334f550SLars Ellenberg unsigned long old, new; 2028e334f550SLars Ellenberg do { 2029e334f550SLars Ellenberg old = *flags; 2030e334f550SLars Ellenberg new = old & ~DRBD_DEVICE_WORK_MASK; 2031e334f550SLars Ellenberg } while (cmpxchg(flags, old, new) != old); 2032e334f550SLars Ellenberg return old & DRBD_DEVICE_WORK_MASK; 2033e334f550SLars Ellenberg } 2034e334f550SLars Ellenberg 2035e334f550SLars Ellenberg static void do_unqueued_work(struct drbd_connection *connection) 2036c7a58db4SLars Ellenberg { 2037c7a58db4SLars Ellenberg struct drbd_peer_device *peer_device; 2038c7a58db4SLars Ellenberg int vnr; 2039c7a58db4SLars Ellenberg 2040c7a58db4SLars Ellenberg rcu_read_lock(); 2041c7a58db4SLars Ellenberg idr_for_each_entry(&connection->peer_devices, peer_device, vnr) { 2042c7a58db4SLars Ellenberg struct drbd_device *device = peer_device->device; 2043e334f550SLars Ellenberg unsigned long todo = get_work_bits(&device->flags); 2044e334f550SLars Ellenberg if (!todo) 2045c7a58db4SLars Ellenberg continue; 20465ab7d2c0SLars Ellenberg 2047c7a58db4SLars Ellenberg kref_get(&device->kref); 2048c7a58db4SLars Ellenberg rcu_read_unlock(); 2049e334f550SLars Ellenberg do_device_work(device, todo); 2050c7a58db4SLars Ellenberg kref_put(&device->kref, drbd_destroy_device); 2051c7a58db4SLars Ellenberg rcu_read_lock(); 2052c7a58db4SLars Ellenberg } 2053c7a58db4SLars Ellenberg rcu_read_unlock(); 2054c7a58db4SLars Ellenberg } 2055c7a58db4SLars Ellenberg 2056a186e478SRashika Kheria static bool dequeue_work_batch(struct drbd_work_queue *queue, struct list_head *work_list) 20578c0785a5SLars Ellenberg { 20588c0785a5SLars Ellenberg spin_lock_irq(&queue->q_lock); 205915e26f6aSLars Ellenberg list_splice_tail_init(&queue->q, work_list); 20608c0785a5SLars Ellenberg spin_unlock_irq(&queue->q_lock); 20618c0785a5SLars Ellenberg return !list_empty(work_list); 20628c0785a5SLars Ellenberg } 20638c0785a5SLars Ellenberg 2064bde89a9eSAndreas Gruenbacher static void wait_for_work(struct drbd_connection *connection, struct list_head *work_list) 2065b6dd1a89SLars Ellenberg { 2066b6dd1a89SLars Ellenberg DEFINE_WAIT(wait); 2067b6dd1a89SLars Ellenberg struct net_conf *nc; 2068b6dd1a89SLars Ellenberg int uncork, cork; 2069b6dd1a89SLars Ellenberg 2070abde9cc6SLars Ellenberg dequeue_work_batch(&connection->sender_work, work_list); 2071b6dd1a89SLars Ellenberg if (!list_empty(work_list)) 2072b6dd1a89SLars Ellenberg return; 2073b6dd1a89SLars Ellenberg 2074b6dd1a89SLars Ellenberg /* Still nothing to do? 2075b6dd1a89SLars Ellenberg * Maybe we still need to close the current epoch, 2076b6dd1a89SLars Ellenberg * even if no new requests are queued yet. 2077b6dd1a89SLars Ellenberg * 2078b6dd1a89SLars Ellenberg * Also, poke TCP, just in case. 2079b6dd1a89SLars Ellenberg * Then wait for new work (or signal). */ 2080b6dd1a89SLars Ellenberg rcu_read_lock(); 2081b6dd1a89SLars Ellenberg nc = rcu_dereference(connection->net_conf); 2082b6dd1a89SLars Ellenberg uncork = nc ? nc->tcp_cork : 0; 2083b6dd1a89SLars Ellenberg rcu_read_unlock(); 2084b6dd1a89SLars Ellenberg if (uncork) { 2085b6dd1a89SLars Ellenberg mutex_lock(&connection->data.mutex); 2086b6dd1a89SLars Ellenberg if (connection->data.socket) 2087b6dd1a89SLars Ellenberg drbd_tcp_uncork(connection->data.socket); 2088b6dd1a89SLars Ellenberg mutex_unlock(&connection->data.mutex); 2089b6dd1a89SLars Ellenberg } 2090b6dd1a89SLars Ellenberg 2091b6dd1a89SLars Ellenberg for (;;) { 2092b6dd1a89SLars Ellenberg int send_barrier; 2093b6dd1a89SLars Ellenberg prepare_to_wait(&connection->sender_work.q_wait, &wait, TASK_INTERRUPTIBLE); 20940500813fSAndreas Gruenbacher spin_lock_irq(&connection->resource->req_lock); 2095b6dd1a89SLars Ellenberg spin_lock(&connection->sender_work.q_lock); /* FIXME get rid of this one? */ 2096bc317a9eSLars Ellenberg if (!list_empty(&connection->sender_work.q)) 20974dd726f0SLars Ellenberg list_splice_tail_init(&connection->sender_work.q, work_list); 2098b6dd1a89SLars Ellenberg spin_unlock(&connection->sender_work.q_lock); /* FIXME get rid of this one? */ 2099b6dd1a89SLars Ellenberg if (!list_empty(work_list) || signal_pending(current)) { 21000500813fSAndreas Gruenbacher spin_unlock_irq(&connection->resource->req_lock); 2101b6dd1a89SLars Ellenberg break; 2102b6dd1a89SLars Ellenberg } 2103f9c78128SLars Ellenberg 2104f9c78128SLars Ellenberg /* We found nothing new to do, no to-be-communicated request, 2105f9c78128SLars Ellenberg * no other work item. We may still need to close the last 2106f9c78128SLars Ellenberg * epoch. Next incoming request epoch will be connection -> 2107f9c78128SLars Ellenberg * current transfer log epoch number. If that is different 2108f9c78128SLars Ellenberg * from the epoch of the last request we communicated, it is 2109f9c78128SLars Ellenberg * safe to send the epoch separating barrier now. 2110f9c78128SLars Ellenberg */ 2111f9c78128SLars Ellenberg send_barrier = 2112f9c78128SLars Ellenberg atomic_read(&connection->current_tle_nr) != 2113f9c78128SLars Ellenberg connection->send.current_epoch_nr; 21140500813fSAndreas Gruenbacher spin_unlock_irq(&connection->resource->req_lock); 2115f9c78128SLars Ellenberg 2116f9c78128SLars Ellenberg if (send_barrier) 2117f9c78128SLars Ellenberg maybe_send_barrier(connection, 2118f9c78128SLars Ellenberg connection->send.current_epoch_nr + 1); 21195ab7d2c0SLars Ellenberg 2120e334f550SLars Ellenberg if (test_bit(DEVICE_WORK_PENDING, &connection->flags)) 21215ab7d2c0SLars Ellenberg break; 21225ab7d2c0SLars Ellenberg 2123a80ca1aeSLars Ellenberg /* drbd_send() may have called flush_signals() */ 2124a80ca1aeSLars Ellenberg if (get_t_state(&connection->worker) != RUNNING) 2125a80ca1aeSLars Ellenberg break; 21265ab7d2c0SLars Ellenberg 2127b6dd1a89SLars Ellenberg schedule(); 2128b6dd1a89SLars Ellenberg /* may be woken up for other things but new work, too, 2129b6dd1a89SLars Ellenberg * e.g. if the current epoch got closed. 2130b6dd1a89SLars Ellenberg * In which case we send the barrier above. */ 2131b6dd1a89SLars Ellenberg } 2132b6dd1a89SLars Ellenberg finish_wait(&connection->sender_work.q_wait, &wait); 2133b6dd1a89SLars Ellenberg 2134b6dd1a89SLars Ellenberg /* someone may have changed the config while we have been waiting above. */ 2135b6dd1a89SLars Ellenberg rcu_read_lock(); 2136b6dd1a89SLars Ellenberg nc = rcu_dereference(connection->net_conf); 2137b6dd1a89SLars Ellenberg cork = nc ? nc->tcp_cork : 0; 2138b6dd1a89SLars Ellenberg rcu_read_unlock(); 2139b6dd1a89SLars Ellenberg mutex_lock(&connection->data.mutex); 2140b6dd1a89SLars Ellenberg if (connection->data.socket) { 2141b6dd1a89SLars Ellenberg if (cork) 2142b6dd1a89SLars Ellenberg drbd_tcp_cork(connection->data.socket); 2143b6dd1a89SLars Ellenberg else if (!uncork) 2144b6dd1a89SLars Ellenberg drbd_tcp_uncork(connection->data.socket); 2145b6dd1a89SLars Ellenberg } 2146b6dd1a89SLars Ellenberg mutex_unlock(&connection->data.mutex); 2147b6dd1a89SLars Ellenberg } 2148b6dd1a89SLars Ellenberg 2149b411b363SPhilipp Reisner int drbd_worker(struct drbd_thread *thi) 2150b411b363SPhilipp Reisner { 2151bde89a9eSAndreas Gruenbacher struct drbd_connection *connection = thi->connection; 21526db7e50aSAndreas Gruenbacher struct drbd_work *w = NULL; 2153c06ece6bSAndreas Gruenbacher struct drbd_peer_device *peer_device; 2154b411b363SPhilipp Reisner LIST_HEAD(work_list); 21558c0785a5SLars Ellenberg int vnr; 2156b411b363SPhilipp Reisner 2157e77a0a5cSAndreas Gruenbacher while (get_t_state(thi) == RUNNING) { 215880822284SPhilipp Reisner drbd_thread_current_set_cpu(thi); 2159b411b363SPhilipp Reisner 2160944410e9SLars Ellenberg if (list_empty(&work_list)) { 2161944410e9SLars Ellenberg update_worker_timing_details(connection, wait_for_work); 2162bde89a9eSAndreas Gruenbacher wait_for_work(connection, &work_list); 2163944410e9SLars Ellenberg } 2164b411b363SPhilipp Reisner 2165944410e9SLars Ellenberg if (test_and_clear_bit(DEVICE_WORK_PENDING, &connection->flags)) { 2166944410e9SLars Ellenberg update_worker_timing_details(connection, do_unqueued_work); 2167e334f550SLars Ellenberg do_unqueued_work(connection); 2168944410e9SLars Ellenberg } 21695ab7d2c0SLars Ellenberg 21708c0785a5SLars Ellenberg if (signal_pending(current)) { 2171b411b363SPhilipp Reisner flush_signals(current); 217219393e10SPhilipp Reisner if (get_t_state(thi) == RUNNING) { 21731ec861ebSAndreas Gruenbacher drbd_warn(connection, "Worker got an unexpected signal\n"); 2174b411b363SPhilipp Reisner continue; 217519393e10SPhilipp Reisner } 2176b411b363SPhilipp Reisner break; 2177b411b363SPhilipp Reisner } 2178b411b363SPhilipp Reisner 2179e77a0a5cSAndreas Gruenbacher if (get_t_state(thi) != RUNNING) 2180b411b363SPhilipp Reisner break; 2181b411b363SPhilipp Reisner 2182729e8b87SLars Ellenberg if (!list_empty(&work_list)) { 21836db7e50aSAndreas Gruenbacher w = list_first_entry(&work_list, struct drbd_work, list); 21846db7e50aSAndreas Gruenbacher list_del_init(&w->list); 2185944410e9SLars Ellenberg update_worker_timing_details(connection, w->cb); 21866db7e50aSAndreas Gruenbacher if (w->cb(w, connection->cstate < C_WF_REPORT_PARAMS) == 0) 21878c0785a5SLars Ellenberg continue; 2188bde89a9eSAndreas Gruenbacher if (connection->cstate >= C_WF_REPORT_PARAMS) 2189bde89a9eSAndreas Gruenbacher conn_request_state(connection, NS(conn, C_NETWORK_FAILURE), CS_HARD); 2190b411b363SPhilipp Reisner } 2191b411b363SPhilipp Reisner } 2192b411b363SPhilipp Reisner 21938c0785a5SLars Ellenberg do { 2194944410e9SLars Ellenberg if (test_and_clear_bit(DEVICE_WORK_PENDING, &connection->flags)) { 2195944410e9SLars Ellenberg update_worker_timing_details(connection, do_unqueued_work); 2196e334f550SLars Ellenberg do_unqueued_work(connection); 2197944410e9SLars Ellenberg } 2198729e8b87SLars Ellenberg if (!list_empty(&work_list)) { 21996db7e50aSAndreas Gruenbacher w = list_first_entry(&work_list, struct drbd_work, list); 22006db7e50aSAndreas Gruenbacher list_del_init(&w->list); 2201944410e9SLars Ellenberg update_worker_timing_details(connection, w->cb); 22026db7e50aSAndreas Gruenbacher w->cb(w, 1); 2203729e8b87SLars Ellenberg } else 2204bde89a9eSAndreas Gruenbacher dequeue_work_batch(&connection->sender_work, &work_list); 2205e334f550SLars Ellenberg } while (!list_empty(&work_list) || test_bit(DEVICE_WORK_PENDING, &connection->flags)); 2206b411b363SPhilipp Reisner 2207c141ebdaSPhilipp Reisner rcu_read_lock(); 2208c06ece6bSAndreas Gruenbacher idr_for_each_entry(&connection->peer_devices, peer_device, vnr) { 2209c06ece6bSAndreas Gruenbacher struct drbd_device *device = peer_device->device; 22100b0ba1efSAndreas Gruenbacher D_ASSERT(device, device->state.disk == D_DISKLESS && device->state.conn == C_STANDALONE); 2211b30ab791SAndreas Gruenbacher kref_get(&device->kref); 2212c141ebdaSPhilipp Reisner rcu_read_unlock(); 2213b30ab791SAndreas Gruenbacher drbd_device_cleanup(device); 221405a10ec7SAndreas Gruenbacher kref_put(&device->kref, drbd_destroy_device); 2215c141ebdaSPhilipp Reisner rcu_read_lock(); 22160e29d163SPhilipp Reisner } 2217c141ebdaSPhilipp Reisner rcu_read_unlock(); 2218b411b363SPhilipp Reisner 2219b411b363SPhilipp Reisner return 0; 2220b411b363SPhilipp Reisner } 2221