18839a0e0STejun Heo /* 24fed947cSTejun Heo * Functions to sequence FLUSH and FUA writes. 3ae1b1539STejun Heo * 4ae1b1539STejun Heo * Copyright (C) 2011 Max Planck Institute for Gravitational Physics 5ae1b1539STejun Heo * Copyright (C) 2011 Tejun Heo <tj@kernel.org> 6ae1b1539STejun Heo * 7ae1b1539STejun Heo * This file is released under the GPLv2. 8ae1b1539STejun Heo * 9ae1b1539STejun Heo * REQ_{FLUSH|FUA} requests are decomposed to sequences consisted of three 10ae1b1539STejun Heo * optional steps - PREFLUSH, DATA and POSTFLUSH - according to the request 11ae1b1539STejun Heo * properties and hardware capability. 12ae1b1539STejun Heo * 1328a8f0d3SMike Christie * If a request doesn't have data, only REQ_PREFLUSH makes sense, which 1428a8f0d3SMike Christie * indicates a simple flush request. If there is data, REQ_PREFLUSH indicates 15ae1b1539STejun Heo * that the device cache should be flushed before the data is executed, and 16ae1b1539STejun Heo * REQ_FUA means that the data must be on non-volatile media on request 17ae1b1539STejun Heo * completion. 18ae1b1539STejun Heo * 19ae1b1539STejun Heo * If the device doesn't have writeback cache, FLUSH and FUA don't make any 20ae1b1539STejun Heo * difference. The requests are either completed immediately if there's no 21ae1b1539STejun Heo * data or executed as normal requests otherwise. 22ae1b1539STejun Heo * 2328a8f0d3SMike Christie * If the device has writeback cache and supports FUA, REQ_PREFLUSH is 24ae1b1539STejun Heo * translated to PREFLUSH but REQ_FUA is passed down directly with DATA. 25ae1b1539STejun Heo * 2628a8f0d3SMike Christie * If the device has writeback cache and doesn't support FUA, REQ_PREFLUSH 2728a8f0d3SMike Christie * is translated to PREFLUSH and REQ_FUA to POSTFLUSH. 28ae1b1539STejun Heo * 29ae1b1539STejun Heo * The actual execution of flush is double buffered. Whenever a request 30ae1b1539STejun Heo * needs to execute PRE or POSTFLUSH, it queues at 317c94e1c1SMing Lei * fq->flush_queue[fq->flush_pending_idx]. Once certain criteria are met, a 323a5e02ceSMike Christie * REQ_OP_FLUSH is issued and the pending_idx is toggled. When the flush 33ae1b1539STejun Heo * completes, all the requests which were pending are proceeded to the next 34ae1b1539STejun Heo * step. This allows arbitrary merging of different types of FLUSH/FUA 35ae1b1539STejun Heo * requests. 36ae1b1539STejun Heo * 37ae1b1539STejun Heo * Currently, the following conditions are used to determine when to issue 38ae1b1539STejun Heo * flush. 39ae1b1539STejun Heo * 40ae1b1539STejun Heo * C1. At any given time, only one flush shall be in progress. This makes 41ae1b1539STejun Heo * double buffering sufficient. 42ae1b1539STejun Heo * 43ae1b1539STejun Heo * C2. Flush is deferred if any request is executing DATA of its sequence. 44ae1b1539STejun Heo * This avoids issuing separate POSTFLUSHes for requests which shared 45ae1b1539STejun Heo * PREFLUSH. 46ae1b1539STejun Heo * 47ae1b1539STejun Heo * C3. The second condition is ignored if there is a request which has 48ae1b1539STejun Heo * waited longer than FLUSH_PENDING_TIMEOUT. This is to avoid 49ae1b1539STejun Heo * starvation in the unlikely case where there are continuous stream of 50ae1b1539STejun Heo * FUA (without FLUSH) requests. 51ae1b1539STejun Heo * 52ae1b1539STejun Heo * For devices which support FUA, it isn't clear whether C2 (and thus C3) 53ae1b1539STejun Heo * is beneficial. 54ae1b1539STejun Heo * 55ae1b1539STejun Heo * Note that a sequenced FLUSH/FUA request with DATA is completed twice. 56ae1b1539STejun Heo * Once while executing DATA and again after the whole sequence is 57ae1b1539STejun Heo * complete. The first completion updates the contained bio but doesn't 58ae1b1539STejun Heo * finish it so that the bio submitter is notified only after the whole 59e8064021SChristoph Hellwig * sequence is complete. This is implemented by testing RQF_FLUSH_SEQ in 60ae1b1539STejun Heo * req_bio_endio(). 61ae1b1539STejun Heo * 62ae1b1539STejun Heo * The above peculiarity requires that each FLUSH/FUA request has only one 63ae1b1539STejun Heo * bio attached to it, which is guaranteed as they aren't allowed to be 64ae1b1539STejun Heo * merged in the usual way. 658839a0e0STejun Heo */ 66ae1b1539STejun Heo 678839a0e0STejun Heo #include <linux/kernel.h> 688839a0e0STejun Heo #include <linux/module.h> 698839a0e0STejun Heo #include <linux/bio.h> 708839a0e0STejun Heo #include <linux/blkdev.h> 718839a0e0STejun Heo #include <linux/gfp.h> 72320ae51fSJens Axboe #include <linux/blk-mq.h> 738839a0e0STejun Heo 748839a0e0STejun Heo #include "blk.h" 75320ae51fSJens Axboe #include "blk-mq.h" 760048b483SMing Lei #include "blk-mq-tag.h" 77bd166ef1SJens Axboe #include "blk-mq-sched.h" 788839a0e0STejun Heo 794fed947cSTejun Heo /* FLUSH/FUA sequences */ 804fed947cSTejun Heo enum { 81ae1b1539STejun Heo REQ_FSEQ_PREFLUSH = (1 << 0), /* pre-flushing in progress */ 82ae1b1539STejun Heo REQ_FSEQ_DATA = (1 << 1), /* data write in progress */ 83ae1b1539STejun Heo REQ_FSEQ_POSTFLUSH = (1 << 2), /* post-flushing in progress */ 84ae1b1539STejun Heo REQ_FSEQ_DONE = (1 << 3), 85ae1b1539STejun Heo 86ae1b1539STejun Heo REQ_FSEQ_ACTIONS = REQ_FSEQ_PREFLUSH | REQ_FSEQ_DATA | 87ae1b1539STejun Heo REQ_FSEQ_POSTFLUSH, 88ae1b1539STejun Heo 89ae1b1539STejun Heo /* 90ae1b1539STejun Heo * If flush has been pending longer than the following timeout, 91ae1b1539STejun Heo * it's issued even if flush_data requests are still in flight. 92ae1b1539STejun Heo */ 93ae1b1539STejun Heo FLUSH_PENDING_TIMEOUT = 5 * HZ, 944fed947cSTejun Heo }; 954fed947cSTejun Heo 960bae352dSMing Lei static bool blk_kick_flush(struct request_queue *q, 970bae352dSMing Lei struct blk_flush_queue *fq); 988839a0e0STejun Heo 99c888a8f9SJens Axboe static unsigned int blk_flush_policy(unsigned long fflags, struct request *rq) 1008839a0e0STejun Heo { 101ae1b1539STejun Heo unsigned int policy = 0; 102ae1b1539STejun Heo 103fa1bf42fSJeff Moyer if (blk_rq_sectors(rq)) 104fa1bf42fSJeff Moyer policy |= REQ_FSEQ_DATA; 105fa1bf42fSJeff Moyer 106c888a8f9SJens Axboe if (fflags & (1UL << QUEUE_FLAG_WC)) { 10728a8f0d3SMike Christie if (rq->cmd_flags & REQ_PREFLUSH) 108ae1b1539STejun Heo policy |= REQ_FSEQ_PREFLUSH; 109c888a8f9SJens Axboe if (!(fflags & (1UL << QUEUE_FLAG_FUA)) && 110c888a8f9SJens Axboe (rq->cmd_flags & REQ_FUA)) 111ae1b1539STejun Heo policy |= REQ_FSEQ_POSTFLUSH; 112ae1b1539STejun Heo } 113ae1b1539STejun Heo return policy; 1148839a0e0STejun Heo } 1158839a0e0STejun Heo 116ae1b1539STejun Heo static unsigned int blk_flush_cur_seq(struct request *rq) 1178839a0e0STejun Heo { 118ae1b1539STejun Heo return 1 << ffz(rq->flush.seq); 1198839a0e0STejun Heo } 1208839a0e0STejun Heo 121ae1b1539STejun Heo static void blk_flush_restore_request(struct request *rq) 12247f70d5aSTejun Heo { 12347f70d5aSTejun Heo /* 124ae1b1539STejun Heo * After flush data completion, @rq->bio is %NULL but we need to 125ae1b1539STejun Heo * complete the bio again. @rq->biotail is guaranteed to equal the 126ae1b1539STejun Heo * original @rq->bio. Restore it. 12747f70d5aSTejun Heo */ 128ae1b1539STejun Heo rq->bio = rq->biotail; 129ae1b1539STejun Heo 130ae1b1539STejun Heo /* make @rq a normal request */ 131e8064021SChristoph Hellwig rq->rq_flags &= ~RQF_FLUSH_SEQ; 1324853abaaSJeff Moyer rq->end_io = rq->flush.saved_end_io; 133320ae51fSJens Axboe } 134320ae51fSJens Axboe 13510beafc1SMike Snitzer static bool blk_flush_queue_rq(struct request *rq, bool add_front) 136320ae51fSJens Axboe { 13718741986SChristoph Hellwig if (rq->q->mq_ops) { 1382b053acaSBart Van Assche blk_mq_add_to_requeue_list(rq, add_front, true); 13918741986SChristoph Hellwig return false; 14018741986SChristoph Hellwig } else { 14110beafc1SMike Snitzer if (add_front) 14210beafc1SMike Snitzer list_add(&rq->queuelist, &rq->q->queue_head); 14310beafc1SMike Snitzer else 14418741986SChristoph Hellwig list_add_tail(&rq->queuelist, &rq->q->queue_head); 14518741986SChristoph Hellwig return true; 14618741986SChristoph Hellwig } 14747f70d5aSTejun Heo } 14847f70d5aSTejun Heo 149ae1b1539STejun Heo /** 150ae1b1539STejun Heo * blk_flush_complete_seq - complete flush sequence 151ae1b1539STejun Heo * @rq: FLUSH/FUA request being sequenced 1520bae352dSMing Lei * @fq: flush queue 153ae1b1539STejun Heo * @seq: sequences to complete (mask of %REQ_FSEQ_*, can be zero) 154ae1b1539STejun Heo * @error: whether an error occurred 155ae1b1539STejun Heo * 156ae1b1539STejun Heo * @rq just completed @seq part of its flush sequence, record the 157ae1b1539STejun Heo * completion and trigger the next step. 158ae1b1539STejun Heo * 159ae1b1539STejun Heo * CONTEXT: 1607c94e1c1SMing Lei * spin_lock_irq(q->queue_lock or fq->mq_flush_lock) 161ae1b1539STejun Heo * 162ae1b1539STejun Heo * RETURNS: 163ae1b1539STejun Heo * %true if requests were added to the dispatch queue, %false otherwise. 164ae1b1539STejun Heo */ 1650bae352dSMing Lei static bool blk_flush_complete_seq(struct request *rq, 1660bae352dSMing Lei struct blk_flush_queue *fq, 1672a842acaSChristoph Hellwig unsigned int seq, blk_status_t error) 1688839a0e0STejun Heo { 169ae1b1539STejun Heo struct request_queue *q = rq->q; 1707c94e1c1SMing Lei struct list_head *pending = &fq->flush_queue[fq->flush_pending_idx]; 171320ae51fSJens Axboe bool queued = false, kicked; 1728839a0e0STejun Heo 173ae1b1539STejun Heo BUG_ON(rq->flush.seq & seq); 174ae1b1539STejun Heo rq->flush.seq |= seq; 1758839a0e0STejun Heo 176ae1b1539STejun Heo if (likely(!error)) 177ae1b1539STejun Heo seq = blk_flush_cur_seq(rq); 178ae1b1539STejun Heo else 179ae1b1539STejun Heo seq = REQ_FSEQ_DONE; 1808839a0e0STejun Heo 181ae1b1539STejun Heo switch (seq) { 182ae1b1539STejun Heo case REQ_FSEQ_PREFLUSH: 183ae1b1539STejun Heo case REQ_FSEQ_POSTFLUSH: 184ae1b1539STejun Heo /* queue for flush */ 185ae1b1539STejun Heo if (list_empty(pending)) 1867c94e1c1SMing Lei fq->flush_pending_since = jiffies; 187ae1b1539STejun Heo list_move_tail(&rq->flush.list, pending); 1888839a0e0STejun Heo break; 189ae1b1539STejun Heo 190ae1b1539STejun Heo case REQ_FSEQ_DATA: 1917c94e1c1SMing Lei list_move_tail(&rq->flush.list, &fq->flush_data_in_flight); 19210beafc1SMike Snitzer queued = blk_flush_queue_rq(rq, true); 193ae1b1539STejun Heo break; 194ae1b1539STejun Heo 195ae1b1539STejun Heo case REQ_FSEQ_DONE: 19609d60c70STejun Heo /* 197ae1b1539STejun Heo * @rq was previously adjusted by blk_flush_issue() for 198ae1b1539STejun Heo * flush sequencing and may already have gone through the 199ae1b1539STejun Heo * flush data request completion path. Restore @rq for 200ae1b1539STejun Heo * normal completion and end it. 20109d60c70STejun Heo */ 202ae1b1539STejun Heo BUG_ON(!list_empty(&rq->queuelist)); 203ae1b1539STejun Heo list_del_init(&rq->flush.list); 204ae1b1539STejun Heo blk_flush_restore_request(rq); 205320ae51fSJens Axboe if (q->mq_ops) 206c8a446adSChristoph Hellwig blk_mq_end_request(rq, error); 207320ae51fSJens Axboe else 208ae1b1539STejun Heo __blk_end_request_all(rq, error); 2098839a0e0STejun Heo break; 210ae1b1539STejun Heo 2118839a0e0STejun Heo default: 2128839a0e0STejun Heo BUG(); 2138839a0e0STejun Heo } 214cde4c406SChristoph Hellwig 2150bae352dSMing Lei kicked = blk_kick_flush(q, fq); 216320ae51fSJens Axboe return kicked | queued; 2178839a0e0STejun Heo } 2188839a0e0STejun Heo 2192a842acaSChristoph Hellwig static void flush_end_io(struct request *flush_rq, blk_status_t error) 2208839a0e0STejun Heo { 221ae1b1539STejun Heo struct request_queue *q = flush_rq->q; 222320ae51fSJens Axboe struct list_head *running; 223ae1b1539STejun Heo bool queued = false; 224ae1b1539STejun Heo struct request *rq, *n; 225320ae51fSJens Axboe unsigned long flags = 0; 226e97c293cSMing Lei struct blk_flush_queue *fq = blk_get_flush_queue(q, flush_rq->mq_ctx); 2278839a0e0STejun Heo 22822302375SShaohua Li if (q->mq_ops) { 2290048b483SMing Lei struct blk_mq_hw_ctx *hctx; 2300048b483SMing Lei 2310048b483SMing Lei /* release the tag's ownership to the req cloned from */ 2327c94e1c1SMing Lei spin_lock_irqsave(&fq->mq_flush_lock, flags); 2337d7e0f90SChristoph Hellwig hctx = blk_mq_map_queue(q, flush_rq->mq_ctx->cpu); 2340048b483SMing Lei blk_mq_tag_set_rq(hctx, flush_rq->tag, fq->orig_rq); 2357ddab5deSMing Lei flush_rq->tag = -1; 23622302375SShaohua Li } 23718741986SChristoph Hellwig 2387c94e1c1SMing Lei running = &fq->flush_queue[fq->flush_running_idx]; 2397c94e1c1SMing Lei BUG_ON(fq->flush_pending_idx == fq->flush_running_idx); 240ae1b1539STejun Heo 241ae1b1539STejun Heo /* account completion of the flush request */ 2427c94e1c1SMing Lei fq->flush_running_idx ^= 1; 243320ae51fSJens Axboe 244320ae51fSJens Axboe if (!q->mq_ops) 245ae1b1539STejun Heo elv_completed_request(q, flush_rq); 246ae1b1539STejun Heo 247ae1b1539STejun Heo /* and push the waiting requests to the next stage */ 248ae1b1539STejun Heo list_for_each_entry_safe(rq, n, running, flush.list) { 249ae1b1539STejun Heo unsigned int seq = blk_flush_cur_seq(rq); 250ae1b1539STejun Heo 251ae1b1539STejun Heo BUG_ON(seq != REQ_FSEQ_PREFLUSH && seq != REQ_FSEQ_POSTFLUSH); 2520bae352dSMing Lei queued |= blk_flush_complete_seq(rq, fq, seq, error); 253ae1b1539STejun Heo } 254ae1b1539STejun Heo 2558839a0e0STejun Heo /* 2563ac0cc45Sshaohua.li@intel.com * Kick the queue to avoid stall for two cases: 2573ac0cc45Sshaohua.li@intel.com * 1. Moving a request silently to empty queue_head may stall the 2583ac0cc45Sshaohua.li@intel.com * queue. 2593ac0cc45Sshaohua.li@intel.com * 2. When flush request is running in non-queueable queue, the 2603ac0cc45Sshaohua.li@intel.com * queue is hold. Restart the queue after flush request is finished 2613ac0cc45Sshaohua.li@intel.com * to avoid stall. 2623ac0cc45Sshaohua.li@intel.com * This function is called from request completion path and calling 2633ac0cc45Sshaohua.li@intel.com * directly into request_fn may confuse the driver. Always use 2643ac0cc45Sshaohua.li@intel.com * kblockd. 2658839a0e0STejun Heo */ 2667c94e1c1SMing Lei if (queued || fq->flush_queue_delayed) { 26718741986SChristoph Hellwig WARN_ON(q->mq_ops); 26824ecfbe2SChristoph Hellwig blk_run_queue_async(q); 269320ae51fSJens Axboe } 2707c94e1c1SMing Lei fq->flush_queue_delayed = 0; 271320ae51fSJens Axboe if (q->mq_ops) 2727c94e1c1SMing Lei spin_unlock_irqrestore(&fq->mq_flush_lock, flags); 273320ae51fSJens Axboe } 274320ae51fSJens Axboe 275ae1b1539STejun Heo /** 276ae1b1539STejun Heo * blk_kick_flush - consider issuing flush request 277ae1b1539STejun Heo * @q: request_queue being kicked 2780bae352dSMing Lei * @fq: flush queue 2794fed947cSTejun Heo * 280ae1b1539STejun Heo * Flush related states of @q have changed, consider issuing flush request. 281ae1b1539STejun Heo * Please read the comment at the top of this file for more info. 282ae1b1539STejun Heo * 283ae1b1539STejun Heo * CONTEXT: 2847c94e1c1SMing Lei * spin_lock_irq(q->queue_lock or fq->mq_flush_lock) 285ae1b1539STejun Heo * 286ae1b1539STejun Heo * RETURNS: 287ae1b1539STejun Heo * %true if flush was issued, %false otherwise. 2888839a0e0STejun Heo */ 2890bae352dSMing Lei static bool blk_kick_flush(struct request_queue *q, struct blk_flush_queue *fq) 290ae1b1539STejun Heo { 2917c94e1c1SMing Lei struct list_head *pending = &fq->flush_queue[fq->flush_pending_idx]; 292ae1b1539STejun Heo struct request *first_rq = 293ae1b1539STejun Heo list_first_entry(pending, struct request, flush.list); 2947c94e1c1SMing Lei struct request *flush_rq = fq->flush_rq; 295ae1b1539STejun Heo 296ae1b1539STejun Heo /* C1 described at the top of this file */ 2977c94e1c1SMing Lei if (fq->flush_pending_idx != fq->flush_running_idx || list_empty(pending)) 298ae1b1539STejun Heo return false; 299ae1b1539STejun Heo 3007520872cSJens Axboe /* C2 and C3 3017520872cSJens Axboe * 3027520872cSJens Axboe * For blk-mq + scheduling, we can risk having all driver tags 3037520872cSJens Axboe * assigned to empty flushes, and we deadlock if we are expecting 3047520872cSJens Axboe * other requests to make progress. Don't defer for that case. 3057520872cSJens Axboe */ 3067c94e1c1SMing Lei if (!list_empty(&fq->flush_data_in_flight) && 3077520872cSJens Axboe !(q->mq_ops && q->elevator) && 308ae1b1539STejun Heo time_before(jiffies, 3097c94e1c1SMing Lei fq->flush_pending_since + FLUSH_PENDING_TIMEOUT)) 310ae1b1539STejun Heo return false; 311ae1b1539STejun Heo 312ae1b1539STejun Heo /* 313ae1b1539STejun Heo * Issue flush and toggle pending_idx. This makes pending_idx 314ae1b1539STejun Heo * different from running_idx, which means flush is in flight. 315ae1b1539STejun Heo */ 3167c94e1c1SMing Lei fq->flush_pending_idx ^= 1; 31718741986SChristoph Hellwig 3187ddab5deSMing Lei blk_rq_init(q, flush_rq); 319f70ced09SMing Lei 320f70ced09SMing Lei /* 321f70ced09SMing Lei * Borrow tag from the first request since they can't 3220048b483SMing Lei * be in flight at the same time. And acquire the tag's 3230048b483SMing Lei * ownership for flush req. 324f70ced09SMing Lei */ 325f70ced09SMing Lei if (q->mq_ops) { 3260048b483SMing Lei struct blk_mq_hw_ctx *hctx; 3270048b483SMing Lei 328f70ced09SMing Lei flush_rq->mq_ctx = first_rq->mq_ctx; 329f70ced09SMing Lei flush_rq->tag = first_rq->tag; 3300048b483SMing Lei fq->orig_rq = first_rq; 3310048b483SMing Lei 3327d7e0f90SChristoph Hellwig hctx = blk_mq_map_queue(q, first_rq->mq_ctx->cpu); 3330048b483SMing Lei blk_mq_tag_set_rq(hctx, first_rq->tag, flush_rq); 334f70ced09SMing Lei } 335320ae51fSJens Axboe 33670fd7614SChristoph Hellwig flush_rq->cmd_flags = REQ_OP_FLUSH | REQ_PREFLUSH; 337e8064021SChristoph Hellwig flush_rq->rq_flags |= RQF_FLUSH_SEQ; 3387ddab5deSMing Lei flush_rq->rq_disk = first_rq->rq_disk; 3397ddab5deSMing Lei flush_rq->end_io = flush_end_io; 340ae1b1539STejun Heo 3417ddab5deSMing Lei return blk_flush_queue_rq(flush_rq, false); 342ae1b1539STejun Heo } 343ae1b1539STejun Heo 3442a842acaSChristoph Hellwig static void flush_data_end_io(struct request *rq, blk_status_t error) 345ae1b1539STejun Heo { 346ae1b1539STejun Heo struct request_queue *q = rq->q; 347e97c293cSMing Lei struct blk_flush_queue *fq = blk_get_flush_queue(q, NULL); 348ae1b1539STejun Heo 3492fff8a92SBart Van Assche lockdep_assert_held(q->queue_lock); 3502fff8a92SBart Van Assche 3518839a0e0STejun Heo /* 35294d7dea4SMing Lei * Updating q->in_flight[] here for making this tag usable 35394d7dea4SMing Lei * early. Because in blk_queue_start_tag(), 35494d7dea4SMing Lei * q->in_flight[BLK_RW_ASYNC] is used to limit async I/O and 35594d7dea4SMing Lei * reserve tags for sync I/O. 35694d7dea4SMing Lei * 35794d7dea4SMing Lei * More importantly this way can avoid the following I/O 35894d7dea4SMing Lei * deadlock: 35994d7dea4SMing Lei * 36094d7dea4SMing Lei * - suppose there are 40 fua requests comming to flush queue 36194d7dea4SMing Lei * and queue depth is 31 36294d7dea4SMing Lei * - 30 rqs are scheduled then blk_queue_start_tag() can't alloc 36394d7dea4SMing Lei * tag for async I/O any more 36494d7dea4SMing Lei * - all the 30 rqs are completed before FLUSH_PENDING_TIMEOUT 36594d7dea4SMing Lei * and flush_data_end_io() is called 36694d7dea4SMing Lei * - the other rqs still can't go ahead if not updating 36794d7dea4SMing Lei * q->in_flight[BLK_RW_ASYNC] here, meantime these rqs 36894d7dea4SMing Lei * are held in flush data queue and make no progress of 36994d7dea4SMing Lei * handling post flush rq 37094d7dea4SMing Lei * - only after the post flush rq is handled, all these rqs 37194d7dea4SMing Lei * can be completed 37294d7dea4SMing Lei */ 37394d7dea4SMing Lei 37494d7dea4SMing Lei elv_completed_request(q, rq); 37594d7dea4SMing Lei 37694d7dea4SMing Lei /* for avoiding double accounting */ 37736869cb9SLinus Torvalds rq->rq_flags &= ~RQF_STARTED; 37894d7dea4SMing Lei 37994d7dea4SMing Lei /* 380e83a46bbSTejun Heo * After populating an empty queue, kick it to avoid stall. Read 381e83a46bbSTejun Heo * the comment in flush_end_io(). 3828839a0e0STejun Heo */ 3830bae352dSMing Lei if (blk_flush_complete_seq(rq, fq, REQ_FSEQ_DATA, error)) 38424ecfbe2SChristoph Hellwig blk_run_queue_async(q); 385ae1b1539STejun Heo } 386ae1b1539STejun Heo 3872a842acaSChristoph Hellwig static void mq_flush_data_end_io(struct request *rq, blk_status_t error) 388320ae51fSJens Axboe { 389320ae51fSJens Axboe struct request_queue *q = rq->q; 390320ae51fSJens Axboe struct blk_mq_hw_ctx *hctx; 391e97c293cSMing Lei struct blk_mq_ctx *ctx = rq->mq_ctx; 392320ae51fSJens Axboe unsigned long flags; 393e97c293cSMing Lei struct blk_flush_queue *fq = blk_get_flush_queue(q, ctx); 394320ae51fSJens Axboe 3957d7e0f90SChristoph Hellwig hctx = blk_mq_map_queue(q, ctx->cpu); 396320ae51fSJens Axboe 397320ae51fSJens Axboe /* 398320ae51fSJens Axboe * After populating an empty queue, kick it to avoid stall. Read 399320ae51fSJens Axboe * the comment in flush_end_io(). 400320ae51fSJens Axboe */ 4017c94e1c1SMing Lei spin_lock_irqsave(&fq->mq_flush_lock, flags); 402bd166ef1SJens Axboe blk_flush_complete_seq(rq, fq, REQ_FSEQ_DATA, error); 4037c94e1c1SMing Lei spin_unlock_irqrestore(&fq->mq_flush_lock, flags); 404bd166ef1SJens Axboe 405bd166ef1SJens Axboe blk_mq_run_hw_queue(hctx, true); 406320ae51fSJens Axboe } 407320ae51fSJens Axboe 408ae1b1539STejun Heo /** 409ae1b1539STejun Heo * blk_insert_flush - insert a new FLUSH/FUA request 410ae1b1539STejun Heo * @rq: request to insert 411ae1b1539STejun Heo * 412b710a480SJens Axboe * To be called from __elv_add_request() for %ELEVATOR_INSERT_FLUSH insertions. 413320ae51fSJens Axboe * or __blk_mq_run_hw_queue() to dispatch request. 414ae1b1539STejun Heo * @rq is being submitted. Analyze what needs to be done and put it on the 415ae1b1539STejun Heo * right queue. 416ae1b1539STejun Heo */ 417ae1b1539STejun Heo void blk_insert_flush(struct request *rq) 418ae1b1539STejun Heo { 419ae1b1539STejun Heo struct request_queue *q = rq->q; 420c888a8f9SJens Axboe unsigned long fflags = q->queue_flags; /* may change, cache */ 421ae1b1539STejun Heo unsigned int policy = blk_flush_policy(fflags, rq); 422e97c293cSMing Lei struct blk_flush_queue *fq = blk_get_flush_queue(q, rq->mq_ctx); 423ae1b1539STejun Heo 4242fff8a92SBart Van Assche if (!q->mq_ops) 4252fff8a92SBart Van Assche lockdep_assert_held(q->queue_lock); 4262fff8a92SBart Van Assche 427ae1b1539STejun Heo /* 428ae1b1539STejun Heo * @policy now records what operations need to be done. Adjust 42928a8f0d3SMike Christie * REQ_PREFLUSH and FUA for the driver. 430ae1b1539STejun Heo */ 43128a8f0d3SMike Christie rq->cmd_flags &= ~REQ_PREFLUSH; 432c888a8f9SJens Axboe if (!(fflags & (1UL << QUEUE_FLAG_FUA))) 4334fed947cSTejun Heo rq->cmd_flags &= ~REQ_FUA; 434ae1b1539STejun Heo 435ae1b1539STejun Heo /* 436ae5b2ec8SJens Axboe * REQ_PREFLUSH|REQ_FUA implies REQ_SYNC, so if we clear any 437ae5b2ec8SJens Axboe * of those flags, we have to set REQ_SYNC to avoid skewing 438ae5b2ec8SJens Axboe * the request accounting. 439ae5b2ec8SJens Axboe */ 440ae5b2ec8SJens Axboe rq->cmd_flags |= REQ_SYNC; 441ae5b2ec8SJens Axboe 442ae5b2ec8SJens Axboe /* 4434853abaaSJeff Moyer * An empty flush handed down from a stacking driver may 4444853abaaSJeff Moyer * translate into nothing if the underlying device does not 4454853abaaSJeff Moyer * advertise a write-back cache. In this case, simply 4464853abaaSJeff Moyer * complete the request. 4474853abaaSJeff Moyer */ 4484853abaaSJeff Moyer if (!policy) { 449320ae51fSJens Axboe if (q->mq_ops) 450c8a446adSChristoph Hellwig blk_mq_end_request(rq, 0); 451320ae51fSJens Axboe else 452d0fac025SChristoph Hellwig __blk_end_request(rq, 0, 0); 4534853abaaSJeff Moyer return; 4544853abaaSJeff Moyer } 4554853abaaSJeff Moyer 456834f9f61SJeff Moyer BUG_ON(rq->bio != rq->biotail); /*assumes zero or single bio rq */ 4574853abaaSJeff Moyer 4584853abaaSJeff Moyer /* 459ae1b1539STejun Heo * If there's data but flush is not necessary, the request can be 460ae1b1539STejun Heo * processed directly without going through flush machinery. Queue 461ae1b1539STejun Heo * for normal execution. 462ae1b1539STejun Heo */ 463ae1b1539STejun Heo if ((policy & REQ_FSEQ_DATA) && 464ae1b1539STejun Heo !(policy & (REQ_FSEQ_PREFLUSH | REQ_FSEQ_POSTFLUSH))) { 465bd166ef1SJens Axboe if (q->mq_ops) 466bd6737f1SJens Axboe blk_mq_sched_insert_request(rq, false, true, false, false); 467bd166ef1SJens Axboe else 468dcd8376cSJens Axboe list_add_tail(&rq->queuelist, &q->queue_head); 469ae1b1539STejun Heo return; 4708839a0e0STejun Heo } 4718839a0e0STejun Heo 4728839a0e0STejun Heo /* 473ae1b1539STejun Heo * @rq should go through flush machinery. Mark it part of flush 474ae1b1539STejun Heo * sequence and submit for further processing. 4758839a0e0STejun Heo */ 476ae1b1539STejun Heo memset(&rq->flush, 0, sizeof(rq->flush)); 477ae1b1539STejun Heo INIT_LIST_HEAD(&rq->flush.list); 478e8064021SChristoph Hellwig rq->rq_flags |= RQF_FLUSH_SEQ; 4794853abaaSJeff Moyer rq->flush.saved_end_io = rq->end_io; /* Usually NULL */ 480320ae51fSJens Axboe if (q->mq_ops) { 481320ae51fSJens Axboe rq->end_io = mq_flush_data_end_io; 482320ae51fSJens Axboe 4837c94e1c1SMing Lei spin_lock_irq(&fq->mq_flush_lock); 4840bae352dSMing Lei blk_flush_complete_seq(rq, fq, REQ_FSEQ_ACTIONS & ~policy, 0); 4857c94e1c1SMing Lei spin_unlock_irq(&fq->mq_flush_lock); 486320ae51fSJens Axboe return; 487320ae51fSJens Axboe } 488ae1b1539STejun Heo rq->end_io = flush_data_end_io; 489ae1b1539STejun Heo 4900bae352dSMing Lei blk_flush_complete_seq(rq, fq, REQ_FSEQ_ACTIONS & ~policy, 0); 491ae1b1539STejun Heo } 492ae1b1539STejun Heo 493ae1b1539STejun Heo /** 4948839a0e0STejun Heo * blkdev_issue_flush - queue a flush 4958839a0e0STejun Heo * @bdev: blockdev to issue flush for 4968839a0e0STejun Heo * @gfp_mask: memory allocation flags (for bio_alloc) 4978839a0e0STejun Heo * @error_sector: error sector 4988839a0e0STejun Heo * 4998839a0e0STejun Heo * Description: 5008839a0e0STejun Heo * Issue a flush for the block device in question. Caller can supply 5018839a0e0STejun Heo * room for storing the error offset in case of a flush error, if they 5021be7d207SEric Biggers * wish to. 5038839a0e0STejun Heo */ 5048839a0e0STejun Heo int blkdev_issue_flush(struct block_device *bdev, gfp_t gfp_mask, 505dd3932edSChristoph Hellwig sector_t *error_sector) 5068839a0e0STejun Heo { 5078839a0e0STejun Heo struct request_queue *q; 5088839a0e0STejun Heo struct bio *bio; 5098839a0e0STejun Heo int ret = 0; 5108839a0e0STejun Heo 5118839a0e0STejun Heo if (bdev->bd_disk == NULL) 5128839a0e0STejun Heo return -ENXIO; 5138839a0e0STejun Heo 5148839a0e0STejun Heo q = bdev_get_queue(bdev); 5158839a0e0STejun Heo if (!q) 5168839a0e0STejun Heo return -ENXIO; 5178839a0e0STejun Heo 5188839a0e0STejun Heo /* 5198839a0e0STejun Heo * some block devices may not have their queue correctly set up here 5208839a0e0STejun Heo * (e.g. loop device without a backing file) and so issuing a flush 5218839a0e0STejun Heo * here will panic. Ensure there is a request function before issuing 522d391a2ddSTejun Heo * the flush. 5238839a0e0STejun Heo */ 5248839a0e0STejun Heo if (!q->make_request_fn) 5258839a0e0STejun Heo return -ENXIO; 5268839a0e0STejun Heo 5278839a0e0STejun Heo bio = bio_alloc(gfp_mask, 0); 52874d46992SChristoph Hellwig bio_set_dev(bio, bdev); 52970fd7614SChristoph Hellwig bio->bi_opf = REQ_OP_WRITE | REQ_PREFLUSH; 5308839a0e0STejun Heo 5314e49ea4aSMike Christie ret = submit_bio_wait(bio); 532dd3932edSChristoph Hellwig 5338839a0e0STejun Heo /* 5348839a0e0STejun Heo * The driver must store the error location in ->bi_sector, if 5358839a0e0STejun Heo * it supports it. For non-stacked drivers, this should be 5368839a0e0STejun Heo * copied from blk_rq_pos(rq). 5378839a0e0STejun Heo */ 5388839a0e0STejun Heo if (error_sector) 5394f024f37SKent Overstreet *error_sector = bio->bi_iter.bi_sector; 5408839a0e0STejun Heo 5418839a0e0STejun Heo bio_put(bio); 5428839a0e0STejun Heo return ret; 5438839a0e0STejun Heo } 5448839a0e0STejun Heo EXPORT_SYMBOL(blkdev_issue_flush); 545320ae51fSJens Axboe 546f70ced09SMing Lei struct blk_flush_queue *blk_alloc_flush_queue(struct request_queue *q, 547f70ced09SMing Lei int node, int cmd_size) 548320ae51fSJens Axboe { 5497c94e1c1SMing Lei struct blk_flush_queue *fq; 5507c94e1c1SMing Lei int rq_sz = sizeof(struct request); 5511bcb1eadSMing Lei 552f70ced09SMing Lei fq = kzalloc_node(sizeof(*fq), GFP_KERNEL, node); 5537c94e1c1SMing Lei if (!fq) 5547c94e1c1SMing Lei goto fail; 5551bcb1eadSMing Lei 5566d247d7fSChristoph Hellwig if (q->mq_ops) 5577c94e1c1SMing Lei spin_lock_init(&fq->mq_flush_lock); 5587c94e1c1SMing Lei 5596d247d7fSChristoph Hellwig rq_sz = round_up(rq_sz + cmd_size, cache_line_size()); 560f70ced09SMing Lei fq->flush_rq = kzalloc_node(rq_sz, GFP_KERNEL, node); 5617c94e1c1SMing Lei if (!fq->flush_rq) 5627c94e1c1SMing Lei goto fail_rq; 5637c94e1c1SMing Lei 5647c94e1c1SMing Lei INIT_LIST_HEAD(&fq->flush_queue[0]); 5657c94e1c1SMing Lei INIT_LIST_HEAD(&fq->flush_queue[1]); 5667c94e1c1SMing Lei INIT_LIST_HEAD(&fq->flush_data_in_flight); 5677c94e1c1SMing Lei 5687c94e1c1SMing Lei return fq; 5697c94e1c1SMing Lei 5707c94e1c1SMing Lei fail_rq: 5717c94e1c1SMing Lei kfree(fq); 5727c94e1c1SMing Lei fail: 5737c94e1c1SMing Lei return NULL; 5747c94e1c1SMing Lei } 5757c94e1c1SMing Lei 576ba483388SMing Lei void blk_free_flush_queue(struct blk_flush_queue *fq) 5777c94e1c1SMing Lei { 5787c94e1c1SMing Lei /* bio based request queue hasn't flush queue */ 5797c94e1c1SMing Lei if (!fq) 5807c94e1c1SMing Lei return; 5817c94e1c1SMing Lei 5827c94e1c1SMing Lei kfree(fq->flush_rq); 5837c94e1c1SMing Lei kfree(fq); 584320ae51fSJens Axboe } 585