1 /* 2 * Copyright (C) 2003 Russell King, All Rights Reserved. 3 * Copyright 2006-2007 Pierre Ossman 4 * 5 * This program is free software; you can redistribute it and/or modify 6 * it under the terms of the GNU General Public License version 2 as 7 * published by the Free Software Foundation. 8 * 9 */ 10 #include <linux/slab.h> 11 #include <linux/module.h> 12 #include <linux/blkdev.h> 13 #include <linux/freezer.h> 14 #include <linux/kthread.h> 15 #include <linux/scatterlist.h> 16 #include <linux/dma-mapping.h> 17 18 #include <linux/mmc/card.h> 19 #include <linux/mmc/host.h> 20 21 #include "queue.h" 22 #include "block.h" 23 #include "core.h" 24 #include "card.h" 25 26 #define MMC_QUEUE_BOUNCESZ 65536 27 28 /* 29 * Prepare a MMC request. This just filters out odd stuff. 30 */ 31 static int mmc_prep_request(struct request_queue *q, struct request *req) 32 { 33 struct mmc_queue *mq = q->queuedata; 34 35 if (mq && (mmc_card_removed(mq->card) || mmc_access_rpmb(mq))) 36 return BLKPREP_KILL; 37 38 req->rq_flags |= RQF_DONTPREP; 39 40 return BLKPREP_OK; 41 } 42 43 static int mmc_queue_thread(void *d) 44 { 45 struct mmc_queue *mq = d; 46 struct request_queue *q = mq->queue; 47 struct mmc_context_info *cntx = &mq->card->host->context_info; 48 49 current->flags |= PF_MEMALLOC; 50 51 down(&mq->thread_sem); 52 do { 53 struct request *req; 54 55 spin_lock_irq(q->queue_lock); 56 set_current_state(TASK_INTERRUPTIBLE); 57 req = blk_fetch_request(q); 58 mq->asleep = false; 59 cntx->is_waiting_last_req = false; 60 cntx->is_new_req = false; 61 if (!req) { 62 /* 63 * Dispatch queue is empty so set flags for 64 * mmc_request_fn() to wake us up. 65 */ 66 if (mq->qcnt) 67 cntx->is_waiting_last_req = true; 68 else 69 mq->asleep = true; 70 } 71 spin_unlock_irq(q->queue_lock); 72 73 if (req || mq->qcnt) { 74 set_current_state(TASK_RUNNING); 75 mmc_blk_issue_rq(mq, req); 76 cond_resched(); 77 } else { 78 if (kthread_should_stop()) { 79 set_current_state(TASK_RUNNING); 80 break; 81 } 82 up(&mq->thread_sem); 83 schedule(); 84 down(&mq->thread_sem); 85 } 86 } while (1); 87 up(&mq->thread_sem); 88 89 return 0; 90 } 91 92 /* 93 * Generic MMC request handler. This is called for any queue on a 94 * particular host. When the host is not busy, we look for a request 95 * on any queue on this host, and attempt to issue it. This may 96 * not be the queue we were asked to process. 97 */ 98 static void mmc_request_fn(struct request_queue *q) 99 { 100 struct mmc_queue *mq = q->queuedata; 101 struct request *req; 102 struct mmc_context_info *cntx; 103 104 if (!mq) { 105 while ((req = blk_fetch_request(q)) != NULL) { 106 req->rq_flags |= RQF_QUIET; 107 __blk_end_request_all(req, BLK_STS_IOERR); 108 } 109 return; 110 } 111 112 cntx = &mq->card->host->context_info; 113 114 if (cntx->is_waiting_last_req) { 115 cntx->is_new_req = true; 116 wake_up_interruptible(&cntx->wait); 117 } 118 119 if (mq->asleep) 120 wake_up_process(mq->thread); 121 } 122 123 static struct scatterlist *mmc_alloc_sg(int sg_len, gfp_t gfp) 124 { 125 struct scatterlist *sg; 126 127 sg = kmalloc_array(sg_len, sizeof(*sg), gfp); 128 if (sg) 129 sg_init_table(sg, sg_len); 130 131 return sg; 132 } 133 134 static void mmc_queue_setup_discard(struct request_queue *q, 135 struct mmc_card *card) 136 { 137 unsigned max_discard; 138 139 max_discard = mmc_calc_max_discard(card); 140 if (!max_discard) 141 return; 142 143 queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, q); 144 blk_queue_max_discard_sectors(q, max_discard); 145 q->limits.discard_granularity = card->pref_erase << 9; 146 /* granularity must not be greater than max. discard */ 147 if (card->pref_erase > max_discard) 148 q->limits.discard_granularity = 0; 149 if (mmc_can_secure_erase_trim(card)) 150 queue_flag_set_unlocked(QUEUE_FLAG_SECERASE, q); 151 } 152 153 static unsigned int mmc_queue_calc_bouncesz(struct mmc_host *host) 154 { 155 unsigned int bouncesz = MMC_QUEUE_BOUNCESZ; 156 157 if (host->max_segs != 1 || (host->caps & MMC_CAP_NO_BOUNCE_BUFF)) 158 return 0; 159 160 if (bouncesz > host->max_req_size) 161 bouncesz = host->max_req_size; 162 if (bouncesz > host->max_seg_size) 163 bouncesz = host->max_seg_size; 164 if (bouncesz > host->max_blk_count * 512) 165 bouncesz = host->max_blk_count * 512; 166 167 if (bouncesz <= 512) 168 return 0; 169 170 return bouncesz; 171 } 172 173 /** 174 * mmc_init_request() - initialize the MMC-specific per-request data 175 * @q: the request queue 176 * @req: the request 177 * @gfp: memory allocation policy 178 */ 179 static int mmc_init_request(struct request_queue *q, struct request *req, 180 gfp_t gfp) 181 { 182 struct mmc_queue_req *mq_rq = req_to_mmc_queue_req(req); 183 struct mmc_queue *mq = q->queuedata; 184 struct mmc_card *card = mq->card; 185 struct mmc_host *host = card->host; 186 187 if (card->bouncesz) { 188 mq_rq->bounce_buf = kmalloc(card->bouncesz, gfp); 189 if (!mq_rq->bounce_buf) 190 return -ENOMEM; 191 if (card->bouncesz > 512) { 192 mq_rq->sg = mmc_alloc_sg(1, gfp); 193 if (!mq_rq->sg) 194 return -ENOMEM; 195 mq_rq->bounce_sg = mmc_alloc_sg(card->bouncesz / 512, 196 gfp); 197 if (!mq_rq->bounce_sg) 198 return -ENOMEM; 199 } 200 } else { 201 mq_rq->bounce_buf = NULL; 202 mq_rq->bounce_sg = NULL; 203 mq_rq->sg = mmc_alloc_sg(host->max_segs, gfp); 204 if (!mq_rq->sg) 205 return -ENOMEM; 206 } 207 208 return 0; 209 } 210 211 static void mmc_exit_request(struct request_queue *q, struct request *req) 212 { 213 struct mmc_queue_req *mq_rq = req_to_mmc_queue_req(req); 214 215 /* It is OK to kfree(NULL) so this will be smooth */ 216 kfree(mq_rq->bounce_sg); 217 mq_rq->bounce_sg = NULL; 218 219 kfree(mq_rq->bounce_buf); 220 mq_rq->bounce_buf = NULL; 221 222 kfree(mq_rq->sg); 223 mq_rq->sg = NULL; 224 } 225 226 /** 227 * mmc_init_queue - initialise a queue structure. 228 * @mq: mmc queue 229 * @card: mmc card to attach this queue 230 * @lock: queue lock 231 * @subname: partition subname 232 * 233 * Initialise a MMC card request queue. 234 */ 235 int mmc_init_queue(struct mmc_queue *mq, struct mmc_card *card, 236 spinlock_t *lock, const char *subname) 237 { 238 struct mmc_host *host = card->host; 239 u64 limit = BLK_BOUNCE_HIGH; 240 int ret = -ENOMEM; 241 242 if (mmc_dev(host)->dma_mask && *mmc_dev(host)->dma_mask) 243 limit = (u64)dma_max_pfn(mmc_dev(host)) << PAGE_SHIFT; 244 245 mq->card = card; 246 mq->queue = blk_alloc_queue(GFP_KERNEL); 247 if (!mq->queue) 248 return -ENOMEM; 249 mq->queue->queue_lock = lock; 250 mq->queue->request_fn = mmc_request_fn; 251 mq->queue->init_rq_fn = mmc_init_request; 252 mq->queue->exit_rq_fn = mmc_exit_request; 253 mq->queue->cmd_size = sizeof(struct mmc_queue_req); 254 mq->queue->queuedata = mq; 255 mq->qcnt = 0; 256 ret = blk_init_allocated_queue(mq->queue); 257 if (ret) { 258 blk_cleanup_queue(mq->queue); 259 return ret; 260 } 261 262 blk_queue_prep_rq(mq->queue, mmc_prep_request); 263 queue_flag_set_unlocked(QUEUE_FLAG_NONROT, mq->queue); 264 queue_flag_clear_unlocked(QUEUE_FLAG_ADD_RANDOM, mq->queue); 265 if (mmc_can_erase(card)) 266 mmc_queue_setup_discard(mq->queue, card); 267 268 card->bouncesz = mmc_queue_calc_bouncesz(host); 269 if (card->bouncesz) { 270 blk_queue_max_hw_sectors(mq->queue, card->bouncesz / 512); 271 blk_queue_max_segments(mq->queue, card->bouncesz / 512); 272 blk_queue_max_segment_size(mq->queue, card->bouncesz); 273 } else { 274 blk_queue_bounce_limit(mq->queue, limit); 275 blk_queue_max_hw_sectors(mq->queue, 276 min(host->max_blk_count, host->max_req_size / 512)); 277 blk_queue_max_segments(mq->queue, host->max_segs); 278 blk_queue_max_segment_size(mq->queue, host->max_seg_size); 279 } 280 281 sema_init(&mq->thread_sem, 1); 282 283 mq->thread = kthread_run(mmc_queue_thread, mq, "mmcqd/%d%s", 284 host->index, subname ? subname : ""); 285 286 if (IS_ERR(mq->thread)) { 287 ret = PTR_ERR(mq->thread); 288 goto cleanup_queue; 289 } 290 291 return 0; 292 293 cleanup_queue: 294 blk_cleanup_queue(mq->queue); 295 return ret; 296 } 297 298 void mmc_cleanup_queue(struct mmc_queue *mq) 299 { 300 struct request_queue *q = mq->queue; 301 unsigned long flags; 302 303 /* Make sure the queue isn't suspended, as that will deadlock */ 304 mmc_queue_resume(mq); 305 306 /* Then terminate our worker thread */ 307 kthread_stop(mq->thread); 308 309 /* Empty the queue */ 310 spin_lock_irqsave(q->queue_lock, flags); 311 q->queuedata = NULL; 312 blk_start_queue(q); 313 spin_unlock_irqrestore(q->queue_lock, flags); 314 315 mq->card = NULL; 316 } 317 EXPORT_SYMBOL(mmc_cleanup_queue); 318 319 /** 320 * mmc_queue_suspend - suspend a MMC request queue 321 * @mq: MMC queue to suspend 322 * 323 * Stop the block request queue, and wait for our thread to 324 * complete any outstanding requests. This ensures that we 325 * won't suspend while a request is being processed. 326 */ 327 void mmc_queue_suspend(struct mmc_queue *mq) 328 { 329 struct request_queue *q = mq->queue; 330 unsigned long flags; 331 332 if (!mq->suspended) { 333 mq->suspended |= true; 334 335 spin_lock_irqsave(q->queue_lock, flags); 336 blk_stop_queue(q); 337 spin_unlock_irqrestore(q->queue_lock, flags); 338 339 down(&mq->thread_sem); 340 } 341 } 342 343 /** 344 * mmc_queue_resume - resume a previously suspended MMC request queue 345 * @mq: MMC queue to resume 346 */ 347 void mmc_queue_resume(struct mmc_queue *mq) 348 { 349 struct request_queue *q = mq->queue; 350 unsigned long flags; 351 352 if (mq->suspended) { 353 mq->suspended = false; 354 355 up(&mq->thread_sem); 356 357 spin_lock_irqsave(q->queue_lock, flags); 358 blk_start_queue(q); 359 spin_unlock_irqrestore(q->queue_lock, flags); 360 } 361 } 362 363 /* 364 * Prepare the sg list(s) to be handed of to the host driver 365 */ 366 unsigned int mmc_queue_map_sg(struct mmc_queue *mq, struct mmc_queue_req *mqrq) 367 { 368 unsigned int sg_len; 369 size_t buflen; 370 struct scatterlist *sg; 371 struct request *req = mmc_queue_req_to_req(mqrq); 372 int i; 373 374 if (!mqrq->bounce_buf) 375 return blk_rq_map_sg(mq->queue, req, mqrq->sg); 376 377 sg_len = blk_rq_map_sg(mq->queue, req, mqrq->bounce_sg); 378 379 mqrq->bounce_sg_len = sg_len; 380 381 buflen = 0; 382 for_each_sg(mqrq->bounce_sg, sg, sg_len, i) 383 buflen += sg->length; 384 385 sg_init_one(mqrq->sg, mqrq->bounce_buf, buflen); 386 387 return 1; 388 } 389 390 /* 391 * If writing, bounce the data to the buffer before the request 392 * is sent to the host driver 393 */ 394 void mmc_queue_bounce_pre(struct mmc_queue_req *mqrq) 395 { 396 if (!mqrq->bounce_buf) 397 return; 398 399 if (rq_data_dir(mmc_queue_req_to_req(mqrq)) != WRITE) 400 return; 401 402 sg_copy_to_buffer(mqrq->bounce_sg, mqrq->bounce_sg_len, 403 mqrq->bounce_buf, mqrq->sg[0].length); 404 } 405 406 /* 407 * If reading, bounce the data from the buffer after the request 408 * has been handled by the host driver 409 */ 410 void mmc_queue_bounce_post(struct mmc_queue_req *mqrq) 411 { 412 if (!mqrq->bounce_buf) 413 return; 414 415 if (rq_data_dir(mmc_queue_req_to_req(mqrq)) != READ) 416 return; 417 418 sg_copy_from_buffer(mqrq->bounce_sg, mqrq->bounce_sg_len, 419 mqrq->bounce_buf, mqrq->sg[0].length); 420 } 421