direct-io.c (cebeb0f1885fa93c44be5d4e0b9b640210ff088c) direct-io.c (31b140398ce56ab41646eda7f02bcb78d6a4c916)
1/*
2 * fs/direct-io.c
3 *
4 * Copyright (C) 2002, Linus Torvalds.
5 *
6 * O_DIRECT
7 *
8 * 04Jul2002 Andrew Morton

--- 1093 unchanged lines hidden (view full) ---

1102 *
1103 * NOTE: if you pass "sdio" to anything by pointer make sure that function
1104 * is always inlined. Otherwise gcc is unable to split the structure into
1105 * individual fields and will generate much worse code. This is important
1106 * for the whole file.
1107 */
1108static inline ssize_t
1109do_blockdev_direct_IO(int rw, struct kiocb *iocb, struct inode *inode,
1/*
2 * fs/direct-io.c
3 *
4 * Copyright (C) 2002, Linus Torvalds.
5 *
6 * O_DIRECT
7 *
8 * 04Jul2002 Andrew Morton

--- 1093 unchanged lines hidden (view full) ---

1102 *
1103 * NOTE: if you pass "sdio" to anything by pointer make sure that function
1104 * is always inlined. Otherwise gcc is unable to split the structure into
1105 * individual fields and will generate much worse code. This is important
1106 * for the whole file.
1107 */
1108static inline ssize_t
1109do_blockdev_direct_IO(int rw, struct kiocb *iocb, struct inode *inode,
1110 struct block_device *bdev, const struct iovec *iov, loff_t offset,
1111 unsigned long nr_segs, get_block_t get_block, dio_iodone_t end_io,
1110 struct block_device *bdev, struct iov_iter *iter, loff_t offset,
1111 get_block_t get_block, dio_iodone_t end_io,
1112 dio_submit_t submit_io, int flags)
1113{
1114 int seg;
1115 size_t size;
1116 unsigned long addr;
1117 unsigned i_blkbits = ACCESS_ONCE(inode->i_blkbits);
1118 unsigned blkbits = i_blkbits;
1119 unsigned blocksize_mask = (1 << blkbits) - 1;

--- 18 unchanged lines hidden (view full) ---

1138 if (bdev)
1139 blkbits = blksize_bits(bdev_logical_block_size(bdev));
1140 blocksize_mask = (1 << blkbits) - 1;
1141 if (offset & blocksize_mask)
1142 goto out;
1143 }
1144
1145 /* Check the memory alignment. Blocks cannot straddle pages */
1112 dio_submit_t submit_io, int flags)
1113{
1114 int seg;
1115 size_t size;
1116 unsigned long addr;
1117 unsigned i_blkbits = ACCESS_ONCE(inode->i_blkbits);
1118 unsigned blkbits = i_blkbits;
1119 unsigned blocksize_mask = (1 << blkbits) - 1;

--- 18 unchanged lines hidden (view full) ---

1138 if (bdev)
1139 blkbits = blksize_bits(bdev_logical_block_size(bdev));
1140 blocksize_mask = (1 << blkbits) - 1;
1141 if (offset & blocksize_mask)
1142 goto out;
1143 }
1144
1145 /* Check the memory alignment. Blocks cannot straddle pages */
1146 for (seg = 0; seg < nr_segs; seg++) {
1147 addr = (unsigned long)iov[seg].iov_base;
1148 size = iov[seg].iov_len;
1146 for (seg = 0; seg < iter->nr_segs; seg++) {
1147 addr = (unsigned long)iter->iov[seg].iov_base;
1148 size = iter->iov[seg].iov_len;
1149 end += size;
1150 if (unlikely((addr & blocksize_mask) ||
1151 (size & blocksize_mask))) {
1152 if (bdev)
1153 blkbits = blksize_bits(
1154 bdev_logical_block_size(bdev));
1155 blocksize_mask = (1 << blkbits) - 1;
1156 if ((addr & blocksize_mask) || (size & blocksize_mask))

--- 94 unchanged lines hidden (view full) ---

1251
1252 /*
1253 * In case of non-aligned buffers, we may need 2 more
1254 * pages since we need to zero out first and last block.
1255 */
1256 if (unlikely(sdio.blkfactor))
1257 sdio.pages_in_io = 2;
1258
1149 end += size;
1150 if (unlikely((addr & blocksize_mask) ||
1151 (size & blocksize_mask))) {
1152 if (bdev)
1153 blkbits = blksize_bits(
1154 bdev_logical_block_size(bdev));
1155 blocksize_mask = (1 << blkbits) - 1;
1156 if ((addr & blocksize_mask) || (size & blocksize_mask))

--- 94 unchanged lines hidden (view full) ---

1251
1252 /*
1253 * In case of non-aligned buffers, we may need 2 more
1254 * pages since we need to zero out first and last block.
1255 */
1256 if (unlikely(sdio.blkfactor))
1257 sdio.pages_in_io = 2;
1258
1259 for (seg = 0; seg < nr_segs; seg++) {
1260 user_addr = (unsigned long)iov[seg].iov_base;
1259 for (seg = 0; seg < iter->nr_segs; seg++) {
1260 user_addr = (unsigned long)iter->iov[seg].iov_base;
1261 sdio.pages_in_io +=
1261 sdio.pages_in_io +=
1262 ((user_addr + iov[seg].iov_len + PAGE_SIZE-1) /
1262 ((user_addr + iter->iov[seg].iov_len + PAGE_SIZE-1) /
1263 PAGE_SIZE - user_addr / PAGE_SIZE);
1264 }
1265
1266 blk_start_plug(&plug);
1267
1263 PAGE_SIZE - user_addr / PAGE_SIZE);
1264 }
1265
1266 blk_start_plug(&plug);
1267
1268 for (seg = 0; seg < nr_segs; seg++) {
1269 user_addr = (unsigned long)iov[seg].iov_base;
1270 sdio.size += bytes = iov[seg].iov_len;
1268 for (seg = 0; seg < iter->nr_segs; seg++) {
1269 user_addr = (unsigned long)iter->iov[seg].iov_base;
1270 sdio.size += bytes = iter->iov[seg].iov_len;
1271
1272 /* Index into the first page of the first block */
1273 sdio.first_block_in_page = (user_addr & ~PAGE_MASK) >> blkbits;
1274 sdio.final_block_in_request = sdio.block_in_file +
1275 (bytes >> blkbits);
1276 /* Page fetching state */
1277 sdio.head = 0;
1278 sdio.tail = 0;

--- 4 unchanged lines hidden (view full) ---

1283 sdio.total_pages++;
1284 bytes -= PAGE_SIZE - (user_addr & (PAGE_SIZE - 1));
1285 }
1286 sdio.total_pages += (bytes + PAGE_SIZE - 1) / PAGE_SIZE;
1287 sdio.curr_user_address = user_addr;
1288
1289 retval = do_direct_IO(dio, &sdio, &map_bh);
1290
1271
1272 /* Index into the first page of the first block */
1273 sdio.first_block_in_page = (user_addr & ~PAGE_MASK) >> blkbits;
1274 sdio.final_block_in_request = sdio.block_in_file +
1275 (bytes >> blkbits);
1276 /* Page fetching state */
1277 sdio.head = 0;
1278 sdio.tail = 0;

--- 4 unchanged lines hidden (view full) ---

1283 sdio.total_pages++;
1284 bytes -= PAGE_SIZE - (user_addr & (PAGE_SIZE - 1));
1285 }
1286 sdio.total_pages += (bytes + PAGE_SIZE - 1) / PAGE_SIZE;
1287 sdio.curr_user_address = user_addr;
1288
1289 retval = do_direct_IO(dio, &sdio, &map_bh);
1290
1291 dio->result += iov[seg].iov_len -
1291 dio->result += iter->iov[seg].iov_len -
1292 ((sdio.final_block_in_request - sdio.block_in_file) <<
1293 blkbits);
1294
1295 if (retval) {
1296 dio_cleanup(dio, &sdio);
1297 break;
1298 }
1299 } /* end iovec loop */

--- 60 unchanged lines hidden (view full) ---

1360 BUG_ON(retval != -EIOCBQUEUED);
1361
1362out:
1363 return retval;
1364}
1365
1366ssize_t
1367__blockdev_direct_IO(int rw, struct kiocb *iocb, struct inode *inode,
1292 ((sdio.final_block_in_request - sdio.block_in_file) <<
1293 blkbits);
1294
1295 if (retval) {
1296 dio_cleanup(dio, &sdio);
1297 break;
1298 }
1299 } /* end iovec loop */

--- 60 unchanged lines hidden (view full) ---

1360 BUG_ON(retval != -EIOCBQUEUED);
1361
1362out:
1363 return retval;
1364}
1365
1366ssize_t
1367__blockdev_direct_IO(int rw, struct kiocb *iocb, struct inode *inode,
1368 struct block_device *bdev, const struct iovec *iov, loff_t offset,
1369 unsigned long nr_segs, get_block_t get_block, dio_iodone_t end_io,
1368 struct block_device *bdev, struct iov_iter *iter, loff_t offset,
1369 get_block_t get_block, dio_iodone_t end_io,
1370 dio_submit_t submit_io, int flags)
1371{
1372 /*
1373 * The block device state is needed in the end to finally
1374 * submit everything. Since it's likely to be cache cold
1375 * prefetch it here as first thing to hide some of the
1376 * latency.
1377 *
1378 * Attempt to prefetch the pieces we likely need later.
1379 */
1380 prefetch(&bdev->bd_disk->part_tbl);
1381 prefetch(bdev->bd_queue);
1382 prefetch((char *)bdev->bd_queue + SMP_CACHE_BYTES);
1383
1370 dio_submit_t submit_io, int flags)
1371{
1372 /*
1373 * The block device state is needed in the end to finally
1374 * submit everything. Since it's likely to be cache cold
1375 * prefetch it here as first thing to hide some of the
1376 * latency.
1377 *
1378 * Attempt to prefetch the pieces we likely need later.
1379 */
1380 prefetch(&bdev->bd_disk->part_tbl);
1381 prefetch(bdev->bd_queue);
1382 prefetch((char *)bdev->bd_queue + SMP_CACHE_BYTES);
1383
1384 return do_blockdev_direct_IO(rw, iocb, inode, bdev, iov, offset,
1385 nr_segs, get_block, end_io,
1386 submit_io, flags);
1384 return do_blockdev_direct_IO(rw, iocb, inode, bdev, iter, offset,
1385 get_block, end_io, submit_io, flags);
1387}
1388
1389EXPORT_SYMBOL(__blockdev_direct_IO);
1390
1391static __init int dio_init(void)
1392{
1393 dio_cache = KMEM_CACHE(dio, SLAB_PANIC);
1394 return 0;
1395}
1396module_init(dio_init)
1386}
1387
1388EXPORT_SYMBOL(__blockdev_direct_IO);
1389
1390static __init int dio_init(void)
1391{
1392 dio_cache = KMEM_CACHE(dio, SLAB_PANIC);
1393 return 0;
1394}
1395module_init(dio_init)