dm.c (55a62eef8d1b50ceff3b7bf46851103bdcc7e5b0) dm.c (14fe594d679c9ba8c8e3d6ad1a3ed9c0ba336df0)
1/*
2 * Copyright (C) 2001, 2002 Sistina Software (UK) Limited.
3 * Copyright (C) 2004-2008 Red Hat, Inc. All rights reserved.
4 *
5 * This file is released under the GPL.
6 */
7
8#include "dm.h"

--- 1047 unchanged lines hidden (view full) ---

1056
1057 if (trim)
1058 bio_integrity_trim(clone, bio_sector_offset(bio, idx, offset), len);
1059}
1060
1061/*
1062 * Creates a little bio that just does part of a bvec.
1063 */
1/*
2 * Copyright (C) 2001, 2002 Sistina Software (UK) Limited.
3 * Copyright (C) 2004-2008 Red Hat, Inc. All rights reserved.
4 *
5 * This file is released under the GPL.
6 */
7
8#include "dm.h"

--- 1047 unchanged lines hidden (view full) ---

1056
1057 if (trim)
1058 bio_integrity_trim(clone, bio_sector_offset(bio, idx, offset), len);
1059}
1060
1061/*
1062 * Creates a little bio that just does part of a bvec.
1063 */
1064static void split_bvec(struct dm_target_io *tio, struct bio *bio,
1065 sector_t sector, unsigned short idx, unsigned int offset,
1066 unsigned int len)
1064static void clone_split_bio(struct dm_target_io *tio, struct bio *bio,
1065 sector_t sector, unsigned short idx,
1066 unsigned offset, unsigned len)
1067{
1068 struct bio *clone = &tio->clone;
1069 struct bio_vec *bv = bio->bi_io_vec + idx;
1070
1071 *clone->bi_io_vec = *bv;
1072
1073 bio_setup_sector(clone, sector, len);
1074

--- 39 unchanged lines hidden (view full) ---

1114 tio->io = ci->io;
1115 tio->ti = ti;
1116 memset(&tio->info, 0, sizeof(tio->info));
1117 tio->target_bio_nr = target_bio_nr;
1118
1119 return tio;
1120}
1121
1067{
1068 struct bio *clone = &tio->clone;
1069 struct bio_vec *bv = bio->bi_io_vec + idx;
1070
1071 *clone->bi_io_vec = *bv;
1072
1073 bio_setup_sector(clone, sector, len);
1074

--- 39 unchanged lines hidden (view full) ---

1114 tio->io = ci->io;
1115 tio->ti = ti;
1116 memset(&tio->info, 0, sizeof(tio->info));
1117 tio->target_bio_nr = target_bio_nr;
1118
1119 return tio;
1120}
1121
1122static void __issue_target_request(struct clone_info *ci, struct dm_target *ti,
1123 unsigned target_bio_nr, sector_t len)
1122static void __clone_and_map_simple_bio(struct clone_info *ci,
1123 struct dm_target *ti,
1124 unsigned target_bio_nr, sector_t len)
1124{
1125 struct dm_target_io *tio = alloc_tio(ci, ti, ci->bio->bi_max_vecs, target_bio_nr);
1126 struct bio *clone = &tio->clone;
1127
1128 /*
1129 * Discard requests require the bio's inline iovecs be initialized.
1130 * ci->bio->bi_max_vecs is BIO_INLINE_VECS anyway, for both flush
1131 * and discard, so no need for concern about wasted bvec allocations.
1132 */
1133 __bio_clone(clone, ci->bio);
1134 if (len)
1135 bio_setup_sector(clone, ci->sector, len);
1136
1137 __map_bio(tio);
1138}
1139
1125{
1126 struct dm_target_io *tio = alloc_tio(ci, ti, ci->bio->bi_max_vecs, target_bio_nr);
1127 struct bio *clone = &tio->clone;
1128
1129 /*
1130 * Discard requests require the bio's inline iovecs be initialized.
1131 * ci->bio->bi_max_vecs is BIO_INLINE_VECS anyway, for both flush
1132 * and discard, so no need for concern about wasted bvec allocations.
1133 */
1134 __bio_clone(clone, ci->bio);
1135 if (len)
1136 bio_setup_sector(clone, ci->sector, len);
1137
1138 __map_bio(tio);
1139}
1140
1140static void __issue_target_bios(struct clone_info *ci, struct dm_target *ti,
1141 unsigned num_bios, sector_t len)
1141static void __send_duplicate_bios(struct clone_info *ci, struct dm_target *ti,
1142 unsigned num_bios, sector_t len)
1142{
1143 unsigned target_bio_nr;
1144
1145 for (target_bio_nr = 0; target_bio_nr < num_bios; target_bio_nr++)
1143{
1144 unsigned target_bio_nr;
1145
1146 for (target_bio_nr = 0; target_bio_nr < num_bios; target_bio_nr++)
1146 __issue_target_request(ci, ti, target_bio_nr, len);
1147 __clone_and_map_simple_bio(ci, ti, target_bio_nr, len);
1147}
1148
1148}
1149
1149static int __clone_and_map_empty_flush(struct clone_info *ci)
1150static int __send_empty_flush(struct clone_info *ci)
1150{
1151 unsigned target_nr = 0;
1152 struct dm_target *ti;
1153
1154 BUG_ON(bio_has_data(ci->bio));
1155 while ((ti = dm_table_get_target(ci->map, target_nr++)))
1151{
1152 unsigned target_nr = 0;
1153 struct dm_target *ti;
1154
1155 BUG_ON(bio_has_data(ci->bio));
1156 while ((ti = dm_table_get_target(ci->map, target_nr++)))
1156 __issue_target_bios(ci, ti, ti->num_flush_bios, 0);
1157 __send_duplicate_bios(ci, ti, ti->num_flush_bios, 0);
1157
1158 return 0;
1159}
1160
1158
1159 return 0;
1160}
1161
1161/*
1162 * Perform all io with a single clone.
1163 */
1164static void __clone_and_map_simple(struct clone_info *ci, struct dm_target *ti)
1162static void __clone_and_map_data_bio(struct clone_info *ci,
1163 struct dm_target *ti)
1165{
1166 struct bio *bio = ci->bio;
1167 struct dm_target_io *tio;
1168
1169 tio = alloc_tio(ci, ti, bio->bi_max_vecs, 0);
1170 clone_bio(tio, bio, ci->sector, ci->idx, bio->bi_vcnt - ci->idx,
1171 ci->sector_count);
1172 __map_bio(tio);

--- 14 unchanged lines hidden (view full) ---

1187
1188typedef bool (*is_split_required_fn)(struct dm_target *ti);
1189
1190static bool is_split_required_for_discard(struct dm_target *ti)
1191{
1192 return ti->split_discard_bios;
1193}
1194
1164{
1165 struct bio *bio = ci->bio;
1166 struct dm_target_io *tio;
1167
1168 tio = alloc_tio(ci, ti, bio->bi_max_vecs, 0);
1169 clone_bio(tio, bio, ci->sector, ci->idx, bio->bi_vcnt - ci->idx,
1170 ci->sector_count);
1171 __map_bio(tio);

--- 14 unchanged lines hidden (view full) ---

1186
1187typedef bool (*is_split_required_fn)(struct dm_target *ti);
1188
1189static bool is_split_required_for_discard(struct dm_target *ti)
1190{
1191 return ti->split_discard_bios;
1192}
1193
1195static int __clone_and_map_changing_extent_only(struct clone_info *ci,
1196 get_num_bios_fn get_num_bios,
1197 is_split_required_fn is_split_required)
1194static int __send_changing_extent_only(struct clone_info *ci,
1195 get_num_bios_fn get_num_bios,
1196 is_split_required_fn is_split_required)
1198{
1199 struct dm_target *ti;
1200 sector_t len;
1201 unsigned num_bios;
1202
1203 do {
1204 ti = dm_table_find_target(ci->map, ci->sector);
1205 if (!dm_target_is_valid(ti))

--- 9 unchanged lines hidden (view full) ---

1215 if (!num_bios)
1216 return -EOPNOTSUPP;
1217
1218 if (is_split_required && !is_split_required(ti))
1219 len = min(ci->sector_count, max_io_len_target_boundary(ci->sector, ti));
1220 else
1221 len = min(ci->sector_count, max_io_len(ci->sector, ti));
1222
1197{
1198 struct dm_target *ti;
1199 sector_t len;
1200 unsigned num_bios;
1201
1202 do {
1203 ti = dm_table_find_target(ci->map, ci->sector);
1204 if (!dm_target_is_valid(ti))

--- 9 unchanged lines hidden (view full) ---

1214 if (!num_bios)
1215 return -EOPNOTSUPP;
1216
1217 if (is_split_required && !is_split_required(ti))
1218 len = min(ci->sector_count, max_io_len_target_boundary(ci->sector, ti));
1219 else
1220 len = min(ci->sector_count, max_io_len(ci->sector, ti));
1221
1223 __issue_target_bios(ci, ti, num_bios, len);
1222 __send_duplicate_bios(ci, ti, num_bios, len);
1224
1225 ci->sector += len;
1226 } while (ci->sector_count -= len);
1227
1228 return 0;
1229}
1230
1223
1224 ci->sector += len;
1225 } while (ci->sector_count -= len);
1226
1227 return 0;
1228}
1229
1231static int __clone_and_map_discard(struct clone_info *ci)
1230static int __send_discard(struct clone_info *ci)
1232{
1231{
1233 return __clone_and_map_changing_extent_only(ci, get_num_discard_bios,
1234 is_split_required_for_discard);
1232 return __send_changing_extent_only(ci, get_num_discard_bios,
1233 is_split_required_for_discard);
1235}
1236
1234}
1235
1237static int __clone_and_map_write_same(struct clone_info *ci)
1236static int __send_write_same(struct clone_info *ci)
1238{
1237{
1239 return __clone_and_map_changing_extent_only(ci, get_num_write_same_bios, NULL);
1238 return __send_changing_extent_only(ci, get_num_write_same_bios, NULL);
1240}
1241
1239}
1240
1242static int __clone_and_map(struct clone_info *ci)
1241static int __split_and_process_non_flush(struct clone_info *ci)
1243{
1244 struct bio *bio = ci->bio;
1245 struct dm_target *ti;
1246 sector_t len = 0, max;
1247 struct dm_target_io *tio;
1248
1249 if (unlikely(bio->bi_rw & REQ_DISCARD))
1242{
1243 struct bio *bio = ci->bio;
1244 struct dm_target *ti;
1245 sector_t len = 0, max;
1246 struct dm_target_io *tio;
1247
1248 if (unlikely(bio->bi_rw & REQ_DISCARD))
1250 return __clone_and_map_discard(ci);
1249 return __send_discard(ci);
1251 else if (unlikely(bio->bi_rw & REQ_WRITE_SAME))
1250 else if (unlikely(bio->bi_rw & REQ_WRITE_SAME))
1252 return __clone_and_map_write_same(ci);
1251 return __send_write_same(ci);
1253
1254 ti = dm_table_find_target(ci->map, ci->sector);
1255 if (!dm_target_is_valid(ti))
1256 return -EIO;
1257
1258 max = max_io_len(ci->sector, ti);
1259
1260 if (ci->sector_count <= max) {
1261 /*
1262 * Optimise for the simple case where we can do all of
1263 * the remaining io with a single clone.
1264 */
1252
1253 ti = dm_table_find_target(ci->map, ci->sector);
1254 if (!dm_target_is_valid(ti))
1255 return -EIO;
1256
1257 max = max_io_len(ci->sector, ti);
1258
1259 if (ci->sector_count <= max) {
1260 /*
1261 * Optimise for the simple case where we can do all of
1262 * the remaining io with a single clone.
1263 */
1265 __clone_and_map_simple(ci, ti);
1264 __clone_and_map_data_bio(ci, ti);
1266
1267 } else if (to_sector(bio->bi_io_vec[ci->idx].bv_len) <= max) {
1268 /*
1269 * There are some bvecs that don't span targets.
1270 * Do as many of these as possible.
1271 */
1272 int i;
1273 sector_t remaining = max;

--- 32 unchanged lines hidden (view full) ---

1306 return -EIO;
1307
1308 max = max_io_len(ci->sector, ti);
1309 }
1310
1311 len = min(remaining, max);
1312
1313 tio = alloc_tio(ci, ti, 1, 0);
1265
1266 } else if (to_sector(bio->bi_io_vec[ci->idx].bv_len) <= max) {
1267 /*
1268 * There are some bvecs that don't span targets.
1269 * Do as many of these as possible.
1270 */
1271 int i;
1272 sector_t remaining = max;

--- 32 unchanged lines hidden (view full) ---

1305 return -EIO;
1306
1307 max = max_io_len(ci->sector, ti);
1308 }
1309
1310 len = min(remaining, max);
1311
1312 tio = alloc_tio(ci, ti, 1, 0);
1314 split_bvec(tio, bio, ci->sector, ci->idx,
1315 bv->bv_offset + offset, len);
1313 clone_split_bio(tio, bio, ci->sector, ci->idx,
1314 bv->bv_offset + offset, len);
1316
1317 __map_bio(tio);
1318
1319 ci->sector += len;
1320 ci->sector_count -= len;
1321 offset += to_bytes(len);
1322 } while (remaining -= len);
1323
1324 ci->idx++;
1325 }
1326
1327 return 0;
1328}
1329
1330/*
1315
1316 __map_bio(tio);
1317
1318 ci->sector += len;
1319 ci->sector_count -= len;
1320 offset += to_bytes(len);
1321 } while (remaining -= len);
1322
1323 ci->idx++;
1324 }
1325
1326 return 0;
1327}
1328
1329/*
1331 * Split the bio into several clones and submit it to targets.
1330 * Entry point to split a bio into clones and submit them to the targets.
1332 */
1333static void __split_and_process_bio(struct mapped_device *md, struct bio *bio)
1334{
1335 struct clone_info ci;
1336 int error = 0;
1337
1338 ci.map = dm_get_live_table(md);
1339 if (unlikely(!ci.map)) {

--- 11 unchanged lines hidden (view full) ---

1351 ci.sector = bio->bi_sector;
1352 ci.idx = bio->bi_idx;
1353
1354 start_io_acct(ci.io);
1355
1356 if (bio->bi_rw & REQ_FLUSH) {
1357 ci.bio = &ci.md->flush_bio;
1358 ci.sector_count = 0;
1331 */
1332static void __split_and_process_bio(struct mapped_device *md, struct bio *bio)
1333{
1334 struct clone_info ci;
1335 int error = 0;
1336
1337 ci.map = dm_get_live_table(md);
1338 if (unlikely(!ci.map)) {

--- 11 unchanged lines hidden (view full) ---

1350 ci.sector = bio->bi_sector;
1351 ci.idx = bio->bi_idx;
1352
1353 start_io_acct(ci.io);
1354
1355 if (bio->bi_rw & REQ_FLUSH) {
1356 ci.bio = &ci.md->flush_bio;
1357 ci.sector_count = 0;
1359 error = __clone_and_map_empty_flush(&ci);
1358 error = __send_empty_flush(&ci);
1360 /* dec_pending submits any data associated with flush */
1361 } else {
1362 ci.bio = bio;
1363 ci.sector_count = bio_sectors(bio);
1364 while (ci.sector_count && !error)
1359 /* dec_pending submits any data associated with flush */
1360 } else {
1361 ci.bio = bio;
1362 ci.sector_count = bio_sectors(bio);
1363 while (ci.sector_count && !error)
1365 error = __clone_and_map(&ci);
1364 error = __split_and_process_non_flush(&ci);
1366 }
1367
1368 /* drop the extra reference count */
1369 dec_pending(ci.io, error);
1370 dm_table_put(ci.map);
1371}
1372/*-----------------------------------------------------------------
1373 * CRUD END

--- 1460 unchanged lines hidden --- 		1365 }
1366
1367 /* drop the extra reference count */
1368 dec_pending(ci.io, error);
1369 dm_table_put(ci.map);
1370}
1371/*-----------------------------------------------------------------
1372 * CRUD END

--- 1460 unchanged lines hidden ---