1 /* 2 * Copyright (C) 2007 Jens Axboe <jens.axboe@oracle.com> 3 * 4 * Scatterlist handling helpers. 5 * 6 * This source code is licensed under the GNU General Public License, 7 * Version 2. See the file COPYING for more details. 8 */ 9 #include <linux/module.h> 10 #include <linux/scatterlist.h> 11 12 /** 13 * sg_next - return the next scatterlist entry in a list 14 * @sg: The current sg entry 15 * 16 * Description: 17 * Usually the next entry will be @sg@ + 1, but if this sg element is part 18 * of a chained scatterlist, it could jump to the start of a new 19 * scatterlist array. 20 * 21 **/ 22 struct scatterlist *sg_next(struct scatterlist *sg) 23 { 24 #ifdef CONFIG_DEBUG_SG 25 BUG_ON(sg->sg_magic != SG_MAGIC); 26 #endif 27 if (sg_is_last(sg)) 28 return NULL; 29 30 sg++; 31 if (unlikely(sg_is_chain(sg))) 32 sg = sg_chain_ptr(sg); 33 34 return sg; 35 } 36 EXPORT_SYMBOL(sg_next); 37 38 /** 39 * sg_last - return the last scatterlist entry in a list 40 * @sgl: First entry in the scatterlist 41 * @nents: Number of entries in the scatterlist 42 * 43 * Description: 44 * Should only be used casually, it (currently) scans the entire list 45 * to get the last entry. 46 * 47 * Note that the @sgl@ pointer passed in need not be the first one, 48 * the important bit is that @nents@ denotes the number of entries that 49 * exist from @sgl@. 50 * 51 **/ 52 struct scatterlist *sg_last(struct scatterlist *sgl, unsigned int nents) 53 { 54 #ifndef ARCH_HAS_SG_CHAIN 55 struct scatterlist *ret = &sgl[nents - 1]; 56 #else 57 struct scatterlist *sg, *ret = NULL; 58 unsigned int i; 59 60 for_each_sg(sgl, sg, nents, i) 61 ret = sg; 62 63 #endif 64 #ifdef CONFIG_DEBUG_SG 65 BUG_ON(sgl[0].sg_magic != SG_MAGIC); 66 BUG_ON(!sg_is_last(ret)); 67 #endif 68 return ret; 69 } 70 EXPORT_SYMBOL(sg_last); 71 72 /** 73 * sg_init_table - Initialize SG table 74 * @sgl: The SG table 75 * @nents: Number of entries in table 76 * 77 * Notes: 78 * If this is part of a chained sg table, sg_mark_end() should be 79 * used only on the last table part. 80 * 81 **/ 82 void sg_init_table(struct scatterlist *sgl, unsigned int nents) 83 { 84 memset(sgl, 0, sizeof(*sgl) * nents); 85 #ifdef CONFIG_DEBUG_SG 86 { 87 unsigned int i; 88 for (i = 0; i < nents; i++) 89 sgl[i].sg_magic = SG_MAGIC; 90 } 91 #endif 92 sg_mark_end(&sgl[nents - 1]); 93 } 94 EXPORT_SYMBOL(sg_init_table); 95 96 /** 97 * sg_init_one - Initialize a single entry sg list 98 * @sg: SG entry 99 * @buf: Virtual address for IO 100 * @buflen: IO length 101 * 102 **/ 103 void sg_init_one(struct scatterlist *sg, const void *buf, unsigned int buflen) 104 { 105 sg_init_table(sg, 1); 106 sg_set_buf(sg, buf, buflen); 107 } 108 EXPORT_SYMBOL(sg_init_one); 109 110 /* 111 * The default behaviour of sg_alloc_table() is to use these kmalloc/kfree 112 * helpers. 113 */ 114 static struct scatterlist *sg_kmalloc(unsigned int nents, gfp_t gfp_mask) 115 { 116 if (nents == SG_MAX_SINGLE_ALLOC) 117 return (struct scatterlist *) __get_free_page(gfp_mask); 118 else 119 return kmalloc(nents * sizeof(struct scatterlist), gfp_mask); 120 } 121 122 static void sg_kfree(struct scatterlist *sg, unsigned int nents) 123 { 124 if (nents == SG_MAX_SINGLE_ALLOC) 125 free_page((unsigned long) sg); 126 else 127 kfree(sg); 128 } 129 130 /** 131 * __sg_free_table - Free a previously mapped sg table 132 * @table: The sg table header to use 133 * @max_ents: The maximum number of entries per single scatterlist 134 * @free_fn: Free function 135 * 136 * Description: 137 * Free an sg table previously allocated and setup with 138 * __sg_alloc_table(). The @max_ents value must be identical to 139 * that previously used with __sg_alloc_table(). 140 * 141 **/ 142 void __sg_free_table(struct sg_table *table, unsigned int max_ents, 143 sg_free_fn *free_fn) 144 { 145 struct scatterlist *sgl, *next; 146 147 if (unlikely(!table->sgl)) 148 return; 149 150 sgl = table->sgl; 151 while (table->orig_nents) { 152 unsigned int alloc_size = table->orig_nents; 153 unsigned int sg_size; 154 155 /* 156 * If we have more than max_ents segments left, 157 * then assign 'next' to the sg table after the current one. 158 * sg_size is then one less than alloc size, since the last 159 * element is the chain pointer. 160 */ 161 if (alloc_size > max_ents) { 162 next = sg_chain_ptr(&sgl[max_ents - 1]); 163 alloc_size = max_ents; 164 sg_size = alloc_size - 1; 165 } else { 166 sg_size = alloc_size; 167 next = NULL; 168 } 169 170 table->orig_nents -= sg_size; 171 free_fn(sgl, alloc_size); 172 sgl = next; 173 } 174 175 table->sgl = NULL; 176 } 177 EXPORT_SYMBOL(__sg_free_table); 178 179 /** 180 * sg_free_table - Free a previously allocated sg table 181 * @table: The mapped sg table header 182 * 183 **/ 184 void sg_free_table(struct sg_table *table) 185 { 186 __sg_free_table(table, SG_MAX_SINGLE_ALLOC, sg_kfree); 187 } 188 EXPORT_SYMBOL(sg_free_table); 189 190 /** 191 * __sg_alloc_table - Allocate and initialize an sg table with given allocator 192 * @table: The sg table header to use 193 * @nents: Number of entries in sg list 194 * @max_ents: The maximum number of entries the allocator returns per call 195 * @gfp_mask: GFP allocation mask 196 * @alloc_fn: Allocator to use 197 * 198 * Description: 199 * This function returns a @table @nents long. The allocator is 200 * defined to return scatterlist chunks of maximum size @max_ents. 201 * Thus if @nents is bigger than @max_ents, the scatterlists will be 202 * chained in units of @max_ents. 203 * 204 * Notes: 205 * If this function returns non-0 (eg failure), the caller must call 206 * __sg_free_table() to cleanup any leftover allocations. 207 * 208 **/ 209 int __sg_alloc_table(struct sg_table *table, unsigned int nents, 210 unsigned int max_ents, gfp_t gfp_mask, 211 sg_alloc_fn *alloc_fn) 212 { 213 struct scatterlist *sg, *prv; 214 unsigned int left; 215 216 #ifndef ARCH_HAS_SG_CHAIN 217 BUG_ON(nents > max_ents); 218 #endif 219 220 memset(table, 0, sizeof(*table)); 221 222 left = nents; 223 prv = NULL; 224 do { 225 unsigned int sg_size, alloc_size = left; 226 227 if (alloc_size > max_ents) { 228 alloc_size = max_ents; 229 sg_size = alloc_size - 1; 230 } else 231 sg_size = alloc_size; 232 233 left -= sg_size; 234 235 sg = alloc_fn(alloc_size, gfp_mask); 236 if (unlikely(!sg)) 237 return -ENOMEM; 238 239 sg_init_table(sg, alloc_size); 240 table->nents = table->orig_nents += sg_size; 241 242 /* 243 * If this is the first mapping, assign the sg table header. 244 * If this is not the first mapping, chain previous part. 245 */ 246 if (prv) 247 sg_chain(prv, max_ents, sg); 248 else 249 table->sgl = sg; 250 251 /* 252 * If no more entries after this one, mark the end 253 */ 254 if (!left) 255 sg_mark_end(&sg[sg_size - 1]); 256 257 /* 258 * only really needed for mempool backed sg allocations (like 259 * SCSI), a possible improvement here would be to pass the 260 * table pointer into the allocator and let that clear these 261 * flags 262 */ 263 gfp_mask &= ~__GFP_WAIT; 264 gfp_mask |= __GFP_HIGH; 265 prv = sg; 266 } while (left); 267 268 return 0; 269 } 270 EXPORT_SYMBOL(__sg_alloc_table); 271 272 /** 273 * sg_alloc_table - Allocate and initialize an sg table 274 * @table: The sg table header to use 275 * @nents: Number of entries in sg list 276 * @gfp_mask: GFP allocation mask 277 * 278 * Description: 279 * Allocate and initialize an sg table. If @nents@ is larger than 280 * SG_MAX_SINGLE_ALLOC a chained sg table will be setup. 281 * 282 **/ 283 int sg_alloc_table(struct sg_table *table, unsigned int nents, gfp_t gfp_mask) 284 { 285 int ret; 286 287 ret = __sg_alloc_table(table, nents, SG_MAX_SINGLE_ALLOC, 288 gfp_mask, sg_kmalloc); 289 if (unlikely(ret)) 290 __sg_free_table(table, SG_MAX_SINGLE_ALLOC, sg_kfree); 291 292 return ret; 293 } 294 EXPORT_SYMBOL(sg_alloc_table); 295