vmalloc.c (913272b3864d6da89c70d9fc2c30ccb57794b369) vmalloc.c (db1aecafef58b5dda39c4228debe2c845e4a27ab)
1/*
2 * linux/mm/vmalloc.c
3 *
4 * Copyright (C) 1993 Linus Torvalds
5 * Support of BIGMEM added by Gerhard Wichert, Siemens AG, July 1999
6 * SMP-safe vmalloc/vfree/ioremap, Tigran Aivazian <tigran@veritas.com>, May 2000
7 * Major rework to support vmap/vunmap, Christoph Hellwig, SGI, August 2002
8 * Numa awareness, Christoph Lameter, SGI, June 2005

--- 242 unchanged lines hidden (view full) ---

251
252struct vmap_area {
253 unsigned long va_start;
254 unsigned long va_end;
255 unsigned long flags;
256 struct rb_node rb_node; /* address sorted rbtree */
257 struct list_head list; /* address sorted list */
258 struct list_head purge_list; /* "lazy purge" list */
1/*
2 * linux/mm/vmalloc.c
3 *
4 * Copyright (C) 1993 Linus Torvalds
5 * Support of BIGMEM added by Gerhard Wichert, Siemens AG, July 1999
6 * SMP-safe vmalloc/vfree/ioremap, Tigran Aivazian <tigran@veritas.com>, May 2000
7 * Major rework to support vmap/vunmap, Christoph Hellwig, SGI, August 2002
8 * Numa awareness, Christoph Lameter, SGI, June 2005

--- 242 unchanged lines hidden (view full) ---

251
252struct vmap_area {
253 unsigned long va_start;
254 unsigned long va_end;
255 unsigned long flags;
256 struct rb_node rb_node; /* address sorted rbtree */
257 struct list_head list; /* address sorted list */
258 struct list_head purge_list; /* "lazy purge" list */
259 void *private;
259 struct vm_struct *vm;
260 struct rcu_head rcu_head;
261};
262
263static DEFINE_SPINLOCK(vmap_area_lock);
264static LIST_HEAD(vmap_area_list);
265static struct rb_root vmap_area_root = RB_ROOT;
266
267/* The vmap cache globals are protected by vmap_area_lock */

--- 845 unchanged lines hidden (view full) ---

1113 vm_unmap_ram(mem, count);
1114 return NULL;
1115 }
1116 return mem;
1117}
1118EXPORT_SYMBOL(vm_map_ram);
1119
1120/**
260 struct rcu_head rcu_head;
261};
262
263static DEFINE_SPINLOCK(vmap_area_lock);
264static LIST_HEAD(vmap_area_list);
265static struct rb_root vmap_area_root = RB_ROOT;
266
267/* The vmap cache globals are protected by vmap_area_lock */

--- 845 unchanged lines hidden (view full) ---

1113 vm_unmap_ram(mem, count);
1114 return NULL;
1115 }
1116 return mem;
1117}
1118EXPORT_SYMBOL(vm_map_ram);
1119
1120/**
1121 * vm_area_add_early - add vmap area early during boot
1122 * @vm: vm_struct to add
1123 *
1124 * This function is used to add fixed kernel vm area to vmlist before
1125 * vmalloc_init() is called. @vm->addr, @vm->size, and @vm->flags
1126 * should contain proper values and the other fields should be zero.
1127 *
1128 * DO NOT USE THIS FUNCTION UNLESS YOU KNOW WHAT YOU'RE DOING.
1129 */
1130void __init vm_area_add_early(struct vm_struct *vm)
1131{
1132 struct vm_struct *tmp, **p;
1133
1134 BUG_ON(vmap_initialized);
1135 for (p = &vmlist; (tmp = *p) != NULL; p = &tmp->next) {
1136 if (tmp->addr >= vm->addr) {
1137 BUG_ON(tmp->addr < vm->addr + vm->size);
1138 break;
1139 } else
1140 BUG_ON(tmp->addr + tmp->size > vm->addr);
1141 }
1142 vm->next = *p;
1143 *p = vm;
1144}
1145
1146/**
1121 * vm_area_register_early - register vmap area early during boot
1122 * @vm: vm_struct to register
1123 * @align: requested alignment
1124 *
1125 * This function is used to register kernel vm area before
1126 * vmalloc_init() is called. @vm->size and @vm->flags should contain
1127 * proper values on entry and other fields should be zero. On return,
1128 * vm->addr contains the allocated address.

--- 5 unchanged lines hidden (view full) ---

1134 static size_t vm_init_off __initdata;
1135 unsigned long addr;
1136
1137 addr = ALIGN(VMALLOC_START + vm_init_off, align);
1138 vm_init_off = PFN_ALIGN(addr + vm->size) - VMALLOC_START;
1139
1140 vm->addr = (void *)addr;
1141
1147 * vm_area_register_early - register vmap area early during boot
1148 * @vm: vm_struct to register
1149 * @align: requested alignment
1150 *
1151 * This function is used to register kernel vm area before
1152 * vmalloc_init() is called. @vm->size and @vm->flags should contain
1153 * proper values on entry and other fields should be zero. On return,
1154 * vm->addr contains the allocated address.

--- 5 unchanged lines hidden (view full) ---

1160 static size_t vm_init_off __initdata;
1161 unsigned long addr;
1162
1163 addr = ALIGN(VMALLOC_START + vm_init_off, align);
1164 vm_init_off = PFN_ALIGN(addr + vm->size) - VMALLOC_START;
1165
1166 vm->addr = (void *)addr;
1167
1142 vm->next = vmlist;
1143 vmlist = vm;
1168 vm_area_add_early(vm);
1144}
1145
1146void __init vmalloc_init(void)
1147{
1148 struct vmap_area *va;
1149 struct vm_struct *tmp;
1150 int i;
1151

--- 103 unchanged lines hidden (view full) ---

1255
1256static void setup_vmalloc_vm(struct vm_struct *vm, struct vmap_area *va,
1257 unsigned long flags, void *caller)
1258{
1259 vm->flags = flags;
1260 vm->addr = (void *)va->va_start;
1261 vm->size = va->va_end - va->va_start;
1262 vm->caller = caller;
1169}
1170
1171void __init vmalloc_init(void)
1172{
1173 struct vmap_area *va;
1174 struct vm_struct *tmp;
1175 int i;
1176

--- 103 unchanged lines hidden (view full) ---

1280
1281static void setup_vmalloc_vm(struct vm_struct *vm, struct vmap_area *va,
1282 unsigned long flags, void *caller)
1283{
1284 vm->flags = flags;
1285 vm->addr = (void *)va->va_start;
1286 vm->size = va->va_end - va->va_start;
1287 vm->caller = caller;
1263 va->private = vm;
1288 va->vm = vm;
1264 va->flags |= VM_VM_AREA;
1265}
1266
1267static void insert_vmalloc_vmlist(struct vm_struct *vm)
1268{
1269 struct vm_struct *tmp, **p;
1270
1271 vm->flags &= ~VM_UNLIST;

--- 13 unchanged lines hidden (view full) ---

1285 setup_vmalloc_vm(vm, va, flags, caller);
1286 insert_vmalloc_vmlist(vm);
1287}
1288
1289static struct vm_struct *__get_vm_area_node(unsigned long size,
1290 unsigned long align, unsigned long flags, unsigned long start,
1291 unsigned long end, int node, gfp_t gfp_mask, void *caller)
1292{
1289 va->flags |= VM_VM_AREA;
1290}
1291
1292static void insert_vmalloc_vmlist(struct vm_struct *vm)
1293{
1294 struct vm_struct *tmp, **p;
1295
1296 vm->flags &= ~VM_UNLIST;

--- 13 unchanged lines hidden (view full) ---

1310 setup_vmalloc_vm(vm, va, flags, caller);
1311 insert_vmalloc_vmlist(vm);
1312}
1313
1314static struct vm_struct *__get_vm_area_node(unsigned long size,
1315 unsigned long align, unsigned long flags, unsigned long start,
1316 unsigned long end, int node, gfp_t gfp_mask, void *caller)
1317{
1293 static struct vmap_area *va;
1318 struct vmap_area *va;
1294 struct vm_struct *area;
1295
1296 BUG_ON(in_interrupt());
1297 if (flags & VM_IOREMAP) {
1298 int bit = fls(size);
1299
1300 if (bit > IOREMAP_MAX_ORDER)
1301 bit = IOREMAP_MAX_ORDER;

--- 76 unchanged lines hidden (view full) ---

1378}
1379
1380static struct vm_struct *find_vm_area(const void *addr)
1381{
1382 struct vmap_area *va;
1383
1384 va = find_vmap_area((unsigned long)addr);
1385 if (va && va->flags & VM_VM_AREA)
1319 struct vm_struct *area;
1320
1321 BUG_ON(in_interrupt());
1322 if (flags & VM_IOREMAP) {
1323 int bit = fls(size);
1324
1325 if (bit > IOREMAP_MAX_ORDER)
1326 bit = IOREMAP_MAX_ORDER;

--- 76 unchanged lines hidden (view full) ---

1403}
1404
1405static struct vm_struct *find_vm_area(const void *addr)
1406{
1407 struct vmap_area *va;
1408
1409 va = find_vmap_area((unsigned long)addr);
1410 if (va && va->flags & VM_VM_AREA)
1386 return va->private;
1411 return va->vm;
1387
1388 return NULL;
1389}
1390
1391/**
1392 * remove_vm_area - find and remove a continuous kernel virtual area
1393 * @addr: base address
1394 *
1395 * Search for the kernel VM area starting at @addr, and remove it.
1396 * This function returns the found VM area, but using it is NOT safe
1397 * on SMP machines, except for its size or flags.
1398 */
1399struct vm_struct *remove_vm_area(const void *addr)
1400{
1401 struct vmap_area *va;
1402
1403 va = find_vmap_area((unsigned long)addr);
1404 if (va && va->flags & VM_VM_AREA) {
1412
1413 return NULL;
1414}
1415
1416/**
1417 * remove_vm_area - find and remove a continuous kernel virtual area
1418 * @addr: base address
1419 *
1420 * Search for the kernel VM area starting at @addr, and remove it.
1421 * This function returns the found VM area, but using it is NOT safe
1422 * on SMP machines, except for its size or flags.
1423 */
1424struct vm_struct *remove_vm_area(const void *addr)
1425{
1426 struct vmap_area *va;
1427
1428 va = find_vmap_area((unsigned long)addr);
1429 if (va && va->flags & VM_VM_AREA) {
1405 struct vm_struct *vm = va->private;
1430 struct vm_struct *vm = va->vm;
1406
1407 if (!(vm->flags & VM_UNLIST)) {
1408 struct vm_struct *tmp, **p;
1409 /*
1410 * remove from list and disallow access to
1411 * this vm_struct before unmap. (address range
1412 * confliction is maintained by vmap.)
1413 */

--- 214 unchanged lines hidden (view full) ---

1628 goto fail;
1629
1630 area = __get_vm_area_node(size, align, VM_ALLOC | VM_UNLIST,
1631 start, end, node, gfp_mask, caller);
1632 if (!area)
1633 goto fail;
1634
1635 addr = __vmalloc_area_node(area, gfp_mask, prot, node, caller);
1431
1432 if (!(vm->flags & VM_UNLIST)) {
1433 struct vm_struct *tmp, **p;
1434 /*
1435 * remove from list and disallow access to
1436 * this vm_struct before unmap. (address range
1437 * confliction is maintained by vmap.)
1438 */

--- 214 unchanged lines hidden (view full) ---

1653 goto fail;
1654
1655 area = __get_vm_area_node(size, align, VM_ALLOC | VM_UNLIST,
1656 start, end, node, gfp_mask, caller);
1657 if (!area)
1658 goto fail;
1659
1660 addr = __vmalloc_area_node(area, gfp_mask, prot, node, caller);
1661 if (!addr)
1662 return NULL;
1636
1637 /*
1638 * In this function, newly allocated vm_struct is not added
1639 * to vmlist at __get_vm_area_node(). so, it is added here.
1640 */
1641 insert_vmalloc_vmlist(area);
1642
1643 /*

--- 965 unchanged lines hidden --- 		1663
1664 /*
1665 * In this function, newly allocated vm_struct is not added
1666 * to vmlist at __get_vm_area_node(). so, it is added here.
1667 */
1668 insert_vmalloc_vmlist(area);
1669
1670 /*

--- 965 unchanged lines hidden ---