filemap.c (e61452365372570253b2b1de84bab0cdb2e62c64) filemap.c (2cf938aae17203426a89b5955bd1c9668657bfa8)
1/*
2 * linux/mm/filemap.c
3 *
4 * Copyright (C) 1994-1999 Linus Torvalds
5 */
6
7/*
8 * This file handles the generic file mmap semantics used by

--- 1241 unchanged lines hidden (view full) ---

1250 void **slot;
1251 unsigned int ret = 0;
1252 struct radix_tree_iter iter;
1253
1254 if (!nr_entries)
1255 return 0;
1256
1257 rcu_read_lock();
1/*
2 * linux/mm/filemap.c
3 *
4 * Copyright (C) 1994-1999 Linus Torvalds
5 */
6
7/*
8 * This file handles the generic file mmap semantics used by

--- 1241 unchanged lines hidden (view full) ---

1250 void **slot;
1251 unsigned int ret = 0;
1252 struct radix_tree_iter iter;
1253
1254 if (!nr_entries)
1255 return 0;
1256
1257 rcu_read_lock();
1258restart:
1259 radix_tree_for_each_slot(slot, &mapping->page_tree, &iter, start) {
1260 struct page *page;
1261repeat:
1262 page = radix_tree_deref_slot(slot);
1263 if (unlikely(!page))
1264 continue;
1265 if (radix_tree_exception(page)) {
1258 radix_tree_for_each_slot(slot, &mapping->page_tree, &iter, start) {
1259 struct page *page;
1260repeat:
1261 page = radix_tree_deref_slot(slot);
1262 if (unlikely(!page))
1263 continue;
1264 if (radix_tree_exception(page)) {
1266 if (radix_tree_deref_retry(page))
1267 goto restart;
1265 if (radix_tree_deref_retry(page)) {
1266 slot = radix_tree_iter_retry(&iter);
1267 continue;
1268 }
1268 /*
1269 * A shadow entry of a recently evicted page, a swap
1270 * entry from shmem/tmpfs or a DAX entry. Return it
1271 * without attempting to raise page count.
1272 */
1273 goto export;
1274 }
1275 if (!page_cache_get_speculative(page))

--- 36 unchanged lines hidden (view full) ---

1312 struct radix_tree_iter iter;
1313 void **slot;
1314 unsigned ret = 0;
1315
1316 if (unlikely(!nr_pages))
1317 return 0;
1318
1319 rcu_read_lock();
1269 /*
1270 * A shadow entry of a recently evicted page, a swap
1271 * entry from shmem/tmpfs or a DAX entry. Return it
1272 * without attempting to raise page count.
1273 */
1274 goto export;
1275 }
1276 if (!page_cache_get_speculative(page))

--- 36 unchanged lines hidden (view full) ---

1313 struct radix_tree_iter iter;
1314 void **slot;
1315 unsigned ret = 0;
1316
1317 if (unlikely(!nr_pages))
1318 return 0;
1319
1320 rcu_read_lock();
1320restart:
1321 radix_tree_for_each_slot(slot, &mapping->page_tree, &iter, start) {
1322 struct page *page;
1323repeat:
1324 page = radix_tree_deref_slot(slot);
1325 if (unlikely(!page))
1326 continue;
1327
1328 if (radix_tree_exception(page)) {
1329 if (radix_tree_deref_retry(page)) {
1321 radix_tree_for_each_slot(slot, &mapping->page_tree, &iter, start) {
1322 struct page *page;
1323repeat:
1324 page = radix_tree_deref_slot(slot);
1325 if (unlikely(!page))
1326 continue;
1327
1328 if (radix_tree_exception(page)) {
1329 if (radix_tree_deref_retry(page)) {
1330 /*
1331 * Transient condition which can only trigger
1332 * when entry at index 0 moves out of or back
1333 * to root: none yet gotten, safe to restart.
1334 */
1335 WARN_ON(iter.index);
1336 goto restart;
1330 slot = radix_tree_iter_retry(&iter);
1331 continue;
1337 }
1338 /*
1339 * A shadow entry of a recently evicted page,
1340 * or a swap entry from shmem/tmpfs. Skip
1341 * over it.
1342 */
1343 continue;
1344 }

--- 34 unchanged lines hidden (view full) ---

1379 struct radix_tree_iter iter;
1380 void **slot;
1381 unsigned int ret = 0;
1382
1383 if (unlikely(!nr_pages))
1384 return 0;
1385
1386 rcu_read_lock();
1332 }
1333 /*
1334 * A shadow entry of a recently evicted page,
1335 * or a swap entry from shmem/tmpfs. Skip
1336 * over it.
1337 */
1338 continue;
1339 }

--- 34 unchanged lines hidden (view full) ---

1374 struct radix_tree_iter iter;
1375 void **slot;
1376 unsigned int ret = 0;
1377
1378 if (unlikely(!nr_pages))
1379 return 0;
1380
1381 rcu_read_lock();
1387restart:
1388 radix_tree_for_each_contig(slot, &mapping->page_tree, &iter, index) {
1389 struct page *page;
1390repeat:
1391 page = radix_tree_deref_slot(slot);
1392 /* The hole, there no reason to continue */
1393 if (unlikely(!page))
1394 break;
1395
1396 if (radix_tree_exception(page)) {
1397 if (radix_tree_deref_retry(page)) {
1382 radix_tree_for_each_contig(slot, &mapping->page_tree, &iter, index) {
1383 struct page *page;
1384repeat:
1385 page = radix_tree_deref_slot(slot);
1386 /* The hole, there no reason to continue */
1387 if (unlikely(!page))
1388 break;
1389
1390 if (radix_tree_exception(page)) {
1391 if (radix_tree_deref_retry(page)) {
1398 /*
1399 * Transient condition which can only trigger
1400 * when entry at index 0 moves out of or back
1401 * to root: none yet gotten, safe to restart.
1402 */
1403 goto restart;
1392 slot = radix_tree_iter_retry(&iter);
1393 continue;
1404 }
1405 /*
1406 * A shadow entry of a recently evicted page,
1407 * or a swap entry from shmem/tmpfs. Stop
1408 * looking for contiguous pages.
1409 */
1410 break;
1411 }

--- 43 unchanged lines hidden (view full) ---

1455 struct radix_tree_iter iter;
1456 void **slot;
1457 unsigned ret = 0;
1458
1459 if (unlikely(!nr_pages))
1460 return 0;
1461
1462 rcu_read_lock();
1394 }
1395 /*
1396 * A shadow entry of a recently evicted page,
1397 * or a swap entry from shmem/tmpfs. Stop
1398 * looking for contiguous pages.
1399 */
1400 break;
1401 }

--- 43 unchanged lines hidden (view full) ---

1445 struct radix_tree_iter iter;
1446 void **slot;
1447 unsigned ret = 0;
1448
1449 if (unlikely(!nr_pages))
1450 return 0;
1451
1452 rcu_read_lock();
1463restart:
1464 radix_tree_for_each_tagged(slot, &mapping->page_tree,
1465 &iter, *index, tag) {
1466 struct page *page;
1467repeat:
1468 page = radix_tree_deref_slot(slot);
1469 if (unlikely(!page))
1470 continue;
1471
1472 if (radix_tree_exception(page)) {
1473 if (radix_tree_deref_retry(page)) {
1453 radix_tree_for_each_tagged(slot, &mapping->page_tree,
1454 &iter, *index, tag) {
1455 struct page *page;
1456repeat:
1457 page = radix_tree_deref_slot(slot);
1458 if (unlikely(!page))
1459 continue;
1460
1461 if (radix_tree_exception(page)) {
1462 if (radix_tree_deref_retry(page)) {
1474 /*
1475 * Transient condition which can only trigger
1476 * when entry at index 0 moves out of or back
1477 * to root: none yet gotten, safe to restart.
1478 */
1479 goto restart;
1463 slot = radix_tree_iter_retry(&iter);
1464 continue;
1480 }
1481 /*
1482 * A shadow entry of a recently evicted page.
1483 *
1484 * Those entries should never be tagged, but
1485 * this tree walk is lockless and the tags are
1486 * looked up in bulk, one radix tree node at a
1487 * time, so there is a sizable window for page

--- 46 unchanged lines hidden (view full) ---

1534 void **slot;
1535 unsigned int ret = 0;
1536 struct radix_tree_iter iter;
1537
1538 if (!nr_entries)
1539 return 0;
1540
1541 rcu_read_lock();
1465 }
1466 /*
1467 * A shadow entry of a recently evicted page.
1468 *
1469 * Those entries should never be tagged, but
1470 * this tree walk is lockless and the tags are
1471 * looked up in bulk, one radix tree node at a
1472 * time, so there is a sizable window for page

--- 46 unchanged lines hidden (view full) ---

1519 void **slot;
1520 unsigned int ret = 0;
1521 struct radix_tree_iter iter;
1522
1523 if (!nr_entries)
1524 return 0;
1525
1526 rcu_read_lock();
1542restart:
1543 radix_tree_for_each_tagged(slot, &mapping->page_tree,
1544 &iter, start, tag) {
1545 struct page *page;
1546repeat:
1547 page = radix_tree_deref_slot(slot);
1548 if (unlikely(!page))
1549 continue;
1550 if (radix_tree_exception(page)) {
1551 if (radix_tree_deref_retry(page)) {
1527 radix_tree_for_each_tagged(slot, &mapping->page_tree,
1528 &iter, start, tag) {
1529 struct page *page;
1530repeat:
1531 page = radix_tree_deref_slot(slot);
1532 if (unlikely(!page))
1533 continue;
1534 if (radix_tree_exception(page)) {
1535 if (radix_tree_deref_retry(page)) {
1552 /*
1553 * Transient condition which can only trigger
1554 * when entry at index 0 moves out of or back
1555 * to root: none yet gotten, safe to restart.
1556 */
1557 goto restart;
1536 slot = radix_tree_iter_retry(&iter);
1537 continue;
1558 }
1559
1560 /*
1561 * A shadow entry of a recently evicted page, a swap
1562 * entry from shmem/tmpfs or a DAX entry. Return it
1563 * without attempting to raise page count.
1564 */
1565 goto export;

--- 600 unchanged lines hidden (view full) ---

2166 radix_tree_for_each_slot(slot, &mapping->page_tree, &iter, vmf->pgoff) {
2167 if (iter.index > vmf->max_pgoff)
2168 break;
2169repeat:
2170 page = radix_tree_deref_slot(slot);
2171 if (unlikely(!page))
2172 goto next;
2173 if (radix_tree_exception(page)) {
1538 }
1539
1540 /*
1541 * A shadow entry of a recently evicted page, a swap
1542 * entry from shmem/tmpfs or a DAX entry. Return it
1543 * without attempting to raise page count.
1544 */
1545 goto export;

--- 600 unchanged lines hidden (view full) ---

2146 radix_tree_for_each_slot(slot, &mapping->page_tree, &iter, vmf->pgoff) {
2147 if (iter.index > vmf->max_pgoff)
2148 break;
2149repeat:
2150 page = radix_tree_deref_slot(slot);
2151 if (unlikely(!page))
2152 goto next;
2153 if (radix_tree_exception(page)) {
2174 if (radix_tree_deref_retry(page))
2175 break;
2176 else
2177 goto next;
2154 if (radix_tree_deref_retry(page)) {
2155 slot = radix_tree_iter_retry(&iter);
2156 continue;
2157 }
2158 goto next;
2178 }
2179
2180 if (!page_cache_get_speculative(page))
2181 goto repeat;
2182
2183 /* Has the page moved? */
2184 if (unlikely(page != *slot)) {
2185 page_cache_release(page);

--- 669 unchanged lines hidden --- 		2159 }
2160
2161 if (!page_cache_get_speculative(page))
2162 goto repeat;
2163
2164 /* Has the page moved? */
2165 if (unlikely(page != *slot)) {
2166 page_cache_release(page);

--- 669 unchanged lines hidden ---