259 	struct swap_iocb *sio = container_of(iocb, struct swap_iocb, iocb);  in sio_write_complete()  local
260 	struct page *page = sio->bvec[0].bv_page;  in sio_write_complete()
263 	if (ret != sio->len) {  in sio_write_complete()
276 		for (p = 0; p < sio->pages; p++) {  in sio_write_complete()
277 			page = sio->bvec[p].bv_page;  in sio_write_complete()
282 		for (p = 0; p < sio->pages; p++)  in sio_write_complete()
283 			count_swpout_vm_event(page_folio(sio->bvec[p].bv_page));  in sio_write_complete()
286 	for (p = 0; p < sio->pages; p++)  in sio_write_complete()
287 		end_page_writeback(sio->bvec[p].bv_page);  in sio_write_complete()
289 	mempool_free(sio, sio_pool);  in sio_write_complete()
294 	struct swap_iocb *sio = NULL;  in swap_writepage_fs()  local
302 		sio = *wbc->swap_plug;  in swap_writepage_fs()
303 	if (sio) {  in swap_writepage_fs()
304 		if (sio->iocb.ki_filp != swap_file ||  in swap_writepage_fs()
305 		    sio->iocb.ki_pos + sio->len != pos) {  in swap_writepage_fs()
306 			swap_write_unplug(sio);  in swap_writepage_fs()
307 			sio = NULL;  in swap_writepage_fs()
310 	if (!sio) {  in swap_writepage_fs()
311 		sio = mempool_alloc(sio_pool, GFP_NOIO);  in swap_writepage_fs()
312 		init_sync_kiocb(&sio->iocb, swap_file);  in swap_writepage_fs()
313 		sio->iocb.ki_complete = sio_write_complete;  in swap_writepage_fs()
314 		sio->iocb.ki_pos = pos;  in swap_writepage_fs()
315 		sio->pages = 0;  in swap_writepage_fs()
316 		sio->len = 0;  in swap_writepage_fs()
318 	bvec_set_page(&sio->bvec[sio->pages], page, thp_size(page), 0);  in swap_writepage_fs()
319 	sio->len += thp_size(page);  in swap_writepage_fs()
320 	sio->pages += 1;  in swap_writepage_fs()
321 	if (sio->pages == ARRAY_SIZE(sio->bvec) || !wbc->swap_plug) {  in swap_writepage_fs()
322 		swap_write_unplug(sio);  in swap_writepage_fs()
323 		sio = NULL;  in swap_writepage_fs()
326 		*wbc->swap_plug = sio;  in swap_writepage_fs()
389 void swap_write_unplug(struct swap_iocb *sio)  in swap_write_unplug()  argument
392 	struct address_space *mapping = sio->iocb.ki_filp->f_mapping;  in swap_write_unplug()
395 	iov_iter_bvec(&from, ITER_SOURCE, sio->bvec, sio->pages, sio->len);  in swap_write_unplug()
396 	ret = mapping->a_ops->swap_rw(&sio->iocb, &from);  in swap_write_unplug()
398 		sio_write_complete(&sio->iocb, ret);  in swap_write_unplug()
403 	struct swap_iocb *sio = container_of(iocb, struct swap_iocb, iocb);  in sio_read_complete()  local
406 	if (ret == sio->len) {  in sio_read_complete()
407 		for (p = 0; p < sio->pages; p++) {  in sio_read_complete()
408 			struct folio *folio = page_folio(sio->bvec[p].bv_page);  in sio_read_complete()
413 		count_vm_events(PSWPIN, sio->pages);  in sio_read_complete()
415 		for (p = 0; p < sio->pages; p++) {  in sio_read_complete()
416 			struct folio *folio = page_folio(sio->bvec[p].bv_page);  in sio_read_complete()
422 	mempool_free(sio, sio_pool);  in sio_read_complete()
429 	struct swap_iocb *sio = NULL;  in swap_readpage_fs()  local
433 		sio = *plug;  in swap_readpage_fs()
434 	if (sio) {  in swap_readpage_fs()
435 		if (sio->iocb.ki_filp != sis->swap_file ||  in swap_readpage_fs()
436 		    sio->iocb.ki_pos + sio->len != pos) {  in swap_readpage_fs()
437 			swap_read_unplug(sio);  in swap_readpage_fs()
438 			sio = NULL;  in swap_readpage_fs()
441 	if (!sio) {  in swap_readpage_fs()
442 		sio = mempool_alloc(sio_pool, GFP_KERNEL);  in swap_readpage_fs()
443 		init_sync_kiocb(&sio->iocb, sis->swap_file);  in swap_readpage_fs()
444 		sio->iocb.ki_pos = pos;  in swap_readpage_fs()
445 		sio->iocb.ki_complete = sio_read_complete;  in swap_readpage_fs()
446 		sio->pages = 0;  in swap_readpage_fs()
447 		sio->len = 0;  in swap_readpage_fs()
449 	bvec_set_page(&sio->bvec[sio->pages], page, thp_size(page), 0);  in swap_readpage_fs()
450 	sio->len += thp_size(page);  in swap_readpage_fs()
451 	sio->pages += 1;  in swap_readpage_fs()
452 	if (sio->pages == ARRAY_SIZE(sio->bvec) || !plug) {  in swap_readpage_fs()
453 		swap_read_unplug(sio);  in swap_readpage_fs()
454 		sio = NULL;  in swap_readpage_fs()
457 		*plug = sio;  in swap_readpage_fs()
534 void __swap_read_unplug(struct swap_iocb *sio)  in __swap_read_unplug()  argument
537 	struct address_space *mapping = sio->iocb.ki_filp->f_mapping;  in __swap_read_unplug()
540 	iov_iter_bvec(&from, ITER_DEST, sio->bvec, sio->pages, sio->len);  in __swap_read_unplug()
541 	ret = mapping->a_ops->swap_rw(&sio->iocb, &from);  in __swap_read_unplug()
543 		sio_read_complete(&sio->iocb, ret);  in __swap_read_unplug()