xref: /openbmc/linux/fs/erofs/zdata.c (revision 8bdc2a19)
1  // SPDX-License-Identifier: GPL-2.0-only
2  /*
3   * Copyright (C) 2018 HUAWEI, Inc.
4   *             https://www.huawei.com/
5   */
6  #include "zdata.h"
7  #include "compress.h"
8  #include <linux/prefetch.h>
9  
10  #include <trace/events/erofs.h>
11  
12  /*
13   * since pclustersize is variable for big pcluster feature, introduce slab
14   * pools implementation for different pcluster sizes.
15   */
16  struct z_erofs_pcluster_slab {
17  	struct kmem_cache *slab;
18  	unsigned int maxpages;
19  	char name[48];
20  };
21  
22  #define _PCLP(n) { .maxpages = n }
23  
24  static struct z_erofs_pcluster_slab pcluster_pool[] __read_mostly = {
25  	_PCLP(1), _PCLP(4), _PCLP(16), _PCLP(64), _PCLP(128),
26  	_PCLP(Z_EROFS_PCLUSTER_MAX_PAGES)
27  };
28  
29  static void z_erofs_destroy_pcluster_pool(void)
30  {
31  	int i;
32  
33  	for (i = 0; i < ARRAY_SIZE(pcluster_pool); ++i) {
34  		if (!pcluster_pool[i].slab)
35  			continue;
36  		kmem_cache_destroy(pcluster_pool[i].slab);
37  		pcluster_pool[i].slab = NULL;
38  	}
39  }
40  
41  static int z_erofs_create_pcluster_pool(void)
42  {
43  	struct z_erofs_pcluster_slab *pcs;
44  	struct z_erofs_pcluster *a;
45  	unsigned int size;
46  
47  	for (pcs = pcluster_pool;
48  	     pcs < pcluster_pool + ARRAY_SIZE(pcluster_pool); ++pcs) {
49  		size = struct_size(a, compressed_pages, pcs->maxpages);
50  
51  		sprintf(pcs->name, "erofs_pcluster-%u", pcs->maxpages);
52  		pcs->slab = kmem_cache_create(pcs->name, size, 0,
53  					      SLAB_RECLAIM_ACCOUNT, NULL);
54  		if (pcs->slab)
55  			continue;
56  
57  		z_erofs_destroy_pcluster_pool();
58  		return -ENOMEM;
59  	}
60  	return 0;
61  }
62  
63  static struct z_erofs_pcluster *z_erofs_alloc_pcluster(unsigned int nrpages)
64  {
65  	int i;
66  
67  	for (i = 0; i < ARRAY_SIZE(pcluster_pool); ++i) {
68  		struct z_erofs_pcluster_slab *pcs = pcluster_pool + i;
69  		struct z_erofs_pcluster *pcl;
70  
71  		if (nrpages > pcs->maxpages)
72  			continue;
73  
74  		pcl = kmem_cache_zalloc(pcs->slab, GFP_NOFS);
75  		if (!pcl)
76  			return ERR_PTR(-ENOMEM);
77  		pcl->pclusterpages = nrpages;
78  		return pcl;
79  	}
80  	return ERR_PTR(-EINVAL);
81  }
82  
83  static void z_erofs_free_pcluster(struct z_erofs_pcluster *pcl)
84  {
85  	unsigned int pclusterpages = z_erofs_pclusterpages(pcl);
86  	int i;
87  
88  	for (i = 0; i < ARRAY_SIZE(pcluster_pool); ++i) {
89  		struct z_erofs_pcluster_slab *pcs = pcluster_pool + i;
90  
91  		if (pclusterpages > pcs->maxpages)
92  			continue;
93  
94  		kmem_cache_free(pcs->slab, pcl);
95  		return;
96  	}
97  	DBG_BUGON(1);
98  }
99  
100  /* how to allocate cached pages for a pcluster */
101  enum z_erofs_cache_alloctype {
102  	DONTALLOC,	/* don't allocate any cached pages */
103  	/*
104  	 * try to use cached I/O if page allocation succeeds or fallback
105  	 * to in-place I/O instead to avoid any direct reclaim.
106  	 */
107  	TRYALLOC,
108  };
109  
110  /*
111   * tagged pointer with 1-bit tag for all compressed pages
112   * tag 0 - the page is just found with an extra page reference
113   */
114  typedef tagptr1_t compressed_page_t;
115  
116  #define tag_compressed_page_justfound(page) \
117  	tagptr_fold(compressed_page_t, page, 1)
118  
119  static struct workqueue_struct *z_erofs_workqueue __read_mostly;
120  
121  void z_erofs_exit_zip_subsystem(void)
122  {
123  	destroy_workqueue(z_erofs_workqueue);
124  	z_erofs_destroy_pcluster_pool();
125  }
126  
127  static inline int z_erofs_init_workqueue(void)
128  {
129  	const unsigned int onlinecpus = num_possible_cpus();
130  
131  	/*
132  	 * no need to spawn too many threads, limiting threads could minimum
133  	 * scheduling overhead, perhaps per-CPU threads should be better?
134  	 */
135  	z_erofs_workqueue = alloc_workqueue("erofs_unzipd",
136  					    WQ_UNBOUND | WQ_HIGHPRI,
137  					    onlinecpus + onlinecpus / 4);
138  	return z_erofs_workqueue ? 0 : -ENOMEM;
139  }
140  
141  int __init z_erofs_init_zip_subsystem(void)
142  {
143  	int err = z_erofs_create_pcluster_pool();
144  
145  	if (err)
146  		return err;
147  	err = z_erofs_init_workqueue();
148  	if (err)
149  		z_erofs_destroy_pcluster_pool();
150  	return err;
151  }
152  
153  enum z_erofs_collectmode {
154  	COLLECT_SECONDARY,
155  	COLLECT_PRIMARY,
156  	/*
157  	 * The current collection was the tail of an exist chain, in addition
158  	 * that the previous processed chained collections are all decided to
159  	 * be hooked up to it.
160  	 * A new chain will be created for the remaining collections which are
161  	 * not processed yet, therefore different from COLLECT_PRIMARY_FOLLOWED,
162  	 * the next collection cannot reuse the whole page safely in
163  	 * the following scenario:
164  	 *  ________________________________________________________________
165  	 * |      tail (partial) page     |       head (partial) page       |
166  	 * |   (belongs to the next cl)   |   (belongs to the current cl)   |
167  	 * |_______PRIMARY_FOLLOWED_______|________PRIMARY_HOOKED___________|
168  	 */
169  	COLLECT_PRIMARY_HOOKED,
170  	/*
171  	 * a weak form of COLLECT_PRIMARY_FOLLOWED, the difference is that it
172  	 * could be dispatched into bypass queue later due to uptodated managed
173  	 * pages. All related online pages cannot be reused for inplace I/O (or
174  	 * pagevec) since it can be directly decoded without I/O submission.
175  	 */
176  	COLLECT_PRIMARY_FOLLOWED_NOINPLACE,
177  	/*
178  	 * The current collection has been linked with the owned chain, and
179  	 * could also be linked with the remaining collections, which means
180  	 * if the processing page is the tail page of the collection, thus
181  	 * the current collection can safely use the whole page (since
182  	 * the previous collection is under control) for in-place I/O, as
183  	 * illustrated below:
184  	 *  ________________________________________________________________
185  	 * |  tail (partial) page |          head (partial) page           |
186  	 * |  (of the current cl) |      (of the previous collection)      |
187  	 * |  PRIMARY_FOLLOWED or |                                        |
188  	 * |_____PRIMARY_HOOKED___|____________PRIMARY_FOLLOWED____________|
189  	 *
190  	 * [  (*) the above page can be used as inplace I/O.               ]
191  	 */
192  	COLLECT_PRIMARY_FOLLOWED,
193  };
194  
195  struct z_erofs_decompress_frontend {
196  	struct inode *const inode;
197  	struct erofs_map_blocks map;
198  
199  	struct z_erofs_pagevec_ctor vector;
200  
201  	struct z_erofs_pcluster *pcl, *tailpcl;
202  	struct z_erofs_collection *cl;
203  	/* a pointer used to pick up inplace I/O pages */
204  	struct page **icpage_ptr;
205  	z_erofs_next_pcluster_t owned_head;
206  
207  	enum z_erofs_collectmode mode;
208  
209  	bool readahead;
210  	/* used for applying cache strategy on the fly */
211  	bool backmost;
212  	erofs_off_t headoffset;
213  };
214  
215  #define DECOMPRESS_FRONTEND_INIT(__i) { \
216  	.inode = __i, .owned_head = Z_EROFS_PCLUSTER_TAIL, \
217  	.mode = COLLECT_PRIMARY_FOLLOWED }
218  
219  static struct page *z_pagemap_global[Z_EROFS_VMAP_GLOBAL_PAGES];
220  static DEFINE_MUTEX(z_pagemap_global_lock);
221  
222  static void z_erofs_bind_cache(struct z_erofs_decompress_frontend *fe,
223  			       enum z_erofs_cache_alloctype type,
224  			       struct page **pagepool)
225  {
226  	struct address_space *mc = MNGD_MAPPING(EROFS_I_SB(fe->inode));
227  	struct z_erofs_pcluster *pcl = fe->pcl;
228  	bool standalone = true;
229  	/*
230  	 * optimistic allocation without direct reclaim since inplace I/O
231  	 * can be used if low memory otherwise.
232  	 */
233  	gfp_t gfp = (mapping_gfp_mask(mc) & ~__GFP_DIRECT_RECLAIM) |
234  			__GFP_NOMEMALLOC | __GFP_NORETRY | __GFP_NOWARN;
235  	struct page **pages;
236  	pgoff_t index;
237  
238  	if (fe->mode < COLLECT_PRIMARY_FOLLOWED)
239  		return;
240  
241  	pages = pcl->compressed_pages;
242  	index = pcl->obj.index;
243  	for (; index < pcl->obj.index + pcl->pclusterpages; ++index, ++pages) {
244  		struct page *page;
245  		compressed_page_t t;
246  		struct page *newpage = NULL;
247  
248  		/* the compressed page was loaded before */
249  		if (READ_ONCE(*pages))
250  			continue;
251  
252  		page = find_get_page(mc, index);
253  
254  		if (page) {
255  			t = tag_compressed_page_justfound(page);
256  		} else {
257  			/* I/O is needed, no possible to decompress directly */
258  			standalone = false;
259  			switch (type) {
260  			case TRYALLOC:
261  				newpage = erofs_allocpage(pagepool, gfp);
262  				if (!newpage)
263  					continue;
264  				set_page_private(newpage,
265  						 Z_EROFS_PREALLOCATED_PAGE);
266  				t = tag_compressed_page_justfound(newpage);
267  				break;
268  			default:        /* DONTALLOC */
269  				continue;
270  			}
271  		}
272  
273  		if (!cmpxchg_relaxed(pages, NULL, tagptr_cast_ptr(t)))
274  			continue;
275  
276  		if (page)
277  			put_page(page);
278  		else if (newpage)
279  			erofs_pagepool_add(pagepool, newpage);
280  	}
281  
282  	/*
283  	 * don't do inplace I/O if all compressed pages are available in
284  	 * managed cache since it can be moved to the bypass queue instead.
285  	 */
286  	if (standalone)
287  		fe->mode = COLLECT_PRIMARY_FOLLOWED_NOINPLACE;
288  }
289  
290  /* called by erofs_shrinker to get rid of all compressed_pages */
291  int erofs_try_to_free_all_cached_pages(struct erofs_sb_info *sbi,
292  				       struct erofs_workgroup *grp)
293  {
294  	struct z_erofs_pcluster *const pcl =
295  		container_of(grp, struct z_erofs_pcluster, obj);
296  	int i;
297  
298  	DBG_BUGON(z_erofs_is_inline_pcluster(pcl));
299  	/*
300  	 * refcount of workgroup is now freezed as 1,
301  	 * therefore no need to worry about available decompression users.
302  	 */
303  	for (i = 0; i < pcl->pclusterpages; ++i) {
304  		struct page *page = pcl->compressed_pages[i];
305  
306  		if (!page)
307  			continue;
308  
309  		/* block other users from reclaiming or migrating the page */
310  		if (!trylock_page(page))
311  			return -EBUSY;
312  
313  		if (!erofs_page_is_managed(sbi, page))
314  			continue;
315  
316  		/* barrier is implied in the following 'unlock_page' */
317  		WRITE_ONCE(pcl->compressed_pages[i], NULL);
318  		detach_page_private(page);
319  		unlock_page(page);
320  	}
321  	return 0;
322  }
323  
324  int erofs_try_to_free_cached_page(struct page *page)
325  {
326  	struct z_erofs_pcluster *const pcl = (void *)page_private(page);
327  	int ret = 0;	/* 0 - busy */
328  
329  	if (erofs_workgroup_try_to_freeze(&pcl->obj, 1)) {
330  		unsigned int i;
331  
332  		DBG_BUGON(z_erofs_is_inline_pcluster(pcl));
333  		for (i = 0; i < pcl->pclusterpages; ++i) {
334  			if (pcl->compressed_pages[i] == page) {
335  				WRITE_ONCE(pcl->compressed_pages[i], NULL);
336  				ret = 1;
337  				break;
338  			}
339  		}
340  		erofs_workgroup_unfreeze(&pcl->obj, 1);
341  
342  		if (ret)
343  			detach_page_private(page);
344  	}
345  	return ret;
346  }
347  
348  /* page_type must be Z_EROFS_PAGE_TYPE_EXCLUSIVE */
349  static bool z_erofs_try_inplace_io(struct z_erofs_decompress_frontend *fe,
350  				   struct page *page)
351  {
352  	struct z_erofs_pcluster *const pcl = fe->pcl;
353  
354  	while (fe->icpage_ptr > pcl->compressed_pages)
355  		if (!cmpxchg(--fe->icpage_ptr, NULL, page))
356  			return true;
357  	return false;
358  }
359  
360  /* callers must be with collection lock held */
361  static int z_erofs_attach_page(struct z_erofs_decompress_frontend *fe,
362  			       struct page *page, enum z_erofs_page_type type,
363  			       bool pvec_safereuse)
364  {
365  	int ret;
366  
367  	/* give priority for inplaceio */
368  	if (fe->mode >= COLLECT_PRIMARY &&
369  	    type == Z_EROFS_PAGE_TYPE_EXCLUSIVE &&
370  	    z_erofs_try_inplace_io(fe, page))
371  		return 0;
372  
373  	ret = z_erofs_pagevec_enqueue(&fe->vector, page, type,
374  				      pvec_safereuse);
375  	fe->cl->vcnt += (unsigned int)ret;
376  	return ret ? 0 : -EAGAIN;
377  }
378  
379  static void z_erofs_try_to_claim_pcluster(struct z_erofs_decompress_frontend *f)
380  {
381  	struct z_erofs_pcluster *pcl = f->pcl;
382  	z_erofs_next_pcluster_t *owned_head = &f->owned_head;
383  
384  	/* type 1, nil pcluster (this pcluster doesn't belong to any chain.) */
385  	if (cmpxchg(&pcl->next, Z_EROFS_PCLUSTER_NIL,
386  		    *owned_head) == Z_EROFS_PCLUSTER_NIL) {
387  		*owned_head = &pcl->next;
388  		/* so we can attach this pcluster to our submission chain. */
389  		f->mode = COLLECT_PRIMARY_FOLLOWED;
390  		return;
391  	}
392  
393  	/*
394  	 * type 2, link to the end of an existing open chain, be careful
395  	 * that its submission is controlled by the original attached chain.
396  	 */
397  	if (cmpxchg(&pcl->next, Z_EROFS_PCLUSTER_TAIL,
398  		    *owned_head) == Z_EROFS_PCLUSTER_TAIL) {
399  		*owned_head = Z_EROFS_PCLUSTER_TAIL;
400  		f->mode = COLLECT_PRIMARY_HOOKED;
401  		f->tailpcl = NULL;
402  		return;
403  	}
404  	/* type 3, it belongs to a chain, but it isn't the end of the chain */
405  	f->mode = COLLECT_PRIMARY;
406  }
407  
408  static int z_erofs_lookup_collection(struct z_erofs_decompress_frontend *fe,
409  				     struct inode *inode,
410  				     struct erofs_map_blocks *map)
411  {
412  	struct z_erofs_pcluster *pcl = fe->pcl;
413  	struct z_erofs_collection *cl;
414  	unsigned int length;
415  
416  	/* to avoid unexpected loop formed by corrupted images */
417  	if (fe->owned_head == &pcl->next || pcl == fe->tailpcl) {
418  		DBG_BUGON(1);
419  		return -EFSCORRUPTED;
420  	}
421  
422  	cl = z_erofs_primarycollection(pcl);
423  	if (cl->pageofs != (map->m_la & ~PAGE_MASK)) {
424  		DBG_BUGON(1);
425  		return -EFSCORRUPTED;
426  	}
427  
428  	length = READ_ONCE(pcl->length);
429  	if (length & Z_EROFS_PCLUSTER_FULL_LENGTH) {
430  		if ((map->m_llen << Z_EROFS_PCLUSTER_LENGTH_BIT) > length) {
431  			DBG_BUGON(1);
432  			return -EFSCORRUPTED;
433  		}
434  	} else {
435  		unsigned int llen = map->m_llen << Z_EROFS_PCLUSTER_LENGTH_BIT;
436  
437  		if (map->m_flags & EROFS_MAP_FULL_MAPPED)
438  			llen |= Z_EROFS_PCLUSTER_FULL_LENGTH;
439  
440  		while (llen > length &&
441  		       length != cmpxchg_relaxed(&pcl->length, length, llen)) {
442  			cpu_relax();
443  			length = READ_ONCE(pcl->length);
444  		}
445  	}
446  	mutex_lock(&cl->lock);
447  	/* used to check tail merging loop due to corrupted images */
448  	if (fe->owned_head == Z_EROFS_PCLUSTER_TAIL)
449  		fe->tailpcl = pcl;
450  
451  	z_erofs_try_to_claim_pcluster(fe);
452  	fe->cl = cl;
453  	return 0;
454  }
455  
456  static int z_erofs_register_collection(struct z_erofs_decompress_frontend *fe,
457  				       struct inode *inode,
458  				       struct erofs_map_blocks *map)
459  {
460  	bool ztailpacking = map->m_flags & EROFS_MAP_META;
461  	struct z_erofs_pcluster *pcl;
462  	struct z_erofs_collection *cl;
463  	struct erofs_workgroup *grp;
464  	int err;
465  
466  	if (!(map->m_flags & EROFS_MAP_ENCODED)) {
467  		DBG_BUGON(1);
468  		return -EFSCORRUPTED;
469  	}
470  
471  	/* no available pcluster, let's allocate one */
472  	pcl = z_erofs_alloc_pcluster(ztailpacking ? 1 :
473  				     map->m_plen >> PAGE_SHIFT);
474  	if (IS_ERR(pcl))
475  		return PTR_ERR(pcl);
476  
477  	atomic_set(&pcl->obj.refcount, 1);
478  	pcl->algorithmformat = map->m_algorithmformat;
479  	pcl->length = (map->m_llen << Z_EROFS_PCLUSTER_LENGTH_BIT) |
480  		(map->m_flags & EROFS_MAP_FULL_MAPPED ?
481  			Z_EROFS_PCLUSTER_FULL_LENGTH : 0);
482  
483  	/* new pclusters should be claimed as type 1, primary and followed */
484  	pcl->next = fe->owned_head;
485  	fe->mode = COLLECT_PRIMARY_FOLLOWED;
486  
487  	cl = z_erofs_primarycollection(pcl);
488  	cl->pageofs = map->m_la & ~PAGE_MASK;
489  
490  	/*
491  	 * lock all primary followed works before visible to others
492  	 * and mutex_trylock *never* fails for a new pcluster.
493  	 */
494  	mutex_init(&cl->lock);
495  	DBG_BUGON(!mutex_trylock(&cl->lock));
496  
497  	if (ztailpacking) {
498  		pcl->obj.index = 0;	/* which indicates ztailpacking */
499  		pcl->pageofs_in = erofs_blkoff(map->m_pa);
500  		pcl->tailpacking_size = map->m_plen;
501  	} else {
502  		pcl->obj.index = map->m_pa >> PAGE_SHIFT;
503  
504  		grp = erofs_insert_workgroup(inode->i_sb, &pcl->obj);
505  		if (IS_ERR(grp)) {
506  			err = PTR_ERR(grp);
507  			goto err_out;
508  		}
509  
510  		if (grp != &pcl->obj) {
511  			fe->pcl = container_of(grp,
512  					struct z_erofs_pcluster, obj);
513  			err = -EEXIST;
514  			goto err_out;
515  		}
516  	}
517  	/* used to check tail merging loop due to corrupted images */
518  	if (fe->owned_head == Z_EROFS_PCLUSTER_TAIL)
519  		fe->tailpcl = pcl;
520  	fe->owned_head = &pcl->next;
521  	fe->pcl = pcl;
522  	fe->cl = cl;
523  	return 0;
524  
525  err_out:
526  	mutex_unlock(&cl->lock);
527  	z_erofs_free_pcluster(pcl);
528  	return err;
529  }
530  
531  static int z_erofs_collector_begin(struct z_erofs_decompress_frontend *fe,
532  				   struct inode *inode,
533  				   struct erofs_map_blocks *map)
534  {
535  	struct erofs_workgroup *grp;
536  	int ret;
537  
538  	DBG_BUGON(fe->cl);
539  
540  	/* must be Z_EROFS_PCLUSTER_TAIL or pointed to previous collection */
541  	DBG_BUGON(fe->owned_head == Z_EROFS_PCLUSTER_NIL);
542  	DBG_BUGON(fe->owned_head == Z_EROFS_PCLUSTER_TAIL_CLOSED);
543  
544  	if (map->m_flags & EROFS_MAP_META) {
545  		if ((map->m_pa & ~PAGE_MASK) + map->m_plen > PAGE_SIZE) {
546  			DBG_BUGON(1);
547  			return -EFSCORRUPTED;
548  		}
549  		goto tailpacking;
550  	}
551  
552  	grp = erofs_find_workgroup(inode->i_sb, map->m_pa >> PAGE_SHIFT);
553  	if (grp) {
554  		fe->pcl = container_of(grp, struct z_erofs_pcluster, obj);
555  	} else {
556  tailpacking:
557  		ret = z_erofs_register_collection(fe, inode, map);
558  		if (!ret)
559  			goto out;
560  		if (ret != -EEXIST)
561  			return ret;
562  	}
563  
564  	ret = z_erofs_lookup_collection(fe, inode, map);
565  	if (ret) {
566  		erofs_workgroup_put(&fe->pcl->obj);
567  		return ret;
568  	}
569  
570  out:
571  	z_erofs_pagevec_ctor_init(&fe->vector, Z_EROFS_NR_INLINE_PAGEVECS,
572  				  fe->cl->pagevec, fe->cl->vcnt);
573  	/* since file-backed online pages are traversed in reverse order */
574  	fe->icpage_ptr = fe->pcl->compressed_pages +
575  			z_erofs_pclusterpages(fe->pcl);
576  	return 0;
577  }
578  
579  /*
580   * keep in mind that no referenced pclusters will be freed
581   * only after a RCU grace period.
582   */
583  static void z_erofs_rcu_callback(struct rcu_head *head)
584  {
585  	struct z_erofs_collection *const cl =
586  		container_of(head, struct z_erofs_collection, rcu);
587  
588  	z_erofs_free_pcluster(container_of(cl, struct z_erofs_pcluster,
589  					   primary_collection));
590  }
591  
592  void erofs_workgroup_free_rcu(struct erofs_workgroup *grp)
593  {
594  	struct z_erofs_pcluster *const pcl =
595  		container_of(grp, struct z_erofs_pcluster, obj);
596  	struct z_erofs_collection *const cl = z_erofs_primarycollection(pcl);
597  
598  	call_rcu(&cl->rcu, z_erofs_rcu_callback);
599  }
600  
601  static void z_erofs_collection_put(struct z_erofs_collection *cl)
602  {
603  	struct z_erofs_pcluster *const pcl =
604  		container_of(cl, struct z_erofs_pcluster, primary_collection);
605  
606  	erofs_workgroup_put(&pcl->obj);
607  }
608  
609  static bool z_erofs_collector_end(struct z_erofs_decompress_frontend *fe)
610  {
611  	struct z_erofs_collection *cl = fe->cl;
612  
613  	if (!cl)
614  		return false;
615  
616  	z_erofs_pagevec_ctor_exit(&fe->vector, false);
617  	mutex_unlock(&cl->lock);
618  
619  	/*
620  	 * if all pending pages are added, don't hold its reference
621  	 * any longer if the pcluster isn't hosted by ourselves.
622  	 */
623  	if (fe->mode < COLLECT_PRIMARY_FOLLOWED_NOINPLACE)
624  		z_erofs_collection_put(cl);
625  
626  	fe->cl = NULL;
627  	return true;
628  }
629  
630  static bool should_alloc_managed_pages(struct z_erofs_decompress_frontend *fe,
631  				       unsigned int cachestrategy,
632  				       erofs_off_t la)
633  {
634  	if (cachestrategy <= EROFS_ZIP_CACHE_DISABLED)
635  		return false;
636  
637  	if (fe->backmost)
638  		return true;
639  
640  	return cachestrategy >= EROFS_ZIP_CACHE_READAROUND &&
641  		la < fe->headoffset;
642  }
643  
644  static int z_erofs_do_read_page(struct z_erofs_decompress_frontend *fe,
645  				struct page *page, struct page **pagepool)
646  {
647  	struct inode *const inode = fe->inode;
648  	struct erofs_sb_info *const sbi = EROFS_I_SB(inode);
649  	struct erofs_map_blocks *const map = &fe->map;
650  	const loff_t offset = page_offset(page);
651  	bool tight = true;
652  
653  	enum z_erofs_cache_alloctype cache_strategy;
654  	enum z_erofs_page_type page_type;
655  	unsigned int cur, end, spiltted, index;
656  	int err = 0;
657  
658  	/* register locked file pages as online pages in pack */
659  	z_erofs_onlinepage_init(page);
660  
661  	spiltted = 0;
662  	end = PAGE_SIZE;
663  repeat:
664  	cur = end - 1;
665  
666  	/* lucky, within the range of the current map_blocks */
667  	if (offset + cur >= map->m_la &&
668  	    offset + cur < map->m_la + map->m_llen) {
669  		/* didn't get a valid collection previously (very rare) */
670  		if (!fe->cl)
671  			goto restart_now;
672  		goto hitted;
673  	}
674  
675  	/* go ahead the next map_blocks */
676  	erofs_dbg("%s: [out-of-range] pos %llu", __func__, offset + cur);
677  
678  	if (z_erofs_collector_end(fe))
679  		fe->backmost = false;
680  
681  	map->m_la = offset + cur;
682  	map->m_llen = 0;
683  	err = z_erofs_map_blocks_iter(inode, map, 0);
684  	if (err)
685  		goto err_out;
686  
687  restart_now:
688  	if (!(map->m_flags & EROFS_MAP_MAPPED))
689  		goto hitted;
690  
691  	err = z_erofs_collector_begin(fe, inode, map);
692  	if (err)
693  		goto err_out;
694  
695  	if (z_erofs_is_inline_pcluster(fe->pcl)) {
696  		void *mp;
697  
698  		mp = erofs_read_metabuf(&fe->map.buf, inode->i_sb,
699  					erofs_blknr(map->m_pa), EROFS_NO_KMAP);
700  		if (IS_ERR(mp)) {
701  			err = PTR_ERR(mp);
702  			erofs_err(inode->i_sb,
703  				  "failed to get inline page, err %d", err);
704  			goto err_out;
705  		}
706  		get_page(fe->map.buf.page);
707  		WRITE_ONCE(fe->pcl->compressed_pages[0], fe->map.buf.page);
708  		fe->mode = COLLECT_PRIMARY_FOLLOWED_NOINPLACE;
709  	} else {
710  		/* bind cache first when cached decompression is preferred */
711  		if (should_alloc_managed_pages(fe, sbi->opt.cache_strategy,
712  					       map->m_la))
713  			cache_strategy = TRYALLOC;
714  		else
715  			cache_strategy = DONTALLOC;
716  
717  		z_erofs_bind_cache(fe, cache_strategy, pagepool);
718  	}
719  hitted:
720  	/*
721  	 * Ensure the current partial page belongs to this submit chain rather
722  	 * than other concurrent submit chains or the noio(bypass) chain since
723  	 * those chains are handled asynchronously thus the page cannot be used
724  	 * for inplace I/O or pagevec (should be processed in strict order.)
725  	 */
726  	tight &= (fe->mode >= COLLECT_PRIMARY_HOOKED &&
727  		  fe->mode != COLLECT_PRIMARY_FOLLOWED_NOINPLACE);
728  
729  	cur = end - min_t(unsigned int, offset + end - map->m_la, end);
730  	if (!(map->m_flags & EROFS_MAP_MAPPED)) {
731  		zero_user_segment(page, cur, end);
732  		goto next_part;
733  	}
734  
735  	/* let's derive page type */
736  	page_type = cur ? Z_EROFS_VLE_PAGE_TYPE_HEAD :
737  		(!spiltted ? Z_EROFS_PAGE_TYPE_EXCLUSIVE :
738  			(tight ? Z_EROFS_PAGE_TYPE_EXCLUSIVE :
739  				Z_EROFS_VLE_PAGE_TYPE_TAIL_SHARED));
740  
741  	if (cur)
742  		tight &= (fe->mode >= COLLECT_PRIMARY_FOLLOWED);
743  
744  retry:
745  	err = z_erofs_attach_page(fe, page, page_type,
746  				  fe->mode >= COLLECT_PRIMARY_FOLLOWED);
747  	/* should allocate an additional short-lived page for pagevec */
748  	if (err == -EAGAIN) {
749  		struct page *const newpage =
750  				alloc_page(GFP_NOFS | __GFP_NOFAIL);
751  
752  		set_page_private(newpage, Z_EROFS_SHORTLIVED_PAGE);
753  		err = z_erofs_attach_page(fe, newpage,
754  					  Z_EROFS_PAGE_TYPE_EXCLUSIVE, true);
755  		if (!err)
756  			goto retry;
757  	}
758  
759  	if (err)
760  		goto err_out;
761  
762  	index = page->index - (map->m_la >> PAGE_SHIFT);
763  
764  	z_erofs_onlinepage_fixup(page, index, true);
765  
766  	/* bump up the number of spiltted parts of a page */
767  	++spiltted;
768  	/* also update nr_pages */
769  	fe->cl->nr_pages = max_t(pgoff_t, fe->cl->nr_pages, index + 1);
770  next_part:
771  	/* can be used for verification */
772  	map->m_llen = offset + cur - map->m_la;
773  
774  	end = cur;
775  	if (end > 0)
776  		goto repeat;
777  
778  out:
779  	z_erofs_onlinepage_endio(page);
780  
781  	erofs_dbg("%s, finish page: %pK spiltted: %u map->m_llen %llu",
782  		  __func__, page, spiltted, map->m_llen);
783  	return err;
784  
785  	/* if some error occurred while processing this page */
786  err_out:
787  	SetPageError(page);
788  	goto out;
789  }
790  
791  static bool z_erofs_get_sync_decompress_policy(struct erofs_sb_info *sbi,
792  				       unsigned int readahead_pages)
793  {
794  	/* auto: enable for read_folio, disable for readahead */
795  	if ((sbi->opt.sync_decompress == EROFS_SYNC_DECOMPRESS_AUTO) &&
796  	    !readahead_pages)
797  		return true;
798  
799  	if ((sbi->opt.sync_decompress == EROFS_SYNC_DECOMPRESS_FORCE_ON) &&
800  	    (readahead_pages <= sbi->opt.max_sync_decompress_pages))
801  		return true;
802  
803  	return false;
804  }
805  
806  static bool z_erofs_page_is_invalidated(struct page *page)
807  {
808  	return !page->mapping && !z_erofs_is_shortlived_page(page);
809  }
810  
811  static int z_erofs_decompress_pcluster(struct super_block *sb,
812  				       struct z_erofs_pcluster *pcl,
813  				       struct page **pagepool)
814  {
815  	struct erofs_sb_info *const sbi = EROFS_SB(sb);
816  	unsigned int pclusterpages = z_erofs_pclusterpages(pcl);
817  	struct z_erofs_pagevec_ctor ctor;
818  	unsigned int i, inputsize, outputsize, llen, nr_pages;
819  	struct page *pages_onstack[Z_EROFS_VMAP_ONSTACK_PAGES];
820  	struct page **pages, **compressed_pages, *page;
821  
822  	enum z_erofs_page_type page_type;
823  	bool overlapped, partial;
824  	struct z_erofs_collection *cl;
825  	int err;
826  
827  	might_sleep();
828  	cl = z_erofs_primarycollection(pcl);
829  	DBG_BUGON(!READ_ONCE(cl->nr_pages));
830  
831  	mutex_lock(&cl->lock);
832  	nr_pages = cl->nr_pages;
833  
834  	if (nr_pages <= Z_EROFS_VMAP_ONSTACK_PAGES) {
835  		pages = pages_onstack;
836  	} else if (nr_pages <= Z_EROFS_VMAP_GLOBAL_PAGES &&
837  		   mutex_trylock(&z_pagemap_global_lock)) {
838  		pages = z_pagemap_global;
839  	} else {
840  		gfp_t gfp_flags = GFP_KERNEL;
841  
842  		if (nr_pages > Z_EROFS_VMAP_GLOBAL_PAGES)
843  			gfp_flags |= __GFP_NOFAIL;
844  
845  		pages = kvmalloc_array(nr_pages, sizeof(struct page *),
846  				       gfp_flags);
847  
848  		/* fallback to global pagemap for the lowmem scenario */
849  		if (!pages) {
850  			mutex_lock(&z_pagemap_global_lock);
851  			pages = z_pagemap_global;
852  		}
853  	}
854  
855  	for (i = 0; i < nr_pages; ++i)
856  		pages[i] = NULL;
857  
858  	err = 0;
859  	z_erofs_pagevec_ctor_init(&ctor, Z_EROFS_NR_INLINE_PAGEVECS,
860  				  cl->pagevec, 0);
861  
862  	for (i = 0; i < cl->vcnt; ++i) {
863  		unsigned int pagenr;
864  
865  		page = z_erofs_pagevec_dequeue(&ctor, &page_type);
866  
867  		/* all pages in pagevec ought to be valid */
868  		DBG_BUGON(!page);
869  		DBG_BUGON(z_erofs_page_is_invalidated(page));
870  
871  		if (z_erofs_put_shortlivedpage(pagepool, page))
872  			continue;
873  
874  		if (page_type == Z_EROFS_VLE_PAGE_TYPE_HEAD)
875  			pagenr = 0;
876  		else
877  			pagenr = z_erofs_onlinepage_index(page);
878  
879  		DBG_BUGON(pagenr >= nr_pages);
880  
881  		/*
882  		 * currently EROFS doesn't support multiref(dedup),
883  		 * so here erroring out one multiref page.
884  		 */
885  		if (pages[pagenr]) {
886  			DBG_BUGON(1);
887  			SetPageError(pages[pagenr]);
888  			z_erofs_onlinepage_endio(pages[pagenr]);
889  			err = -EFSCORRUPTED;
890  		}
891  		pages[pagenr] = page;
892  	}
893  	z_erofs_pagevec_ctor_exit(&ctor, true);
894  
895  	overlapped = false;
896  	compressed_pages = pcl->compressed_pages;
897  
898  	for (i = 0; i < pclusterpages; ++i) {
899  		unsigned int pagenr;
900  
901  		page = compressed_pages[i];
902  		/* all compressed pages ought to be valid */
903  		DBG_BUGON(!page);
904  
905  		if (z_erofs_is_inline_pcluster(pcl)) {
906  			if (!PageUptodate(page))
907  				err = -EIO;
908  			continue;
909  		}
910  
911  		DBG_BUGON(z_erofs_page_is_invalidated(page));
912  		if (!z_erofs_is_shortlived_page(page)) {
913  			if (erofs_page_is_managed(sbi, page)) {
914  				if (!PageUptodate(page))
915  					err = -EIO;
916  				continue;
917  			}
918  
919  			/*
920  			 * only if non-head page can be selected
921  			 * for inplace decompression
922  			 */
923  			pagenr = z_erofs_onlinepage_index(page);
924  
925  			DBG_BUGON(pagenr >= nr_pages);
926  			if (pages[pagenr]) {
927  				DBG_BUGON(1);
928  				SetPageError(pages[pagenr]);
929  				z_erofs_onlinepage_endio(pages[pagenr]);
930  				err = -EFSCORRUPTED;
931  			}
932  			pages[pagenr] = page;
933  
934  			overlapped = true;
935  		}
936  
937  		/* PG_error needs checking for all non-managed pages */
938  		if (PageError(page)) {
939  			DBG_BUGON(PageUptodate(page));
940  			err = -EIO;
941  		}
942  	}
943  
944  	if (err)
945  		goto out;
946  
947  	llen = pcl->length >> Z_EROFS_PCLUSTER_LENGTH_BIT;
948  	if (nr_pages << PAGE_SHIFT >= cl->pageofs + llen) {
949  		outputsize = llen;
950  		partial = !(pcl->length & Z_EROFS_PCLUSTER_FULL_LENGTH);
951  	} else {
952  		outputsize = (nr_pages << PAGE_SHIFT) - cl->pageofs;
953  		partial = true;
954  	}
955  
956  	if (z_erofs_is_inline_pcluster(pcl))
957  		inputsize = pcl->tailpacking_size;
958  	else
959  		inputsize = pclusterpages * PAGE_SIZE;
960  
961  	err = z_erofs_decompress(&(struct z_erofs_decompress_req) {
962  					.sb = sb,
963  					.in = compressed_pages,
964  					.out = pages,
965  					.pageofs_in = pcl->pageofs_in,
966  					.pageofs_out = cl->pageofs,
967  					.inputsize = inputsize,
968  					.outputsize = outputsize,
969  					.alg = pcl->algorithmformat,
970  					.inplace_io = overlapped,
971  					.partial_decoding = partial
972  				 }, pagepool);
973  
974  out:
975  	/* must handle all compressed pages before actual file pages */
976  	if (z_erofs_is_inline_pcluster(pcl)) {
977  		page = compressed_pages[0];
978  		WRITE_ONCE(compressed_pages[0], NULL);
979  		put_page(page);
980  	} else {
981  		for (i = 0; i < pclusterpages; ++i) {
982  			page = compressed_pages[i];
983  
984  			if (erofs_page_is_managed(sbi, page))
985  				continue;
986  
987  			/* recycle all individual short-lived pages */
988  			(void)z_erofs_put_shortlivedpage(pagepool, page);
989  			WRITE_ONCE(compressed_pages[i], NULL);
990  		}
991  	}
992  
993  	for (i = 0; i < nr_pages; ++i) {
994  		page = pages[i];
995  		if (!page)
996  			continue;
997  
998  		DBG_BUGON(z_erofs_page_is_invalidated(page));
999  
1000  		/* recycle all individual short-lived pages */
1001  		if (z_erofs_put_shortlivedpage(pagepool, page))
1002  			continue;
1003  
1004  		if (err < 0)
1005  			SetPageError(page);
1006  
1007  		z_erofs_onlinepage_endio(page);
1008  	}
1009  
1010  	if (pages == z_pagemap_global)
1011  		mutex_unlock(&z_pagemap_global_lock);
1012  	else if (pages != pages_onstack)
1013  		kvfree(pages);
1014  
1015  	cl->nr_pages = 0;
1016  	cl->vcnt = 0;
1017  
1018  	/* all cl locks MUST be taken before the following line */
1019  	WRITE_ONCE(pcl->next, Z_EROFS_PCLUSTER_NIL);
1020  
1021  	/* all cl locks SHOULD be released right now */
1022  	mutex_unlock(&cl->lock);
1023  
1024  	z_erofs_collection_put(cl);
1025  	return err;
1026  }
1027  
1028  static void z_erofs_decompress_queue(const struct z_erofs_decompressqueue *io,
1029  				     struct page **pagepool)
1030  {
1031  	z_erofs_next_pcluster_t owned = io->head;
1032  
1033  	while (owned != Z_EROFS_PCLUSTER_TAIL_CLOSED) {
1034  		struct z_erofs_pcluster *pcl;
1035  
1036  		/* no possible that 'owned' equals Z_EROFS_WORK_TPTR_TAIL */
1037  		DBG_BUGON(owned == Z_EROFS_PCLUSTER_TAIL);
1038  
1039  		/* no possible that 'owned' equals NULL */
1040  		DBG_BUGON(owned == Z_EROFS_PCLUSTER_NIL);
1041  
1042  		pcl = container_of(owned, struct z_erofs_pcluster, next);
1043  		owned = READ_ONCE(pcl->next);
1044  
1045  		z_erofs_decompress_pcluster(io->sb, pcl, pagepool);
1046  	}
1047  }
1048  
1049  static void z_erofs_decompressqueue_work(struct work_struct *work)
1050  {
1051  	struct z_erofs_decompressqueue *bgq =
1052  		container_of(work, struct z_erofs_decompressqueue, u.work);
1053  	struct page *pagepool = NULL;
1054  
1055  	DBG_BUGON(bgq->head == Z_EROFS_PCLUSTER_TAIL_CLOSED);
1056  	z_erofs_decompress_queue(bgq, &pagepool);
1057  
1058  	erofs_release_pages(&pagepool);
1059  	kvfree(bgq);
1060  }
1061  
1062  static void z_erofs_decompress_kickoff(struct z_erofs_decompressqueue *io,
1063  				       bool sync, int bios)
1064  {
1065  	struct erofs_sb_info *const sbi = EROFS_SB(io->sb);
1066  
1067  	/* wake up the caller thread for sync decompression */
1068  	if (sync) {
1069  		if (!atomic_add_return(bios, &io->pending_bios))
1070  			complete(&io->u.done);
1071  
1072  		return;
1073  	}
1074  
1075  	if (atomic_add_return(bios, &io->pending_bios))
1076  		return;
1077  	/* Use workqueue and sync decompression for atomic contexts only */
1078  	if (in_atomic() || irqs_disabled()) {
1079  		queue_work(z_erofs_workqueue, &io->u.work);
1080  		/* enable sync decompression for readahead */
1081  		if (sbi->opt.sync_decompress == EROFS_SYNC_DECOMPRESS_AUTO)
1082  			sbi->opt.sync_decompress = EROFS_SYNC_DECOMPRESS_FORCE_ON;
1083  		return;
1084  	}
1085  	z_erofs_decompressqueue_work(&io->u.work);
1086  }
1087  
1088  static struct page *pickup_page_for_submission(struct z_erofs_pcluster *pcl,
1089  					       unsigned int nr,
1090  					       struct page **pagepool,
1091  					       struct address_space *mc)
1092  {
1093  	const pgoff_t index = pcl->obj.index;
1094  	gfp_t gfp = mapping_gfp_mask(mc);
1095  	bool tocache = false;
1096  
1097  	struct address_space *mapping;
1098  	struct page *oldpage, *page;
1099  
1100  	compressed_page_t t;
1101  	int justfound;
1102  
1103  repeat:
1104  	page = READ_ONCE(pcl->compressed_pages[nr]);
1105  	oldpage = page;
1106  
1107  	if (!page)
1108  		goto out_allocpage;
1109  
1110  	/* process the target tagged pointer */
1111  	t = tagptr_init(compressed_page_t, page);
1112  	justfound = tagptr_unfold_tags(t);
1113  	page = tagptr_unfold_ptr(t);
1114  
1115  	/*
1116  	 * preallocated cached pages, which is used to avoid direct reclaim
1117  	 * otherwise, it will go inplace I/O path instead.
1118  	 */
1119  	if (page->private == Z_EROFS_PREALLOCATED_PAGE) {
1120  		WRITE_ONCE(pcl->compressed_pages[nr], page);
1121  		set_page_private(page, 0);
1122  		tocache = true;
1123  		goto out_tocache;
1124  	}
1125  	mapping = READ_ONCE(page->mapping);
1126  
1127  	/*
1128  	 * file-backed online pages in plcuster are all locked steady,
1129  	 * therefore it is impossible for `mapping' to be NULL.
1130  	 */
1131  	if (mapping && mapping != mc)
1132  		/* ought to be unmanaged pages */
1133  		goto out;
1134  
1135  	/* directly return for shortlived page as well */
1136  	if (z_erofs_is_shortlived_page(page))
1137  		goto out;
1138  
1139  	lock_page(page);
1140  
1141  	/* only true if page reclaim goes wrong, should never happen */
1142  	DBG_BUGON(justfound && PagePrivate(page));
1143  
1144  	/* the page is still in manage cache */
1145  	if (page->mapping == mc) {
1146  		WRITE_ONCE(pcl->compressed_pages[nr], page);
1147  
1148  		ClearPageError(page);
1149  		if (!PagePrivate(page)) {
1150  			/*
1151  			 * impossible to be !PagePrivate(page) for
1152  			 * the current restriction as well if
1153  			 * the page is already in compressed_pages[].
1154  			 */
1155  			DBG_BUGON(!justfound);
1156  
1157  			justfound = 0;
1158  			set_page_private(page, (unsigned long)pcl);
1159  			SetPagePrivate(page);
1160  		}
1161  
1162  		/* no need to submit io if it is already up-to-date */
1163  		if (PageUptodate(page)) {
1164  			unlock_page(page);
1165  			page = NULL;
1166  		}
1167  		goto out;
1168  	}
1169  
1170  	/*
1171  	 * the managed page has been truncated, it's unsafe to
1172  	 * reuse this one, let's allocate a new cache-managed page.
1173  	 */
1174  	DBG_BUGON(page->mapping);
1175  	DBG_BUGON(!justfound);
1176  
1177  	tocache = true;
1178  	unlock_page(page);
1179  	put_page(page);
1180  out_allocpage:
1181  	page = erofs_allocpage(pagepool, gfp | __GFP_NOFAIL);
1182  	if (oldpage != cmpxchg(&pcl->compressed_pages[nr], oldpage, page)) {
1183  		erofs_pagepool_add(pagepool, page);
1184  		cond_resched();
1185  		goto repeat;
1186  	}
1187  out_tocache:
1188  	if (!tocache || add_to_page_cache_lru(page, mc, index + nr, gfp)) {
1189  		/* turn into temporary page if fails (1 ref) */
1190  		set_page_private(page, Z_EROFS_SHORTLIVED_PAGE);
1191  		goto out;
1192  	}
1193  	attach_page_private(page, pcl);
1194  	/* drop a refcount added by allocpage (then we have 2 refs here) */
1195  	put_page(page);
1196  
1197  out:	/* the only exit (for tracing and debugging) */
1198  	return page;
1199  }
1200  
1201  static struct z_erofs_decompressqueue *
1202  jobqueue_init(struct super_block *sb,
1203  	      struct z_erofs_decompressqueue *fgq, bool *fg)
1204  {
1205  	struct z_erofs_decompressqueue *q;
1206  
1207  	if (fg && !*fg) {
1208  		q = kvzalloc(sizeof(*q), GFP_KERNEL | __GFP_NOWARN);
1209  		if (!q) {
1210  			*fg = true;
1211  			goto fg_out;
1212  		}
1213  		INIT_WORK(&q->u.work, z_erofs_decompressqueue_work);
1214  	} else {
1215  fg_out:
1216  		q = fgq;
1217  		init_completion(&fgq->u.done);
1218  		atomic_set(&fgq->pending_bios, 0);
1219  	}
1220  	q->sb = sb;
1221  	q->head = Z_EROFS_PCLUSTER_TAIL_CLOSED;
1222  	return q;
1223  }
1224  
1225  /* define decompression jobqueue types */
1226  enum {
1227  	JQ_BYPASS,
1228  	JQ_SUBMIT,
1229  	NR_JOBQUEUES,
1230  };
1231  
1232  static void *jobqueueset_init(struct super_block *sb,
1233  			      struct z_erofs_decompressqueue *q[],
1234  			      struct z_erofs_decompressqueue *fgq, bool *fg)
1235  {
1236  	/*
1237  	 * if managed cache is enabled, bypass jobqueue is needed,
1238  	 * no need to read from device for all pclusters in this queue.
1239  	 */
1240  	q[JQ_BYPASS] = jobqueue_init(sb, fgq + JQ_BYPASS, NULL);
1241  	q[JQ_SUBMIT] = jobqueue_init(sb, fgq + JQ_SUBMIT, fg);
1242  
1243  	return tagptr_cast_ptr(tagptr_fold(tagptr1_t, q[JQ_SUBMIT], *fg));
1244  }
1245  
1246  static void move_to_bypass_jobqueue(struct z_erofs_pcluster *pcl,
1247  				    z_erofs_next_pcluster_t qtail[],
1248  				    z_erofs_next_pcluster_t owned_head)
1249  {
1250  	z_erofs_next_pcluster_t *const submit_qtail = qtail[JQ_SUBMIT];
1251  	z_erofs_next_pcluster_t *const bypass_qtail = qtail[JQ_BYPASS];
1252  
1253  	DBG_BUGON(owned_head == Z_EROFS_PCLUSTER_TAIL_CLOSED);
1254  	if (owned_head == Z_EROFS_PCLUSTER_TAIL)
1255  		owned_head = Z_EROFS_PCLUSTER_TAIL_CLOSED;
1256  
1257  	WRITE_ONCE(pcl->next, Z_EROFS_PCLUSTER_TAIL_CLOSED);
1258  
1259  	WRITE_ONCE(*submit_qtail, owned_head);
1260  	WRITE_ONCE(*bypass_qtail, &pcl->next);
1261  
1262  	qtail[JQ_BYPASS] = &pcl->next;
1263  }
1264  
1265  static void z_erofs_decompressqueue_endio(struct bio *bio)
1266  {
1267  	tagptr1_t t = tagptr_init(tagptr1_t, bio->bi_private);
1268  	struct z_erofs_decompressqueue *q = tagptr_unfold_ptr(t);
1269  	blk_status_t err = bio->bi_status;
1270  	struct bio_vec *bvec;
1271  	struct bvec_iter_all iter_all;
1272  
1273  	bio_for_each_segment_all(bvec, bio, iter_all) {
1274  		struct page *page = bvec->bv_page;
1275  
1276  		DBG_BUGON(PageUptodate(page));
1277  		DBG_BUGON(z_erofs_page_is_invalidated(page));
1278  
1279  		if (err)
1280  			SetPageError(page);
1281  
1282  		if (erofs_page_is_managed(EROFS_SB(q->sb), page)) {
1283  			if (!err)
1284  				SetPageUptodate(page);
1285  			unlock_page(page);
1286  		}
1287  	}
1288  	z_erofs_decompress_kickoff(q, tagptr_unfold_tags(t), -1);
1289  	bio_put(bio);
1290  }
1291  
1292  static void z_erofs_submit_queue(struct super_block *sb,
1293  				 struct z_erofs_decompress_frontend *f,
1294  				 struct page **pagepool,
1295  				 struct z_erofs_decompressqueue *fgq,
1296  				 bool *force_fg)
1297  {
1298  	struct erofs_sb_info *const sbi = EROFS_SB(sb);
1299  	z_erofs_next_pcluster_t qtail[NR_JOBQUEUES];
1300  	struct z_erofs_decompressqueue *q[NR_JOBQUEUES];
1301  	void *bi_private;
1302  	z_erofs_next_pcluster_t owned_head = f->owned_head;
1303  	/* bio is NULL initially, so no need to initialize last_{index,bdev} */
1304  	pgoff_t last_index;
1305  	struct block_device *last_bdev;
1306  	unsigned int nr_bios = 0;
1307  	struct bio *bio = NULL;
1308  
1309  	bi_private = jobqueueset_init(sb, q, fgq, force_fg);
1310  	qtail[JQ_BYPASS] = &q[JQ_BYPASS]->head;
1311  	qtail[JQ_SUBMIT] = &q[JQ_SUBMIT]->head;
1312  
1313  	/* by default, all need io submission */
1314  	q[JQ_SUBMIT]->head = owned_head;
1315  
1316  	do {
1317  		struct erofs_map_dev mdev;
1318  		struct z_erofs_pcluster *pcl;
1319  		pgoff_t cur, end;
1320  		unsigned int i = 0;
1321  		bool bypass = true;
1322  
1323  		/* no possible 'owned_head' equals the following */
1324  		DBG_BUGON(owned_head == Z_EROFS_PCLUSTER_TAIL_CLOSED);
1325  		DBG_BUGON(owned_head == Z_EROFS_PCLUSTER_NIL);
1326  
1327  		pcl = container_of(owned_head, struct z_erofs_pcluster, next);
1328  
1329  		/* close the main owned chain at first */
1330  		owned_head = cmpxchg(&pcl->next, Z_EROFS_PCLUSTER_TAIL,
1331  				     Z_EROFS_PCLUSTER_TAIL_CLOSED);
1332  		if (z_erofs_is_inline_pcluster(pcl)) {
1333  			move_to_bypass_jobqueue(pcl, qtail, owned_head);
1334  			continue;
1335  		}
1336  
1337  		/* no device id here, thus it will always succeed */
1338  		mdev = (struct erofs_map_dev) {
1339  			.m_pa = blknr_to_addr(pcl->obj.index),
1340  		};
1341  		(void)erofs_map_dev(sb, &mdev);
1342  
1343  		cur = erofs_blknr(mdev.m_pa);
1344  		end = cur + pcl->pclusterpages;
1345  
1346  		do {
1347  			struct page *page;
1348  
1349  			page = pickup_page_for_submission(pcl, i++, pagepool,
1350  							  MNGD_MAPPING(sbi));
1351  			if (!page)
1352  				continue;
1353  
1354  			if (bio && (cur != last_index + 1 ||
1355  				    last_bdev != mdev.m_bdev)) {
1356  submit_bio_retry:
1357  				submit_bio(bio);
1358  				bio = NULL;
1359  			}
1360  
1361  			if (!bio) {
1362  				bio = bio_alloc(mdev.m_bdev, BIO_MAX_VECS,
1363  						REQ_OP_READ, GFP_NOIO);
1364  				bio->bi_end_io = z_erofs_decompressqueue_endio;
1365  
1366  				last_bdev = mdev.m_bdev;
1367  				bio->bi_iter.bi_sector = (sector_t)cur <<
1368  					LOG_SECTORS_PER_BLOCK;
1369  				bio->bi_private = bi_private;
1370  				if (f->readahead)
1371  					bio->bi_opf |= REQ_RAHEAD;
1372  				++nr_bios;
1373  			}
1374  
1375  			if (bio_add_page(bio, page, PAGE_SIZE, 0) < PAGE_SIZE)
1376  				goto submit_bio_retry;
1377  
1378  			last_index = cur;
1379  			bypass = false;
1380  		} while (++cur < end);
1381  
1382  		if (!bypass)
1383  			qtail[JQ_SUBMIT] = &pcl->next;
1384  		else
1385  			move_to_bypass_jobqueue(pcl, qtail, owned_head);
1386  	} while (owned_head != Z_EROFS_PCLUSTER_TAIL);
1387  
1388  	if (bio)
1389  		submit_bio(bio);
1390  
1391  	/*
1392  	 * although background is preferred, no one is pending for submission.
1393  	 * don't issue workqueue for decompression but drop it directly instead.
1394  	 */
1395  	if (!*force_fg && !nr_bios) {
1396  		kvfree(q[JQ_SUBMIT]);
1397  		return;
1398  	}
1399  	z_erofs_decompress_kickoff(q[JQ_SUBMIT], *force_fg, nr_bios);
1400  }
1401  
1402  static void z_erofs_runqueue(struct super_block *sb,
1403  			     struct z_erofs_decompress_frontend *f,
1404  			     struct page **pagepool, bool force_fg)
1405  {
1406  	struct z_erofs_decompressqueue io[NR_JOBQUEUES];
1407  
1408  	if (f->owned_head == Z_EROFS_PCLUSTER_TAIL)
1409  		return;
1410  	z_erofs_submit_queue(sb, f, pagepool, io, &force_fg);
1411  
1412  	/* handle bypass queue (no i/o pclusters) immediately */
1413  	z_erofs_decompress_queue(&io[JQ_BYPASS], pagepool);
1414  
1415  	if (!force_fg)
1416  		return;
1417  
1418  	/* wait until all bios are completed */
1419  	wait_for_completion_io(&io[JQ_SUBMIT].u.done);
1420  
1421  	/* handle synchronous decompress queue in the caller context */
1422  	z_erofs_decompress_queue(&io[JQ_SUBMIT], pagepool);
1423  }
1424  
1425  /*
1426   * Since partial uptodate is still unimplemented for now, we have to use
1427   * approximate readmore strategies as a start.
1428   */
1429  static void z_erofs_pcluster_readmore(struct z_erofs_decompress_frontend *f,
1430  				      struct readahead_control *rac,
1431  				      erofs_off_t end,
1432  				      struct page **pagepool,
1433  				      bool backmost)
1434  {
1435  	struct inode *inode = f->inode;
1436  	struct erofs_map_blocks *map = &f->map;
1437  	erofs_off_t cur;
1438  	int err;
1439  
1440  	if (backmost) {
1441  		map->m_la = end;
1442  		err = z_erofs_map_blocks_iter(inode, map,
1443  					      EROFS_GET_BLOCKS_READMORE);
1444  		if (err)
1445  			return;
1446  
1447  		/* expend ra for the trailing edge if readahead */
1448  		if (rac) {
1449  			loff_t newstart = readahead_pos(rac);
1450  
1451  			cur = round_up(map->m_la + map->m_llen, PAGE_SIZE);
1452  			readahead_expand(rac, newstart, cur - newstart);
1453  			return;
1454  		}
1455  		end = round_up(end, PAGE_SIZE);
1456  	} else {
1457  		end = round_up(map->m_la, PAGE_SIZE);
1458  
1459  		if (!map->m_llen)
1460  			return;
1461  	}
1462  
1463  	cur = map->m_la + map->m_llen - 1;
1464  	while (cur >= end) {
1465  		pgoff_t index = cur >> PAGE_SHIFT;
1466  		struct page *page;
1467  
1468  		page = erofs_grab_cache_page_nowait(inode->i_mapping, index);
1469  		if (!page)
1470  			goto skip;
1471  
1472  		if (PageUptodate(page)) {
1473  			unlock_page(page);
1474  			put_page(page);
1475  			goto skip;
1476  		}
1477  
1478  		err = z_erofs_do_read_page(f, page, pagepool);
1479  		if (err)
1480  			erofs_err(inode->i_sb,
1481  				  "readmore error at page %lu @ nid %llu",
1482  				  index, EROFS_I(inode)->nid);
1483  		put_page(page);
1484  skip:
1485  		if (cur < PAGE_SIZE)
1486  			break;
1487  		cur = (index << PAGE_SHIFT) - 1;
1488  	}
1489  }
1490  
1491  static int z_erofs_read_folio(struct file *file, struct folio *folio)
1492  {
1493  	struct page *page = &folio->page;
1494  	struct inode *const inode = page->mapping->host;
1495  	struct erofs_sb_info *const sbi = EROFS_I_SB(inode);
1496  	struct z_erofs_decompress_frontend f = DECOMPRESS_FRONTEND_INIT(inode);
1497  	struct page *pagepool = NULL;
1498  	int err;
1499  
1500  	trace_erofs_readpage(page, false);
1501  	f.headoffset = (erofs_off_t)page->index << PAGE_SHIFT;
1502  
1503  	z_erofs_pcluster_readmore(&f, NULL, f.headoffset + PAGE_SIZE - 1,
1504  				  &pagepool, true);
1505  	err = z_erofs_do_read_page(&f, page, &pagepool);
1506  	z_erofs_pcluster_readmore(&f, NULL, 0, &pagepool, false);
1507  
1508  	(void)z_erofs_collector_end(&f);
1509  
1510  	/* if some compressed cluster ready, need submit them anyway */
1511  	z_erofs_runqueue(inode->i_sb, &f, &pagepool,
1512  			 z_erofs_get_sync_decompress_policy(sbi, 0));
1513  
1514  	if (err)
1515  		erofs_err(inode->i_sb, "failed to read, err [%d]", err);
1516  
1517  	erofs_put_metabuf(&f.map.buf);
1518  	erofs_release_pages(&pagepool);
1519  	return err;
1520  }
1521  
1522  static void z_erofs_readahead(struct readahead_control *rac)
1523  {
1524  	struct inode *const inode = rac->mapping->host;
1525  	struct erofs_sb_info *const sbi = EROFS_I_SB(inode);
1526  	struct z_erofs_decompress_frontend f = DECOMPRESS_FRONTEND_INIT(inode);
1527  	struct page *pagepool = NULL, *head = NULL, *page;
1528  	unsigned int nr_pages;
1529  
1530  	f.readahead = true;
1531  	f.headoffset = readahead_pos(rac);
1532  
1533  	z_erofs_pcluster_readmore(&f, rac, f.headoffset +
1534  				  readahead_length(rac) - 1, &pagepool, true);
1535  	nr_pages = readahead_count(rac);
1536  	trace_erofs_readpages(inode, readahead_index(rac), nr_pages, false);
1537  
1538  	while ((page = readahead_page(rac))) {
1539  		set_page_private(page, (unsigned long)head);
1540  		head = page;
1541  	}
1542  
1543  	while (head) {
1544  		struct page *page = head;
1545  		int err;
1546  
1547  		/* traversal in reverse order */
1548  		head = (void *)page_private(page);
1549  
1550  		err = z_erofs_do_read_page(&f, page, &pagepool);
1551  		if (err)
1552  			erofs_err(inode->i_sb,
1553  				  "readahead error at page %lu @ nid %llu",
1554  				  page->index, EROFS_I(inode)->nid);
1555  		put_page(page);
1556  	}
1557  	z_erofs_pcluster_readmore(&f, rac, 0, &pagepool, false);
1558  	(void)z_erofs_collector_end(&f);
1559  
1560  	z_erofs_runqueue(inode->i_sb, &f, &pagepool,
1561  			 z_erofs_get_sync_decompress_policy(sbi, nr_pages));
1562  	erofs_put_metabuf(&f.map.buf);
1563  	erofs_release_pages(&pagepool);
1564  }
1565  
1566  const struct address_space_operations z_erofs_aops = {
1567  	.read_folio = z_erofs_read_folio,
1568  	.readahead = z_erofs_readahead,
1569  };
1570