1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) Gao Xiang <xiang@kernel.org> 4 * 5 * For low-latency decompression algorithms (e.g. lz4), reserve consecutive 6 * per-CPU virtual memory (in pages) in advance to store such inplace I/O 7 * data if inplace decompression is failed (due to unmet inplace margin for 8 * example). 9 */ 10 #include "internal.h" 11 12 struct erofs_pcpubuf { 13 raw_spinlock_t lock; 14 void *ptr; 15 struct page **pages; 16 unsigned int nrpages; 17 }; 18 19 static DEFINE_PER_CPU(struct erofs_pcpubuf, erofs_pcb); 20 21 void *erofs_get_pcpubuf(unsigned int requiredpages) 22 __acquires(pcb->lock) 23 { 24 struct erofs_pcpubuf *pcb = &get_cpu_var(erofs_pcb); 25 26 raw_spin_lock(&pcb->lock); 27 /* check if the per-CPU buffer is too small */ 28 if (requiredpages > pcb->nrpages) { 29 raw_spin_unlock(&pcb->lock); 30 put_cpu_var(erofs_pcb); 31 /* (for sparse checker) pretend pcb->lock is still taken */ 32 __acquire(pcb->lock); 33 return NULL; 34 } 35 return pcb->ptr; 36 } 37 38 void erofs_put_pcpubuf(void *ptr) __releases(pcb->lock) 39 { 40 struct erofs_pcpubuf *pcb = &per_cpu(erofs_pcb, smp_processor_id()); 41 42 DBG_BUGON(pcb->ptr != ptr); 43 raw_spin_unlock(&pcb->lock); 44 put_cpu_var(erofs_pcb); 45 } 46 47 /* the next step: support per-CPU page buffers hotplug */ 48 int erofs_pcpubuf_growsize(unsigned int nrpages) 49 { 50 static DEFINE_MUTEX(pcb_resize_mutex); 51 static unsigned int pcb_nrpages; 52 LIST_HEAD(pagepool); 53 int delta, cpu, ret, i; 54 55 mutex_lock(&pcb_resize_mutex); 56 delta = nrpages - pcb_nrpages; 57 ret = 0; 58 /* avoid shrinking pcpubuf, since no idea how many fses rely on */ 59 if (delta <= 0) 60 goto out; 61 62 for_each_possible_cpu(cpu) { 63 struct erofs_pcpubuf *pcb = &per_cpu(erofs_pcb, cpu); 64 struct page **pages, **oldpages; 65 void *ptr, *old_ptr; 66 67 pages = kmalloc_array(nrpages, sizeof(*pages), GFP_KERNEL); 68 if (!pages) { 69 ret = -ENOMEM; 70 break; 71 } 72 73 for (i = 0; i < nrpages; ++i) { 74 pages[i] = erofs_allocpage(&pagepool, GFP_KERNEL); 75 if (!pages[i]) { 76 ret = -ENOMEM; 77 oldpages = pages; 78 goto free_pagearray; 79 } 80 } 81 ptr = vmap(pages, nrpages, VM_MAP, PAGE_KERNEL); 82 if (!ptr) { 83 ret = -ENOMEM; 84 oldpages = pages; 85 goto free_pagearray; 86 } 87 raw_spin_lock(&pcb->lock); 88 old_ptr = pcb->ptr; 89 pcb->ptr = ptr; 90 oldpages = pcb->pages; 91 pcb->pages = pages; 92 i = pcb->nrpages; 93 pcb->nrpages = nrpages; 94 raw_spin_unlock(&pcb->lock); 95 96 if (!oldpages) { 97 DBG_BUGON(old_ptr); 98 continue; 99 } 100 101 if (old_ptr) 102 vunmap(old_ptr); 103 free_pagearray: 104 while (i) 105 list_add(&oldpages[--i]->lru, &pagepool); 106 kfree(oldpages); 107 if (ret) 108 break; 109 } 110 pcb_nrpages = nrpages; 111 put_pages_list(&pagepool); 112 out: 113 mutex_unlock(&pcb_resize_mutex); 114 return ret; 115 } 116 117 void erofs_pcpubuf_init(void) 118 { 119 int cpu; 120 121 for_each_possible_cpu(cpu) { 122 struct erofs_pcpubuf *pcb = &per_cpu(erofs_pcb, cpu); 123 124 raw_spin_lock_init(&pcb->lock); 125 } 126 } 127 128 void erofs_pcpubuf_exit(void) 129 { 130 int cpu, i; 131 132 for_each_possible_cpu(cpu) { 133 struct erofs_pcpubuf *pcb = &per_cpu(erofs_pcb, cpu); 134 135 if (pcb->ptr) { 136 vunmap(pcb->ptr); 137 pcb->ptr = NULL; 138 } 139 if (!pcb->pages) 140 continue; 141 142 for (i = 0; i < pcb->nrpages; ++i) 143 if (pcb->pages[i]) 144 put_page(pcb->pages[i]); 145 kfree(pcb->pages); 146 pcb->pages = NULL; 147 } 148 } 149