1*52488734SGao Xiang // SPDX-License-Identifier: GPL-2.0-only 2*52488734SGao Xiang /* 3*52488734SGao Xiang * Copyright (C) Gao Xiang <xiang@kernel.org> 4*52488734SGao Xiang * 5*52488734SGao Xiang * For low-latency decompression algorithms (e.g. lz4), reserve consecutive 6*52488734SGao Xiang * per-CPU virtual memory (in pages) in advance to store such inplace I/O 7*52488734SGao Xiang * data if inplace decompression is failed (due to unmet inplace margin for 8*52488734SGao Xiang * example). 9*52488734SGao Xiang */ 10*52488734SGao Xiang #include "internal.h" 11*52488734SGao Xiang 12*52488734SGao Xiang struct erofs_pcpubuf { 13*52488734SGao Xiang raw_spinlock_t lock; 14*52488734SGao Xiang void *ptr; 15*52488734SGao Xiang struct page **pages; 16*52488734SGao Xiang unsigned int nrpages; 17*52488734SGao Xiang }; 18*52488734SGao Xiang 19*52488734SGao Xiang static DEFINE_PER_CPU(struct erofs_pcpubuf, erofs_pcb); 20*52488734SGao Xiang 21*52488734SGao Xiang void *erofs_get_pcpubuf(unsigned int requiredpages) 22*52488734SGao Xiang __acquires(pcb->lock) 23*52488734SGao Xiang { 24*52488734SGao Xiang struct erofs_pcpubuf *pcb = &get_cpu_var(erofs_pcb); 25*52488734SGao Xiang 26*52488734SGao Xiang raw_spin_lock(&pcb->lock); 27*52488734SGao Xiang /* check if the per-CPU buffer is too small */ 28*52488734SGao Xiang if (requiredpages > pcb->nrpages) { 29*52488734SGao Xiang raw_spin_unlock(&pcb->lock); 30*52488734SGao Xiang put_cpu_var(erofs_pcb); 31*52488734SGao Xiang /* (for sparse checker) pretend pcb->lock is still taken */ 32*52488734SGao Xiang __acquire(pcb->lock); 33*52488734SGao Xiang return NULL; 34*52488734SGao Xiang } 35*52488734SGao Xiang return pcb->ptr; 36*52488734SGao Xiang } 37*52488734SGao Xiang 38*52488734SGao Xiang void erofs_put_pcpubuf(void *ptr) __releases(pcb->lock) 39*52488734SGao Xiang { 40*52488734SGao Xiang struct erofs_pcpubuf *pcb = &per_cpu(erofs_pcb, smp_processor_id()); 41*52488734SGao Xiang 42*52488734SGao Xiang DBG_BUGON(pcb->ptr != ptr); 43*52488734SGao Xiang raw_spin_unlock(&pcb->lock); 44*52488734SGao Xiang put_cpu_var(erofs_pcb); 45*52488734SGao Xiang } 46*52488734SGao Xiang 47*52488734SGao Xiang /* the next step: support per-CPU page buffers hotplug */ 48*52488734SGao Xiang int erofs_pcpubuf_growsize(unsigned int nrpages) 49*52488734SGao Xiang { 50*52488734SGao Xiang static DEFINE_MUTEX(pcb_resize_mutex); 51*52488734SGao Xiang static unsigned int pcb_nrpages; 52*52488734SGao Xiang LIST_HEAD(pagepool); 53*52488734SGao Xiang int delta, cpu, ret, i; 54*52488734SGao Xiang 55*52488734SGao Xiang mutex_lock(&pcb_resize_mutex); 56*52488734SGao Xiang delta = nrpages - pcb_nrpages; 57*52488734SGao Xiang ret = 0; 58*52488734SGao Xiang /* avoid shrinking pcpubuf, since no idea how many fses rely on */ 59*52488734SGao Xiang if (delta <= 0) 60*52488734SGao Xiang goto out; 61*52488734SGao Xiang 62*52488734SGao Xiang for_each_possible_cpu(cpu) { 63*52488734SGao Xiang struct erofs_pcpubuf *pcb = &per_cpu(erofs_pcb, cpu); 64*52488734SGao Xiang struct page **pages, **oldpages; 65*52488734SGao Xiang void *ptr, *old_ptr; 66*52488734SGao Xiang 67*52488734SGao Xiang pages = kmalloc_array(nrpages, sizeof(*pages), GFP_KERNEL); 68*52488734SGao Xiang if (!pages) { 69*52488734SGao Xiang ret = -ENOMEM; 70*52488734SGao Xiang break; 71*52488734SGao Xiang } 72*52488734SGao Xiang 73*52488734SGao Xiang for (i = 0; i < nrpages; ++i) { 74*52488734SGao Xiang pages[i] = erofs_allocpage(&pagepool, GFP_KERNEL); 75*52488734SGao Xiang if (!pages[i]) { 76*52488734SGao Xiang ret = -ENOMEM; 77*52488734SGao Xiang oldpages = pages; 78*52488734SGao Xiang goto free_pagearray; 79*52488734SGao Xiang } 80*52488734SGao Xiang } 81*52488734SGao Xiang ptr = vmap(pages, nrpages, VM_MAP, PAGE_KERNEL); 82*52488734SGao Xiang if (!ptr) { 83*52488734SGao Xiang ret = -ENOMEM; 84*52488734SGao Xiang oldpages = pages; 85*52488734SGao Xiang goto free_pagearray; 86*52488734SGao Xiang } 87*52488734SGao Xiang raw_spin_lock(&pcb->lock); 88*52488734SGao Xiang old_ptr = pcb->ptr; 89*52488734SGao Xiang pcb->ptr = ptr; 90*52488734SGao Xiang oldpages = pcb->pages; 91*52488734SGao Xiang pcb->pages = pages; 92*52488734SGao Xiang i = pcb->nrpages; 93*52488734SGao Xiang pcb->nrpages = nrpages; 94*52488734SGao Xiang raw_spin_unlock(&pcb->lock); 95*52488734SGao Xiang 96*52488734SGao Xiang if (!oldpages) { 97*52488734SGao Xiang DBG_BUGON(old_ptr); 98*52488734SGao Xiang continue; 99*52488734SGao Xiang } 100*52488734SGao Xiang 101*52488734SGao Xiang if (old_ptr) 102*52488734SGao Xiang vunmap(old_ptr); 103*52488734SGao Xiang free_pagearray: 104*52488734SGao Xiang while (i) 105*52488734SGao Xiang list_add(&oldpages[--i]->lru, &pagepool); 106*52488734SGao Xiang kfree(oldpages); 107*52488734SGao Xiang if (ret) 108*52488734SGao Xiang break; 109*52488734SGao Xiang } 110*52488734SGao Xiang pcb_nrpages = nrpages; 111*52488734SGao Xiang put_pages_list(&pagepool); 112*52488734SGao Xiang out: 113*52488734SGao Xiang mutex_unlock(&pcb_resize_mutex); 114*52488734SGao Xiang return ret; 115*52488734SGao Xiang } 116*52488734SGao Xiang 117*52488734SGao Xiang void erofs_pcpubuf_init(void) 118*52488734SGao Xiang { 119*52488734SGao Xiang int cpu; 120*52488734SGao Xiang 121*52488734SGao Xiang for_each_possible_cpu(cpu) { 122*52488734SGao Xiang struct erofs_pcpubuf *pcb = &per_cpu(erofs_pcb, cpu); 123*52488734SGao Xiang 124*52488734SGao Xiang raw_spin_lock_init(&pcb->lock); 125*52488734SGao Xiang } 126*52488734SGao Xiang } 127*52488734SGao Xiang 128*52488734SGao Xiang void erofs_pcpubuf_exit(void) 129*52488734SGao Xiang { 130*52488734SGao Xiang int cpu, i; 131*52488734SGao Xiang 132*52488734SGao Xiang for_each_possible_cpu(cpu) { 133*52488734SGao Xiang struct erofs_pcpubuf *pcb = &per_cpu(erofs_pcb, cpu); 134*52488734SGao Xiang 135*52488734SGao Xiang if (pcb->ptr) { 136*52488734SGao Xiang vunmap(pcb->ptr); 137*52488734SGao Xiang pcb->ptr = NULL; 138*52488734SGao Xiang } 139*52488734SGao Xiang if (!pcb->pages) 140*52488734SGao Xiang continue; 141*52488734SGao Xiang 142*52488734SGao Xiang for (i = 0; i < pcb->nrpages; ++i) 143*52488734SGao Xiang if (pcb->pages[i]) 144*52488734SGao Xiang put_page(pcb->pages[i]); 145*52488734SGao Xiang kfree(pcb->pages); 146*52488734SGao Xiang pcb->pages = NULL; 147*52488734SGao Xiang } 148*52488734SGao Xiang } 149