1 // SPDX-License-Identifier: GPL-2.0+ 2 /* 3 * VAS Fault handling. 4 * Copyright 2019, IBM Corporation 5 */ 6 7 #define pr_fmt(fmt) "vas: " fmt 8 9 #include <linux/kernel.h> 10 #include <linux/types.h> 11 #include <linux/slab.h> 12 #include <linux/uaccess.h> 13 #include <linux/kthread.h> 14 #include <linux/mmu_context.h> 15 #include <asm/icswx.h> 16 17 #include "vas.h" 18 19 /* 20 * The maximum FIFO size for fault window can be 8MB 21 * (VAS_RX_FIFO_SIZE_MAX). Using 4MB FIFO since each VAS 22 * instance will be having fault window. 23 * 8MB FIFO can be used if expects more faults for each VAS 24 * instance. 25 */ 26 #define VAS_FAULT_WIN_FIFO_SIZE (4 << 20) 27 28 /* 29 * Process valid CRBs in fault FIFO. 30 * NX process user space requests, return credit and update the status 31 * in CRB. If it encounters transalation error when accessing CRB or 32 * request buffers, raises interrupt on the CPU to handle the fault. 33 * It takes credit on fault window, updates nx_fault_stamp in CRB with 34 * the following information and pastes CRB in fault FIFO. 35 * 36 * pswid - window ID of the window on which the request is sent. 37 * fault_storage_addr - fault address 38 * 39 * It can raise a single interrupt for multiple faults. Expects OS to 40 * process all valid faults and return credit for each fault on user 41 * space and fault windows. This fault FIFO control will be done with 42 * credit mechanism. NX can continuously paste CRBs until credits are not 43 * available on fault window. Otherwise, returns with RMA_reject. 44 * 45 * Total credits available on fault window: FIFO_SIZE(4MB)/CRBS_SIZE(128) 46 * 47 */ 48 irqreturn_t vas_fault_thread_fn(int irq, void *data) 49 { 50 struct vas_instance *vinst = data; 51 struct coprocessor_request_block *crb, *entry; 52 struct coprocessor_request_block buf; 53 struct vas_window *window; 54 unsigned long flags; 55 void *fifo; 56 57 crb = &buf; 58 59 /* 60 * VAS can interrupt with multiple page faults. So process all 61 * valid CRBs within fault FIFO until reaches invalid CRB. 62 * We use CCW[0] and pswid to validate validate CRBs: 63 * 64 * CCW[0] Reserved bit. When NX pastes CRB, CCW[0]=0 65 * OS sets this bit to 1 after reading CRB. 66 * pswid NX assigns window ID. Set pswid to -1 after 67 * reading CRB from fault FIFO. 68 * 69 * We exit this function if no valid CRBs are available to process. 70 * So acquire fault_lock and reset fifo_in_progress to 0 before 71 * exit. 72 * In case kernel receives another interrupt with different page 73 * fault, interrupt handler returns with IRQ_HANDLED if 74 * fifo_in_progress is set. Means these new faults will be 75 * handled by the current thread. Otherwise set fifo_in_progress 76 * and return IRQ_WAKE_THREAD to wake up thread. 77 */ 78 while (true) { 79 spin_lock_irqsave(&vinst->fault_lock, flags); 80 /* 81 * Advance the fault fifo pointer to next CRB. 82 * Use CRB_SIZE rather than sizeof(*crb) since the latter is 83 * aligned to CRB_ALIGN (256) but the CRB written to by VAS is 84 * only CRB_SIZE in len. 85 */ 86 fifo = vinst->fault_fifo + (vinst->fault_crbs * CRB_SIZE); 87 entry = fifo; 88 89 if ((entry->stamp.nx.pswid == cpu_to_be32(FIFO_INVALID_ENTRY)) 90 || (entry->ccw & cpu_to_be32(CCW0_INVALID))) { 91 vinst->fifo_in_progress = 0; 92 spin_unlock_irqrestore(&vinst->fault_lock, flags); 93 return IRQ_HANDLED; 94 } 95 96 spin_unlock_irqrestore(&vinst->fault_lock, flags); 97 vinst->fault_crbs++; 98 if (vinst->fault_crbs == (vinst->fault_fifo_size / CRB_SIZE)) 99 vinst->fault_crbs = 0; 100 101 memcpy(crb, fifo, CRB_SIZE); 102 entry->stamp.nx.pswid = cpu_to_be32(FIFO_INVALID_ENTRY); 103 entry->ccw |= cpu_to_be32(CCW0_INVALID); 104 105 pr_devel("VAS[%d] fault_fifo %p, fifo %p, fault_crbs %d\n", 106 vinst->vas_id, vinst->fault_fifo, fifo, 107 vinst->fault_crbs); 108 109 window = vas_pswid_to_window(vinst, 110 be32_to_cpu(crb->stamp.nx.pswid)); 111 112 if (IS_ERR(window)) { 113 /* 114 * We got an interrupt about a specific send 115 * window but we can't find that window and we can't 116 * even clean it up (return credit on user space 117 * window). 118 * But we should not get here. 119 * TODO: Disable IRQ. 120 */ 121 pr_err("VAS[%d] fault_fifo %p, fifo %p, pswid 0x%x, fault_crbs %d bad CRB?\n", 122 vinst->vas_id, vinst->fault_fifo, fifo, 123 be32_to_cpu(crb->stamp.nx.pswid), 124 vinst->fault_crbs); 125 126 WARN_ON_ONCE(1); 127 } 128 129 } 130 } 131 132 irqreturn_t vas_fault_handler(int irq, void *dev_id) 133 { 134 struct vas_instance *vinst = dev_id; 135 irqreturn_t ret = IRQ_WAKE_THREAD; 136 unsigned long flags; 137 138 /* 139 * NX can generate an interrupt for multiple faults. So the 140 * fault handler thread process all CRBs until finds invalid 141 * entry. In case if NX sees continuous faults, it is possible 142 * that the thread function entered with the first interrupt 143 * can execute and process all valid CRBs. 144 * So wake up thread only if the fault thread is not in progress. 145 */ 146 spin_lock_irqsave(&vinst->fault_lock, flags); 147 148 if (vinst->fifo_in_progress) 149 ret = IRQ_HANDLED; 150 else 151 vinst->fifo_in_progress = 1; 152 153 spin_unlock_irqrestore(&vinst->fault_lock, flags); 154 155 return ret; 156 } 157 158 /* 159 * Fault window is opened per VAS instance. NX pastes fault CRB in fault 160 * FIFO upon page faults. 161 */ 162 int vas_setup_fault_window(struct vas_instance *vinst) 163 { 164 struct vas_rx_win_attr attr; 165 166 vinst->fault_fifo_size = VAS_FAULT_WIN_FIFO_SIZE; 167 vinst->fault_fifo = kzalloc(vinst->fault_fifo_size, GFP_KERNEL); 168 if (!vinst->fault_fifo) { 169 pr_err("Unable to alloc %d bytes for fault_fifo\n", 170 vinst->fault_fifo_size); 171 return -ENOMEM; 172 } 173 174 /* 175 * Invalidate all CRB entries. NX pastes valid entry for each fault. 176 */ 177 memset(vinst->fault_fifo, FIFO_INVALID_ENTRY, vinst->fault_fifo_size); 178 vas_init_rx_win_attr(&attr, VAS_COP_TYPE_FAULT); 179 180 attr.rx_fifo_size = vinst->fault_fifo_size; 181 attr.rx_fifo = vinst->fault_fifo; 182 183 /* 184 * Max creds is based on number of CRBs can fit in the FIFO. 185 * (fault_fifo_size/CRB_SIZE). If 8MB FIFO is used, max creds 186 * will be 0xffff since the receive creds field is 16bits wide. 187 */ 188 attr.wcreds_max = vinst->fault_fifo_size / CRB_SIZE; 189 attr.lnotify_lpid = 0; 190 attr.lnotify_pid = mfspr(SPRN_PID); 191 attr.lnotify_tid = mfspr(SPRN_PID); 192 193 vinst->fault_win = vas_rx_win_open(vinst->vas_id, VAS_COP_TYPE_FAULT, 194 &attr); 195 196 if (IS_ERR(vinst->fault_win)) { 197 pr_err("VAS: Error %ld opening FaultWin\n", 198 PTR_ERR(vinst->fault_win)); 199 kfree(vinst->fault_fifo); 200 return PTR_ERR(vinst->fault_win); 201 } 202 203 pr_devel("VAS: Created FaultWin %d, LPID/PID/TID [%d/%d/%d]\n", 204 vinst->fault_win->winid, attr.lnotify_lpid, 205 attr.lnotify_pid, attr.lnotify_tid); 206 207 return 0; 208 } 209