10d17de03SHaren Myneni // SPDX-License-Identifier: GPL-2.0+ 20d17de03SHaren Myneni /* 30d17de03SHaren Myneni * VAS Fault handling. 40d17de03SHaren Myneni * Copyright 2019, IBM Corporation 50d17de03SHaren Myneni */ 60d17de03SHaren Myneni 70d17de03SHaren Myneni #define pr_fmt(fmt) "vas: " fmt 80d17de03SHaren Myneni 90d17de03SHaren Myneni #include <linux/kernel.h> 100d17de03SHaren Myneni #include <linux/types.h> 110d17de03SHaren Myneni #include <linux/slab.h> 120d17de03SHaren Myneni #include <linux/uaccess.h> 130d17de03SHaren Myneni #include <linux/kthread.h> 149774628aSHaren Myneni #include <linux/mmu_context.h> 150d17de03SHaren Myneni #include <asm/icswx.h> 160d17de03SHaren Myneni 170d17de03SHaren Myneni #include "vas.h" 180d17de03SHaren Myneni 190d17de03SHaren Myneni /* 200d17de03SHaren Myneni * The maximum FIFO size for fault window can be 8MB 210d17de03SHaren Myneni * (VAS_RX_FIFO_SIZE_MAX). Using 4MB FIFO since each VAS 220d17de03SHaren Myneni * instance will be having fault window. 230d17de03SHaren Myneni * 8MB FIFO can be used if expects more faults for each VAS 240d17de03SHaren Myneni * instance. 250d17de03SHaren Myneni */ 260d17de03SHaren Myneni #define VAS_FAULT_WIN_FIFO_SIZE (4 << 20) 270d17de03SHaren Myneni 280d17de03SHaren Myneni /* 299774628aSHaren Myneni * Process valid CRBs in fault FIFO. 309774628aSHaren Myneni * NX process user space requests, return credit and update the status 319774628aSHaren Myneni * in CRB. If it encounters transalation error when accessing CRB or 329774628aSHaren Myneni * request buffers, raises interrupt on the CPU to handle the fault. 339774628aSHaren Myneni * It takes credit on fault window, updates nx_fault_stamp in CRB with 349774628aSHaren Myneni * the following information and pastes CRB in fault FIFO. 359774628aSHaren Myneni * 369774628aSHaren Myneni * pswid - window ID of the window on which the request is sent. 379774628aSHaren Myneni * fault_storage_addr - fault address 389774628aSHaren Myneni * 399774628aSHaren Myneni * It can raise a single interrupt for multiple faults. Expects OS to 409774628aSHaren Myneni * process all valid faults and return credit for each fault on user 419774628aSHaren Myneni * space and fault windows. This fault FIFO control will be done with 429774628aSHaren Myneni * credit mechanism. NX can continuously paste CRBs until credits are not 439774628aSHaren Myneni * available on fault window. Otherwise, returns with RMA_reject. 449774628aSHaren Myneni * 459774628aSHaren Myneni * Total credits available on fault window: FIFO_SIZE(4MB)/CRBS_SIZE(128) 469774628aSHaren Myneni * 479774628aSHaren Myneni */ 489774628aSHaren Myneni irqreturn_t vas_fault_thread_fn(int irq, void *data) 499774628aSHaren Myneni { 509774628aSHaren Myneni struct vas_instance *vinst = data; 519774628aSHaren Myneni struct coprocessor_request_block *crb, *entry; 529774628aSHaren Myneni struct coprocessor_request_block buf; 539774628aSHaren Myneni struct vas_window *window; 549774628aSHaren Myneni unsigned long flags; 559774628aSHaren Myneni void *fifo; 569774628aSHaren Myneni 579774628aSHaren Myneni crb = &buf; 589774628aSHaren Myneni 599774628aSHaren Myneni /* 609774628aSHaren Myneni * VAS can interrupt with multiple page faults. So process all 619774628aSHaren Myneni * valid CRBs within fault FIFO until reaches invalid CRB. 629774628aSHaren Myneni * We use CCW[0] and pswid to validate validate CRBs: 639774628aSHaren Myneni * 649774628aSHaren Myneni * CCW[0] Reserved bit. When NX pastes CRB, CCW[0]=0 659774628aSHaren Myneni * OS sets this bit to 1 after reading CRB. 669774628aSHaren Myneni * pswid NX assigns window ID. Set pswid to -1 after 679774628aSHaren Myneni * reading CRB from fault FIFO. 689774628aSHaren Myneni * 699774628aSHaren Myneni * We exit this function if no valid CRBs are available to process. 709774628aSHaren Myneni * So acquire fault_lock and reset fifo_in_progress to 0 before 719774628aSHaren Myneni * exit. 729774628aSHaren Myneni * In case kernel receives another interrupt with different page 739774628aSHaren Myneni * fault, interrupt handler returns with IRQ_HANDLED if 749774628aSHaren Myneni * fifo_in_progress is set. Means these new faults will be 759774628aSHaren Myneni * handled by the current thread. Otherwise set fifo_in_progress 769774628aSHaren Myneni * and return IRQ_WAKE_THREAD to wake up thread. 779774628aSHaren Myneni */ 789774628aSHaren Myneni while (true) { 799774628aSHaren Myneni spin_lock_irqsave(&vinst->fault_lock, flags); 809774628aSHaren Myneni /* 819774628aSHaren Myneni * Advance the fault fifo pointer to next CRB. 829774628aSHaren Myneni * Use CRB_SIZE rather than sizeof(*crb) since the latter is 839774628aSHaren Myneni * aligned to CRB_ALIGN (256) but the CRB written to by VAS is 849774628aSHaren Myneni * only CRB_SIZE in len. 859774628aSHaren Myneni */ 869774628aSHaren Myneni fifo = vinst->fault_fifo + (vinst->fault_crbs * CRB_SIZE); 879774628aSHaren Myneni entry = fifo; 889774628aSHaren Myneni 899774628aSHaren Myneni if ((entry->stamp.nx.pswid == cpu_to_be32(FIFO_INVALID_ENTRY)) 909774628aSHaren Myneni || (entry->ccw & cpu_to_be32(CCW0_INVALID))) { 919774628aSHaren Myneni vinst->fifo_in_progress = 0; 929774628aSHaren Myneni spin_unlock_irqrestore(&vinst->fault_lock, flags); 939774628aSHaren Myneni return IRQ_HANDLED; 949774628aSHaren Myneni } 959774628aSHaren Myneni 969774628aSHaren Myneni spin_unlock_irqrestore(&vinst->fault_lock, flags); 979774628aSHaren Myneni vinst->fault_crbs++; 989774628aSHaren Myneni if (vinst->fault_crbs == (vinst->fault_fifo_size / CRB_SIZE)) 999774628aSHaren Myneni vinst->fault_crbs = 0; 1009774628aSHaren Myneni 1019774628aSHaren Myneni memcpy(crb, fifo, CRB_SIZE); 1029774628aSHaren Myneni entry->stamp.nx.pswid = cpu_to_be32(FIFO_INVALID_ENTRY); 1039774628aSHaren Myneni entry->ccw |= cpu_to_be32(CCW0_INVALID); 1049774628aSHaren Myneni 1059774628aSHaren Myneni pr_devel("VAS[%d] fault_fifo %p, fifo %p, fault_crbs %d\n", 1069774628aSHaren Myneni vinst->vas_id, vinst->fault_fifo, fifo, 1079774628aSHaren Myneni vinst->fault_crbs); 1089774628aSHaren Myneni 1099774628aSHaren Myneni window = vas_pswid_to_window(vinst, 1109774628aSHaren Myneni be32_to_cpu(crb->stamp.nx.pswid)); 1119774628aSHaren Myneni 1129774628aSHaren Myneni if (IS_ERR(window)) { 1139774628aSHaren Myneni /* 1149774628aSHaren Myneni * We got an interrupt about a specific send 1159774628aSHaren Myneni * window but we can't find that window and we can't 1169774628aSHaren Myneni * even clean it up (return credit on user space 1179774628aSHaren Myneni * window). 1189774628aSHaren Myneni * But we should not get here. 1199774628aSHaren Myneni * TODO: Disable IRQ. 1209774628aSHaren Myneni */ 1219774628aSHaren Myneni pr_err("VAS[%d] fault_fifo %p, fifo %p, pswid 0x%x, fault_crbs %d bad CRB?\n", 1229774628aSHaren Myneni vinst->vas_id, vinst->fault_fifo, fifo, 1239774628aSHaren Myneni be32_to_cpu(crb->stamp.nx.pswid), 1249774628aSHaren Myneni vinst->fault_crbs); 1259774628aSHaren Myneni 1269774628aSHaren Myneni WARN_ON_ONCE(1); 1279774628aSHaren Myneni } 1289774628aSHaren Myneni 1299774628aSHaren Myneni } 1309774628aSHaren Myneni } 1319774628aSHaren Myneni 1329774628aSHaren Myneni irqreturn_t vas_fault_handler(int irq, void *dev_id) 1339774628aSHaren Myneni { 1349774628aSHaren Myneni struct vas_instance *vinst = dev_id; 1359774628aSHaren Myneni irqreturn_t ret = IRQ_WAKE_THREAD; 1369774628aSHaren Myneni unsigned long flags; 1379774628aSHaren Myneni 1389774628aSHaren Myneni /* 1399774628aSHaren Myneni * NX can generate an interrupt for multiple faults. So the 1409774628aSHaren Myneni * fault handler thread process all CRBs until finds invalid 1419774628aSHaren Myneni * entry. In case if NX sees continuous faults, it is possible 1429774628aSHaren Myneni * that the thread function entered with the first interrupt 1439774628aSHaren Myneni * can execute and process all valid CRBs. 1449774628aSHaren Myneni * So wake up thread only if the fault thread is not in progress. 1459774628aSHaren Myneni */ 1469774628aSHaren Myneni spin_lock_irqsave(&vinst->fault_lock, flags); 1479774628aSHaren Myneni 1489774628aSHaren Myneni if (vinst->fifo_in_progress) 1499774628aSHaren Myneni ret = IRQ_HANDLED; 1509774628aSHaren Myneni else 1519774628aSHaren Myneni vinst->fifo_in_progress = 1; 1529774628aSHaren Myneni 1539774628aSHaren Myneni spin_unlock_irqrestore(&vinst->fault_lock, flags); 1549774628aSHaren Myneni 1559774628aSHaren Myneni return ret; 1569774628aSHaren Myneni } 1579774628aSHaren Myneni 1589774628aSHaren Myneni /* 1590d17de03SHaren Myneni * Fault window is opened per VAS instance. NX pastes fault CRB in fault 1600d17de03SHaren Myneni * FIFO upon page faults. 1610d17de03SHaren Myneni */ 1620d17de03SHaren Myneni int vas_setup_fault_window(struct vas_instance *vinst) 1630d17de03SHaren Myneni { 1640d17de03SHaren Myneni struct vas_rx_win_attr attr; 1650d17de03SHaren Myneni 1660d17de03SHaren Myneni vinst->fault_fifo_size = VAS_FAULT_WIN_FIFO_SIZE; 1670d17de03SHaren Myneni vinst->fault_fifo = kzalloc(vinst->fault_fifo_size, GFP_KERNEL); 1680d17de03SHaren Myneni if (!vinst->fault_fifo) { 1690d17de03SHaren Myneni pr_err("Unable to alloc %d bytes for fault_fifo\n", 1700d17de03SHaren Myneni vinst->fault_fifo_size); 1710d17de03SHaren Myneni return -ENOMEM; 1720d17de03SHaren Myneni } 1730d17de03SHaren Myneni 1740d17de03SHaren Myneni /* 1750d17de03SHaren Myneni * Invalidate all CRB entries. NX pastes valid entry for each fault. 1760d17de03SHaren Myneni */ 1770d17de03SHaren Myneni memset(vinst->fault_fifo, FIFO_INVALID_ENTRY, vinst->fault_fifo_size); 1780d17de03SHaren Myneni vas_init_rx_win_attr(&attr, VAS_COP_TYPE_FAULT); 1790d17de03SHaren Myneni 1800d17de03SHaren Myneni attr.rx_fifo_size = vinst->fault_fifo_size; 1810d17de03SHaren Myneni attr.rx_fifo = vinst->fault_fifo; 1820d17de03SHaren Myneni 1830d17de03SHaren Myneni /* 1840d17de03SHaren Myneni * Max creds is based on number of CRBs can fit in the FIFO. 1850d17de03SHaren Myneni * (fault_fifo_size/CRB_SIZE). If 8MB FIFO is used, max creds 1860d17de03SHaren Myneni * will be 0xffff since the receive creds field is 16bits wide. 1870d17de03SHaren Myneni */ 1880d17de03SHaren Myneni attr.wcreds_max = vinst->fault_fifo_size / CRB_SIZE; 1890d17de03SHaren Myneni attr.lnotify_lpid = 0; 1900d17de03SHaren Myneni attr.lnotify_pid = mfspr(SPRN_PID); 1910d17de03SHaren Myneni attr.lnotify_tid = mfspr(SPRN_PID); 1920d17de03SHaren Myneni 1930d17de03SHaren Myneni vinst->fault_win = vas_rx_win_open(vinst->vas_id, VAS_COP_TYPE_FAULT, 1940d17de03SHaren Myneni &attr); 1950d17de03SHaren Myneni 1960d17de03SHaren Myneni if (IS_ERR(vinst->fault_win)) { 1970d17de03SHaren Myneni pr_err("VAS: Error %ld opening FaultWin\n", 1980d17de03SHaren Myneni PTR_ERR(vinst->fault_win)); 1990d17de03SHaren Myneni kfree(vinst->fault_fifo); 2000d17de03SHaren Myneni return PTR_ERR(vinst->fault_win); 2010d17de03SHaren Myneni } 2020d17de03SHaren Myneni 2030d17de03SHaren Myneni pr_devel("VAS: Created FaultWin %d, LPID/PID/TID [%d/%d/%d]\n", 2040d17de03SHaren Myneni vinst->fault_win->winid, attr.lnotify_lpid, 2050d17de03SHaren Myneni attr.lnotify_pid, attr.lnotify_tid); 2060d17de03SHaren Myneni 2070d17de03SHaren Myneni return 0; 2080d17de03SHaren Myneni } 209