10d17de03SHaren Myneni // SPDX-License-Identifier: GPL-2.0+
20d17de03SHaren Myneni /*
30d17de03SHaren Myneni * VAS Fault handling.
40d17de03SHaren Myneni * Copyright 2019, IBM Corporation
50d17de03SHaren Myneni */
60d17de03SHaren Myneni
70d17de03SHaren Myneni #define pr_fmt(fmt) "vas: " fmt
80d17de03SHaren Myneni
90d17de03SHaren Myneni #include <linux/kernel.h>
100d17de03SHaren Myneni #include <linux/types.h>
110d17de03SHaren Myneni #include <linux/slab.h>
120d17de03SHaren Myneni #include <linux/uaccess.h>
130d17de03SHaren Myneni #include <linux/kthread.h>
14c96c4436SHaren Myneni #include <linux/sched/signal.h>
159774628aSHaren Myneni #include <linux/mmu_context.h>
160d17de03SHaren Myneni #include <asm/icswx.h>
170d17de03SHaren Myneni
180d17de03SHaren Myneni #include "vas.h"
190d17de03SHaren Myneni
200d17de03SHaren Myneni /*
210d17de03SHaren Myneni * The maximum FIFO size for fault window can be 8MB
220d17de03SHaren Myneni * (VAS_RX_FIFO_SIZE_MAX). Using 4MB FIFO since each VAS
230d17de03SHaren Myneni * instance will be having fault window.
240d17de03SHaren Myneni * 8MB FIFO can be used if expects more faults for each VAS
250d17de03SHaren Myneni * instance.
260d17de03SHaren Myneni */
270d17de03SHaren Myneni #define VAS_FAULT_WIN_FIFO_SIZE (4 << 20)
280d17de03SHaren Myneni
dump_fifo(struct vas_instance * vinst,void * entry)29cf33e1e9SHaren Myneni static void dump_fifo(struct vas_instance *vinst, void *entry)
30cf33e1e9SHaren Myneni {
31cf33e1e9SHaren Myneni unsigned long *end = vinst->fault_fifo + vinst->fault_fifo_size;
32cf33e1e9SHaren Myneni unsigned long *fifo = entry;
33cf33e1e9SHaren Myneni int i;
34cf33e1e9SHaren Myneni
35cf33e1e9SHaren Myneni pr_err("Fault fifo size %d, Max crbs %d\n", vinst->fault_fifo_size,
36cf33e1e9SHaren Myneni vinst->fault_fifo_size / CRB_SIZE);
37cf33e1e9SHaren Myneni
38cf33e1e9SHaren Myneni /* Dump 10 CRB entries or until end of FIFO */
39cf33e1e9SHaren Myneni pr_err("Fault FIFO Dump:\n");
40cf33e1e9SHaren Myneni for (i = 0; i < 10*(CRB_SIZE/8) && fifo < end; i += 4, fifo += 4) {
41cf33e1e9SHaren Myneni pr_err("[%.3d, %p]: 0x%.16lx 0x%.16lx 0x%.16lx 0x%.16lx\n",
42cf33e1e9SHaren Myneni i, fifo, *fifo, *(fifo+1), *(fifo+2), *(fifo+3));
43cf33e1e9SHaren Myneni }
44cf33e1e9SHaren Myneni }
45cf33e1e9SHaren Myneni
46c96c4436SHaren Myneni /*
479774628aSHaren Myneni * Process valid CRBs in fault FIFO.
489774628aSHaren Myneni * NX process user space requests, return credit and update the status
499774628aSHaren Myneni * in CRB. If it encounters transalation error when accessing CRB or
509774628aSHaren Myneni * request buffers, raises interrupt on the CPU to handle the fault.
519774628aSHaren Myneni * It takes credit on fault window, updates nx_fault_stamp in CRB with
529774628aSHaren Myneni * the following information and pastes CRB in fault FIFO.
539774628aSHaren Myneni *
549774628aSHaren Myneni * pswid - window ID of the window on which the request is sent.
559774628aSHaren Myneni * fault_storage_addr - fault address
569774628aSHaren Myneni *
579774628aSHaren Myneni * It can raise a single interrupt for multiple faults. Expects OS to
589774628aSHaren Myneni * process all valid faults and return credit for each fault on user
599774628aSHaren Myneni * space and fault windows. This fault FIFO control will be done with
609774628aSHaren Myneni * credit mechanism. NX can continuously paste CRBs until credits are not
619774628aSHaren Myneni * available on fault window. Otherwise, returns with RMA_reject.
629774628aSHaren Myneni *
639774628aSHaren Myneni * Total credits available on fault window: FIFO_SIZE(4MB)/CRBS_SIZE(128)
649774628aSHaren Myneni *
659774628aSHaren Myneni */
vas_fault_thread_fn(int irq,void * data)669774628aSHaren Myneni irqreturn_t vas_fault_thread_fn(int irq, void *data)
679774628aSHaren Myneni {
689774628aSHaren Myneni struct vas_instance *vinst = data;
699774628aSHaren Myneni struct coprocessor_request_block *crb, *entry;
709774628aSHaren Myneni struct coprocessor_request_block buf;
717bc6f71bSHaren Myneni struct pnv_vas_window *window;
729774628aSHaren Myneni unsigned long flags;
739774628aSHaren Myneni void *fifo;
749774628aSHaren Myneni
759774628aSHaren Myneni crb = &buf;
769774628aSHaren Myneni
779774628aSHaren Myneni /*
789774628aSHaren Myneni * VAS can interrupt with multiple page faults. So process all
799774628aSHaren Myneni * valid CRBs within fault FIFO until reaches invalid CRB.
80*2b461880SMichael Ellerman * We use CCW[0] and pswid to validate CRBs:
819774628aSHaren Myneni *
829774628aSHaren Myneni * CCW[0] Reserved bit. When NX pastes CRB, CCW[0]=0
839774628aSHaren Myneni * OS sets this bit to 1 after reading CRB.
849774628aSHaren Myneni * pswid NX assigns window ID. Set pswid to -1 after
859774628aSHaren Myneni * reading CRB from fault FIFO.
869774628aSHaren Myneni *
879774628aSHaren Myneni * We exit this function if no valid CRBs are available to process.
889774628aSHaren Myneni * So acquire fault_lock and reset fifo_in_progress to 0 before
899774628aSHaren Myneni * exit.
909774628aSHaren Myneni * In case kernel receives another interrupt with different page
919774628aSHaren Myneni * fault, interrupt handler returns with IRQ_HANDLED if
929774628aSHaren Myneni * fifo_in_progress is set. Means these new faults will be
939774628aSHaren Myneni * handled by the current thread. Otherwise set fifo_in_progress
949774628aSHaren Myneni * and return IRQ_WAKE_THREAD to wake up thread.
959774628aSHaren Myneni */
969774628aSHaren Myneni while (true) {
979774628aSHaren Myneni spin_lock_irqsave(&vinst->fault_lock, flags);
989774628aSHaren Myneni /*
999774628aSHaren Myneni * Advance the fault fifo pointer to next CRB.
1009774628aSHaren Myneni * Use CRB_SIZE rather than sizeof(*crb) since the latter is
1019774628aSHaren Myneni * aligned to CRB_ALIGN (256) but the CRB written to by VAS is
1029774628aSHaren Myneni * only CRB_SIZE in len.
1039774628aSHaren Myneni */
1049774628aSHaren Myneni fifo = vinst->fault_fifo + (vinst->fault_crbs * CRB_SIZE);
1059774628aSHaren Myneni entry = fifo;
1069774628aSHaren Myneni
1079774628aSHaren Myneni if ((entry->stamp.nx.pswid == cpu_to_be32(FIFO_INVALID_ENTRY))
1089774628aSHaren Myneni || (entry->ccw & cpu_to_be32(CCW0_INVALID))) {
1099774628aSHaren Myneni vinst->fifo_in_progress = 0;
1109774628aSHaren Myneni spin_unlock_irqrestore(&vinst->fault_lock, flags);
1119774628aSHaren Myneni return IRQ_HANDLED;
1129774628aSHaren Myneni }
1139774628aSHaren Myneni
1149774628aSHaren Myneni spin_unlock_irqrestore(&vinst->fault_lock, flags);
1159774628aSHaren Myneni vinst->fault_crbs++;
1169774628aSHaren Myneni if (vinst->fault_crbs == (vinst->fault_fifo_size / CRB_SIZE))
1179774628aSHaren Myneni vinst->fault_crbs = 0;
1189774628aSHaren Myneni
1199774628aSHaren Myneni memcpy(crb, fifo, CRB_SIZE);
1209774628aSHaren Myneni entry->stamp.nx.pswid = cpu_to_be32(FIFO_INVALID_ENTRY);
1219774628aSHaren Myneni entry->ccw |= cpu_to_be32(CCW0_INVALID);
122461862efSHaren Myneni /*
123461862efSHaren Myneni * Return credit for the fault window.
124461862efSHaren Myneni */
125461862efSHaren Myneni vas_return_credit(vinst->fault_win, false);
1269774628aSHaren Myneni
1279774628aSHaren Myneni pr_devel("VAS[%d] fault_fifo %p, fifo %p, fault_crbs %d\n",
1289774628aSHaren Myneni vinst->vas_id, vinst->fault_fifo, fifo,
1299774628aSHaren Myneni vinst->fault_crbs);
1309774628aSHaren Myneni
1313b267973SHaren Myneni vas_dump_crb(crb);
1329774628aSHaren Myneni window = vas_pswid_to_window(vinst,
1339774628aSHaren Myneni be32_to_cpu(crb->stamp.nx.pswid));
1349774628aSHaren Myneni
1359774628aSHaren Myneni if (IS_ERR(window)) {
1369774628aSHaren Myneni /*
1379774628aSHaren Myneni * We got an interrupt about a specific send
1389774628aSHaren Myneni * window but we can't find that window and we can't
1399774628aSHaren Myneni * even clean it up (return credit on user space
1409774628aSHaren Myneni * window).
1419774628aSHaren Myneni * But we should not get here.
1429774628aSHaren Myneni * TODO: Disable IRQ.
1439774628aSHaren Myneni */
144cf33e1e9SHaren Myneni dump_fifo(vinst, (void *)entry);
1459774628aSHaren Myneni pr_err("VAS[%d] fault_fifo %p, fifo %p, pswid 0x%x, fault_crbs %d bad CRB?\n",
1469774628aSHaren Myneni vinst->vas_id, vinst->fault_fifo, fifo,
1479774628aSHaren Myneni be32_to_cpu(crb->stamp.nx.pswid),
1489774628aSHaren Myneni vinst->fault_crbs);
1499774628aSHaren Myneni
1509774628aSHaren Myneni WARN_ON_ONCE(1);
151c96c4436SHaren Myneni } else {
1523b267973SHaren Myneni /*
1533b267973SHaren Myneni * NX sees faults only with user space windows.
1543b267973SHaren Myneni */
1553b267973SHaren Myneni if (window->user_win)
1567bc6f71bSHaren Myneni vas_update_csb(crb, &window->vas_win.task_ref);
1573b267973SHaren Myneni else
1583b267973SHaren Myneni WARN_ON_ONCE(!window->user_win);
1593b267973SHaren Myneni
160461862efSHaren Myneni /*
161461862efSHaren Myneni * Return credit for send window after processing
162461862efSHaren Myneni * fault CRB.
163461862efSHaren Myneni */
164461862efSHaren Myneni vas_return_credit(window, true);
1659774628aSHaren Myneni }
1669774628aSHaren Myneni }
1679774628aSHaren Myneni }
1689774628aSHaren Myneni
vas_fault_handler(int irq,void * dev_id)1699774628aSHaren Myneni irqreturn_t vas_fault_handler(int irq, void *dev_id)
1709774628aSHaren Myneni {
1719774628aSHaren Myneni struct vas_instance *vinst = dev_id;
1729774628aSHaren Myneni irqreturn_t ret = IRQ_WAKE_THREAD;
1739774628aSHaren Myneni unsigned long flags;
1749774628aSHaren Myneni
1759774628aSHaren Myneni /*
1769774628aSHaren Myneni * NX can generate an interrupt for multiple faults. So the
1779774628aSHaren Myneni * fault handler thread process all CRBs until finds invalid
1789774628aSHaren Myneni * entry. In case if NX sees continuous faults, it is possible
1799774628aSHaren Myneni * that the thread function entered with the first interrupt
1809774628aSHaren Myneni * can execute and process all valid CRBs.
1819774628aSHaren Myneni * So wake up thread only if the fault thread is not in progress.
1829774628aSHaren Myneni */
1839774628aSHaren Myneni spin_lock_irqsave(&vinst->fault_lock, flags);
1849774628aSHaren Myneni
1859774628aSHaren Myneni if (vinst->fifo_in_progress)
1869774628aSHaren Myneni ret = IRQ_HANDLED;
1879774628aSHaren Myneni else
1889774628aSHaren Myneni vinst->fifo_in_progress = 1;
1899774628aSHaren Myneni
1909774628aSHaren Myneni spin_unlock_irqrestore(&vinst->fault_lock, flags);
1919774628aSHaren Myneni
1929774628aSHaren Myneni return ret;
1939774628aSHaren Myneni }
1949774628aSHaren Myneni
1959774628aSHaren Myneni /*
1960d17de03SHaren Myneni * Fault window is opened per VAS instance. NX pastes fault CRB in fault
1970d17de03SHaren Myneni * FIFO upon page faults.
1980d17de03SHaren Myneni */
vas_setup_fault_window(struct vas_instance * vinst)1990d17de03SHaren Myneni int vas_setup_fault_window(struct vas_instance *vinst)
2000d17de03SHaren Myneni {
2010d17de03SHaren Myneni struct vas_rx_win_attr attr;
2027bc6f71bSHaren Myneni struct vas_window *win;
2030d17de03SHaren Myneni
2040d17de03SHaren Myneni vinst->fault_fifo_size = VAS_FAULT_WIN_FIFO_SIZE;
2050d17de03SHaren Myneni vinst->fault_fifo = kzalloc(vinst->fault_fifo_size, GFP_KERNEL);
2060d17de03SHaren Myneni if (!vinst->fault_fifo) {
2070d17de03SHaren Myneni pr_err("Unable to alloc %d bytes for fault_fifo\n",
2080d17de03SHaren Myneni vinst->fault_fifo_size);
2090d17de03SHaren Myneni return -ENOMEM;
2100d17de03SHaren Myneni }
2110d17de03SHaren Myneni
2120d17de03SHaren Myneni /*
2130d17de03SHaren Myneni * Invalidate all CRB entries. NX pastes valid entry for each fault.
2140d17de03SHaren Myneni */
2150d17de03SHaren Myneni memset(vinst->fault_fifo, FIFO_INVALID_ENTRY, vinst->fault_fifo_size);
2160d17de03SHaren Myneni vas_init_rx_win_attr(&attr, VAS_COP_TYPE_FAULT);
2170d17de03SHaren Myneni
2180d17de03SHaren Myneni attr.rx_fifo_size = vinst->fault_fifo_size;
219c127d130SHaren Myneni attr.rx_fifo = __pa(vinst->fault_fifo);
2200d17de03SHaren Myneni
2210d17de03SHaren Myneni /*
2220d17de03SHaren Myneni * Max creds is based on number of CRBs can fit in the FIFO.
2230d17de03SHaren Myneni * (fault_fifo_size/CRB_SIZE). If 8MB FIFO is used, max creds
2240d17de03SHaren Myneni * will be 0xffff since the receive creds field is 16bits wide.
2250d17de03SHaren Myneni */
2260d17de03SHaren Myneni attr.wcreds_max = vinst->fault_fifo_size / CRB_SIZE;
2270d17de03SHaren Myneni attr.lnotify_lpid = 0;
2280d17de03SHaren Myneni attr.lnotify_pid = mfspr(SPRN_PID);
2290d17de03SHaren Myneni attr.lnotify_tid = mfspr(SPRN_PID);
2300d17de03SHaren Myneni
2317bc6f71bSHaren Myneni win = vas_rx_win_open(vinst->vas_id, VAS_COP_TYPE_FAULT, &attr);
2327bc6f71bSHaren Myneni if (IS_ERR(win)) {
2337bc6f71bSHaren Myneni pr_err("VAS: Error %ld opening FaultWin\n", PTR_ERR(win));
2340d17de03SHaren Myneni kfree(vinst->fault_fifo);
2357bc6f71bSHaren Myneni return PTR_ERR(win);
2360d17de03SHaren Myneni }
2370d17de03SHaren Myneni
2387bc6f71bSHaren Myneni vinst->fault_win = container_of(win, struct pnv_vas_window, vas_win);
2397bc6f71bSHaren Myneni
2400d17de03SHaren Myneni pr_devel("VAS: Created FaultWin %d, LPID/PID/TID [%d/%d/%d]\n",
2417bc6f71bSHaren Myneni vinst->fault_win->vas_win.winid, attr.lnotify_lpid,
2420d17de03SHaren Myneni attr.lnotify_pid, attr.lnotify_tid);
2430d17de03SHaren Myneni
2440d17de03SHaren Myneni return 0;
2450d17de03SHaren Myneni }
246