1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * SN Platform GRU Driver 4 * 5 * Dump GRU State 6 * 7 * Copyright (c) 2008 Silicon Graphics, Inc. All Rights Reserved. 8 */ 9 10 #include <linux/kernel.h> 11 #include <linux/mm.h> 12 #include <linux/spinlock.h> 13 #include <linux/uaccess.h> 14 #include <linux/delay.h> 15 #include <linux/bitops.h> 16 #include <asm/uv/uv_hub.h> 17 18 #include <linux/nospec.h> 19 20 #include "gru.h" 21 #include "grutables.h" 22 #include "gruhandles.h" 23 #include "grulib.h" 24 25 #define CCH_LOCK_ATTEMPTS 10 26 27 static int gru_user_copy_handle(void __user **dp, void *s) 28 { 29 if (copy_to_user(*dp, s, GRU_HANDLE_BYTES)) 30 return -1; 31 *dp += GRU_HANDLE_BYTES; 32 return 0; 33 } 34 35 static int gru_dump_context_data(void *grubase, 36 struct gru_context_configuration_handle *cch, 37 void __user *ubuf, int ctxnum, int dsrcnt, 38 int flush_cbrs) 39 { 40 void *cb, *cbe, *tfh, *gseg; 41 int i, scr; 42 43 gseg = grubase + ctxnum * GRU_GSEG_STRIDE; 44 cb = gseg + GRU_CB_BASE; 45 cbe = grubase + GRU_CBE_BASE; 46 tfh = grubase + GRU_TFH_BASE; 47 48 for_each_cbr_in_allocation_map(i, &cch->cbr_allocation_map, scr) { 49 if (flush_cbrs) 50 gru_flush_cache(cb); 51 if (gru_user_copy_handle(&ubuf, cb)) 52 goto fail; 53 if (gru_user_copy_handle(&ubuf, tfh + i * GRU_HANDLE_STRIDE)) 54 goto fail; 55 if (gru_user_copy_handle(&ubuf, cbe + i * GRU_HANDLE_STRIDE)) 56 goto fail; 57 cb += GRU_HANDLE_STRIDE; 58 } 59 if (dsrcnt) 60 memcpy(ubuf, gseg + GRU_DS_BASE, dsrcnt * GRU_HANDLE_STRIDE); 61 return 0; 62 63 fail: 64 return -EFAULT; 65 } 66 67 static int gru_dump_tfm(struct gru_state *gru, 68 void __user *ubuf, void __user *ubufend) 69 { 70 struct gru_tlb_fault_map *tfm; 71 int i; 72 73 if (GRU_NUM_TFM * GRU_CACHE_LINE_BYTES > ubufend - ubuf) 74 return -EFBIG; 75 76 for (i = 0; i < GRU_NUM_TFM; i++) { 77 tfm = get_tfm(gru->gs_gru_base_vaddr, i); 78 if (gru_user_copy_handle(&ubuf, tfm)) 79 goto fail; 80 } 81 return GRU_NUM_TFM * GRU_CACHE_LINE_BYTES; 82 83 fail: 84 return -EFAULT; 85 } 86 87 static int gru_dump_tgh(struct gru_state *gru, 88 void __user *ubuf, void __user *ubufend) 89 { 90 struct gru_tlb_global_handle *tgh; 91 int i; 92 93 if (GRU_NUM_TGH * GRU_CACHE_LINE_BYTES > ubufend - ubuf) 94 return -EFBIG; 95 96 for (i = 0; i < GRU_NUM_TGH; i++) { 97 tgh = get_tgh(gru->gs_gru_base_vaddr, i); 98 if (gru_user_copy_handle(&ubuf, tgh)) 99 goto fail; 100 } 101 return GRU_NUM_TGH * GRU_CACHE_LINE_BYTES; 102 103 fail: 104 return -EFAULT; 105 } 106 107 static int gru_dump_context(struct gru_state *gru, int ctxnum, 108 void __user *ubuf, void __user *ubufend, char data_opt, 109 char lock_cch, char flush_cbrs) 110 { 111 struct gru_dump_context_header hdr; 112 struct gru_dump_context_header __user *uhdr = ubuf; 113 struct gru_context_configuration_handle *cch, *ubufcch; 114 struct gru_thread_state *gts; 115 int try, cch_locked, cbrcnt = 0, dsrcnt = 0, bytes = 0, ret = 0; 116 void *grubase; 117 118 memset(&hdr, 0, sizeof(hdr)); 119 grubase = gru->gs_gru_base_vaddr; 120 cch = get_cch(grubase, ctxnum); 121 for (try = 0; try < CCH_LOCK_ATTEMPTS; try++) { 122 cch_locked = trylock_cch_handle(cch); 123 if (cch_locked) 124 break; 125 msleep(1); 126 } 127 128 ubuf += sizeof(hdr); 129 ubufcch = ubuf; 130 if (gru_user_copy_handle(&ubuf, cch)) { 131 if (cch_locked) 132 unlock_cch_handle(cch); 133 return -EFAULT; 134 } 135 if (cch_locked) 136 ubufcch->delresp = 0; 137 bytes = sizeof(hdr) + GRU_CACHE_LINE_BYTES; 138 139 if (cch_locked || !lock_cch) { 140 gts = gru->gs_gts[ctxnum]; 141 if (gts && gts->ts_vma) { 142 hdr.pid = gts->ts_tgid_owner; 143 hdr.vaddr = gts->ts_vma->vm_start; 144 } 145 if (cch->state != CCHSTATE_INACTIVE) { 146 cbrcnt = hweight64(cch->cbr_allocation_map) * 147 GRU_CBR_AU_SIZE; 148 dsrcnt = data_opt ? hweight32(cch->dsr_allocation_map) * 149 GRU_DSR_AU_CL : 0; 150 } 151 bytes += (3 * cbrcnt + dsrcnt) * GRU_CACHE_LINE_BYTES; 152 if (bytes > ubufend - ubuf) 153 ret = -EFBIG; 154 else 155 ret = gru_dump_context_data(grubase, cch, ubuf, ctxnum, 156 dsrcnt, flush_cbrs); 157 } 158 if (cch_locked) 159 unlock_cch_handle(cch); 160 if (ret) 161 return ret; 162 163 hdr.magic = GRU_DUMP_MAGIC; 164 hdr.gid = gru->gs_gid; 165 hdr.ctxnum = ctxnum; 166 hdr.cbrcnt = cbrcnt; 167 hdr.dsrcnt = dsrcnt; 168 hdr.cch_locked = cch_locked; 169 if (copy_to_user(uhdr, &hdr, sizeof(hdr))) 170 return -EFAULT; 171 172 return bytes; 173 } 174 175 int gru_dump_chiplet_request(unsigned long arg) 176 { 177 struct gru_state *gru; 178 struct gru_dump_chiplet_state_req req; 179 void __user *ubuf; 180 void __user *ubufend; 181 int ctxnum, ret, cnt = 0; 182 183 if (copy_from_user(&req, (void __user *)arg, sizeof(req))) 184 return -EFAULT; 185 186 /* Currently, only dump by gid is implemented */ 187 if (req.gid >= gru_max_gids) 188 return -EINVAL; 189 req.gid = array_index_nospec(req.gid, gru_max_gids); 190 191 gru = GID_TO_GRU(req.gid); 192 ubuf = req.buf; 193 ubufend = req.buf + req.buflen; 194 195 ret = gru_dump_tfm(gru, ubuf, ubufend); 196 if (ret < 0) 197 goto fail; 198 ubuf += ret; 199 200 ret = gru_dump_tgh(gru, ubuf, ubufend); 201 if (ret < 0) 202 goto fail; 203 ubuf += ret; 204 205 for (ctxnum = 0; ctxnum < GRU_NUM_CCH; ctxnum++) { 206 if (req.ctxnum == ctxnum || req.ctxnum < 0) { 207 ret = gru_dump_context(gru, ctxnum, ubuf, ubufend, 208 req.data_opt, req.lock_cch, 209 req.flush_cbrs); 210 if (ret < 0) 211 goto fail; 212 ubuf += ret; 213 cnt++; 214 } 215 } 216 217 if (copy_to_user((void __user *)arg, &req, sizeof(req))) 218 return -EFAULT; 219 return cnt; 220 221 fail: 222 return ret; 223 } 224