1 /* 2 * Copyright 2022 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 */ 23 #include "amdgpu.h" 24 #include "amdgpu_xcp.h" 25 #include "amdgpu_drv.h" 26 27 #include <drm/drm_drv.h> 28 #include "../amdxcp/amdgpu_xcp_drv.h" 29 30 static int __amdgpu_xcp_run(struct amdgpu_xcp_mgr *xcp_mgr, 31 struct amdgpu_xcp_ip *xcp_ip, int xcp_state) 32 { 33 int (*run_func)(void *handle, uint32_t inst_mask); 34 int ret = 0; 35 36 if (!xcp_ip || !xcp_ip->valid || !xcp_ip->ip_funcs) 37 return 0; 38 39 run_func = NULL; 40 41 switch (xcp_state) { 42 case AMDGPU_XCP_PREPARE_SUSPEND: 43 run_func = xcp_ip->ip_funcs->prepare_suspend; 44 break; 45 case AMDGPU_XCP_SUSPEND: 46 run_func = xcp_ip->ip_funcs->suspend; 47 break; 48 case AMDGPU_XCP_PREPARE_RESUME: 49 run_func = xcp_ip->ip_funcs->prepare_resume; 50 break; 51 case AMDGPU_XCP_RESUME: 52 run_func = xcp_ip->ip_funcs->resume; 53 break; 54 } 55 56 if (run_func) 57 ret = run_func(xcp_mgr->adev, xcp_ip->inst_mask); 58 59 return ret; 60 } 61 62 static int amdgpu_xcp_run_transition(struct amdgpu_xcp_mgr *xcp_mgr, int xcp_id, 63 int state) 64 { 65 struct amdgpu_xcp_ip *xcp_ip; 66 struct amdgpu_xcp *xcp; 67 int i, ret; 68 69 if (xcp_id >= MAX_XCP || !xcp_mgr->xcp[xcp_id].valid) 70 return -EINVAL; 71 72 xcp = &xcp_mgr->xcp[xcp_id]; 73 for (i = 0; i < AMDGPU_XCP_MAX_BLOCKS; ++i) { 74 xcp_ip = &xcp->ip[i]; 75 ret = __amdgpu_xcp_run(xcp_mgr, xcp_ip, state); 76 if (ret) 77 break; 78 } 79 80 return ret; 81 } 82 83 int amdgpu_xcp_prepare_suspend(struct amdgpu_xcp_mgr *xcp_mgr, int xcp_id) 84 { 85 return amdgpu_xcp_run_transition(xcp_mgr, xcp_id, 86 AMDGPU_XCP_PREPARE_SUSPEND); 87 } 88 89 int amdgpu_xcp_suspend(struct amdgpu_xcp_mgr *xcp_mgr, int xcp_id) 90 { 91 return amdgpu_xcp_run_transition(xcp_mgr, xcp_id, AMDGPU_XCP_SUSPEND); 92 } 93 94 int amdgpu_xcp_prepare_resume(struct amdgpu_xcp_mgr *xcp_mgr, int xcp_id) 95 { 96 return amdgpu_xcp_run_transition(xcp_mgr, xcp_id, 97 AMDGPU_XCP_PREPARE_RESUME); 98 } 99 100 int amdgpu_xcp_resume(struct amdgpu_xcp_mgr *xcp_mgr, int xcp_id) 101 { 102 return amdgpu_xcp_run_transition(xcp_mgr, xcp_id, AMDGPU_XCP_RESUME); 103 } 104 105 static void __amdgpu_xcp_add_block(struct amdgpu_xcp_mgr *xcp_mgr, int xcp_id, 106 struct amdgpu_xcp_ip *ip) 107 { 108 struct amdgpu_xcp *xcp; 109 110 if (!ip) 111 return; 112 113 xcp = &xcp_mgr->xcp[xcp_id]; 114 xcp->ip[ip->ip_id] = *ip; 115 xcp->ip[ip->ip_id].valid = true; 116 117 xcp->valid = true; 118 } 119 120 int amdgpu_xcp_init(struct amdgpu_xcp_mgr *xcp_mgr, int num_xcps, int mode) 121 { 122 struct amdgpu_device *adev = xcp_mgr->adev; 123 struct amdgpu_xcp_ip ip; 124 uint8_t mem_id; 125 int i, j, ret; 126 127 if (!num_xcps || num_xcps > MAX_XCP) 128 return -EINVAL; 129 130 xcp_mgr->mode = mode; 131 132 for (i = 0; i < MAX_XCP; ++i) 133 xcp_mgr->xcp[i].valid = false; 134 135 /* This is needed for figuring out memory id of xcp */ 136 xcp_mgr->num_xcp_per_mem_partition = num_xcps / xcp_mgr->adev->gmc.num_mem_partitions; 137 138 for (i = 0; i < num_xcps; ++i) { 139 for (j = AMDGPU_XCP_GFXHUB; j < AMDGPU_XCP_MAX_BLOCKS; ++j) { 140 ret = xcp_mgr->funcs->get_ip_details(xcp_mgr, i, j, 141 &ip); 142 if (ret) 143 continue; 144 145 __amdgpu_xcp_add_block(xcp_mgr, i, &ip); 146 } 147 148 xcp_mgr->xcp[i].id = i; 149 150 if (xcp_mgr->funcs->get_xcp_mem_id) { 151 ret = xcp_mgr->funcs->get_xcp_mem_id( 152 xcp_mgr, &xcp_mgr->xcp[i], &mem_id); 153 if (ret) 154 continue; 155 else 156 xcp_mgr->xcp[i].mem_id = mem_id; 157 } 158 } 159 160 xcp_mgr->num_xcps = num_xcps; 161 amdgpu_xcp_update_partition_sched_list(adev); 162 163 return 0; 164 } 165 166 int amdgpu_xcp_switch_partition_mode(struct amdgpu_xcp_mgr *xcp_mgr, int mode) 167 { 168 int ret, curr_mode, num_xcps = 0; 169 170 if (!xcp_mgr || mode == AMDGPU_XCP_MODE_NONE) 171 return -EINVAL; 172 173 if (xcp_mgr->mode == mode) 174 return 0; 175 176 if (!xcp_mgr->funcs || !xcp_mgr->funcs->switch_partition_mode) 177 return 0; 178 179 mutex_lock(&xcp_mgr->xcp_lock); 180 181 curr_mode = xcp_mgr->mode; 182 /* State set to transient mode */ 183 xcp_mgr->mode = AMDGPU_XCP_MODE_TRANS; 184 185 ret = xcp_mgr->funcs->switch_partition_mode(xcp_mgr, mode, &num_xcps); 186 187 if (ret) { 188 /* Failed, get whatever mode it's at now */ 189 if (xcp_mgr->funcs->query_partition_mode) 190 xcp_mgr->mode = amdgpu_xcp_query_partition_mode( 191 xcp_mgr, AMDGPU_XCP_FL_LOCKED); 192 else 193 xcp_mgr->mode = curr_mode; 194 195 goto out; 196 } 197 198 out: 199 mutex_unlock(&xcp_mgr->xcp_lock); 200 201 return ret; 202 } 203 204 int amdgpu_xcp_query_partition_mode(struct amdgpu_xcp_mgr *xcp_mgr, u32 flags) 205 { 206 int mode; 207 208 if (xcp_mgr->mode == AMDGPU_XCP_MODE_NONE) 209 return xcp_mgr->mode; 210 211 if (!xcp_mgr->funcs || !xcp_mgr->funcs->query_partition_mode) 212 return xcp_mgr->mode; 213 214 if (!(flags & AMDGPU_XCP_FL_LOCKED)) 215 mutex_lock(&xcp_mgr->xcp_lock); 216 mode = xcp_mgr->funcs->query_partition_mode(xcp_mgr); 217 if (xcp_mgr->mode != AMDGPU_XCP_MODE_TRANS && mode != xcp_mgr->mode) 218 dev_WARN( 219 xcp_mgr->adev->dev, 220 "Cached partition mode %d not matching with device mode %d", 221 xcp_mgr->mode, mode); 222 223 if (!(flags & AMDGPU_XCP_FL_LOCKED)) 224 mutex_unlock(&xcp_mgr->xcp_lock); 225 226 return mode; 227 } 228 229 static int amdgpu_xcp_dev_alloc(struct amdgpu_device *adev) 230 { 231 struct drm_device *p_ddev; 232 struct drm_device *ddev; 233 int i, ret; 234 235 ddev = adev_to_drm(adev); 236 237 for (i = 0; i < MAX_XCP; i++) { 238 ret = amdgpu_xcp_drm_dev_alloc(&p_ddev); 239 if (ret) 240 return ret; 241 242 /* Redirect all IOCTLs to the primary device */ 243 adev->xcp_mgr->xcp[i].rdev = p_ddev->render->dev; 244 adev->xcp_mgr->xcp[i].pdev = p_ddev->primary->dev; 245 adev->xcp_mgr->xcp[i].driver = (struct drm_driver *)p_ddev->driver; 246 adev->xcp_mgr->xcp[i].vma_offset_manager = p_ddev->vma_offset_manager; 247 p_ddev->render->dev = ddev; 248 p_ddev->primary->dev = ddev; 249 p_ddev->vma_offset_manager = ddev->vma_offset_manager; 250 p_ddev->driver = &amdgpu_partition_driver; 251 adev->xcp_mgr->xcp[i].ddev = p_ddev; 252 } 253 254 return 0; 255 } 256 257 int amdgpu_xcp_mgr_init(struct amdgpu_device *adev, int init_mode, 258 int init_num_xcps, 259 struct amdgpu_xcp_mgr_funcs *xcp_funcs) 260 { 261 struct amdgpu_xcp_mgr *xcp_mgr; 262 263 if (!xcp_funcs || !xcp_funcs->switch_partition_mode || 264 !xcp_funcs->get_ip_details) 265 return -EINVAL; 266 267 xcp_mgr = kzalloc(sizeof(*xcp_mgr), GFP_KERNEL); 268 269 if (!xcp_mgr) 270 return -ENOMEM; 271 272 xcp_mgr->adev = adev; 273 xcp_mgr->funcs = xcp_funcs; 274 xcp_mgr->mode = init_mode; 275 mutex_init(&xcp_mgr->xcp_lock); 276 277 if (init_mode != AMDGPU_XCP_MODE_NONE) 278 amdgpu_xcp_init(xcp_mgr, init_num_xcps, init_mode); 279 280 adev->xcp_mgr = xcp_mgr; 281 282 return amdgpu_xcp_dev_alloc(adev); 283 } 284 285 int amdgpu_xcp_get_partition(struct amdgpu_xcp_mgr *xcp_mgr, 286 enum AMDGPU_XCP_IP_BLOCK ip, int instance) 287 { 288 struct amdgpu_xcp *xcp; 289 int i, id_mask = 0; 290 291 if (ip >= AMDGPU_XCP_MAX_BLOCKS) 292 return -EINVAL; 293 294 for (i = 0; i < xcp_mgr->num_xcps; ++i) { 295 xcp = &xcp_mgr->xcp[i]; 296 if ((xcp->valid) && (xcp->ip[ip].valid) && 297 (xcp->ip[ip].inst_mask & BIT(instance))) 298 id_mask |= BIT(i); 299 } 300 301 if (!id_mask) 302 id_mask = -ENXIO; 303 304 return id_mask; 305 } 306 307 int amdgpu_xcp_get_inst_details(struct amdgpu_xcp *xcp, 308 enum AMDGPU_XCP_IP_BLOCK ip, 309 uint32_t *inst_mask) 310 { 311 if (!xcp->valid || !inst_mask || !(xcp->ip[ip].valid)) 312 return -EINVAL; 313 314 *inst_mask = xcp->ip[ip].inst_mask; 315 316 return 0; 317 } 318 319 int amdgpu_xcp_dev_register(struct amdgpu_device *adev, 320 const struct pci_device_id *ent) 321 { 322 int i, ret; 323 324 if (!adev->xcp_mgr) 325 return 0; 326 327 for (i = 0; i < MAX_XCP; i++) { 328 ret = drm_dev_register(adev->xcp_mgr->xcp[i].ddev, ent->driver_data); 329 if (ret) 330 return ret; 331 } 332 333 return 0; 334 } 335 336 void amdgpu_xcp_dev_unplug(struct amdgpu_device *adev) 337 { 338 struct drm_device *p_ddev; 339 int i; 340 341 if (!adev->xcp_mgr) 342 return; 343 344 for (i = 0; i < MAX_XCP; i++) { 345 p_ddev = adev->xcp_mgr->xcp[i].ddev; 346 drm_dev_unplug(p_ddev); 347 p_ddev->render->dev = adev->xcp_mgr->xcp[i].rdev; 348 p_ddev->primary->dev = adev->xcp_mgr->xcp[i].pdev; 349 p_ddev->driver = adev->xcp_mgr->xcp[i].driver; 350 p_ddev->vma_offset_manager = adev->xcp_mgr->xcp[i].vma_offset_manager; 351 } 352 } 353 354 int amdgpu_xcp_open_device(struct amdgpu_device *adev, 355 struct amdgpu_fpriv *fpriv, 356 struct drm_file *file_priv) 357 { 358 int i; 359 360 if (!adev->xcp_mgr) 361 return 0; 362 363 fpriv->xcp_id = ~0; 364 for (i = 0; i < MAX_XCP; ++i) { 365 if (!adev->xcp_mgr->xcp[i].ddev) 366 break; 367 368 if (file_priv->minor == adev->xcp_mgr->xcp[i].ddev->render) { 369 if (adev->xcp_mgr->xcp[i].valid == FALSE) { 370 dev_err(adev->dev, "renderD%d partition %d not valid!", 371 file_priv->minor->index, i); 372 return -ENOENT; 373 } 374 dev_dbg(adev->dev, "renderD%d partition %d opened!", 375 file_priv->minor->index, i); 376 fpriv->xcp_id = i; 377 break; 378 } 379 } 380 381 fpriv->vm.mem_id = fpriv->xcp_id == ~0 ? -1 : 382 adev->xcp_mgr->xcp[fpriv->xcp_id].mem_id; 383 return 0; 384 } 385 386 void amdgpu_xcp_release_sched(struct amdgpu_device *adev, 387 struct amdgpu_ctx_entity *entity) 388 { 389 struct drm_gpu_scheduler *sched; 390 struct amdgpu_ring *ring; 391 392 if (!adev->xcp_mgr) 393 return; 394 395 sched = entity->entity.rq->sched; 396 if (sched->ready) { 397 ring = to_amdgpu_ring(entity->entity.rq->sched); 398 atomic_dec(&adev->xcp_mgr->xcp[ring->xcp_id].ref_cnt); 399 } 400 } 401 402