1 /*
2  * Copyright 2022 Advanced Micro Devices, Inc.
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice shall be included in
12  * all copies or substantial portions of the Software.
13  *
14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20  * OTHER DEALINGS IN THE SOFTWARE.
21  *
22  */
23 #include "amdgpu.h"
24 #include "amdgpu_xcp.h"
25 #include "amdgpu_drv.h"
26 
27 #include <drm/drm_drv.h>
28 #include "../amdxcp/amdgpu_xcp_drv.h"
29 
30 static int __amdgpu_xcp_run(struct amdgpu_xcp_mgr *xcp_mgr,
31 			    struct amdgpu_xcp_ip *xcp_ip, int xcp_state)
32 {
33 	int (*run_func)(void *handle, uint32_t inst_mask);
34 	int ret = 0;
35 
36 	if (!xcp_ip || !xcp_ip->valid || !xcp_ip->ip_funcs)
37 		return 0;
38 
39 	run_func = NULL;
40 
41 	switch (xcp_state) {
42 	case AMDGPU_XCP_PREPARE_SUSPEND:
43 		run_func = xcp_ip->ip_funcs->prepare_suspend;
44 		break;
45 	case AMDGPU_XCP_SUSPEND:
46 		run_func = xcp_ip->ip_funcs->suspend;
47 		break;
48 	case AMDGPU_XCP_PREPARE_RESUME:
49 		run_func = xcp_ip->ip_funcs->prepare_resume;
50 		break;
51 	case AMDGPU_XCP_RESUME:
52 		run_func = xcp_ip->ip_funcs->resume;
53 		break;
54 	}
55 
56 	if (run_func)
57 		ret = run_func(xcp_mgr->adev, xcp_ip->inst_mask);
58 
59 	return ret;
60 }
61 
62 static int amdgpu_xcp_run_transition(struct amdgpu_xcp_mgr *xcp_mgr, int xcp_id,
63 				     int state)
64 {
65 	struct amdgpu_xcp_ip *xcp_ip;
66 	struct amdgpu_xcp *xcp;
67 	int i, ret;
68 
69 	if (xcp_id >= MAX_XCP || !xcp_mgr->xcp[xcp_id].valid)
70 		return -EINVAL;
71 
72 	xcp = &xcp_mgr->xcp[xcp_id];
73 	for (i = 0; i < AMDGPU_XCP_MAX_BLOCKS; ++i) {
74 		xcp_ip = &xcp->ip[i];
75 		ret = __amdgpu_xcp_run(xcp_mgr, xcp_ip, state);
76 		if (ret)
77 			break;
78 	}
79 
80 	return ret;
81 }
82 
83 int amdgpu_xcp_prepare_suspend(struct amdgpu_xcp_mgr *xcp_mgr, int xcp_id)
84 {
85 	return amdgpu_xcp_run_transition(xcp_mgr, xcp_id,
86 					 AMDGPU_XCP_PREPARE_SUSPEND);
87 }
88 
89 int amdgpu_xcp_suspend(struct amdgpu_xcp_mgr *xcp_mgr, int xcp_id)
90 {
91 	return amdgpu_xcp_run_transition(xcp_mgr, xcp_id, AMDGPU_XCP_SUSPEND);
92 }
93 
94 int amdgpu_xcp_prepare_resume(struct amdgpu_xcp_mgr *xcp_mgr, int xcp_id)
95 {
96 	return amdgpu_xcp_run_transition(xcp_mgr, xcp_id,
97 					 AMDGPU_XCP_PREPARE_RESUME);
98 }
99 
100 int amdgpu_xcp_resume(struct amdgpu_xcp_mgr *xcp_mgr, int xcp_id)
101 {
102 	return amdgpu_xcp_run_transition(xcp_mgr, xcp_id, AMDGPU_XCP_RESUME);
103 }
104 
105 static void __amdgpu_xcp_add_block(struct amdgpu_xcp_mgr *xcp_mgr, int xcp_id,
106 				   struct amdgpu_xcp_ip *ip)
107 {
108 	struct amdgpu_xcp *xcp;
109 
110 	if (!ip)
111 		return;
112 
113 	xcp = &xcp_mgr->xcp[xcp_id];
114 	xcp->ip[ip->ip_id] = *ip;
115 	xcp->ip[ip->ip_id].valid = true;
116 
117 	xcp->valid = true;
118 }
119 
120 int amdgpu_xcp_init(struct amdgpu_xcp_mgr *xcp_mgr, int num_xcps, int mode)
121 {
122 	struct amdgpu_device *adev = xcp_mgr->adev;
123 	struct amdgpu_xcp_ip ip;
124 	uint8_t mem_id;
125 	int i, j, ret;
126 
127 	if (!num_xcps || num_xcps > MAX_XCP)
128 		return -EINVAL;
129 
130 	xcp_mgr->mode = mode;
131 
132 	for (i = 0; i < MAX_XCP; ++i)
133 		xcp_mgr->xcp[i].valid = false;
134 
135 	/* This is needed for figuring out memory id of xcp */
136 	xcp_mgr->num_xcp_per_mem_partition = num_xcps / xcp_mgr->adev->gmc.num_mem_partitions;
137 
138 	for (i = 0; i < num_xcps; ++i) {
139 		for (j = AMDGPU_XCP_GFXHUB; j < AMDGPU_XCP_MAX_BLOCKS; ++j) {
140 			ret = xcp_mgr->funcs->get_ip_details(xcp_mgr, i, j,
141 							     &ip);
142 			if (ret)
143 				continue;
144 
145 			__amdgpu_xcp_add_block(xcp_mgr, i, &ip);
146 		}
147 
148 		xcp_mgr->xcp[i].id = i;
149 
150 		if (xcp_mgr->funcs->get_xcp_mem_id) {
151 			ret = xcp_mgr->funcs->get_xcp_mem_id(
152 				xcp_mgr, &xcp_mgr->xcp[i], &mem_id);
153 			if (ret)
154 				continue;
155 			else
156 				xcp_mgr->xcp[i].mem_id = mem_id;
157 		}
158 	}
159 
160 	xcp_mgr->num_xcps = num_xcps;
161 	amdgpu_xcp_update_partition_sched_list(adev);
162 
163 	return 0;
164 }
165 
166 int amdgpu_xcp_switch_partition_mode(struct amdgpu_xcp_mgr *xcp_mgr, int mode)
167 {
168 	int ret, curr_mode, num_xcps = 0;
169 
170 	if (!xcp_mgr || mode == AMDGPU_XCP_MODE_NONE)
171 		return -EINVAL;
172 
173 	if (xcp_mgr->mode == mode)
174 		return 0;
175 
176 	if (!xcp_mgr->funcs || !xcp_mgr->funcs->switch_partition_mode)
177 		return 0;
178 
179 	mutex_lock(&xcp_mgr->xcp_lock);
180 
181 	curr_mode = xcp_mgr->mode;
182 	/* State set to transient mode */
183 	xcp_mgr->mode = AMDGPU_XCP_MODE_TRANS;
184 
185 	ret = xcp_mgr->funcs->switch_partition_mode(xcp_mgr, mode, &num_xcps);
186 
187 	if (ret) {
188 		/* Failed, get whatever mode it's at now */
189 		if (xcp_mgr->funcs->query_partition_mode)
190 			xcp_mgr->mode = amdgpu_xcp_query_partition_mode(
191 				xcp_mgr, AMDGPU_XCP_FL_LOCKED);
192 		else
193 			xcp_mgr->mode = curr_mode;
194 
195 		goto out;
196 	}
197 
198 out:
199 	mutex_unlock(&xcp_mgr->xcp_lock);
200 
201 	return ret;
202 }
203 
204 int amdgpu_xcp_query_partition_mode(struct amdgpu_xcp_mgr *xcp_mgr, u32 flags)
205 {
206 	int mode;
207 
208 	if (xcp_mgr->mode == AMDGPU_XCP_MODE_NONE)
209 		return xcp_mgr->mode;
210 
211 	if (!xcp_mgr->funcs || !xcp_mgr->funcs->query_partition_mode)
212 		return xcp_mgr->mode;
213 
214 	if (!(flags & AMDGPU_XCP_FL_LOCKED))
215 		mutex_lock(&xcp_mgr->xcp_lock);
216 	mode = xcp_mgr->funcs->query_partition_mode(xcp_mgr);
217 	if (xcp_mgr->mode != AMDGPU_XCP_MODE_TRANS && mode != xcp_mgr->mode)
218 		dev_WARN(
219 			xcp_mgr->adev->dev,
220 			"Cached partition mode %d not matching with device mode %d",
221 			xcp_mgr->mode, mode);
222 
223 	if (!(flags & AMDGPU_XCP_FL_LOCKED))
224 		mutex_unlock(&xcp_mgr->xcp_lock);
225 
226 	return mode;
227 }
228 
229 static int amdgpu_xcp_dev_alloc(struct amdgpu_device *adev)
230 {
231 	struct drm_device *p_ddev;
232 	struct drm_device *ddev;
233 	int i, ret;
234 
235 	ddev = adev_to_drm(adev);
236 
237 	/* xcp #0 shares drm device setting with adev */
238 	adev->xcp_mgr->xcp->ddev = ddev;
239 
240 	for (i = 1; i < MAX_XCP; i++) {
241 		ret = amdgpu_xcp_drm_dev_alloc(&p_ddev);
242 		if (ret)
243 			return ret;
244 
245 		/* Redirect all IOCTLs to the primary device */
246 		adev->xcp_mgr->xcp[i].rdev = p_ddev->render->dev;
247 		adev->xcp_mgr->xcp[i].pdev = p_ddev->primary->dev;
248 		adev->xcp_mgr->xcp[i].driver = (struct drm_driver *)p_ddev->driver;
249 		adev->xcp_mgr->xcp[i].vma_offset_manager = p_ddev->vma_offset_manager;
250 		p_ddev->render->dev = ddev;
251 		p_ddev->primary->dev = ddev;
252 		p_ddev->vma_offset_manager = ddev->vma_offset_manager;
253 		p_ddev->driver = &amdgpu_partition_driver;
254 		adev->xcp_mgr->xcp[i].ddev = p_ddev;
255 	}
256 
257 	return 0;
258 }
259 
260 int amdgpu_xcp_mgr_init(struct amdgpu_device *adev, int init_mode,
261 			int init_num_xcps,
262 			struct amdgpu_xcp_mgr_funcs *xcp_funcs)
263 {
264 	struct amdgpu_xcp_mgr *xcp_mgr;
265 
266 	if (!xcp_funcs || !xcp_funcs->switch_partition_mode ||
267 	    !xcp_funcs->get_ip_details)
268 		return -EINVAL;
269 
270 	xcp_mgr = kzalloc(sizeof(*xcp_mgr), GFP_KERNEL);
271 
272 	if (!xcp_mgr)
273 		return -ENOMEM;
274 
275 	xcp_mgr->adev = adev;
276 	xcp_mgr->funcs = xcp_funcs;
277 	xcp_mgr->mode = init_mode;
278 	mutex_init(&xcp_mgr->xcp_lock);
279 
280 	if (init_mode != AMDGPU_XCP_MODE_NONE)
281 		amdgpu_xcp_init(xcp_mgr, init_num_xcps, init_mode);
282 
283 	adev->xcp_mgr = xcp_mgr;
284 
285 	return amdgpu_xcp_dev_alloc(adev);
286 }
287 
288 int amdgpu_xcp_get_partition(struct amdgpu_xcp_mgr *xcp_mgr,
289 			     enum AMDGPU_XCP_IP_BLOCK ip, int instance)
290 {
291 	struct amdgpu_xcp *xcp;
292 	int i, id_mask = 0;
293 
294 	if (ip >= AMDGPU_XCP_MAX_BLOCKS)
295 		return -EINVAL;
296 
297 	for (i = 0; i < xcp_mgr->num_xcps; ++i) {
298 		xcp = &xcp_mgr->xcp[i];
299 		if ((xcp->valid) && (xcp->ip[ip].valid) &&
300 		    (xcp->ip[ip].inst_mask & BIT(instance)))
301 			id_mask |= BIT(i);
302 	}
303 
304 	if (!id_mask)
305 		id_mask = -ENXIO;
306 
307 	return id_mask;
308 }
309 
310 int amdgpu_xcp_get_inst_details(struct amdgpu_xcp *xcp,
311 				enum AMDGPU_XCP_IP_BLOCK ip,
312 				uint32_t *inst_mask)
313 {
314 	if (!xcp->valid || !inst_mask || !(xcp->ip[ip].valid))
315 		return -EINVAL;
316 
317 	*inst_mask = xcp->ip[ip].inst_mask;
318 
319 	return 0;
320 }
321 
322 int amdgpu_xcp_dev_register(struct amdgpu_device *adev,
323 			const struct pci_device_id *ent)
324 {
325 	int i, ret;
326 
327 	if (!adev->xcp_mgr)
328 		return 0;
329 
330 	for (i = 1; i < MAX_XCP; i++) {
331 		ret = drm_dev_register(adev->xcp_mgr->xcp[i].ddev, ent->driver_data);
332 		if (ret)
333 			return ret;
334 	}
335 
336 	return 0;
337 }
338 
339 void amdgpu_xcp_dev_unplug(struct amdgpu_device *adev)
340 {
341 	struct drm_device *p_ddev;
342 	int i;
343 
344 	if (!adev->xcp_mgr)
345 		return;
346 
347 	for (i = 1; i < MAX_XCP; i++) {
348 		p_ddev = adev->xcp_mgr->xcp[i].ddev;
349 		drm_dev_unplug(p_ddev);
350 		p_ddev->render->dev = adev->xcp_mgr->xcp[i].rdev;
351 		p_ddev->primary->dev = adev->xcp_mgr->xcp[i].pdev;
352 		p_ddev->driver =  adev->xcp_mgr->xcp[i].driver;
353 		p_ddev->vma_offset_manager = adev->xcp_mgr->xcp[i].vma_offset_manager;
354 	}
355 }
356 
357 int amdgpu_xcp_open_device(struct amdgpu_device *adev,
358 			   struct amdgpu_fpriv *fpriv,
359 			   struct drm_file *file_priv)
360 {
361 	int i;
362 
363 	if (!adev->xcp_mgr)
364 		return 0;
365 
366 	fpriv->xcp_id = ~0;
367 	for (i = 0; i < MAX_XCP; ++i) {
368 		if (!adev->xcp_mgr->xcp[i].ddev)
369 			break;
370 
371 		if (file_priv->minor == adev->xcp_mgr->xcp[i].ddev->render) {
372 			if (adev->xcp_mgr->xcp[i].valid == FALSE) {
373 				dev_err(adev->dev, "renderD%d partition %d not valid!",
374 						file_priv->minor->index, i);
375 				return -ENOENT;
376 			}
377 			dev_dbg(adev->dev, "renderD%d partition %d opened!",
378 					file_priv->minor->index, i);
379 			fpriv->xcp_id = i;
380 			break;
381 		}
382 	}
383 
384 	fpriv->vm.mem_id = fpriv->xcp_id == ~0 ? -1 :
385 				adev->xcp_mgr->xcp[fpriv->xcp_id].mem_id;
386 	return 0;
387 }
388 
389 void amdgpu_xcp_release_sched(struct amdgpu_device *adev,
390 				  struct amdgpu_ctx_entity *entity)
391 {
392 	struct drm_gpu_scheduler *sched;
393 	struct amdgpu_ring *ring;
394 
395 	if (!adev->xcp_mgr)
396 		return;
397 
398 	sched = entity->entity.rq->sched;
399 	if (sched->ready) {
400 		ring = to_amdgpu_ring(entity->entity.rq->sched);
401 		atomic_dec(&adev->xcp_mgr->xcp[ring->xcp_id].ref_cnt);
402 	}
403 }
404 
405