1 /*
2  * Copyright 2014 Advanced Micro Devices, Inc.
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice shall be included in
12  * all copies or substantial portions of the Software.
13  *
14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20  * OTHER DEALINGS IN THE SOFTWARE.
21  *
22  */
23 
24 #include <linux/slab.h>
25 #include <linux/list.h>
26 #include "kfd_device_queue_manager.h"
27 #include "kfd_priv.h"
28 #include "kfd_kernel_queue.h"
29 
30 static inline struct process_queue_node *get_queue_by_qid(
31 			struct process_queue_manager *pqm, unsigned int qid)
32 {
33 	struct process_queue_node *pqn;
34 
35 	list_for_each_entry(pqn, &pqm->queues, process_queue_list) {
36 		if ((pqn->q && pqn->q->properties.queue_id == qid) ||
37 		    (pqn->kq && pqn->kq->queue->properties.queue_id == qid))
38 			return pqn;
39 	}
40 
41 	return NULL;
42 }
43 
44 static int find_available_queue_slot(struct process_queue_manager *pqm,
45 					unsigned int *qid)
46 {
47 	unsigned long found;
48 
49 	found = find_first_zero_bit(pqm->queue_slot_bitmap,
50 			KFD_MAX_NUM_OF_QUEUES_PER_PROCESS);
51 
52 	pr_debug("The new slot id %lu\n", found);
53 
54 	if (found >= KFD_MAX_NUM_OF_QUEUES_PER_PROCESS) {
55 		pr_info("Cannot open more queues for process with pasid %d\n",
56 				pqm->process->pasid);
57 		return -ENOMEM;
58 	}
59 
60 	set_bit(found, pqm->queue_slot_bitmap);
61 	*qid = found;
62 
63 	return 0;
64 }
65 
66 void kfd_process_dequeue_from_device(struct kfd_process_device *pdd)
67 {
68 	struct kfd_dev *dev = pdd->dev;
69 
70 	if (pdd->already_dequeued)
71 		return;
72 
73 	dev->dqm->ops.process_termination(dev->dqm, &pdd->qpd);
74 	pdd->already_dequeued = true;
75 }
76 
77 void kfd_process_dequeue_from_all_devices(struct kfd_process *p)
78 {
79 	struct kfd_process_device *pdd;
80 
81 	list_for_each_entry(pdd, &p->per_device_data, per_device_list)
82 		kfd_process_dequeue_from_device(pdd);
83 }
84 
85 int pqm_init(struct process_queue_manager *pqm, struct kfd_process *p)
86 {
87 	INIT_LIST_HEAD(&pqm->queues);
88 	pqm->queue_slot_bitmap =
89 			kzalloc(DIV_ROUND_UP(KFD_MAX_NUM_OF_QUEUES_PER_PROCESS,
90 					BITS_PER_BYTE), GFP_KERNEL);
91 	if (!pqm->queue_slot_bitmap)
92 		return -ENOMEM;
93 	pqm->process = p;
94 
95 	return 0;
96 }
97 
98 void pqm_uninit(struct process_queue_manager *pqm)
99 {
100 	struct process_queue_node *pqn, *next;
101 
102 	list_for_each_entry_safe(pqn, next, &pqm->queues, process_queue_list) {
103 		uninit_queue(pqn->q);
104 		list_del(&pqn->process_queue_list);
105 		kfree(pqn);
106 	}
107 
108 	kfree(pqm->queue_slot_bitmap);
109 	pqm->queue_slot_bitmap = NULL;
110 }
111 
112 static int create_cp_queue(struct process_queue_manager *pqm,
113 				struct kfd_dev *dev, struct queue **q,
114 				struct queue_properties *q_properties,
115 				struct file *f, unsigned int qid)
116 {
117 	int retval;
118 
119 	/* Doorbell initialized in user space*/
120 	q_properties->doorbell_ptr = NULL;
121 
122 	/* let DQM handle it*/
123 	q_properties->vmid = 0;
124 	q_properties->queue_id = qid;
125 
126 	retval = init_queue(q, q_properties);
127 	if (retval != 0)
128 		return retval;
129 
130 	(*q)->device = dev;
131 	(*q)->process = pqm->process;
132 
133 	pr_debug("PQM After init queue");
134 
135 	return retval;
136 }
137 
138 int pqm_create_queue(struct process_queue_manager *pqm,
139 			    struct kfd_dev *dev,
140 			    struct file *f,
141 			    struct queue_properties *properties,
142 			    unsigned int *qid)
143 {
144 	int retval;
145 	struct kfd_process_device *pdd;
146 	struct queue *q;
147 	struct process_queue_node *pqn;
148 	struct kernel_queue *kq;
149 	enum kfd_queue_type type = properties->type;
150 	unsigned int max_queues = 127; /* HWS limit */
151 
152 	q = NULL;
153 	kq = NULL;
154 
155 	pdd = kfd_get_process_device_data(dev, pqm->process);
156 	if (!pdd) {
157 		pr_err("Process device data doesn't exist\n");
158 		return -1;
159 	}
160 
161 	/*
162 	 * for debug process, verify that it is within the static queues limit
163 	 * currently limit is set to half of the total avail HQD slots
164 	 * If we are just about to create DIQ, the is_debug flag is not set yet
165 	 * Hence we also check the type as well
166 	 */
167 	if ((pdd->qpd.is_debug) || (type == KFD_QUEUE_TYPE_DIQ))
168 		max_queues = dev->device_info->max_no_of_hqd/2;
169 
170 	if (pdd->qpd.queue_count >= max_queues)
171 		return -ENOSPC;
172 
173 	retval = find_available_queue_slot(pqm, qid);
174 	if (retval != 0)
175 		return retval;
176 
177 	if (list_empty(&pdd->qpd.queues_list) &&
178 	    list_empty(&pdd->qpd.priv_queue_list))
179 		dev->dqm->ops.register_process(dev->dqm, &pdd->qpd);
180 
181 	pqn = kzalloc(sizeof(*pqn), GFP_KERNEL);
182 	if (!pqn) {
183 		retval = -ENOMEM;
184 		goto err_allocate_pqn;
185 	}
186 
187 	switch (type) {
188 	case KFD_QUEUE_TYPE_SDMA:
189 		if (dev->dqm->queue_count >= get_num_sdma_queues(dev->dqm)) {
190 			pr_err("Over-subscription is not allowed for SDMA.\n");
191 			retval = -EPERM;
192 			goto err_create_queue;
193 		}
194 
195 		retval = create_cp_queue(pqm, dev, &q, properties, f, *qid);
196 		if (retval != 0)
197 			goto err_create_queue;
198 		pqn->q = q;
199 		pqn->kq = NULL;
200 		retval = dev->dqm->ops.create_queue(dev->dqm, q, &pdd->qpd);
201 		pr_debug("DQM returned %d for create_queue\n", retval);
202 		print_queue(q);
203 		break;
204 
205 	case KFD_QUEUE_TYPE_COMPUTE:
206 		/* check if there is over subscription */
207 		if ((dev->dqm->sched_policy ==
208 		     KFD_SCHED_POLICY_HWS_NO_OVERSUBSCRIPTION) &&
209 		((dev->dqm->processes_count >= dev->vm_info.vmid_num_kfd) ||
210 		(dev->dqm->queue_count >= get_queues_num(dev->dqm)))) {
211 			pr_debug("Over-subscription is not allowed when amdkfd.sched_policy == 1\n");
212 			retval = -EPERM;
213 			goto err_create_queue;
214 		}
215 
216 		retval = create_cp_queue(pqm, dev, &q, properties, f, *qid);
217 		if (retval != 0)
218 			goto err_create_queue;
219 		pqn->q = q;
220 		pqn->kq = NULL;
221 		retval = dev->dqm->ops.create_queue(dev->dqm, q, &pdd->qpd);
222 		pr_debug("DQM returned %d for create_queue\n", retval);
223 		print_queue(q);
224 		break;
225 	case KFD_QUEUE_TYPE_DIQ:
226 		kq = kernel_queue_init(dev, KFD_QUEUE_TYPE_DIQ);
227 		if (!kq) {
228 			retval = -ENOMEM;
229 			goto err_create_queue;
230 		}
231 		kq->queue->properties.queue_id = *qid;
232 		pqn->kq = kq;
233 		pqn->q = NULL;
234 		retval = dev->dqm->ops.create_kernel_queue(dev->dqm,
235 							kq, &pdd->qpd);
236 		break;
237 	default:
238 		WARN(1, "Invalid queue type %d", type);
239 		retval = -EINVAL;
240 	}
241 
242 	if (retval != 0) {
243 		pr_err("Pasid %d DQM create queue %d failed. ret %d\n",
244 			pqm->process->pasid, type, retval);
245 		goto err_create_queue;
246 	}
247 
248 	if (q)
249 		/* Return the doorbell offset within the doorbell page
250 		 * to the caller so it can be passed up to user mode
251 		 * (in bytes).
252 		 */
253 		properties->doorbell_off =
254 			(q->properties.doorbell_off * sizeof(uint32_t)) &
255 			(kfd_doorbell_process_slice(dev) - 1);
256 
257 	pr_debug("PQM After DQM create queue\n");
258 
259 	list_add(&pqn->process_queue_list, &pqm->queues);
260 
261 	if (q) {
262 		pr_debug("PQM done creating queue\n");
263 		print_queue_properties(&q->properties);
264 	}
265 
266 	return retval;
267 
268 err_create_queue:
269 	kfree(pqn);
270 err_allocate_pqn:
271 	/* check if queues list is empty unregister process from device */
272 	clear_bit(*qid, pqm->queue_slot_bitmap);
273 	if (list_empty(&pdd->qpd.queues_list) &&
274 	    list_empty(&pdd->qpd.priv_queue_list))
275 		dev->dqm->ops.unregister_process(dev->dqm, &pdd->qpd);
276 	return retval;
277 }
278 
279 int pqm_destroy_queue(struct process_queue_manager *pqm, unsigned int qid)
280 {
281 	struct process_queue_node *pqn;
282 	struct kfd_process_device *pdd;
283 	struct device_queue_manager *dqm;
284 	struct kfd_dev *dev;
285 	int retval;
286 
287 	dqm = NULL;
288 
289 	retval = 0;
290 
291 	pqn = get_queue_by_qid(pqm, qid);
292 	if (!pqn) {
293 		pr_err("Queue id does not match any known queue\n");
294 		return -EINVAL;
295 	}
296 
297 	dev = NULL;
298 	if (pqn->kq)
299 		dev = pqn->kq->dev;
300 	if (pqn->q)
301 		dev = pqn->q->device;
302 	if (WARN_ON(!dev))
303 		return -ENODEV;
304 
305 	pdd = kfd_get_process_device_data(dev, pqm->process);
306 	if (!pdd) {
307 		pr_err("Process device data doesn't exist\n");
308 		return -1;
309 	}
310 
311 	if (pqn->kq) {
312 		/* destroy kernel queue (DIQ) */
313 		dqm = pqn->kq->dev->dqm;
314 		dqm->ops.destroy_kernel_queue(dqm, pqn->kq, &pdd->qpd);
315 		kernel_queue_uninit(pqn->kq);
316 	}
317 
318 	if (pqn->q) {
319 		dqm = pqn->q->device->dqm;
320 		retval = dqm->ops.destroy_queue(dqm, &pdd->qpd, pqn->q);
321 		if (retval) {
322 			pr_err("Pasid %d destroy queue %d failed, ret %d\n",
323 				pqm->process->pasid,
324 				pqn->q->properties.queue_id, retval);
325 			if (retval != -ETIME)
326 				goto err_destroy_queue;
327 		}
328 		kfree(pqn->q->properties.cu_mask);
329 		pqn->q->properties.cu_mask = NULL;
330 		uninit_queue(pqn->q);
331 	}
332 
333 	list_del(&pqn->process_queue_list);
334 	kfree(pqn);
335 	clear_bit(qid, pqm->queue_slot_bitmap);
336 
337 	if (list_empty(&pdd->qpd.queues_list) &&
338 	    list_empty(&pdd->qpd.priv_queue_list))
339 		dqm->ops.unregister_process(dqm, &pdd->qpd);
340 
341 err_destroy_queue:
342 	return retval;
343 }
344 
345 int pqm_update_queue(struct process_queue_manager *pqm, unsigned int qid,
346 			struct queue_properties *p)
347 {
348 	int retval;
349 	struct process_queue_node *pqn;
350 
351 	pqn = get_queue_by_qid(pqm, qid);
352 	if (!pqn) {
353 		pr_debug("No queue %d exists for update operation\n", qid);
354 		return -EFAULT;
355 	}
356 
357 	pqn->q->properties.queue_address = p->queue_address;
358 	pqn->q->properties.queue_size = p->queue_size;
359 	pqn->q->properties.queue_percent = p->queue_percent;
360 	pqn->q->properties.priority = p->priority;
361 
362 	retval = pqn->q->device->dqm->ops.update_queue(pqn->q->device->dqm,
363 							pqn->q);
364 	if (retval != 0)
365 		return retval;
366 
367 	return 0;
368 }
369 
370 int pqm_set_cu_mask(struct process_queue_manager *pqm, unsigned int qid,
371 			struct queue_properties *p)
372 {
373 	int retval;
374 	struct process_queue_node *pqn;
375 
376 	pqn = get_queue_by_qid(pqm, qid);
377 	if (!pqn) {
378 		pr_debug("No queue %d exists for update operation\n", qid);
379 		return -EFAULT;
380 	}
381 
382 	/* Free the old CU mask memory if it is already allocated, then
383 	 * allocate memory for the new CU mask.
384 	 */
385 	kfree(pqn->q->properties.cu_mask);
386 
387 	pqn->q->properties.cu_mask_count = p->cu_mask_count;
388 	pqn->q->properties.cu_mask = p->cu_mask;
389 
390 	retval = pqn->q->device->dqm->ops.update_queue(pqn->q->device->dqm,
391 							pqn->q);
392 	if (retval != 0)
393 		return retval;
394 
395 	return 0;
396 }
397 
398 struct kernel_queue *pqm_get_kernel_queue(
399 					struct process_queue_manager *pqm,
400 					unsigned int qid)
401 {
402 	struct process_queue_node *pqn;
403 
404 	pqn = get_queue_by_qid(pqm, qid);
405 	if (pqn && pqn->kq)
406 		return pqn->kq;
407 
408 	return NULL;
409 }
410 
411 #if defined(CONFIG_DEBUG_FS)
412 
413 int pqm_debugfs_mqds(struct seq_file *m, void *data)
414 {
415 	struct process_queue_manager *pqm = data;
416 	struct process_queue_node *pqn;
417 	struct queue *q;
418 	enum KFD_MQD_TYPE mqd_type;
419 	struct mqd_manager *mqd_mgr;
420 	int r = 0;
421 
422 	list_for_each_entry(pqn, &pqm->queues, process_queue_list) {
423 		if (pqn->q) {
424 			q = pqn->q;
425 			switch (q->properties.type) {
426 			case KFD_QUEUE_TYPE_SDMA:
427 				seq_printf(m, "  SDMA queue on device %x\n",
428 					   q->device->id);
429 				mqd_type = KFD_MQD_TYPE_SDMA;
430 				break;
431 			case KFD_QUEUE_TYPE_COMPUTE:
432 				seq_printf(m, "  Compute queue on device %x\n",
433 					   q->device->id);
434 				mqd_type = KFD_MQD_TYPE_CP;
435 				break;
436 			default:
437 				seq_printf(m,
438 				"  Bad user queue type %d on device %x\n",
439 					   q->properties.type, q->device->id);
440 				continue;
441 			}
442 			mqd_mgr = q->device->dqm->ops.get_mqd_manager(
443 				q->device->dqm, mqd_type);
444 		} else if (pqn->kq) {
445 			q = pqn->kq->queue;
446 			mqd_mgr = pqn->kq->mqd_mgr;
447 			switch (q->properties.type) {
448 			case KFD_QUEUE_TYPE_DIQ:
449 				seq_printf(m, "  DIQ on device %x\n",
450 					   pqn->kq->dev->id);
451 				mqd_type = KFD_MQD_TYPE_HIQ;
452 				break;
453 			default:
454 				seq_printf(m,
455 				"  Bad kernel queue type %d on device %x\n",
456 					   q->properties.type,
457 					   pqn->kq->dev->id);
458 				continue;
459 			}
460 		} else {
461 			seq_printf(m,
462 		"  Weird: Queue node with neither kernel nor user queue\n");
463 			continue;
464 		}
465 
466 		r = mqd_mgr->debugfs_show_mqd(m, q->mqd);
467 		if (r != 0)
468 			break;
469 	}
470 
471 	return r;
472 }
473 
474 #endif
475