xref: /openbmc/linux/drivers/firmware/tegra/bpmp.c (revision 835fd614) 
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Copyright (c) 2016, NVIDIA CORPORATION.  All rights reserved.
4  */
5 
6 #include <linux/clk/tegra.h>
7 #include <linux/genalloc.h>
8 #include <linux/mailbox_client.h>
9 #include <linux/module.h>
10 #include <linux/of.h>
11 #include <linux/of_address.h>
12 #include <linux/of_device.h>
13 #include <linux/platform_device.h>
14 #include <linux/pm.h>
15 #include <linux/semaphore.h>
16 #include <linux/sched/clock.h>
17 
18 #include <soc/tegra/bpmp.h>
19 #include <soc/tegra/bpmp-abi.h>
20 #include <soc/tegra/ivc.h>
21 
22 #include "bpmp-private.h"
23 
24 #define MSG_ACK		BIT(0)
25 #define MSG_RING	BIT(1)
26 #define TAG_SZ		32
27 
28 static inline struct tegra_bpmp *
29 mbox_client_to_bpmp(struct mbox_client *client)
30 {
31 	return container_of(client, struct tegra_bpmp, mbox.client);
32 }
33 
34 static inline const struct tegra_bpmp_ops *
35 channel_to_ops(struct tegra_bpmp_channel *channel)
36 {
37 	struct tegra_bpmp *bpmp = channel->bpmp;
38 
39 	return bpmp->soc->ops;
40 }
41 
42 struct tegra_bpmp *tegra_bpmp_get(struct device *dev)
43 {
44 	struct platform_device *pdev;
45 	struct tegra_bpmp *bpmp;
46 	struct device_node *np;
47 
48 	np = of_parse_phandle(dev->of_node, "nvidia,bpmp", 0);
49 	if (!np)
50 		return ERR_PTR(-ENOENT);
51 
52 	pdev = of_find_device_by_node(np);
53 	if (!pdev) {
54 		bpmp = ERR_PTR(-ENODEV);
55 		goto put;
56 	}
57 
58 	bpmp = platform_get_drvdata(pdev);
59 	if (!bpmp) {
60 		bpmp = ERR_PTR(-EPROBE_DEFER);
61 		put_device(&pdev->dev);
62 		goto put;
63 	}
64 
65 put:
66 	of_node_put(np);
67 	return bpmp;
68 }
69 EXPORT_SYMBOL_GPL(tegra_bpmp_get);
70 
71 void tegra_bpmp_put(struct tegra_bpmp *bpmp)
72 {
73 	if (bpmp)
74 		put_device(bpmp->dev);
75 }
76 EXPORT_SYMBOL_GPL(tegra_bpmp_put);
77 
78 static int
79 tegra_bpmp_channel_get_thread_index(struct tegra_bpmp_channel *channel)
80 {
81 	struct tegra_bpmp *bpmp = channel->bpmp;
82 	unsigned int count;
83 	int index;
84 
85 	count = bpmp->soc->channels.thread.count;
86 
87 	index = channel - channel->bpmp->threaded_channels;
88 	if (index < 0 || index >= count)
89 		return -EINVAL;
90 
91 	return index;
92 }
93 
94 static bool tegra_bpmp_message_valid(const struct tegra_bpmp_message *msg)
95 {
96 	return (msg->tx.size <= MSG_DATA_MIN_SZ) &&
97 	       (msg->rx.size <= MSG_DATA_MIN_SZ) &&
98 	       (msg->tx.size == 0 || msg->tx.data) &&
99 	       (msg->rx.size == 0 || msg->rx.data);
100 }
101 
102 static bool tegra_bpmp_is_response_ready(struct tegra_bpmp_channel *channel)
103 {
104 	const struct tegra_bpmp_ops *ops = channel_to_ops(channel);
105 
106 	return ops->is_response_ready(channel);
107 }
108 
109 static bool tegra_bpmp_is_request_ready(struct tegra_bpmp_channel *channel)
110 {
111 	const struct tegra_bpmp_ops *ops = channel_to_ops(channel);
112 
113 	return ops->is_request_ready(channel);
114 }
115 
116 static int tegra_bpmp_wait_response(struct tegra_bpmp_channel *channel)
117 {
118 	unsigned long timeout = channel->bpmp->soc->channels.cpu_tx.timeout;
119 	ktime_t end;
120 
121 	end = ktime_add_us(ktime_get(), timeout);
122 
123 	do {
124 		if (tegra_bpmp_is_response_ready(channel))
125 			return 0;
126 	} while (ktime_before(ktime_get(), end));
127 
128 	return -ETIMEDOUT;
129 }
130 
131 static int tegra_bpmp_ack_response(struct tegra_bpmp_channel *channel)
132 {
133 	const struct tegra_bpmp_ops *ops = channel_to_ops(channel);
134 
135 	return ops->ack_response(channel);
136 }
137 
138 static int tegra_bpmp_ack_request(struct tegra_bpmp_channel *channel)
139 {
140 	const struct tegra_bpmp_ops *ops = channel_to_ops(channel);
141 
142 	return ops->ack_request(channel);
143 }
144 
145 static bool
146 tegra_bpmp_is_request_channel_free(struct tegra_bpmp_channel *channel)
147 {
148 	const struct tegra_bpmp_ops *ops = channel_to_ops(channel);
149 
150 	return ops->is_request_channel_free(channel);
151 }
152 
153 static bool
154 tegra_bpmp_is_response_channel_free(struct tegra_bpmp_channel *channel)
155 {
156 	const struct tegra_bpmp_ops *ops = channel_to_ops(channel);
157 
158 	return ops->is_response_channel_free(channel);
159 }
160 
161 static int
162 tegra_bpmp_wait_request_channel_free(struct tegra_bpmp_channel *channel)
163 {
164 	unsigned long timeout = channel->bpmp->soc->channels.cpu_tx.timeout;
165 	ktime_t start, now;
166 
167 	start = ns_to_ktime(local_clock());
168 
169 	do {
170 		if (tegra_bpmp_is_request_channel_free(channel))
171 			return 0;
172 
173 		now = ns_to_ktime(local_clock());
174 	} while (ktime_us_delta(now, start) < timeout);
175 
176 	return -ETIMEDOUT;
177 }
178 
179 static int tegra_bpmp_post_request(struct tegra_bpmp_channel *channel)
180 {
181 	const struct tegra_bpmp_ops *ops = channel_to_ops(channel);
182 
183 	return ops->post_request(channel);
184 }
185 
186 static int tegra_bpmp_post_response(struct tegra_bpmp_channel *channel)
187 {
188 	const struct tegra_bpmp_ops *ops = channel_to_ops(channel);
189 
190 	return ops->post_response(channel);
191 }
192 
193 static int tegra_bpmp_ring_doorbell(struct tegra_bpmp *bpmp)
194 {
195 	return bpmp->soc->ops->ring_doorbell(bpmp);
196 }
197 
198 static ssize_t __tegra_bpmp_channel_read(struct tegra_bpmp_channel *channel,
199 					 void *data, size_t size, int *ret)
200 {
201 	int err;
202 
203 	if (data && size > 0)
204 		memcpy(data, channel->ib->data, size);
205 
206 	err = tegra_bpmp_ack_response(channel);
207 	if (err < 0)
208 		return err;
209 
210 	*ret = channel->ib->code;
211 
212 	return 0;
213 }
214 
215 static ssize_t tegra_bpmp_channel_read(struct tegra_bpmp_channel *channel,
216 				       void *data, size_t size, int *ret)
217 {
218 	struct tegra_bpmp *bpmp = channel->bpmp;
219 	unsigned long flags;
220 	ssize_t err;
221 	int index;
222 
223 	index = tegra_bpmp_channel_get_thread_index(channel);
224 	if (index < 0) {
225 		err = index;
226 		goto unlock;
227 	}
228 
229 	spin_lock_irqsave(&bpmp->lock, flags);
230 	err = __tegra_bpmp_channel_read(channel, data, size, ret);
231 	clear_bit(index, bpmp->threaded.allocated);
232 	spin_unlock_irqrestore(&bpmp->lock, flags);
233 
234 unlock:
235 	up(&bpmp->threaded.lock);
236 
237 	return err;
238 }
239 
240 static ssize_t __tegra_bpmp_channel_write(struct tegra_bpmp_channel *channel,
241 					  unsigned int mrq, unsigned long flags,
242 					  const void *data, size_t size)
243 {
244 	channel->ob->code = mrq;
245 	channel->ob->flags = flags;
246 
247 	if (data && size > 0)
248 		memcpy(channel->ob->data, data, size);
249 
250 	return tegra_bpmp_post_request(channel);
251 }
252 
253 static struct tegra_bpmp_channel *
254 tegra_bpmp_write_threaded(struct tegra_bpmp *bpmp, unsigned int mrq,
255 			  const void *data, size_t size)
256 {
257 	unsigned long timeout = bpmp->soc->channels.thread.timeout;
258 	unsigned int count = bpmp->soc->channels.thread.count;
259 	struct tegra_bpmp_channel *channel;
260 	unsigned long flags;
261 	unsigned int index;
262 	int err;
263 
264 	err = down_timeout(&bpmp->threaded.lock, usecs_to_jiffies(timeout));
265 	if (err < 0)
266 		return ERR_PTR(err);
267 
268 	spin_lock_irqsave(&bpmp->lock, flags);
269 
270 	index = find_first_zero_bit(bpmp->threaded.allocated, count);
271 	if (index == count) {
272 		err = -EBUSY;
273 		goto unlock;
274 	}
275 
276 	channel = &bpmp->threaded_channels[index];
277 
278 	if (!tegra_bpmp_is_request_channel_free(channel)) {
279 		err = -EBUSY;
280 		goto unlock;
281 	}
282 
283 	set_bit(index, bpmp->threaded.allocated);
284 
285 	err = __tegra_bpmp_channel_write(channel, mrq, MSG_ACK | MSG_RING,
286 					 data, size);
287 	if (err < 0)
288 		goto clear_allocated;
289 
290 	set_bit(index, bpmp->threaded.busy);
291 
292 	spin_unlock_irqrestore(&bpmp->lock, flags);
293 	return channel;
294 
295 clear_allocated:
296 	clear_bit(index, bpmp->threaded.allocated);
297 unlock:
298 	spin_unlock_irqrestore(&bpmp->lock, flags);
299 	up(&bpmp->threaded.lock);
300 
301 	return ERR_PTR(err);
302 }
303 
304 static ssize_t tegra_bpmp_channel_write(struct tegra_bpmp_channel *channel,
305 					unsigned int mrq, unsigned long flags,
306 					const void *data, size_t size)
307 {
308 	int err;
309 
310 	err = tegra_bpmp_wait_request_channel_free(channel);
311 	if (err < 0)
312 		return err;
313 
314 	return __tegra_bpmp_channel_write(channel, mrq, flags, data, size);
315 }
316 
317 int tegra_bpmp_transfer_atomic(struct tegra_bpmp *bpmp,
318 			       struct tegra_bpmp_message *msg)
319 {
320 	struct tegra_bpmp_channel *channel;
321 	int err;
322 
323 	if (WARN_ON(!irqs_disabled()))
324 		return -EPERM;
325 
326 	if (!tegra_bpmp_message_valid(msg))
327 		return -EINVAL;
328 
329 	channel = bpmp->tx_channel;
330 
331 	spin_lock(&bpmp->atomic_tx_lock);
332 
333 	err = tegra_bpmp_channel_write(channel, msg->mrq, MSG_ACK,
334 				       msg->tx.data, msg->tx.size);
335 	if (err < 0) {
336 		spin_unlock(&bpmp->atomic_tx_lock);
337 		return err;
338 	}
339 
340 	spin_unlock(&bpmp->atomic_tx_lock);
341 
342 	err = tegra_bpmp_ring_doorbell(bpmp);
343 	if (err < 0)
344 		return err;
345 
346 	err = tegra_bpmp_wait_response(channel);
347 	if (err < 0)
348 		return err;
349 
350 	return __tegra_bpmp_channel_read(channel, msg->rx.data, msg->rx.size,
351 					 &msg->rx.ret);
352 }
353 EXPORT_SYMBOL_GPL(tegra_bpmp_transfer_atomic);
354 
355 int tegra_bpmp_transfer(struct tegra_bpmp *bpmp,
356 			struct tegra_bpmp_message *msg)
357 {
358 	struct tegra_bpmp_channel *channel;
359 	unsigned long timeout;
360 	int err;
361 
362 	if (WARN_ON(irqs_disabled()))
363 		return -EPERM;
364 
365 	if (!tegra_bpmp_message_valid(msg))
366 		return -EINVAL;
367 
368 	channel = tegra_bpmp_write_threaded(bpmp, msg->mrq, msg->tx.data,
369 					    msg->tx.size);
370 	if (IS_ERR(channel))
371 		return PTR_ERR(channel);
372 
373 	err = tegra_bpmp_ring_doorbell(bpmp);
374 	if (err < 0)
375 		return err;
376 
377 	timeout = usecs_to_jiffies(bpmp->soc->channels.thread.timeout);
378 
379 	err = wait_for_completion_timeout(&channel->completion, timeout);
380 	if (err == 0)
381 		return -ETIMEDOUT;
382 
383 	return tegra_bpmp_channel_read(channel, msg->rx.data, msg->rx.size,
384 				       &msg->rx.ret);
385 }
386 EXPORT_SYMBOL_GPL(tegra_bpmp_transfer);
387 
388 static struct tegra_bpmp_mrq *tegra_bpmp_find_mrq(struct tegra_bpmp *bpmp,
389 						  unsigned int mrq)
390 {
391 	struct tegra_bpmp_mrq *entry;
392 
393 	list_for_each_entry(entry, &bpmp->mrqs, list)
394 		if (entry->mrq == mrq)
395 			return entry;
396 
397 	return NULL;
398 }
399 
400 void tegra_bpmp_mrq_return(struct tegra_bpmp_channel *channel, int code,
401 			   const void *data, size_t size)
402 {
403 	unsigned long flags = channel->ib->flags;
404 	struct tegra_bpmp *bpmp = channel->bpmp;
405 	int err;
406 
407 	if (WARN_ON(size > MSG_DATA_MIN_SZ))
408 		return;
409 
410 	err = tegra_bpmp_ack_request(channel);
411 	if (WARN_ON(err < 0))
412 		return;
413 
414 	if ((flags & MSG_ACK) == 0)
415 		return;
416 
417 	if (WARN_ON(!tegra_bpmp_is_response_channel_free(channel)))
418 		return;
419 
420 	channel->ob->code = code;
421 
422 	if (data && size > 0)
423 		memcpy(channel->ob->data, data, size);
424 
425 	err = tegra_bpmp_post_response(channel);
426 	if (WARN_ON(err < 0))
427 		return;
428 
429 	if (flags & MSG_RING) {
430 		err = tegra_bpmp_ring_doorbell(bpmp);
431 		if (WARN_ON(err < 0))
432 			return;
433 	}
434 }
435 EXPORT_SYMBOL_GPL(tegra_bpmp_mrq_return);
436 
437 static void tegra_bpmp_handle_mrq(struct tegra_bpmp *bpmp,
438 				  unsigned int mrq,
439 				  struct tegra_bpmp_channel *channel)
440 {
441 	struct tegra_bpmp_mrq *entry;
442 	u32 zero = 0;
443 
444 	spin_lock(&bpmp->lock);
445 
446 	entry = tegra_bpmp_find_mrq(bpmp, mrq);
447 	if (!entry) {
448 		spin_unlock(&bpmp->lock);
449 		tegra_bpmp_mrq_return(channel, -EINVAL, &zero, sizeof(zero));
450 		return;
451 	}
452 
453 	entry->handler(mrq, channel, entry->data);
454 
455 	spin_unlock(&bpmp->lock);
456 }
457 
458 int tegra_bpmp_request_mrq(struct tegra_bpmp *bpmp, unsigned int mrq,
459 			   tegra_bpmp_mrq_handler_t handler, void *data)
460 {
461 	struct tegra_bpmp_mrq *entry;
462 	unsigned long flags;
463 
464 	if (!handler)
465 		return -EINVAL;
466 
467 	entry = devm_kzalloc(bpmp->dev, sizeof(*entry), GFP_KERNEL);
468 	if (!entry)
469 		return -ENOMEM;
470 
471 	spin_lock_irqsave(&bpmp->lock, flags);
472 
473 	entry->mrq = mrq;
474 	entry->handler = handler;
475 	entry->data = data;
476 	list_add(&entry->list, &bpmp->mrqs);
477 
478 	spin_unlock_irqrestore(&bpmp->lock, flags);
479 
480 	return 0;
481 }
482 EXPORT_SYMBOL_GPL(tegra_bpmp_request_mrq);
483 
484 void tegra_bpmp_free_mrq(struct tegra_bpmp *bpmp, unsigned int mrq, void *data)
485 {
486 	struct tegra_bpmp_mrq *entry;
487 	unsigned long flags;
488 
489 	spin_lock_irqsave(&bpmp->lock, flags);
490 
491 	entry = tegra_bpmp_find_mrq(bpmp, mrq);
492 	if (!entry)
493 		goto unlock;
494 
495 	list_del(&entry->list);
496 	devm_kfree(bpmp->dev, entry);
497 
498 unlock:
499 	spin_unlock_irqrestore(&bpmp->lock, flags);
500 }
501 EXPORT_SYMBOL_GPL(tegra_bpmp_free_mrq);
502 
503 bool tegra_bpmp_mrq_is_supported(struct tegra_bpmp *bpmp, unsigned int mrq)
504 {
505 	struct mrq_query_abi_request req = { .mrq = cpu_to_le32(mrq) };
506 	struct mrq_query_abi_response resp;
507 	struct tegra_bpmp_message msg = {
508 		.mrq = MRQ_QUERY_ABI,
509 		.tx = {
510 			.data = &req,
511 			.size = sizeof(req),
512 		},
513 		.rx = {
514 			.data = &resp,
515 			.size = sizeof(resp),
516 		},
517 	};
518 	int err;
519 
520 	err = tegra_bpmp_transfer(bpmp, &msg);
521 	if (err || msg.rx.ret)
522 		return false;
523 
524 	return resp.status == 0;
525 }
526 EXPORT_SYMBOL_GPL(tegra_bpmp_mrq_is_supported);
527 
528 static void tegra_bpmp_mrq_handle_ping(unsigned int mrq,
529 				       struct tegra_bpmp_channel *channel,
530 				       void *data)
531 {
532 	struct mrq_ping_request *request;
533 	struct mrq_ping_response response;
534 
535 	request = (struct mrq_ping_request *)channel->ib->data;
536 
537 	memset(&response, 0, sizeof(response));
538 	response.reply = request->challenge << 1;
539 
540 	tegra_bpmp_mrq_return(channel, 0, &response, sizeof(response));
541 }
542 
543 static int tegra_bpmp_ping(struct tegra_bpmp *bpmp)
544 {
545 	struct mrq_ping_response response;
546 	struct mrq_ping_request request;
547 	struct tegra_bpmp_message msg;
548 	unsigned long flags;
549 	ktime_t start, end;
550 	int err;
551 
552 	memset(&request, 0, sizeof(request));
553 	request.challenge = 1;
554 
555 	memset(&response, 0, sizeof(response));
556 
557 	memset(&msg, 0, sizeof(msg));
558 	msg.mrq = MRQ_PING;
559 	msg.tx.data = &request;
560 	msg.tx.size = sizeof(request);
561 	msg.rx.data = &response;
562 	msg.rx.size = sizeof(response);
563 
564 	local_irq_save(flags);
565 	start = ktime_get();
566 	err = tegra_bpmp_transfer_atomic(bpmp, &msg);
567 	end = ktime_get();
568 	local_irq_restore(flags);
569 
570 	if (!err)
571 		dev_dbg(bpmp->dev,
572 			"ping ok: challenge: %u, response: %u, time: %lld\n",
573 			request.challenge, response.reply,
574 			ktime_to_us(ktime_sub(end, start)));
575 
576 	return err;
577 }
578 
579 /* deprecated version of tag query */
580 static int tegra_bpmp_get_firmware_tag_old(struct tegra_bpmp *bpmp, char *tag,
581 					   size_t size)
582 {
583 	struct mrq_query_tag_request request;
584 	struct tegra_bpmp_message msg;
585 	unsigned long flags;
586 	dma_addr_t phys;
587 	void *virt;
588 	int err;
589 
590 	if (size != TAG_SZ)
591 		return -EINVAL;
592 
593 	virt = dma_alloc_coherent(bpmp->dev, TAG_SZ, &phys,
594 				  GFP_KERNEL | GFP_DMA32);
595 	if (!virt)
596 		return -ENOMEM;
597 
598 	memset(&request, 0, sizeof(request));
599 	request.addr = phys;
600 
601 	memset(&msg, 0, sizeof(msg));
602 	msg.mrq = MRQ_QUERY_TAG;
603 	msg.tx.data = &request;
604 	msg.tx.size = sizeof(request);
605 
606 	local_irq_save(flags);
607 	err = tegra_bpmp_transfer_atomic(bpmp, &msg);
608 	local_irq_restore(flags);
609 
610 	if (err == 0)
611 		memcpy(tag, virt, TAG_SZ);
612 
613 	dma_free_coherent(bpmp->dev, TAG_SZ, virt, phys);
614 
615 	return err;
616 }
617 
618 static int tegra_bpmp_get_firmware_tag(struct tegra_bpmp *bpmp, char *tag,
619 				       size_t size)
620 {
621 	if (tegra_bpmp_mrq_is_supported(bpmp, MRQ_QUERY_FW_TAG)) {
622 		struct mrq_query_fw_tag_response resp;
623 		struct tegra_bpmp_message msg = {
624 			.mrq = MRQ_QUERY_FW_TAG,
625 			.rx = {
626 				.data = &resp,
627 				.size = sizeof(resp),
628 			},
629 		};
630 		int err;
631 
632 		if (size != sizeof(resp.tag))
633 			return -EINVAL;
634 
635 		err = tegra_bpmp_transfer(bpmp, &msg);
636 
637 		if (err)
638 			return err;
639 		if (msg.rx.ret < 0)
640 			return -EINVAL;
641 
642 		memcpy(tag, resp.tag, sizeof(resp.tag));
643 		return 0;
644 	}
645 
646 	return tegra_bpmp_get_firmware_tag_old(bpmp, tag, size);
647 }
648 
649 static void tegra_bpmp_channel_signal(struct tegra_bpmp_channel *channel)
650 {
651 	unsigned long flags = channel->ob->flags;
652 
653 	if ((flags & MSG_RING) == 0)
654 		return;
655 
656 	complete(&channel->completion);
657 }
658 
659 void tegra_bpmp_handle_rx(struct tegra_bpmp *bpmp)
660 {
661 	struct tegra_bpmp_channel *channel;
662 	unsigned int i, count;
663 	unsigned long *busy;
664 
665 	channel = bpmp->rx_channel;
666 	count = bpmp->soc->channels.thread.count;
667 	busy = bpmp->threaded.busy;
668 
669 	if (tegra_bpmp_is_request_ready(channel))
670 		tegra_bpmp_handle_mrq(bpmp, channel->ib->code, channel);
671 
672 	spin_lock(&bpmp->lock);
673 
674 	for_each_set_bit(i, busy, count) {
675 		struct tegra_bpmp_channel *channel;
676 
677 		channel = &bpmp->threaded_channels[i];
678 
679 		if (tegra_bpmp_is_response_ready(channel)) {
680 			tegra_bpmp_channel_signal(channel);
681 			clear_bit(i, busy);
682 		}
683 	}
684 
685 	spin_unlock(&bpmp->lock);
686 }
687 
688 static int tegra_bpmp_probe(struct platform_device *pdev)
689 {
690 	struct tegra_bpmp *bpmp;
691 	char tag[TAG_SZ];
692 	size_t size;
693 	int err;
694 
695 	bpmp = devm_kzalloc(&pdev->dev, sizeof(*bpmp), GFP_KERNEL);
696 	if (!bpmp)
697 		return -ENOMEM;
698 
699 	bpmp->soc = of_device_get_match_data(&pdev->dev);
700 	bpmp->dev = &pdev->dev;
701 
702 	INIT_LIST_HEAD(&bpmp->mrqs);
703 	spin_lock_init(&bpmp->lock);
704 
705 	bpmp->threaded.count = bpmp->soc->channels.thread.count;
706 	sema_init(&bpmp->threaded.lock, bpmp->threaded.count);
707 
708 	size = BITS_TO_LONGS(bpmp->threaded.count) * sizeof(long);
709 
710 	bpmp->threaded.allocated = devm_kzalloc(&pdev->dev, size, GFP_KERNEL);
711 	if (!bpmp->threaded.allocated)
712 		return -ENOMEM;
713 
714 	bpmp->threaded.busy = devm_kzalloc(&pdev->dev, size, GFP_KERNEL);
715 	if (!bpmp->threaded.busy)
716 		return -ENOMEM;
717 
718 	spin_lock_init(&bpmp->atomic_tx_lock);
719 	bpmp->tx_channel = devm_kzalloc(&pdev->dev, sizeof(*bpmp->tx_channel),
720 					GFP_KERNEL);
721 	if (!bpmp->tx_channel)
722 		return -ENOMEM;
723 
724 	bpmp->rx_channel = devm_kzalloc(&pdev->dev, sizeof(*bpmp->rx_channel),
725 	                                GFP_KERNEL);
726 	if (!bpmp->rx_channel)
727 		return -ENOMEM;
728 
729 	bpmp->threaded_channels = devm_kcalloc(&pdev->dev, bpmp->threaded.count,
730 					       sizeof(*bpmp->threaded_channels),
731 					       GFP_KERNEL);
732 	if (!bpmp->threaded_channels)
733 		return -ENOMEM;
734 
735 	err = bpmp->soc->ops->init(bpmp);
736 	if (err < 0)
737 		return err;
738 
739 	err = tegra_bpmp_request_mrq(bpmp, MRQ_PING,
740 				     tegra_bpmp_mrq_handle_ping, bpmp);
741 	if (err < 0)
742 		goto deinit;
743 
744 	err = tegra_bpmp_ping(bpmp);
745 	if (err < 0) {
746 		dev_err(&pdev->dev, "failed to ping BPMP: %d\n", err);
747 		goto free_mrq;
748 	}
749 
750 	err = tegra_bpmp_get_firmware_tag(bpmp, tag, sizeof(tag));
751 	if (err < 0) {
752 		dev_err(&pdev->dev, "failed to get firmware tag: %d\n", err);
753 		goto free_mrq;
754 	}
755 
756 	dev_info(&pdev->dev, "firmware: %.*s\n", (int)sizeof(tag), tag);
757 
758 	platform_set_drvdata(pdev, bpmp);
759 
760 	err = of_platform_default_populate(pdev->dev.of_node, NULL, &pdev->dev);
761 	if (err < 0)
762 		goto free_mrq;
763 
764 	if (of_find_property(pdev->dev.of_node, "#clock-cells", NULL)) {
765 		err = tegra_bpmp_init_clocks(bpmp);
766 		if (err < 0)
767 			goto free_mrq;
768 	}
769 
770 	if (of_find_property(pdev->dev.of_node, "#reset-cells", NULL)) {
771 		err = tegra_bpmp_init_resets(bpmp);
772 		if (err < 0)
773 			goto free_mrq;
774 	}
775 
776 	if (of_find_property(pdev->dev.of_node, "#power-domain-cells", NULL)) {
777 		err = tegra_bpmp_init_powergates(bpmp);
778 		if (err < 0)
779 			goto free_mrq;
780 	}
781 
782 	err = tegra_bpmp_init_debugfs(bpmp);
783 	if (err < 0)
784 		dev_err(&pdev->dev, "debugfs initialization failed: %d\n", err);
785 
786 	return 0;
787 
788 free_mrq:
789 	tegra_bpmp_free_mrq(bpmp, MRQ_PING, bpmp);
790 deinit:
791 	if (bpmp->soc->ops->deinit)
792 		bpmp->soc->ops->deinit(bpmp);
793 
794 	return err;
795 }
796 
797 static int __maybe_unused tegra_bpmp_resume(struct device *dev)
798 {
799 	struct tegra_bpmp *bpmp = dev_get_drvdata(dev);
800 
801 	if (bpmp->soc->ops->resume)
802 		return bpmp->soc->ops->resume(bpmp);
803 	else
804 		return 0;
805 }
806 
807 static const struct dev_pm_ops tegra_bpmp_pm_ops = {
808 	.resume_noirq = tegra_bpmp_resume,
809 };
810 
811 #if IS_ENABLED(CONFIG_ARCH_TEGRA_186_SOC) || \
812     IS_ENABLED(CONFIG_ARCH_TEGRA_194_SOC)
813 static const struct tegra_bpmp_soc tegra186_soc = {
814 	.channels = {
815 		.cpu_tx = {
816 			.offset = 3,
817 			.timeout = 60 * USEC_PER_SEC,
818 		},
819 		.thread = {
820 			.offset = 0,
821 			.count = 3,
822 			.timeout = 600 * USEC_PER_SEC,
823 		},
824 		.cpu_rx = {
825 			.offset = 13,
826 			.timeout = 0,
827 		},
828 	},
829 	.ops = &tegra186_bpmp_ops,
830 	.num_resets = 193,
831 };
832 #endif
833 
834 #if IS_ENABLED(CONFIG_ARCH_TEGRA_210_SOC)
835 static const struct tegra_bpmp_soc tegra210_soc = {
836 	.channels = {
837 		.cpu_tx = {
838 			.offset = 0,
839 			.count = 1,
840 			.timeout = 60 * USEC_PER_SEC,
841 		},
842 		.thread = {
843 			.offset = 4,
844 			.count = 1,
845 			.timeout = 600 * USEC_PER_SEC,
846 		},
847 		.cpu_rx = {
848 			.offset = 8,
849 			.count = 1,
850 			.timeout = 0,
851 		},
852 	},
853 	.ops = &tegra210_bpmp_ops,
854 };
855 #endif
856 
857 static const struct of_device_id tegra_bpmp_match[] = {
858 #if IS_ENABLED(CONFIG_ARCH_TEGRA_186_SOC) || \
859     IS_ENABLED(CONFIG_ARCH_TEGRA_194_SOC) || \
860     IS_ENABLED(CONFIG_ARCH_TEGRA_234_SOC)
861 	{ .compatible = "nvidia,tegra186-bpmp", .data = &tegra186_soc },
862 #endif
863 #if IS_ENABLED(CONFIG_ARCH_TEGRA_210_SOC)
864 	{ .compatible = "nvidia,tegra210-bpmp", .data = &tegra210_soc },
865 #endif
866 	{ }
867 };
868 
869 static struct platform_driver tegra_bpmp_driver = {
870 	.driver = {
871 		.name = "tegra-bpmp",
872 		.of_match_table = tegra_bpmp_match,
873 		.pm = &tegra_bpmp_pm_ops,
874 		.suppress_bind_attrs = true,
875 	},
876 	.probe = tegra_bpmp_probe,
877 };
878 builtin_platform_driver(tegra_bpmp_driver);
879