Loading...
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Copyright (c) 2016, NVIDIA CORPORATION. All rights reserved.
4 */
5
6#include <linux/clk/tegra.h>
7#include <linux/genalloc.h>
8#include <linux/mailbox_client.h>
9#include <linux/module.h>
10#include <linux/of.h>
11#include <linux/of_platform.h>
12#include <linux/platform_device.h>
13#include <linux/pm.h>
14#include <linux/semaphore.h>
15#include <linux/sched/clock.h>
16
17#include <soc/tegra/bpmp.h>
18#include <soc/tegra/bpmp-abi.h>
19#include <soc/tegra/ivc.h>
20
21#include "bpmp-private.h"
22
23#define MSG_ACK BIT(0)
24#define MSG_RING BIT(1)
25#define TAG_SZ 32
26
27static inline struct tegra_bpmp *
28mbox_client_to_bpmp(struct mbox_client *client)
29{
30 return container_of(client, struct tegra_bpmp, mbox.client);
31}
32
33static inline const struct tegra_bpmp_ops *
34channel_to_ops(struct tegra_bpmp_channel *channel)
35{
36 struct tegra_bpmp *bpmp = channel->bpmp;
37
38 return bpmp->soc->ops;
39}
40
41struct tegra_bpmp *tegra_bpmp_get(struct device *dev)
42{
43 struct platform_device *pdev;
44 struct tegra_bpmp *bpmp;
45 struct device_node *np;
46
47 np = of_parse_phandle(dev->of_node, "nvidia,bpmp", 0);
48 if (!np)
49 return ERR_PTR(-ENOENT);
50
51 pdev = of_find_device_by_node(np);
52 if (!pdev) {
53 bpmp = ERR_PTR(-ENODEV);
54 goto put;
55 }
56
57 bpmp = platform_get_drvdata(pdev);
58 if (!bpmp) {
59 bpmp = ERR_PTR(-EPROBE_DEFER);
60 put_device(&pdev->dev);
61 goto put;
62 }
63
64put:
65 of_node_put(np);
66 return bpmp;
67}
68EXPORT_SYMBOL_GPL(tegra_bpmp_get);
69
70void tegra_bpmp_put(struct tegra_bpmp *bpmp)
71{
72 if (bpmp)
73 put_device(bpmp->dev);
74}
75EXPORT_SYMBOL_GPL(tegra_bpmp_put);
76
77static int
78tegra_bpmp_channel_get_thread_index(struct tegra_bpmp_channel *channel)
79{
80 struct tegra_bpmp *bpmp = channel->bpmp;
81 unsigned int count;
82 int index;
83
84 count = bpmp->soc->channels.thread.count;
85
86 index = channel - channel->bpmp->threaded_channels;
87 if (index < 0 || index >= count)
88 return -EINVAL;
89
90 return index;
91}
92
93static bool tegra_bpmp_message_valid(const struct tegra_bpmp_message *msg)
94{
95 return (msg->tx.size <= MSG_DATA_MIN_SZ) &&
96 (msg->rx.size <= MSG_DATA_MIN_SZ) &&
97 (msg->tx.size == 0 || msg->tx.data) &&
98 (msg->rx.size == 0 || msg->rx.data);
99}
100
101static bool tegra_bpmp_is_response_ready(struct tegra_bpmp_channel *channel)
102{
103 const struct tegra_bpmp_ops *ops = channel_to_ops(channel);
104
105 return ops->is_response_ready(channel);
106}
107
108static bool tegra_bpmp_is_request_ready(struct tegra_bpmp_channel *channel)
109{
110 const struct tegra_bpmp_ops *ops = channel_to_ops(channel);
111
112 return ops->is_request_ready(channel);
113}
114
115static int tegra_bpmp_wait_response(struct tegra_bpmp_channel *channel)
116{
117 unsigned long timeout = channel->bpmp->soc->channels.cpu_tx.timeout;
118 ktime_t end;
119
120 end = ktime_add_us(ktime_get(), timeout);
121
122 do {
123 if (tegra_bpmp_is_response_ready(channel))
124 return 0;
125 } while (ktime_before(ktime_get(), end));
126
127 return -ETIMEDOUT;
128}
129
130static int tegra_bpmp_ack_response(struct tegra_bpmp_channel *channel)
131{
132 const struct tegra_bpmp_ops *ops = channel_to_ops(channel);
133
134 return ops->ack_response(channel);
135}
136
137static int tegra_bpmp_ack_request(struct tegra_bpmp_channel *channel)
138{
139 const struct tegra_bpmp_ops *ops = channel_to_ops(channel);
140
141 return ops->ack_request(channel);
142}
143
144static bool
145tegra_bpmp_is_request_channel_free(struct tegra_bpmp_channel *channel)
146{
147 const struct tegra_bpmp_ops *ops = channel_to_ops(channel);
148
149 return ops->is_request_channel_free(channel);
150}
151
152static bool
153tegra_bpmp_is_response_channel_free(struct tegra_bpmp_channel *channel)
154{
155 const struct tegra_bpmp_ops *ops = channel_to_ops(channel);
156
157 return ops->is_response_channel_free(channel);
158}
159
160static int
161tegra_bpmp_wait_request_channel_free(struct tegra_bpmp_channel *channel)
162{
163 unsigned long timeout = channel->bpmp->soc->channels.cpu_tx.timeout;
164 ktime_t start, now;
165
166 start = ns_to_ktime(local_clock());
167
168 do {
169 if (tegra_bpmp_is_request_channel_free(channel))
170 return 0;
171
172 now = ns_to_ktime(local_clock());
173 } while (ktime_us_delta(now, start) < timeout);
174
175 return -ETIMEDOUT;
176}
177
178static int tegra_bpmp_post_request(struct tegra_bpmp_channel *channel)
179{
180 const struct tegra_bpmp_ops *ops = channel_to_ops(channel);
181
182 return ops->post_request(channel);
183}
184
185static int tegra_bpmp_post_response(struct tegra_bpmp_channel *channel)
186{
187 const struct tegra_bpmp_ops *ops = channel_to_ops(channel);
188
189 return ops->post_response(channel);
190}
191
192static int tegra_bpmp_ring_doorbell(struct tegra_bpmp *bpmp)
193{
194 return bpmp->soc->ops->ring_doorbell(bpmp);
195}
196
197static ssize_t __tegra_bpmp_channel_read(struct tegra_bpmp_channel *channel,
198 void *data, size_t size, int *ret)
199{
200 int err;
201
202 if (data && size > 0)
203 tegra_bpmp_mb_read(data, &channel->ib, size);
204
205 err = tegra_bpmp_ack_response(channel);
206 if (err < 0)
207 return err;
208
209 *ret = tegra_bpmp_mb_read_field(&channel->ib, code);
210
211 return 0;
212}
213
214static ssize_t tegra_bpmp_channel_read(struct tegra_bpmp_channel *channel,
215 void *data, size_t size, int *ret)
216{
217 struct tegra_bpmp *bpmp = channel->bpmp;
218 unsigned long flags;
219 ssize_t err;
220 int index;
221
222 index = tegra_bpmp_channel_get_thread_index(channel);
223 if (index < 0) {
224 err = index;
225 goto unlock;
226 }
227
228 spin_lock_irqsave(&bpmp->lock, flags);
229 err = __tegra_bpmp_channel_read(channel, data, size, ret);
230 clear_bit(index, bpmp->threaded.allocated);
231 spin_unlock_irqrestore(&bpmp->lock, flags);
232
233unlock:
234 up(&bpmp->threaded.lock);
235
236 return err;
237}
238
239static ssize_t __tegra_bpmp_channel_write(struct tegra_bpmp_channel *channel,
240 unsigned int mrq, unsigned long flags,
241 const void *data, size_t size)
242{
243 tegra_bpmp_mb_write_field(&channel->ob, code, mrq);
244 tegra_bpmp_mb_write_field(&channel->ob, flags, flags);
245
246 if (data && size > 0)
247 tegra_bpmp_mb_write(&channel->ob, data, size);
248
249 return tegra_bpmp_post_request(channel);
250}
251
252static struct tegra_bpmp_channel *
253tegra_bpmp_write_threaded(struct tegra_bpmp *bpmp, unsigned int mrq,
254 const void *data, size_t size)
255{
256 unsigned long timeout = bpmp->soc->channels.thread.timeout;
257 unsigned int count = bpmp->soc->channels.thread.count;
258 struct tegra_bpmp_channel *channel;
259 unsigned long flags;
260 unsigned int index;
261 int err;
262
263 err = down_timeout(&bpmp->threaded.lock, usecs_to_jiffies(timeout));
264 if (err < 0)
265 return ERR_PTR(err);
266
267 spin_lock_irqsave(&bpmp->lock, flags);
268
269 index = find_first_zero_bit(bpmp->threaded.allocated, count);
270 if (index == count) {
271 err = -EBUSY;
272 goto unlock;
273 }
274
275 channel = &bpmp->threaded_channels[index];
276
277 if (!tegra_bpmp_is_request_channel_free(channel)) {
278 err = -EBUSY;
279 goto unlock;
280 }
281
282 set_bit(index, bpmp->threaded.allocated);
283
284 err = __tegra_bpmp_channel_write(channel, mrq, MSG_ACK | MSG_RING,
285 data, size);
286 if (err < 0)
287 goto clear_allocated;
288
289 set_bit(index, bpmp->threaded.busy);
290
291 spin_unlock_irqrestore(&bpmp->lock, flags);
292 return channel;
293
294clear_allocated:
295 clear_bit(index, bpmp->threaded.allocated);
296unlock:
297 spin_unlock_irqrestore(&bpmp->lock, flags);
298 up(&bpmp->threaded.lock);
299
300 return ERR_PTR(err);
301}
302
303static ssize_t tegra_bpmp_channel_write(struct tegra_bpmp_channel *channel,
304 unsigned int mrq, unsigned long flags,
305 const void *data, size_t size)
306{
307 int err;
308
309 err = tegra_bpmp_wait_request_channel_free(channel);
310 if (err < 0)
311 return err;
312
313 return __tegra_bpmp_channel_write(channel, mrq, flags, data, size);
314}
315
316static int __maybe_unused tegra_bpmp_resume(struct device *dev);
317
318int tegra_bpmp_transfer_atomic(struct tegra_bpmp *bpmp,
319 struct tegra_bpmp_message *msg)
320{
321 struct tegra_bpmp_channel *channel;
322 int err;
323
324 if (WARN_ON(!irqs_disabled()))
325 return -EPERM;
326
327 if (!tegra_bpmp_message_valid(msg))
328 return -EINVAL;
329
330 if (bpmp->suspended) {
331 /* Reset BPMP IPC channels during resume based on flags passed */
332 if (msg->flags & TEGRA_BPMP_MESSAGE_RESET)
333 tegra_bpmp_resume(bpmp->dev);
334 else
335 return -EAGAIN;
336 }
337
338 channel = bpmp->tx_channel;
339
340 spin_lock(&bpmp->atomic_tx_lock);
341
342 err = tegra_bpmp_channel_write(channel, msg->mrq, MSG_ACK,
343 msg->tx.data, msg->tx.size);
344 if (err < 0) {
345 spin_unlock(&bpmp->atomic_tx_lock);
346 return err;
347 }
348
349 spin_unlock(&bpmp->atomic_tx_lock);
350
351 err = tegra_bpmp_ring_doorbell(bpmp);
352 if (err < 0)
353 return err;
354
355 err = tegra_bpmp_wait_response(channel);
356 if (err < 0)
357 return err;
358
359 return __tegra_bpmp_channel_read(channel, msg->rx.data, msg->rx.size,
360 &msg->rx.ret);
361}
362EXPORT_SYMBOL_GPL(tegra_bpmp_transfer_atomic);
363
364int tegra_bpmp_transfer(struct tegra_bpmp *bpmp,
365 struct tegra_bpmp_message *msg)
366{
367 struct tegra_bpmp_channel *channel;
368 unsigned long timeout;
369 int err;
370
371 if (WARN_ON(irqs_disabled()))
372 return -EPERM;
373
374 if (!tegra_bpmp_message_valid(msg))
375 return -EINVAL;
376
377 if (bpmp->suspended) {
378 /* Reset BPMP IPC channels during resume based on flags passed */
379 if (msg->flags & TEGRA_BPMP_MESSAGE_RESET)
380 tegra_bpmp_resume(bpmp->dev);
381 else
382 return -EAGAIN;
383 }
384
385 channel = tegra_bpmp_write_threaded(bpmp, msg->mrq, msg->tx.data,
386 msg->tx.size);
387 if (IS_ERR(channel))
388 return PTR_ERR(channel);
389
390 err = tegra_bpmp_ring_doorbell(bpmp);
391 if (err < 0)
392 return err;
393
394 timeout = usecs_to_jiffies(bpmp->soc->channels.thread.timeout);
395
396 err = wait_for_completion_timeout(&channel->completion, timeout);
397 if (err == 0)
398 return -ETIMEDOUT;
399
400 return tegra_bpmp_channel_read(channel, msg->rx.data, msg->rx.size,
401 &msg->rx.ret);
402}
403EXPORT_SYMBOL_GPL(tegra_bpmp_transfer);
404
405static struct tegra_bpmp_mrq *tegra_bpmp_find_mrq(struct tegra_bpmp *bpmp,
406 unsigned int mrq)
407{
408 struct tegra_bpmp_mrq *entry;
409
410 list_for_each_entry(entry, &bpmp->mrqs, list)
411 if (entry->mrq == mrq)
412 return entry;
413
414 return NULL;
415}
416
417void tegra_bpmp_mrq_return(struct tegra_bpmp_channel *channel, int code,
418 const void *data, size_t size)
419{
420 unsigned long flags = tegra_bpmp_mb_read_field(&channel->ib, flags);
421 struct tegra_bpmp *bpmp = channel->bpmp;
422 int err;
423
424 if (WARN_ON(size > MSG_DATA_MIN_SZ))
425 return;
426
427 err = tegra_bpmp_ack_request(channel);
428 if (WARN_ON(err < 0))
429 return;
430
431 if ((flags & MSG_ACK) == 0)
432 return;
433
434 if (WARN_ON(!tegra_bpmp_is_response_channel_free(channel)))
435 return;
436
437 tegra_bpmp_mb_write_field(&channel->ob, code, code);
438
439 if (data && size > 0)
440 tegra_bpmp_mb_write(&channel->ob, data, size);
441
442 err = tegra_bpmp_post_response(channel);
443 if (WARN_ON(err < 0))
444 return;
445
446 if (flags & MSG_RING) {
447 err = tegra_bpmp_ring_doorbell(bpmp);
448 if (WARN_ON(err < 0))
449 return;
450 }
451}
452EXPORT_SYMBOL_GPL(tegra_bpmp_mrq_return);
453
454static void tegra_bpmp_handle_mrq(struct tegra_bpmp *bpmp,
455 unsigned int mrq,
456 struct tegra_bpmp_channel *channel)
457{
458 struct tegra_bpmp_mrq *entry;
459 u32 zero = 0;
460
461 spin_lock(&bpmp->lock);
462
463 entry = tegra_bpmp_find_mrq(bpmp, mrq);
464 if (!entry) {
465 spin_unlock(&bpmp->lock);
466 tegra_bpmp_mrq_return(channel, -EINVAL, &zero, sizeof(zero));
467 return;
468 }
469
470 entry->handler(mrq, channel, entry->data);
471
472 spin_unlock(&bpmp->lock);
473}
474
475int tegra_bpmp_request_mrq(struct tegra_bpmp *bpmp, unsigned int mrq,
476 tegra_bpmp_mrq_handler_t handler, void *data)
477{
478 struct tegra_bpmp_mrq *entry;
479 unsigned long flags;
480
481 if (!handler)
482 return -EINVAL;
483
484 entry = devm_kzalloc(bpmp->dev, sizeof(*entry), GFP_KERNEL);
485 if (!entry)
486 return -ENOMEM;
487
488 spin_lock_irqsave(&bpmp->lock, flags);
489
490 entry->mrq = mrq;
491 entry->handler = handler;
492 entry->data = data;
493 list_add(&entry->list, &bpmp->mrqs);
494
495 spin_unlock_irqrestore(&bpmp->lock, flags);
496
497 return 0;
498}
499EXPORT_SYMBOL_GPL(tegra_bpmp_request_mrq);
500
501void tegra_bpmp_free_mrq(struct tegra_bpmp *bpmp, unsigned int mrq, void *data)
502{
503 struct tegra_bpmp_mrq *entry;
504 unsigned long flags;
505
506 spin_lock_irqsave(&bpmp->lock, flags);
507
508 entry = tegra_bpmp_find_mrq(bpmp, mrq);
509 if (!entry)
510 goto unlock;
511
512 list_del(&entry->list);
513 devm_kfree(bpmp->dev, entry);
514
515unlock:
516 spin_unlock_irqrestore(&bpmp->lock, flags);
517}
518EXPORT_SYMBOL_GPL(tegra_bpmp_free_mrq);
519
520bool tegra_bpmp_mrq_is_supported(struct tegra_bpmp *bpmp, unsigned int mrq)
521{
522 struct mrq_query_abi_request req = { .mrq = mrq };
523 struct mrq_query_abi_response resp;
524 struct tegra_bpmp_message msg = {
525 .mrq = MRQ_QUERY_ABI,
526 .tx = {
527 .data = &req,
528 .size = sizeof(req),
529 },
530 .rx = {
531 .data = &resp,
532 .size = sizeof(resp),
533 },
534 };
535 int err;
536
537 err = tegra_bpmp_transfer(bpmp, &msg);
538 if (err || msg.rx.ret)
539 return false;
540
541 return resp.status == 0;
542}
543EXPORT_SYMBOL_GPL(tegra_bpmp_mrq_is_supported);
544
545static void tegra_bpmp_mrq_handle_ping(unsigned int mrq,
546 struct tegra_bpmp_channel *channel,
547 void *data)
548{
549 struct mrq_ping_request request;
550 struct mrq_ping_response response;
551
552 tegra_bpmp_mb_read(&request, &channel->ib, sizeof(request));
553
554 memset(&response, 0, sizeof(response));
555 response.reply = request.challenge << 1;
556
557 tegra_bpmp_mrq_return(channel, 0, &response, sizeof(response));
558}
559
560static int tegra_bpmp_ping(struct tegra_bpmp *bpmp)
561{
562 struct mrq_ping_response response;
563 struct mrq_ping_request request;
564 struct tegra_bpmp_message msg;
565 unsigned long flags;
566 ktime_t start, end;
567 int err;
568
569 memset(&request, 0, sizeof(request));
570 request.challenge = 1;
571
572 memset(&response, 0, sizeof(response));
573
574 memset(&msg, 0, sizeof(msg));
575 msg.mrq = MRQ_PING;
576 msg.tx.data = &request;
577 msg.tx.size = sizeof(request);
578 msg.rx.data = &response;
579 msg.rx.size = sizeof(response);
580
581 local_irq_save(flags);
582 start = ktime_get();
583 err = tegra_bpmp_transfer_atomic(bpmp, &msg);
584 end = ktime_get();
585 local_irq_restore(flags);
586
587 if (!err)
588 dev_dbg(bpmp->dev,
589 "ping ok: challenge: %u, response: %u, time: %lld\n",
590 request.challenge, response.reply,
591 ktime_to_us(ktime_sub(end, start)));
592
593 return err;
594}
595
596/* deprecated version of tag query */
597static int tegra_bpmp_get_firmware_tag_old(struct tegra_bpmp *bpmp, char *tag,
598 size_t size)
599{
600 struct mrq_query_tag_request request;
601 struct tegra_bpmp_message msg;
602 unsigned long flags;
603 dma_addr_t phys;
604 void *virt;
605 int err;
606
607 if (size != TAG_SZ)
608 return -EINVAL;
609
610 virt = dma_alloc_coherent(bpmp->dev, TAG_SZ, &phys,
611 GFP_KERNEL | GFP_DMA32);
612 if (!virt)
613 return -ENOMEM;
614
615 memset(&request, 0, sizeof(request));
616 request.addr = phys;
617
618 memset(&msg, 0, sizeof(msg));
619 msg.mrq = MRQ_QUERY_TAG;
620 msg.tx.data = &request;
621 msg.tx.size = sizeof(request);
622
623 local_irq_save(flags);
624 err = tegra_bpmp_transfer_atomic(bpmp, &msg);
625 local_irq_restore(flags);
626
627 if (err == 0)
628 memcpy(tag, virt, TAG_SZ);
629
630 dma_free_coherent(bpmp->dev, TAG_SZ, virt, phys);
631
632 return err;
633}
634
635static int tegra_bpmp_get_firmware_tag(struct tegra_bpmp *bpmp, char *tag,
636 size_t size)
637{
638 if (tegra_bpmp_mrq_is_supported(bpmp, MRQ_QUERY_FW_TAG)) {
639 struct mrq_query_fw_tag_response resp;
640 struct tegra_bpmp_message msg = {
641 .mrq = MRQ_QUERY_FW_TAG,
642 .rx = {
643 .data = &resp,
644 .size = sizeof(resp),
645 },
646 };
647 int err;
648
649 if (size != sizeof(resp.tag))
650 return -EINVAL;
651
652 err = tegra_bpmp_transfer(bpmp, &msg);
653
654 if (err)
655 return err;
656 if (msg.rx.ret < 0)
657 return -EINVAL;
658
659 memcpy(tag, resp.tag, sizeof(resp.tag));
660 return 0;
661 }
662
663 return tegra_bpmp_get_firmware_tag_old(bpmp, tag, size);
664}
665
666static void tegra_bpmp_channel_signal(struct tegra_bpmp_channel *channel)
667{
668 unsigned long flags = tegra_bpmp_mb_read_field(&channel->ob, flags);
669
670 if ((flags & MSG_RING) == 0)
671 return;
672
673 complete(&channel->completion);
674}
675
676void tegra_bpmp_handle_rx(struct tegra_bpmp *bpmp)
677{
678 struct tegra_bpmp_channel *channel;
679 unsigned int i, count;
680 unsigned long *busy;
681
682 channel = bpmp->rx_channel;
683 count = bpmp->soc->channels.thread.count;
684 busy = bpmp->threaded.busy;
685
686 if (tegra_bpmp_is_request_ready(channel)) {
687 unsigned int mrq = tegra_bpmp_mb_read_field(&channel->ib, code);
688
689 tegra_bpmp_handle_mrq(bpmp, mrq, channel);
690 }
691
692 spin_lock(&bpmp->lock);
693
694 for_each_set_bit(i, busy, count) {
695 struct tegra_bpmp_channel *channel;
696
697 channel = &bpmp->threaded_channels[i];
698
699 if (tegra_bpmp_is_response_ready(channel)) {
700 tegra_bpmp_channel_signal(channel);
701 clear_bit(i, busy);
702 }
703 }
704
705 spin_unlock(&bpmp->lock);
706}
707
708static int tegra_bpmp_probe(struct platform_device *pdev)
709{
710 struct tegra_bpmp *bpmp;
711 char tag[TAG_SZ];
712 size_t size;
713 int err;
714
715 bpmp = devm_kzalloc(&pdev->dev, sizeof(*bpmp), GFP_KERNEL);
716 if (!bpmp)
717 return -ENOMEM;
718
719 bpmp->soc = of_device_get_match_data(&pdev->dev);
720 bpmp->dev = &pdev->dev;
721
722 INIT_LIST_HEAD(&bpmp->mrqs);
723 spin_lock_init(&bpmp->lock);
724
725 bpmp->threaded.count = bpmp->soc->channels.thread.count;
726 sema_init(&bpmp->threaded.lock, bpmp->threaded.count);
727
728 size = BITS_TO_LONGS(bpmp->threaded.count) * sizeof(long);
729
730 bpmp->threaded.allocated = devm_kzalloc(&pdev->dev, size, GFP_KERNEL);
731 if (!bpmp->threaded.allocated)
732 return -ENOMEM;
733
734 bpmp->threaded.busy = devm_kzalloc(&pdev->dev, size, GFP_KERNEL);
735 if (!bpmp->threaded.busy)
736 return -ENOMEM;
737
738 spin_lock_init(&bpmp->atomic_tx_lock);
739 bpmp->tx_channel = devm_kzalloc(&pdev->dev, sizeof(*bpmp->tx_channel),
740 GFP_KERNEL);
741 if (!bpmp->tx_channel)
742 return -ENOMEM;
743
744 bpmp->rx_channel = devm_kzalloc(&pdev->dev, sizeof(*bpmp->rx_channel),
745 GFP_KERNEL);
746 if (!bpmp->rx_channel)
747 return -ENOMEM;
748
749 bpmp->threaded_channels = devm_kcalloc(&pdev->dev, bpmp->threaded.count,
750 sizeof(*bpmp->threaded_channels),
751 GFP_KERNEL);
752 if (!bpmp->threaded_channels)
753 return -ENOMEM;
754
755 platform_set_drvdata(pdev, bpmp);
756
757 err = bpmp->soc->ops->init(bpmp);
758 if (err < 0)
759 return err;
760
761 err = tegra_bpmp_request_mrq(bpmp, MRQ_PING,
762 tegra_bpmp_mrq_handle_ping, bpmp);
763 if (err < 0)
764 goto deinit;
765
766 err = tegra_bpmp_ping(bpmp);
767 if (err < 0) {
768 dev_err(&pdev->dev, "failed to ping BPMP: %d\n", err);
769 goto free_mrq;
770 }
771
772 err = tegra_bpmp_get_firmware_tag(bpmp, tag, sizeof(tag));
773 if (err < 0) {
774 dev_err(&pdev->dev, "failed to get firmware tag: %d\n", err);
775 goto free_mrq;
776 }
777
778 dev_info(&pdev->dev, "firmware: %.*s\n", (int)sizeof(tag), tag);
779
780 err = of_platform_default_populate(pdev->dev.of_node, NULL, &pdev->dev);
781 if (err < 0)
782 goto free_mrq;
783
784 if (of_property_present(pdev->dev.of_node, "#clock-cells")) {
785 err = tegra_bpmp_init_clocks(bpmp);
786 if (err < 0)
787 goto free_mrq;
788 }
789
790 if (of_property_present(pdev->dev.of_node, "#reset-cells")) {
791 err = tegra_bpmp_init_resets(bpmp);
792 if (err < 0)
793 goto free_mrq;
794 }
795
796 if (of_property_present(pdev->dev.of_node, "#power-domain-cells")) {
797 err = tegra_bpmp_init_powergates(bpmp);
798 if (err < 0)
799 goto free_mrq;
800 }
801
802 err = tegra_bpmp_init_debugfs(bpmp);
803 if (err < 0)
804 dev_err(&pdev->dev, "debugfs initialization failed: %d\n", err);
805
806 return 0;
807
808free_mrq:
809 tegra_bpmp_free_mrq(bpmp, MRQ_PING, bpmp);
810deinit:
811 if (bpmp->soc->ops->deinit)
812 bpmp->soc->ops->deinit(bpmp);
813
814 return err;
815}
816
817static int __maybe_unused tegra_bpmp_suspend(struct device *dev)
818{
819 struct tegra_bpmp *bpmp = dev_get_drvdata(dev);
820
821 bpmp->suspended = true;
822
823 return 0;
824}
825
826static int __maybe_unused tegra_bpmp_resume(struct device *dev)
827{
828 struct tegra_bpmp *bpmp = dev_get_drvdata(dev);
829
830 bpmp->suspended = false;
831
832 if (bpmp->soc->ops->resume)
833 return bpmp->soc->ops->resume(bpmp);
834 else
835 return 0;
836}
837
838static const struct dev_pm_ops tegra_bpmp_pm_ops = {
839 .suspend_noirq = tegra_bpmp_suspend,
840 .resume_noirq = tegra_bpmp_resume,
841};
842
843#if IS_ENABLED(CONFIG_ARCH_TEGRA_186_SOC) || \
844 IS_ENABLED(CONFIG_ARCH_TEGRA_194_SOC) || \
845 IS_ENABLED(CONFIG_ARCH_TEGRA_234_SOC)
846static const struct tegra_bpmp_soc tegra186_soc = {
847 .channels = {
848 .cpu_tx = {
849 .offset = 3,
850 .timeout = 60 * USEC_PER_SEC,
851 },
852 .thread = {
853 .offset = 0,
854 .count = 3,
855 .timeout = 600 * USEC_PER_SEC,
856 },
857 .cpu_rx = {
858 .offset = 13,
859 .timeout = 0,
860 },
861 },
862 .ops = &tegra186_bpmp_ops,
863 .num_resets = 193,
864};
865#endif
866
867#if IS_ENABLED(CONFIG_ARCH_TEGRA_210_SOC)
868static const struct tegra_bpmp_soc tegra210_soc = {
869 .channels = {
870 .cpu_tx = {
871 .offset = 0,
872 .count = 1,
873 .timeout = 60 * USEC_PER_SEC,
874 },
875 .thread = {
876 .offset = 4,
877 .count = 1,
878 .timeout = 600 * USEC_PER_SEC,
879 },
880 .cpu_rx = {
881 .offset = 8,
882 .count = 1,
883 .timeout = 0,
884 },
885 },
886 .ops = &tegra210_bpmp_ops,
887};
888#endif
889
890static const struct of_device_id tegra_bpmp_match[] = {
891#if IS_ENABLED(CONFIG_ARCH_TEGRA_186_SOC) || \
892 IS_ENABLED(CONFIG_ARCH_TEGRA_194_SOC) || \
893 IS_ENABLED(CONFIG_ARCH_TEGRA_234_SOC)
894 { .compatible = "nvidia,tegra186-bpmp", .data = &tegra186_soc },
895#endif
896#if IS_ENABLED(CONFIG_ARCH_TEGRA_210_SOC)
897 { .compatible = "nvidia,tegra210-bpmp", .data = &tegra210_soc },
898#endif
899 { }
900};
901
902static struct platform_driver tegra_bpmp_driver = {
903 .driver = {
904 .name = "tegra-bpmp",
905 .of_match_table = tegra_bpmp_match,
906 .pm = &tegra_bpmp_pm_ops,
907 .suppress_bind_attrs = true,
908 },
909 .probe = tegra_bpmp_probe,
910};
911builtin_platform_driver(tegra_bpmp_driver);
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Copyright (c) 2016, NVIDIA CORPORATION. All rights reserved.
4 */
5
6#include <linux/clk/tegra.h>
7#include <linux/genalloc.h>
8#include <linux/mailbox_client.h>
9#include <linux/module.h>
10#include <linux/of.h>
11#include <linux/of_address.h>
12#include <linux/of_device.h>
13#include <linux/platform_device.h>
14#include <linux/pm.h>
15#include <linux/semaphore.h>
16#include <linux/sched/clock.h>
17
18#include <soc/tegra/bpmp.h>
19#include <soc/tegra/bpmp-abi.h>
20#include <soc/tegra/ivc.h>
21
22#include "bpmp-private.h"
23
24#define MSG_ACK BIT(0)
25#define MSG_RING BIT(1)
26#define TAG_SZ 32
27
28static inline struct tegra_bpmp *
29mbox_client_to_bpmp(struct mbox_client *client)
30{
31 return container_of(client, struct tegra_bpmp, mbox.client);
32}
33
34static inline const struct tegra_bpmp_ops *
35channel_to_ops(struct tegra_bpmp_channel *channel)
36{
37 struct tegra_bpmp *bpmp = channel->bpmp;
38
39 return bpmp->soc->ops;
40}
41
42struct tegra_bpmp *tegra_bpmp_get(struct device *dev)
43{
44 struct platform_device *pdev;
45 struct tegra_bpmp *bpmp;
46 struct device_node *np;
47
48 np = of_parse_phandle(dev->of_node, "nvidia,bpmp", 0);
49 if (!np)
50 return ERR_PTR(-ENOENT);
51
52 pdev = of_find_device_by_node(np);
53 if (!pdev) {
54 bpmp = ERR_PTR(-ENODEV);
55 goto put;
56 }
57
58 bpmp = platform_get_drvdata(pdev);
59 if (!bpmp) {
60 bpmp = ERR_PTR(-EPROBE_DEFER);
61 put_device(&pdev->dev);
62 goto put;
63 }
64
65put:
66 of_node_put(np);
67 return bpmp;
68}
69EXPORT_SYMBOL_GPL(tegra_bpmp_get);
70
71void tegra_bpmp_put(struct tegra_bpmp *bpmp)
72{
73 if (bpmp)
74 put_device(bpmp->dev);
75}
76EXPORT_SYMBOL_GPL(tegra_bpmp_put);
77
78static int
79tegra_bpmp_channel_get_thread_index(struct tegra_bpmp_channel *channel)
80{
81 struct tegra_bpmp *bpmp = channel->bpmp;
82 unsigned int count;
83 int index;
84
85 count = bpmp->soc->channels.thread.count;
86
87 index = channel - channel->bpmp->threaded_channels;
88 if (index < 0 || index >= count)
89 return -EINVAL;
90
91 return index;
92}
93
94static bool tegra_bpmp_message_valid(const struct tegra_bpmp_message *msg)
95{
96 return (msg->tx.size <= MSG_DATA_MIN_SZ) &&
97 (msg->rx.size <= MSG_DATA_MIN_SZ) &&
98 (msg->tx.size == 0 || msg->tx.data) &&
99 (msg->rx.size == 0 || msg->rx.data);
100}
101
102static bool tegra_bpmp_is_response_ready(struct tegra_bpmp_channel *channel)
103{
104 const struct tegra_bpmp_ops *ops = channel_to_ops(channel);
105
106 return ops->is_response_ready(channel);
107}
108
109static bool tegra_bpmp_is_request_ready(struct tegra_bpmp_channel *channel)
110{
111 const struct tegra_bpmp_ops *ops = channel_to_ops(channel);
112
113 return ops->is_request_ready(channel);
114}
115
116static int tegra_bpmp_wait_response(struct tegra_bpmp_channel *channel)
117{
118 unsigned long timeout = channel->bpmp->soc->channels.cpu_tx.timeout;
119 ktime_t end;
120
121 end = ktime_add_us(ktime_get(), timeout);
122
123 do {
124 if (tegra_bpmp_is_response_ready(channel))
125 return 0;
126 } while (ktime_before(ktime_get(), end));
127
128 return -ETIMEDOUT;
129}
130
131static int tegra_bpmp_ack_response(struct tegra_bpmp_channel *channel)
132{
133 const struct tegra_bpmp_ops *ops = channel_to_ops(channel);
134
135 return ops->ack_response(channel);
136}
137
138static int tegra_bpmp_ack_request(struct tegra_bpmp_channel *channel)
139{
140 const struct tegra_bpmp_ops *ops = channel_to_ops(channel);
141
142 return ops->ack_request(channel);
143}
144
145static bool
146tegra_bpmp_is_request_channel_free(struct tegra_bpmp_channel *channel)
147{
148 const struct tegra_bpmp_ops *ops = channel_to_ops(channel);
149
150 return ops->is_request_channel_free(channel);
151}
152
153static bool
154tegra_bpmp_is_response_channel_free(struct tegra_bpmp_channel *channel)
155{
156 const struct tegra_bpmp_ops *ops = channel_to_ops(channel);
157
158 return ops->is_response_channel_free(channel);
159}
160
161static int
162tegra_bpmp_wait_request_channel_free(struct tegra_bpmp_channel *channel)
163{
164 unsigned long timeout = channel->bpmp->soc->channels.cpu_tx.timeout;
165 ktime_t start, now;
166
167 start = ns_to_ktime(local_clock());
168
169 do {
170 if (tegra_bpmp_is_request_channel_free(channel))
171 return 0;
172
173 now = ns_to_ktime(local_clock());
174 } while (ktime_us_delta(now, start) < timeout);
175
176 return -ETIMEDOUT;
177}
178
179static int tegra_bpmp_post_request(struct tegra_bpmp_channel *channel)
180{
181 const struct tegra_bpmp_ops *ops = channel_to_ops(channel);
182
183 return ops->post_request(channel);
184}
185
186static int tegra_bpmp_post_response(struct tegra_bpmp_channel *channel)
187{
188 const struct tegra_bpmp_ops *ops = channel_to_ops(channel);
189
190 return ops->post_response(channel);
191}
192
193static int tegra_bpmp_ring_doorbell(struct tegra_bpmp *bpmp)
194{
195 return bpmp->soc->ops->ring_doorbell(bpmp);
196}
197
198static ssize_t __tegra_bpmp_channel_read(struct tegra_bpmp_channel *channel,
199 void *data, size_t size, int *ret)
200{
201 int err;
202
203 if (data && size > 0)
204 tegra_bpmp_mb_read(data, &channel->ib, size);
205
206 err = tegra_bpmp_ack_response(channel);
207 if (err < 0)
208 return err;
209
210 *ret = tegra_bpmp_mb_read_field(&channel->ib, code);
211
212 return 0;
213}
214
215static ssize_t tegra_bpmp_channel_read(struct tegra_bpmp_channel *channel,
216 void *data, size_t size, int *ret)
217{
218 struct tegra_bpmp *bpmp = channel->bpmp;
219 unsigned long flags;
220 ssize_t err;
221 int index;
222
223 index = tegra_bpmp_channel_get_thread_index(channel);
224 if (index < 0) {
225 err = index;
226 goto unlock;
227 }
228
229 spin_lock_irqsave(&bpmp->lock, flags);
230 err = __tegra_bpmp_channel_read(channel, data, size, ret);
231 clear_bit(index, bpmp->threaded.allocated);
232 spin_unlock_irqrestore(&bpmp->lock, flags);
233
234unlock:
235 up(&bpmp->threaded.lock);
236
237 return err;
238}
239
240static ssize_t __tegra_bpmp_channel_write(struct tegra_bpmp_channel *channel,
241 unsigned int mrq, unsigned long flags,
242 const void *data, size_t size)
243{
244 tegra_bpmp_mb_write_field(&channel->ob, code, mrq);
245 tegra_bpmp_mb_write_field(&channel->ob, flags, flags);
246
247 if (data && size > 0)
248 tegra_bpmp_mb_write(&channel->ob, data, size);
249
250 return tegra_bpmp_post_request(channel);
251}
252
253static struct tegra_bpmp_channel *
254tegra_bpmp_write_threaded(struct tegra_bpmp *bpmp, unsigned int mrq,
255 const void *data, size_t size)
256{
257 unsigned long timeout = bpmp->soc->channels.thread.timeout;
258 unsigned int count = bpmp->soc->channels.thread.count;
259 struct tegra_bpmp_channel *channel;
260 unsigned long flags;
261 unsigned int index;
262 int err;
263
264 err = down_timeout(&bpmp->threaded.lock, usecs_to_jiffies(timeout));
265 if (err < 0)
266 return ERR_PTR(err);
267
268 spin_lock_irqsave(&bpmp->lock, flags);
269
270 index = find_first_zero_bit(bpmp->threaded.allocated, count);
271 if (index == count) {
272 err = -EBUSY;
273 goto unlock;
274 }
275
276 channel = &bpmp->threaded_channels[index];
277
278 if (!tegra_bpmp_is_request_channel_free(channel)) {
279 err = -EBUSY;
280 goto unlock;
281 }
282
283 set_bit(index, bpmp->threaded.allocated);
284
285 err = __tegra_bpmp_channel_write(channel, mrq, MSG_ACK | MSG_RING,
286 data, size);
287 if (err < 0)
288 goto clear_allocated;
289
290 set_bit(index, bpmp->threaded.busy);
291
292 spin_unlock_irqrestore(&bpmp->lock, flags);
293 return channel;
294
295clear_allocated:
296 clear_bit(index, bpmp->threaded.allocated);
297unlock:
298 spin_unlock_irqrestore(&bpmp->lock, flags);
299 up(&bpmp->threaded.lock);
300
301 return ERR_PTR(err);
302}
303
304static ssize_t tegra_bpmp_channel_write(struct tegra_bpmp_channel *channel,
305 unsigned int mrq, unsigned long flags,
306 const void *data, size_t size)
307{
308 int err;
309
310 err = tegra_bpmp_wait_request_channel_free(channel);
311 if (err < 0)
312 return err;
313
314 return __tegra_bpmp_channel_write(channel, mrq, flags, data, size);
315}
316
317int tegra_bpmp_transfer_atomic(struct tegra_bpmp *bpmp,
318 struct tegra_bpmp_message *msg)
319{
320 struct tegra_bpmp_channel *channel;
321 int err;
322
323 if (WARN_ON(!irqs_disabled()))
324 return -EPERM;
325
326 if (!tegra_bpmp_message_valid(msg))
327 return -EINVAL;
328
329 channel = bpmp->tx_channel;
330
331 spin_lock(&bpmp->atomic_tx_lock);
332
333 err = tegra_bpmp_channel_write(channel, msg->mrq, MSG_ACK,
334 msg->tx.data, msg->tx.size);
335 if (err < 0) {
336 spin_unlock(&bpmp->atomic_tx_lock);
337 return err;
338 }
339
340 spin_unlock(&bpmp->atomic_tx_lock);
341
342 err = tegra_bpmp_ring_doorbell(bpmp);
343 if (err < 0)
344 return err;
345
346 err = tegra_bpmp_wait_response(channel);
347 if (err < 0)
348 return err;
349
350 return __tegra_bpmp_channel_read(channel, msg->rx.data, msg->rx.size,
351 &msg->rx.ret);
352}
353EXPORT_SYMBOL_GPL(tegra_bpmp_transfer_atomic);
354
355int tegra_bpmp_transfer(struct tegra_bpmp *bpmp,
356 struct tegra_bpmp_message *msg)
357{
358 struct tegra_bpmp_channel *channel;
359 unsigned long timeout;
360 int err;
361
362 if (WARN_ON(irqs_disabled()))
363 return -EPERM;
364
365 if (!tegra_bpmp_message_valid(msg))
366 return -EINVAL;
367
368 channel = tegra_bpmp_write_threaded(bpmp, msg->mrq, msg->tx.data,
369 msg->tx.size);
370 if (IS_ERR(channel))
371 return PTR_ERR(channel);
372
373 err = tegra_bpmp_ring_doorbell(bpmp);
374 if (err < 0)
375 return err;
376
377 timeout = usecs_to_jiffies(bpmp->soc->channels.thread.timeout);
378
379 err = wait_for_completion_timeout(&channel->completion, timeout);
380 if (err == 0)
381 return -ETIMEDOUT;
382
383 return tegra_bpmp_channel_read(channel, msg->rx.data, msg->rx.size,
384 &msg->rx.ret);
385}
386EXPORT_SYMBOL_GPL(tegra_bpmp_transfer);
387
388static struct tegra_bpmp_mrq *tegra_bpmp_find_mrq(struct tegra_bpmp *bpmp,
389 unsigned int mrq)
390{
391 struct tegra_bpmp_mrq *entry;
392
393 list_for_each_entry(entry, &bpmp->mrqs, list)
394 if (entry->mrq == mrq)
395 return entry;
396
397 return NULL;
398}
399
400void tegra_bpmp_mrq_return(struct tegra_bpmp_channel *channel, int code,
401 const void *data, size_t size)
402{
403 unsigned long flags = tegra_bpmp_mb_read_field(&channel->ib, flags);
404 struct tegra_bpmp *bpmp = channel->bpmp;
405 int err;
406
407 if (WARN_ON(size > MSG_DATA_MIN_SZ))
408 return;
409
410 err = tegra_bpmp_ack_request(channel);
411 if (WARN_ON(err < 0))
412 return;
413
414 if ((flags & MSG_ACK) == 0)
415 return;
416
417 if (WARN_ON(!tegra_bpmp_is_response_channel_free(channel)))
418 return;
419
420 tegra_bpmp_mb_write_field(&channel->ob, code, code);
421
422 if (data && size > 0)
423 tegra_bpmp_mb_write(&channel->ob, data, size);
424
425 err = tegra_bpmp_post_response(channel);
426 if (WARN_ON(err < 0))
427 return;
428
429 if (flags & MSG_RING) {
430 err = tegra_bpmp_ring_doorbell(bpmp);
431 if (WARN_ON(err < 0))
432 return;
433 }
434}
435EXPORT_SYMBOL_GPL(tegra_bpmp_mrq_return);
436
437static void tegra_bpmp_handle_mrq(struct tegra_bpmp *bpmp,
438 unsigned int mrq,
439 struct tegra_bpmp_channel *channel)
440{
441 struct tegra_bpmp_mrq *entry;
442 u32 zero = 0;
443
444 spin_lock(&bpmp->lock);
445
446 entry = tegra_bpmp_find_mrq(bpmp, mrq);
447 if (!entry) {
448 spin_unlock(&bpmp->lock);
449 tegra_bpmp_mrq_return(channel, -EINVAL, &zero, sizeof(zero));
450 return;
451 }
452
453 entry->handler(mrq, channel, entry->data);
454
455 spin_unlock(&bpmp->lock);
456}
457
458int tegra_bpmp_request_mrq(struct tegra_bpmp *bpmp, unsigned int mrq,
459 tegra_bpmp_mrq_handler_t handler, void *data)
460{
461 struct tegra_bpmp_mrq *entry;
462 unsigned long flags;
463
464 if (!handler)
465 return -EINVAL;
466
467 entry = devm_kzalloc(bpmp->dev, sizeof(*entry), GFP_KERNEL);
468 if (!entry)
469 return -ENOMEM;
470
471 spin_lock_irqsave(&bpmp->lock, flags);
472
473 entry->mrq = mrq;
474 entry->handler = handler;
475 entry->data = data;
476 list_add(&entry->list, &bpmp->mrqs);
477
478 spin_unlock_irqrestore(&bpmp->lock, flags);
479
480 return 0;
481}
482EXPORT_SYMBOL_GPL(tegra_bpmp_request_mrq);
483
484void tegra_bpmp_free_mrq(struct tegra_bpmp *bpmp, unsigned int mrq, void *data)
485{
486 struct tegra_bpmp_mrq *entry;
487 unsigned long flags;
488
489 spin_lock_irqsave(&bpmp->lock, flags);
490
491 entry = tegra_bpmp_find_mrq(bpmp, mrq);
492 if (!entry)
493 goto unlock;
494
495 list_del(&entry->list);
496 devm_kfree(bpmp->dev, entry);
497
498unlock:
499 spin_unlock_irqrestore(&bpmp->lock, flags);
500}
501EXPORT_SYMBOL_GPL(tegra_bpmp_free_mrq);
502
503bool tegra_bpmp_mrq_is_supported(struct tegra_bpmp *bpmp, unsigned int mrq)
504{
505 struct mrq_query_abi_request req = { .mrq = mrq };
506 struct mrq_query_abi_response resp;
507 struct tegra_bpmp_message msg = {
508 .mrq = MRQ_QUERY_ABI,
509 .tx = {
510 .data = &req,
511 .size = sizeof(req),
512 },
513 .rx = {
514 .data = &resp,
515 .size = sizeof(resp),
516 },
517 };
518 int err;
519
520 err = tegra_bpmp_transfer(bpmp, &msg);
521 if (err || msg.rx.ret)
522 return false;
523
524 return resp.status == 0;
525}
526EXPORT_SYMBOL_GPL(tegra_bpmp_mrq_is_supported);
527
528static void tegra_bpmp_mrq_handle_ping(unsigned int mrq,
529 struct tegra_bpmp_channel *channel,
530 void *data)
531{
532 struct mrq_ping_request request;
533 struct mrq_ping_response response;
534
535 tegra_bpmp_mb_read(&request, &channel->ib, sizeof(request));
536
537 memset(&response, 0, sizeof(response));
538 response.reply = request.challenge << 1;
539
540 tegra_bpmp_mrq_return(channel, 0, &response, sizeof(response));
541}
542
543static int tegra_bpmp_ping(struct tegra_bpmp *bpmp)
544{
545 struct mrq_ping_response response;
546 struct mrq_ping_request request;
547 struct tegra_bpmp_message msg;
548 unsigned long flags;
549 ktime_t start, end;
550 int err;
551
552 memset(&request, 0, sizeof(request));
553 request.challenge = 1;
554
555 memset(&response, 0, sizeof(response));
556
557 memset(&msg, 0, sizeof(msg));
558 msg.mrq = MRQ_PING;
559 msg.tx.data = &request;
560 msg.tx.size = sizeof(request);
561 msg.rx.data = &response;
562 msg.rx.size = sizeof(response);
563
564 local_irq_save(flags);
565 start = ktime_get();
566 err = tegra_bpmp_transfer_atomic(bpmp, &msg);
567 end = ktime_get();
568 local_irq_restore(flags);
569
570 if (!err)
571 dev_dbg(bpmp->dev,
572 "ping ok: challenge: %u, response: %u, time: %lld\n",
573 request.challenge, response.reply,
574 ktime_to_us(ktime_sub(end, start)));
575
576 return err;
577}
578
579/* deprecated version of tag query */
580static int tegra_bpmp_get_firmware_tag_old(struct tegra_bpmp *bpmp, char *tag,
581 size_t size)
582{
583 struct mrq_query_tag_request request;
584 struct tegra_bpmp_message msg;
585 unsigned long flags;
586 dma_addr_t phys;
587 void *virt;
588 int err;
589
590 if (size != TAG_SZ)
591 return -EINVAL;
592
593 virt = dma_alloc_coherent(bpmp->dev, TAG_SZ, &phys,
594 GFP_KERNEL | GFP_DMA32);
595 if (!virt)
596 return -ENOMEM;
597
598 memset(&request, 0, sizeof(request));
599 request.addr = phys;
600
601 memset(&msg, 0, sizeof(msg));
602 msg.mrq = MRQ_QUERY_TAG;
603 msg.tx.data = &request;
604 msg.tx.size = sizeof(request);
605
606 local_irq_save(flags);
607 err = tegra_bpmp_transfer_atomic(bpmp, &msg);
608 local_irq_restore(flags);
609
610 if (err == 0)
611 memcpy(tag, virt, TAG_SZ);
612
613 dma_free_coherent(bpmp->dev, TAG_SZ, virt, phys);
614
615 return err;
616}
617
618static int tegra_bpmp_get_firmware_tag(struct tegra_bpmp *bpmp, char *tag,
619 size_t size)
620{
621 if (tegra_bpmp_mrq_is_supported(bpmp, MRQ_QUERY_FW_TAG)) {
622 struct mrq_query_fw_tag_response resp;
623 struct tegra_bpmp_message msg = {
624 .mrq = MRQ_QUERY_FW_TAG,
625 .rx = {
626 .data = &resp,
627 .size = sizeof(resp),
628 },
629 };
630 int err;
631
632 if (size != sizeof(resp.tag))
633 return -EINVAL;
634
635 err = tegra_bpmp_transfer(bpmp, &msg);
636
637 if (err)
638 return err;
639 if (msg.rx.ret < 0)
640 return -EINVAL;
641
642 memcpy(tag, resp.tag, sizeof(resp.tag));
643 return 0;
644 }
645
646 return tegra_bpmp_get_firmware_tag_old(bpmp, tag, size);
647}
648
649static void tegra_bpmp_channel_signal(struct tegra_bpmp_channel *channel)
650{
651 unsigned long flags = tegra_bpmp_mb_read_field(&channel->ob, flags);
652
653 if ((flags & MSG_RING) == 0)
654 return;
655
656 complete(&channel->completion);
657}
658
659void tegra_bpmp_handle_rx(struct tegra_bpmp *bpmp)
660{
661 struct tegra_bpmp_channel *channel;
662 unsigned int i, count;
663 unsigned long *busy;
664
665 channel = bpmp->rx_channel;
666 count = bpmp->soc->channels.thread.count;
667 busy = bpmp->threaded.busy;
668
669 if (tegra_bpmp_is_request_ready(channel)) {
670 unsigned int mrq = tegra_bpmp_mb_read_field(&channel->ib, code);
671
672 tegra_bpmp_handle_mrq(bpmp, mrq, channel);
673 }
674
675 spin_lock(&bpmp->lock);
676
677 for_each_set_bit(i, busy, count) {
678 struct tegra_bpmp_channel *channel;
679
680 channel = &bpmp->threaded_channels[i];
681
682 if (tegra_bpmp_is_response_ready(channel)) {
683 tegra_bpmp_channel_signal(channel);
684 clear_bit(i, busy);
685 }
686 }
687
688 spin_unlock(&bpmp->lock);
689}
690
691static int tegra_bpmp_probe(struct platform_device *pdev)
692{
693 struct tegra_bpmp *bpmp;
694 char tag[TAG_SZ];
695 size_t size;
696 int err;
697
698 bpmp = devm_kzalloc(&pdev->dev, sizeof(*bpmp), GFP_KERNEL);
699 if (!bpmp)
700 return -ENOMEM;
701
702 bpmp->soc = of_device_get_match_data(&pdev->dev);
703 bpmp->dev = &pdev->dev;
704
705 INIT_LIST_HEAD(&bpmp->mrqs);
706 spin_lock_init(&bpmp->lock);
707
708 bpmp->threaded.count = bpmp->soc->channels.thread.count;
709 sema_init(&bpmp->threaded.lock, bpmp->threaded.count);
710
711 size = BITS_TO_LONGS(bpmp->threaded.count) * sizeof(long);
712
713 bpmp->threaded.allocated = devm_kzalloc(&pdev->dev, size, GFP_KERNEL);
714 if (!bpmp->threaded.allocated)
715 return -ENOMEM;
716
717 bpmp->threaded.busy = devm_kzalloc(&pdev->dev, size, GFP_KERNEL);
718 if (!bpmp->threaded.busy)
719 return -ENOMEM;
720
721 spin_lock_init(&bpmp->atomic_tx_lock);
722 bpmp->tx_channel = devm_kzalloc(&pdev->dev, sizeof(*bpmp->tx_channel),
723 GFP_KERNEL);
724 if (!bpmp->tx_channel)
725 return -ENOMEM;
726
727 bpmp->rx_channel = devm_kzalloc(&pdev->dev, sizeof(*bpmp->rx_channel),
728 GFP_KERNEL);
729 if (!bpmp->rx_channel)
730 return -ENOMEM;
731
732 bpmp->threaded_channels = devm_kcalloc(&pdev->dev, bpmp->threaded.count,
733 sizeof(*bpmp->threaded_channels),
734 GFP_KERNEL);
735 if (!bpmp->threaded_channels)
736 return -ENOMEM;
737
738 err = bpmp->soc->ops->init(bpmp);
739 if (err < 0)
740 return err;
741
742 err = tegra_bpmp_request_mrq(bpmp, MRQ_PING,
743 tegra_bpmp_mrq_handle_ping, bpmp);
744 if (err < 0)
745 goto deinit;
746
747 err = tegra_bpmp_ping(bpmp);
748 if (err < 0) {
749 dev_err(&pdev->dev, "failed to ping BPMP: %d\n", err);
750 goto free_mrq;
751 }
752
753 err = tegra_bpmp_get_firmware_tag(bpmp, tag, sizeof(tag));
754 if (err < 0) {
755 dev_err(&pdev->dev, "failed to get firmware tag: %d\n", err);
756 goto free_mrq;
757 }
758
759 dev_info(&pdev->dev, "firmware: %.*s\n", (int)sizeof(tag), tag);
760
761 platform_set_drvdata(pdev, bpmp);
762
763 err = of_platform_default_populate(pdev->dev.of_node, NULL, &pdev->dev);
764 if (err < 0)
765 goto free_mrq;
766
767 if (of_find_property(pdev->dev.of_node, "#clock-cells", NULL)) {
768 err = tegra_bpmp_init_clocks(bpmp);
769 if (err < 0)
770 goto free_mrq;
771 }
772
773 if (of_find_property(pdev->dev.of_node, "#reset-cells", NULL)) {
774 err = tegra_bpmp_init_resets(bpmp);
775 if (err < 0)
776 goto free_mrq;
777 }
778
779 if (of_find_property(pdev->dev.of_node, "#power-domain-cells", NULL)) {
780 err = tegra_bpmp_init_powergates(bpmp);
781 if (err < 0)
782 goto free_mrq;
783 }
784
785 err = tegra_bpmp_init_debugfs(bpmp);
786 if (err < 0)
787 dev_err(&pdev->dev, "debugfs initialization failed: %d\n", err);
788
789 return 0;
790
791free_mrq:
792 tegra_bpmp_free_mrq(bpmp, MRQ_PING, bpmp);
793deinit:
794 if (bpmp->soc->ops->deinit)
795 bpmp->soc->ops->deinit(bpmp);
796
797 return err;
798}
799
800static int __maybe_unused tegra_bpmp_resume(struct device *dev)
801{
802 struct tegra_bpmp *bpmp = dev_get_drvdata(dev);
803
804 if (bpmp->soc->ops->resume)
805 return bpmp->soc->ops->resume(bpmp);
806 else
807 return 0;
808}
809
810static const struct dev_pm_ops tegra_bpmp_pm_ops = {
811 .resume_noirq = tegra_bpmp_resume,
812};
813
814#if IS_ENABLED(CONFIG_ARCH_TEGRA_186_SOC) || \
815 IS_ENABLED(CONFIG_ARCH_TEGRA_194_SOC) || \
816 IS_ENABLED(CONFIG_ARCH_TEGRA_234_SOC)
817static const struct tegra_bpmp_soc tegra186_soc = {
818 .channels = {
819 .cpu_tx = {
820 .offset = 3,
821 .timeout = 60 * USEC_PER_SEC,
822 },
823 .thread = {
824 .offset = 0,
825 .count = 3,
826 .timeout = 600 * USEC_PER_SEC,
827 },
828 .cpu_rx = {
829 .offset = 13,
830 .timeout = 0,
831 },
832 },
833 .ops = &tegra186_bpmp_ops,
834 .num_resets = 193,
835};
836#endif
837
838#if IS_ENABLED(CONFIG_ARCH_TEGRA_210_SOC)
839static const struct tegra_bpmp_soc tegra210_soc = {
840 .channels = {
841 .cpu_tx = {
842 .offset = 0,
843 .count = 1,
844 .timeout = 60 * USEC_PER_SEC,
845 },
846 .thread = {
847 .offset = 4,
848 .count = 1,
849 .timeout = 600 * USEC_PER_SEC,
850 },
851 .cpu_rx = {
852 .offset = 8,
853 .count = 1,
854 .timeout = 0,
855 },
856 },
857 .ops = &tegra210_bpmp_ops,
858};
859#endif
860
861static const struct of_device_id tegra_bpmp_match[] = {
862#if IS_ENABLED(CONFIG_ARCH_TEGRA_186_SOC) || \
863 IS_ENABLED(CONFIG_ARCH_TEGRA_194_SOC) || \
864 IS_ENABLED(CONFIG_ARCH_TEGRA_234_SOC)
865 { .compatible = "nvidia,tegra186-bpmp", .data = &tegra186_soc },
866#endif
867#if IS_ENABLED(CONFIG_ARCH_TEGRA_210_SOC)
868 { .compatible = "nvidia,tegra210-bpmp", .data = &tegra210_soc },
869#endif
870 { }
871};
872
873static struct platform_driver tegra_bpmp_driver = {
874 .driver = {
875 .name = "tegra-bpmp",
876 .of_match_table = tegra_bpmp_match,
877 .pm = &tegra_bpmp_pm_ops,
878 .suppress_bind_attrs = true,
879 },
880 .probe = tegra_bpmp_probe,
881};
882builtin_platform_driver(tegra_bpmp_driver);