Loading...
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Copyright (c) 2016, NVIDIA CORPORATION. All rights reserved.
4 */
5
6#include <linux/clk/tegra.h>
7#include <linux/genalloc.h>
8#include <linux/mailbox_client.h>
9#include <linux/module.h>
10#include <linux/of.h>
11#include <linux/of_platform.h>
12#include <linux/platform_device.h>
13#include <linux/pm.h>
14#include <linux/semaphore.h>
15#include <linux/sched/clock.h>
16
17#include <soc/tegra/bpmp.h>
18#include <soc/tegra/bpmp-abi.h>
19#include <soc/tegra/ivc.h>
20
21#include "bpmp-private.h"
22
23#define MSG_ACK BIT(0)
24#define MSG_RING BIT(1)
25#define TAG_SZ 32
26
27static inline const struct tegra_bpmp_ops *
28channel_to_ops(struct tegra_bpmp_channel *channel)
29{
30 struct tegra_bpmp *bpmp = channel->bpmp;
31
32 return bpmp->soc->ops;
33}
34
35struct tegra_bpmp *tegra_bpmp_get(struct device *dev)
36{
37 struct platform_device *pdev;
38 struct tegra_bpmp *bpmp;
39 struct device_node *np;
40
41 np = of_parse_phandle(dev->of_node, "nvidia,bpmp", 0);
42 if (!np)
43 return ERR_PTR(-ENOENT);
44
45 pdev = of_find_device_by_node(np);
46 if (!pdev) {
47 bpmp = ERR_PTR(-ENODEV);
48 goto put;
49 }
50
51 bpmp = platform_get_drvdata(pdev);
52 if (!bpmp) {
53 bpmp = ERR_PTR(-EPROBE_DEFER);
54 put_device(&pdev->dev);
55 goto put;
56 }
57
58put:
59 of_node_put(np);
60 return bpmp;
61}
62EXPORT_SYMBOL_GPL(tegra_bpmp_get);
63
64void tegra_bpmp_put(struct tegra_bpmp *bpmp)
65{
66 if (bpmp)
67 put_device(bpmp->dev);
68}
69EXPORT_SYMBOL_GPL(tegra_bpmp_put);
70
71static int
72tegra_bpmp_channel_get_thread_index(struct tegra_bpmp_channel *channel)
73{
74 struct tegra_bpmp *bpmp = channel->bpmp;
75 unsigned int count;
76 int index;
77
78 count = bpmp->soc->channels.thread.count;
79
80 index = channel - channel->bpmp->threaded_channels;
81 if (index < 0 || index >= count)
82 return -EINVAL;
83
84 return index;
85}
86
87static bool tegra_bpmp_message_valid(const struct tegra_bpmp_message *msg)
88{
89 return (msg->tx.size <= MSG_DATA_MIN_SZ) &&
90 (msg->rx.size <= MSG_DATA_MIN_SZ) &&
91 (msg->tx.size == 0 || msg->tx.data) &&
92 (msg->rx.size == 0 || msg->rx.data);
93}
94
95static bool tegra_bpmp_is_response_ready(struct tegra_bpmp_channel *channel)
96{
97 const struct tegra_bpmp_ops *ops = channel_to_ops(channel);
98
99 return ops->is_response_ready(channel);
100}
101
102static bool tegra_bpmp_is_request_ready(struct tegra_bpmp_channel *channel)
103{
104 const struct tegra_bpmp_ops *ops = channel_to_ops(channel);
105
106 return ops->is_request_ready(channel);
107}
108
109static int tegra_bpmp_wait_response(struct tegra_bpmp_channel *channel)
110{
111 unsigned long timeout = channel->bpmp->soc->channels.cpu_tx.timeout;
112 ktime_t end;
113
114 end = ktime_add_us(ktime_get(), timeout);
115
116 do {
117 if (tegra_bpmp_is_response_ready(channel))
118 return 0;
119 } while (ktime_before(ktime_get(), end));
120
121 return -ETIMEDOUT;
122}
123
124static int tegra_bpmp_ack_response(struct tegra_bpmp_channel *channel)
125{
126 const struct tegra_bpmp_ops *ops = channel_to_ops(channel);
127
128 return ops->ack_response(channel);
129}
130
131static int tegra_bpmp_ack_request(struct tegra_bpmp_channel *channel)
132{
133 const struct tegra_bpmp_ops *ops = channel_to_ops(channel);
134
135 return ops->ack_request(channel);
136}
137
138static bool
139tegra_bpmp_is_request_channel_free(struct tegra_bpmp_channel *channel)
140{
141 const struct tegra_bpmp_ops *ops = channel_to_ops(channel);
142
143 return ops->is_request_channel_free(channel);
144}
145
146static bool
147tegra_bpmp_is_response_channel_free(struct tegra_bpmp_channel *channel)
148{
149 const struct tegra_bpmp_ops *ops = channel_to_ops(channel);
150
151 return ops->is_response_channel_free(channel);
152}
153
154static int
155tegra_bpmp_wait_request_channel_free(struct tegra_bpmp_channel *channel)
156{
157 unsigned long timeout = channel->bpmp->soc->channels.cpu_tx.timeout;
158 ktime_t start, now;
159
160 start = ns_to_ktime(local_clock());
161
162 do {
163 if (tegra_bpmp_is_request_channel_free(channel))
164 return 0;
165
166 now = ns_to_ktime(local_clock());
167 } while (ktime_us_delta(now, start) < timeout);
168
169 return -ETIMEDOUT;
170}
171
172static int tegra_bpmp_post_request(struct tegra_bpmp_channel *channel)
173{
174 const struct tegra_bpmp_ops *ops = channel_to_ops(channel);
175
176 return ops->post_request(channel);
177}
178
179static int tegra_bpmp_post_response(struct tegra_bpmp_channel *channel)
180{
181 const struct tegra_bpmp_ops *ops = channel_to_ops(channel);
182
183 return ops->post_response(channel);
184}
185
186static int tegra_bpmp_ring_doorbell(struct tegra_bpmp *bpmp)
187{
188 return bpmp->soc->ops->ring_doorbell(bpmp);
189}
190
191static ssize_t __tegra_bpmp_channel_read(struct tegra_bpmp_channel *channel,
192 void *data, size_t size, int *ret)
193{
194 int err;
195
196 if (data && size > 0)
197 tegra_bpmp_mb_read(data, &channel->ib, size);
198
199 err = tegra_bpmp_ack_response(channel);
200 if (err < 0)
201 return err;
202
203 *ret = tegra_bpmp_mb_read_field(&channel->ib, code);
204
205 return 0;
206}
207
208static ssize_t tegra_bpmp_channel_read(struct tegra_bpmp_channel *channel,
209 void *data, size_t size, int *ret)
210{
211 struct tegra_bpmp *bpmp = channel->bpmp;
212 unsigned long flags;
213 ssize_t err;
214 int index;
215
216 index = tegra_bpmp_channel_get_thread_index(channel);
217 if (index < 0) {
218 err = index;
219 goto unlock;
220 }
221
222 spin_lock_irqsave(&bpmp->lock, flags);
223 err = __tegra_bpmp_channel_read(channel, data, size, ret);
224 clear_bit(index, bpmp->threaded.allocated);
225 spin_unlock_irqrestore(&bpmp->lock, flags);
226
227unlock:
228 up(&bpmp->threaded.lock);
229
230 return err;
231}
232
233static ssize_t __tegra_bpmp_channel_write(struct tegra_bpmp_channel *channel,
234 unsigned int mrq, unsigned long flags,
235 const void *data, size_t size)
236{
237 tegra_bpmp_mb_write_field(&channel->ob, code, mrq);
238 tegra_bpmp_mb_write_field(&channel->ob, flags, flags);
239
240 if (data && size > 0)
241 tegra_bpmp_mb_write(&channel->ob, data, size);
242
243 return tegra_bpmp_post_request(channel);
244}
245
246static struct tegra_bpmp_channel *
247tegra_bpmp_write_threaded(struct tegra_bpmp *bpmp, unsigned int mrq,
248 const void *data, size_t size)
249{
250 unsigned long timeout = bpmp->soc->channels.thread.timeout;
251 unsigned int count = bpmp->soc->channels.thread.count;
252 struct tegra_bpmp_channel *channel;
253 unsigned long flags;
254 unsigned int index;
255 int err;
256
257 err = down_timeout(&bpmp->threaded.lock, usecs_to_jiffies(timeout));
258 if (err < 0)
259 return ERR_PTR(err);
260
261 spin_lock_irqsave(&bpmp->lock, flags);
262
263 index = find_first_zero_bit(bpmp->threaded.allocated, count);
264 if (index == count) {
265 err = -EBUSY;
266 goto unlock;
267 }
268
269 channel = &bpmp->threaded_channels[index];
270
271 if (!tegra_bpmp_is_request_channel_free(channel)) {
272 err = -EBUSY;
273 goto unlock;
274 }
275
276 set_bit(index, bpmp->threaded.allocated);
277
278 err = __tegra_bpmp_channel_write(channel, mrq, MSG_ACK | MSG_RING,
279 data, size);
280 if (err < 0)
281 goto clear_allocated;
282
283 set_bit(index, bpmp->threaded.busy);
284
285 spin_unlock_irqrestore(&bpmp->lock, flags);
286 return channel;
287
288clear_allocated:
289 clear_bit(index, bpmp->threaded.allocated);
290unlock:
291 spin_unlock_irqrestore(&bpmp->lock, flags);
292 up(&bpmp->threaded.lock);
293
294 return ERR_PTR(err);
295}
296
297static ssize_t tegra_bpmp_channel_write(struct tegra_bpmp_channel *channel,
298 unsigned int mrq, unsigned long flags,
299 const void *data, size_t size)
300{
301 int err;
302
303 err = tegra_bpmp_wait_request_channel_free(channel);
304 if (err < 0)
305 return err;
306
307 return __tegra_bpmp_channel_write(channel, mrq, flags, data, size);
308}
309
310static int __maybe_unused tegra_bpmp_resume(struct device *dev);
311
312int tegra_bpmp_transfer_atomic(struct tegra_bpmp *bpmp,
313 struct tegra_bpmp_message *msg)
314{
315 struct tegra_bpmp_channel *channel;
316 int err;
317
318 if (WARN_ON(!irqs_disabled()))
319 return -EPERM;
320
321 if (!tegra_bpmp_message_valid(msg))
322 return -EINVAL;
323
324 if (bpmp->suspended) {
325 /* Reset BPMP IPC channels during resume based on flags passed */
326 if (msg->flags & TEGRA_BPMP_MESSAGE_RESET)
327 tegra_bpmp_resume(bpmp->dev);
328 else
329 return -EAGAIN;
330 }
331
332 channel = bpmp->tx_channel;
333
334 spin_lock(&bpmp->atomic_tx_lock);
335
336 err = tegra_bpmp_channel_write(channel, msg->mrq, MSG_ACK,
337 msg->tx.data, msg->tx.size);
338 if (err < 0) {
339 spin_unlock(&bpmp->atomic_tx_lock);
340 return err;
341 }
342
343 spin_unlock(&bpmp->atomic_tx_lock);
344
345 err = tegra_bpmp_ring_doorbell(bpmp);
346 if (err < 0)
347 return err;
348
349 err = tegra_bpmp_wait_response(channel);
350 if (err < 0)
351 return err;
352
353 return __tegra_bpmp_channel_read(channel, msg->rx.data, msg->rx.size,
354 &msg->rx.ret);
355}
356EXPORT_SYMBOL_GPL(tegra_bpmp_transfer_atomic);
357
358int tegra_bpmp_transfer(struct tegra_bpmp *bpmp,
359 struct tegra_bpmp_message *msg)
360{
361 struct tegra_bpmp_channel *channel;
362 unsigned long timeout;
363 int err;
364
365 if (WARN_ON(irqs_disabled()))
366 return -EPERM;
367
368 if (!tegra_bpmp_message_valid(msg))
369 return -EINVAL;
370
371 if (bpmp->suspended) {
372 /* Reset BPMP IPC channels during resume based on flags passed */
373 if (msg->flags & TEGRA_BPMP_MESSAGE_RESET)
374 tegra_bpmp_resume(bpmp->dev);
375 else
376 return -EAGAIN;
377 }
378
379 channel = tegra_bpmp_write_threaded(bpmp, msg->mrq, msg->tx.data,
380 msg->tx.size);
381 if (IS_ERR(channel))
382 return PTR_ERR(channel);
383
384 err = tegra_bpmp_ring_doorbell(bpmp);
385 if (err < 0)
386 return err;
387
388 timeout = usecs_to_jiffies(bpmp->soc->channels.thread.timeout);
389
390 err = wait_for_completion_timeout(&channel->completion, timeout);
391 if (err == 0)
392 return -ETIMEDOUT;
393
394 return tegra_bpmp_channel_read(channel, msg->rx.data, msg->rx.size,
395 &msg->rx.ret);
396}
397EXPORT_SYMBOL_GPL(tegra_bpmp_transfer);
398
399static struct tegra_bpmp_mrq *tegra_bpmp_find_mrq(struct tegra_bpmp *bpmp,
400 unsigned int mrq)
401{
402 struct tegra_bpmp_mrq *entry;
403
404 list_for_each_entry(entry, &bpmp->mrqs, list)
405 if (entry->mrq == mrq)
406 return entry;
407
408 return NULL;
409}
410
411void tegra_bpmp_mrq_return(struct tegra_bpmp_channel *channel, int code,
412 const void *data, size_t size)
413{
414 unsigned long flags = tegra_bpmp_mb_read_field(&channel->ib, flags);
415 struct tegra_bpmp *bpmp = channel->bpmp;
416 int err;
417
418 if (WARN_ON(size > MSG_DATA_MIN_SZ))
419 return;
420
421 err = tegra_bpmp_ack_request(channel);
422 if (WARN_ON(err < 0))
423 return;
424
425 if ((flags & MSG_ACK) == 0)
426 return;
427
428 if (WARN_ON(!tegra_bpmp_is_response_channel_free(channel)))
429 return;
430
431 tegra_bpmp_mb_write_field(&channel->ob, code, code);
432
433 if (data && size > 0)
434 tegra_bpmp_mb_write(&channel->ob, data, size);
435
436 err = tegra_bpmp_post_response(channel);
437 if (WARN_ON(err < 0))
438 return;
439
440 if (flags & MSG_RING) {
441 err = tegra_bpmp_ring_doorbell(bpmp);
442 if (WARN_ON(err < 0))
443 return;
444 }
445}
446EXPORT_SYMBOL_GPL(tegra_bpmp_mrq_return);
447
448static void tegra_bpmp_handle_mrq(struct tegra_bpmp *bpmp,
449 unsigned int mrq,
450 struct tegra_bpmp_channel *channel)
451{
452 struct tegra_bpmp_mrq *entry;
453 u32 zero = 0;
454
455 spin_lock(&bpmp->lock);
456
457 entry = tegra_bpmp_find_mrq(bpmp, mrq);
458 if (!entry) {
459 spin_unlock(&bpmp->lock);
460 tegra_bpmp_mrq_return(channel, -EINVAL, &zero, sizeof(zero));
461 return;
462 }
463
464 entry->handler(mrq, channel, entry->data);
465
466 spin_unlock(&bpmp->lock);
467}
468
469int tegra_bpmp_request_mrq(struct tegra_bpmp *bpmp, unsigned int mrq,
470 tegra_bpmp_mrq_handler_t handler, void *data)
471{
472 struct tegra_bpmp_mrq *entry;
473 unsigned long flags;
474
475 if (!handler)
476 return -EINVAL;
477
478 entry = devm_kzalloc(bpmp->dev, sizeof(*entry), GFP_KERNEL);
479 if (!entry)
480 return -ENOMEM;
481
482 spin_lock_irqsave(&bpmp->lock, flags);
483
484 entry->mrq = mrq;
485 entry->handler = handler;
486 entry->data = data;
487 list_add(&entry->list, &bpmp->mrqs);
488
489 spin_unlock_irqrestore(&bpmp->lock, flags);
490
491 return 0;
492}
493EXPORT_SYMBOL_GPL(tegra_bpmp_request_mrq);
494
495void tegra_bpmp_free_mrq(struct tegra_bpmp *bpmp, unsigned int mrq, void *data)
496{
497 struct tegra_bpmp_mrq *entry;
498 unsigned long flags;
499
500 spin_lock_irqsave(&bpmp->lock, flags);
501
502 entry = tegra_bpmp_find_mrq(bpmp, mrq);
503 if (!entry)
504 goto unlock;
505
506 list_del(&entry->list);
507 devm_kfree(bpmp->dev, entry);
508
509unlock:
510 spin_unlock_irqrestore(&bpmp->lock, flags);
511}
512EXPORT_SYMBOL_GPL(tegra_bpmp_free_mrq);
513
514bool tegra_bpmp_mrq_is_supported(struct tegra_bpmp *bpmp, unsigned int mrq)
515{
516 struct mrq_query_abi_request req = { .mrq = mrq };
517 struct mrq_query_abi_response resp;
518 struct tegra_bpmp_message msg = {
519 .mrq = MRQ_QUERY_ABI,
520 .tx = {
521 .data = &req,
522 .size = sizeof(req),
523 },
524 .rx = {
525 .data = &resp,
526 .size = sizeof(resp),
527 },
528 };
529 int err;
530
531 err = tegra_bpmp_transfer(bpmp, &msg);
532 if (err || msg.rx.ret)
533 return false;
534
535 return resp.status == 0;
536}
537EXPORT_SYMBOL_GPL(tegra_bpmp_mrq_is_supported);
538
539static void tegra_bpmp_mrq_handle_ping(unsigned int mrq,
540 struct tegra_bpmp_channel *channel,
541 void *data)
542{
543 struct mrq_ping_request request;
544 struct mrq_ping_response response;
545
546 tegra_bpmp_mb_read(&request, &channel->ib, sizeof(request));
547
548 memset(&response, 0, sizeof(response));
549 response.reply = request.challenge << 1;
550
551 tegra_bpmp_mrq_return(channel, 0, &response, sizeof(response));
552}
553
554static int tegra_bpmp_ping(struct tegra_bpmp *bpmp)
555{
556 struct mrq_ping_response response;
557 struct mrq_ping_request request;
558 struct tegra_bpmp_message msg;
559 unsigned long flags;
560 ktime_t start, end;
561 int err;
562
563 memset(&request, 0, sizeof(request));
564 request.challenge = 1;
565
566 memset(&response, 0, sizeof(response));
567
568 memset(&msg, 0, sizeof(msg));
569 msg.mrq = MRQ_PING;
570 msg.tx.data = &request;
571 msg.tx.size = sizeof(request);
572 msg.rx.data = &response;
573 msg.rx.size = sizeof(response);
574
575 local_irq_save(flags);
576 start = ktime_get();
577 err = tegra_bpmp_transfer_atomic(bpmp, &msg);
578 end = ktime_get();
579 local_irq_restore(flags);
580
581 if (!err)
582 dev_dbg(bpmp->dev,
583 "ping ok: challenge: %u, response: %u, time: %lld\n",
584 request.challenge, response.reply,
585 ktime_to_us(ktime_sub(end, start)));
586
587 return err;
588}
589
590/* deprecated version of tag query */
591static int tegra_bpmp_get_firmware_tag_old(struct tegra_bpmp *bpmp, char *tag,
592 size_t size)
593{
594 struct mrq_query_tag_request request;
595 struct tegra_bpmp_message msg;
596 unsigned long flags;
597 dma_addr_t phys;
598 void *virt;
599 int err;
600
601 if (size != TAG_SZ)
602 return -EINVAL;
603
604 virt = dma_alloc_coherent(bpmp->dev, TAG_SZ, &phys,
605 GFP_KERNEL | GFP_DMA32);
606 if (!virt)
607 return -ENOMEM;
608
609 memset(&request, 0, sizeof(request));
610 request.addr = phys;
611
612 memset(&msg, 0, sizeof(msg));
613 msg.mrq = MRQ_QUERY_TAG;
614 msg.tx.data = &request;
615 msg.tx.size = sizeof(request);
616
617 local_irq_save(flags);
618 err = tegra_bpmp_transfer_atomic(bpmp, &msg);
619 local_irq_restore(flags);
620
621 if (err == 0)
622 memcpy(tag, virt, TAG_SZ);
623
624 dma_free_coherent(bpmp->dev, TAG_SZ, virt, phys);
625
626 return err;
627}
628
629static int tegra_bpmp_get_firmware_tag(struct tegra_bpmp *bpmp, char *tag,
630 size_t size)
631{
632 if (tegra_bpmp_mrq_is_supported(bpmp, MRQ_QUERY_FW_TAG)) {
633 struct mrq_query_fw_tag_response resp;
634 struct tegra_bpmp_message msg = {
635 .mrq = MRQ_QUERY_FW_TAG,
636 .rx = {
637 .data = &resp,
638 .size = sizeof(resp),
639 },
640 };
641 int err;
642
643 if (size != sizeof(resp.tag))
644 return -EINVAL;
645
646 err = tegra_bpmp_transfer(bpmp, &msg);
647
648 if (err)
649 return err;
650 if (msg.rx.ret < 0)
651 return -EINVAL;
652
653 memcpy(tag, resp.tag, sizeof(resp.tag));
654 return 0;
655 }
656
657 return tegra_bpmp_get_firmware_tag_old(bpmp, tag, size);
658}
659
660static void tegra_bpmp_channel_signal(struct tegra_bpmp_channel *channel)
661{
662 unsigned long flags = tegra_bpmp_mb_read_field(&channel->ob, flags);
663
664 if ((flags & MSG_RING) == 0)
665 return;
666
667 complete(&channel->completion);
668}
669
670void tegra_bpmp_handle_rx(struct tegra_bpmp *bpmp)
671{
672 struct tegra_bpmp_channel *channel;
673 unsigned int i, count;
674 unsigned long *busy;
675
676 channel = bpmp->rx_channel;
677 count = bpmp->soc->channels.thread.count;
678 busy = bpmp->threaded.busy;
679
680 if (tegra_bpmp_is_request_ready(channel)) {
681 unsigned int mrq = tegra_bpmp_mb_read_field(&channel->ib, code);
682
683 tegra_bpmp_handle_mrq(bpmp, mrq, channel);
684 }
685
686 spin_lock(&bpmp->lock);
687
688 for_each_set_bit(i, busy, count) {
689 struct tegra_bpmp_channel *channel;
690
691 channel = &bpmp->threaded_channels[i];
692
693 if (tegra_bpmp_is_response_ready(channel)) {
694 tegra_bpmp_channel_signal(channel);
695 clear_bit(i, busy);
696 }
697 }
698
699 spin_unlock(&bpmp->lock);
700}
701
702static int tegra_bpmp_probe(struct platform_device *pdev)
703{
704 struct tegra_bpmp *bpmp;
705 char tag[TAG_SZ];
706 size_t size;
707 int err;
708
709 bpmp = devm_kzalloc(&pdev->dev, sizeof(*bpmp), GFP_KERNEL);
710 if (!bpmp)
711 return -ENOMEM;
712
713 bpmp->soc = of_device_get_match_data(&pdev->dev);
714 bpmp->dev = &pdev->dev;
715
716 INIT_LIST_HEAD(&bpmp->mrqs);
717 spin_lock_init(&bpmp->lock);
718
719 bpmp->threaded.count = bpmp->soc->channels.thread.count;
720 sema_init(&bpmp->threaded.lock, bpmp->threaded.count);
721
722 size = BITS_TO_LONGS(bpmp->threaded.count) * sizeof(long);
723
724 bpmp->threaded.allocated = devm_kzalloc(&pdev->dev, size, GFP_KERNEL);
725 if (!bpmp->threaded.allocated)
726 return -ENOMEM;
727
728 bpmp->threaded.busy = devm_kzalloc(&pdev->dev, size, GFP_KERNEL);
729 if (!bpmp->threaded.busy)
730 return -ENOMEM;
731
732 spin_lock_init(&bpmp->atomic_tx_lock);
733 bpmp->tx_channel = devm_kzalloc(&pdev->dev, sizeof(*bpmp->tx_channel),
734 GFP_KERNEL);
735 if (!bpmp->tx_channel)
736 return -ENOMEM;
737
738 bpmp->rx_channel = devm_kzalloc(&pdev->dev, sizeof(*bpmp->rx_channel),
739 GFP_KERNEL);
740 if (!bpmp->rx_channel)
741 return -ENOMEM;
742
743 bpmp->threaded_channels = devm_kcalloc(&pdev->dev, bpmp->threaded.count,
744 sizeof(*bpmp->threaded_channels),
745 GFP_KERNEL);
746 if (!bpmp->threaded_channels)
747 return -ENOMEM;
748
749 platform_set_drvdata(pdev, bpmp);
750
751 err = bpmp->soc->ops->init(bpmp);
752 if (err < 0)
753 return err;
754
755 err = tegra_bpmp_request_mrq(bpmp, MRQ_PING,
756 tegra_bpmp_mrq_handle_ping, bpmp);
757 if (err < 0)
758 goto deinit;
759
760 err = tegra_bpmp_ping(bpmp);
761 if (err < 0) {
762 dev_err(&pdev->dev, "failed to ping BPMP: %d\n", err);
763 goto free_mrq;
764 }
765
766 err = tegra_bpmp_get_firmware_tag(bpmp, tag, sizeof(tag));
767 if (err < 0) {
768 dev_err(&pdev->dev, "failed to get firmware tag: %d\n", err);
769 goto free_mrq;
770 }
771
772 dev_info(&pdev->dev, "firmware: %.*s\n", (int)sizeof(tag), tag);
773
774 err = of_platform_default_populate(pdev->dev.of_node, NULL, &pdev->dev);
775 if (err < 0)
776 goto free_mrq;
777
778 if (of_property_present(pdev->dev.of_node, "#clock-cells")) {
779 err = tegra_bpmp_init_clocks(bpmp);
780 if (err < 0)
781 goto free_mrq;
782 }
783
784 if (of_property_present(pdev->dev.of_node, "#reset-cells")) {
785 err = tegra_bpmp_init_resets(bpmp);
786 if (err < 0)
787 goto free_mrq;
788 }
789
790 if (of_property_present(pdev->dev.of_node, "#power-domain-cells")) {
791 err = tegra_bpmp_init_powergates(bpmp);
792 if (err < 0)
793 goto free_mrq;
794 }
795
796 err = tegra_bpmp_init_debugfs(bpmp);
797 if (err < 0)
798 dev_err(&pdev->dev, "debugfs initialization failed: %d\n", err);
799
800 return 0;
801
802free_mrq:
803 tegra_bpmp_free_mrq(bpmp, MRQ_PING, bpmp);
804deinit:
805 if (bpmp->soc->ops->deinit)
806 bpmp->soc->ops->deinit(bpmp);
807
808 return err;
809}
810
811static int __maybe_unused tegra_bpmp_suspend(struct device *dev)
812{
813 struct tegra_bpmp *bpmp = dev_get_drvdata(dev);
814
815 bpmp->suspended = true;
816
817 return 0;
818}
819
820static int __maybe_unused tegra_bpmp_resume(struct device *dev)
821{
822 struct tegra_bpmp *bpmp = dev_get_drvdata(dev);
823
824 bpmp->suspended = false;
825
826 if (bpmp->soc->ops->resume)
827 return bpmp->soc->ops->resume(bpmp);
828 else
829 return 0;
830}
831
832static const struct dev_pm_ops tegra_bpmp_pm_ops = {
833 .suspend_noirq = tegra_bpmp_suspend,
834 .resume_noirq = tegra_bpmp_resume,
835};
836
837#if IS_ENABLED(CONFIG_ARCH_TEGRA_186_SOC) || \
838 IS_ENABLED(CONFIG_ARCH_TEGRA_194_SOC) || \
839 IS_ENABLED(CONFIG_ARCH_TEGRA_234_SOC)
840static const struct tegra_bpmp_soc tegra186_soc = {
841 .channels = {
842 .cpu_tx = {
843 .offset = 3,
844 .timeout = 60 * USEC_PER_SEC,
845 },
846 .thread = {
847 .offset = 0,
848 .count = 3,
849 .timeout = 600 * USEC_PER_SEC,
850 },
851 .cpu_rx = {
852 .offset = 13,
853 .timeout = 0,
854 },
855 },
856 .ops = &tegra186_bpmp_ops,
857 .num_resets = 193,
858};
859#endif
860
861#if IS_ENABLED(CONFIG_ARCH_TEGRA_210_SOC)
862static const struct tegra_bpmp_soc tegra210_soc = {
863 .channels = {
864 .cpu_tx = {
865 .offset = 0,
866 .count = 1,
867 .timeout = 60 * USEC_PER_SEC,
868 },
869 .thread = {
870 .offset = 4,
871 .count = 1,
872 .timeout = 600 * USEC_PER_SEC,
873 },
874 .cpu_rx = {
875 .offset = 8,
876 .count = 1,
877 .timeout = 0,
878 },
879 },
880 .ops = &tegra210_bpmp_ops,
881};
882#endif
883
884static const struct of_device_id tegra_bpmp_match[] = {
885#if IS_ENABLED(CONFIG_ARCH_TEGRA_186_SOC) || \
886 IS_ENABLED(CONFIG_ARCH_TEGRA_194_SOC) || \
887 IS_ENABLED(CONFIG_ARCH_TEGRA_234_SOC)
888 { .compatible = "nvidia,tegra186-bpmp", .data = &tegra186_soc },
889#endif
890#if IS_ENABLED(CONFIG_ARCH_TEGRA_210_SOC)
891 { .compatible = "nvidia,tegra210-bpmp", .data = &tegra210_soc },
892#endif
893 { }
894};
895
896static struct platform_driver tegra_bpmp_driver = {
897 .driver = {
898 .name = "tegra-bpmp",
899 .of_match_table = tegra_bpmp_match,
900 .pm = &tegra_bpmp_pm_ops,
901 .suppress_bind_attrs = true,
902 },
903 .probe = tegra_bpmp_probe,
904};
905builtin_platform_driver(tegra_bpmp_driver);
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Copyright (c) 2016, NVIDIA CORPORATION. All rights reserved.
4 */
5
6#include <linux/clk/tegra.h>
7#include <linux/genalloc.h>
8#include <linux/mailbox_client.h>
9#include <linux/module.h>
10#include <linux/of.h>
11#include <linux/of_address.h>
12#include <linux/of_device.h>
13#include <linux/platform_device.h>
14#include <linux/pm.h>
15#include <linux/semaphore.h>
16#include <linux/sched/clock.h>
17
18#include <soc/tegra/bpmp.h>
19#include <soc/tegra/bpmp-abi.h>
20#include <soc/tegra/ivc.h>
21
22#include "bpmp-private.h"
23
24#define MSG_ACK BIT(0)
25#define MSG_RING BIT(1)
26#define TAG_SZ 32
27
28static inline struct tegra_bpmp *
29mbox_client_to_bpmp(struct mbox_client *client)
30{
31 return container_of(client, struct tegra_bpmp, mbox.client);
32}
33
34static inline const struct tegra_bpmp_ops *
35channel_to_ops(struct tegra_bpmp_channel *channel)
36{
37 struct tegra_bpmp *bpmp = channel->bpmp;
38
39 return bpmp->soc->ops;
40}
41
42struct tegra_bpmp *tegra_bpmp_get(struct device *dev)
43{
44 struct platform_device *pdev;
45 struct tegra_bpmp *bpmp;
46 struct device_node *np;
47
48 np = of_parse_phandle(dev->of_node, "nvidia,bpmp", 0);
49 if (!np)
50 return ERR_PTR(-ENOENT);
51
52 pdev = of_find_device_by_node(np);
53 if (!pdev) {
54 bpmp = ERR_PTR(-ENODEV);
55 goto put;
56 }
57
58 bpmp = platform_get_drvdata(pdev);
59 if (!bpmp) {
60 bpmp = ERR_PTR(-EPROBE_DEFER);
61 put_device(&pdev->dev);
62 goto put;
63 }
64
65put:
66 of_node_put(np);
67 return bpmp;
68}
69EXPORT_SYMBOL_GPL(tegra_bpmp_get);
70
71void tegra_bpmp_put(struct tegra_bpmp *bpmp)
72{
73 if (bpmp)
74 put_device(bpmp->dev);
75}
76EXPORT_SYMBOL_GPL(tegra_bpmp_put);
77
78static int
79tegra_bpmp_channel_get_thread_index(struct tegra_bpmp_channel *channel)
80{
81 struct tegra_bpmp *bpmp = channel->bpmp;
82 unsigned int count;
83 int index;
84
85 count = bpmp->soc->channels.thread.count;
86
87 index = channel - channel->bpmp->threaded_channels;
88 if (index < 0 || index >= count)
89 return -EINVAL;
90
91 return index;
92}
93
94static bool tegra_bpmp_message_valid(const struct tegra_bpmp_message *msg)
95{
96 return (msg->tx.size <= MSG_DATA_MIN_SZ) &&
97 (msg->rx.size <= MSG_DATA_MIN_SZ) &&
98 (msg->tx.size == 0 || msg->tx.data) &&
99 (msg->rx.size == 0 || msg->rx.data);
100}
101
102static bool tegra_bpmp_is_response_ready(struct tegra_bpmp_channel *channel)
103{
104 const struct tegra_bpmp_ops *ops = channel_to_ops(channel);
105
106 return ops->is_response_ready(channel);
107}
108
109static bool tegra_bpmp_is_request_ready(struct tegra_bpmp_channel *channel)
110{
111 const struct tegra_bpmp_ops *ops = channel_to_ops(channel);
112
113 return ops->is_request_ready(channel);
114}
115
116static int tegra_bpmp_wait_response(struct tegra_bpmp_channel *channel)
117{
118 unsigned long timeout = channel->bpmp->soc->channels.cpu_tx.timeout;
119 ktime_t end;
120
121 end = ktime_add_us(ktime_get(), timeout);
122
123 do {
124 if (tegra_bpmp_is_response_ready(channel))
125 return 0;
126 } while (ktime_before(ktime_get(), end));
127
128 return -ETIMEDOUT;
129}
130
131static int tegra_bpmp_ack_response(struct tegra_bpmp_channel *channel)
132{
133 const struct tegra_bpmp_ops *ops = channel_to_ops(channel);
134
135 return ops->ack_response(channel);
136}
137
138static int tegra_bpmp_ack_request(struct tegra_bpmp_channel *channel)
139{
140 const struct tegra_bpmp_ops *ops = channel_to_ops(channel);
141
142 return ops->ack_request(channel);
143}
144
145static bool
146tegra_bpmp_is_request_channel_free(struct tegra_bpmp_channel *channel)
147{
148 const struct tegra_bpmp_ops *ops = channel_to_ops(channel);
149
150 return ops->is_request_channel_free(channel);
151}
152
153static bool
154tegra_bpmp_is_response_channel_free(struct tegra_bpmp_channel *channel)
155{
156 const struct tegra_bpmp_ops *ops = channel_to_ops(channel);
157
158 return ops->is_response_channel_free(channel);
159}
160
161static int
162tegra_bpmp_wait_request_channel_free(struct tegra_bpmp_channel *channel)
163{
164 unsigned long timeout = channel->bpmp->soc->channels.cpu_tx.timeout;
165 ktime_t start, now;
166
167 start = ns_to_ktime(local_clock());
168
169 do {
170 if (tegra_bpmp_is_request_channel_free(channel))
171 return 0;
172
173 now = ns_to_ktime(local_clock());
174 } while (ktime_us_delta(now, start) < timeout);
175
176 return -ETIMEDOUT;
177}
178
179static int tegra_bpmp_post_request(struct tegra_bpmp_channel *channel)
180{
181 const struct tegra_bpmp_ops *ops = channel_to_ops(channel);
182
183 return ops->post_request(channel);
184}
185
186static int tegra_bpmp_post_response(struct tegra_bpmp_channel *channel)
187{
188 const struct tegra_bpmp_ops *ops = channel_to_ops(channel);
189
190 return ops->post_response(channel);
191}
192
193static int tegra_bpmp_ring_doorbell(struct tegra_bpmp *bpmp)
194{
195 return bpmp->soc->ops->ring_doorbell(bpmp);
196}
197
198static ssize_t __tegra_bpmp_channel_read(struct tegra_bpmp_channel *channel,
199 void *data, size_t size, int *ret)
200{
201 int err;
202
203 if (data && size > 0)
204 memcpy(data, channel->ib->data, size);
205
206 err = tegra_bpmp_ack_response(channel);
207 if (err < 0)
208 return err;
209
210 *ret = channel->ib->code;
211
212 return 0;
213}
214
215static ssize_t tegra_bpmp_channel_read(struct tegra_bpmp_channel *channel,
216 void *data, size_t size, int *ret)
217{
218 struct tegra_bpmp *bpmp = channel->bpmp;
219 unsigned long flags;
220 ssize_t err;
221 int index;
222
223 index = tegra_bpmp_channel_get_thread_index(channel);
224 if (index < 0) {
225 err = index;
226 goto unlock;
227 }
228
229 spin_lock_irqsave(&bpmp->lock, flags);
230 err = __tegra_bpmp_channel_read(channel, data, size, ret);
231 clear_bit(index, bpmp->threaded.allocated);
232 spin_unlock_irqrestore(&bpmp->lock, flags);
233
234unlock:
235 up(&bpmp->threaded.lock);
236
237 return err;
238}
239
240static ssize_t __tegra_bpmp_channel_write(struct tegra_bpmp_channel *channel,
241 unsigned int mrq, unsigned long flags,
242 const void *data, size_t size)
243{
244 channel->ob->code = mrq;
245 channel->ob->flags = flags;
246
247 if (data && size > 0)
248 memcpy(channel->ob->data, data, size);
249
250 return tegra_bpmp_post_request(channel);
251}
252
253static struct tegra_bpmp_channel *
254tegra_bpmp_write_threaded(struct tegra_bpmp *bpmp, unsigned int mrq,
255 const void *data, size_t size)
256{
257 unsigned long timeout = bpmp->soc->channels.thread.timeout;
258 unsigned int count = bpmp->soc->channels.thread.count;
259 struct tegra_bpmp_channel *channel;
260 unsigned long flags;
261 unsigned int index;
262 int err;
263
264 err = down_timeout(&bpmp->threaded.lock, usecs_to_jiffies(timeout));
265 if (err < 0)
266 return ERR_PTR(err);
267
268 spin_lock_irqsave(&bpmp->lock, flags);
269
270 index = find_first_zero_bit(bpmp->threaded.allocated, count);
271 if (index == count) {
272 err = -EBUSY;
273 goto unlock;
274 }
275
276 channel = &bpmp->threaded_channels[index];
277
278 if (!tegra_bpmp_is_request_channel_free(channel)) {
279 err = -EBUSY;
280 goto unlock;
281 }
282
283 set_bit(index, bpmp->threaded.allocated);
284
285 err = __tegra_bpmp_channel_write(channel, mrq, MSG_ACK | MSG_RING,
286 data, size);
287 if (err < 0)
288 goto clear_allocated;
289
290 set_bit(index, bpmp->threaded.busy);
291
292 spin_unlock_irqrestore(&bpmp->lock, flags);
293 return channel;
294
295clear_allocated:
296 clear_bit(index, bpmp->threaded.allocated);
297unlock:
298 spin_unlock_irqrestore(&bpmp->lock, flags);
299 up(&bpmp->threaded.lock);
300
301 return ERR_PTR(err);
302}
303
304static ssize_t tegra_bpmp_channel_write(struct tegra_bpmp_channel *channel,
305 unsigned int mrq, unsigned long flags,
306 const void *data, size_t size)
307{
308 int err;
309
310 err = tegra_bpmp_wait_request_channel_free(channel);
311 if (err < 0)
312 return err;
313
314 return __tegra_bpmp_channel_write(channel, mrq, flags, data, size);
315}
316
317int tegra_bpmp_transfer_atomic(struct tegra_bpmp *bpmp,
318 struct tegra_bpmp_message *msg)
319{
320 struct tegra_bpmp_channel *channel;
321 int err;
322
323 if (WARN_ON(!irqs_disabled()))
324 return -EPERM;
325
326 if (!tegra_bpmp_message_valid(msg))
327 return -EINVAL;
328
329 channel = bpmp->tx_channel;
330
331 spin_lock(&bpmp->atomic_tx_lock);
332
333 err = tegra_bpmp_channel_write(channel, msg->mrq, MSG_ACK,
334 msg->tx.data, msg->tx.size);
335 if (err < 0) {
336 spin_unlock(&bpmp->atomic_tx_lock);
337 return err;
338 }
339
340 spin_unlock(&bpmp->atomic_tx_lock);
341
342 err = tegra_bpmp_ring_doorbell(bpmp);
343 if (err < 0)
344 return err;
345
346 err = tegra_bpmp_wait_response(channel);
347 if (err < 0)
348 return err;
349
350 return __tegra_bpmp_channel_read(channel, msg->rx.data, msg->rx.size,
351 &msg->rx.ret);
352}
353EXPORT_SYMBOL_GPL(tegra_bpmp_transfer_atomic);
354
355int tegra_bpmp_transfer(struct tegra_bpmp *bpmp,
356 struct tegra_bpmp_message *msg)
357{
358 struct tegra_bpmp_channel *channel;
359 unsigned long timeout;
360 int err;
361
362 if (WARN_ON(irqs_disabled()))
363 return -EPERM;
364
365 if (!tegra_bpmp_message_valid(msg))
366 return -EINVAL;
367
368 channel = tegra_bpmp_write_threaded(bpmp, msg->mrq, msg->tx.data,
369 msg->tx.size);
370 if (IS_ERR(channel))
371 return PTR_ERR(channel);
372
373 err = tegra_bpmp_ring_doorbell(bpmp);
374 if (err < 0)
375 return err;
376
377 timeout = usecs_to_jiffies(bpmp->soc->channels.thread.timeout);
378
379 err = wait_for_completion_timeout(&channel->completion, timeout);
380 if (err == 0)
381 return -ETIMEDOUT;
382
383 return tegra_bpmp_channel_read(channel, msg->rx.data, msg->rx.size,
384 &msg->rx.ret);
385}
386EXPORT_SYMBOL_GPL(tegra_bpmp_transfer);
387
388static struct tegra_bpmp_mrq *tegra_bpmp_find_mrq(struct tegra_bpmp *bpmp,
389 unsigned int mrq)
390{
391 struct tegra_bpmp_mrq *entry;
392
393 list_for_each_entry(entry, &bpmp->mrqs, list)
394 if (entry->mrq == mrq)
395 return entry;
396
397 return NULL;
398}
399
400void tegra_bpmp_mrq_return(struct tegra_bpmp_channel *channel, int code,
401 const void *data, size_t size)
402{
403 unsigned long flags = channel->ib->flags;
404 struct tegra_bpmp *bpmp = channel->bpmp;
405 int err;
406
407 if (WARN_ON(size > MSG_DATA_MIN_SZ))
408 return;
409
410 err = tegra_bpmp_ack_request(channel);
411 if (WARN_ON(err < 0))
412 return;
413
414 if ((flags & MSG_ACK) == 0)
415 return;
416
417 if (WARN_ON(!tegra_bpmp_is_response_channel_free(channel)))
418 return;
419
420 channel->ob->code = code;
421
422 if (data && size > 0)
423 memcpy(channel->ob->data, data, size);
424
425 err = tegra_bpmp_post_response(channel);
426 if (WARN_ON(err < 0))
427 return;
428
429 if (flags & MSG_RING) {
430 err = tegra_bpmp_ring_doorbell(bpmp);
431 if (WARN_ON(err < 0))
432 return;
433 }
434}
435EXPORT_SYMBOL_GPL(tegra_bpmp_mrq_return);
436
437static void tegra_bpmp_handle_mrq(struct tegra_bpmp *bpmp,
438 unsigned int mrq,
439 struct tegra_bpmp_channel *channel)
440{
441 struct tegra_bpmp_mrq *entry;
442 u32 zero = 0;
443
444 spin_lock(&bpmp->lock);
445
446 entry = tegra_bpmp_find_mrq(bpmp, mrq);
447 if (!entry) {
448 spin_unlock(&bpmp->lock);
449 tegra_bpmp_mrq_return(channel, -EINVAL, &zero, sizeof(zero));
450 return;
451 }
452
453 entry->handler(mrq, channel, entry->data);
454
455 spin_unlock(&bpmp->lock);
456}
457
458int tegra_bpmp_request_mrq(struct tegra_bpmp *bpmp, unsigned int mrq,
459 tegra_bpmp_mrq_handler_t handler, void *data)
460{
461 struct tegra_bpmp_mrq *entry;
462 unsigned long flags;
463
464 if (!handler)
465 return -EINVAL;
466
467 entry = devm_kzalloc(bpmp->dev, sizeof(*entry), GFP_KERNEL);
468 if (!entry)
469 return -ENOMEM;
470
471 spin_lock_irqsave(&bpmp->lock, flags);
472
473 entry->mrq = mrq;
474 entry->handler = handler;
475 entry->data = data;
476 list_add(&entry->list, &bpmp->mrqs);
477
478 spin_unlock_irqrestore(&bpmp->lock, flags);
479
480 return 0;
481}
482EXPORT_SYMBOL_GPL(tegra_bpmp_request_mrq);
483
484void tegra_bpmp_free_mrq(struct tegra_bpmp *bpmp, unsigned int mrq, void *data)
485{
486 struct tegra_bpmp_mrq *entry;
487 unsigned long flags;
488
489 spin_lock_irqsave(&bpmp->lock, flags);
490
491 entry = tegra_bpmp_find_mrq(bpmp, mrq);
492 if (!entry)
493 goto unlock;
494
495 list_del(&entry->list);
496 devm_kfree(bpmp->dev, entry);
497
498unlock:
499 spin_unlock_irqrestore(&bpmp->lock, flags);
500}
501EXPORT_SYMBOL_GPL(tegra_bpmp_free_mrq);
502
503bool tegra_bpmp_mrq_is_supported(struct tegra_bpmp *bpmp, unsigned int mrq)
504{
505 struct mrq_query_abi_request req = { .mrq = cpu_to_le32(mrq) };
506 struct mrq_query_abi_response resp;
507 struct tegra_bpmp_message msg = {
508 .mrq = MRQ_QUERY_ABI,
509 .tx = {
510 .data = &req,
511 .size = sizeof(req),
512 },
513 .rx = {
514 .data = &resp,
515 .size = sizeof(resp),
516 },
517 };
518 int err;
519
520 err = tegra_bpmp_transfer(bpmp, &msg);
521 if (err || msg.rx.ret)
522 return false;
523
524 return resp.status == 0;
525}
526EXPORT_SYMBOL_GPL(tegra_bpmp_mrq_is_supported);
527
528static void tegra_bpmp_mrq_handle_ping(unsigned int mrq,
529 struct tegra_bpmp_channel *channel,
530 void *data)
531{
532 struct mrq_ping_request *request;
533 struct mrq_ping_response response;
534
535 request = (struct mrq_ping_request *)channel->ib->data;
536
537 memset(&response, 0, sizeof(response));
538 response.reply = request->challenge << 1;
539
540 tegra_bpmp_mrq_return(channel, 0, &response, sizeof(response));
541}
542
543static int tegra_bpmp_ping(struct tegra_bpmp *bpmp)
544{
545 struct mrq_ping_response response;
546 struct mrq_ping_request request;
547 struct tegra_bpmp_message msg;
548 unsigned long flags;
549 ktime_t start, end;
550 int err;
551
552 memset(&request, 0, sizeof(request));
553 request.challenge = 1;
554
555 memset(&response, 0, sizeof(response));
556
557 memset(&msg, 0, sizeof(msg));
558 msg.mrq = MRQ_PING;
559 msg.tx.data = &request;
560 msg.tx.size = sizeof(request);
561 msg.rx.data = &response;
562 msg.rx.size = sizeof(response);
563
564 local_irq_save(flags);
565 start = ktime_get();
566 err = tegra_bpmp_transfer_atomic(bpmp, &msg);
567 end = ktime_get();
568 local_irq_restore(flags);
569
570 if (!err)
571 dev_dbg(bpmp->dev,
572 "ping ok: challenge: %u, response: %u, time: %lld\n",
573 request.challenge, response.reply,
574 ktime_to_us(ktime_sub(end, start)));
575
576 return err;
577}
578
579/* deprecated version of tag query */
580static int tegra_bpmp_get_firmware_tag_old(struct tegra_bpmp *bpmp, char *tag,
581 size_t size)
582{
583 struct mrq_query_tag_request request;
584 struct tegra_bpmp_message msg;
585 unsigned long flags;
586 dma_addr_t phys;
587 void *virt;
588 int err;
589
590 if (size != TAG_SZ)
591 return -EINVAL;
592
593 virt = dma_alloc_coherent(bpmp->dev, TAG_SZ, &phys,
594 GFP_KERNEL | GFP_DMA32);
595 if (!virt)
596 return -ENOMEM;
597
598 memset(&request, 0, sizeof(request));
599 request.addr = phys;
600
601 memset(&msg, 0, sizeof(msg));
602 msg.mrq = MRQ_QUERY_TAG;
603 msg.tx.data = &request;
604 msg.tx.size = sizeof(request);
605
606 local_irq_save(flags);
607 err = tegra_bpmp_transfer_atomic(bpmp, &msg);
608 local_irq_restore(flags);
609
610 if (err == 0)
611 memcpy(tag, virt, TAG_SZ);
612
613 dma_free_coherent(bpmp->dev, TAG_SZ, virt, phys);
614
615 return err;
616}
617
618static int tegra_bpmp_get_firmware_tag(struct tegra_bpmp *bpmp, char *tag,
619 size_t size)
620{
621 if (tegra_bpmp_mrq_is_supported(bpmp, MRQ_QUERY_FW_TAG)) {
622 struct mrq_query_fw_tag_response resp;
623 struct tegra_bpmp_message msg = {
624 .mrq = MRQ_QUERY_FW_TAG,
625 .rx = {
626 .data = &resp,
627 .size = sizeof(resp),
628 },
629 };
630 int err;
631
632 if (size != sizeof(resp.tag))
633 return -EINVAL;
634
635 err = tegra_bpmp_transfer(bpmp, &msg);
636
637 if (err)
638 return err;
639 if (msg.rx.ret < 0)
640 return -EINVAL;
641
642 memcpy(tag, resp.tag, sizeof(resp.tag));
643 return 0;
644 }
645
646 return tegra_bpmp_get_firmware_tag_old(bpmp, tag, size);
647}
648
649static void tegra_bpmp_channel_signal(struct tegra_bpmp_channel *channel)
650{
651 unsigned long flags = channel->ob->flags;
652
653 if ((flags & MSG_RING) == 0)
654 return;
655
656 complete(&channel->completion);
657}
658
659void tegra_bpmp_handle_rx(struct tegra_bpmp *bpmp)
660{
661 struct tegra_bpmp_channel *channel;
662 unsigned int i, count;
663 unsigned long *busy;
664
665 channel = bpmp->rx_channel;
666 count = bpmp->soc->channels.thread.count;
667 busy = bpmp->threaded.busy;
668
669 if (tegra_bpmp_is_request_ready(channel))
670 tegra_bpmp_handle_mrq(bpmp, channel->ib->code, channel);
671
672 spin_lock(&bpmp->lock);
673
674 for_each_set_bit(i, busy, count) {
675 struct tegra_bpmp_channel *channel;
676
677 channel = &bpmp->threaded_channels[i];
678
679 if (tegra_bpmp_is_response_ready(channel)) {
680 tegra_bpmp_channel_signal(channel);
681 clear_bit(i, busy);
682 }
683 }
684
685 spin_unlock(&bpmp->lock);
686}
687
688static int tegra_bpmp_probe(struct platform_device *pdev)
689{
690 struct tegra_bpmp *bpmp;
691 char tag[TAG_SZ];
692 size_t size;
693 int err;
694
695 bpmp = devm_kzalloc(&pdev->dev, sizeof(*bpmp), GFP_KERNEL);
696 if (!bpmp)
697 return -ENOMEM;
698
699 bpmp->soc = of_device_get_match_data(&pdev->dev);
700 bpmp->dev = &pdev->dev;
701
702 INIT_LIST_HEAD(&bpmp->mrqs);
703 spin_lock_init(&bpmp->lock);
704
705 bpmp->threaded.count = bpmp->soc->channels.thread.count;
706 sema_init(&bpmp->threaded.lock, bpmp->threaded.count);
707
708 size = BITS_TO_LONGS(bpmp->threaded.count) * sizeof(long);
709
710 bpmp->threaded.allocated = devm_kzalloc(&pdev->dev, size, GFP_KERNEL);
711 if (!bpmp->threaded.allocated)
712 return -ENOMEM;
713
714 bpmp->threaded.busy = devm_kzalloc(&pdev->dev, size, GFP_KERNEL);
715 if (!bpmp->threaded.busy)
716 return -ENOMEM;
717
718 spin_lock_init(&bpmp->atomic_tx_lock);
719 bpmp->tx_channel = devm_kzalloc(&pdev->dev, sizeof(*bpmp->tx_channel),
720 GFP_KERNEL);
721 if (!bpmp->tx_channel)
722 return -ENOMEM;
723
724 bpmp->rx_channel = devm_kzalloc(&pdev->dev, sizeof(*bpmp->rx_channel),
725 GFP_KERNEL);
726 if (!bpmp->rx_channel)
727 return -ENOMEM;
728
729 bpmp->threaded_channels = devm_kcalloc(&pdev->dev, bpmp->threaded.count,
730 sizeof(*bpmp->threaded_channels),
731 GFP_KERNEL);
732 if (!bpmp->threaded_channels)
733 return -ENOMEM;
734
735 err = bpmp->soc->ops->init(bpmp);
736 if (err < 0)
737 return err;
738
739 err = tegra_bpmp_request_mrq(bpmp, MRQ_PING,
740 tegra_bpmp_mrq_handle_ping, bpmp);
741 if (err < 0)
742 goto deinit;
743
744 err = tegra_bpmp_ping(bpmp);
745 if (err < 0) {
746 dev_err(&pdev->dev, "failed to ping BPMP: %d\n", err);
747 goto free_mrq;
748 }
749
750 err = tegra_bpmp_get_firmware_tag(bpmp, tag, sizeof(tag));
751 if (err < 0) {
752 dev_err(&pdev->dev, "failed to get firmware tag: %d\n", err);
753 goto free_mrq;
754 }
755
756 dev_info(&pdev->dev, "firmware: %.*s\n", (int)sizeof(tag), tag);
757
758 platform_set_drvdata(pdev, bpmp);
759
760 err = of_platform_default_populate(pdev->dev.of_node, NULL, &pdev->dev);
761 if (err < 0)
762 goto free_mrq;
763
764 if (of_find_property(pdev->dev.of_node, "#clock-cells", NULL)) {
765 err = tegra_bpmp_init_clocks(bpmp);
766 if (err < 0)
767 goto free_mrq;
768 }
769
770 if (of_find_property(pdev->dev.of_node, "#reset-cells", NULL)) {
771 err = tegra_bpmp_init_resets(bpmp);
772 if (err < 0)
773 goto free_mrq;
774 }
775
776 if (of_find_property(pdev->dev.of_node, "#power-domain-cells", NULL)) {
777 err = tegra_bpmp_init_powergates(bpmp);
778 if (err < 0)
779 goto free_mrq;
780 }
781
782 err = tegra_bpmp_init_debugfs(bpmp);
783 if (err < 0)
784 dev_err(&pdev->dev, "debugfs initialization failed: %d\n", err);
785
786 return 0;
787
788free_mrq:
789 tegra_bpmp_free_mrq(bpmp, MRQ_PING, bpmp);
790deinit:
791 if (bpmp->soc->ops->deinit)
792 bpmp->soc->ops->deinit(bpmp);
793
794 return err;
795}
796
797static int __maybe_unused tegra_bpmp_resume(struct device *dev)
798{
799 struct tegra_bpmp *bpmp = dev_get_drvdata(dev);
800
801 if (bpmp->soc->ops->resume)
802 return bpmp->soc->ops->resume(bpmp);
803 else
804 return 0;
805}
806
807static const struct dev_pm_ops tegra_bpmp_pm_ops = {
808 .resume_noirq = tegra_bpmp_resume,
809};
810
811#if IS_ENABLED(CONFIG_ARCH_TEGRA_186_SOC) || \
812 IS_ENABLED(CONFIG_ARCH_TEGRA_194_SOC) || \
813 IS_ENABLED(CONFIG_ARCH_TEGRA_234_SOC)
814static const struct tegra_bpmp_soc tegra186_soc = {
815 .channels = {
816 .cpu_tx = {
817 .offset = 3,
818 .timeout = 60 * USEC_PER_SEC,
819 },
820 .thread = {
821 .offset = 0,
822 .count = 3,
823 .timeout = 600 * USEC_PER_SEC,
824 },
825 .cpu_rx = {
826 .offset = 13,
827 .timeout = 0,
828 },
829 },
830 .ops = &tegra186_bpmp_ops,
831 .num_resets = 193,
832};
833#endif
834
835#if IS_ENABLED(CONFIG_ARCH_TEGRA_210_SOC)
836static const struct tegra_bpmp_soc tegra210_soc = {
837 .channels = {
838 .cpu_tx = {
839 .offset = 0,
840 .count = 1,
841 .timeout = 60 * USEC_PER_SEC,
842 },
843 .thread = {
844 .offset = 4,
845 .count = 1,
846 .timeout = 600 * USEC_PER_SEC,
847 },
848 .cpu_rx = {
849 .offset = 8,
850 .count = 1,
851 .timeout = 0,
852 },
853 },
854 .ops = &tegra210_bpmp_ops,
855};
856#endif
857
858static const struct of_device_id tegra_bpmp_match[] = {
859#if IS_ENABLED(CONFIG_ARCH_TEGRA_186_SOC) || \
860 IS_ENABLED(CONFIG_ARCH_TEGRA_194_SOC) || \
861 IS_ENABLED(CONFIG_ARCH_TEGRA_234_SOC)
862 { .compatible = "nvidia,tegra186-bpmp", .data = &tegra186_soc },
863#endif
864#if IS_ENABLED(CONFIG_ARCH_TEGRA_210_SOC)
865 { .compatible = "nvidia,tegra210-bpmp", .data = &tegra210_soc },
866#endif
867 { }
868};
869
870static struct platform_driver tegra_bpmp_driver = {
871 .driver = {
872 .name = "tegra-bpmp",
873 .of_match_table = tegra_bpmp_match,
874 .pm = &tegra_bpmp_pm_ops,
875 .suppress_bind_attrs = true,
876 },
877 .probe = tegra_bpmp_probe,
878};
879builtin_platform_driver(tegra_bpmp_driver);