Loading...
1// SPDX-License-Identifier: GPL-2.0-only
2//
3// Copyright(c) 2021-2022 Intel Corporation
4//
5// Authors: Cezary Rojewski <cezary.rojewski@intel.com>
6// Amadeusz Slawinski <amadeuszx.slawinski@linux.intel.com>
7//
8
9#include <linux/io-64-nonatomic-lo-hi.h>
10#include <linux/slab.h>
11#include <sound/hdaudio_ext.h>
12#include "avs.h"
13#include "messages.h"
14#include "registers.h"
15#include "trace.h"
16
17#define AVS_IPC_TIMEOUT_MS 300
18#define AVS_D0IX_DELAY_MS 300
19
20static int
21avs_dsp_set_d0ix(struct avs_dev *adev, bool enable)
22{
23 struct avs_ipc *ipc = adev->ipc;
24 int ret;
25
26 /* Is transition required? */
27 if (ipc->in_d0ix == enable)
28 return 0;
29
30 ret = avs_dsp_op(adev, set_d0ix, enable);
31 if (ret) {
32 /* Prevent further d0ix attempts on conscious IPC failure. */
33 if (ret == -AVS_EIPC)
34 atomic_inc(&ipc->d0ix_disable_depth);
35
36 ipc->in_d0ix = false;
37 return ret;
38 }
39
40 ipc->in_d0ix = enable;
41 return 0;
42}
43
44static void avs_dsp_schedule_d0ix(struct avs_dev *adev, struct avs_ipc_msg *tx)
45{
46 if (atomic_read(&adev->ipc->d0ix_disable_depth))
47 return;
48
49 mod_delayed_work(system_power_efficient_wq, &adev->ipc->d0ix_work,
50 msecs_to_jiffies(AVS_D0IX_DELAY_MS));
51}
52
53static void avs_dsp_d0ix_work(struct work_struct *work)
54{
55 struct avs_ipc *ipc = container_of(work, struct avs_ipc, d0ix_work.work);
56
57 avs_dsp_set_d0ix(to_avs_dev(ipc->dev), true);
58}
59
60static int avs_dsp_wake_d0i0(struct avs_dev *adev, struct avs_ipc_msg *tx)
61{
62 struct avs_ipc *ipc = adev->ipc;
63
64 if (!atomic_read(&ipc->d0ix_disable_depth)) {
65 cancel_delayed_work_sync(&ipc->d0ix_work);
66 return avs_dsp_set_d0ix(adev, false);
67 }
68
69 return 0;
70}
71
72int avs_dsp_disable_d0ix(struct avs_dev *adev)
73{
74 struct avs_ipc *ipc = adev->ipc;
75
76 /* Prevent PG only on the first disable. */
77 if (atomic_inc_return(&ipc->d0ix_disable_depth) == 1) {
78 cancel_delayed_work_sync(&ipc->d0ix_work);
79 return avs_dsp_set_d0ix(adev, false);
80 }
81
82 return 0;
83}
84
85int avs_dsp_enable_d0ix(struct avs_dev *adev)
86{
87 struct avs_ipc *ipc = adev->ipc;
88
89 if (atomic_dec_and_test(&ipc->d0ix_disable_depth))
90 queue_delayed_work(system_power_efficient_wq, &ipc->d0ix_work,
91 msecs_to_jiffies(AVS_D0IX_DELAY_MS));
92 return 0;
93}
94
95static void avs_dsp_recovery(struct avs_dev *adev)
96{
97 struct avs_soc_component *acomp;
98 unsigned int core_mask;
99 int ret;
100
101 mutex_lock(&adev->comp_list_mutex);
102 /* disconnect all running streams */
103 list_for_each_entry(acomp, &adev->comp_list, node) {
104 struct snd_soc_pcm_runtime *rtd;
105 struct snd_soc_card *card;
106
107 card = acomp->base.card;
108 if (!card)
109 continue;
110
111 for_each_card_rtds(card, rtd) {
112 struct snd_pcm *pcm;
113 int dir;
114
115 pcm = rtd->pcm;
116 if (!pcm || rtd->dai_link->no_pcm)
117 continue;
118
119 for_each_pcm_streams(dir) {
120 struct snd_pcm_substream *substream;
121
122 substream = pcm->streams[dir].substream;
123 if (!substream || !substream->runtime)
124 continue;
125
126 /* No need for _irq() as we are in nonatomic context. */
127 snd_pcm_stream_lock(substream);
128 snd_pcm_stop(substream, SNDRV_PCM_STATE_DISCONNECTED);
129 snd_pcm_stream_unlock(substream);
130 }
131 }
132 }
133 mutex_unlock(&adev->comp_list_mutex);
134
135 /* forcibly shutdown all cores */
136 core_mask = GENMASK(adev->hw_cfg.dsp_cores - 1, 0);
137 avs_dsp_core_disable(adev, core_mask);
138
139 /* attempt dsp reboot */
140 ret = avs_dsp_boot_firmware(adev, true);
141 if (ret < 0)
142 dev_err(adev->dev, "dsp reboot failed: %d\n", ret);
143
144 pm_runtime_mark_last_busy(adev->dev);
145 pm_runtime_enable(adev->dev);
146 pm_request_autosuspend(adev->dev);
147
148 atomic_set(&adev->ipc->recovering, 0);
149}
150
151static void avs_dsp_recovery_work(struct work_struct *work)
152{
153 struct avs_ipc *ipc = container_of(work, struct avs_ipc, recovery_work);
154
155 avs_dsp_recovery(to_avs_dev(ipc->dev));
156}
157
158static void avs_dsp_exception_caught(struct avs_dev *adev, union avs_notify_msg *msg)
159{
160 struct avs_ipc *ipc = adev->ipc;
161
162 /* Account for the double-exception case. */
163 ipc->ready = false;
164
165 if (!atomic_add_unless(&ipc->recovering, 1, 1)) {
166 dev_err(adev->dev, "dsp recovery is already in progress\n");
167 return;
168 }
169
170 dev_crit(adev->dev, "communication severed, rebooting dsp..\n");
171
172 cancel_delayed_work_sync(&ipc->d0ix_work);
173 ipc->in_d0ix = false;
174 /* Re-enabled on recovery completion. */
175 pm_runtime_disable(adev->dev);
176
177 /* Process received notification. */
178 avs_dsp_op(adev, coredump, msg);
179
180 schedule_work(&ipc->recovery_work);
181}
182
183static void avs_dsp_receive_rx(struct avs_dev *adev, u64 header)
184{
185 struct avs_ipc *ipc = adev->ipc;
186 union avs_reply_msg msg = AVS_MSG(header);
187 u64 reg;
188
189 reg = readq(avs_sram_addr(adev, AVS_FW_REGS_WINDOW));
190 trace_avs_ipc_reply_msg(header, reg);
191
192 ipc->rx.header = header;
193 /* Abort copying payload if request processing was unsuccessful. */
194 if (!msg.status) {
195 /* update size in case of LARGE_CONFIG_GET */
196 if (msg.msg_target == AVS_MOD_MSG &&
197 msg.global_msg_type == AVS_MOD_LARGE_CONFIG_GET)
198 ipc->rx.size = min_t(u32, AVS_MAILBOX_SIZE,
199 msg.ext.large_config.data_off_size);
200
201 memcpy_fromio(ipc->rx.data, avs_uplink_addr(adev), ipc->rx.size);
202 trace_avs_msg_payload(ipc->rx.data, ipc->rx.size);
203 }
204}
205
206static void avs_dsp_process_notification(struct avs_dev *adev, u64 header)
207{
208 struct avs_notify_mod_data mod_data;
209 union avs_notify_msg msg = AVS_MSG(header);
210 size_t data_size = 0;
211 void *data = NULL;
212 u64 reg;
213
214 reg = readq(avs_sram_addr(adev, AVS_FW_REGS_WINDOW));
215 trace_avs_ipc_notify_msg(header, reg);
216
217 /* Ignore spurious notifications until handshake is established. */
218 if (!adev->ipc->ready && msg.notify_msg_type != AVS_NOTIFY_FW_READY) {
219 dev_dbg(adev->dev, "FW not ready, skip notification: 0x%08x\n", msg.primary);
220 return;
221 }
222
223 /* Calculate notification payload size. */
224 switch (msg.notify_msg_type) {
225 case AVS_NOTIFY_FW_READY:
226 break;
227
228 case AVS_NOTIFY_PHRASE_DETECTED:
229 data_size = sizeof(struct avs_notify_voice_data);
230 break;
231
232 case AVS_NOTIFY_RESOURCE_EVENT:
233 data_size = sizeof(struct avs_notify_res_data);
234 break;
235
236 case AVS_NOTIFY_LOG_BUFFER_STATUS:
237 case AVS_NOTIFY_EXCEPTION_CAUGHT:
238 break;
239
240 case AVS_NOTIFY_MODULE_EVENT:
241 /* To know the total payload size, header needs to be read first. */
242 memcpy_fromio(&mod_data, avs_uplink_addr(adev), sizeof(mod_data));
243 data_size = sizeof(mod_data) + mod_data.data_size;
244 break;
245
246 default:
247 dev_info(adev->dev, "unknown notification: 0x%08x\n", msg.primary);
248 break;
249 }
250
251 if (data_size) {
252 data = kmalloc(data_size, GFP_KERNEL);
253 if (!data)
254 return;
255
256 memcpy_fromio(data, avs_uplink_addr(adev), data_size);
257 trace_avs_msg_payload(data, data_size);
258 }
259
260 /* Perform notification-specific operations. */
261 switch (msg.notify_msg_type) {
262 case AVS_NOTIFY_FW_READY:
263 dev_dbg(adev->dev, "FW READY 0x%08x\n", msg.primary);
264 adev->ipc->ready = true;
265 complete(&adev->fw_ready);
266 break;
267
268 case AVS_NOTIFY_LOG_BUFFER_STATUS:
269 avs_log_buffer_status_locked(adev, &msg);
270 break;
271
272 case AVS_NOTIFY_EXCEPTION_CAUGHT:
273 avs_dsp_exception_caught(adev, &msg);
274 break;
275
276 default:
277 break;
278 }
279
280 kfree(data);
281}
282
283void avs_dsp_process_response(struct avs_dev *adev, u64 header)
284{
285 struct avs_ipc *ipc = adev->ipc;
286
287 /*
288 * Response may either be solicited - a reply for a request that has
289 * been sent beforehand - or unsolicited (notification).
290 */
291 if (avs_msg_is_reply(header)) {
292 /* Response processing is invoked from IRQ thread. */
293 spin_lock_irq(&ipc->rx_lock);
294 avs_dsp_receive_rx(adev, header);
295 ipc->rx_completed = true;
296 spin_unlock_irq(&ipc->rx_lock);
297 } else {
298 avs_dsp_process_notification(adev, header);
299 }
300
301 complete(&ipc->busy_completion);
302}
303
304static bool avs_ipc_is_busy(struct avs_ipc *ipc)
305{
306 struct avs_dev *adev = to_avs_dev(ipc->dev);
307 const struct avs_spec *const spec = adev->spec;
308 u32 hipc_rsp;
309
310 hipc_rsp = snd_hdac_adsp_readl(adev, spec->hipc->rsp_offset);
311 return hipc_rsp & spec->hipc->rsp_busy_mask;
312}
313
314static int avs_ipc_wait_busy_completion(struct avs_ipc *ipc, int timeout)
315{
316 u32 repeats_left = 128; /* to avoid infinite looping */
317 int ret;
318
319again:
320 ret = wait_for_completion_timeout(&ipc->busy_completion, msecs_to_jiffies(timeout));
321
322 /* DSP could be unresponsive at this point. */
323 if (!ipc->ready)
324 return -EPERM;
325
326 if (!ret) {
327 if (!avs_ipc_is_busy(ipc))
328 return -ETIMEDOUT;
329 /*
330 * Firmware did its job, either notification or reply
331 * has been received - now wait until it's processed.
332 */
333 wait_for_completion_killable(&ipc->busy_completion);
334 }
335
336 /* Ongoing notification's bottom-half may cause early wakeup */
337 spin_lock(&ipc->rx_lock);
338 if (!ipc->rx_completed) {
339 if (repeats_left) {
340 /* Reply delayed due to notification. */
341 repeats_left--;
342 reinit_completion(&ipc->busy_completion);
343 spin_unlock(&ipc->rx_lock);
344 goto again;
345 }
346
347 spin_unlock(&ipc->rx_lock);
348 return -ETIMEDOUT;
349 }
350
351 spin_unlock(&ipc->rx_lock);
352 return 0;
353}
354
355static void avs_ipc_msg_init(struct avs_ipc *ipc, struct avs_ipc_msg *reply)
356{
357 lockdep_assert_held(&ipc->rx_lock);
358
359 ipc->rx.header = 0;
360 ipc->rx.size = reply ? reply->size : 0;
361 ipc->rx_completed = false;
362
363 reinit_completion(&ipc->done_completion);
364 reinit_completion(&ipc->busy_completion);
365}
366
367static void avs_dsp_send_tx(struct avs_dev *adev, struct avs_ipc_msg *tx, bool read_fwregs)
368{
369 const struct avs_spec *const spec = adev->spec;
370 u64 reg = ULONG_MAX;
371
372 tx->header |= spec->hipc->req_busy_mask;
373 if (read_fwregs)
374 reg = readq(avs_sram_addr(adev, AVS_FW_REGS_WINDOW));
375
376 trace_avs_request(tx, reg);
377
378 if (tx->size)
379 memcpy_toio(avs_downlink_addr(adev), tx->data, tx->size);
380 snd_hdac_adsp_writel(adev, spec->hipc->req_ext_offset, tx->header >> 32);
381 snd_hdac_adsp_writel(adev, spec->hipc->req_offset, tx->header & UINT_MAX);
382}
383
384static int avs_dsp_do_send_msg(struct avs_dev *adev, struct avs_ipc_msg *request,
385 struct avs_ipc_msg *reply, int timeout, const char *name)
386{
387 struct avs_ipc *ipc = adev->ipc;
388 int ret;
389
390 if (!ipc->ready)
391 return -EPERM;
392
393 mutex_lock(&ipc->msg_mutex);
394
395 spin_lock(&ipc->rx_lock);
396 avs_ipc_msg_init(ipc, reply);
397 avs_dsp_send_tx(adev, request, true);
398 spin_unlock(&ipc->rx_lock);
399
400 ret = avs_ipc_wait_busy_completion(ipc, timeout);
401 if (ret) {
402 if (ret == -ETIMEDOUT) {
403 union avs_notify_msg msg = AVS_NOTIFICATION(EXCEPTION_CAUGHT);
404
405 /* Same treatment as on exception, just stack_dump=0. */
406 avs_dsp_exception_caught(adev, &msg);
407 }
408 goto exit;
409 }
410
411 ret = ipc->rx.rsp.status;
412 /*
413 * If IPC channel is blocked e.g.: due to ongoing recovery,
414 * -EPERM error code is expected and thus it's not an actual error.
415 *
416 * Unsupported IPCs are of no harm either.
417 */
418 if (ret == -EPERM || ret == AVS_IPC_NOT_SUPPORTED)
419 dev_dbg(adev->dev, "%s (0x%08x 0x%08x) failed: %d\n",
420 name, request->glb.primary, request->glb.ext.val, ret);
421 else if (ret)
422 dev_err(adev->dev, "%s (0x%08x 0x%08x) failed: %d\n",
423 name, request->glb.primary, request->glb.ext.val, ret);
424
425 if (reply) {
426 reply->header = ipc->rx.header;
427 reply->size = ipc->rx.size;
428 if (reply->data && ipc->rx.size)
429 memcpy(reply->data, ipc->rx.data, reply->size);
430 }
431
432exit:
433 mutex_unlock(&ipc->msg_mutex);
434 return ret;
435}
436
437static int avs_dsp_send_msg_sequence(struct avs_dev *adev, struct avs_ipc_msg *request,
438 struct avs_ipc_msg *reply, int timeout, bool wake_d0i0,
439 bool schedule_d0ix, const char *name)
440{
441 int ret;
442
443 trace_avs_d0ix("wake", wake_d0i0, request->header);
444 if (wake_d0i0) {
445 ret = avs_dsp_wake_d0i0(adev, request);
446 if (ret)
447 return ret;
448 }
449
450 ret = avs_dsp_do_send_msg(adev, request, reply, timeout, name);
451 if (ret)
452 return ret;
453
454 trace_avs_d0ix("schedule", schedule_d0ix, request->header);
455 if (schedule_d0ix)
456 avs_dsp_schedule_d0ix(adev, request);
457
458 return 0;
459}
460
461int avs_dsp_send_msg_timeout(struct avs_dev *adev, struct avs_ipc_msg *request,
462 struct avs_ipc_msg *reply, int timeout, const char *name)
463{
464 bool wake_d0i0 = avs_dsp_op(adev, d0ix_toggle, request, true);
465 bool schedule_d0ix = avs_dsp_op(adev, d0ix_toggle, request, false);
466
467 return avs_dsp_send_msg_sequence(adev, request, reply, timeout, wake_d0i0, schedule_d0ix,
468 name);
469}
470
471int avs_dsp_send_msg(struct avs_dev *adev, struct avs_ipc_msg *request,
472 struct avs_ipc_msg *reply, const char *name)
473{
474 return avs_dsp_send_msg_timeout(adev, request, reply, adev->ipc->default_timeout_ms, name);
475}
476
477int avs_dsp_send_pm_msg_timeout(struct avs_dev *adev, struct avs_ipc_msg *request,
478 struct avs_ipc_msg *reply, int timeout, bool wake_d0i0,
479 const char *name)
480{
481 return avs_dsp_send_msg_sequence(adev, request, reply, timeout, wake_d0i0, false, name);
482}
483
484int avs_dsp_send_pm_msg(struct avs_dev *adev, struct avs_ipc_msg *request,
485 struct avs_ipc_msg *reply, bool wake_d0i0, const char *name)
486{
487 return avs_dsp_send_pm_msg_timeout(adev, request, reply, adev->ipc->default_timeout_ms,
488 wake_d0i0, name);
489}
490
491static int avs_dsp_do_send_rom_msg(struct avs_dev *adev, struct avs_ipc_msg *request, int timeout,
492 const char *name)
493{
494 struct avs_ipc *ipc = adev->ipc;
495 int ret;
496
497 mutex_lock(&ipc->msg_mutex);
498
499 spin_lock(&ipc->rx_lock);
500 avs_ipc_msg_init(ipc, NULL);
501 /*
502 * with hw still stalled, memory windows may not be
503 * configured properly so avoid accessing SRAM
504 */
505 avs_dsp_send_tx(adev, request, false);
506 spin_unlock(&ipc->rx_lock);
507
508 /* ROM messages must be sent before main core is unstalled */
509 ret = avs_dsp_op(adev, stall, AVS_MAIN_CORE_MASK, false);
510 if (!ret) {
511 ret = wait_for_completion_timeout(&ipc->done_completion, msecs_to_jiffies(timeout));
512 ret = ret ? 0 : -ETIMEDOUT;
513 }
514 if (ret)
515 dev_err(adev->dev, "%s (0x%08x 0x%08x) failed: %d\n",
516 name, request->glb.primary, request->glb.ext.val, ret);
517
518 mutex_unlock(&ipc->msg_mutex);
519
520 return ret;
521}
522
523int avs_dsp_send_rom_msg_timeout(struct avs_dev *adev, struct avs_ipc_msg *request, int timeout,
524 const char *name)
525{
526 return avs_dsp_do_send_rom_msg(adev, request, timeout, name);
527}
528
529int avs_dsp_send_rom_msg(struct avs_dev *adev, struct avs_ipc_msg *request, const char *name)
530{
531 return avs_dsp_send_rom_msg_timeout(adev, request, adev->ipc->default_timeout_ms, name);
532}
533
534void avs_dsp_interrupt_control(struct avs_dev *adev, bool enable)
535{
536 const struct avs_spec *const spec = adev->spec;
537 u32 value, mask;
538
539 /*
540 * No particular bit setting order. All of these are required
541 * to have a functional SW <-> FW communication.
542 */
543 value = enable ? AVS_ADSP_ADSPIC_IPC : 0;
544 snd_hdac_adsp_updatel(adev, AVS_ADSP_REG_ADSPIC, AVS_ADSP_ADSPIC_IPC, value);
545
546 mask = AVS_ADSP_HIPCCTL_DONE | AVS_ADSP_HIPCCTL_BUSY;
547 value = enable ? mask : 0;
548 snd_hdac_adsp_updatel(adev, spec->hipc->ctl_offset, mask, value);
549}
550
551int avs_ipc_init(struct avs_ipc *ipc, struct device *dev)
552{
553 ipc->rx.data = devm_kzalloc(dev, AVS_MAILBOX_SIZE, GFP_KERNEL);
554 if (!ipc->rx.data)
555 return -ENOMEM;
556
557 ipc->dev = dev;
558 ipc->ready = false;
559 ipc->default_timeout_ms = AVS_IPC_TIMEOUT_MS;
560 INIT_WORK(&ipc->recovery_work, avs_dsp_recovery_work);
561 INIT_DELAYED_WORK(&ipc->d0ix_work, avs_dsp_d0ix_work);
562 init_completion(&ipc->done_completion);
563 init_completion(&ipc->busy_completion);
564 spin_lock_init(&ipc->rx_lock);
565 mutex_init(&ipc->msg_mutex);
566
567 return 0;
568}
569
570void avs_ipc_block(struct avs_ipc *ipc)
571{
572 ipc->ready = false;
573 cancel_work_sync(&ipc->recovery_work);
574 cancel_delayed_work_sync(&ipc->d0ix_work);
575 ipc->in_d0ix = false;
576}
1// SPDX-License-Identifier: GPL-2.0-only
2//
3// Copyright(c) 2021-2022 Intel Corporation. All rights reserved.
4//
5// Authors: Cezary Rojewski <cezary.rojewski@intel.com>
6// Amadeusz Slawinski <amadeuszx.slawinski@linux.intel.com>
7//
8
9#include <linux/io-64-nonatomic-lo-hi.h>
10#include <linux/slab.h>
11#include <sound/hdaudio_ext.h>
12#include "avs.h"
13#include "messages.h"
14#include "registers.h"
15#include "trace.h"
16
17#define AVS_IPC_TIMEOUT_MS 300
18#define AVS_D0IX_DELAY_MS 300
19
20static int
21avs_dsp_set_d0ix(struct avs_dev *adev, bool enable)
22{
23 struct avs_ipc *ipc = adev->ipc;
24 int ret;
25
26 /* Is transition required? */
27 if (ipc->in_d0ix == enable)
28 return 0;
29
30 ret = avs_dsp_op(adev, set_d0ix, enable);
31 if (ret) {
32 /* Prevent further d0ix attempts on conscious IPC failure. */
33 if (ret == -AVS_EIPC)
34 atomic_inc(&ipc->d0ix_disable_depth);
35
36 ipc->in_d0ix = false;
37 return ret;
38 }
39
40 ipc->in_d0ix = enable;
41 return 0;
42}
43
44static void avs_dsp_schedule_d0ix(struct avs_dev *adev, struct avs_ipc_msg *tx)
45{
46 if (atomic_read(&adev->ipc->d0ix_disable_depth))
47 return;
48
49 mod_delayed_work(system_power_efficient_wq, &adev->ipc->d0ix_work,
50 msecs_to_jiffies(AVS_D0IX_DELAY_MS));
51}
52
53static void avs_dsp_d0ix_work(struct work_struct *work)
54{
55 struct avs_ipc *ipc = container_of(work, struct avs_ipc, d0ix_work.work);
56
57 avs_dsp_set_d0ix(to_avs_dev(ipc->dev), true);
58}
59
60static int avs_dsp_wake_d0i0(struct avs_dev *adev, struct avs_ipc_msg *tx)
61{
62 struct avs_ipc *ipc = adev->ipc;
63
64 if (!atomic_read(&ipc->d0ix_disable_depth)) {
65 cancel_delayed_work_sync(&ipc->d0ix_work);
66 return avs_dsp_set_d0ix(adev, false);
67 }
68
69 return 0;
70}
71
72int avs_dsp_disable_d0ix(struct avs_dev *adev)
73{
74 struct avs_ipc *ipc = adev->ipc;
75
76 /* Prevent PG only on the first disable. */
77 if (atomic_inc_return(&ipc->d0ix_disable_depth) == 1) {
78 cancel_delayed_work_sync(&ipc->d0ix_work);
79 return avs_dsp_set_d0ix(adev, false);
80 }
81
82 return 0;
83}
84
85int avs_dsp_enable_d0ix(struct avs_dev *adev)
86{
87 struct avs_ipc *ipc = adev->ipc;
88
89 if (atomic_dec_and_test(&ipc->d0ix_disable_depth))
90 queue_delayed_work(system_power_efficient_wq, &ipc->d0ix_work,
91 msecs_to_jiffies(AVS_D0IX_DELAY_MS));
92 return 0;
93}
94
95static void avs_dsp_recovery(struct avs_dev *adev)
96{
97 struct avs_soc_component *acomp;
98 unsigned int core_mask;
99 int ret;
100
101 mutex_lock(&adev->comp_list_mutex);
102 /* disconnect all running streams */
103 list_for_each_entry(acomp, &adev->comp_list, node) {
104 struct snd_soc_pcm_runtime *rtd;
105 struct snd_soc_card *card;
106
107 card = acomp->base.card;
108 if (!card)
109 continue;
110
111 for_each_card_rtds(card, rtd) {
112 struct snd_pcm *pcm;
113 int dir;
114
115 pcm = rtd->pcm;
116 if (!pcm || rtd->dai_link->no_pcm)
117 continue;
118
119 for_each_pcm_streams(dir) {
120 struct snd_pcm_substream *substream;
121
122 substream = pcm->streams[dir].substream;
123 if (!substream || !substream->runtime)
124 continue;
125
126 /* No need for _irq() as we are in nonatomic context. */
127 snd_pcm_stream_lock(substream);
128 snd_pcm_stop(substream, SNDRV_PCM_STATE_DISCONNECTED);
129 snd_pcm_stream_unlock(substream);
130 }
131 }
132 }
133 mutex_unlock(&adev->comp_list_mutex);
134
135 /* forcibly shutdown all cores */
136 core_mask = GENMASK(adev->hw_cfg.dsp_cores - 1, 0);
137 avs_dsp_core_disable(adev, core_mask);
138
139 /* attempt dsp reboot */
140 ret = avs_dsp_boot_firmware(adev, true);
141 if (ret < 0)
142 dev_err(adev->dev, "dsp reboot failed: %d\n", ret);
143
144 pm_runtime_mark_last_busy(adev->dev);
145 pm_runtime_enable(adev->dev);
146 pm_request_autosuspend(adev->dev);
147
148 atomic_set(&adev->ipc->recovering, 0);
149}
150
151static void avs_dsp_recovery_work(struct work_struct *work)
152{
153 struct avs_ipc *ipc = container_of(work, struct avs_ipc, recovery_work);
154
155 avs_dsp_recovery(to_avs_dev(ipc->dev));
156}
157
158static void avs_dsp_exception_caught(struct avs_dev *adev, union avs_notify_msg *msg)
159{
160 struct avs_ipc *ipc = adev->ipc;
161
162 /* Account for the double-exception case. */
163 ipc->ready = false;
164
165 if (!atomic_add_unless(&ipc->recovering, 1, 1)) {
166 dev_err(adev->dev, "dsp recovery is already in progress\n");
167 return;
168 }
169
170 dev_crit(adev->dev, "communication severed, rebooting dsp..\n");
171
172 cancel_delayed_work_sync(&ipc->d0ix_work);
173 ipc->in_d0ix = false;
174 /* Re-enabled on recovery completion. */
175 pm_runtime_disable(adev->dev);
176
177 /* Process received notification. */
178 avs_dsp_op(adev, coredump, msg);
179
180 schedule_work(&ipc->recovery_work);
181}
182
183static void avs_dsp_receive_rx(struct avs_dev *adev, u64 header)
184{
185 struct avs_ipc *ipc = adev->ipc;
186 union avs_reply_msg msg = AVS_MSG(header);
187 u64 reg;
188
189 reg = readq(avs_sram_addr(adev, AVS_FW_REGS_WINDOW));
190 trace_avs_ipc_reply_msg(header, reg);
191
192 ipc->rx.header = header;
193 /* Abort copying payload if request processing was unsuccessful. */
194 if (!msg.status) {
195 /* update size in case of LARGE_CONFIG_GET */
196 if (msg.msg_target == AVS_MOD_MSG &&
197 msg.global_msg_type == AVS_MOD_LARGE_CONFIG_GET)
198 ipc->rx.size = min_t(u32, AVS_MAILBOX_SIZE,
199 msg.ext.large_config.data_off_size);
200
201 memcpy_fromio(ipc->rx.data, avs_uplink_addr(adev), ipc->rx.size);
202 trace_avs_msg_payload(ipc->rx.data, ipc->rx.size);
203 }
204}
205
206static void avs_dsp_process_notification(struct avs_dev *adev, u64 header)
207{
208 struct avs_notify_mod_data mod_data;
209 union avs_notify_msg msg = AVS_MSG(header);
210 size_t data_size = 0;
211 void *data = NULL;
212 u64 reg;
213
214 reg = readq(avs_sram_addr(adev, AVS_FW_REGS_WINDOW));
215 trace_avs_ipc_notify_msg(header, reg);
216
217 /* Ignore spurious notifications until handshake is established. */
218 if (!adev->ipc->ready && msg.notify_msg_type != AVS_NOTIFY_FW_READY) {
219 dev_dbg(adev->dev, "FW not ready, skip notification: 0x%08x\n", msg.primary);
220 return;
221 }
222
223 /* Calculate notification payload size. */
224 switch (msg.notify_msg_type) {
225 case AVS_NOTIFY_FW_READY:
226 break;
227
228 case AVS_NOTIFY_PHRASE_DETECTED:
229 data_size = sizeof(struct avs_notify_voice_data);
230 break;
231
232 case AVS_NOTIFY_RESOURCE_EVENT:
233 data_size = sizeof(struct avs_notify_res_data);
234 break;
235
236 case AVS_NOTIFY_LOG_BUFFER_STATUS:
237 case AVS_NOTIFY_EXCEPTION_CAUGHT:
238 break;
239
240 case AVS_NOTIFY_MODULE_EVENT:
241 /* To know the total payload size, header needs to be read first. */
242 memcpy_fromio(&mod_data, avs_uplink_addr(adev), sizeof(mod_data));
243 data_size = sizeof(mod_data) + mod_data.data_size;
244 break;
245
246 default:
247 dev_info(adev->dev, "unknown notification: 0x%08x\n", msg.primary);
248 break;
249 }
250
251 if (data_size) {
252 data = kmalloc(data_size, GFP_KERNEL);
253 if (!data)
254 return;
255
256 memcpy_fromio(data, avs_uplink_addr(adev), data_size);
257 trace_avs_msg_payload(data, data_size);
258 }
259
260 /* Perform notification-specific operations. */
261 switch (msg.notify_msg_type) {
262 case AVS_NOTIFY_FW_READY:
263 dev_dbg(adev->dev, "FW READY 0x%08x\n", msg.primary);
264 adev->ipc->ready = true;
265 complete(&adev->fw_ready);
266 break;
267
268 case AVS_NOTIFY_LOG_BUFFER_STATUS:
269 avs_log_buffer_status_locked(adev, &msg);
270 break;
271
272 case AVS_NOTIFY_EXCEPTION_CAUGHT:
273 avs_dsp_exception_caught(adev, &msg);
274 break;
275
276 default:
277 break;
278 }
279
280 kfree(data);
281}
282
283void avs_dsp_process_response(struct avs_dev *adev, u64 header)
284{
285 struct avs_ipc *ipc = adev->ipc;
286
287 /*
288 * Response may either be solicited - a reply for a request that has
289 * been sent beforehand - or unsolicited (notification).
290 */
291 if (avs_msg_is_reply(header)) {
292 /* Response processing is invoked from IRQ thread. */
293 spin_lock_irq(&ipc->rx_lock);
294 avs_dsp_receive_rx(adev, header);
295 ipc->rx_completed = true;
296 spin_unlock_irq(&ipc->rx_lock);
297 } else {
298 avs_dsp_process_notification(adev, header);
299 }
300
301 complete(&ipc->busy_completion);
302}
303
304irqreturn_t avs_irq_handler(struct avs_dev *adev)
305{
306 struct avs_ipc *ipc = adev->ipc;
307 const struct avs_spec *const spec = adev->spec;
308 u32 adspis, hipc_rsp, hipc_ack;
309 irqreturn_t ret = IRQ_NONE;
310
311 adspis = snd_hdac_adsp_readl(adev, AVS_ADSP_REG_ADSPIS);
312 if (adspis == UINT_MAX || !(adspis & AVS_ADSP_ADSPIS_IPC))
313 return ret;
314
315 hipc_ack = snd_hdac_adsp_readl(adev, spec->hipc->ack_offset);
316 hipc_rsp = snd_hdac_adsp_readl(adev, spec->hipc->rsp_offset);
317
318 /* DSP acked host's request */
319 if (hipc_ack & spec->hipc->ack_done_mask) {
320 /*
321 * As an extra precaution, mask done interrupt. Code executed
322 * due to complete() found below does not assume any masking.
323 */
324 snd_hdac_adsp_updatel(adev, spec->hipc->ctl_offset,
325 AVS_ADSP_HIPCCTL_DONE, 0);
326
327 complete(&ipc->done_completion);
328
329 /* tell DSP it has our attention */
330 snd_hdac_adsp_updatel(adev, spec->hipc->ack_offset,
331 spec->hipc->ack_done_mask,
332 spec->hipc->ack_done_mask);
333 /* unmask done interrupt */
334 snd_hdac_adsp_updatel(adev, spec->hipc->ctl_offset,
335 AVS_ADSP_HIPCCTL_DONE,
336 AVS_ADSP_HIPCCTL_DONE);
337 ret = IRQ_HANDLED;
338 }
339
340 /* DSP sent new response to process */
341 if (hipc_rsp & spec->hipc->rsp_busy_mask) {
342 /* mask busy interrupt */
343 snd_hdac_adsp_updatel(adev, spec->hipc->ctl_offset,
344 AVS_ADSP_HIPCCTL_BUSY, 0);
345
346 ret = IRQ_WAKE_THREAD;
347 }
348
349 return ret;
350}
351
352static bool avs_ipc_is_busy(struct avs_ipc *ipc)
353{
354 struct avs_dev *adev = to_avs_dev(ipc->dev);
355 const struct avs_spec *const spec = adev->spec;
356 u32 hipc_rsp;
357
358 hipc_rsp = snd_hdac_adsp_readl(adev, spec->hipc->rsp_offset);
359 return hipc_rsp & spec->hipc->rsp_busy_mask;
360}
361
362static int avs_ipc_wait_busy_completion(struct avs_ipc *ipc, int timeout)
363{
364 u32 repeats_left = 128; /* to avoid infinite looping */
365 int ret;
366
367again:
368 ret = wait_for_completion_timeout(&ipc->busy_completion, msecs_to_jiffies(timeout));
369
370 /* DSP could be unresponsive at this point. */
371 if (!ipc->ready)
372 return -EPERM;
373
374 if (!ret) {
375 if (!avs_ipc_is_busy(ipc))
376 return -ETIMEDOUT;
377 /*
378 * Firmware did its job, either notification or reply
379 * has been received - now wait until it's processed.
380 */
381 wait_for_completion_killable(&ipc->busy_completion);
382 }
383
384 /* Ongoing notification's bottom-half may cause early wakeup */
385 spin_lock(&ipc->rx_lock);
386 if (!ipc->rx_completed) {
387 if (repeats_left) {
388 /* Reply delayed due to notification. */
389 repeats_left--;
390 reinit_completion(&ipc->busy_completion);
391 spin_unlock(&ipc->rx_lock);
392 goto again;
393 }
394
395 spin_unlock(&ipc->rx_lock);
396 return -ETIMEDOUT;
397 }
398
399 spin_unlock(&ipc->rx_lock);
400 return 0;
401}
402
403static void avs_ipc_msg_init(struct avs_ipc *ipc, struct avs_ipc_msg *reply)
404{
405 lockdep_assert_held(&ipc->rx_lock);
406
407 ipc->rx.header = 0;
408 ipc->rx.size = reply ? reply->size : 0;
409 ipc->rx_completed = false;
410
411 reinit_completion(&ipc->done_completion);
412 reinit_completion(&ipc->busy_completion);
413}
414
415static void avs_dsp_send_tx(struct avs_dev *adev, struct avs_ipc_msg *tx, bool read_fwregs)
416{
417 const struct avs_spec *const spec = adev->spec;
418 u64 reg = ULONG_MAX;
419
420 tx->header |= spec->hipc->req_busy_mask;
421 if (read_fwregs)
422 reg = readq(avs_sram_addr(adev, AVS_FW_REGS_WINDOW));
423
424 trace_avs_request(tx, reg);
425
426 if (tx->size)
427 memcpy_toio(avs_downlink_addr(adev), tx->data, tx->size);
428 snd_hdac_adsp_writel(adev, spec->hipc->req_ext_offset, tx->header >> 32);
429 snd_hdac_adsp_writel(adev, spec->hipc->req_offset, tx->header & UINT_MAX);
430}
431
432static int avs_dsp_do_send_msg(struct avs_dev *adev, struct avs_ipc_msg *request,
433 struct avs_ipc_msg *reply, int timeout, const char *name)
434{
435 struct avs_ipc *ipc = adev->ipc;
436 int ret;
437
438 if (!ipc->ready)
439 return -EPERM;
440
441 mutex_lock(&ipc->msg_mutex);
442
443 spin_lock(&ipc->rx_lock);
444 avs_ipc_msg_init(ipc, reply);
445 avs_dsp_send_tx(adev, request, true);
446 spin_unlock(&ipc->rx_lock);
447
448 ret = avs_ipc_wait_busy_completion(ipc, timeout);
449 if (ret) {
450 if (ret == -ETIMEDOUT) {
451 union avs_notify_msg msg = AVS_NOTIFICATION(EXCEPTION_CAUGHT);
452
453 /* Same treatment as on exception, just stack_dump=0. */
454 avs_dsp_exception_caught(adev, &msg);
455 }
456 goto exit;
457 }
458
459 ret = ipc->rx.rsp.status;
460 /*
461 * If IPC channel is blocked e.g.: due to ongoing recovery,
462 * -EPERM error code is expected and thus it's not an actual error.
463 *
464 * Unsupported IPCs are of no harm either.
465 */
466 if (ret == -EPERM || ret == AVS_IPC_NOT_SUPPORTED)
467 dev_dbg(adev->dev, "%s (0x%08x 0x%08x) failed: %d\n",
468 name, request->glb.primary, request->glb.ext.val, ret);
469 else if (ret)
470 dev_err(adev->dev, "%s (0x%08x 0x%08x) failed: %d\n",
471 name, request->glb.primary, request->glb.ext.val, ret);
472
473 if (reply) {
474 reply->header = ipc->rx.header;
475 reply->size = ipc->rx.size;
476 if (reply->data && ipc->rx.size)
477 memcpy(reply->data, ipc->rx.data, reply->size);
478 }
479
480exit:
481 mutex_unlock(&ipc->msg_mutex);
482 return ret;
483}
484
485static int avs_dsp_send_msg_sequence(struct avs_dev *adev, struct avs_ipc_msg *request,
486 struct avs_ipc_msg *reply, int timeout, bool wake_d0i0,
487 bool schedule_d0ix, const char *name)
488{
489 int ret;
490
491 trace_avs_d0ix("wake", wake_d0i0, request->header);
492 if (wake_d0i0) {
493 ret = avs_dsp_wake_d0i0(adev, request);
494 if (ret)
495 return ret;
496 }
497
498 ret = avs_dsp_do_send_msg(adev, request, reply, timeout, name);
499 if (ret)
500 return ret;
501
502 trace_avs_d0ix("schedule", schedule_d0ix, request->header);
503 if (schedule_d0ix)
504 avs_dsp_schedule_d0ix(adev, request);
505
506 return 0;
507}
508
509int avs_dsp_send_msg_timeout(struct avs_dev *adev, struct avs_ipc_msg *request,
510 struct avs_ipc_msg *reply, int timeout, const char *name)
511{
512 bool wake_d0i0 = avs_dsp_op(adev, d0ix_toggle, request, true);
513 bool schedule_d0ix = avs_dsp_op(adev, d0ix_toggle, request, false);
514
515 return avs_dsp_send_msg_sequence(adev, request, reply, timeout, wake_d0i0, schedule_d0ix,
516 name);
517}
518
519int avs_dsp_send_msg(struct avs_dev *adev, struct avs_ipc_msg *request,
520 struct avs_ipc_msg *reply, const char *name)
521{
522 return avs_dsp_send_msg_timeout(adev, request, reply, adev->ipc->default_timeout_ms, name);
523}
524
525int avs_dsp_send_pm_msg_timeout(struct avs_dev *adev, struct avs_ipc_msg *request,
526 struct avs_ipc_msg *reply, int timeout, bool wake_d0i0,
527 const char *name)
528{
529 return avs_dsp_send_msg_sequence(adev, request, reply, timeout, wake_d0i0, false, name);
530}
531
532int avs_dsp_send_pm_msg(struct avs_dev *adev, struct avs_ipc_msg *request,
533 struct avs_ipc_msg *reply, bool wake_d0i0, const char *name)
534{
535 return avs_dsp_send_pm_msg_timeout(adev, request, reply, adev->ipc->default_timeout_ms,
536 wake_d0i0, name);
537}
538
539static int avs_dsp_do_send_rom_msg(struct avs_dev *adev, struct avs_ipc_msg *request, int timeout,
540 const char *name)
541{
542 struct avs_ipc *ipc = adev->ipc;
543 int ret;
544
545 mutex_lock(&ipc->msg_mutex);
546
547 spin_lock(&ipc->rx_lock);
548 avs_ipc_msg_init(ipc, NULL);
549 /*
550 * with hw still stalled, memory windows may not be
551 * configured properly so avoid accessing SRAM
552 */
553 avs_dsp_send_tx(adev, request, false);
554 spin_unlock(&ipc->rx_lock);
555
556 /* ROM messages must be sent before main core is unstalled */
557 ret = avs_dsp_op(adev, stall, AVS_MAIN_CORE_MASK, false);
558 if (!ret) {
559 ret = wait_for_completion_timeout(&ipc->done_completion, msecs_to_jiffies(timeout));
560 ret = ret ? 0 : -ETIMEDOUT;
561 }
562 if (ret)
563 dev_err(adev->dev, "%s (0x%08x 0x%08x) failed: %d\n",
564 name, request->glb.primary, request->glb.ext.val, ret);
565
566 mutex_unlock(&ipc->msg_mutex);
567
568 return ret;
569}
570
571int avs_dsp_send_rom_msg_timeout(struct avs_dev *adev, struct avs_ipc_msg *request, int timeout,
572 const char *name)
573{
574 return avs_dsp_do_send_rom_msg(adev, request, timeout, name);
575}
576
577int avs_dsp_send_rom_msg(struct avs_dev *adev, struct avs_ipc_msg *request, const char *name)
578{
579 return avs_dsp_send_rom_msg_timeout(adev, request, adev->ipc->default_timeout_ms, name);
580}
581
582void avs_dsp_interrupt_control(struct avs_dev *adev, bool enable)
583{
584 const struct avs_spec *const spec = adev->spec;
585 u32 value, mask;
586
587 /*
588 * No particular bit setting order. All of these are required
589 * to have a functional SW <-> FW communication.
590 */
591 value = enable ? AVS_ADSP_ADSPIC_IPC : 0;
592 snd_hdac_adsp_updatel(adev, AVS_ADSP_REG_ADSPIC, AVS_ADSP_ADSPIC_IPC, value);
593
594 mask = AVS_ADSP_HIPCCTL_DONE | AVS_ADSP_HIPCCTL_BUSY;
595 value = enable ? mask : 0;
596 snd_hdac_adsp_updatel(adev, spec->hipc->ctl_offset, mask, value);
597}
598
599int avs_ipc_init(struct avs_ipc *ipc, struct device *dev)
600{
601 ipc->rx.data = devm_kzalloc(dev, AVS_MAILBOX_SIZE, GFP_KERNEL);
602 if (!ipc->rx.data)
603 return -ENOMEM;
604
605 ipc->dev = dev;
606 ipc->ready = false;
607 ipc->default_timeout_ms = AVS_IPC_TIMEOUT_MS;
608 INIT_WORK(&ipc->recovery_work, avs_dsp_recovery_work);
609 INIT_DELAYED_WORK(&ipc->d0ix_work, avs_dsp_d0ix_work);
610 init_completion(&ipc->done_completion);
611 init_completion(&ipc->busy_completion);
612 spin_lock_init(&ipc->rx_lock);
613 mutex_init(&ipc->msg_mutex);
614
615 return 0;
616}
617
618void avs_ipc_block(struct avs_ipc *ipc)
619{
620 ipc->ready = false;
621 cancel_work_sync(&ipc->recovery_work);
622 cancel_delayed_work_sync(&ipc->d0ix_work);
623 ipc->in_d0ix = false;
624}