Loading...
Note: File does not exist in v4.17.
1// SPDX-License-Identifier: GPL-2.0
2
3/* Copyright (c) 2012-2018, The Linux Foundation. All rights reserved.
4 * Copyright (C) 2019-2022 Linaro Ltd.
5 */
6
7#include <linux/types.h>
8#include <linux/bitfield.h>
9#include <linux/bug.h>
10#include <linux/dma-mapping.h>
11#include <linux/iommu.h>
12#include <linux/io.h>
13#include <linux/soc/qcom/smem.h>
14
15#include "ipa.h"
16#include "ipa_reg.h"
17#include "ipa_data.h"
18#include "ipa_cmd.h"
19#include "ipa_mem.h"
20#include "ipa_table.h"
21#include "gsi_trans.h"
22
23/* "Canary" value placed between memory regions to detect overflow */
24#define IPA_MEM_CANARY_VAL cpu_to_le32(0xdeadbeef)
25
26/* SMEM host id representing the modem. */
27#define QCOM_SMEM_HOST_MODEM 1
28
29const struct ipa_mem *ipa_mem_find(struct ipa *ipa, enum ipa_mem_id mem_id)
30{
31 u32 i;
32
33 for (i = 0; i < ipa->mem_count; i++) {
34 const struct ipa_mem *mem = &ipa->mem[i];
35
36 if (mem->id == mem_id)
37 return mem;
38 }
39
40 return NULL;
41}
42
43/* Add an immediate command to a transaction that zeroes a memory region */
44static void
45ipa_mem_zero_region_add(struct gsi_trans *trans, enum ipa_mem_id mem_id)
46{
47 struct ipa *ipa = container_of(trans->gsi, struct ipa, gsi);
48 const struct ipa_mem *mem = ipa_mem_find(ipa, mem_id);
49 dma_addr_t addr = ipa->zero_addr;
50
51 if (!mem->size)
52 return;
53
54 ipa_cmd_dma_shared_mem_add(trans, mem->offset, mem->size, addr, true);
55}
56
57/**
58 * ipa_mem_setup() - Set up IPA AP and modem shared memory areas
59 * @ipa: IPA pointer
60 *
61 * Set up the shared memory regions in IPA local memory. This involves
62 * zero-filling memory regions, and in the case of header memory, telling
63 * the IPA where it's located.
64 *
65 * This function performs the initial setup of this memory. If the modem
66 * crashes, its regions are re-zeroed in ipa_mem_zero_modem().
67 *
68 * The AP informs the modem where its portions of memory are located
69 * in a QMI exchange that occurs at modem startup.
70 *
71 * There is no need for a matching ipa_mem_teardown() function.
72 *
73 * Return: 0 if successful, or a negative error code
74 */
75int ipa_mem_setup(struct ipa *ipa)
76{
77 dma_addr_t addr = ipa->zero_addr;
78 const struct ipa_reg *reg;
79 const struct ipa_mem *mem;
80 struct gsi_trans *trans;
81 u32 offset;
82 u16 size;
83 u32 val;
84
85 /* Get a transaction to define the header memory region and to zero
86 * the processing context and modem memory regions.
87 */
88 trans = ipa_cmd_trans_alloc(ipa, 4);
89 if (!trans) {
90 dev_err(&ipa->pdev->dev, "no transaction for memory setup\n");
91 return -EBUSY;
92 }
93
94 /* Initialize IPA-local header memory. The AP header region, if
95 * present, is contiguous with and follows the modem header region,
96 * and they are initialized together.
97 */
98 mem = ipa_mem_find(ipa, IPA_MEM_MODEM_HEADER);
99 offset = mem->offset;
100 size = mem->size;
101 mem = ipa_mem_find(ipa, IPA_MEM_AP_HEADER);
102 if (mem)
103 size += mem->size;
104
105 ipa_cmd_hdr_init_local_add(trans, offset, size, addr);
106
107 ipa_mem_zero_region_add(trans, IPA_MEM_MODEM_PROC_CTX);
108 ipa_mem_zero_region_add(trans, IPA_MEM_AP_PROC_CTX);
109 ipa_mem_zero_region_add(trans, IPA_MEM_MODEM);
110
111 gsi_trans_commit_wait(trans);
112
113 /* Tell the hardware where the processing context area is located */
114 mem = ipa_mem_find(ipa, IPA_MEM_MODEM_PROC_CTX);
115 offset = ipa->mem_offset + mem->offset;
116
117 reg = ipa_reg(ipa, LOCAL_PKT_PROC_CNTXT);
118 val = ipa_reg_encode(reg, IPA_BASE_ADDR, offset);
119 iowrite32(val, ipa->reg_virt + ipa_reg_offset(reg));
120
121 return 0;
122}
123
124/* Is the given memory region ID is valid for the current IPA version? */
125static bool ipa_mem_id_valid(struct ipa *ipa, enum ipa_mem_id mem_id)
126{
127 enum ipa_version version = ipa->version;
128
129 switch (mem_id) {
130 case IPA_MEM_UC_SHARED:
131 case IPA_MEM_UC_INFO:
132 case IPA_MEM_V4_FILTER_HASHED:
133 case IPA_MEM_V4_FILTER:
134 case IPA_MEM_V6_FILTER_HASHED:
135 case IPA_MEM_V6_FILTER:
136 case IPA_MEM_V4_ROUTE_HASHED:
137 case IPA_MEM_V4_ROUTE:
138 case IPA_MEM_V6_ROUTE_HASHED:
139 case IPA_MEM_V6_ROUTE:
140 case IPA_MEM_MODEM_HEADER:
141 case IPA_MEM_AP_HEADER:
142 case IPA_MEM_MODEM_PROC_CTX:
143 case IPA_MEM_AP_PROC_CTX:
144 case IPA_MEM_MODEM:
145 case IPA_MEM_UC_EVENT_RING:
146 case IPA_MEM_PDN_CONFIG:
147 case IPA_MEM_STATS_QUOTA_MODEM:
148 case IPA_MEM_STATS_QUOTA_AP:
149 case IPA_MEM_END_MARKER: /* pseudo region */
150 break;
151
152 case IPA_MEM_STATS_TETHERING:
153 case IPA_MEM_STATS_DROP:
154 if (version < IPA_VERSION_4_0)
155 return false;
156 break;
157
158 case IPA_MEM_STATS_V4_FILTER:
159 case IPA_MEM_STATS_V6_FILTER:
160 case IPA_MEM_STATS_V4_ROUTE:
161 case IPA_MEM_STATS_V6_ROUTE:
162 if (version < IPA_VERSION_4_0 || version > IPA_VERSION_4_2)
163 return false;
164 break;
165
166 case IPA_MEM_NAT_TABLE:
167 case IPA_MEM_STATS_FILTER_ROUTE:
168 if (version < IPA_VERSION_4_5)
169 return false;
170 break;
171
172 default:
173 return false;
174 }
175
176 return true;
177}
178
179/* Must the given memory region be present in the configuration? */
180static bool ipa_mem_id_required(struct ipa *ipa, enum ipa_mem_id mem_id)
181{
182 switch (mem_id) {
183 case IPA_MEM_UC_SHARED:
184 case IPA_MEM_UC_INFO:
185 case IPA_MEM_V4_FILTER_HASHED:
186 case IPA_MEM_V4_FILTER:
187 case IPA_MEM_V6_FILTER_HASHED:
188 case IPA_MEM_V6_FILTER:
189 case IPA_MEM_V4_ROUTE_HASHED:
190 case IPA_MEM_V4_ROUTE:
191 case IPA_MEM_V6_ROUTE_HASHED:
192 case IPA_MEM_V6_ROUTE:
193 case IPA_MEM_MODEM_HEADER:
194 case IPA_MEM_MODEM_PROC_CTX:
195 case IPA_MEM_AP_PROC_CTX:
196 case IPA_MEM_MODEM:
197 return true;
198
199 case IPA_MEM_PDN_CONFIG:
200 case IPA_MEM_STATS_QUOTA_MODEM:
201 return ipa->version >= IPA_VERSION_4_0;
202
203 case IPA_MEM_STATS_TETHERING:
204 return ipa->version >= IPA_VERSION_4_0 &&
205 ipa->version != IPA_VERSION_5_0;
206
207 default:
208 return false; /* Anything else is optional */
209 }
210}
211
212static bool ipa_mem_valid_one(struct ipa *ipa, const struct ipa_mem *mem)
213{
214 struct device *dev = &ipa->pdev->dev;
215 enum ipa_mem_id mem_id = mem->id;
216 u16 size_multiple;
217
218 /* Make sure the memory region is valid for this version of IPA */
219 if (!ipa_mem_id_valid(ipa, mem_id)) {
220 dev_err(dev, "region id %u not valid\n", mem_id);
221 return false;
222 }
223
224 if (!mem->size && !mem->canary_count) {
225 dev_err(dev, "empty memory region %u\n", mem_id);
226 return false;
227 }
228
229 /* Other than modem memory, sizes must be a multiple of 8 */
230 size_multiple = mem_id == IPA_MEM_MODEM ? 4 : 8;
231 if (mem->size % size_multiple)
232 dev_err(dev, "region %u size not a multiple of %u bytes\n",
233 mem_id, size_multiple);
234 else if (mem->offset % 8)
235 dev_err(dev, "region %u offset not 8-byte aligned\n", mem_id);
236 else if (mem->offset < mem->canary_count * sizeof(__le32))
237 dev_err(dev, "region %u offset too small for %hu canaries\n",
238 mem_id, mem->canary_count);
239 else if (mem_id == IPA_MEM_END_MARKER && mem->size)
240 dev_err(dev, "non-zero end marker region size\n");
241 else
242 return true;
243
244 return false;
245}
246
247/* Verify each defined memory region is valid. */
248static bool ipa_mem_valid(struct ipa *ipa, const struct ipa_mem_data *mem_data)
249{
250 DECLARE_BITMAP(regions, IPA_MEM_COUNT) = { };
251 struct device *dev = &ipa->pdev->dev;
252 enum ipa_mem_id mem_id;
253 u32 i;
254
255 if (mem_data->local_count > IPA_MEM_COUNT) {
256 dev_err(dev, "too many memory regions (%u > %u)\n",
257 mem_data->local_count, IPA_MEM_COUNT);
258 return false;
259 }
260
261 for (i = 0; i < mem_data->local_count; i++) {
262 const struct ipa_mem *mem = &mem_data->local[i];
263
264 if (__test_and_set_bit(mem->id, regions)) {
265 dev_err(dev, "duplicate memory region %u\n", mem->id);
266 return false;
267 }
268
269 /* Defined regions have non-zero size and/or canary count */
270 if (!ipa_mem_valid_one(ipa, mem))
271 return false;
272 }
273
274 /* Now see if any required regions are not defined */
275 for_each_clear_bit(mem_id, regions, IPA_MEM_COUNT) {
276 if (ipa_mem_id_required(ipa, mem_id))
277 dev_err(dev, "required memory region %u missing\n",
278 mem_id);
279 }
280
281 return true;
282}
283
284/* Do all memory regions fit within the IPA local memory? */
285static bool ipa_mem_size_valid(struct ipa *ipa)
286{
287 struct device *dev = &ipa->pdev->dev;
288 u32 limit = ipa->mem_size;
289 u32 i;
290
291 for (i = 0; i < ipa->mem_count; i++) {
292 const struct ipa_mem *mem = &ipa->mem[i];
293
294 if (mem->offset + mem->size <= limit)
295 continue;
296
297 dev_err(dev, "region %u ends beyond memory limit (0x%08x)\n",
298 mem->id, limit);
299
300 return false;
301 }
302
303 return true;
304}
305
306/**
307 * ipa_mem_config() - Configure IPA shared memory
308 * @ipa: IPA pointer
309 *
310 * Return: 0 if successful, or a negative error code
311 */
312int ipa_mem_config(struct ipa *ipa)
313{
314 struct device *dev = &ipa->pdev->dev;
315 const struct ipa_reg *reg;
316 const struct ipa_mem *mem;
317 dma_addr_t addr;
318 u32 mem_size;
319 void *virt;
320 u32 val;
321 u32 i;
322
323 /* Check the advertised location and size of the shared memory area */
324 reg = ipa_reg(ipa, SHARED_MEM_SIZE);
325 val = ioread32(ipa->reg_virt + ipa_reg_offset(reg));
326
327 /* The fields in the register are in 8 byte units */
328 ipa->mem_offset = 8 * ipa_reg_decode(reg, MEM_BADDR, val);
329
330 /* Make sure the end is within the region's mapped space */
331 mem_size = 8 * ipa_reg_decode(reg, MEM_SIZE, val);
332
333 /* If the sizes don't match, issue a warning */
334 if (ipa->mem_offset + mem_size < ipa->mem_size) {
335 dev_warn(dev, "limiting IPA memory size to 0x%08x\n",
336 mem_size);
337 ipa->mem_size = mem_size;
338 } else if (ipa->mem_offset + mem_size > ipa->mem_size) {
339 dev_dbg(dev, "ignoring larger reported memory size: 0x%08x\n",
340 mem_size);
341 }
342
343 /* We know our memory size; make sure regions are all in range */
344 if (!ipa_mem_size_valid(ipa))
345 return -EINVAL;
346
347 /* Prealloc DMA memory for zeroing regions */
348 virt = dma_alloc_coherent(dev, IPA_MEM_MAX, &addr, GFP_KERNEL);
349 if (!virt)
350 return -ENOMEM;
351 ipa->zero_addr = addr;
352 ipa->zero_virt = virt;
353 ipa->zero_size = IPA_MEM_MAX;
354
355 /* For each defined region, write "canary" values in the
356 * space prior to the region's base address if indicated.
357 */
358 for (i = 0; i < ipa->mem_count; i++) {
359 u16 canary_count = ipa->mem[i].canary_count;
360 __le32 *canary;
361
362 if (!canary_count)
363 continue;
364
365 /* Write canary values in the space before the region */
366 canary = ipa->mem_virt + ipa->mem_offset + ipa->mem[i].offset;
367 do
368 *--canary = IPA_MEM_CANARY_VAL;
369 while (--canary_count);
370 }
371
372 /* Verify the microcontroller ring alignment (if defined) */
373 mem = ipa_mem_find(ipa, IPA_MEM_UC_EVENT_RING);
374 if (mem && mem->offset % 1024) {
375 dev_err(dev, "microcontroller ring not 1024-byte aligned\n");
376 goto err_dma_free;
377 }
378
379 return 0;
380
381err_dma_free:
382 dma_free_coherent(dev, IPA_MEM_MAX, ipa->zero_virt, ipa->zero_addr);
383
384 return -EINVAL;
385}
386
387/* Inverse of ipa_mem_config() */
388void ipa_mem_deconfig(struct ipa *ipa)
389{
390 struct device *dev = &ipa->pdev->dev;
391
392 dma_free_coherent(dev, ipa->zero_size, ipa->zero_virt, ipa->zero_addr);
393 ipa->zero_size = 0;
394 ipa->zero_virt = NULL;
395 ipa->zero_addr = 0;
396}
397
398/**
399 * ipa_mem_zero_modem() - Zero IPA-local memory regions owned by the modem
400 * @ipa: IPA pointer
401 *
402 * Zero regions of IPA-local memory used by the modem. These are configured
403 * (and initially zeroed) by ipa_mem_setup(), but if the modem crashes and
404 * restarts via SSR we need to re-initialize them. A QMI message tells the
405 * modem where to find regions of IPA local memory it needs to know about
406 * (these included).
407 */
408int ipa_mem_zero_modem(struct ipa *ipa)
409{
410 struct gsi_trans *trans;
411
412 /* Get a transaction to zero the modem memory, modem header,
413 * and modem processing context regions.
414 */
415 trans = ipa_cmd_trans_alloc(ipa, 3);
416 if (!trans) {
417 dev_err(&ipa->pdev->dev,
418 "no transaction to zero modem memory\n");
419 return -EBUSY;
420 }
421
422 ipa_mem_zero_region_add(trans, IPA_MEM_MODEM_HEADER);
423 ipa_mem_zero_region_add(trans, IPA_MEM_MODEM_PROC_CTX);
424 ipa_mem_zero_region_add(trans, IPA_MEM_MODEM);
425
426 gsi_trans_commit_wait(trans);
427
428 return 0;
429}
430
431/**
432 * ipa_imem_init() - Initialize IMEM memory used by the IPA
433 * @ipa: IPA pointer
434 * @addr: Physical address of the IPA region in IMEM
435 * @size: Size (bytes) of the IPA region in IMEM
436 *
437 * IMEM is a block of shared memory separate from system DRAM, and
438 * a portion of this memory is available for the IPA to use. The
439 * modem accesses this memory directly, but the IPA accesses it
440 * via the IOMMU, using the AP's credentials.
441 *
442 * If this region exists (size > 0) we map it for read/write access
443 * through the IOMMU using the IPA device.
444 *
445 * Note: @addr and @size are not guaranteed to be page-aligned.
446 */
447static int ipa_imem_init(struct ipa *ipa, unsigned long addr, size_t size)
448{
449 struct device *dev = &ipa->pdev->dev;
450 struct iommu_domain *domain;
451 unsigned long iova;
452 phys_addr_t phys;
453 int ret;
454
455 if (!size)
456 return 0; /* IMEM memory not used */
457
458 domain = iommu_get_domain_for_dev(dev);
459 if (!domain) {
460 dev_err(dev, "no IOMMU domain found for IMEM\n");
461 return -EINVAL;
462 }
463
464 /* Align the address down and the size up to page boundaries */
465 phys = addr & PAGE_MASK;
466 size = PAGE_ALIGN(size + addr - phys);
467 iova = phys; /* We just want a direct mapping */
468
469 ret = iommu_map(domain, iova, phys, size, IOMMU_READ | IOMMU_WRITE);
470 if (ret)
471 return ret;
472
473 ipa->imem_iova = iova;
474 ipa->imem_size = size;
475
476 return 0;
477}
478
479static void ipa_imem_exit(struct ipa *ipa)
480{
481 struct iommu_domain *domain;
482 struct device *dev;
483
484 if (!ipa->imem_size)
485 return;
486
487 dev = &ipa->pdev->dev;
488 domain = iommu_get_domain_for_dev(dev);
489 if (domain) {
490 size_t size;
491
492 size = iommu_unmap(domain, ipa->imem_iova, ipa->imem_size);
493 if (size != ipa->imem_size)
494 dev_warn(dev, "unmapped %zu IMEM bytes, expected %zu\n",
495 size, ipa->imem_size);
496 } else {
497 dev_err(dev, "couldn't get IPA IOMMU domain for IMEM\n");
498 }
499
500 ipa->imem_size = 0;
501 ipa->imem_iova = 0;
502}
503
504/**
505 * ipa_smem_init() - Initialize SMEM memory used by the IPA
506 * @ipa: IPA pointer
507 * @item: Item ID of SMEM memory
508 * @size: Size (bytes) of SMEM memory region
509 *
510 * SMEM is a managed block of shared DRAM, from which numbered "items"
511 * can be allocated. One item is designated for use by the IPA.
512 *
513 * The modem accesses SMEM memory directly, but the IPA accesses it
514 * via the IOMMU, using the AP's credentials.
515 *
516 * If size provided is non-zero, we allocate it and map it for
517 * access through the IOMMU.
518 *
519 * Note: @size and the item address are is not guaranteed to be page-aligned.
520 */
521static int ipa_smem_init(struct ipa *ipa, u32 item, size_t size)
522{
523 struct device *dev = &ipa->pdev->dev;
524 struct iommu_domain *domain;
525 unsigned long iova;
526 phys_addr_t phys;
527 phys_addr_t addr;
528 size_t actual;
529 void *virt;
530 int ret;
531
532 if (!size)
533 return 0; /* SMEM memory not used */
534
535 /* SMEM is memory shared between the AP and another system entity
536 * (in this case, the modem). An allocation from SMEM is persistent
537 * until the AP reboots; there is no way to free an allocated SMEM
538 * region. Allocation only reserves the space; to use it you need
539 * to "get" a pointer it (this does not imply reference counting).
540 * The item might have already been allocated, in which case we
541 * use it unless the size isn't what we expect.
542 */
543 ret = qcom_smem_alloc(QCOM_SMEM_HOST_MODEM, item, size);
544 if (ret && ret != -EEXIST) {
545 dev_err(dev, "error %d allocating size %zu SMEM item %u\n",
546 ret, size, item);
547 return ret;
548 }
549
550 /* Now get the address of the SMEM memory region */
551 virt = qcom_smem_get(QCOM_SMEM_HOST_MODEM, item, &actual);
552 if (IS_ERR(virt)) {
553 ret = PTR_ERR(virt);
554 dev_err(dev, "error %d getting SMEM item %u\n", ret, item);
555 return ret;
556 }
557
558 /* In case the region was already allocated, verify the size */
559 if (ret && actual != size) {
560 dev_err(dev, "SMEM item %u has size %zu, expected %zu\n",
561 item, actual, size);
562 return -EINVAL;
563 }
564
565 domain = iommu_get_domain_for_dev(dev);
566 if (!domain) {
567 dev_err(dev, "no IOMMU domain found for SMEM\n");
568 return -EINVAL;
569 }
570
571 /* Align the address down and the size up to a page boundary */
572 addr = qcom_smem_virt_to_phys(virt);
573 phys = addr & PAGE_MASK;
574 size = PAGE_ALIGN(size + addr - phys);
575 iova = phys; /* We just want a direct mapping */
576
577 ret = iommu_map(domain, iova, phys, size, IOMMU_READ | IOMMU_WRITE);
578 if (ret)
579 return ret;
580
581 ipa->smem_iova = iova;
582 ipa->smem_size = size;
583
584 return 0;
585}
586
587static void ipa_smem_exit(struct ipa *ipa)
588{
589 struct device *dev = &ipa->pdev->dev;
590 struct iommu_domain *domain;
591
592 domain = iommu_get_domain_for_dev(dev);
593 if (domain) {
594 size_t size;
595
596 size = iommu_unmap(domain, ipa->smem_iova, ipa->smem_size);
597 if (size != ipa->smem_size)
598 dev_warn(dev, "unmapped %zu SMEM bytes, expected %zu\n",
599 size, ipa->smem_size);
600
601 } else {
602 dev_err(dev, "couldn't get IPA IOMMU domain for SMEM\n");
603 }
604
605 ipa->smem_size = 0;
606 ipa->smem_iova = 0;
607}
608
609/* Perform memory region-related initialization */
610int ipa_mem_init(struct ipa *ipa, const struct ipa_mem_data *mem_data)
611{
612 struct device *dev = &ipa->pdev->dev;
613 struct resource *res;
614 int ret;
615
616 /* Make sure the set of defined memory regions is valid */
617 if (!ipa_mem_valid(ipa, mem_data))
618 return -EINVAL;
619
620 ipa->mem_count = mem_data->local_count;
621 ipa->mem = mem_data->local;
622
623 /* Check the route and filter table memory regions */
624 if (!ipa_table_mem_valid(ipa, false))
625 return -EINVAL;
626 if (!ipa_table_mem_valid(ipa, true))
627 return -EINVAL;
628
629 ret = dma_set_mask_and_coherent(&ipa->pdev->dev, DMA_BIT_MASK(64));
630 if (ret) {
631 dev_err(dev, "error %d setting DMA mask\n", ret);
632 return ret;
633 }
634
635 res = platform_get_resource_byname(ipa->pdev, IORESOURCE_MEM,
636 "ipa-shared");
637 if (!res) {
638 dev_err(dev,
639 "DT error getting \"ipa-shared\" memory property\n");
640 return -ENODEV;
641 }
642
643 ipa->mem_virt = memremap(res->start, resource_size(res), MEMREMAP_WC);
644 if (!ipa->mem_virt) {
645 dev_err(dev, "unable to remap \"ipa-shared\" memory\n");
646 return -ENOMEM;
647 }
648
649 ipa->mem_addr = res->start;
650 ipa->mem_size = resource_size(res);
651
652 ret = ipa_imem_init(ipa, mem_data->imem_addr, mem_data->imem_size);
653 if (ret)
654 goto err_unmap;
655
656 ret = ipa_smem_init(ipa, mem_data->smem_id, mem_data->smem_size);
657 if (ret)
658 goto err_imem_exit;
659
660 return 0;
661
662err_imem_exit:
663 ipa_imem_exit(ipa);
664err_unmap:
665 memunmap(ipa->mem_virt);
666
667 return ret;
668}
669
670/* Inverse of ipa_mem_init() */
671void ipa_mem_exit(struct ipa *ipa)
672{
673 ipa_smem_exit(ipa);
674 ipa_imem_exit(ipa);
675 memunmap(ipa->mem_virt);
676}