Loading...
1// SPDX-License-Identifier: GPL-2.0
2
3/* Copyright (c) 2012-2018, The Linux Foundation. All rights reserved.
4 * Copyright (C) 2019-2024 Linaro Ltd.
5 */
6
7#include <linux/dma-mapping.h>
8#include <linux/io.h>
9#include <linux/iommu.h>
10#include <linux/platform_device.h>
11#include <linux/types.h>
12
13#include <linux/soc/qcom/smem.h>
14
15#include "gsi_trans.h"
16#include "ipa.h"
17#include "ipa_cmd.h"
18#include "ipa_data.h"
19#include "ipa_mem.h"
20#include "ipa_reg.h"
21#include "ipa_table.h"
22
23/* "Canary" value placed between memory regions to detect overflow */
24#define IPA_MEM_CANARY_VAL cpu_to_le32(0xdeadbeef)
25
26/* SMEM host id representing the modem. */
27#define QCOM_SMEM_HOST_MODEM 1
28
29const struct ipa_mem *ipa_mem_find(struct ipa *ipa, enum ipa_mem_id mem_id)
30{
31 u32 i;
32
33 for (i = 0; i < ipa->mem_count; i++) {
34 const struct ipa_mem *mem = &ipa->mem[i];
35
36 if (mem->id == mem_id)
37 return mem;
38 }
39
40 return NULL;
41}
42
43/* Add an immediate command to a transaction that zeroes a memory region */
44static void
45ipa_mem_zero_region_add(struct gsi_trans *trans, enum ipa_mem_id mem_id)
46{
47 struct ipa *ipa = container_of(trans->gsi, struct ipa, gsi);
48 const struct ipa_mem *mem = ipa_mem_find(ipa, mem_id);
49 dma_addr_t addr = ipa->zero_addr;
50
51 if (!mem->size)
52 return;
53
54 ipa_cmd_dma_shared_mem_add(trans, mem->offset, mem->size, addr, true);
55}
56
57/**
58 * ipa_mem_setup() - Set up IPA AP and modem shared memory areas
59 * @ipa: IPA pointer
60 *
61 * Set up the shared memory regions in IPA local memory. This involves
62 * zero-filling memory regions, and in the case of header memory, telling
63 * the IPA where it's located.
64 *
65 * This function performs the initial setup of this memory. If the modem
66 * crashes, its regions are re-zeroed in ipa_mem_zero_modem().
67 *
68 * The AP informs the modem where its portions of memory are located
69 * in a QMI exchange that occurs at modem startup.
70 *
71 * There is no need for a matching ipa_mem_teardown() function.
72 *
73 * Return: 0 if successful, or a negative error code
74 */
75int ipa_mem_setup(struct ipa *ipa)
76{
77 dma_addr_t addr = ipa->zero_addr;
78 const struct ipa_mem *mem;
79 struct gsi_trans *trans;
80 const struct reg *reg;
81 u32 offset;
82 u16 size;
83 u32 val;
84
85 /* Get a transaction to define the header memory region and to zero
86 * the processing context and modem memory regions.
87 */
88 trans = ipa_cmd_trans_alloc(ipa, 4);
89 if (!trans) {
90 dev_err(ipa->dev, "no transaction for memory setup\n");
91 return -EBUSY;
92 }
93
94 /* Initialize IPA-local header memory. The AP header region, if
95 * present, is contiguous with and follows the modem header region,
96 * and they are initialized together.
97 */
98 mem = ipa_mem_find(ipa, IPA_MEM_MODEM_HEADER);
99 offset = mem->offset;
100 size = mem->size;
101 mem = ipa_mem_find(ipa, IPA_MEM_AP_HEADER);
102 if (mem)
103 size += mem->size;
104
105 ipa_cmd_hdr_init_local_add(trans, offset, size, addr);
106
107 ipa_mem_zero_region_add(trans, IPA_MEM_MODEM_PROC_CTX);
108 ipa_mem_zero_region_add(trans, IPA_MEM_AP_PROC_CTX);
109 ipa_mem_zero_region_add(trans, IPA_MEM_MODEM);
110
111 gsi_trans_commit_wait(trans);
112
113 /* Tell the hardware where the processing context area is located */
114 mem = ipa_mem_find(ipa, IPA_MEM_MODEM_PROC_CTX);
115 offset = ipa->mem_offset + mem->offset;
116
117 reg = ipa_reg(ipa, LOCAL_PKT_PROC_CNTXT);
118 val = reg_encode(reg, IPA_BASE_ADDR, offset);
119 iowrite32(val, ipa->reg_virt + reg_offset(reg));
120
121 return 0;
122}
123
124/* Is the given memory region ID is valid for the current IPA version? */
125static bool ipa_mem_id_valid(struct ipa *ipa, enum ipa_mem_id mem_id)
126{
127 enum ipa_version version = ipa->version;
128
129 switch (mem_id) {
130 case IPA_MEM_UC_SHARED:
131 case IPA_MEM_UC_INFO:
132 case IPA_MEM_V4_FILTER_HASHED:
133 case IPA_MEM_V4_FILTER:
134 case IPA_MEM_V6_FILTER_HASHED:
135 case IPA_MEM_V6_FILTER:
136 case IPA_MEM_V4_ROUTE_HASHED:
137 case IPA_MEM_V4_ROUTE:
138 case IPA_MEM_V6_ROUTE_HASHED:
139 case IPA_MEM_V6_ROUTE:
140 case IPA_MEM_MODEM_HEADER:
141 case IPA_MEM_AP_HEADER:
142 case IPA_MEM_MODEM_PROC_CTX:
143 case IPA_MEM_AP_PROC_CTX:
144 case IPA_MEM_MODEM:
145 case IPA_MEM_UC_EVENT_RING:
146 case IPA_MEM_PDN_CONFIG:
147 case IPA_MEM_STATS_QUOTA_MODEM:
148 case IPA_MEM_STATS_QUOTA_AP:
149 case IPA_MEM_END_MARKER: /* pseudo region */
150 break;
151
152 case IPA_MEM_STATS_TETHERING:
153 case IPA_MEM_STATS_DROP:
154 if (version < IPA_VERSION_4_0)
155 return false;
156 break;
157
158 case IPA_MEM_STATS_V4_FILTER:
159 case IPA_MEM_STATS_V6_FILTER:
160 case IPA_MEM_STATS_V4_ROUTE:
161 case IPA_MEM_STATS_V6_ROUTE:
162 if (version < IPA_VERSION_4_0 || version > IPA_VERSION_4_2)
163 return false;
164 break;
165
166 case IPA_MEM_AP_V4_FILTER:
167 case IPA_MEM_AP_V6_FILTER:
168 if (version < IPA_VERSION_5_0)
169 return false;
170 break;
171
172 case IPA_MEM_NAT_TABLE:
173 case IPA_MEM_STATS_FILTER_ROUTE:
174 if (version < IPA_VERSION_4_5)
175 return false;
176 break;
177
178 default:
179 return false;
180 }
181
182 return true;
183}
184
185/* Must the given memory region be present in the configuration? */
186static bool ipa_mem_id_required(struct ipa *ipa, enum ipa_mem_id mem_id)
187{
188 switch (mem_id) {
189 case IPA_MEM_UC_SHARED:
190 case IPA_MEM_UC_INFO:
191 case IPA_MEM_V4_FILTER_HASHED:
192 case IPA_MEM_V4_FILTER:
193 case IPA_MEM_V6_FILTER_HASHED:
194 case IPA_MEM_V6_FILTER:
195 case IPA_MEM_V4_ROUTE_HASHED:
196 case IPA_MEM_V4_ROUTE:
197 case IPA_MEM_V6_ROUTE_HASHED:
198 case IPA_MEM_V6_ROUTE:
199 case IPA_MEM_MODEM_HEADER:
200 case IPA_MEM_MODEM_PROC_CTX:
201 case IPA_MEM_AP_PROC_CTX:
202 case IPA_MEM_MODEM:
203 return true;
204
205 case IPA_MEM_PDN_CONFIG:
206 case IPA_MEM_STATS_QUOTA_MODEM:
207 return ipa->version >= IPA_VERSION_4_0;
208
209 case IPA_MEM_STATS_TETHERING:
210 return ipa->version >= IPA_VERSION_4_0 &&
211 ipa->version != IPA_VERSION_5_0;
212
213 default:
214 return false; /* Anything else is optional */
215 }
216}
217
218static bool ipa_mem_valid_one(struct ipa *ipa, const struct ipa_mem *mem)
219{
220 enum ipa_mem_id mem_id = mem->id;
221 struct device *dev = ipa->dev;
222 u16 size_multiple;
223
224 /* Make sure the memory region is valid for this version of IPA */
225 if (!ipa_mem_id_valid(ipa, mem_id)) {
226 dev_err(dev, "region id %u not valid\n", mem_id);
227 return false;
228 }
229
230 if (!mem->size && !mem->canary_count) {
231 dev_err(dev, "empty memory region %u\n", mem_id);
232 return false;
233 }
234
235 /* Other than modem memory, sizes must be a multiple of 8 */
236 size_multiple = mem_id == IPA_MEM_MODEM ? 4 : 8;
237 if (mem->size % size_multiple)
238 dev_err(dev, "region %u size not a multiple of %u bytes\n",
239 mem_id, size_multiple);
240 else if (mem->offset % 8)
241 dev_err(dev, "region %u offset not 8-byte aligned\n", mem_id);
242 else if (mem->offset < mem->canary_count * sizeof(__le32))
243 dev_err(dev, "region %u offset too small for %hu canaries\n",
244 mem_id, mem->canary_count);
245 else if (mem_id == IPA_MEM_END_MARKER && mem->size)
246 dev_err(dev, "non-zero end marker region size\n");
247 else
248 return true;
249
250 return false;
251}
252
253/* Verify each defined memory region is valid. */
254static bool ipa_mem_valid(struct ipa *ipa, const struct ipa_mem_data *mem_data)
255{
256 DECLARE_BITMAP(regions, IPA_MEM_COUNT) = { };
257 struct device *dev = ipa->dev;
258 enum ipa_mem_id mem_id;
259 u32 i;
260
261 if (mem_data->local_count > IPA_MEM_COUNT) {
262 dev_err(dev, "too many memory regions (%u > %u)\n",
263 mem_data->local_count, IPA_MEM_COUNT);
264 return false;
265 }
266
267 for (i = 0; i < mem_data->local_count; i++) {
268 const struct ipa_mem *mem = &mem_data->local[i];
269
270 if (__test_and_set_bit(mem->id, regions)) {
271 dev_err(dev, "duplicate memory region %u\n", mem->id);
272 return false;
273 }
274
275 /* Defined regions have non-zero size and/or canary count */
276 if (!ipa_mem_valid_one(ipa, mem))
277 return false;
278 }
279
280 /* Now see if any required regions are not defined */
281 for_each_clear_bit(mem_id, regions, IPA_MEM_COUNT) {
282 if (ipa_mem_id_required(ipa, mem_id))
283 dev_err(dev, "required memory region %u missing\n",
284 mem_id);
285 }
286
287 return true;
288}
289
290/* Do all memory regions fit within the IPA local memory? */
291static bool ipa_mem_size_valid(struct ipa *ipa)
292{
293 struct device *dev = ipa->dev;
294 u32 limit = ipa->mem_size;
295 u32 i;
296
297 for (i = 0; i < ipa->mem_count; i++) {
298 const struct ipa_mem *mem = &ipa->mem[i];
299
300 if (mem->offset + mem->size <= limit)
301 continue;
302
303 dev_err(dev, "region %u ends beyond memory limit (0x%08x)\n",
304 mem->id, limit);
305
306 return false;
307 }
308
309 return true;
310}
311
312/**
313 * ipa_mem_config() - Configure IPA shared memory
314 * @ipa: IPA pointer
315 *
316 * Return: 0 if successful, or a negative error code
317 */
318int ipa_mem_config(struct ipa *ipa)
319{
320 struct device *dev = ipa->dev;
321 const struct ipa_mem *mem;
322 const struct reg *reg;
323 dma_addr_t addr;
324 u32 mem_size;
325 void *virt;
326 u32 val;
327 u32 i;
328
329 /* Check the advertised location and size of the shared memory area */
330 reg = ipa_reg(ipa, SHARED_MEM_SIZE);
331 val = ioread32(ipa->reg_virt + reg_offset(reg));
332
333 /* The fields in the register are in 8 byte units */
334 ipa->mem_offset = 8 * reg_decode(reg, MEM_BADDR, val);
335
336 /* Make sure the end is within the region's mapped space */
337 mem_size = 8 * reg_decode(reg, MEM_SIZE, val);
338
339 /* If the sizes don't match, issue a warning */
340 if (ipa->mem_offset + mem_size < ipa->mem_size) {
341 dev_warn(dev, "limiting IPA memory size to 0x%08x\n",
342 mem_size);
343 ipa->mem_size = mem_size;
344 } else if (ipa->mem_offset + mem_size > ipa->mem_size) {
345 dev_dbg(dev, "ignoring larger reported memory size: 0x%08x\n",
346 mem_size);
347 }
348
349 /* We know our memory size; make sure regions are all in range */
350 if (!ipa_mem_size_valid(ipa))
351 return -EINVAL;
352
353 /* Prealloc DMA memory for zeroing regions */
354 virt = dma_alloc_coherent(dev, IPA_MEM_MAX, &addr, GFP_KERNEL);
355 if (!virt)
356 return -ENOMEM;
357 ipa->zero_addr = addr;
358 ipa->zero_virt = virt;
359 ipa->zero_size = IPA_MEM_MAX;
360
361 /* For each defined region, write "canary" values in the
362 * space prior to the region's base address if indicated.
363 */
364 for (i = 0; i < ipa->mem_count; i++) {
365 u16 canary_count = ipa->mem[i].canary_count;
366 __le32 *canary;
367
368 if (!canary_count)
369 continue;
370
371 /* Write canary values in the space before the region */
372 canary = ipa->mem_virt + ipa->mem_offset + ipa->mem[i].offset;
373 do
374 *--canary = IPA_MEM_CANARY_VAL;
375 while (--canary_count);
376 }
377
378 /* Verify the microcontroller ring alignment (if defined) */
379 mem = ipa_mem_find(ipa, IPA_MEM_UC_EVENT_RING);
380 if (mem && mem->offset % 1024) {
381 dev_err(dev, "microcontroller ring not 1024-byte aligned\n");
382 goto err_dma_free;
383 }
384
385 return 0;
386
387err_dma_free:
388 dma_free_coherent(dev, IPA_MEM_MAX, ipa->zero_virt, ipa->zero_addr);
389
390 return -EINVAL;
391}
392
393/* Inverse of ipa_mem_config() */
394void ipa_mem_deconfig(struct ipa *ipa)
395{
396 struct device *dev = ipa->dev;
397
398 dma_free_coherent(dev, ipa->zero_size, ipa->zero_virt, ipa->zero_addr);
399 ipa->zero_size = 0;
400 ipa->zero_virt = NULL;
401 ipa->zero_addr = 0;
402}
403
404/**
405 * ipa_mem_zero_modem() - Zero IPA-local memory regions owned by the modem
406 * @ipa: IPA pointer
407 *
408 * Zero regions of IPA-local memory used by the modem. These are configured
409 * (and initially zeroed) by ipa_mem_setup(), but if the modem crashes and
410 * restarts via SSR we need to re-initialize them. A QMI message tells the
411 * modem where to find regions of IPA local memory it needs to know about
412 * (these included).
413 */
414int ipa_mem_zero_modem(struct ipa *ipa)
415{
416 struct gsi_trans *trans;
417
418 /* Get a transaction to zero the modem memory, modem header,
419 * and modem processing context regions.
420 */
421 trans = ipa_cmd_trans_alloc(ipa, 3);
422 if (!trans) {
423 dev_err(ipa->dev, "no transaction to zero modem memory\n");
424 return -EBUSY;
425 }
426
427 ipa_mem_zero_region_add(trans, IPA_MEM_MODEM_HEADER);
428 ipa_mem_zero_region_add(trans, IPA_MEM_MODEM_PROC_CTX);
429 ipa_mem_zero_region_add(trans, IPA_MEM_MODEM);
430
431 gsi_trans_commit_wait(trans);
432
433 return 0;
434}
435
436/**
437 * ipa_imem_init() - Initialize IMEM memory used by the IPA
438 * @ipa: IPA pointer
439 * @addr: Physical address of the IPA region in IMEM
440 * @size: Size (bytes) of the IPA region in IMEM
441 *
442 * IMEM is a block of shared memory separate from system DRAM, and
443 * a portion of this memory is available for the IPA to use. The
444 * modem accesses this memory directly, but the IPA accesses it
445 * via the IOMMU, using the AP's credentials.
446 *
447 * If this region exists (size > 0) we map it for read/write access
448 * through the IOMMU using the IPA device.
449 *
450 * Note: @addr and @size are not guaranteed to be page-aligned.
451 */
452static int ipa_imem_init(struct ipa *ipa, unsigned long addr, size_t size)
453{
454 struct device *dev = ipa->dev;
455 struct iommu_domain *domain;
456 unsigned long iova;
457 phys_addr_t phys;
458 int ret;
459
460 if (!size)
461 return 0; /* IMEM memory not used */
462
463 domain = iommu_get_domain_for_dev(dev);
464 if (!domain) {
465 dev_err(dev, "no IOMMU domain found for IMEM\n");
466 return -EINVAL;
467 }
468
469 /* Align the address down and the size up to page boundaries */
470 phys = addr & PAGE_MASK;
471 size = PAGE_ALIGN(size + addr - phys);
472 iova = phys; /* We just want a direct mapping */
473
474 ret = iommu_map(domain, iova, phys, size, IOMMU_READ | IOMMU_WRITE,
475 GFP_KERNEL);
476 if (ret)
477 return ret;
478
479 ipa->imem_iova = iova;
480 ipa->imem_size = size;
481
482 return 0;
483}
484
485static void ipa_imem_exit(struct ipa *ipa)
486{
487 struct device *dev = ipa->dev;
488 struct iommu_domain *domain;
489
490 if (!ipa->imem_size)
491 return;
492
493 domain = iommu_get_domain_for_dev(dev);
494 if (domain) {
495 size_t size;
496
497 size = iommu_unmap(domain, ipa->imem_iova, ipa->imem_size);
498 if (size != ipa->imem_size)
499 dev_warn(dev, "unmapped %zu IMEM bytes, expected %zu\n",
500 size, ipa->imem_size);
501 } else {
502 dev_err(dev, "couldn't get IPA IOMMU domain for IMEM\n");
503 }
504
505 ipa->imem_size = 0;
506 ipa->imem_iova = 0;
507}
508
509/**
510 * ipa_smem_init() - Initialize SMEM memory used by the IPA
511 * @ipa: IPA pointer
512 * @item: Item ID of SMEM memory
513 * @size: Size (bytes) of SMEM memory region
514 *
515 * SMEM is a managed block of shared DRAM, from which numbered "items"
516 * can be allocated. One item is designated for use by the IPA.
517 *
518 * The modem accesses SMEM memory directly, but the IPA accesses it
519 * via the IOMMU, using the AP's credentials.
520 *
521 * If size provided is non-zero, we allocate it and map it for
522 * access through the IOMMU.
523 *
524 * Note: @size and the item address are is not guaranteed to be page-aligned.
525 */
526static int ipa_smem_init(struct ipa *ipa, u32 item, size_t size)
527{
528 struct device *dev = ipa->dev;
529 struct iommu_domain *domain;
530 unsigned long iova;
531 phys_addr_t phys;
532 phys_addr_t addr;
533 size_t actual;
534 void *virt;
535 int ret;
536
537 if (!size)
538 return 0; /* SMEM memory not used */
539
540 /* SMEM is memory shared between the AP and another system entity
541 * (in this case, the modem). An allocation from SMEM is persistent
542 * until the AP reboots; there is no way to free an allocated SMEM
543 * region. Allocation only reserves the space; to use it you need
544 * to "get" a pointer it (this does not imply reference counting).
545 * The item might have already been allocated, in which case we
546 * use it unless the size isn't what we expect.
547 */
548 ret = qcom_smem_alloc(QCOM_SMEM_HOST_MODEM, item, size);
549 if (ret && ret != -EEXIST) {
550 dev_err(dev, "error %d allocating size %zu SMEM item %u\n",
551 ret, size, item);
552 return ret;
553 }
554
555 /* Now get the address of the SMEM memory region */
556 virt = qcom_smem_get(QCOM_SMEM_HOST_MODEM, item, &actual);
557 if (IS_ERR(virt)) {
558 ret = PTR_ERR(virt);
559 dev_err(dev, "error %d getting SMEM item %u\n", ret, item);
560 return ret;
561 }
562
563 /* In case the region was already allocated, verify the size */
564 if (ret && actual != size) {
565 dev_err(dev, "SMEM item %u has size %zu, expected %zu\n",
566 item, actual, size);
567 return -EINVAL;
568 }
569
570 domain = iommu_get_domain_for_dev(dev);
571 if (!domain) {
572 dev_err(dev, "no IOMMU domain found for SMEM\n");
573 return -EINVAL;
574 }
575
576 /* Align the address down and the size up to a page boundary */
577 addr = qcom_smem_virt_to_phys(virt);
578 phys = addr & PAGE_MASK;
579 size = PAGE_ALIGN(size + addr - phys);
580 iova = phys; /* We just want a direct mapping */
581
582 ret = iommu_map(domain, iova, phys, size, IOMMU_READ | IOMMU_WRITE,
583 GFP_KERNEL);
584 if (ret)
585 return ret;
586
587 ipa->smem_iova = iova;
588 ipa->smem_size = size;
589
590 return 0;
591}
592
593static void ipa_smem_exit(struct ipa *ipa)
594{
595 struct device *dev = ipa->dev;
596 struct iommu_domain *domain;
597
598 domain = iommu_get_domain_for_dev(dev);
599 if (domain) {
600 size_t size;
601
602 size = iommu_unmap(domain, ipa->smem_iova, ipa->smem_size);
603 if (size != ipa->smem_size)
604 dev_warn(dev, "unmapped %zu SMEM bytes, expected %zu\n",
605 size, ipa->smem_size);
606
607 } else {
608 dev_err(dev, "couldn't get IPA IOMMU domain for SMEM\n");
609 }
610
611 ipa->smem_size = 0;
612 ipa->smem_iova = 0;
613}
614
615/* Perform memory region-related initialization */
616int ipa_mem_init(struct ipa *ipa, struct platform_device *pdev,
617 const struct ipa_mem_data *mem_data)
618{
619 struct device *dev = &pdev->dev;
620 struct resource *res;
621 int ret;
622
623 /* Make sure the set of defined memory regions is valid */
624 if (!ipa_mem_valid(ipa, mem_data))
625 return -EINVAL;
626
627 ipa->mem_count = mem_data->local_count;
628 ipa->mem = mem_data->local;
629
630 /* Check the route and filter table memory regions */
631 if (!ipa_table_mem_valid(ipa, false))
632 return -EINVAL;
633 if (!ipa_table_mem_valid(ipa, true))
634 return -EINVAL;
635
636 ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64));
637 if (ret) {
638 dev_err(dev, "error %d setting DMA mask\n", ret);
639 return ret;
640 }
641
642 res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "ipa-shared");
643 if (!res) {
644 dev_err(dev,
645 "DT error getting \"ipa-shared\" memory property\n");
646 return -ENODEV;
647 }
648
649 ipa->mem_virt = memremap(res->start, resource_size(res), MEMREMAP_WC);
650 if (!ipa->mem_virt) {
651 dev_err(dev, "unable to remap \"ipa-shared\" memory\n");
652 return -ENOMEM;
653 }
654
655 ipa->mem_addr = res->start;
656 ipa->mem_size = resource_size(res);
657
658 ret = ipa_imem_init(ipa, mem_data->imem_addr, mem_data->imem_size);
659 if (ret)
660 goto err_unmap;
661
662 ret = ipa_smem_init(ipa, mem_data->smem_id, mem_data->smem_size);
663 if (ret)
664 goto err_imem_exit;
665
666 return 0;
667
668err_imem_exit:
669 ipa_imem_exit(ipa);
670err_unmap:
671 memunmap(ipa->mem_virt);
672
673 return ret;
674}
675
676/* Inverse of ipa_mem_init() */
677void ipa_mem_exit(struct ipa *ipa)
678{
679 ipa_smem_exit(ipa);
680 ipa_imem_exit(ipa);
681 memunmap(ipa->mem_virt);
682}
1// SPDX-License-Identifier: GPL-2.0
2
3/* Copyright (c) 2012-2018, The Linux Foundation. All rights reserved.
4 * Copyright (C) 2019-2021 Linaro Ltd.
5 */
6
7#include <linux/types.h>
8#include <linux/bitfield.h>
9#include <linux/bug.h>
10#include <linux/dma-mapping.h>
11#include <linux/iommu.h>
12#include <linux/io.h>
13#include <linux/soc/qcom/smem.h>
14
15#include "ipa.h"
16#include "ipa_reg.h"
17#include "ipa_data.h"
18#include "ipa_cmd.h"
19#include "ipa_mem.h"
20#include "ipa_table.h"
21#include "gsi_trans.h"
22
23/* "Canary" value placed between memory regions to detect overflow */
24#define IPA_MEM_CANARY_VAL cpu_to_le32(0xdeadbeef)
25
26/* SMEM host id representing the modem. */
27#define QCOM_SMEM_HOST_MODEM 1
28
29const struct ipa_mem *ipa_mem_find(struct ipa *ipa, enum ipa_mem_id mem_id)
30{
31 u32 i;
32
33 for (i = 0; i < ipa->mem_count; i++) {
34 const struct ipa_mem *mem = &ipa->mem[i];
35
36 if (mem->id == mem_id)
37 return mem;
38 }
39
40 return NULL;
41}
42
43/* Add an immediate command to a transaction that zeroes a memory region */
44static void
45ipa_mem_zero_region_add(struct gsi_trans *trans, enum ipa_mem_id mem_id)
46{
47 struct ipa *ipa = container_of(trans->gsi, struct ipa, gsi);
48 const struct ipa_mem *mem = ipa_mem_find(ipa, mem_id);
49 dma_addr_t addr = ipa->zero_addr;
50
51 if (!mem->size)
52 return;
53
54 ipa_cmd_dma_shared_mem_add(trans, mem->offset, mem->size, addr, true);
55}
56
57/**
58 * ipa_mem_setup() - Set up IPA AP and modem shared memory areas
59 * @ipa: IPA pointer
60 *
61 * Set up the shared memory regions in IPA local memory. This involves
62 * zero-filling memory regions, and in the case of header memory, telling
63 * the IPA where it's located.
64 *
65 * This function performs the initial setup of this memory. If the modem
66 * crashes, its regions are re-zeroed in ipa_mem_zero_modem().
67 *
68 * The AP informs the modem where its portions of memory are located
69 * in a QMI exchange that occurs at modem startup.
70 *
71 * There is no need for a matching ipa_mem_teardown() function.
72 *
73 * Return: 0 if successful, or a negative error code
74 */
75int ipa_mem_setup(struct ipa *ipa)
76{
77 dma_addr_t addr = ipa->zero_addr;
78 const struct ipa_mem *mem;
79 struct gsi_trans *trans;
80 u32 offset;
81 u16 size;
82 u32 val;
83
84 /* Get a transaction to define the header memory region and to zero
85 * the processing context and modem memory regions.
86 */
87 trans = ipa_cmd_trans_alloc(ipa, 4);
88 if (!trans) {
89 dev_err(&ipa->pdev->dev, "no transaction for memory setup\n");
90 return -EBUSY;
91 }
92
93 /* Initialize IPA-local header memory. The AP header region, if
94 * present, is contiguous with and follows the modem header region,
95 * and they are initialized together.
96 */
97 mem = ipa_mem_find(ipa, IPA_MEM_MODEM_HEADER);
98 offset = mem->offset;
99 size = mem->size;
100 mem = ipa_mem_find(ipa, IPA_MEM_AP_HEADER);
101 if (mem)
102 size += mem->size;
103
104 ipa_cmd_hdr_init_local_add(trans, offset, size, addr);
105
106 ipa_mem_zero_region_add(trans, IPA_MEM_MODEM_PROC_CTX);
107 ipa_mem_zero_region_add(trans, IPA_MEM_AP_PROC_CTX);
108 ipa_mem_zero_region_add(trans, IPA_MEM_MODEM);
109
110 gsi_trans_commit_wait(trans);
111
112 /* Tell the hardware where the processing context area is located */
113 mem = ipa_mem_find(ipa, IPA_MEM_MODEM_PROC_CTX);
114 offset = ipa->mem_offset + mem->offset;
115 val = proc_cntxt_base_addr_encoded(ipa->version, offset);
116 iowrite32(val, ipa->reg_virt + IPA_REG_LOCAL_PKT_PROC_CNTXT_OFFSET);
117
118 return 0;
119}
120
121/* Is the given memory region ID is valid for the current IPA version? */
122static bool ipa_mem_id_valid(struct ipa *ipa, enum ipa_mem_id mem_id)
123{
124 enum ipa_version version = ipa->version;
125
126 switch (mem_id) {
127 case IPA_MEM_UC_SHARED:
128 case IPA_MEM_UC_INFO:
129 case IPA_MEM_V4_FILTER_HASHED:
130 case IPA_MEM_V4_FILTER:
131 case IPA_MEM_V6_FILTER_HASHED:
132 case IPA_MEM_V6_FILTER:
133 case IPA_MEM_V4_ROUTE_HASHED:
134 case IPA_MEM_V4_ROUTE:
135 case IPA_MEM_V6_ROUTE_HASHED:
136 case IPA_MEM_V6_ROUTE:
137 case IPA_MEM_MODEM_HEADER:
138 case IPA_MEM_AP_HEADER:
139 case IPA_MEM_MODEM_PROC_CTX:
140 case IPA_MEM_AP_PROC_CTX:
141 case IPA_MEM_MODEM:
142 case IPA_MEM_UC_EVENT_RING:
143 case IPA_MEM_PDN_CONFIG:
144 case IPA_MEM_STATS_QUOTA_MODEM:
145 case IPA_MEM_STATS_QUOTA_AP:
146 case IPA_MEM_END_MARKER: /* pseudo region */
147 break;
148
149 case IPA_MEM_STATS_TETHERING:
150 case IPA_MEM_STATS_DROP:
151 if (version < IPA_VERSION_4_0)
152 return false;
153 break;
154
155 case IPA_MEM_STATS_V4_FILTER:
156 case IPA_MEM_STATS_V6_FILTER:
157 case IPA_MEM_STATS_V4_ROUTE:
158 case IPA_MEM_STATS_V6_ROUTE:
159 if (version < IPA_VERSION_4_0 || version > IPA_VERSION_4_2)
160 return false;
161 break;
162
163 case IPA_MEM_NAT_TABLE:
164 case IPA_MEM_STATS_FILTER_ROUTE:
165 if (version < IPA_VERSION_4_5)
166 return false;
167 break;
168
169 default:
170 return false;
171 }
172
173 return true;
174}
175
176/* Must the given memory region be present in the configuration? */
177static bool ipa_mem_id_required(struct ipa *ipa, enum ipa_mem_id mem_id)
178{
179 switch (mem_id) {
180 case IPA_MEM_UC_SHARED:
181 case IPA_MEM_UC_INFO:
182 case IPA_MEM_V4_FILTER_HASHED:
183 case IPA_MEM_V4_FILTER:
184 case IPA_MEM_V6_FILTER_HASHED:
185 case IPA_MEM_V6_FILTER:
186 case IPA_MEM_V4_ROUTE_HASHED:
187 case IPA_MEM_V4_ROUTE:
188 case IPA_MEM_V6_ROUTE_HASHED:
189 case IPA_MEM_V6_ROUTE:
190 case IPA_MEM_MODEM_HEADER:
191 case IPA_MEM_MODEM_PROC_CTX:
192 case IPA_MEM_AP_PROC_CTX:
193 case IPA_MEM_MODEM:
194 return true;
195
196 case IPA_MEM_PDN_CONFIG:
197 case IPA_MEM_STATS_QUOTA_MODEM:
198 case IPA_MEM_STATS_TETHERING:
199 return ipa->version >= IPA_VERSION_4_0;
200
201 default:
202 return false; /* Anything else is optional */
203 }
204}
205
206static bool ipa_mem_valid_one(struct ipa *ipa, const struct ipa_mem *mem)
207{
208 struct device *dev = &ipa->pdev->dev;
209 enum ipa_mem_id mem_id = mem->id;
210 u16 size_multiple;
211
212 /* Make sure the memory region is valid for this version of IPA */
213 if (!ipa_mem_id_valid(ipa, mem_id)) {
214 dev_err(dev, "region id %u not valid\n", mem_id);
215 return false;
216 }
217
218 if (!mem->size && !mem->canary_count) {
219 dev_err(dev, "empty memory region %u\n", mem_id);
220 return false;
221 }
222
223 /* Other than modem memory, sizes must be a multiple of 8 */
224 size_multiple = mem_id == IPA_MEM_MODEM ? 4 : 8;
225 if (mem->size % size_multiple)
226 dev_err(dev, "region %u size not a multiple of %u bytes\n",
227 mem_id, size_multiple);
228 else if (mem->offset % 8)
229 dev_err(dev, "region %u offset not 8-byte aligned\n", mem_id);
230 else if (mem->offset < mem->canary_count * sizeof(__le32))
231 dev_err(dev, "region %u offset too small for %hu canaries\n",
232 mem_id, mem->canary_count);
233 else if (mem_id == IPA_MEM_END_MARKER && mem->size)
234 dev_err(dev, "non-zero end marker region size\n");
235 else
236 return true;
237
238 return false;
239}
240
241/* Verify each defined memory region is valid. */
242static bool ipa_mem_valid(struct ipa *ipa, const struct ipa_mem_data *mem_data)
243{
244 DECLARE_BITMAP(regions, IPA_MEM_COUNT) = { };
245 struct device *dev = &ipa->pdev->dev;
246 enum ipa_mem_id mem_id;
247 u32 i;
248
249 if (mem_data->local_count > IPA_MEM_COUNT) {
250 dev_err(dev, "too many memory regions (%u > %u)\n",
251 mem_data->local_count, IPA_MEM_COUNT);
252 return false;
253 }
254
255 for (i = 0; i < mem_data->local_count; i++) {
256 const struct ipa_mem *mem = &mem_data->local[i];
257
258 if (__test_and_set_bit(mem->id, regions)) {
259 dev_err(dev, "duplicate memory region %u\n", mem->id);
260 return false;
261 }
262
263 /* Defined regions have non-zero size and/or canary count */
264 if (!ipa_mem_valid_one(ipa, mem))
265 return false;
266 }
267
268 /* Now see if any required regions are not defined */
269 for (mem_id = find_first_zero_bit(regions, IPA_MEM_COUNT);
270 mem_id < IPA_MEM_COUNT;
271 mem_id = find_next_zero_bit(regions, IPA_MEM_COUNT, mem_id + 1)) {
272 if (ipa_mem_id_required(ipa, mem_id))
273 dev_err(dev, "required memory region %u missing\n",
274 mem_id);
275 }
276
277 return true;
278}
279
280/* Do all memory regions fit within the IPA local memory? */
281static bool ipa_mem_size_valid(struct ipa *ipa)
282{
283 struct device *dev = &ipa->pdev->dev;
284 u32 limit = ipa->mem_size;
285 u32 i;
286
287 for (i = 0; i < ipa->mem_count; i++) {
288 const struct ipa_mem *mem = &ipa->mem[i];
289
290 if (mem->offset + mem->size <= limit)
291 continue;
292
293 dev_err(dev, "region %u ends beyond memory limit (0x%08x)\n",
294 mem->id, limit);
295
296 return false;
297 }
298
299 return true;
300}
301
302/**
303 * ipa_mem_config() - Configure IPA shared memory
304 * @ipa: IPA pointer
305 *
306 * Return: 0 if successful, or a negative error code
307 */
308int ipa_mem_config(struct ipa *ipa)
309{
310 struct device *dev = &ipa->pdev->dev;
311 const struct ipa_mem *mem;
312 dma_addr_t addr;
313 u32 mem_size;
314 void *virt;
315 u32 val;
316 u32 i;
317
318 /* Check the advertised location and size of the shared memory area */
319 val = ioread32(ipa->reg_virt + IPA_REG_SHARED_MEM_SIZE_OFFSET);
320
321 /* The fields in the register are in 8 byte units */
322 ipa->mem_offset = 8 * u32_get_bits(val, SHARED_MEM_BADDR_FMASK);
323 /* Make sure the end is within the region's mapped space */
324 mem_size = 8 * u32_get_bits(val, SHARED_MEM_SIZE_FMASK);
325
326 /* If the sizes don't match, issue a warning */
327 if (ipa->mem_offset + mem_size < ipa->mem_size) {
328 dev_warn(dev, "limiting IPA memory size to 0x%08x\n",
329 mem_size);
330 ipa->mem_size = mem_size;
331 } else if (ipa->mem_offset + mem_size > ipa->mem_size) {
332 dev_dbg(dev, "ignoring larger reported memory size: 0x%08x\n",
333 mem_size);
334 }
335
336 /* We know our memory size; make sure regions are all in range */
337 if (!ipa_mem_size_valid(ipa))
338 return -EINVAL;
339
340 /* Prealloc DMA memory for zeroing regions */
341 virt = dma_alloc_coherent(dev, IPA_MEM_MAX, &addr, GFP_KERNEL);
342 if (!virt)
343 return -ENOMEM;
344 ipa->zero_addr = addr;
345 ipa->zero_virt = virt;
346 ipa->zero_size = IPA_MEM_MAX;
347
348 /* For each defined region, write "canary" values in the
349 * space prior to the region's base address if indicated.
350 */
351 for (i = 0; i < ipa->mem_count; i++) {
352 u16 canary_count = ipa->mem[i].canary_count;
353 __le32 *canary;
354
355 if (!canary_count)
356 continue;
357
358 /* Write canary values in the space before the region */
359 canary = ipa->mem_virt + ipa->mem_offset + ipa->mem[i].offset;
360 do
361 *--canary = IPA_MEM_CANARY_VAL;
362 while (--canary_count);
363 }
364
365 /* Make sure filter and route table memory regions are valid */
366 if (!ipa_table_valid(ipa))
367 goto err_dma_free;
368
369 /* Validate memory-related properties relevant to immediate commands */
370 if (!ipa_cmd_data_valid(ipa))
371 goto err_dma_free;
372
373 /* Verify the microcontroller ring alignment (if defined) */
374 mem = ipa_mem_find(ipa, IPA_MEM_UC_EVENT_RING);
375 if (mem && mem->offset % 1024) {
376 dev_err(dev, "microcontroller ring not 1024-byte aligned\n");
377 goto err_dma_free;
378 }
379
380 return 0;
381
382err_dma_free:
383 dma_free_coherent(dev, IPA_MEM_MAX, ipa->zero_virt, ipa->zero_addr);
384
385 return -EINVAL;
386}
387
388/* Inverse of ipa_mem_config() */
389void ipa_mem_deconfig(struct ipa *ipa)
390{
391 struct device *dev = &ipa->pdev->dev;
392
393 dma_free_coherent(dev, ipa->zero_size, ipa->zero_virt, ipa->zero_addr);
394 ipa->zero_size = 0;
395 ipa->zero_virt = NULL;
396 ipa->zero_addr = 0;
397}
398
399/**
400 * ipa_mem_zero_modem() - Zero IPA-local memory regions owned by the modem
401 * @ipa: IPA pointer
402 *
403 * Zero regions of IPA-local memory used by the modem. These are configured
404 * (and initially zeroed) by ipa_mem_setup(), but if the modem crashes and
405 * restarts via SSR we need to re-initialize them. A QMI message tells the
406 * modem where to find regions of IPA local memory it needs to know about
407 * (these included).
408 */
409int ipa_mem_zero_modem(struct ipa *ipa)
410{
411 struct gsi_trans *trans;
412
413 /* Get a transaction to zero the modem memory, modem header,
414 * and modem processing context regions.
415 */
416 trans = ipa_cmd_trans_alloc(ipa, 3);
417 if (!trans) {
418 dev_err(&ipa->pdev->dev,
419 "no transaction to zero modem memory\n");
420 return -EBUSY;
421 }
422
423 ipa_mem_zero_region_add(trans, IPA_MEM_MODEM_HEADER);
424 ipa_mem_zero_region_add(trans, IPA_MEM_MODEM_PROC_CTX);
425 ipa_mem_zero_region_add(trans, IPA_MEM_MODEM);
426
427 gsi_trans_commit_wait(trans);
428
429 return 0;
430}
431
432/**
433 * ipa_imem_init() - Initialize IMEM memory used by the IPA
434 * @ipa: IPA pointer
435 * @addr: Physical address of the IPA region in IMEM
436 * @size: Size (bytes) of the IPA region in IMEM
437 *
438 * IMEM is a block of shared memory separate from system DRAM, and
439 * a portion of this memory is available for the IPA to use. The
440 * modem accesses this memory directly, but the IPA accesses it
441 * via the IOMMU, using the AP's credentials.
442 *
443 * If this region exists (size > 0) we map it for read/write access
444 * through the IOMMU using the IPA device.
445 *
446 * Note: @addr and @size are not guaranteed to be page-aligned.
447 */
448static int ipa_imem_init(struct ipa *ipa, unsigned long addr, size_t size)
449{
450 struct device *dev = &ipa->pdev->dev;
451 struct iommu_domain *domain;
452 unsigned long iova;
453 phys_addr_t phys;
454 int ret;
455
456 if (!size)
457 return 0; /* IMEM memory not used */
458
459 domain = iommu_get_domain_for_dev(dev);
460 if (!domain) {
461 dev_err(dev, "no IOMMU domain found for IMEM\n");
462 return -EINVAL;
463 }
464
465 /* Align the address down and the size up to page boundaries */
466 phys = addr & PAGE_MASK;
467 size = PAGE_ALIGN(size + addr - phys);
468 iova = phys; /* We just want a direct mapping */
469
470 ret = iommu_map(domain, iova, phys, size, IOMMU_READ | IOMMU_WRITE);
471 if (ret)
472 return ret;
473
474 ipa->imem_iova = iova;
475 ipa->imem_size = size;
476
477 return 0;
478}
479
480static void ipa_imem_exit(struct ipa *ipa)
481{
482 struct iommu_domain *domain;
483 struct device *dev;
484
485 if (!ipa->imem_size)
486 return;
487
488 dev = &ipa->pdev->dev;
489 domain = iommu_get_domain_for_dev(dev);
490 if (domain) {
491 size_t size;
492
493 size = iommu_unmap(domain, ipa->imem_iova, ipa->imem_size);
494 if (size != ipa->imem_size)
495 dev_warn(dev, "unmapped %zu IMEM bytes, expected %zu\n",
496 size, ipa->imem_size);
497 } else {
498 dev_err(dev, "couldn't get IPA IOMMU domain for IMEM\n");
499 }
500
501 ipa->imem_size = 0;
502 ipa->imem_iova = 0;
503}
504
505/**
506 * ipa_smem_init() - Initialize SMEM memory used by the IPA
507 * @ipa: IPA pointer
508 * @item: Item ID of SMEM memory
509 * @size: Size (bytes) of SMEM memory region
510 *
511 * SMEM is a managed block of shared DRAM, from which numbered "items"
512 * can be allocated. One item is designated for use by the IPA.
513 *
514 * The modem accesses SMEM memory directly, but the IPA accesses it
515 * via the IOMMU, using the AP's credentials.
516 *
517 * If size provided is non-zero, we allocate it and map it for
518 * access through the IOMMU.
519 *
520 * Note: @size and the item address are is not guaranteed to be page-aligned.
521 */
522static int ipa_smem_init(struct ipa *ipa, u32 item, size_t size)
523{
524 struct device *dev = &ipa->pdev->dev;
525 struct iommu_domain *domain;
526 unsigned long iova;
527 phys_addr_t phys;
528 phys_addr_t addr;
529 size_t actual;
530 void *virt;
531 int ret;
532
533 if (!size)
534 return 0; /* SMEM memory not used */
535
536 /* SMEM is memory shared between the AP and another system entity
537 * (in this case, the modem). An allocation from SMEM is persistent
538 * until the AP reboots; there is no way to free an allocated SMEM
539 * region. Allocation only reserves the space; to use it you need
540 * to "get" a pointer it (this does not imply reference counting).
541 * The item might have already been allocated, in which case we
542 * use it unless the size isn't what we expect.
543 */
544 ret = qcom_smem_alloc(QCOM_SMEM_HOST_MODEM, item, size);
545 if (ret && ret != -EEXIST) {
546 dev_err(dev, "error %d allocating size %zu SMEM item %u\n",
547 ret, size, item);
548 return ret;
549 }
550
551 /* Now get the address of the SMEM memory region */
552 virt = qcom_smem_get(QCOM_SMEM_HOST_MODEM, item, &actual);
553 if (IS_ERR(virt)) {
554 ret = PTR_ERR(virt);
555 dev_err(dev, "error %d getting SMEM item %u\n", ret, item);
556 return ret;
557 }
558
559 /* In case the region was already allocated, verify the size */
560 if (ret && actual != size) {
561 dev_err(dev, "SMEM item %u has size %zu, expected %zu\n",
562 item, actual, size);
563 return -EINVAL;
564 }
565
566 domain = iommu_get_domain_for_dev(dev);
567 if (!domain) {
568 dev_err(dev, "no IOMMU domain found for SMEM\n");
569 return -EINVAL;
570 }
571
572 /* Align the address down and the size up to a page boundary */
573 addr = qcom_smem_virt_to_phys(virt) & PAGE_MASK;
574 phys = addr & PAGE_MASK;
575 size = PAGE_ALIGN(size + addr - phys);
576 iova = phys; /* We just want a direct mapping */
577
578 ret = iommu_map(domain, iova, phys, size, IOMMU_READ | IOMMU_WRITE);
579 if (ret)
580 return ret;
581
582 ipa->smem_iova = iova;
583 ipa->smem_size = size;
584
585 return 0;
586}
587
588static void ipa_smem_exit(struct ipa *ipa)
589{
590 struct device *dev = &ipa->pdev->dev;
591 struct iommu_domain *domain;
592
593 domain = iommu_get_domain_for_dev(dev);
594 if (domain) {
595 size_t size;
596
597 size = iommu_unmap(domain, ipa->smem_iova, ipa->smem_size);
598 if (size != ipa->smem_size)
599 dev_warn(dev, "unmapped %zu SMEM bytes, expected %zu\n",
600 size, ipa->smem_size);
601
602 } else {
603 dev_err(dev, "couldn't get IPA IOMMU domain for SMEM\n");
604 }
605
606 ipa->smem_size = 0;
607 ipa->smem_iova = 0;
608}
609
610/* Perform memory region-related initialization */
611int ipa_mem_init(struct ipa *ipa, const struct ipa_mem_data *mem_data)
612{
613 struct device *dev = &ipa->pdev->dev;
614 struct resource *res;
615 int ret;
616
617 /* Make sure the set of defined memory regions is valid */
618 if (!ipa_mem_valid(ipa, mem_data))
619 return -EINVAL;
620
621 ipa->mem_count = mem_data->local_count;
622 ipa->mem = mem_data->local;
623
624 ret = dma_set_mask_and_coherent(&ipa->pdev->dev, DMA_BIT_MASK(64));
625 if (ret) {
626 dev_err(dev, "error %d setting DMA mask\n", ret);
627 return ret;
628 }
629
630 res = platform_get_resource_byname(ipa->pdev, IORESOURCE_MEM,
631 "ipa-shared");
632 if (!res) {
633 dev_err(dev,
634 "DT error getting \"ipa-shared\" memory property\n");
635 return -ENODEV;
636 }
637
638 ipa->mem_virt = memremap(res->start, resource_size(res), MEMREMAP_WC);
639 if (!ipa->mem_virt) {
640 dev_err(dev, "unable to remap \"ipa-shared\" memory\n");
641 return -ENOMEM;
642 }
643
644 ipa->mem_addr = res->start;
645 ipa->mem_size = resource_size(res);
646
647 ret = ipa_imem_init(ipa, mem_data->imem_addr, mem_data->imem_size);
648 if (ret)
649 goto err_unmap;
650
651 ret = ipa_smem_init(ipa, mem_data->smem_id, mem_data->smem_size);
652 if (ret)
653 goto err_imem_exit;
654
655 return 0;
656
657err_imem_exit:
658 ipa_imem_exit(ipa);
659err_unmap:
660 memunmap(ipa->mem_virt);
661
662 return ret;
663}
664
665/* Inverse of ipa_mem_init() */
666void ipa_mem_exit(struct ipa *ipa)
667{
668 ipa_smem_exit(ipa);
669 ipa_imem_exit(ipa);
670 memunmap(ipa->mem_virt);
671}