1// SPDX-License-Identifier: GPL-2.0
  2
  3/* Copyright (c) 2012-2018, The Linux Foundation. All rights reserved.
  4 * Copyright (C) 2019-2021 Linaro Ltd.
  5 */
  6
  7#include <linux/types.h>
  8#include <linux/bitfield.h>
  9#include <linux/bug.h>
 10#include <linux/dma-mapping.h>
 11#include <linux/iommu.h>
 12#include <linux/io.h>
 13#include <linux/soc/qcom/smem.h>
 14
 15#include "ipa.h"
 16#include "ipa_reg.h"
 17#include "ipa_data.h"
 18#include "ipa_cmd.h"
 19#include "ipa_mem.h"
 20#include "ipa_table.h"
 21#include "gsi_trans.h"
 22
 23/* "Canary" value placed between memory regions to detect overflow */
 24#define IPA_MEM_CANARY_VAL		cpu_to_le32(0xdeadbeef)
 25
 26/* SMEM host id representing the modem. */
 27#define QCOM_SMEM_HOST_MODEM	1
 28
 29const struct ipa_mem *ipa_mem_find(struct ipa *ipa, enum ipa_mem_id mem_id)
 30{
 31	u32 i;
 32
 33	for (i = 0; i < ipa->mem_count; i++) {
 34		const struct ipa_mem *mem = &ipa->mem[i];
 35
 36		if (mem->id == mem_id)
 37			return mem;
 38	}
 39
 40	return NULL;
 41}
 42
 43/* Add an immediate command to a transaction that zeroes a memory region */
 44static void
 45ipa_mem_zero_region_add(struct gsi_trans *trans, enum ipa_mem_id mem_id)
 46{
 47	struct ipa *ipa = container_of(trans->gsi, struct ipa, gsi);
 48	const struct ipa_mem *mem = ipa_mem_find(ipa, mem_id);
 49	dma_addr_t addr = ipa->zero_addr;
 50
 51	if (!mem->size)
 52		return;
 53
 54	ipa_cmd_dma_shared_mem_add(trans, mem->offset, mem->size, addr, true);
 55}
 56
 57/**
 58 * ipa_mem_setup() - Set up IPA AP and modem shared memory areas
 59 * @ipa:	IPA pointer
 60 *
 61 * Set up the shared memory regions in IPA local memory.  This involves
 62 * zero-filling memory regions, and in the case of header memory, telling
 63 * the IPA where it's located.
 64 *
 65 * This function performs the initial setup of this memory.  If the modem
 66 * crashes, its regions are re-zeroed in ipa_mem_zero_modem().
 67 *
 68 * The AP informs the modem where its portions of memory are located
 69 * in a QMI exchange that occurs at modem startup.
 70 *
 71 * There is no need for a matching ipa_mem_teardown() function.
 72 *
 73 * Return:	0 if successful, or a negative error code
 74 */
 75int ipa_mem_setup(struct ipa *ipa)
 76{
 77	dma_addr_t addr = ipa->zero_addr;
 78	const struct ipa_mem *mem;
 79	struct gsi_trans *trans;
 80	u32 offset;
 81	u16 size;
 82	u32 val;
 83
 84	/* Get a transaction to define the header memory region and to zero
 85	 * the processing context and modem memory regions.
 86	 */
 87	trans = ipa_cmd_trans_alloc(ipa, 4);
 88	if (!trans) {
 89		dev_err(&ipa->pdev->dev, "no transaction for memory setup\n");
 90		return -EBUSY;
 91	}
 92
 93	/* Initialize IPA-local header memory.  The AP header region, if
 94	 * present, is contiguous with and follows the modem header region,
 95	 * and they are initialized together.
 96	 */
 97	mem = ipa_mem_find(ipa, IPA_MEM_MODEM_HEADER);
 98	offset = mem->offset;
 99	size = mem->size;
100	mem = ipa_mem_find(ipa, IPA_MEM_AP_HEADER);
101	if (mem)
102		size += mem->size;
103
104	ipa_cmd_hdr_init_local_add(trans, offset, size, addr);
105
106	ipa_mem_zero_region_add(trans, IPA_MEM_MODEM_PROC_CTX);
107	ipa_mem_zero_region_add(trans, IPA_MEM_AP_PROC_CTX);
108	ipa_mem_zero_region_add(trans, IPA_MEM_MODEM);
109
110	gsi_trans_commit_wait(trans);
111
112	/* Tell the hardware where the processing context area is located */
113	mem = ipa_mem_find(ipa, IPA_MEM_MODEM_PROC_CTX);
114	offset = ipa->mem_offset + mem->offset;
115	val = proc_cntxt_base_addr_encoded(ipa->version, offset);
116	iowrite32(val, ipa->reg_virt + IPA_REG_LOCAL_PKT_PROC_CNTXT_OFFSET);
117
118	return 0;
119}
120
121/* Is the given memory region ID is valid for the current IPA version? */
122static bool ipa_mem_id_valid(struct ipa *ipa, enum ipa_mem_id mem_id)
123{
124	enum ipa_version version = ipa->version;
125
126	switch (mem_id) {
127	case IPA_MEM_UC_SHARED:
128	case IPA_MEM_UC_INFO:
129	case IPA_MEM_V4_FILTER_HASHED:
130	case IPA_MEM_V4_FILTER:
131	case IPA_MEM_V6_FILTER_HASHED:
132	case IPA_MEM_V6_FILTER:
133	case IPA_MEM_V4_ROUTE_HASHED:
134	case IPA_MEM_V4_ROUTE:
135	case IPA_MEM_V6_ROUTE_HASHED:
136	case IPA_MEM_V6_ROUTE:
137	case IPA_MEM_MODEM_HEADER:
138	case IPA_MEM_AP_HEADER:
139	case IPA_MEM_MODEM_PROC_CTX:
140	case IPA_MEM_AP_PROC_CTX:
141	case IPA_MEM_MODEM:
142	case IPA_MEM_UC_EVENT_RING:
143	case IPA_MEM_PDN_CONFIG:
144	case IPA_MEM_STATS_QUOTA_MODEM:
145	case IPA_MEM_STATS_QUOTA_AP:
146	case IPA_MEM_END_MARKER:	/* pseudo region */
147		break;
148
149	case IPA_MEM_STATS_TETHERING:
150	case IPA_MEM_STATS_DROP:
151		if (version < IPA_VERSION_4_0)
152			return false;
153		break;
154
155	case IPA_MEM_STATS_V4_FILTER:
156	case IPA_MEM_STATS_V6_FILTER:
157	case IPA_MEM_STATS_V4_ROUTE:
158	case IPA_MEM_STATS_V6_ROUTE:
159		if (version < IPA_VERSION_4_0 || version > IPA_VERSION_4_2)
160			return false;
161		break;
162
163	case IPA_MEM_NAT_TABLE:
164	case IPA_MEM_STATS_FILTER_ROUTE:
165		if (version < IPA_VERSION_4_5)
166			return false;
167		break;
168
169	default:
170		return false;
171	}
172
173	return true;
174}
175
176/* Must the given memory region be present in the configuration? */
177static bool ipa_mem_id_required(struct ipa *ipa, enum ipa_mem_id mem_id)
178{
179	switch (mem_id) {
180	case IPA_MEM_UC_SHARED:
181	case IPA_MEM_UC_INFO:
182	case IPA_MEM_V4_FILTER_HASHED:
183	case IPA_MEM_V4_FILTER:
184	case IPA_MEM_V6_FILTER_HASHED:
185	case IPA_MEM_V6_FILTER:
186	case IPA_MEM_V4_ROUTE_HASHED:
187	case IPA_MEM_V4_ROUTE:
188	case IPA_MEM_V6_ROUTE_HASHED:
189	case IPA_MEM_V6_ROUTE:
190	case IPA_MEM_MODEM_HEADER:
191	case IPA_MEM_MODEM_PROC_CTX:
192	case IPA_MEM_AP_PROC_CTX:
193	case IPA_MEM_MODEM:
194		return true;
195
196	case IPA_MEM_PDN_CONFIG:
197	case IPA_MEM_STATS_QUOTA_MODEM:
198	case IPA_MEM_STATS_TETHERING:
199		return ipa->version >= IPA_VERSION_4_0;
200
201	default:
202		return false;		/* Anything else is optional */
203	}
204}
205
206static bool ipa_mem_valid_one(struct ipa *ipa, const struct ipa_mem *mem)
207{
208	struct device *dev = &ipa->pdev->dev;
209	enum ipa_mem_id mem_id = mem->id;
210	u16 size_multiple;
211
212	/* Make sure the memory region is valid for this version of IPA */
213	if (!ipa_mem_id_valid(ipa, mem_id)) {
214		dev_err(dev, "region id %u not valid\n", mem_id);
215		return false;
216	}
217
218	if (!mem->size && !mem->canary_count) {
219		dev_err(dev, "empty memory region %u\n", mem_id);
220		return false;
221	}
222
223	/* Other than modem memory, sizes must be a multiple of 8 */
224	size_multiple = mem_id == IPA_MEM_MODEM ? 4 : 8;
225	if (mem->size % size_multiple)
226		dev_err(dev, "region %u size not a multiple of %u bytes\n",
227			mem_id, size_multiple);
228	else if (mem->offset % 8)
229		dev_err(dev, "region %u offset not 8-byte aligned\n", mem_id);
230	else if (mem->offset < mem->canary_count * sizeof(__le32))
231		dev_err(dev, "region %u offset too small for %hu canaries\n",
232			mem_id, mem->canary_count);
233	else if (mem_id == IPA_MEM_END_MARKER && mem->size)
234		dev_err(dev, "non-zero end marker region size\n");
235	else
236		return true;
237
238	return false;
239}
240
241/* Verify each defined memory region is valid. */
242static bool ipa_mem_valid(struct ipa *ipa, const struct ipa_mem_data *mem_data)
243{
244	DECLARE_BITMAP(regions, IPA_MEM_COUNT) = { };
245	struct device *dev = &ipa->pdev->dev;
246	enum ipa_mem_id mem_id;
247	u32 i;
248
249	if (mem_data->local_count > IPA_MEM_COUNT) {
250		dev_err(dev, "too many memory regions (%u > %u)\n",
251			mem_data->local_count, IPA_MEM_COUNT);
252		return false;
253	}
254
255	for (i = 0; i < mem_data->local_count; i++) {
256		const struct ipa_mem *mem = &mem_data->local[i];
257
258		if (__test_and_set_bit(mem->id, regions)) {
259			dev_err(dev, "duplicate memory region %u\n", mem->id);
260			return false;
261		}
262
263		/* Defined regions have non-zero size and/or canary count */
264		if (!ipa_mem_valid_one(ipa, mem))
265			return false;
266	}
267
268	/* Now see if any required regions are not defined */
269	for (mem_id = find_first_zero_bit(regions, IPA_MEM_COUNT);
270	     mem_id < IPA_MEM_COUNT;
271	     mem_id = find_next_zero_bit(regions, IPA_MEM_COUNT, mem_id + 1)) {
272		if (ipa_mem_id_required(ipa, mem_id))
273			dev_err(dev, "required memory region %u missing\n",
274				mem_id);
275	}
276
277	return true;
278}
279
280/* Do all memory regions fit within the IPA local memory? */
281static bool ipa_mem_size_valid(struct ipa *ipa)
282{
283	struct device *dev = &ipa->pdev->dev;
284	u32 limit = ipa->mem_size;
285	u32 i;
286
287	for (i = 0; i < ipa->mem_count; i++) {
288		const struct ipa_mem *mem = &ipa->mem[i];
289
290		if (mem->offset + mem->size <= limit)
291			continue;
292
293		dev_err(dev, "region %u ends beyond memory limit (0x%08x)\n",
294			mem->id, limit);
295
296		return false;
297	}
298
299	return true;
300}
301
302/**
303 * ipa_mem_config() - Configure IPA shared memory
304 * @ipa:	IPA pointer
305 *
306 * Return:	0 if successful, or a negative error code
307 */
308int ipa_mem_config(struct ipa *ipa)
309{
310	struct device *dev = &ipa->pdev->dev;
311	const struct ipa_mem *mem;
312	dma_addr_t addr;
313	u32 mem_size;
314	void *virt;
315	u32 val;
316	u32 i;
317
318	/* Check the advertised location and size of the shared memory area */
319	val = ioread32(ipa->reg_virt + IPA_REG_SHARED_MEM_SIZE_OFFSET);
320
321	/* The fields in the register are in 8 byte units */
322	ipa->mem_offset = 8 * u32_get_bits(val, SHARED_MEM_BADDR_FMASK);
323	/* Make sure the end is within the region's mapped space */
324	mem_size = 8 * u32_get_bits(val, SHARED_MEM_SIZE_FMASK);
325
326	/* If the sizes don't match, issue a warning */
327	if (ipa->mem_offset + mem_size < ipa->mem_size) {
328		dev_warn(dev, "limiting IPA memory size to 0x%08x\n",
329			 mem_size);
330		ipa->mem_size = mem_size;
331	} else if (ipa->mem_offset + mem_size > ipa->mem_size) {
332		dev_dbg(dev, "ignoring larger reported memory size: 0x%08x\n",
333			mem_size);
334	}
335
336	/* We know our memory size; make sure regions are all in range */
337	if (!ipa_mem_size_valid(ipa))
338		return -EINVAL;
339
340	/* Prealloc DMA memory for zeroing regions */
341	virt = dma_alloc_coherent(dev, IPA_MEM_MAX, &addr, GFP_KERNEL);
342	if (!virt)
343		return -ENOMEM;
344	ipa->zero_addr = addr;
345	ipa->zero_virt = virt;
346	ipa->zero_size = IPA_MEM_MAX;
347
348	/* For each defined region, write "canary" values in the
349	 * space prior to the region's base address if indicated.
350	 */
351	for (i = 0; i < ipa->mem_count; i++) {
352		u16 canary_count = ipa->mem[i].canary_count;
353		__le32 *canary;
354
355		if (!canary_count)
356			continue;
357
358		/* Write canary values in the space before the region */
359		canary = ipa->mem_virt + ipa->mem_offset + ipa->mem[i].offset;
360		do
361			*--canary = IPA_MEM_CANARY_VAL;
362		while (--canary_count);
363	}
364
365	/* Make sure filter and route table memory regions are valid */
366	if (!ipa_table_valid(ipa))
367		goto err_dma_free;
368
369	/* Validate memory-related properties relevant to immediate commands */
370	if (!ipa_cmd_data_valid(ipa))
371		goto err_dma_free;
372
373	/* Verify the microcontroller ring alignment (if defined) */
374	mem = ipa_mem_find(ipa, IPA_MEM_UC_EVENT_RING);
375	if (mem && mem->offset % 1024) {
376		dev_err(dev, "microcontroller ring not 1024-byte aligned\n");
377		goto err_dma_free;
378	}
379
380	return 0;
381
382err_dma_free:
383	dma_free_coherent(dev, IPA_MEM_MAX, ipa->zero_virt, ipa->zero_addr);
384
385	return -EINVAL;
386}
387
388/* Inverse of ipa_mem_config() */
389void ipa_mem_deconfig(struct ipa *ipa)
390{
391	struct device *dev = &ipa->pdev->dev;
392
393	dma_free_coherent(dev, ipa->zero_size, ipa->zero_virt, ipa->zero_addr);
394	ipa->zero_size = 0;
395	ipa->zero_virt = NULL;
396	ipa->zero_addr = 0;
397}
398
399/**
400 * ipa_mem_zero_modem() - Zero IPA-local memory regions owned by the modem
401 * @ipa:	IPA pointer
402 *
403 * Zero regions of IPA-local memory used by the modem.  These are configured
404 * (and initially zeroed) by ipa_mem_setup(), but if the modem crashes and
405 * restarts via SSR we need to re-initialize them.  A QMI message tells the
406 * modem where to find regions of IPA local memory it needs to know about
407 * (these included).
408 */
409int ipa_mem_zero_modem(struct ipa *ipa)
410{
411	struct gsi_trans *trans;
412
413	/* Get a transaction to zero the modem memory, modem header,
414	 * and modem processing context regions.
415	 */
416	trans = ipa_cmd_trans_alloc(ipa, 3);
417	if (!trans) {
418		dev_err(&ipa->pdev->dev,
419			"no transaction to zero modem memory\n");
420		return -EBUSY;
421	}
422
423	ipa_mem_zero_region_add(trans, IPA_MEM_MODEM_HEADER);
424	ipa_mem_zero_region_add(trans, IPA_MEM_MODEM_PROC_CTX);
425	ipa_mem_zero_region_add(trans, IPA_MEM_MODEM);
426
427	gsi_trans_commit_wait(trans);
428
429	return 0;
430}
431
432/**
433 * ipa_imem_init() - Initialize IMEM memory used by the IPA
434 * @ipa:	IPA pointer
435 * @addr:	Physical address of the IPA region in IMEM
436 * @size:	Size (bytes) of the IPA region in IMEM
437 *
438 * IMEM is a block of shared memory separate from system DRAM, and
439 * a portion of this memory is available for the IPA to use.  The
440 * modem accesses this memory directly, but the IPA accesses it
441 * via the IOMMU, using the AP's credentials.
442 *
443 * If this region exists (size > 0) we map it for read/write access
444 * through the IOMMU using the IPA device.
445 *
446 * Note: @addr and @size are not guaranteed to be page-aligned.
447 */
448static int ipa_imem_init(struct ipa *ipa, unsigned long addr, size_t size)
449{
450	struct device *dev = &ipa->pdev->dev;
451	struct iommu_domain *domain;
452	unsigned long iova;
453	phys_addr_t phys;
454	int ret;
455
456	if (!size)
457		return 0;	/* IMEM memory not used */
458
459	domain = iommu_get_domain_for_dev(dev);
460	if (!domain) {
461		dev_err(dev, "no IOMMU domain found for IMEM\n");
462		return -EINVAL;
463	}
464
465	/* Align the address down and the size up to page boundaries */
466	phys = addr & PAGE_MASK;
467	size = PAGE_ALIGN(size + addr - phys);
468	iova = phys;	/* We just want a direct mapping */
469
470	ret = iommu_map(domain, iova, phys, size, IOMMU_READ | IOMMU_WRITE);
471	if (ret)
472		return ret;
473
474	ipa->imem_iova = iova;
475	ipa->imem_size = size;
476
477	return 0;
478}
479
480static void ipa_imem_exit(struct ipa *ipa)
481{
482	struct iommu_domain *domain;
483	struct device *dev;
484
485	if (!ipa->imem_size)
486		return;
487
488	dev = &ipa->pdev->dev;
489	domain = iommu_get_domain_for_dev(dev);
490	if (domain) {
491		size_t size;
492
493		size = iommu_unmap(domain, ipa->imem_iova, ipa->imem_size);
494		if (size != ipa->imem_size)
495			dev_warn(dev, "unmapped %zu IMEM bytes, expected %zu\n",
496				 size, ipa->imem_size);
497	} else {
498		dev_err(dev, "couldn't get IPA IOMMU domain for IMEM\n");
499	}
500
501	ipa->imem_size = 0;
502	ipa->imem_iova = 0;
503}
504
505/**
506 * ipa_smem_init() - Initialize SMEM memory used by the IPA
507 * @ipa:	IPA pointer
508 * @item:	Item ID of SMEM memory
509 * @size:	Size (bytes) of SMEM memory region
510 *
511 * SMEM is a managed block of shared DRAM, from which numbered "items"
512 * can be allocated.  One item is designated for use by the IPA.
513 *
514 * The modem accesses SMEM memory directly, but the IPA accesses it
515 * via the IOMMU, using the AP's credentials.
516 *
517 * If size provided is non-zero, we allocate it and map it for
518 * access through the IOMMU.
519 *
520 * Note: @size and the item address are is not guaranteed to be page-aligned.
521 */
522static int ipa_smem_init(struct ipa *ipa, u32 item, size_t size)
523{
524	struct device *dev = &ipa->pdev->dev;
525	struct iommu_domain *domain;
526	unsigned long iova;
527	phys_addr_t phys;
528	phys_addr_t addr;
529	size_t actual;
530	void *virt;
531	int ret;
532
533	if (!size)
534		return 0;	/* SMEM memory not used */
535
536	/* SMEM is memory shared between the AP and another system entity
537	 * (in this case, the modem).  An allocation from SMEM is persistent
538	 * until the AP reboots; there is no way to free an allocated SMEM
539	 * region.  Allocation only reserves the space; to use it you need
540	 * to "get" a pointer it (this does not imply reference counting).
541	 * The item might have already been allocated, in which case we
542	 * use it unless the size isn't what we expect.
543	 */
544	ret = qcom_smem_alloc(QCOM_SMEM_HOST_MODEM, item, size);
545	if (ret && ret != -EEXIST) {
546		dev_err(dev, "error %d allocating size %zu SMEM item %u\n",
547			ret, size, item);
548		return ret;
549	}
550
551	/* Now get the address of the SMEM memory region */
552	virt = qcom_smem_get(QCOM_SMEM_HOST_MODEM, item, &actual);
553	if (IS_ERR(virt)) {
554		ret = PTR_ERR(virt);
555		dev_err(dev, "error %d getting SMEM item %u\n", ret, item);
556		return ret;
557	}
558
559	/* In case the region was already allocated, verify the size */
560	if (ret && actual != size) {
561		dev_err(dev, "SMEM item %u has size %zu, expected %zu\n",
562			item, actual, size);
563		return -EINVAL;
564	}
565
566	domain = iommu_get_domain_for_dev(dev);
567	if (!domain) {
568		dev_err(dev, "no IOMMU domain found for SMEM\n");
569		return -EINVAL;
570	}
571
572	/* Align the address down and the size up to a page boundary */
573	addr = qcom_smem_virt_to_phys(virt) & PAGE_MASK;
574	phys = addr & PAGE_MASK;
575	size = PAGE_ALIGN(size + addr - phys);
576	iova = phys;	/* We just want a direct mapping */
577
578	ret = iommu_map(domain, iova, phys, size, IOMMU_READ | IOMMU_WRITE);
579	if (ret)
580		return ret;
581
582	ipa->smem_iova = iova;
583	ipa->smem_size = size;
584
585	return 0;
586}
587
588static void ipa_smem_exit(struct ipa *ipa)
589{
590	struct device *dev = &ipa->pdev->dev;
591	struct iommu_domain *domain;
592
593	domain = iommu_get_domain_for_dev(dev);
594	if (domain) {
595		size_t size;
596
597		size = iommu_unmap(domain, ipa->smem_iova, ipa->smem_size);
598		if (size != ipa->smem_size)
599			dev_warn(dev, "unmapped %zu SMEM bytes, expected %zu\n",
600				 size, ipa->smem_size);
601
602	} else {
603		dev_err(dev, "couldn't get IPA IOMMU domain for SMEM\n");
604	}
605
606	ipa->smem_size = 0;
607	ipa->smem_iova = 0;
608}
609
610/* Perform memory region-related initialization */
611int ipa_mem_init(struct ipa *ipa, const struct ipa_mem_data *mem_data)
612{
613	struct device *dev = &ipa->pdev->dev;
614	struct resource *res;
615	int ret;
616
617	/* Make sure the set of defined memory regions is valid */
618	if (!ipa_mem_valid(ipa, mem_data))
619		return -EINVAL;
620
621	ipa->mem_count = mem_data->local_count;
622	ipa->mem = mem_data->local;
623
624	ret = dma_set_mask_and_coherent(&ipa->pdev->dev, DMA_BIT_MASK(64));
625	if (ret) {
626		dev_err(dev, "error %d setting DMA mask\n", ret);
627		return ret;
628	}
629
630	res = platform_get_resource_byname(ipa->pdev, IORESOURCE_MEM,
631					   "ipa-shared");
632	if (!res) {
633		dev_err(dev,
634			"DT error getting \"ipa-shared\" memory property\n");
635		return -ENODEV;
636	}
637
638	ipa->mem_virt = memremap(res->start, resource_size(res), MEMREMAP_WC);
639	if (!ipa->mem_virt) {
640		dev_err(dev, "unable to remap \"ipa-shared\" memory\n");
641		return -ENOMEM;
642	}
643
644	ipa->mem_addr = res->start;
645	ipa->mem_size = resource_size(res);
646
647	ret = ipa_imem_init(ipa, mem_data->imem_addr, mem_data->imem_size);
648	if (ret)
649		goto err_unmap;
650
651	ret = ipa_smem_init(ipa, mem_data->smem_id, mem_data->smem_size);
652	if (ret)
653		goto err_imem_exit;
654
655	return 0;
656
657err_imem_exit:
658	ipa_imem_exit(ipa);
659err_unmap:
660	memunmap(ipa->mem_virt);
661
662	return ret;
663}
664
665/* Inverse of ipa_mem_init() */
666void ipa_mem_exit(struct ipa *ipa)
667{
668	ipa_smem_exit(ipa);
669	ipa_imem_exit(ipa);
670	memunmap(ipa->mem_virt);
671}