Loading...
1// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
2// Copyright (C) 2016-2018, Allwinner Technology CO., LTD.
3// Copyright (C) 2019-2020, Cerno
4
5#include <linux/bitfield.h>
6#include <linux/bug.h>
7#include <linux/clk.h>
8#include <linux/device.h>
9#include <linux/dma-direction.h>
10#include <linux/dma-mapping.h>
11#include <linux/err.h>
12#include <linux/errno.h>
13#include <linux/interrupt.h>
14#include <linux/iommu.h>
15#include <linux/iopoll.h>
16#include <linux/ioport.h>
17#include <linux/log2.h>
18#include <linux/module.h>
19#include <linux/of_platform.h>
20#include <linux/platform_device.h>
21#include <linux/pm.h>
22#include <linux/pm_runtime.h>
23#include <linux/reset.h>
24#include <linux/sizes.h>
25#include <linux/slab.h>
26#include <linux/spinlock.h>
27#include <linux/types.h>
28
29#include "iommu-pages.h"
30
31#define IOMMU_RESET_REG 0x010
32#define IOMMU_RESET_RELEASE_ALL 0xffffffff
33#define IOMMU_ENABLE_REG 0x020
34#define IOMMU_ENABLE_ENABLE BIT(0)
35
36#define IOMMU_BYPASS_REG 0x030
37#define IOMMU_AUTO_GATING_REG 0x040
38#define IOMMU_AUTO_GATING_ENABLE BIT(0)
39
40#define IOMMU_WBUF_CTRL_REG 0x044
41#define IOMMU_OOO_CTRL_REG 0x048
42#define IOMMU_4KB_BDY_PRT_CTRL_REG 0x04c
43#define IOMMU_TTB_REG 0x050
44#define IOMMU_TLB_ENABLE_REG 0x060
45#define IOMMU_TLB_PREFETCH_REG 0x070
46#define IOMMU_TLB_PREFETCH_MASTER_ENABLE(m) BIT(m)
47
48#define IOMMU_TLB_FLUSH_REG 0x080
49#define IOMMU_TLB_FLUSH_PTW_CACHE BIT(17)
50#define IOMMU_TLB_FLUSH_MACRO_TLB BIT(16)
51#define IOMMU_TLB_FLUSH_MICRO_TLB(i) (BIT(i) & GENMASK(5, 0))
52
53#define IOMMU_TLB_IVLD_ADDR_REG 0x090
54#define IOMMU_TLB_IVLD_ADDR_MASK_REG 0x094
55#define IOMMU_TLB_IVLD_ENABLE_REG 0x098
56#define IOMMU_TLB_IVLD_ENABLE_ENABLE BIT(0)
57
58#define IOMMU_PC_IVLD_ADDR_REG 0x0a0
59#define IOMMU_PC_IVLD_ENABLE_REG 0x0a8
60#define IOMMU_PC_IVLD_ENABLE_ENABLE BIT(0)
61
62#define IOMMU_DM_AUT_CTRL_REG(d) (0x0b0 + ((d) / 2) * 4)
63#define IOMMU_DM_AUT_CTRL_RD_UNAVAIL(d, m) (1 << (((d & 1) * 16) + ((m) * 2)))
64#define IOMMU_DM_AUT_CTRL_WR_UNAVAIL(d, m) (1 << (((d & 1) * 16) + ((m) * 2) + 1))
65
66#define IOMMU_DM_AUT_OVWT_REG 0x0d0
67#define IOMMU_INT_ENABLE_REG 0x100
68#define IOMMU_INT_CLR_REG 0x104
69#define IOMMU_INT_STA_REG 0x108
70#define IOMMU_INT_ERR_ADDR_REG(i) (0x110 + (i) * 4)
71#define IOMMU_INT_ERR_ADDR_L1_REG 0x130
72#define IOMMU_INT_ERR_ADDR_L2_REG 0x134
73#define IOMMU_INT_ERR_DATA_REG(i) (0x150 + (i) * 4)
74#define IOMMU_L1PG_INT_REG 0x0180
75#define IOMMU_L2PG_INT_REG 0x0184
76
77#define IOMMU_INT_INVALID_L2PG BIT(17)
78#define IOMMU_INT_INVALID_L1PG BIT(16)
79#define IOMMU_INT_MASTER_PERMISSION(m) BIT(m)
80#define IOMMU_INT_MASTER_MASK (IOMMU_INT_MASTER_PERMISSION(0) | \
81 IOMMU_INT_MASTER_PERMISSION(1) | \
82 IOMMU_INT_MASTER_PERMISSION(2) | \
83 IOMMU_INT_MASTER_PERMISSION(3) | \
84 IOMMU_INT_MASTER_PERMISSION(4) | \
85 IOMMU_INT_MASTER_PERMISSION(5))
86#define IOMMU_INT_MASK (IOMMU_INT_INVALID_L1PG | \
87 IOMMU_INT_INVALID_L2PG | \
88 IOMMU_INT_MASTER_MASK)
89
90#define PT_ENTRY_SIZE sizeof(u32)
91
92#define NUM_DT_ENTRIES 4096
93#define DT_SIZE (NUM_DT_ENTRIES * PT_ENTRY_SIZE)
94
95#define NUM_PT_ENTRIES 256
96#define PT_SIZE (NUM_PT_ENTRIES * PT_ENTRY_SIZE)
97
98#define SPAGE_SIZE 4096
99
100struct sun50i_iommu {
101 struct iommu_device iommu;
102
103 /* Lock to modify the IOMMU registers */
104 spinlock_t iommu_lock;
105
106 struct device *dev;
107 void __iomem *base;
108 struct reset_control *reset;
109 struct clk *clk;
110
111 struct iommu_domain *domain;
112 struct kmem_cache *pt_pool;
113};
114
115struct sun50i_iommu_domain {
116 struct iommu_domain domain;
117
118 /* Number of devices attached to the domain */
119 refcount_t refcnt;
120
121 /* L1 Page Table */
122 u32 *dt;
123 dma_addr_t dt_dma;
124
125 struct sun50i_iommu *iommu;
126};
127
128static struct sun50i_iommu_domain *to_sun50i_domain(struct iommu_domain *domain)
129{
130 return container_of(domain, struct sun50i_iommu_domain, domain);
131}
132
133static struct sun50i_iommu *sun50i_iommu_from_dev(struct device *dev)
134{
135 return dev_iommu_priv_get(dev);
136}
137
138static u32 iommu_read(struct sun50i_iommu *iommu, u32 offset)
139{
140 return readl(iommu->base + offset);
141}
142
143static void iommu_write(struct sun50i_iommu *iommu, u32 offset, u32 value)
144{
145 writel(value, iommu->base + offset);
146}
147
148/*
149 * The Allwinner H6 IOMMU uses a 2-level page table.
150 *
151 * The first level is the usual Directory Table (DT), that consists of
152 * 4096 4-bytes Directory Table Entries (DTE), each pointing to a Page
153 * Table (PT).
154 *
155 * Each PT consits of 256 4-bytes Page Table Entries (PTE), each
156 * pointing to a 4kB page of physical memory.
157 *
158 * The IOMMU supports a single DT, pointed by the IOMMU_TTB_REG
159 * register that contains its physical address.
160 */
161
162#define SUN50I_IOVA_DTE_MASK GENMASK(31, 20)
163#define SUN50I_IOVA_PTE_MASK GENMASK(19, 12)
164#define SUN50I_IOVA_PAGE_MASK GENMASK(11, 0)
165
166static u32 sun50i_iova_get_dte_index(dma_addr_t iova)
167{
168 return FIELD_GET(SUN50I_IOVA_DTE_MASK, iova);
169}
170
171static u32 sun50i_iova_get_pte_index(dma_addr_t iova)
172{
173 return FIELD_GET(SUN50I_IOVA_PTE_MASK, iova);
174}
175
176static u32 sun50i_iova_get_page_offset(dma_addr_t iova)
177{
178 return FIELD_GET(SUN50I_IOVA_PAGE_MASK, iova);
179}
180
181/*
182 * Each Directory Table Entry has a Page Table address and a valid
183 * bit:
184
185 * +---------------------+-----------+-+
186 * | PT address | Reserved |V|
187 * +---------------------+-----------+-+
188 * 31:10 - Page Table address
189 * 9:2 - Reserved
190 * 1:0 - 1 if the entry is valid
191 */
192
193#define SUN50I_DTE_PT_ADDRESS_MASK GENMASK(31, 10)
194#define SUN50I_DTE_PT_ATTRS GENMASK(1, 0)
195#define SUN50I_DTE_PT_VALID 1
196
197static phys_addr_t sun50i_dte_get_pt_address(u32 dte)
198{
199 return (phys_addr_t)dte & SUN50I_DTE_PT_ADDRESS_MASK;
200}
201
202static bool sun50i_dte_is_pt_valid(u32 dte)
203{
204 return (dte & SUN50I_DTE_PT_ATTRS) == SUN50I_DTE_PT_VALID;
205}
206
207static u32 sun50i_mk_dte(dma_addr_t pt_dma)
208{
209 return (pt_dma & SUN50I_DTE_PT_ADDRESS_MASK) | SUN50I_DTE_PT_VALID;
210}
211
212/*
213 * Each PTE has a Page address, an authority index and a valid bit:
214 *
215 * +----------------+-----+-----+-----+---+-----+
216 * | Page address | Rsv | ACI | Rsv | V | Rsv |
217 * +----------------+-----+-----+-----+---+-----+
218 * 31:12 - Page address
219 * 11:8 - Reserved
220 * 7:4 - Authority Control Index
221 * 3:2 - Reserved
222 * 1 - 1 if the entry is valid
223 * 0 - Reserved
224 *
225 * The way permissions work is that the IOMMU has 16 "domains" that
226 * can be configured to give each masters either read or write
227 * permissions through the IOMMU_DM_AUT_CTRL_REG registers. The domain
228 * 0 seems like the default domain, and its permissions in the
229 * IOMMU_DM_AUT_CTRL_REG are only read-only, so it's not really
230 * useful to enforce any particular permission.
231 *
232 * Each page entry will then have a reference to the domain they are
233 * affected to, so that we can actually enforce them on a per-page
234 * basis.
235 *
236 * In order to make it work with the IOMMU framework, we will be using
237 * 4 different domains, starting at 1: RD_WR, RD, WR and NONE
238 * depending on the permission we want to enforce. Each domain will
239 * have each master setup in the same way, since the IOMMU framework
240 * doesn't seem to restrict page access on a per-device basis. And
241 * then we will use the relevant domain index when generating the page
242 * table entry depending on the permissions we want to be enforced.
243 */
244
245enum sun50i_iommu_aci {
246 SUN50I_IOMMU_ACI_DO_NOT_USE = 0,
247 SUN50I_IOMMU_ACI_NONE,
248 SUN50I_IOMMU_ACI_RD,
249 SUN50I_IOMMU_ACI_WR,
250 SUN50I_IOMMU_ACI_RD_WR,
251};
252
253#define SUN50I_PTE_PAGE_ADDRESS_MASK GENMASK(31, 12)
254#define SUN50I_PTE_ACI_MASK GENMASK(7, 4)
255#define SUN50I_PTE_PAGE_VALID BIT(1)
256
257static phys_addr_t sun50i_pte_get_page_address(u32 pte)
258{
259 return (phys_addr_t)pte & SUN50I_PTE_PAGE_ADDRESS_MASK;
260}
261
262static enum sun50i_iommu_aci sun50i_get_pte_aci(u32 pte)
263{
264 return FIELD_GET(SUN50I_PTE_ACI_MASK, pte);
265}
266
267static bool sun50i_pte_is_page_valid(u32 pte)
268{
269 return pte & SUN50I_PTE_PAGE_VALID;
270}
271
272static u32 sun50i_mk_pte(phys_addr_t page, int prot)
273{
274 enum sun50i_iommu_aci aci;
275 u32 flags = 0;
276
277 if ((prot & (IOMMU_READ | IOMMU_WRITE)) == (IOMMU_READ | IOMMU_WRITE))
278 aci = SUN50I_IOMMU_ACI_RD_WR;
279 else if (prot & IOMMU_READ)
280 aci = SUN50I_IOMMU_ACI_RD;
281 else if (prot & IOMMU_WRITE)
282 aci = SUN50I_IOMMU_ACI_WR;
283 else
284 aci = SUN50I_IOMMU_ACI_NONE;
285
286 flags |= FIELD_PREP(SUN50I_PTE_ACI_MASK, aci);
287 page &= SUN50I_PTE_PAGE_ADDRESS_MASK;
288 return page | flags | SUN50I_PTE_PAGE_VALID;
289}
290
291static void sun50i_table_flush(struct sun50i_iommu_domain *sun50i_domain,
292 void *vaddr, unsigned int count)
293{
294 struct sun50i_iommu *iommu = sun50i_domain->iommu;
295 dma_addr_t dma = virt_to_phys(vaddr);
296 size_t size = count * PT_ENTRY_SIZE;
297
298 dma_sync_single_for_device(iommu->dev, dma, size, DMA_TO_DEVICE);
299}
300
301static void sun50i_iommu_zap_iova(struct sun50i_iommu *iommu,
302 unsigned long iova)
303{
304 u32 reg;
305 int ret;
306
307 iommu_write(iommu, IOMMU_TLB_IVLD_ADDR_REG, iova);
308 iommu_write(iommu, IOMMU_TLB_IVLD_ADDR_MASK_REG, GENMASK(31, 12));
309 iommu_write(iommu, IOMMU_TLB_IVLD_ENABLE_REG,
310 IOMMU_TLB_IVLD_ENABLE_ENABLE);
311
312 ret = readl_poll_timeout_atomic(iommu->base + IOMMU_TLB_IVLD_ENABLE_REG,
313 reg, !reg, 1, 2000);
314 if (ret)
315 dev_warn(iommu->dev, "TLB invalidation timed out!\n");
316}
317
318static void sun50i_iommu_zap_ptw_cache(struct sun50i_iommu *iommu,
319 unsigned long iova)
320{
321 u32 reg;
322 int ret;
323
324 iommu_write(iommu, IOMMU_PC_IVLD_ADDR_REG, iova);
325 iommu_write(iommu, IOMMU_PC_IVLD_ENABLE_REG,
326 IOMMU_PC_IVLD_ENABLE_ENABLE);
327
328 ret = readl_poll_timeout_atomic(iommu->base + IOMMU_PC_IVLD_ENABLE_REG,
329 reg, !reg, 1, 2000);
330 if (ret)
331 dev_warn(iommu->dev, "PTW cache invalidation timed out!\n");
332}
333
334static void sun50i_iommu_zap_range(struct sun50i_iommu *iommu,
335 unsigned long iova, size_t size)
336{
337 assert_spin_locked(&iommu->iommu_lock);
338
339 iommu_write(iommu, IOMMU_AUTO_GATING_REG, 0);
340
341 sun50i_iommu_zap_iova(iommu, iova);
342 sun50i_iommu_zap_iova(iommu, iova + SPAGE_SIZE);
343 if (size > SPAGE_SIZE) {
344 sun50i_iommu_zap_iova(iommu, iova + size);
345 sun50i_iommu_zap_iova(iommu, iova + size + SPAGE_SIZE);
346 }
347 sun50i_iommu_zap_ptw_cache(iommu, iova);
348 sun50i_iommu_zap_ptw_cache(iommu, iova + SZ_1M);
349 if (size > SZ_1M) {
350 sun50i_iommu_zap_ptw_cache(iommu, iova + size);
351 sun50i_iommu_zap_ptw_cache(iommu, iova + size + SZ_1M);
352 }
353
354 iommu_write(iommu, IOMMU_AUTO_GATING_REG, IOMMU_AUTO_GATING_ENABLE);
355}
356
357static int sun50i_iommu_flush_all_tlb(struct sun50i_iommu *iommu)
358{
359 u32 reg;
360 int ret;
361
362 assert_spin_locked(&iommu->iommu_lock);
363
364 iommu_write(iommu,
365 IOMMU_TLB_FLUSH_REG,
366 IOMMU_TLB_FLUSH_PTW_CACHE |
367 IOMMU_TLB_FLUSH_MACRO_TLB |
368 IOMMU_TLB_FLUSH_MICRO_TLB(5) |
369 IOMMU_TLB_FLUSH_MICRO_TLB(4) |
370 IOMMU_TLB_FLUSH_MICRO_TLB(3) |
371 IOMMU_TLB_FLUSH_MICRO_TLB(2) |
372 IOMMU_TLB_FLUSH_MICRO_TLB(1) |
373 IOMMU_TLB_FLUSH_MICRO_TLB(0));
374
375 ret = readl_poll_timeout_atomic(iommu->base + IOMMU_TLB_FLUSH_REG,
376 reg, !reg,
377 1, 2000);
378 if (ret)
379 dev_warn(iommu->dev, "TLB Flush timed out!\n");
380
381 return ret;
382}
383
384static void sun50i_iommu_flush_iotlb_all(struct iommu_domain *domain)
385{
386 struct sun50i_iommu_domain *sun50i_domain = to_sun50i_domain(domain);
387 struct sun50i_iommu *iommu = sun50i_domain->iommu;
388 unsigned long flags;
389
390 /*
391 * At boot, we'll have a first call into .flush_iotlb_all right after
392 * .probe_device, and since we link our (single) domain to our iommu in
393 * the .attach_device callback, we don't have that pointer set.
394 *
395 * It shouldn't really be any trouble to ignore it though since we flush
396 * all caches as part of the device powerup.
397 */
398 if (!iommu)
399 return;
400
401 spin_lock_irqsave(&iommu->iommu_lock, flags);
402 sun50i_iommu_flush_all_tlb(iommu);
403 spin_unlock_irqrestore(&iommu->iommu_lock, flags);
404}
405
406static int sun50i_iommu_iotlb_sync_map(struct iommu_domain *domain,
407 unsigned long iova, size_t size)
408{
409 struct sun50i_iommu_domain *sun50i_domain = to_sun50i_domain(domain);
410 struct sun50i_iommu *iommu = sun50i_domain->iommu;
411 unsigned long flags;
412
413 spin_lock_irqsave(&iommu->iommu_lock, flags);
414 sun50i_iommu_zap_range(iommu, iova, size);
415 spin_unlock_irqrestore(&iommu->iommu_lock, flags);
416
417 return 0;
418}
419
420static void sun50i_iommu_iotlb_sync(struct iommu_domain *domain,
421 struct iommu_iotlb_gather *gather)
422{
423 sun50i_iommu_flush_iotlb_all(domain);
424}
425
426static int sun50i_iommu_enable(struct sun50i_iommu *iommu)
427{
428 struct sun50i_iommu_domain *sun50i_domain;
429 unsigned long flags;
430 int ret;
431
432 if (!iommu->domain)
433 return 0;
434
435 sun50i_domain = to_sun50i_domain(iommu->domain);
436
437 ret = reset_control_deassert(iommu->reset);
438 if (ret)
439 return ret;
440
441 ret = clk_prepare_enable(iommu->clk);
442 if (ret)
443 goto err_reset_assert;
444
445 spin_lock_irqsave(&iommu->iommu_lock, flags);
446
447 iommu_write(iommu, IOMMU_TTB_REG, sun50i_domain->dt_dma);
448 iommu_write(iommu, IOMMU_TLB_PREFETCH_REG,
449 IOMMU_TLB_PREFETCH_MASTER_ENABLE(0) |
450 IOMMU_TLB_PREFETCH_MASTER_ENABLE(1) |
451 IOMMU_TLB_PREFETCH_MASTER_ENABLE(2) |
452 IOMMU_TLB_PREFETCH_MASTER_ENABLE(3) |
453 IOMMU_TLB_PREFETCH_MASTER_ENABLE(4) |
454 IOMMU_TLB_PREFETCH_MASTER_ENABLE(5));
455 iommu_write(iommu, IOMMU_BYPASS_REG, 0);
456 iommu_write(iommu, IOMMU_INT_ENABLE_REG, IOMMU_INT_MASK);
457 iommu_write(iommu, IOMMU_DM_AUT_CTRL_REG(SUN50I_IOMMU_ACI_NONE),
458 IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 0) |
459 IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 0) |
460 IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 1) |
461 IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 1) |
462 IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 2) |
463 IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 2) |
464 IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 3) |
465 IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 3) |
466 IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 4) |
467 IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 4) |
468 IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 5) |
469 IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 5));
470
471 iommu_write(iommu, IOMMU_DM_AUT_CTRL_REG(SUN50I_IOMMU_ACI_RD),
472 IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_RD, 0) |
473 IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_RD, 1) |
474 IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_RD, 2) |
475 IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_RD, 3) |
476 IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_RD, 4) |
477 IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_RD, 5));
478
479 iommu_write(iommu, IOMMU_DM_AUT_CTRL_REG(SUN50I_IOMMU_ACI_WR),
480 IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_WR, 0) |
481 IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_WR, 1) |
482 IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_WR, 2) |
483 IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_WR, 3) |
484 IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_WR, 4) |
485 IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_WR, 5));
486
487 ret = sun50i_iommu_flush_all_tlb(iommu);
488 if (ret) {
489 spin_unlock_irqrestore(&iommu->iommu_lock, flags);
490 goto err_clk_disable;
491 }
492
493 iommu_write(iommu, IOMMU_AUTO_GATING_REG, IOMMU_AUTO_GATING_ENABLE);
494 iommu_write(iommu, IOMMU_ENABLE_REG, IOMMU_ENABLE_ENABLE);
495
496 spin_unlock_irqrestore(&iommu->iommu_lock, flags);
497
498 return 0;
499
500err_clk_disable:
501 clk_disable_unprepare(iommu->clk);
502
503err_reset_assert:
504 reset_control_assert(iommu->reset);
505
506 return ret;
507}
508
509static void sun50i_iommu_disable(struct sun50i_iommu *iommu)
510{
511 unsigned long flags;
512
513 spin_lock_irqsave(&iommu->iommu_lock, flags);
514
515 iommu_write(iommu, IOMMU_ENABLE_REG, 0);
516 iommu_write(iommu, IOMMU_TTB_REG, 0);
517
518 spin_unlock_irqrestore(&iommu->iommu_lock, flags);
519
520 clk_disable_unprepare(iommu->clk);
521 reset_control_assert(iommu->reset);
522}
523
524static void *sun50i_iommu_alloc_page_table(struct sun50i_iommu *iommu,
525 gfp_t gfp)
526{
527 dma_addr_t pt_dma;
528 u32 *page_table;
529
530 page_table = kmem_cache_zalloc(iommu->pt_pool, gfp);
531 if (!page_table)
532 return ERR_PTR(-ENOMEM);
533
534 pt_dma = dma_map_single(iommu->dev, page_table, PT_SIZE, DMA_TO_DEVICE);
535 if (dma_mapping_error(iommu->dev, pt_dma)) {
536 dev_err(iommu->dev, "Couldn't map L2 Page Table\n");
537 kmem_cache_free(iommu->pt_pool, page_table);
538 return ERR_PTR(-ENOMEM);
539 }
540
541 /* We rely on the physical address and DMA address being the same */
542 WARN_ON(pt_dma != virt_to_phys(page_table));
543
544 return page_table;
545}
546
547static void sun50i_iommu_free_page_table(struct sun50i_iommu *iommu,
548 u32 *page_table)
549{
550 phys_addr_t pt_phys = virt_to_phys(page_table);
551
552 dma_unmap_single(iommu->dev, pt_phys, PT_SIZE, DMA_TO_DEVICE);
553 kmem_cache_free(iommu->pt_pool, page_table);
554}
555
556static u32 *sun50i_dte_get_page_table(struct sun50i_iommu_domain *sun50i_domain,
557 dma_addr_t iova, gfp_t gfp)
558{
559 struct sun50i_iommu *iommu = sun50i_domain->iommu;
560 u32 *page_table;
561 u32 *dte_addr;
562 u32 old_dte;
563 u32 dte;
564
565 dte_addr = &sun50i_domain->dt[sun50i_iova_get_dte_index(iova)];
566 dte = *dte_addr;
567 if (sun50i_dte_is_pt_valid(dte)) {
568 phys_addr_t pt_phys = sun50i_dte_get_pt_address(dte);
569 return (u32 *)phys_to_virt(pt_phys);
570 }
571
572 page_table = sun50i_iommu_alloc_page_table(iommu, gfp);
573 if (IS_ERR(page_table))
574 return page_table;
575
576 dte = sun50i_mk_dte(virt_to_phys(page_table));
577 old_dte = cmpxchg(dte_addr, 0, dte);
578 if (old_dte) {
579 phys_addr_t installed_pt_phys =
580 sun50i_dte_get_pt_address(old_dte);
581 u32 *installed_pt = phys_to_virt(installed_pt_phys);
582 u32 *drop_pt = page_table;
583
584 page_table = installed_pt;
585 dte = old_dte;
586 sun50i_iommu_free_page_table(iommu, drop_pt);
587 }
588
589 sun50i_table_flush(sun50i_domain, page_table, NUM_PT_ENTRIES);
590 sun50i_table_flush(sun50i_domain, dte_addr, 1);
591
592 return page_table;
593}
594
595static int sun50i_iommu_map(struct iommu_domain *domain, unsigned long iova,
596 phys_addr_t paddr, size_t size, size_t count,
597 int prot, gfp_t gfp, size_t *mapped)
598{
599 struct sun50i_iommu_domain *sun50i_domain = to_sun50i_domain(domain);
600 struct sun50i_iommu *iommu = sun50i_domain->iommu;
601 u32 pte_index;
602 u32 *page_table, *pte_addr;
603 int ret = 0;
604
605 /* the IOMMU can only handle 32-bit addresses, both input and output */
606 if ((uint64_t)paddr >> 32) {
607 ret = -EINVAL;
608 dev_warn_once(iommu->dev,
609 "attempt to map address beyond 4GB\n");
610 goto out;
611 }
612
613 page_table = sun50i_dte_get_page_table(sun50i_domain, iova, gfp);
614 if (IS_ERR(page_table)) {
615 ret = PTR_ERR(page_table);
616 goto out;
617 }
618
619 pte_index = sun50i_iova_get_pte_index(iova);
620 pte_addr = &page_table[pte_index];
621 if (unlikely(sun50i_pte_is_page_valid(*pte_addr))) {
622 phys_addr_t page_phys = sun50i_pte_get_page_address(*pte_addr);
623 dev_err(iommu->dev,
624 "iova %pad already mapped to %pa cannot remap to %pa prot: %#x\n",
625 &iova, &page_phys, &paddr, prot);
626 ret = -EBUSY;
627 goto out;
628 }
629
630 *pte_addr = sun50i_mk_pte(paddr, prot);
631 sun50i_table_flush(sun50i_domain, pte_addr, 1);
632 *mapped = size;
633
634out:
635 return ret;
636}
637
638static size_t sun50i_iommu_unmap(struct iommu_domain *domain, unsigned long iova,
639 size_t size, size_t count, struct iommu_iotlb_gather *gather)
640{
641 struct sun50i_iommu_domain *sun50i_domain = to_sun50i_domain(domain);
642 phys_addr_t pt_phys;
643 u32 *pte_addr;
644 u32 dte;
645
646 dte = sun50i_domain->dt[sun50i_iova_get_dte_index(iova)];
647 if (!sun50i_dte_is_pt_valid(dte))
648 return 0;
649
650 pt_phys = sun50i_dte_get_pt_address(dte);
651 pte_addr = (u32 *)phys_to_virt(pt_phys) + sun50i_iova_get_pte_index(iova);
652
653 if (!sun50i_pte_is_page_valid(*pte_addr))
654 return 0;
655
656 memset(pte_addr, 0, sizeof(*pte_addr));
657 sun50i_table_flush(sun50i_domain, pte_addr, 1);
658
659 return SZ_4K;
660}
661
662static phys_addr_t sun50i_iommu_iova_to_phys(struct iommu_domain *domain,
663 dma_addr_t iova)
664{
665 struct sun50i_iommu_domain *sun50i_domain = to_sun50i_domain(domain);
666 phys_addr_t pt_phys;
667 u32 *page_table;
668 u32 dte, pte;
669
670 dte = sun50i_domain->dt[sun50i_iova_get_dte_index(iova)];
671 if (!sun50i_dte_is_pt_valid(dte))
672 return 0;
673
674 pt_phys = sun50i_dte_get_pt_address(dte);
675 page_table = (u32 *)phys_to_virt(pt_phys);
676 pte = page_table[sun50i_iova_get_pte_index(iova)];
677 if (!sun50i_pte_is_page_valid(pte))
678 return 0;
679
680 return sun50i_pte_get_page_address(pte) +
681 sun50i_iova_get_page_offset(iova);
682}
683
684static struct iommu_domain *
685sun50i_iommu_domain_alloc_paging(struct device *dev)
686{
687 struct sun50i_iommu_domain *sun50i_domain;
688
689 sun50i_domain = kzalloc(sizeof(*sun50i_domain), GFP_KERNEL);
690 if (!sun50i_domain)
691 return NULL;
692
693 sun50i_domain->dt = iommu_alloc_pages(GFP_KERNEL | GFP_DMA32,
694 get_order(DT_SIZE));
695 if (!sun50i_domain->dt)
696 goto err_free_domain;
697
698 refcount_set(&sun50i_domain->refcnt, 1);
699
700 sun50i_domain->domain.geometry.aperture_start = 0;
701 sun50i_domain->domain.geometry.aperture_end = DMA_BIT_MASK(32);
702 sun50i_domain->domain.geometry.force_aperture = true;
703
704 return &sun50i_domain->domain;
705
706err_free_domain:
707 kfree(sun50i_domain);
708
709 return NULL;
710}
711
712static void sun50i_iommu_domain_free(struct iommu_domain *domain)
713{
714 struct sun50i_iommu_domain *sun50i_domain = to_sun50i_domain(domain);
715
716 iommu_free_pages(sun50i_domain->dt, get_order(DT_SIZE));
717 sun50i_domain->dt = NULL;
718
719 kfree(sun50i_domain);
720}
721
722static int sun50i_iommu_attach_domain(struct sun50i_iommu *iommu,
723 struct sun50i_iommu_domain *sun50i_domain)
724{
725 iommu->domain = &sun50i_domain->domain;
726 sun50i_domain->iommu = iommu;
727
728 sun50i_domain->dt_dma = dma_map_single(iommu->dev, sun50i_domain->dt,
729 DT_SIZE, DMA_TO_DEVICE);
730 if (dma_mapping_error(iommu->dev, sun50i_domain->dt_dma)) {
731 dev_err(iommu->dev, "Couldn't map L1 Page Table\n");
732 return -ENOMEM;
733 }
734
735 return sun50i_iommu_enable(iommu);
736}
737
738static void sun50i_iommu_detach_domain(struct sun50i_iommu *iommu,
739 struct sun50i_iommu_domain *sun50i_domain)
740{
741 unsigned int i;
742
743 for (i = 0; i < NUM_DT_ENTRIES; i++) {
744 phys_addr_t pt_phys;
745 u32 *page_table;
746 u32 *dte_addr;
747 u32 dte;
748
749 dte_addr = &sun50i_domain->dt[i];
750 dte = *dte_addr;
751 if (!sun50i_dte_is_pt_valid(dte))
752 continue;
753
754 memset(dte_addr, 0, sizeof(*dte_addr));
755 sun50i_table_flush(sun50i_domain, dte_addr, 1);
756
757 pt_phys = sun50i_dte_get_pt_address(dte);
758 page_table = phys_to_virt(pt_phys);
759 sun50i_iommu_free_page_table(iommu, page_table);
760 }
761
762
763 sun50i_iommu_disable(iommu);
764
765 dma_unmap_single(iommu->dev, virt_to_phys(sun50i_domain->dt),
766 DT_SIZE, DMA_TO_DEVICE);
767
768 iommu->domain = NULL;
769}
770
771static int sun50i_iommu_identity_attach(struct iommu_domain *identity_domain,
772 struct device *dev)
773{
774 struct sun50i_iommu *iommu = dev_iommu_priv_get(dev);
775 struct sun50i_iommu_domain *sun50i_domain;
776
777 dev_dbg(dev, "Detaching from IOMMU domain\n");
778
779 if (iommu->domain == identity_domain)
780 return 0;
781
782 sun50i_domain = to_sun50i_domain(iommu->domain);
783 if (refcount_dec_and_test(&sun50i_domain->refcnt))
784 sun50i_iommu_detach_domain(iommu, sun50i_domain);
785 return 0;
786}
787
788static struct iommu_domain_ops sun50i_iommu_identity_ops = {
789 .attach_dev = sun50i_iommu_identity_attach,
790};
791
792static struct iommu_domain sun50i_iommu_identity_domain = {
793 .type = IOMMU_DOMAIN_IDENTITY,
794 .ops = &sun50i_iommu_identity_ops,
795};
796
797static int sun50i_iommu_attach_device(struct iommu_domain *domain,
798 struct device *dev)
799{
800 struct sun50i_iommu_domain *sun50i_domain = to_sun50i_domain(domain);
801 struct sun50i_iommu *iommu;
802
803 iommu = sun50i_iommu_from_dev(dev);
804 if (!iommu)
805 return -ENODEV;
806
807 dev_dbg(dev, "Attaching to IOMMU domain\n");
808
809 refcount_inc(&sun50i_domain->refcnt);
810
811 if (iommu->domain == domain)
812 return 0;
813
814 sun50i_iommu_identity_attach(&sun50i_iommu_identity_domain, dev);
815
816 sun50i_iommu_attach_domain(iommu, sun50i_domain);
817
818 return 0;
819}
820
821static struct iommu_device *sun50i_iommu_probe_device(struct device *dev)
822{
823 struct sun50i_iommu *iommu;
824
825 iommu = sun50i_iommu_from_dev(dev);
826 if (!iommu)
827 return ERR_PTR(-ENODEV);
828
829 return &iommu->iommu;
830}
831
832static int sun50i_iommu_of_xlate(struct device *dev,
833 const struct of_phandle_args *args)
834{
835 struct platform_device *iommu_pdev = of_find_device_by_node(args->np);
836 unsigned id = args->args[0];
837
838 dev_iommu_priv_set(dev, platform_get_drvdata(iommu_pdev));
839
840 return iommu_fwspec_add_ids(dev, &id, 1);
841}
842
843static const struct iommu_ops sun50i_iommu_ops = {
844 .identity_domain = &sun50i_iommu_identity_domain,
845 .pgsize_bitmap = SZ_4K,
846 .device_group = generic_single_device_group,
847 .domain_alloc_paging = sun50i_iommu_domain_alloc_paging,
848 .of_xlate = sun50i_iommu_of_xlate,
849 .probe_device = sun50i_iommu_probe_device,
850 .default_domain_ops = &(const struct iommu_domain_ops) {
851 .attach_dev = sun50i_iommu_attach_device,
852 .flush_iotlb_all = sun50i_iommu_flush_iotlb_all,
853 .iotlb_sync_map = sun50i_iommu_iotlb_sync_map,
854 .iotlb_sync = sun50i_iommu_iotlb_sync,
855 .iova_to_phys = sun50i_iommu_iova_to_phys,
856 .map_pages = sun50i_iommu_map,
857 .unmap_pages = sun50i_iommu_unmap,
858 .free = sun50i_iommu_domain_free,
859 }
860};
861
862static void sun50i_iommu_report_fault(struct sun50i_iommu *iommu,
863 unsigned master, phys_addr_t iova,
864 unsigned prot)
865{
866 dev_err(iommu->dev, "Page fault for %pad (master %d, dir %s)\n",
867 &iova, master, (prot == IOMMU_FAULT_WRITE) ? "wr" : "rd");
868
869 if (iommu->domain)
870 report_iommu_fault(iommu->domain, iommu->dev, iova, prot);
871 else
872 dev_err(iommu->dev, "Page fault while iommu not attached to any domain?\n");
873
874 sun50i_iommu_zap_range(iommu, iova, SPAGE_SIZE);
875}
876
877static phys_addr_t sun50i_iommu_handle_pt_irq(struct sun50i_iommu *iommu,
878 unsigned addr_reg,
879 unsigned blame_reg)
880{
881 phys_addr_t iova;
882 unsigned master;
883 u32 blame;
884
885 assert_spin_locked(&iommu->iommu_lock);
886
887 iova = iommu_read(iommu, addr_reg);
888 blame = iommu_read(iommu, blame_reg);
889 master = ilog2(blame & IOMMU_INT_MASTER_MASK);
890
891 /*
892 * If the address is not in the page table, we can't get what
893 * operation triggered the fault. Assume it's a read
894 * operation.
895 */
896 sun50i_iommu_report_fault(iommu, master, iova, IOMMU_FAULT_READ);
897
898 return iova;
899}
900
901static phys_addr_t sun50i_iommu_handle_perm_irq(struct sun50i_iommu *iommu)
902{
903 enum sun50i_iommu_aci aci;
904 phys_addr_t iova;
905 unsigned master;
906 unsigned dir;
907 u32 blame;
908
909 assert_spin_locked(&iommu->iommu_lock);
910
911 blame = iommu_read(iommu, IOMMU_INT_STA_REG);
912 master = ilog2(blame & IOMMU_INT_MASTER_MASK);
913 iova = iommu_read(iommu, IOMMU_INT_ERR_ADDR_REG(master));
914 aci = sun50i_get_pte_aci(iommu_read(iommu,
915 IOMMU_INT_ERR_DATA_REG(master)));
916
917 switch (aci) {
918 /*
919 * If we are in the read-only domain, then it means we
920 * tried to write.
921 */
922 case SUN50I_IOMMU_ACI_RD:
923 dir = IOMMU_FAULT_WRITE;
924 break;
925
926 /*
927 * If we are in the write-only domain, then it means
928 * we tried to read.
929 */
930 case SUN50I_IOMMU_ACI_WR:
931
932 /*
933 * If we are in the domain without any permission, we
934 * can't really tell. Let's default to a read
935 * operation.
936 */
937 case SUN50I_IOMMU_ACI_NONE:
938
939 /* WTF? */
940 case SUN50I_IOMMU_ACI_RD_WR:
941 default:
942 dir = IOMMU_FAULT_READ;
943 break;
944 }
945
946 /*
947 * If the address is not in the page table, we can't get what
948 * operation triggered the fault. Assume it's a read
949 * operation.
950 */
951 sun50i_iommu_report_fault(iommu, master, iova, dir);
952
953 return iova;
954}
955
956static irqreturn_t sun50i_iommu_irq(int irq, void *dev_id)
957{
958 u32 status, l1_status, l2_status, resets;
959 struct sun50i_iommu *iommu = dev_id;
960
961 spin_lock(&iommu->iommu_lock);
962
963 status = iommu_read(iommu, IOMMU_INT_STA_REG);
964 if (!(status & IOMMU_INT_MASK)) {
965 spin_unlock(&iommu->iommu_lock);
966 return IRQ_NONE;
967 }
968
969 l1_status = iommu_read(iommu, IOMMU_L1PG_INT_REG);
970 l2_status = iommu_read(iommu, IOMMU_L2PG_INT_REG);
971
972 if (status & IOMMU_INT_INVALID_L2PG)
973 sun50i_iommu_handle_pt_irq(iommu,
974 IOMMU_INT_ERR_ADDR_L2_REG,
975 IOMMU_L2PG_INT_REG);
976 else if (status & IOMMU_INT_INVALID_L1PG)
977 sun50i_iommu_handle_pt_irq(iommu,
978 IOMMU_INT_ERR_ADDR_L1_REG,
979 IOMMU_L1PG_INT_REG);
980 else
981 sun50i_iommu_handle_perm_irq(iommu);
982
983 iommu_write(iommu, IOMMU_INT_CLR_REG, status);
984
985 resets = (status | l1_status | l2_status) & IOMMU_INT_MASTER_MASK;
986 iommu_write(iommu, IOMMU_RESET_REG, ~resets);
987 iommu_write(iommu, IOMMU_RESET_REG, IOMMU_RESET_RELEASE_ALL);
988
989 spin_unlock(&iommu->iommu_lock);
990
991 return IRQ_HANDLED;
992}
993
994static int sun50i_iommu_probe(struct platform_device *pdev)
995{
996 struct sun50i_iommu *iommu;
997 int ret, irq;
998
999 iommu = devm_kzalloc(&pdev->dev, sizeof(*iommu), GFP_KERNEL);
1000 if (!iommu)
1001 return -ENOMEM;
1002 spin_lock_init(&iommu->iommu_lock);
1003 iommu->domain = &sun50i_iommu_identity_domain;
1004 platform_set_drvdata(pdev, iommu);
1005 iommu->dev = &pdev->dev;
1006
1007 iommu->pt_pool = kmem_cache_create(dev_name(&pdev->dev),
1008 PT_SIZE, PT_SIZE,
1009 SLAB_HWCACHE_ALIGN | SLAB_CACHE_DMA32,
1010 NULL);
1011 if (!iommu->pt_pool)
1012 return -ENOMEM;
1013
1014 iommu->base = devm_platform_ioremap_resource(pdev, 0);
1015 if (IS_ERR(iommu->base)) {
1016 ret = PTR_ERR(iommu->base);
1017 goto err_free_cache;
1018 }
1019
1020 irq = platform_get_irq(pdev, 0);
1021 if (irq < 0) {
1022 ret = irq;
1023 goto err_free_cache;
1024 }
1025
1026 iommu->clk = devm_clk_get(&pdev->dev, NULL);
1027 if (IS_ERR(iommu->clk)) {
1028 dev_err(&pdev->dev, "Couldn't get our clock.\n");
1029 ret = PTR_ERR(iommu->clk);
1030 goto err_free_cache;
1031 }
1032
1033 iommu->reset = devm_reset_control_get(&pdev->dev, NULL);
1034 if (IS_ERR(iommu->reset)) {
1035 dev_err(&pdev->dev, "Couldn't get our reset line.\n");
1036 ret = PTR_ERR(iommu->reset);
1037 goto err_free_cache;
1038 }
1039
1040 ret = iommu_device_sysfs_add(&iommu->iommu, &pdev->dev,
1041 NULL, dev_name(&pdev->dev));
1042 if (ret)
1043 goto err_free_cache;
1044
1045 ret = iommu_device_register(&iommu->iommu, &sun50i_iommu_ops, &pdev->dev);
1046 if (ret)
1047 goto err_remove_sysfs;
1048
1049 ret = devm_request_irq(&pdev->dev, irq, sun50i_iommu_irq, 0,
1050 dev_name(&pdev->dev), iommu);
1051 if (ret < 0)
1052 goto err_unregister;
1053
1054 return 0;
1055
1056err_unregister:
1057 iommu_device_unregister(&iommu->iommu);
1058
1059err_remove_sysfs:
1060 iommu_device_sysfs_remove(&iommu->iommu);
1061
1062err_free_cache:
1063 kmem_cache_destroy(iommu->pt_pool);
1064
1065 return ret;
1066}
1067
1068static const struct of_device_id sun50i_iommu_dt[] = {
1069 { .compatible = "allwinner,sun50i-h6-iommu", },
1070 { .compatible = "allwinner,sun50i-h616-iommu", },
1071 { /* sentinel */ },
1072};
1073MODULE_DEVICE_TABLE(of, sun50i_iommu_dt);
1074
1075static struct platform_driver sun50i_iommu_driver = {
1076 .driver = {
1077 .name = "sun50i-iommu",
1078 .of_match_table = sun50i_iommu_dt,
1079 .suppress_bind_attrs = true,
1080 }
1081};
1082builtin_platform_driver_probe(sun50i_iommu_driver, sun50i_iommu_probe);
1083
1084MODULE_DESCRIPTION("Allwinner H6 IOMMU driver");
1085MODULE_AUTHOR("Maxime Ripard <maxime@cerno.tech>");
1086MODULE_AUTHOR("zhuxianbin <zhuxianbin@allwinnertech.com>");
1// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
2// Copyright (C) 2016-2018, Allwinner Technology CO., LTD.
3// Copyright (C) 2019-2020, Cerno
4
5#include <linux/bitfield.h>
6#include <linux/bug.h>
7#include <linux/clk.h>
8#include <linux/device.h>
9#include <linux/dma-direction.h>
10#include <linux/dma-mapping.h>
11#include <linux/err.h>
12#include <linux/errno.h>
13#include <linux/interrupt.h>
14#include <linux/iommu.h>
15#include <linux/iopoll.h>
16#include <linux/ioport.h>
17#include <linux/log2.h>
18#include <linux/module.h>
19#include <linux/of_platform.h>
20#include <linux/platform_device.h>
21#include <linux/pm.h>
22#include <linux/pm_runtime.h>
23#include <linux/reset.h>
24#include <linux/sizes.h>
25#include <linux/slab.h>
26#include <linux/spinlock.h>
27#include <linux/types.h>
28
29#define IOMMU_RESET_REG 0x010
30#define IOMMU_RESET_RELEASE_ALL 0xffffffff
31#define IOMMU_ENABLE_REG 0x020
32#define IOMMU_ENABLE_ENABLE BIT(0)
33
34#define IOMMU_BYPASS_REG 0x030
35#define IOMMU_AUTO_GATING_REG 0x040
36#define IOMMU_AUTO_GATING_ENABLE BIT(0)
37
38#define IOMMU_WBUF_CTRL_REG 0x044
39#define IOMMU_OOO_CTRL_REG 0x048
40#define IOMMU_4KB_BDY_PRT_CTRL_REG 0x04c
41#define IOMMU_TTB_REG 0x050
42#define IOMMU_TLB_ENABLE_REG 0x060
43#define IOMMU_TLB_PREFETCH_REG 0x070
44#define IOMMU_TLB_PREFETCH_MASTER_ENABLE(m) BIT(m)
45
46#define IOMMU_TLB_FLUSH_REG 0x080
47#define IOMMU_TLB_FLUSH_PTW_CACHE BIT(17)
48#define IOMMU_TLB_FLUSH_MACRO_TLB BIT(16)
49#define IOMMU_TLB_FLUSH_MICRO_TLB(i) (BIT(i) & GENMASK(5, 0))
50
51#define IOMMU_TLB_IVLD_ADDR_REG 0x090
52#define IOMMU_TLB_IVLD_ADDR_MASK_REG 0x094
53#define IOMMU_TLB_IVLD_ENABLE_REG 0x098
54#define IOMMU_TLB_IVLD_ENABLE_ENABLE BIT(0)
55
56#define IOMMU_PC_IVLD_ADDR_REG 0x0a0
57#define IOMMU_PC_IVLD_ENABLE_REG 0x0a8
58#define IOMMU_PC_IVLD_ENABLE_ENABLE BIT(0)
59
60#define IOMMU_DM_AUT_CTRL_REG(d) (0x0b0 + ((d) / 2) * 4)
61#define IOMMU_DM_AUT_CTRL_RD_UNAVAIL(d, m) (1 << (((d & 1) * 16) + ((m) * 2)))
62#define IOMMU_DM_AUT_CTRL_WR_UNAVAIL(d, m) (1 << (((d & 1) * 16) + ((m) * 2) + 1))
63
64#define IOMMU_DM_AUT_OVWT_REG 0x0d0
65#define IOMMU_INT_ENABLE_REG 0x100
66#define IOMMU_INT_CLR_REG 0x104
67#define IOMMU_INT_STA_REG 0x108
68#define IOMMU_INT_ERR_ADDR_REG(i) (0x110 + (i) * 4)
69#define IOMMU_INT_ERR_ADDR_L1_REG 0x130
70#define IOMMU_INT_ERR_ADDR_L2_REG 0x134
71#define IOMMU_INT_ERR_DATA_REG(i) (0x150 + (i) * 4)
72#define IOMMU_L1PG_INT_REG 0x0180
73#define IOMMU_L2PG_INT_REG 0x0184
74
75#define IOMMU_INT_INVALID_L2PG BIT(17)
76#define IOMMU_INT_INVALID_L1PG BIT(16)
77#define IOMMU_INT_MASTER_PERMISSION(m) BIT(m)
78#define IOMMU_INT_MASTER_MASK (IOMMU_INT_MASTER_PERMISSION(0) | \
79 IOMMU_INT_MASTER_PERMISSION(1) | \
80 IOMMU_INT_MASTER_PERMISSION(2) | \
81 IOMMU_INT_MASTER_PERMISSION(3) | \
82 IOMMU_INT_MASTER_PERMISSION(4) | \
83 IOMMU_INT_MASTER_PERMISSION(5))
84#define IOMMU_INT_MASK (IOMMU_INT_INVALID_L1PG | \
85 IOMMU_INT_INVALID_L2PG | \
86 IOMMU_INT_MASTER_MASK)
87
88#define PT_ENTRY_SIZE sizeof(u32)
89
90#define NUM_DT_ENTRIES 4096
91#define DT_SIZE (NUM_DT_ENTRIES * PT_ENTRY_SIZE)
92
93#define NUM_PT_ENTRIES 256
94#define PT_SIZE (NUM_PT_ENTRIES * PT_ENTRY_SIZE)
95
96#define SPAGE_SIZE 4096
97
98struct sun50i_iommu {
99 struct iommu_device iommu;
100
101 /* Lock to modify the IOMMU registers */
102 spinlock_t iommu_lock;
103
104 struct device *dev;
105 void __iomem *base;
106 struct reset_control *reset;
107 struct clk *clk;
108
109 struct iommu_domain *domain;
110 struct iommu_group *group;
111 struct kmem_cache *pt_pool;
112};
113
114struct sun50i_iommu_domain {
115 struct iommu_domain domain;
116
117 /* Number of devices attached to the domain */
118 refcount_t refcnt;
119
120 /* L1 Page Table */
121 u32 *dt;
122 dma_addr_t dt_dma;
123
124 struct sun50i_iommu *iommu;
125};
126
127static struct sun50i_iommu_domain *to_sun50i_domain(struct iommu_domain *domain)
128{
129 return container_of(domain, struct sun50i_iommu_domain, domain);
130}
131
132static struct sun50i_iommu *sun50i_iommu_from_dev(struct device *dev)
133{
134 return dev_iommu_priv_get(dev);
135}
136
137static u32 iommu_read(struct sun50i_iommu *iommu, u32 offset)
138{
139 return readl(iommu->base + offset);
140}
141
142static void iommu_write(struct sun50i_iommu *iommu, u32 offset, u32 value)
143{
144 writel(value, iommu->base + offset);
145}
146
147/*
148 * The Allwinner H6 IOMMU uses a 2-level page table.
149 *
150 * The first level is the usual Directory Table (DT), that consists of
151 * 4096 4-bytes Directory Table Entries (DTE), each pointing to a Page
152 * Table (PT).
153 *
154 * Each PT consits of 256 4-bytes Page Table Entries (PTE), each
155 * pointing to a 4kB page of physical memory.
156 *
157 * The IOMMU supports a single DT, pointed by the IOMMU_TTB_REG
158 * register that contains its physical address.
159 */
160
161#define SUN50I_IOVA_DTE_MASK GENMASK(31, 20)
162#define SUN50I_IOVA_PTE_MASK GENMASK(19, 12)
163#define SUN50I_IOVA_PAGE_MASK GENMASK(11, 0)
164
165static u32 sun50i_iova_get_dte_index(dma_addr_t iova)
166{
167 return FIELD_GET(SUN50I_IOVA_DTE_MASK, iova);
168}
169
170static u32 sun50i_iova_get_pte_index(dma_addr_t iova)
171{
172 return FIELD_GET(SUN50I_IOVA_PTE_MASK, iova);
173}
174
175static u32 sun50i_iova_get_page_offset(dma_addr_t iova)
176{
177 return FIELD_GET(SUN50I_IOVA_PAGE_MASK, iova);
178}
179
180/*
181 * Each Directory Table Entry has a Page Table address and a valid
182 * bit:
183
184 * +---------------------+-----------+-+
185 * | PT address | Reserved |V|
186 * +---------------------+-----------+-+
187 * 31:10 - Page Table address
188 * 9:2 - Reserved
189 * 1:0 - 1 if the entry is valid
190 */
191
192#define SUN50I_DTE_PT_ADDRESS_MASK GENMASK(31, 10)
193#define SUN50I_DTE_PT_ATTRS GENMASK(1, 0)
194#define SUN50I_DTE_PT_VALID 1
195
196static phys_addr_t sun50i_dte_get_pt_address(u32 dte)
197{
198 return (phys_addr_t)dte & SUN50I_DTE_PT_ADDRESS_MASK;
199}
200
201static bool sun50i_dte_is_pt_valid(u32 dte)
202{
203 return (dte & SUN50I_DTE_PT_ATTRS) == SUN50I_DTE_PT_VALID;
204}
205
206static u32 sun50i_mk_dte(dma_addr_t pt_dma)
207{
208 return (pt_dma & SUN50I_DTE_PT_ADDRESS_MASK) | SUN50I_DTE_PT_VALID;
209}
210
211/*
212 * Each PTE has a Page address, an authority index and a valid bit:
213 *
214 * +----------------+-----+-----+-----+---+-----+
215 * | Page address | Rsv | ACI | Rsv | V | Rsv |
216 * +----------------+-----+-----+-----+---+-----+
217 * 31:12 - Page address
218 * 11:8 - Reserved
219 * 7:4 - Authority Control Index
220 * 3:2 - Reserved
221 * 1 - 1 if the entry is valid
222 * 0 - Reserved
223 *
224 * The way permissions work is that the IOMMU has 16 "domains" that
225 * can be configured to give each masters either read or write
226 * permissions through the IOMMU_DM_AUT_CTRL_REG registers. The domain
227 * 0 seems like the default domain, and its permissions in the
228 * IOMMU_DM_AUT_CTRL_REG are only read-only, so it's not really
229 * useful to enforce any particular permission.
230 *
231 * Each page entry will then have a reference to the domain they are
232 * affected to, so that we can actually enforce them on a per-page
233 * basis.
234 *
235 * In order to make it work with the IOMMU framework, we will be using
236 * 4 different domains, starting at 1: RD_WR, RD, WR and NONE
237 * depending on the permission we want to enforce. Each domain will
238 * have each master setup in the same way, since the IOMMU framework
239 * doesn't seem to restrict page access on a per-device basis. And
240 * then we will use the relevant domain index when generating the page
241 * table entry depending on the permissions we want to be enforced.
242 */
243
244enum sun50i_iommu_aci {
245 SUN50I_IOMMU_ACI_DO_NOT_USE = 0,
246 SUN50I_IOMMU_ACI_NONE,
247 SUN50I_IOMMU_ACI_RD,
248 SUN50I_IOMMU_ACI_WR,
249 SUN50I_IOMMU_ACI_RD_WR,
250};
251
252#define SUN50I_PTE_PAGE_ADDRESS_MASK GENMASK(31, 12)
253#define SUN50I_PTE_ACI_MASK GENMASK(7, 4)
254#define SUN50I_PTE_PAGE_VALID BIT(1)
255
256static phys_addr_t sun50i_pte_get_page_address(u32 pte)
257{
258 return (phys_addr_t)pte & SUN50I_PTE_PAGE_ADDRESS_MASK;
259}
260
261static enum sun50i_iommu_aci sun50i_get_pte_aci(u32 pte)
262{
263 return FIELD_GET(SUN50I_PTE_ACI_MASK, pte);
264}
265
266static bool sun50i_pte_is_page_valid(u32 pte)
267{
268 return pte & SUN50I_PTE_PAGE_VALID;
269}
270
271static u32 sun50i_mk_pte(phys_addr_t page, int prot)
272{
273 enum sun50i_iommu_aci aci;
274 u32 flags = 0;
275
276 if ((prot & (IOMMU_READ | IOMMU_WRITE)) == (IOMMU_READ | IOMMU_WRITE))
277 aci = SUN50I_IOMMU_ACI_RD_WR;
278 else if (prot & IOMMU_READ)
279 aci = SUN50I_IOMMU_ACI_RD;
280 else if (prot & IOMMU_WRITE)
281 aci = SUN50I_IOMMU_ACI_WR;
282 else
283 aci = SUN50I_IOMMU_ACI_NONE;
284
285 flags |= FIELD_PREP(SUN50I_PTE_ACI_MASK, aci);
286 page &= SUN50I_PTE_PAGE_ADDRESS_MASK;
287 return page | flags | SUN50I_PTE_PAGE_VALID;
288}
289
290static void sun50i_table_flush(struct sun50i_iommu_domain *sun50i_domain,
291 void *vaddr, unsigned int count)
292{
293 struct sun50i_iommu *iommu = sun50i_domain->iommu;
294 dma_addr_t dma = virt_to_phys(vaddr);
295 size_t size = count * PT_ENTRY_SIZE;
296
297 dma_sync_single_for_device(iommu->dev, dma, size, DMA_TO_DEVICE);
298}
299
300static void sun50i_iommu_zap_iova(struct sun50i_iommu *iommu,
301 unsigned long iova)
302{
303 u32 reg;
304 int ret;
305
306 iommu_write(iommu, IOMMU_TLB_IVLD_ADDR_REG, iova);
307 iommu_write(iommu, IOMMU_TLB_IVLD_ADDR_MASK_REG, GENMASK(31, 12));
308 iommu_write(iommu, IOMMU_TLB_IVLD_ENABLE_REG,
309 IOMMU_TLB_IVLD_ENABLE_ENABLE);
310
311 ret = readl_poll_timeout_atomic(iommu->base + IOMMU_TLB_IVLD_ENABLE_REG,
312 reg, !reg, 1, 2000);
313 if (ret)
314 dev_warn(iommu->dev, "TLB invalidation timed out!\n");
315}
316
317static void sun50i_iommu_zap_ptw_cache(struct sun50i_iommu *iommu,
318 unsigned long iova)
319{
320 u32 reg;
321 int ret;
322
323 iommu_write(iommu, IOMMU_PC_IVLD_ADDR_REG, iova);
324 iommu_write(iommu, IOMMU_PC_IVLD_ENABLE_REG,
325 IOMMU_PC_IVLD_ENABLE_ENABLE);
326
327 ret = readl_poll_timeout_atomic(iommu->base + IOMMU_PC_IVLD_ENABLE_REG,
328 reg, !reg, 1, 2000);
329 if (ret)
330 dev_warn(iommu->dev, "PTW cache invalidation timed out!\n");
331}
332
333static void sun50i_iommu_zap_range(struct sun50i_iommu *iommu,
334 unsigned long iova, size_t size)
335{
336 assert_spin_locked(&iommu->iommu_lock);
337
338 iommu_write(iommu, IOMMU_AUTO_GATING_REG, 0);
339
340 sun50i_iommu_zap_iova(iommu, iova);
341 sun50i_iommu_zap_iova(iommu, iova + SPAGE_SIZE);
342 if (size > SPAGE_SIZE) {
343 sun50i_iommu_zap_iova(iommu, iova + size);
344 sun50i_iommu_zap_iova(iommu, iova + size + SPAGE_SIZE);
345 }
346 sun50i_iommu_zap_ptw_cache(iommu, iova);
347 sun50i_iommu_zap_ptw_cache(iommu, iova + SZ_1M);
348 if (size > SZ_1M) {
349 sun50i_iommu_zap_ptw_cache(iommu, iova + size);
350 sun50i_iommu_zap_ptw_cache(iommu, iova + size + SZ_1M);
351 }
352
353 iommu_write(iommu, IOMMU_AUTO_GATING_REG, IOMMU_AUTO_GATING_ENABLE);
354}
355
356static int sun50i_iommu_flush_all_tlb(struct sun50i_iommu *iommu)
357{
358 u32 reg;
359 int ret;
360
361 assert_spin_locked(&iommu->iommu_lock);
362
363 iommu_write(iommu,
364 IOMMU_TLB_FLUSH_REG,
365 IOMMU_TLB_FLUSH_PTW_CACHE |
366 IOMMU_TLB_FLUSH_MACRO_TLB |
367 IOMMU_TLB_FLUSH_MICRO_TLB(5) |
368 IOMMU_TLB_FLUSH_MICRO_TLB(4) |
369 IOMMU_TLB_FLUSH_MICRO_TLB(3) |
370 IOMMU_TLB_FLUSH_MICRO_TLB(2) |
371 IOMMU_TLB_FLUSH_MICRO_TLB(1) |
372 IOMMU_TLB_FLUSH_MICRO_TLB(0));
373
374 ret = readl_poll_timeout_atomic(iommu->base + IOMMU_TLB_FLUSH_REG,
375 reg, !reg,
376 1, 2000);
377 if (ret)
378 dev_warn(iommu->dev, "TLB Flush timed out!\n");
379
380 return ret;
381}
382
383static void sun50i_iommu_flush_iotlb_all(struct iommu_domain *domain)
384{
385 struct sun50i_iommu_domain *sun50i_domain = to_sun50i_domain(domain);
386 struct sun50i_iommu *iommu = sun50i_domain->iommu;
387 unsigned long flags;
388
389 /*
390 * At boot, we'll have a first call into .flush_iotlb_all right after
391 * .probe_device, and since we link our (single) domain to our iommu in
392 * the .attach_device callback, we don't have that pointer set.
393 *
394 * It shouldn't really be any trouble to ignore it though since we flush
395 * all caches as part of the device powerup.
396 */
397 if (!iommu)
398 return;
399
400 spin_lock_irqsave(&iommu->iommu_lock, flags);
401 sun50i_iommu_flush_all_tlb(iommu);
402 spin_unlock_irqrestore(&iommu->iommu_lock, flags);
403}
404
405static void sun50i_iommu_iotlb_sync_map(struct iommu_domain *domain,
406 unsigned long iova, size_t size)
407{
408 struct sun50i_iommu_domain *sun50i_domain = to_sun50i_domain(domain);
409 struct sun50i_iommu *iommu = sun50i_domain->iommu;
410 unsigned long flags;
411
412 spin_lock_irqsave(&iommu->iommu_lock, flags);
413 sun50i_iommu_zap_range(iommu, iova, size);
414 spin_unlock_irqrestore(&iommu->iommu_lock, flags);
415}
416
417static void sun50i_iommu_iotlb_sync(struct iommu_domain *domain,
418 struct iommu_iotlb_gather *gather)
419{
420 sun50i_iommu_flush_iotlb_all(domain);
421}
422
423static int sun50i_iommu_enable(struct sun50i_iommu *iommu)
424{
425 struct sun50i_iommu_domain *sun50i_domain;
426 unsigned long flags;
427 int ret;
428
429 if (!iommu->domain)
430 return 0;
431
432 sun50i_domain = to_sun50i_domain(iommu->domain);
433
434 ret = reset_control_deassert(iommu->reset);
435 if (ret)
436 return ret;
437
438 ret = clk_prepare_enable(iommu->clk);
439 if (ret)
440 goto err_reset_assert;
441
442 spin_lock_irqsave(&iommu->iommu_lock, flags);
443
444 iommu_write(iommu, IOMMU_TTB_REG, sun50i_domain->dt_dma);
445 iommu_write(iommu, IOMMU_TLB_PREFETCH_REG,
446 IOMMU_TLB_PREFETCH_MASTER_ENABLE(0) |
447 IOMMU_TLB_PREFETCH_MASTER_ENABLE(1) |
448 IOMMU_TLB_PREFETCH_MASTER_ENABLE(2) |
449 IOMMU_TLB_PREFETCH_MASTER_ENABLE(3) |
450 IOMMU_TLB_PREFETCH_MASTER_ENABLE(4) |
451 IOMMU_TLB_PREFETCH_MASTER_ENABLE(5));
452 iommu_write(iommu, IOMMU_INT_ENABLE_REG, IOMMU_INT_MASK);
453 iommu_write(iommu, IOMMU_DM_AUT_CTRL_REG(SUN50I_IOMMU_ACI_NONE),
454 IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 0) |
455 IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 0) |
456 IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 1) |
457 IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 1) |
458 IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 2) |
459 IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 2) |
460 IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 3) |
461 IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 3) |
462 IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 4) |
463 IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 4) |
464 IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 5) |
465 IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 5));
466
467 iommu_write(iommu, IOMMU_DM_AUT_CTRL_REG(SUN50I_IOMMU_ACI_RD),
468 IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_RD, 0) |
469 IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_RD, 1) |
470 IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_RD, 2) |
471 IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_RD, 3) |
472 IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_RD, 4) |
473 IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_RD, 5));
474
475 iommu_write(iommu, IOMMU_DM_AUT_CTRL_REG(SUN50I_IOMMU_ACI_WR),
476 IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_WR, 0) |
477 IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_WR, 1) |
478 IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_WR, 2) |
479 IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_WR, 3) |
480 IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_WR, 4) |
481 IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_WR, 5));
482
483 ret = sun50i_iommu_flush_all_tlb(iommu);
484 if (ret) {
485 spin_unlock_irqrestore(&iommu->iommu_lock, flags);
486 goto err_clk_disable;
487 }
488
489 iommu_write(iommu, IOMMU_AUTO_GATING_REG, IOMMU_AUTO_GATING_ENABLE);
490 iommu_write(iommu, IOMMU_ENABLE_REG, IOMMU_ENABLE_ENABLE);
491
492 spin_unlock_irqrestore(&iommu->iommu_lock, flags);
493
494 return 0;
495
496err_clk_disable:
497 clk_disable_unprepare(iommu->clk);
498
499err_reset_assert:
500 reset_control_assert(iommu->reset);
501
502 return ret;
503}
504
505static void sun50i_iommu_disable(struct sun50i_iommu *iommu)
506{
507 unsigned long flags;
508
509 spin_lock_irqsave(&iommu->iommu_lock, flags);
510
511 iommu_write(iommu, IOMMU_ENABLE_REG, 0);
512 iommu_write(iommu, IOMMU_TTB_REG, 0);
513
514 spin_unlock_irqrestore(&iommu->iommu_lock, flags);
515
516 clk_disable_unprepare(iommu->clk);
517 reset_control_assert(iommu->reset);
518}
519
520static void *sun50i_iommu_alloc_page_table(struct sun50i_iommu *iommu,
521 gfp_t gfp)
522{
523 dma_addr_t pt_dma;
524 u32 *page_table;
525
526 page_table = kmem_cache_zalloc(iommu->pt_pool, gfp);
527 if (!page_table)
528 return ERR_PTR(-ENOMEM);
529
530 pt_dma = dma_map_single(iommu->dev, page_table, PT_SIZE, DMA_TO_DEVICE);
531 if (dma_mapping_error(iommu->dev, pt_dma)) {
532 dev_err(iommu->dev, "Couldn't map L2 Page Table\n");
533 kmem_cache_free(iommu->pt_pool, page_table);
534 return ERR_PTR(-ENOMEM);
535 }
536
537 /* We rely on the physical address and DMA address being the same */
538 WARN_ON(pt_dma != virt_to_phys(page_table));
539
540 return page_table;
541}
542
543static void sun50i_iommu_free_page_table(struct sun50i_iommu *iommu,
544 u32 *page_table)
545{
546 phys_addr_t pt_phys = virt_to_phys(page_table);
547
548 dma_unmap_single(iommu->dev, pt_phys, PT_SIZE, DMA_TO_DEVICE);
549 kmem_cache_free(iommu->pt_pool, page_table);
550}
551
552static u32 *sun50i_dte_get_page_table(struct sun50i_iommu_domain *sun50i_domain,
553 dma_addr_t iova, gfp_t gfp)
554{
555 struct sun50i_iommu *iommu = sun50i_domain->iommu;
556 u32 *page_table;
557 u32 *dte_addr;
558 u32 old_dte;
559 u32 dte;
560
561 dte_addr = &sun50i_domain->dt[sun50i_iova_get_dte_index(iova)];
562 dte = *dte_addr;
563 if (sun50i_dte_is_pt_valid(dte)) {
564 phys_addr_t pt_phys = sun50i_dte_get_pt_address(dte);
565 return (u32 *)phys_to_virt(pt_phys);
566 }
567
568 page_table = sun50i_iommu_alloc_page_table(iommu, gfp);
569 if (IS_ERR(page_table))
570 return page_table;
571
572 dte = sun50i_mk_dte(virt_to_phys(page_table));
573 old_dte = cmpxchg(dte_addr, 0, dte);
574 if (old_dte) {
575 phys_addr_t installed_pt_phys =
576 sun50i_dte_get_pt_address(old_dte);
577 u32 *installed_pt = phys_to_virt(installed_pt_phys);
578 u32 *drop_pt = page_table;
579
580 page_table = installed_pt;
581 dte = old_dte;
582 sun50i_iommu_free_page_table(iommu, drop_pt);
583 }
584
585 sun50i_table_flush(sun50i_domain, page_table, NUM_PT_ENTRIES);
586 sun50i_table_flush(sun50i_domain, dte_addr, 1);
587
588 return page_table;
589}
590
591static int sun50i_iommu_map(struct iommu_domain *domain, unsigned long iova,
592 phys_addr_t paddr, size_t size, int prot, gfp_t gfp)
593{
594 struct sun50i_iommu_domain *sun50i_domain = to_sun50i_domain(domain);
595 struct sun50i_iommu *iommu = sun50i_domain->iommu;
596 u32 pte_index;
597 u32 *page_table, *pte_addr;
598 int ret = 0;
599
600 page_table = sun50i_dte_get_page_table(sun50i_domain, iova, gfp);
601 if (IS_ERR(page_table)) {
602 ret = PTR_ERR(page_table);
603 goto out;
604 }
605
606 pte_index = sun50i_iova_get_pte_index(iova);
607 pte_addr = &page_table[pte_index];
608 if (unlikely(sun50i_pte_is_page_valid(*pte_addr))) {
609 phys_addr_t page_phys = sun50i_pte_get_page_address(*pte_addr);
610 dev_err(iommu->dev,
611 "iova %pad already mapped to %pa cannot remap to %pa prot: %#x\n",
612 &iova, &page_phys, &paddr, prot);
613 ret = -EBUSY;
614 goto out;
615 }
616
617 *pte_addr = sun50i_mk_pte(paddr, prot);
618 sun50i_table_flush(sun50i_domain, pte_addr, 1);
619
620out:
621 return ret;
622}
623
624static size_t sun50i_iommu_unmap(struct iommu_domain *domain, unsigned long iova,
625 size_t size, struct iommu_iotlb_gather *gather)
626{
627 struct sun50i_iommu_domain *sun50i_domain = to_sun50i_domain(domain);
628 phys_addr_t pt_phys;
629 u32 *pte_addr;
630 u32 dte;
631
632 dte = sun50i_domain->dt[sun50i_iova_get_dte_index(iova)];
633 if (!sun50i_dte_is_pt_valid(dte))
634 return 0;
635
636 pt_phys = sun50i_dte_get_pt_address(dte);
637 pte_addr = (u32 *)phys_to_virt(pt_phys) + sun50i_iova_get_pte_index(iova);
638
639 if (!sun50i_pte_is_page_valid(*pte_addr))
640 return 0;
641
642 memset(pte_addr, 0, sizeof(*pte_addr));
643 sun50i_table_flush(sun50i_domain, pte_addr, 1);
644
645 return SZ_4K;
646}
647
648static phys_addr_t sun50i_iommu_iova_to_phys(struct iommu_domain *domain,
649 dma_addr_t iova)
650{
651 struct sun50i_iommu_domain *sun50i_domain = to_sun50i_domain(domain);
652 phys_addr_t pt_phys;
653 u32 *page_table;
654 u32 dte, pte;
655
656 dte = sun50i_domain->dt[sun50i_iova_get_dte_index(iova)];
657 if (!sun50i_dte_is_pt_valid(dte))
658 return 0;
659
660 pt_phys = sun50i_dte_get_pt_address(dte);
661 page_table = (u32 *)phys_to_virt(pt_phys);
662 pte = page_table[sun50i_iova_get_pte_index(iova)];
663 if (!sun50i_pte_is_page_valid(pte))
664 return 0;
665
666 return sun50i_pte_get_page_address(pte) +
667 sun50i_iova_get_page_offset(iova);
668}
669
670static struct iommu_domain *sun50i_iommu_domain_alloc(unsigned type)
671{
672 struct sun50i_iommu_domain *sun50i_domain;
673
674 if (type != IOMMU_DOMAIN_DMA &&
675 type != IOMMU_DOMAIN_UNMANAGED)
676 return NULL;
677
678 sun50i_domain = kzalloc(sizeof(*sun50i_domain), GFP_KERNEL);
679 if (!sun50i_domain)
680 return NULL;
681
682 sun50i_domain->dt = (u32 *)__get_free_pages(GFP_KERNEL | __GFP_ZERO,
683 get_order(DT_SIZE));
684 if (!sun50i_domain->dt)
685 goto err_free_domain;
686
687 refcount_set(&sun50i_domain->refcnt, 1);
688
689 sun50i_domain->domain.geometry.aperture_start = 0;
690 sun50i_domain->domain.geometry.aperture_end = DMA_BIT_MASK(32);
691 sun50i_domain->domain.geometry.force_aperture = true;
692
693 return &sun50i_domain->domain;
694
695err_free_domain:
696 kfree(sun50i_domain);
697
698 return NULL;
699}
700
701static void sun50i_iommu_domain_free(struct iommu_domain *domain)
702{
703 struct sun50i_iommu_domain *sun50i_domain = to_sun50i_domain(domain);
704
705 free_pages((unsigned long)sun50i_domain->dt, get_order(DT_SIZE));
706 sun50i_domain->dt = NULL;
707
708 kfree(sun50i_domain);
709}
710
711static int sun50i_iommu_attach_domain(struct sun50i_iommu *iommu,
712 struct sun50i_iommu_domain *sun50i_domain)
713{
714 iommu->domain = &sun50i_domain->domain;
715 sun50i_domain->iommu = iommu;
716
717 sun50i_domain->dt_dma = dma_map_single(iommu->dev, sun50i_domain->dt,
718 DT_SIZE, DMA_TO_DEVICE);
719 if (dma_mapping_error(iommu->dev, sun50i_domain->dt_dma)) {
720 dev_err(iommu->dev, "Couldn't map L1 Page Table\n");
721 return -ENOMEM;
722 }
723
724 return sun50i_iommu_enable(iommu);
725}
726
727static void sun50i_iommu_detach_domain(struct sun50i_iommu *iommu,
728 struct sun50i_iommu_domain *sun50i_domain)
729{
730 unsigned int i;
731
732 for (i = 0; i < NUM_DT_ENTRIES; i++) {
733 phys_addr_t pt_phys;
734 u32 *page_table;
735 u32 *dte_addr;
736 u32 dte;
737
738 dte_addr = &sun50i_domain->dt[i];
739 dte = *dte_addr;
740 if (!sun50i_dte_is_pt_valid(dte))
741 continue;
742
743 memset(dte_addr, 0, sizeof(*dte_addr));
744 sun50i_table_flush(sun50i_domain, dte_addr, 1);
745
746 pt_phys = sun50i_dte_get_pt_address(dte);
747 page_table = phys_to_virt(pt_phys);
748 sun50i_iommu_free_page_table(iommu, page_table);
749 }
750
751
752 sun50i_iommu_disable(iommu);
753
754 dma_unmap_single(iommu->dev, virt_to_phys(sun50i_domain->dt),
755 DT_SIZE, DMA_TO_DEVICE);
756
757 iommu->domain = NULL;
758}
759
760static void sun50i_iommu_detach_device(struct iommu_domain *domain,
761 struct device *dev)
762{
763 struct sun50i_iommu_domain *sun50i_domain = to_sun50i_domain(domain);
764 struct sun50i_iommu *iommu = dev_iommu_priv_get(dev);
765
766 dev_dbg(dev, "Detaching from IOMMU domain\n");
767
768 if (iommu->domain != domain)
769 return;
770
771 if (refcount_dec_and_test(&sun50i_domain->refcnt))
772 sun50i_iommu_detach_domain(iommu, sun50i_domain);
773}
774
775static int sun50i_iommu_attach_device(struct iommu_domain *domain,
776 struct device *dev)
777{
778 struct sun50i_iommu_domain *sun50i_domain = to_sun50i_domain(domain);
779 struct sun50i_iommu *iommu;
780
781 iommu = sun50i_iommu_from_dev(dev);
782 if (!iommu)
783 return -ENODEV;
784
785 dev_dbg(dev, "Attaching to IOMMU domain\n");
786
787 refcount_inc(&sun50i_domain->refcnt);
788
789 if (iommu->domain == domain)
790 return 0;
791
792 if (iommu->domain)
793 sun50i_iommu_detach_device(iommu->domain, dev);
794
795 sun50i_iommu_attach_domain(iommu, sun50i_domain);
796
797 return 0;
798}
799
800static struct iommu_device *sun50i_iommu_probe_device(struct device *dev)
801{
802 struct sun50i_iommu *iommu;
803
804 iommu = sun50i_iommu_from_dev(dev);
805 if (!iommu)
806 return ERR_PTR(-ENODEV);
807
808 return &iommu->iommu;
809}
810
811static struct iommu_group *sun50i_iommu_device_group(struct device *dev)
812{
813 struct sun50i_iommu *iommu = sun50i_iommu_from_dev(dev);
814
815 return iommu_group_ref_get(iommu->group);
816}
817
818static int sun50i_iommu_of_xlate(struct device *dev,
819 struct of_phandle_args *args)
820{
821 struct platform_device *iommu_pdev = of_find_device_by_node(args->np);
822 unsigned id = args->args[0];
823
824 dev_iommu_priv_set(dev, platform_get_drvdata(iommu_pdev));
825
826 return iommu_fwspec_add_ids(dev, &id, 1);
827}
828
829static const struct iommu_ops sun50i_iommu_ops = {
830 .pgsize_bitmap = SZ_4K,
831 .device_group = sun50i_iommu_device_group,
832 .domain_alloc = sun50i_iommu_domain_alloc,
833 .of_xlate = sun50i_iommu_of_xlate,
834 .probe_device = sun50i_iommu_probe_device,
835 .default_domain_ops = &(const struct iommu_domain_ops) {
836 .attach_dev = sun50i_iommu_attach_device,
837 .detach_dev = sun50i_iommu_detach_device,
838 .flush_iotlb_all = sun50i_iommu_flush_iotlb_all,
839 .iotlb_sync_map = sun50i_iommu_iotlb_sync_map,
840 .iotlb_sync = sun50i_iommu_iotlb_sync,
841 .iova_to_phys = sun50i_iommu_iova_to_phys,
842 .map = sun50i_iommu_map,
843 .unmap = sun50i_iommu_unmap,
844 .free = sun50i_iommu_domain_free,
845 }
846};
847
848static void sun50i_iommu_report_fault(struct sun50i_iommu *iommu,
849 unsigned master, phys_addr_t iova,
850 unsigned prot)
851{
852 dev_err(iommu->dev, "Page fault for %pad (master %d, dir %s)\n",
853 &iova, master, (prot == IOMMU_FAULT_WRITE) ? "wr" : "rd");
854
855 if (iommu->domain)
856 report_iommu_fault(iommu->domain, iommu->dev, iova, prot);
857 else
858 dev_err(iommu->dev, "Page fault while iommu not attached to any domain?\n");
859
860 sun50i_iommu_zap_range(iommu, iova, SPAGE_SIZE);
861}
862
863static phys_addr_t sun50i_iommu_handle_pt_irq(struct sun50i_iommu *iommu,
864 unsigned addr_reg,
865 unsigned blame_reg)
866{
867 phys_addr_t iova;
868 unsigned master;
869 u32 blame;
870
871 assert_spin_locked(&iommu->iommu_lock);
872
873 iova = iommu_read(iommu, addr_reg);
874 blame = iommu_read(iommu, blame_reg);
875 master = ilog2(blame & IOMMU_INT_MASTER_MASK);
876
877 /*
878 * If the address is not in the page table, we can't get what
879 * operation triggered the fault. Assume it's a read
880 * operation.
881 */
882 sun50i_iommu_report_fault(iommu, master, iova, IOMMU_FAULT_READ);
883
884 return iova;
885}
886
887static phys_addr_t sun50i_iommu_handle_perm_irq(struct sun50i_iommu *iommu)
888{
889 enum sun50i_iommu_aci aci;
890 phys_addr_t iova;
891 unsigned master;
892 unsigned dir;
893 u32 blame;
894
895 assert_spin_locked(&iommu->iommu_lock);
896
897 blame = iommu_read(iommu, IOMMU_INT_STA_REG);
898 master = ilog2(blame & IOMMU_INT_MASTER_MASK);
899 iova = iommu_read(iommu, IOMMU_INT_ERR_ADDR_REG(master));
900 aci = sun50i_get_pte_aci(iommu_read(iommu,
901 IOMMU_INT_ERR_DATA_REG(master)));
902
903 switch (aci) {
904 /*
905 * If we are in the read-only domain, then it means we
906 * tried to write.
907 */
908 case SUN50I_IOMMU_ACI_RD:
909 dir = IOMMU_FAULT_WRITE;
910 break;
911
912 /*
913 * If we are in the write-only domain, then it means
914 * we tried to read.
915 */
916 case SUN50I_IOMMU_ACI_WR:
917
918 /*
919 * If we are in the domain without any permission, we
920 * can't really tell. Let's default to a read
921 * operation.
922 */
923 case SUN50I_IOMMU_ACI_NONE:
924
925 /* WTF? */
926 case SUN50I_IOMMU_ACI_RD_WR:
927 default:
928 dir = IOMMU_FAULT_READ;
929 break;
930 }
931
932 /*
933 * If the address is not in the page table, we can't get what
934 * operation triggered the fault. Assume it's a read
935 * operation.
936 */
937 sun50i_iommu_report_fault(iommu, master, iova, dir);
938
939 return iova;
940}
941
942static irqreturn_t sun50i_iommu_irq(int irq, void *dev_id)
943{
944 u32 status, l1_status, l2_status, resets;
945 struct sun50i_iommu *iommu = dev_id;
946
947 spin_lock(&iommu->iommu_lock);
948
949 status = iommu_read(iommu, IOMMU_INT_STA_REG);
950 if (!(status & IOMMU_INT_MASK)) {
951 spin_unlock(&iommu->iommu_lock);
952 return IRQ_NONE;
953 }
954
955 l1_status = iommu_read(iommu, IOMMU_L1PG_INT_REG);
956 l2_status = iommu_read(iommu, IOMMU_L2PG_INT_REG);
957
958 if (status & IOMMU_INT_INVALID_L2PG)
959 sun50i_iommu_handle_pt_irq(iommu,
960 IOMMU_INT_ERR_ADDR_L2_REG,
961 IOMMU_L2PG_INT_REG);
962 else if (status & IOMMU_INT_INVALID_L1PG)
963 sun50i_iommu_handle_pt_irq(iommu,
964 IOMMU_INT_ERR_ADDR_L1_REG,
965 IOMMU_L1PG_INT_REG);
966 else
967 sun50i_iommu_handle_perm_irq(iommu);
968
969 iommu_write(iommu, IOMMU_INT_CLR_REG, status);
970
971 resets = (status | l1_status | l2_status) & IOMMU_INT_MASTER_MASK;
972 iommu_write(iommu, IOMMU_RESET_REG, ~resets);
973 iommu_write(iommu, IOMMU_RESET_REG, IOMMU_RESET_RELEASE_ALL);
974
975 spin_unlock(&iommu->iommu_lock);
976
977 return IRQ_HANDLED;
978}
979
980static int sun50i_iommu_probe(struct platform_device *pdev)
981{
982 struct sun50i_iommu *iommu;
983 int ret, irq;
984
985 iommu = devm_kzalloc(&pdev->dev, sizeof(*iommu), GFP_KERNEL);
986 if (!iommu)
987 return -ENOMEM;
988 spin_lock_init(&iommu->iommu_lock);
989 platform_set_drvdata(pdev, iommu);
990 iommu->dev = &pdev->dev;
991
992 iommu->pt_pool = kmem_cache_create(dev_name(&pdev->dev),
993 PT_SIZE, PT_SIZE,
994 SLAB_HWCACHE_ALIGN,
995 NULL);
996 if (!iommu->pt_pool)
997 return -ENOMEM;
998
999 iommu->group = iommu_group_alloc();
1000 if (IS_ERR(iommu->group)) {
1001 ret = PTR_ERR(iommu->group);
1002 goto err_free_cache;
1003 }
1004
1005 iommu->base = devm_platform_ioremap_resource(pdev, 0);
1006 if (IS_ERR(iommu->base)) {
1007 ret = PTR_ERR(iommu->base);
1008 goto err_free_group;
1009 }
1010
1011 irq = platform_get_irq(pdev, 0);
1012 if (irq < 0) {
1013 ret = irq;
1014 goto err_free_group;
1015 }
1016
1017 iommu->clk = devm_clk_get(&pdev->dev, NULL);
1018 if (IS_ERR(iommu->clk)) {
1019 dev_err(&pdev->dev, "Couldn't get our clock.\n");
1020 ret = PTR_ERR(iommu->clk);
1021 goto err_free_group;
1022 }
1023
1024 iommu->reset = devm_reset_control_get(&pdev->dev, NULL);
1025 if (IS_ERR(iommu->reset)) {
1026 dev_err(&pdev->dev, "Couldn't get our reset line.\n");
1027 ret = PTR_ERR(iommu->reset);
1028 goto err_free_group;
1029 }
1030
1031 ret = iommu_device_sysfs_add(&iommu->iommu, &pdev->dev,
1032 NULL, dev_name(&pdev->dev));
1033 if (ret)
1034 goto err_free_group;
1035
1036 ret = iommu_device_register(&iommu->iommu, &sun50i_iommu_ops, &pdev->dev);
1037 if (ret)
1038 goto err_remove_sysfs;
1039
1040 ret = devm_request_irq(&pdev->dev, irq, sun50i_iommu_irq, 0,
1041 dev_name(&pdev->dev), iommu);
1042 if (ret < 0)
1043 goto err_unregister;
1044
1045 return 0;
1046
1047err_unregister:
1048 iommu_device_unregister(&iommu->iommu);
1049
1050err_remove_sysfs:
1051 iommu_device_sysfs_remove(&iommu->iommu);
1052
1053err_free_group:
1054 iommu_group_put(iommu->group);
1055
1056err_free_cache:
1057 kmem_cache_destroy(iommu->pt_pool);
1058
1059 return ret;
1060}
1061
1062static const struct of_device_id sun50i_iommu_dt[] = {
1063 { .compatible = "allwinner,sun50i-h6-iommu", },
1064 { /* sentinel */ },
1065};
1066MODULE_DEVICE_TABLE(of, sun50i_iommu_dt);
1067
1068static struct platform_driver sun50i_iommu_driver = {
1069 .driver = {
1070 .name = "sun50i-iommu",
1071 .of_match_table = sun50i_iommu_dt,
1072 .suppress_bind_attrs = true,
1073 }
1074};
1075builtin_platform_driver_probe(sun50i_iommu_driver, sun50i_iommu_probe);
1076
1077MODULE_DESCRIPTION("Allwinner H6 IOMMU driver");
1078MODULE_AUTHOR("Maxime Ripard <maxime@cerno.tech>");
1079MODULE_AUTHOR("zhuxianbin <zhuxianbin@allwinnertech.com>");
1080MODULE_LICENSE("Dual BSD/GPL");