Loading...
Note: File does not exist in v3.1.
1/*
2 * Copyright (C) 2016 Etnaviv Project
3 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms of the GNU General Public License version 2 as published by
6 * the Free Software Foundation.
7 *
8 * This program is distributed in the hope that it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
11 * more details.
12 *
13 * You should have received a copy of the GNU General Public License along with
14 * this program. If not, see <http://www.gnu.org/licenses/>.
15 */
16
17#include <linux/platform_device.h>
18#include <linux/sizes.h>
19#include <linux/slab.h>
20#include <linux/dma-mapping.h>
21#include <linux/bitops.h>
22
23#include "etnaviv_cmdbuf.h"
24#include "etnaviv_gpu.h"
25#include "etnaviv_mmu.h"
26#include "etnaviv_iommu.h"
27#include "state.xml.h"
28#include "state_hi.xml.h"
29
30#define MMUv2_PTE_PRESENT BIT(0)
31#define MMUv2_PTE_EXCEPTION BIT(1)
32#define MMUv2_PTE_WRITEABLE BIT(2)
33
34#define MMUv2_MTLB_MASK 0xffc00000
35#define MMUv2_MTLB_SHIFT 22
36#define MMUv2_STLB_MASK 0x003ff000
37#define MMUv2_STLB_SHIFT 12
38
39#define MMUv2_MAX_STLB_ENTRIES 1024
40
41struct etnaviv_iommuv2_domain {
42 struct etnaviv_iommu_domain base;
43 /* P(age) T(able) A(rray) */
44 u64 *pta_cpu;
45 dma_addr_t pta_dma;
46 /* M(aster) TLB aka first level pagetable */
47 u32 *mtlb_cpu;
48 dma_addr_t mtlb_dma;
49 /* S(lave) TLB aka second level pagetable */
50 u32 *stlb_cpu[1024];
51 dma_addr_t stlb_dma[1024];
52};
53
54static struct etnaviv_iommuv2_domain *
55to_etnaviv_domain(struct etnaviv_iommu_domain *domain)
56{
57 return container_of(domain, struct etnaviv_iommuv2_domain, base);
58}
59
60static int etnaviv_iommuv2_map(struct etnaviv_iommu_domain *domain,
61 unsigned long iova, phys_addr_t paddr,
62 size_t size, int prot)
63{
64 struct etnaviv_iommuv2_domain *etnaviv_domain =
65 to_etnaviv_domain(domain);
66 int mtlb_entry, stlb_entry;
67 u32 entry = (u32)paddr | MMUv2_PTE_PRESENT;
68
69 if (size != SZ_4K)
70 return -EINVAL;
71
72 if (prot & ETNAVIV_PROT_WRITE)
73 entry |= MMUv2_PTE_WRITEABLE;
74
75 mtlb_entry = (iova & MMUv2_MTLB_MASK) >> MMUv2_MTLB_SHIFT;
76 stlb_entry = (iova & MMUv2_STLB_MASK) >> MMUv2_STLB_SHIFT;
77
78 etnaviv_domain->stlb_cpu[mtlb_entry][stlb_entry] = entry;
79
80 return 0;
81}
82
83static size_t etnaviv_iommuv2_unmap(struct etnaviv_iommu_domain *domain,
84 unsigned long iova, size_t size)
85{
86 struct etnaviv_iommuv2_domain *etnaviv_domain =
87 to_etnaviv_domain(domain);
88 int mtlb_entry, stlb_entry;
89
90 if (size != SZ_4K)
91 return -EINVAL;
92
93 mtlb_entry = (iova & MMUv2_MTLB_MASK) >> MMUv2_MTLB_SHIFT;
94 stlb_entry = (iova & MMUv2_STLB_MASK) >> MMUv2_STLB_SHIFT;
95
96 etnaviv_domain->stlb_cpu[mtlb_entry][stlb_entry] = MMUv2_PTE_EXCEPTION;
97
98 return SZ_4K;
99}
100
101static int etnaviv_iommuv2_init(struct etnaviv_iommuv2_domain *etnaviv_domain)
102{
103 u32 *p;
104 int ret, i, j;
105
106 /* allocate scratch page */
107 etnaviv_domain->base.bad_page_cpu = dma_alloc_coherent(
108 etnaviv_domain->base.dev,
109 SZ_4K,
110 &etnaviv_domain->base.bad_page_dma,
111 GFP_KERNEL);
112 if (!etnaviv_domain->base.bad_page_cpu) {
113 ret = -ENOMEM;
114 goto fail_mem;
115 }
116 p = etnaviv_domain->base.bad_page_cpu;
117 for (i = 0; i < SZ_4K / 4; i++)
118 *p++ = 0xdead55aa;
119
120 etnaviv_domain->pta_cpu = dma_alloc_coherent(etnaviv_domain->base.dev,
121 SZ_4K,
122 &etnaviv_domain->pta_dma,
123 GFP_KERNEL);
124 if (!etnaviv_domain->pta_cpu) {
125 ret = -ENOMEM;
126 goto fail_mem;
127 }
128
129 etnaviv_domain->mtlb_cpu = dma_alloc_coherent(etnaviv_domain->base.dev,
130 SZ_4K,
131 &etnaviv_domain->mtlb_dma,
132 GFP_KERNEL);
133 if (!etnaviv_domain->mtlb_cpu) {
134 ret = -ENOMEM;
135 goto fail_mem;
136 }
137
138 /* pre-populate STLB pages (may want to switch to on-demand later) */
139 for (i = 0; i < MMUv2_MAX_STLB_ENTRIES; i++) {
140 etnaviv_domain->stlb_cpu[i] =
141 dma_alloc_coherent(etnaviv_domain->base.dev,
142 SZ_4K,
143 &etnaviv_domain->stlb_dma[i],
144 GFP_KERNEL);
145 if (!etnaviv_domain->stlb_cpu[i]) {
146 ret = -ENOMEM;
147 goto fail_mem;
148 }
149 p = etnaviv_domain->stlb_cpu[i];
150 for (j = 0; j < SZ_4K / 4; j++)
151 *p++ = MMUv2_PTE_EXCEPTION;
152
153 etnaviv_domain->mtlb_cpu[i] = etnaviv_domain->stlb_dma[i] |
154 MMUv2_PTE_PRESENT;
155 }
156
157 return 0;
158
159fail_mem:
160 if (etnaviv_domain->base.bad_page_cpu)
161 dma_free_coherent(etnaviv_domain->base.dev, SZ_4K,
162 etnaviv_domain->base.bad_page_cpu,
163 etnaviv_domain->base.bad_page_dma);
164
165 if (etnaviv_domain->pta_cpu)
166 dma_free_coherent(etnaviv_domain->base.dev, SZ_4K,
167 etnaviv_domain->pta_cpu,
168 etnaviv_domain->pta_dma);
169
170 if (etnaviv_domain->mtlb_cpu)
171 dma_free_coherent(etnaviv_domain->base.dev, SZ_4K,
172 etnaviv_domain->mtlb_cpu,
173 etnaviv_domain->mtlb_dma);
174
175 for (i = 0; i < MMUv2_MAX_STLB_ENTRIES; i++) {
176 if (etnaviv_domain->stlb_cpu[i])
177 dma_free_coherent(etnaviv_domain->base.dev, SZ_4K,
178 etnaviv_domain->stlb_cpu[i],
179 etnaviv_domain->stlb_dma[i]);
180 }
181
182 return ret;
183}
184
185static void etnaviv_iommuv2_domain_free(struct etnaviv_iommu_domain *domain)
186{
187 struct etnaviv_iommuv2_domain *etnaviv_domain =
188 to_etnaviv_domain(domain);
189 int i;
190
191 dma_free_coherent(etnaviv_domain->base.dev, SZ_4K,
192 etnaviv_domain->base.bad_page_cpu,
193 etnaviv_domain->base.bad_page_dma);
194
195 dma_free_coherent(etnaviv_domain->base.dev, SZ_4K,
196 etnaviv_domain->pta_cpu,
197 etnaviv_domain->pta_dma);
198
199 dma_free_coherent(etnaviv_domain->base.dev, SZ_4K,
200 etnaviv_domain->mtlb_cpu,
201 etnaviv_domain->mtlb_dma);
202
203 for (i = 0; i < MMUv2_MAX_STLB_ENTRIES; i++) {
204 if (etnaviv_domain->stlb_cpu[i])
205 dma_free_coherent(etnaviv_domain->base.dev, SZ_4K,
206 etnaviv_domain->stlb_cpu[i],
207 etnaviv_domain->stlb_dma[i]);
208 }
209
210 vfree(etnaviv_domain);
211}
212
213static size_t etnaviv_iommuv2_dump_size(struct etnaviv_iommu_domain *domain)
214{
215 struct etnaviv_iommuv2_domain *etnaviv_domain =
216 to_etnaviv_domain(domain);
217 size_t dump_size = SZ_4K;
218 int i;
219
220 for (i = 0; i < MMUv2_MAX_STLB_ENTRIES; i++)
221 if (etnaviv_domain->mtlb_cpu[i] & MMUv2_PTE_PRESENT)
222 dump_size += SZ_4K;
223
224 return dump_size;
225}
226
227static void etnaviv_iommuv2_dump(struct etnaviv_iommu_domain *domain, void *buf)
228{
229 struct etnaviv_iommuv2_domain *etnaviv_domain =
230 to_etnaviv_domain(domain);
231 int i;
232
233 memcpy(buf, etnaviv_domain->mtlb_cpu, SZ_4K);
234 buf += SZ_4K;
235 for (i = 0; i < MMUv2_MAX_STLB_ENTRIES; i++, buf += SZ_4K)
236 if (etnaviv_domain->mtlb_cpu[i] & MMUv2_PTE_PRESENT)
237 memcpy(buf, etnaviv_domain->stlb_cpu[i], SZ_4K);
238}
239
240static void etnaviv_iommuv2_restore_nonsec(struct etnaviv_gpu *gpu)
241{
242 struct etnaviv_iommuv2_domain *etnaviv_domain =
243 to_etnaviv_domain(gpu->mmu->domain);
244 u16 prefetch;
245
246 /* If the MMU is already enabled the state is still there. */
247 if (gpu_read(gpu, VIVS_MMUv2_CONTROL) & VIVS_MMUv2_CONTROL_ENABLE)
248 return;
249
250 prefetch = etnaviv_buffer_config_mmuv2(gpu,
251 (u32)etnaviv_domain->mtlb_dma,
252 (u32)etnaviv_domain->base.bad_page_dma);
253 etnaviv_gpu_start_fe(gpu, (u32)etnaviv_cmdbuf_get_pa(&gpu->buffer),
254 prefetch);
255 etnaviv_gpu_wait_idle(gpu, 100);
256
257 gpu_write(gpu, VIVS_MMUv2_CONTROL, VIVS_MMUv2_CONTROL_ENABLE);
258}
259
260static void etnaviv_iommuv2_restore_sec(struct etnaviv_gpu *gpu)
261{
262 struct etnaviv_iommuv2_domain *etnaviv_domain =
263 to_etnaviv_domain(gpu->mmu->domain);
264 u16 prefetch;
265
266 /* If the MMU is already enabled the state is still there. */
267 if (gpu_read(gpu, VIVS_MMUv2_SEC_CONTROL) & VIVS_MMUv2_SEC_CONTROL_ENABLE)
268 return;
269
270 gpu_write(gpu, VIVS_MMUv2_PTA_ADDRESS_LOW,
271 lower_32_bits(etnaviv_domain->pta_dma));
272 gpu_write(gpu, VIVS_MMUv2_PTA_ADDRESS_HIGH,
273 upper_32_bits(etnaviv_domain->pta_dma));
274 gpu_write(gpu, VIVS_MMUv2_PTA_CONTROL, VIVS_MMUv2_PTA_CONTROL_ENABLE);
275
276 gpu_write(gpu, VIVS_MMUv2_NONSEC_SAFE_ADDR_LOW,
277 lower_32_bits(etnaviv_domain->base.bad_page_dma));
278 gpu_write(gpu, VIVS_MMUv2_SEC_SAFE_ADDR_LOW,
279 lower_32_bits(etnaviv_domain->base.bad_page_dma));
280 gpu_write(gpu, VIVS_MMUv2_SAFE_ADDRESS_CONFIG,
281 VIVS_MMUv2_SAFE_ADDRESS_CONFIG_NON_SEC_SAFE_ADDR_HIGH(
282 upper_32_bits(etnaviv_domain->base.bad_page_dma)) |
283 VIVS_MMUv2_SAFE_ADDRESS_CONFIG_SEC_SAFE_ADDR_HIGH(
284 upper_32_bits(etnaviv_domain->base.bad_page_dma)));
285
286 etnaviv_domain->pta_cpu[0] = etnaviv_domain->mtlb_dma |
287 VIVS_MMUv2_CONFIGURATION_MODE_MODE4_K;
288
289 /* trigger a PTA load through the FE */
290 prefetch = etnaviv_buffer_config_pta(gpu);
291 etnaviv_gpu_start_fe(gpu, (u32)etnaviv_cmdbuf_get_pa(&gpu->buffer),
292 prefetch);
293 etnaviv_gpu_wait_idle(gpu, 100);
294
295 gpu_write(gpu, VIVS_MMUv2_SEC_CONTROL, VIVS_MMUv2_SEC_CONTROL_ENABLE);
296}
297
298void etnaviv_iommuv2_restore(struct etnaviv_gpu *gpu)
299{
300 switch (gpu->sec_mode) {
301 case ETNA_SEC_NONE:
302 etnaviv_iommuv2_restore_nonsec(gpu);
303 break;
304 case ETNA_SEC_KERNEL:
305 etnaviv_iommuv2_restore_sec(gpu);
306 break;
307 default:
308 WARN(1, "unhandled GPU security mode\n");
309 break;
310 }
311}
312
313static const struct etnaviv_iommu_domain_ops etnaviv_iommuv2_ops = {
314 .free = etnaviv_iommuv2_domain_free,
315 .map = etnaviv_iommuv2_map,
316 .unmap = etnaviv_iommuv2_unmap,
317 .dump_size = etnaviv_iommuv2_dump_size,
318 .dump = etnaviv_iommuv2_dump,
319};
320
321struct etnaviv_iommu_domain *
322etnaviv_iommuv2_domain_alloc(struct etnaviv_gpu *gpu)
323{
324 struct etnaviv_iommuv2_domain *etnaviv_domain;
325 struct etnaviv_iommu_domain *domain;
326 int ret;
327
328 etnaviv_domain = vzalloc(sizeof(*etnaviv_domain));
329 if (!etnaviv_domain)
330 return NULL;
331
332 domain = &etnaviv_domain->base;
333
334 domain->dev = gpu->dev;
335 domain->base = 0;
336 domain->size = (u64)SZ_1G * 4;
337 domain->ops = &etnaviv_iommuv2_ops;
338
339 ret = etnaviv_iommuv2_init(etnaviv_domain);
340 if (ret)
341 goto out_free;
342
343 return &etnaviv_domain->base;
344
345out_free:
346 vfree(etnaviv_domain);
347 return NULL;
348}