1// SPDX-License-Identifier: GPL-2.0+
  2/* Copyright (C) 2017-2018 Broadcom */
  3
  4/**
  5 * DOC: Broadcom V3D MMU
  6 *
  7 * The V3D 3.x hardware (compared to VC4) now includes an MMU.  It has
  8 * a single level of page tables for the V3D's 4GB address space to
  9 * map to AXI bus addresses, thus it could need up to 4MB of
 10 * physically contiguous memory to store the PTEs.
 11 *
 12 * Because the 4MB of contiguous memory for page tables is precious,
 13 * and switching between them is expensive, we load all BOs into the
 14 * same 4GB address space.
 15 *
 16 * To protect clients from each other, we should use the GMP to
 17 * quickly mask out (at 128kb granularity) what pages are available to
 18 * each client.  This is not yet implemented.
 19 */
 20
 21#include "v3d_drv.h"
 22#include "v3d_regs.h"
 23
 24/* Note: All PTEs for the 1MB superpage must be filled with the
 25 * superpage bit set.
 26 */
 27#define V3D_PTE_SUPERPAGE BIT(31)
 28#define V3D_PTE_BIGPAGE BIT(30)
 29#define V3D_PTE_WRITEABLE BIT(29)
 30#define V3D_PTE_VALID BIT(28)
 31
 32static bool v3d_mmu_is_aligned(u32 page, u32 page_address, size_t alignment)
 33{
 34	return IS_ALIGNED(page, alignment >> V3D_MMU_PAGE_SHIFT) &&
 35		IS_ALIGNED(page_address, alignment >> V3D_MMU_PAGE_SHIFT);
 36}
 37
 38int v3d_mmu_flush_all(struct v3d_dev *v3d)
 39{
 40	int ret;
 41
 42	V3D_WRITE(V3D_MMUC_CONTROL, V3D_MMUC_CONTROL_FLUSH |
 43		  V3D_MMUC_CONTROL_ENABLE);
 44
 45	ret = wait_for(!(V3D_READ(V3D_MMUC_CONTROL) &
 46			 V3D_MMUC_CONTROL_FLUSHING), 100);
 47	if (ret) {
 48		dev_err(v3d->drm.dev, "MMUC flush wait idle failed\n");
 49		return ret;
 50	}
 51
 52	V3D_WRITE(V3D_MMU_CTL, V3D_READ(V3D_MMU_CTL) |
 53		  V3D_MMU_CTL_TLB_CLEAR);
 54
 55	ret = wait_for(!(V3D_READ(V3D_MMU_CTL) &
 56			 V3D_MMU_CTL_TLB_CLEARING), 100);
 57	if (ret)
 58		dev_err(v3d->drm.dev, "MMU TLB clear wait idle failed\n");
 59
 60	return ret;
 61}
 62
 63int v3d_mmu_set_page_table(struct v3d_dev *v3d)
 64{
 65	V3D_WRITE(V3D_MMU_PT_PA_BASE, v3d->pt_paddr >> V3D_MMU_PAGE_SHIFT);
 66	V3D_WRITE(V3D_MMU_CTL,
 67		  V3D_MMU_CTL_ENABLE |
 68		  V3D_MMU_CTL_PT_INVALID_ENABLE |
 69		  V3D_MMU_CTL_PT_INVALID_ABORT |
 70		  V3D_MMU_CTL_PT_INVALID_INT |
 71		  V3D_MMU_CTL_WRITE_VIOLATION_ABORT |
 72		  V3D_MMU_CTL_WRITE_VIOLATION_INT |
 73		  V3D_MMU_CTL_CAP_EXCEEDED_ABORT |
 74		  V3D_MMU_CTL_CAP_EXCEEDED_INT);
 75	V3D_WRITE(V3D_MMU_ILLEGAL_ADDR,
 76		  (v3d->mmu_scratch_paddr >> V3D_MMU_PAGE_SHIFT) |
 77		  V3D_MMU_ILLEGAL_ADDR_ENABLE);
 78	V3D_WRITE(V3D_MMUC_CONTROL, V3D_MMUC_CONTROL_ENABLE);
 79
 80	return v3d_mmu_flush_all(v3d);
 81}
 82
 83void v3d_mmu_insert_ptes(struct v3d_bo *bo)
 84{
 85	struct drm_gem_shmem_object *shmem_obj = &bo->base;
 86	struct v3d_dev *v3d = to_v3d_dev(shmem_obj->base.dev);
 87	u32 page = bo->node.start;
 88	struct scatterlist *sgl;
 89	unsigned int count;
 90
 91	for_each_sgtable_dma_sg(shmem_obj->sgt, sgl, count) {
 92		dma_addr_t dma_addr = sg_dma_address(sgl);
 93		u32 pfn = dma_addr >> V3D_MMU_PAGE_SHIFT;
 94		unsigned int len = sg_dma_len(sgl);
 95
 96		while (len > 0) {
 97			u32 page_prot = V3D_PTE_WRITEABLE | V3D_PTE_VALID;
 98			u32 page_address = page_prot | pfn;
 99			unsigned int i, page_size;
100
101			BUG_ON(pfn + V3D_PAGE_FACTOR >= BIT(24));
102
103			if (len >= SZ_1M &&
104			    v3d_mmu_is_aligned(page, page_address, SZ_1M)) {
105				page_size = SZ_1M;
106				page_address |= V3D_PTE_SUPERPAGE;
107			} else if (len >= SZ_64K &&
108				   v3d_mmu_is_aligned(page, page_address, SZ_64K)) {
109				page_size = SZ_64K;
110				page_address |= V3D_PTE_BIGPAGE;
111			} else {
112				page_size = SZ_4K;
113			}
114
115			for (i = 0; i < page_size >> V3D_MMU_PAGE_SHIFT; i++) {
116				v3d->pt[page++] = page_address + i;
117				pfn++;
118			}
119
120			len -= page_size;
121		}
122	}
123
124	WARN_ON_ONCE(page - bo->node.start !=
125		     shmem_obj->base.size >> V3D_MMU_PAGE_SHIFT);
126
127	if (v3d_mmu_flush_all(v3d))
128		dev_err(v3d->drm.dev, "MMU flush timeout\n");
129}
130
131void v3d_mmu_remove_ptes(struct v3d_bo *bo)
132{
133	struct v3d_dev *v3d = to_v3d_dev(bo->base.base.dev);
134	u32 npages = bo->base.base.size >> V3D_MMU_PAGE_SHIFT;
135	u32 page;
136
137	for (page = bo->node.start; page < bo->node.start + npages; page++)
138		v3d->pt[page] = 0;
139
140	if (v3d_mmu_flush_all(v3d))
141		dev_err(v3d->drm.dev, "MMU flush timeout\n");
142}