1// SPDX-License-Identifier: GPL-2.0 OR MIT
  2/* Copyright 2017-2019 Qiang Yu <yuq825@gmail.com> */
  3
  4#include <linux/interrupt.h>
  5#include <linux/iopoll.h>
  6#include <linux/device.h>
  7
  8#include "lima_device.h"
  9#include "lima_mmu.h"
 10#include "lima_vm.h"
 11#include "lima_object.h"
 12#include "lima_regs.h"
 13
 14#define mmu_write(reg, data) writel(data, ip->iomem + reg)
 15#define mmu_read(reg) readl(ip->iomem + reg)
 16
 17#define lima_mmu_send_command(cmd, addr, val, cond)	     \
 18({							     \
 19	int __ret;					     \
 20							     \
 21	mmu_write(LIMA_MMU_COMMAND, cmd);		     \
 22	__ret = readl_poll_timeout(ip->iomem + (addr), val,  \
 23				  cond, 0, 100);	     \
 24	if (__ret)					     \
 25		dev_err(dev->dev,			     \
 26			"mmu command %x timeout\n", cmd);    \
 27	__ret;						     \
 28})
 29
 30static irqreturn_t lima_mmu_irq_handler(int irq, void *data)
 31{
 32	struct lima_ip *ip = data;
 33	struct lima_device *dev = ip->dev;
 34	u32 status = mmu_read(LIMA_MMU_INT_STATUS);
 35	struct lima_sched_pipe *pipe;
 36
 37	/* for shared irq case */
 38	if (!status)
 39		return IRQ_NONE;
 40
 41	if (status & LIMA_MMU_INT_PAGE_FAULT) {
 42		u32 fault = mmu_read(LIMA_MMU_PAGE_FAULT_ADDR);
 43
 44		dev_err(dev->dev, "mmu page fault at 0x%x from bus id %d of type %s on %s\n",
 45			fault, LIMA_MMU_STATUS_BUS_ID(status),
 46			status & LIMA_MMU_STATUS_PAGE_FAULT_IS_WRITE ? "write" : "read",
 47			lima_ip_name(ip));
 48	}
 49
 50	if (status & LIMA_MMU_INT_READ_BUS_ERROR)
 51		dev_err(dev->dev, "mmu %s irq bus error\n", lima_ip_name(ip));
 52
 53	/* mask all interrupts before resume */
 54	mmu_write(LIMA_MMU_INT_MASK, 0);
 55	mmu_write(LIMA_MMU_INT_CLEAR, status);
 56
 57	pipe = dev->pipe + (ip->id == lima_ip_gpmmu ? lima_pipe_gp : lima_pipe_pp);
 58	lima_sched_pipe_mmu_error(pipe);
 59
 60	return IRQ_HANDLED;
 61}
 62
 63int lima_mmu_init(struct lima_ip *ip)
 64{
 65	struct lima_device *dev = ip->dev;
 66	int err;
 67	u32 v;
 68
 69	if (ip->id == lima_ip_ppmmu_bcast)
 70		return 0;
 71
 72	mmu_write(LIMA_MMU_DTE_ADDR, 0xCAFEBABE);
 73	if (mmu_read(LIMA_MMU_DTE_ADDR) != 0xCAFEB000) {
 74		dev_err(dev->dev, "mmu %s dte write test fail\n", lima_ip_name(ip));
 75		return -EIO;
 76	}
 77
 78	mmu_write(LIMA_MMU_COMMAND, LIMA_MMU_COMMAND_HARD_RESET);
 79	err = lima_mmu_send_command(LIMA_MMU_COMMAND_HARD_RESET,
 80				    LIMA_MMU_DTE_ADDR, v, v == 0);
 81	if (err)
 82		return err;
 83
 84	err = devm_request_irq(dev->dev, ip->irq, lima_mmu_irq_handler,
 85			       IRQF_SHARED, lima_ip_name(ip), ip);
 86	if (err) {
 87		dev_err(dev->dev, "mmu %s fail to request irq\n", lima_ip_name(ip));
 88		return err;
 89	}
 90
 91	mmu_write(LIMA_MMU_INT_MASK, LIMA_MMU_INT_PAGE_FAULT | LIMA_MMU_INT_READ_BUS_ERROR);
 92	mmu_write(LIMA_MMU_DTE_ADDR, dev->empty_vm->pd.dma);
 93	return lima_mmu_send_command(LIMA_MMU_COMMAND_ENABLE_PAGING,
 94				     LIMA_MMU_STATUS, v,
 95				     v & LIMA_MMU_STATUS_PAGING_ENABLED);
 96}
 97
 98void lima_mmu_fini(struct lima_ip *ip)
 99{
100
101}
102
103void lima_mmu_switch_vm(struct lima_ip *ip, struct lima_vm *vm)
104{
105	struct lima_device *dev = ip->dev;
106	u32 v;
107
108	lima_mmu_send_command(LIMA_MMU_COMMAND_ENABLE_STALL,
109			      LIMA_MMU_STATUS, v,
110			      v & LIMA_MMU_STATUS_STALL_ACTIVE);
111
112	if (vm)
113		mmu_write(LIMA_MMU_DTE_ADDR, vm->pd.dma);
114
115	/* flush the TLB */
116	mmu_write(LIMA_MMU_COMMAND, LIMA_MMU_COMMAND_ZAP_CACHE);
117
118	lima_mmu_send_command(LIMA_MMU_COMMAND_DISABLE_STALL,
119			      LIMA_MMU_STATUS, v,
120			      !(v & LIMA_MMU_STATUS_STALL_ACTIVE));
121}
122
123void lima_mmu_page_fault_resume(struct lima_ip *ip)
124{
125	struct lima_device *dev = ip->dev;
126	u32 status = mmu_read(LIMA_MMU_STATUS);
127	u32 v;
128
129	if (status & LIMA_MMU_STATUS_PAGE_FAULT_ACTIVE) {
130		dev_info(dev->dev, "mmu resume\n");
131
132		mmu_write(LIMA_MMU_INT_MASK, 0);
133		mmu_write(LIMA_MMU_DTE_ADDR, 0xCAFEBABE);
134		lima_mmu_send_command(LIMA_MMU_COMMAND_HARD_RESET,
135				      LIMA_MMU_DTE_ADDR, v, v == 0);
136		mmu_write(LIMA_MMU_INT_MASK, LIMA_MMU_INT_PAGE_FAULT | LIMA_MMU_INT_READ_BUS_ERROR);
137		mmu_write(LIMA_MMU_DTE_ADDR, dev->empty_vm->pd.dma);
138		lima_mmu_send_command(LIMA_MMU_COMMAND_ENABLE_PAGING,
139				      LIMA_MMU_STATUS, v,
140				      v & LIMA_MMU_STATUS_PAGING_ENABLED);
141	}
142}