1// SPDX-License-Identifier: MIT
  2/*
  3 * Copyright © 2021-2023 Intel Corporation
  4 */
  5
  6#include "xe_mmio.h"
  7
  8#include <linux/delay.h>
  9#include <linux/io-64-nonatomic-lo-hi.h>
 10#include <linux/minmax.h>
 11#include <linux/pci.h>
 12
 13#include <drm/drm_managed.h>
 14#include <drm/drm_print.h>
 15
 16#include "regs/xe_bars.h"
 17#include "regs/xe_regs.h"
 18#include "xe_device.h"
 19#include "xe_gt.h"
 20#include "xe_gt_printk.h"
 21#include "xe_gt_sriov_vf.h"
 22#include "xe_macros.h"
 23#include "xe_sriov.h"
 24#include "xe_trace.h"
 25
 26static void tiles_fini(void *arg)
 27{
 28	struct xe_device *xe = arg;
 29	struct xe_tile *tile;
 30	int id;
 31
 32	for_each_remote_tile(tile, xe, id)
 33		tile->mmio.regs = NULL;
 34}
 35
 36/*
 37 * On multi-tile devices, partition the BAR space for MMIO on each tile,
 38 * possibly accounting for register override on the number of tiles available.
 39 * tile_mmio_size contains both the tile's 4MB register space, as well as
 40 * additional space for the GTT and other (possibly unused) regions).
 41 * Resulting memory layout is like below:
 42 *
 43 * .----------------------. <- tile_count * tile_mmio_size
 44 * |         ....         |
 45 * |----------------------| <- 2 * tile_mmio_size
 46 * |   tile1 GTT + other  |
 47 * |----------------------| <- 1 * tile_mmio_size + 4MB
 48 * |   tile1->mmio.regs   |
 49 * |----------------------| <- 1 * tile_mmio_size
 50 * |   tile0 GTT + other  |
 51 * |----------------------| <- 4MB
 52 * |   tile0->mmio.regs   |
 53 * '----------------------' <- 0MB
 54 */
 55static void mmio_multi_tile_setup(struct xe_device *xe, size_t tile_mmio_size)
 56{
 57	struct xe_tile *tile;
 58	void __iomem *regs;
 59	u8 id;
 60
 61	/*
 62	 * Nothing to be done as tile 0 has already been setup earlier with the
 63	 * entire BAR mapped - see xe_mmio_init()
 64	 */
 65	if (xe->info.tile_count == 1)
 66		return;
 67
 68	/* Possibly override number of tile based on configuration register */
 69	if (!xe->info.skip_mtcfg) {
 70		struct xe_mmio *mmio = xe_root_tile_mmio(xe);
 71		u8 tile_count;
 72		u32 mtcfg;
 73
 74		/*
 75		 * Although the per-tile mmio regs are not yet initialized, this
 76		 * is fine as it's going to the root tile's mmio, that's
 77		 * guaranteed to be initialized earlier in xe_mmio_init()
 78		 */
 79		mtcfg = xe_mmio_read64_2x32(mmio, XEHP_MTCFG_ADDR);
 80		tile_count = REG_FIELD_GET(TILE_COUNT, mtcfg) + 1;
 81
 82		if (tile_count < xe->info.tile_count) {
 83			drm_info(&xe->drm, "tile_count: %d, reduced_tile_count %d\n",
 84					xe->info.tile_count, tile_count);
 85			xe->info.tile_count = tile_count;
 86
 87			/*
 88			 * FIXME: Needs some work for standalone media, but
 89			 * should be impossible with multi-tile for now:
 90			 * multi-tile platform with standalone media doesn't
 91			 * exist
 92			 */
 93			xe->info.gt_count = xe->info.tile_count;
 94		}
 95	}
 96
 97	regs = xe->mmio.regs;
 98	for_each_tile(tile, xe, id) {
 99		tile->mmio.regs_size = SZ_4M;
100		tile->mmio.regs = regs;
101		tile->mmio.tile = tile;
102		regs += tile_mmio_size;
103	}
104}
105
106/*
107 * On top of all the multi-tile MMIO space there can be a platform-dependent
108 * extension for each tile, resulting in a layout like below:
109 *
110 * .----------------------. <- ext_base + tile_count * tile_mmio_ext_size
111 * |         ....         |
112 * |----------------------| <- ext_base + 2 * tile_mmio_ext_size
113 * | tile1->mmio_ext.regs |
114 * |----------------------| <- ext_base + 1 * tile_mmio_ext_size
115 * | tile0->mmio_ext.regs |
116 * |======================| <- ext_base = tile_count * tile_mmio_size
117 * |                      |
118 * |       mmio.regs      |
119 * |                      |
120 * '----------------------' <- 0MB
121 *
122 * Set up the tile[]->mmio_ext pointers/sizes.
123 */
124static void mmio_extension_setup(struct xe_device *xe, size_t tile_mmio_size,
125				 size_t tile_mmio_ext_size)
126{
127	struct xe_tile *tile;
128	void __iomem *regs;
129	u8 id;
130
131	if (!xe->info.has_mmio_ext)
132		return;
133
134	regs = xe->mmio.regs + tile_mmio_size * xe->info.tile_count;
135	for_each_tile(tile, xe, id) {
136		tile->mmio_ext.regs_size = tile_mmio_ext_size;
137		tile->mmio_ext.regs = regs;
138		tile->mmio_ext.tile = tile;
139		regs += tile_mmio_ext_size;
140	}
141}
142
143int xe_mmio_probe_tiles(struct xe_device *xe)
144{
145	size_t tile_mmio_size = SZ_16M;
146	size_t tile_mmio_ext_size = xe->info.tile_mmio_ext_size;
147
148	mmio_multi_tile_setup(xe, tile_mmio_size);
149	mmio_extension_setup(xe, tile_mmio_size, tile_mmio_ext_size);
150
151	return devm_add_action_or_reset(xe->drm.dev, tiles_fini, xe);
152}
153
154static void mmio_fini(void *arg)
155{
156	struct xe_device *xe = arg;
157	struct xe_tile *root_tile = xe_device_get_root_tile(xe);
158
159	pci_iounmap(to_pci_dev(xe->drm.dev), xe->mmio.regs);
160	xe->mmio.regs = NULL;
161	root_tile->mmio.regs = NULL;
162}
163
164int xe_mmio_init(struct xe_device *xe)
165{
166	struct xe_tile *root_tile = xe_device_get_root_tile(xe);
167	struct pci_dev *pdev = to_pci_dev(xe->drm.dev);
168
169	/*
170	 * Map the entire BAR.
171	 * The first 16MB of the BAR, belong to the root tile, and include:
172	 * registers (0-4MB), reserved space (4MB-8MB) and GGTT (8MB-16MB).
173	 */
174	xe->mmio.size = pci_resource_len(pdev, GTTMMADR_BAR);
175	xe->mmio.regs = pci_iomap(pdev, GTTMMADR_BAR, 0);
176	if (xe->mmio.regs == NULL) {
177		drm_err(&xe->drm, "failed to map registers\n");
178		return -EIO;
179	}
180
181	/* Setup first tile; other tiles (if present) will be setup later. */
182	root_tile->mmio.regs_size = SZ_4M;
183	root_tile->mmio.regs = xe->mmio.regs;
184	root_tile->mmio.tile = root_tile;
185
186	return devm_add_action_or_reset(xe->drm.dev, mmio_fini, xe);
187}
188
189static void mmio_flush_pending_writes(struct xe_mmio *mmio)
190{
191#define DUMMY_REG_OFFSET	0x130030
192	int i;
193
194	if (mmio->tile->xe->info.platform != XE_LUNARLAKE)
195		return;
196
197	/* 4 dummy writes */
198	for (i = 0; i < 4; i++)
199		writel(0, mmio->regs + DUMMY_REG_OFFSET);
200}
201
202u8 xe_mmio_read8(struct xe_mmio *mmio, struct xe_reg reg)
203{
204	u32 addr = xe_mmio_adjusted_addr(mmio, reg.addr);
205	u8 val;
206
207	/* Wa_15015404425 */
208	mmio_flush_pending_writes(mmio);
209
210	val = readb(mmio->regs + addr);
211	trace_xe_reg_rw(mmio, false, addr, val, sizeof(val));
212
213	return val;
214}
215
216u16 xe_mmio_read16(struct xe_mmio *mmio, struct xe_reg reg)
217{
218	u32 addr = xe_mmio_adjusted_addr(mmio, reg.addr);
219	u16 val;
220
221	/* Wa_15015404425 */
222	mmio_flush_pending_writes(mmio);
223
224	val = readw(mmio->regs + addr);
225	trace_xe_reg_rw(mmio, false, addr, val, sizeof(val));
226
227	return val;
228}
229
230void xe_mmio_write32(struct xe_mmio *mmio, struct xe_reg reg, u32 val)
231{
232	u32 addr = xe_mmio_adjusted_addr(mmio, reg.addr);
233
234	trace_xe_reg_rw(mmio, true, addr, val, sizeof(val));
235
236	if (!reg.vf && mmio->sriov_vf_gt)
237		xe_gt_sriov_vf_write32(mmio->sriov_vf_gt, reg, val);
238	else
239		writel(val, mmio->regs + addr);
240}
241
242u32 xe_mmio_read32(struct xe_mmio *mmio, struct xe_reg reg)
243{
244	u32 addr = xe_mmio_adjusted_addr(mmio, reg.addr);
245	u32 val;
246
247	/* Wa_15015404425 */
248	mmio_flush_pending_writes(mmio);
249
250	if (!reg.vf && mmio->sriov_vf_gt)
251		val = xe_gt_sriov_vf_read32(mmio->sriov_vf_gt, reg);
252	else
253		val = readl(mmio->regs + addr);
254
255	trace_xe_reg_rw(mmio, false, addr, val, sizeof(val));
256
257	return val;
258}
259
260u32 xe_mmio_rmw32(struct xe_mmio *mmio, struct xe_reg reg, u32 clr, u32 set)
261{
262	u32 old, reg_val;
263
264	old = xe_mmio_read32(mmio, reg);
265	reg_val = (old & ~clr) | set;
266	xe_mmio_write32(mmio, reg, reg_val);
267
268	return old;
269}
270
271int xe_mmio_write32_and_verify(struct xe_mmio *mmio,
272			       struct xe_reg reg, u32 val, u32 mask, u32 eval)
273{
274	u32 reg_val;
275
276	xe_mmio_write32(mmio, reg, val);
277	reg_val = xe_mmio_read32(mmio, reg);
278
279	return (reg_val & mask) != eval ? -EINVAL : 0;
280}
281
282bool xe_mmio_in_range(const struct xe_mmio *mmio,
283		      const struct xe_mmio_range *range,
284		      struct xe_reg reg)
285{
286	u32 addr = xe_mmio_adjusted_addr(mmio, reg.addr);
287
288	return range && addr >= range->start && addr <= range->end;
289}
290
291/**
292 * xe_mmio_read64_2x32() - Read a 64-bit register as two 32-bit reads
293 * @mmio: MMIO target
294 * @reg: register to read value from
295 *
296 * Although Intel GPUs have some 64-bit registers, the hardware officially
297 * only supports GTTMMADR register reads of 32 bits or smaller.  Even if
298 * a readq operation may return a reasonable value, that violation of the
299 * spec shouldn't be relied upon and all 64-bit register reads should be
300 * performed as two 32-bit reads of the upper and lower dwords.
301 *
302 * When reading registers that may be changing (such as
303 * counters), a rollover of the lower dword between the two 32-bit reads
304 * can be problematic.  This function attempts to ensure the upper dword has
305 * stabilized before returning the 64-bit value.
306 *
307 * Note that because this function may re-read the register multiple times
308 * while waiting for the value to stabilize it should not be used to read
309 * any registers where read operations have side effects.
310 *
311 * Returns the value of the 64-bit register.
312 */
313u64 xe_mmio_read64_2x32(struct xe_mmio *mmio, struct xe_reg reg)
314{
315	struct xe_reg reg_udw = { .addr = reg.addr + 0x4 };
316	u32 ldw, udw, oldudw, retries;
317
318	reg.addr = xe_mmio_adjusted_addr(mmio, reg.addr);
319	reg_udw.addr = xe_mmio_adjusted_addr(mmio, reg_udw.addr);
320
321	/* we shouldn't adjust just one register address */
322	xe_tile_assert(mmio->tile, reg_udw.addr == reg.addr + 0x4);
323
324	oldudw = xe_mmio_read32(mmio, reg_udw);
325	for (retries = 5; retries; --retries) {
326		ldw = xe_mmio_read32(mmio, reg);
327		udw = xe_mmio_read32(mmio, reg_udw);
328
329		if (udw == oldudw)
330			break;
331
332		oldudw = udw;
333	}
334
335	drm_WARN(&mmio->tile->xe->drm, retries == 0,
336		 "64-bit read of %#x did not stabilize\n", reg.addr);
337
338	return (u64)udw << 32 | ldw;
339}
340
341static int __xe_mmio_wait32(struct xe_mmio *mmio, struct xe_reg reg, u32 mask, u32 val, u32 timeout_us,
342			    u32 *out_val, bool atomic, bool expect_match)
343{
344	ktime_t cur = ktime_get_raw();
345	const ktime_t end = ktime_add_us(cur, timeout_us);
346	int ret = -ETIMEDOUT;
347	s64 wait = 10;
348	u32 read;
349	bool check;
350
351	for (;;) {
352		read = xe_mmio_read32(mmio, reg);
353
354		check = (read & mask) == val;
355		if (!expect_match)
356			check = !check;
357
358		if (check) {
359			ret = 0;
360			break;
361		}
362
363		cur = ktime_get_raw();
364		if (!ktime_before(cur, end))
365			break;
366
367		if (ktime_after(ktime_add_us(cur, wait), end))
368			wait = ktime_us_delta(end, cur);
369
370		if (atomic)
371			udelay(wait);
372		else
373			usleep_range(wait, wait << 1);
374		wait <<= 1;
375	}
376
377	if (ret != 0) {
378		read = xe_mmio_read32(mmio, reg);
379
380		check = (read & mask) == val;
381		if (!expect_match)
382			check = !check;
383
384		if (check)
385			ret = 0;
386	}
387
388	if (out_val)
389		*out_val = read;
390
391	return ret;
392}
393
394/**
395 * xe_mmio_wait32() - Wait for a register to match the desired masked value
396 * @mmio: MMIO target
397 * @reg: register to read value from
398 * @mask: mask to be applied to the value read from the register
399 * @val: desired value after applying the mask
400 * @timeout_us: time out after this period of time. Wait logic tries to be
401 * smart, applying an exponential backoff until @timeout_us is reached.
402 * @out_val: if not NULL, points where to store the last unmasked value
403 * @atomic: needs to be true if calling from an atomic context
404 *
405 * This function polls for the desired masked value and returns zero on success
406 * or -ETIMEDOUT if timed out.
407 *
408 * Note that @timeout_us represents the minimum amount of time to wait before
409 * giving up. The actual time taken by this function can be a little more than
410 * @timeout_us for different reasons, specially in non-atomic contexts. Thus,
411 * it is possible that this function succeeds even after @timeout_us has passed.
412 */
413int xe_mmio_wait32(struct xe_mmio *mmio, struct xe_reg reg, u32 mask, u32 val, u32 timeout_us,
414		   u32 *out_val, bool atomic)
415{
416	return __xe_mmio_wait32(mmio, reg, mask, val, timeout_us, out_val, atomic, true);
417}
418
419/**
420 * xe_mmio_wait32_not() - Wait for a register to return anything other than the given masked value
421 * @mmio: MMIO target
422 * @reg: register to read value from
423 * @mask: mask to be applied to the value read from the register
424 * @val: value not to be matched after applying the mask
425 * @timeout_us: time out after this period of time
426 * @out_val: if not NULL, points where to store the last unmasked value
427 * @atomic: needs to be true if calling from an atomic context
428 *
429 * This function works exactly like xe_mmio_wait32() with the exception that
430 * @val is expected not to be matched.
431 */
432int xe_mmio_wait32_not(struct xe_mmio *mmio, struct xe_reg reg, u32 mask, u32 val, u32 timeout_us,
433		       u32 *out_val, bool atomic)
434{
435	return __xe_mmio_wait32(mmio, reg, mask, val, timeout_us, out_val, atomic, false);
436}