1// SPDX-License-Identifier: MIT
  2/*
  3 * Copyright © 2023 Intel Corporation
  4 */
  5
  6#include <linux/align.h>
  7#include <linux/bitfield.h>
  8#include <linux/log2.h>
  9#include <linux/sizes.h>
 10
 11#include "xe_lmtt_types.h"
 12#include "xe_macros.h"
 13
 14/**
 15 * DOC: Multi-Level LMTT Structure
 16 *
 17 * LMHAW (Local Memory Host Address Width) is 48 bit (256TB)
 18 *
 19 * LMGAW (Local Memory Guest Address Width) is 48 bit (256TB)
 20 *
 21 * The following figure illustrates the structure and function of the ML LMTT::
 22 *
 23 *           LMTT L3 Directory
 24 *           (1 Entry per VF)                                       LMTT L1 Leaf
 25 *            +-----------+                                         +-----------+
 26 *            |           |             LMTT L2 (per VF)            |           |
 27 *            |           |              +-----------+              |           |
 28 *            |           |              |           |     index:   +===========+
 29 *            |           |              |           |     GDPA --> |    PTE    | => LMEM PF offset
 30 *            |           |              |           |     34:21    +===========+
 31 *            |           |    index:    |           |              |           |
 32 *            |           |    LMEM VF   +===========+              |           |
 33 *            |           |    offset -> |    PTE    |  ----------> +-----------+
 34 *            |           |    GAW-1:35  +===========+              /           \.
 35 *   index:   +===========+              |           |             /              \.
 36 *   VFID --> |    PDE    |  --------->  +-----------+            /                 \.
 37 *            +===========+             /           /            /                    \.
 38 *            |           |           /            /            /                       \.
 39 *            +-----------+  <== [LMTT Directory Ptr]          /                          \.
 40 *           /             \      /              /            /                             \.
 41 *          /                \  /               /       +-----------+-----------------+------+---+
 42 *         /                  /\               /        | 31:HAW-16 |        HAW-17:5 |  4:1 | 0 |
 43 *        /                 /    \            /         +===========+=================+======+===+
 44 *       /                /        \         /          |  Reserved | LMEM Page (2MB) | Rsvd | V |
 45 *      /                                   /           +-----------+-----------------+------+---+
 46 *     /                                   /
 47 *  +-----------+-----------------+------+---+
 48 *  | 63:HAW-12 |        HAW-13:4 |  3:1 | 0 |
 49 *  +===========+=================+======+===+
 50 *  |  Reserved | LMTT Ptr (64KB) | Rsvd | V |
 51 *  +-----------+-----------------+------+---+
 52 *
 53 */
 54
 55typedef u64 lmtt_ml_pde_t;
 56typedef u32 lmtt_ml_pte_t;
 57
 58#define LMTT_ML_HAW			48 /* 256 TiB */
 59
 60#define LMTT_ML_PDE_MAX_NUM		64 /* SRIOV with PF and 63 VFs, index 0 (PF) is unused */
 61#define LMTT_ML_PDE_LMTT_PTR		GENMASK_ULL(LMTT_ML_HAW - 13, 4)
 62#define LMTT_ML_PDE_VALID		BIT(0)
 63
 64#define LMTT_ML_PDE_L2_SHIFT		35
 65#define LMTT_ML_PDE_L2_MAX_NUM		BIT_ULL(LMTT_ML_HAW - 35)
 66
 67#define LMTT_ML_PTE_MAX_NUM		BIT(35 - ilog2(SZ_2M))
 68#define LMTT_ML_PTE_LMEM_PAGE		GENMASK(LMTT_ML_HAW - 17, 5)
 69#define LMTT_ML_PTE_VALID		BIT(0)
 70
 71static unsigned int lmtt_ml_root_pd_level(void)
 72{
 73	return 2; /* implementation is 0-based */
 74}
 75
 76static unsigned int lmtt_ml_pte_num(unsigned int level)
 77{
 78	switch (level) {
 79	case 2:
 80		return LMTT_ML_PDE_MAX_NUM;
 81	case 1:
 82		BUILD_BUG_ON(LMTT_ML_HAW == 48 && LMTT_ML_PDE_L2_MAX_NUM != SZ_8K);
 83		return LMTT_ML_PDE_L2_MAX_NUM;
 84	case 0:
 85		BUILD_BUG_ON(LMTT_ML_PTE_MAX_NUM != SZ_16K);
 86		return LMTT_ML_PTE_MAX_NUM;
 87	default:
 88		return 0;
 89	}
 90}
 91
 92static unsigned int lmtt_ml_pte_size(unsigned int level)
 93{
 94	switch (level) {
 95	case 2:
 96	case 1:
 97		return sizeof(lmtt_ml_pde_t);
 98	case 0:
 99		return sizeof(lmtt_ml_pte_t);
100	default:
101		return 0;
102	}
103}
104
105static unsigned int lmtt_ml_pte_shift(unsigned int level)
106{
107	switch (level) {
108	case 1:
109		BUILD_BUG_ON(BIT_ULL(LMTT_ML_PDE_L2_SHIFT) != SZ_32G);
110		return ilog2(SZ_32G);
111	case 0:
112		return ilog2(SZ_2M);
113	default:
114		return 0;
115	}
116}
117
118static unsigned int lmtt_ml_pte_index(u64 addr, unsigned int level)
119{
120	addr >>= lmtt_ml_pte_shift(level);
121
122	switch (level) {
123	case 1:
124		/* SZ_32G increments */
125		BUILD_BUG_ON_NOT_POWER_OF_2(LMTT_ML_PDE_L2_MAX_NUM);
126		return addr & (LMTT_ML_PDE_L2_MAX_NUM - 1);
127	case 0:
128		/* SZ_2M increments */
129		BUILD_BUG_ON_NOT_POWER_OF_2(LMTT_ML_PTE_MAX_NUM);
130		return addr & (LMTT_ML_PTE_MAX_NUM - 1);
131	default:
132		return 0;
133	}
134}
135
136static u64 lmtt_ml_pte_encode(unsigned long offset, unsigned int level)
137{
138	switch (level) {
139	case 0:
140		XE_WARN_ON(!IS_ALIGNED(offset, SZ_2M));
141		XE_WARN_ON(!FIELD_FIT(LMTT_ML_PTE_LMEM_PAGE, offset / SZ_2M));
142		return FIELD_PREP(LMTT_ML_PTE_LMEM_PAGE, offset / SZ_2M) | LMTT_ML_PTE_VALID;
143	case 1:
144	case 2:
145		XE_WARN_ON(!IS_ALIGNED(offset, SZ_64K));
146		XE_WARN_ON(!FIELD_FIT(LMTT_ML_PDE_LMTT_PTR, offset / SZ_64K));
147		return FIELD_PREP(LMTT_ML_PDE_LMTT_PTR, offset / SZ_64K) | LMTT_ML_PDE_VALID;
148	default:
149		XE_WARN_ON(true);
150		return 0;
151	}
152}
153
154const struct xe_lmtt_ops lmtt_ml_ops = {
155	.lmtt_root_pd_level = lmtt_ml_root_pd_level,
156	.lmtt_pte_num = lmtt_ml_pte_num,
157	.lmtt_pte_size = lmtt_ml_pte_size,
158	.lmtt_pte_shift = lmtt_ml_pte_shift,
159	.lmtt_pte_index = lmtt_ml_pte_index,
160	.lmtt_pte_encode = lmtt_ml_pte_encode,
161};