1/*
  2 * Copyright 2014 Advanced Micro Devices, Inc.
  3 *
  4 * Permission is hereby granted, free of charge, to any person obtaining a
  5 * copy of this software and associated documentation files (the "Software"),
  6 * to deal in the Software without restriction, including without limitation
  7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  8 * and/or sell copies of the Software, and to permit persons to whom the
  9 * Software is furnished to do so, subject to the following conditions:
 10 *
 11 * The above copyright notice and this permission notice shall be included in
 12 * all copies or substantial portions of the Software.
 13 *
 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
 20 * OTHER DEALINGS IN THE SOFTWARE.
 21 *
 22 */
 23
 24#include "kfd_device_queue_manager.h"
 25#include "cik_regs.h"
 26#include "oss/oss_2_4_sh_mask.h"
 27#include "gca/gfx_7_2_sh_mask.h"
 28
 29static bool set_cache_memory_policy_cik(struct device_queue_manager *dqm,
 30				   struct qcm_process_device *qpd,
 31				   enum cache_policy default_policy,
 32				   enum cache_policy alternate_policy,
 33				   void __user *alternate_aperture_base,
 34				   uint64_t alternate_aperture_size);
 35static int update_qpd_cik(struct device_queue_manager *dqm,
 36					struct qcm_process_device *qpd);
 37static int update_qpd_cik_hawaii(struct device_queue_manager *dqm,
 38					struct qcm_process_device *qpd);
 39static void init_sdma_vm(struct device_queue_manager *dqm, struct queue *q,
 40				struct qcm_process_device *qpd);
 41static void init_sdma_vm_hawaii(struct device_queue_manager *dqm,
 42				struct queue *q,
 43				struct qcm_process_device *qpd);
 44
 45void device_queue_manager_init_cik(
 46		struct device_queue_manager_asic_ops *asic_ops)
 47{
 48	asic_ops->set_cache_memory_policy = set_cache_memory_policy_cik;
 49	asic_ops->update_qpd = update_qpd_cik;
 50	asic_ops->init_sdma_vm = init_sdma_vm;
 51}
 52
 53void device_queue_manager_init_cik_hawaii(
 54		struct device_queue_manager_asic_ops *asic_ops)
 55{
 56	asic_ops->set_cache_memory_policy = set_cache_memory_policy_cik;
 57	asic_ops->update_qpd = update_qpd_cik_hawaii;
 58	asic_ops->init_sdma_vm = init_sdma_vm_hawaii;
 59}
 60
 61static uint32_t compute_sh_mem_bases_64bit(unsigned int top_address_nybble)
 62{
 63	/* In 64-bit mode, we can only control the top 3 bits of the LDS,
 64	 * scratch and GPUVM apertures.
 65	 * The hardware fills in the remaining 59 bits according to the
 66	 * following pattern:
 67	 * LDS:		X0000000'00000000 - X0000001'00000000 (4GB)
 68	 * Scratch:	X0000001'00000000 - X0000002'00000000 (4GB)
 69	 * GPUVM:	Y0010000'00000000 - Y0020000'00000000 (1TB)
 70	 *
 71	 * (where X/Y is the configurable nybble with the low-bit 0)
 72	 *
 73	 * LDS and scratch will have the same top nybble programmed in the
 74	 * top 3 bits of SH_MEM_BASES.PRIVATE_BASE.
 75	 * GPUVM can have a different top nybble programmed in the
 76	 * top 3 bits of SH_MEM_BASES.SHARED_BASE.
 77	 * We don't bother to support different top nybbles
 78	 * for LDS/Scratch and GPUVM.
 79	 */
 80
 81	WARN_ON((top_address_nybble & 1) || top_address_nybble > 0xE ||
 82		top_address_nybble == 0);
 83
 84	return PRIVATE_BASE(top_address_nybble << 12) |
 85			SHARED_BASE(top_address_nybble << 12);
 86}
 87
 88static bool set_cache_memory_policy_cik(struct device_queue_manager *dqm,
 89				   struct qcm_process_device *qpd,
 90				   enum cache_policy default_policy,
 91				   enum cache_policy alternate_policy,
 92				   void __user *alternate_aperture_base,
 93				   uint64_t alternate_aperture_size)
 94{
 95	uint32_t default_mtype;
 96	uint32_t ape1_mtype;
 97
 98	default_mtype = (default_policy == cache_policy_coherent) ?
 99			MTYPE_NONCACHED :
100			MTYPE_CACHED;
101
102	ape1_mtype = (alternate_policy == cache_policy_coherent) ?
103			MTYPE_NONCACHED :
104			MTYPE_CACHED;
105
106	qpd->sh_mem_config = (qpd->sh_mem_config & PTR32)
107			| ALIGNMENT_MODE(SH_MEM_ALIGNMENT_MODE_UNALIGNED)
108			| DEFAULT_MTYPE(default_mtype)
109			| APE1_MTYPE(ape1_mtype);
110
111	return true;
112}
113
114static int update_qpd_cik(struct device_queue_manager *dqm,
115		struct qcm_process_device *qpd)
116{
117	struct kfd_process_device *pdd;
118	unsigned int temp;
119
120	pdd = qpd_to_pdd(qpd);
121
122	/* check if sh_mem_config register already configured */
123	if (qpd->sh_mem_config == 0) {
124		qpd->sh_mem_config =
125			ALIGNMENT_MODE(SH_MEM_ALIGNMENT_MODE_UNALIGNED) |
126			DEFAULT_MTYPE(MTYPE_NONCACHED) |
127			APE1_MTYPE(MTYPE_NONCACHED);
128		qpd->sh_mem_ape1_limit = 0;
129		qpd->sh_mem_ape1_base = 0;
130	}
131
132	if (qpd->pqm->process->is_32bit_user_mode) {
133		temp = get_sh_mem_bases_32(pdd);
134		qpd->sh_mem_bases = SHARED_BASE(temp);
135		qpd->sh_mem_config |= PTR32;
136	} else {
137		temp = get_sh_mem_bases_nybble_64(pdd);
138		qpd->sh_mem_bases = compute_sh_mem_bases_64bit(temp);
139		qpd->sh_mem_config |= 1  << SH_MEM_CONFIG__PRIVATE_ATC__SHIFT;
140	}
141
142	pr_debug("is32bit process: %d sh_mem_bases nybble: 0x%X and register 0x%X\n",
143		qpd->pqm->process->is_32bit_user_mode, temp, qpd->sh_mem_bases);
144
145	return 0;
146}
147
148static int update_qpd_cik_hawaii(struct device_queue_manager *dqm,
149		struct qcm_process_device *qpd)
150{
151	struct kfd_process_device *pdd;
152	unsigned int temp;
153
154	pdd = qpd_to_pdd(qpd);
155
156	/* check if sh_mem_config register already configured */
157	if (qpd->sh_mem_config == 0) {
158		qpd->sh_mem_config =
159			ALIGNMENT_MODE(SH_MEM_ALIGNMENT_MODE_UNALIGNED) |
160			DEFAULT_MTYPE(MTYPE_NONCACHED) |
161			APE1_MTYPE(MTYPE_NONCACHED);
162		qpd->sh_mem_ape1_limit = 0;
163		qpd->sh_mem_ape1_base = 0;
164	}
165
166	/* On dGPU we're always in GPUVM64 addressing mode with 64-bit
167	 * aperture addresses.
168	 */
169	temp = get_sh_mem_bases_nybble_64(pdd);
170	qpd->sh_mem_bases = compute_sh_mem_bases_64bit(temp);
171
172	pr_debug("is32bit process: %d sh_mem_bases nybble: 0x%X and register 0x%X\n",
173		qpd->pqm->process->is_32bit_user_mode, temp, qpd->sh_mem_bases);
174
175	return 0;
176}
177
178static void init_sdma_vm(struct device_queue_manager *dqm, struct queue *q,
179				struct qcm_process_device *qpd)
180{
181	uint32_t value = (1 << SDMA0_RLC0_VIRTUAL_ADDR__ATC__SHIFT);
182
183	if (q->process->is_32bit_user_mode)
184		value |= (1 << SDMA0_RLC0_VIRTUAL_ADDR__PTR32__SHIFT) |
185				get_sh_mem_bases_32(qpd_to_pdd(qpd));
186	else
187		value |= ((get_sh_mem_bases_nybble_64(qpd_to_pdd(qpd))) <<
188				SDMA0_RLC0_VIRTUAL_ADDR__SHARED_BASE__SHIFT) &
189				SDMA0_RLC0_VIRTUAL_ADDR__SHARED_BASE_MASK;
190
191	q->properties.sdma_vm_addr = value;
192}
193
194static void init_sdma_vm_hawaii(struct device_queue_manager *dqm,
195				struct queue *q,
196				struct qcm_process_device *qpd)
197{
198	/* On dGPU we're always in GPUVM64 addressing mode with 64-bit
199	 * aperture addresses.
200	 */
201	q->properties.sdma_vm_addr =
202		((get_sh_mem_bases_nybble_64(qpd_to_pdd(qpd))) <<
203		 SDMA0_RLC0_VIRTUAL_ADDR__SHARED_BASE__SHIFT) &
204		SDMA0_RLC0_VIRTUAL_ADDR__SHARED_BASE_MASK;
205}