1/*
  2 * Copyright 2014 Advanced Micro Devices, Inc.
  3 *
  4 * Permission is hereby granted, free of charge, to any person obtaining a
  5 * copy of this software and associated documentation files (the "Software"),
  6 * to deal in the Software without restriction, including without limitation
  7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  8 * and/or sell copies of the Software, and to permit persons to whom the
  9 * Software is furnished to do so, subject to the following conditions:
 10 *
 11 * The above copyright notice and this permission notice shall be included in
 12 * all copies or substantial portions of the Software.
 13 *
 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
 20 * OTHER DEALINGS IN THE SOFTWARE.
 21 *
 22 */
 23
 24#include <linux/printk.h>
 25#include <linux/slab.h>
 26#include "kfd_priv.h"
 27#include "kfd_mqd_manager.h"
 28#include "vi_structs.h"
 29#include "gca/gfx_8_0_sh_mask.h"
 30#include "gca/gfx_8_0_enum.h"
 31
 32#define CP_MQD_CONTROL__PRIV_STATE__SHIFT 0x8
 33
 34static inline struct vi_mqd *get_mqd(void *mqd)
 35{
 36	return (struct vi_mqd *)mqd;
 37}
 38
 39static int init_mqd(struct mqd_manager *mm, void **mqd,
 40			struct kfd_mem_obj **mqd_mem_obj, uint64_t *gart_addr,
 41			struct queue_properties *q)
 42{
 43	int retval;
 44	uint64_t addr;
 45	struct vi_mqd *m;
 46
 47	retval = kfd_gtt_sa_allocate(mm->dev, sizeof(struct vi_mqd),
 48			mqd_mem_obj);
 49	if (retval != 0)
 50		return -ENOMEM;
 51
 52	m = (struct vi_mqd *) (*mqd_mem_obj)->cpu_ptr;
 53	addr = (*mqd_mem_obj)->gpu_addr;
 54
 55	memset(m, 0, sizeof(struct vi_mqd));
 56
 57	m->header = 0xC0310800;
 58	m->compute_pipelinestat_enable = 1;
 59	m->compute_static_thread_mgmt_se0 = 0xFFFFFFFF;
 60	m->compute_static_thread_mgmt_se1 = 0xFFFFFFFF;
 61	m->compute_static_thread_mgmt_se2 = 0xFFFFFFFF;
 62	m->compute_static_thread_mgmt_se3 = 0xFFFFFFFF;
 63
 64	m->cp_hqd_persistent_state = CP_HQD_PERSISTENT_STATE__PRELOAD_REQ_MASK |
 65			0x53 << CP_HQD_PERSISTENT_STATE__PRELOAD_SIZE__SHIFT;
 66
 67	m->cp_mqd_control = 1 << CP_MQD_CONTROL__PRIV_STATE__SHIFT |
 68			MTYPE_UC << CP_MQD_CONTROL__MTYPE__SHIFT;
 69
 70	m->cp_mqd_base_addr_lo        = lower_32_bits(addr);
 71	m->cp_mqd_base_addr_hi        = upper_32_bits(addr);
 72
 73	m->cp_hqd_quantum = 1 << CP_HQD_QUANTUM__QUANTUM_EN__SHIFT |
 74			1 << CP_HQD_QUANTUM__QUANTUM_SCALE__SHIFT |
 75			10 << CP_HQD_QUANTUM__QUANTUM_DURATION__SHIFT;
 76
 77	m->cp_hqd_pipe_priority = 1;
 78	m->cp_hqd_queue_priority = 15;
 79
 80	m->cp_hqd_eop_rptr = 1 << CP_HQD_EOP_RPTR__INIT_FETCHER__SHIFT;
 81
 82	if (q->format == KFD_QUEUE_FORMAT_AQL)
 83		m->cp_hqd_iq_rptr = 1;
 84
 85	*mqd = m;
 86	if (gart_addr != NULL)
 87		*gart_addr = addr;
 88	retval = mm->update_mqd(mm, m, q);
 89
 90	return retval;
 91}
 92
 93static int load_mqd(struct mqd_manager *mm, void *mqd,
 94			uint32_t pipe_id, uint32_t queue_id,
 95			uint32_t __user *wptr)
 96{
 97	return mm->dev->kfd2kgd->hqd_load
 98		(mm->dev->kgd, mqd, pipe_id, queue_id, wptr);
 99}
100
101static int __update_mqd(struct mqd_manager *mm, void *mqd,
102			struct queue_properties *q, unsigned int mtype,
103			unsigned int atc_bit)
104{
105	struct vi_mqd *m;
106
107	BUG_ON(!mm || !q || !mqd);
108
109	pr_debug("kfd: In func %s\n", __func__);
110
111	m = get_mqd(mqd);
112
113	m->cp_hqd_pq_control = 5 << CP_HQD_PQ_CONTROL__RPTR_BLOCK_SIZE__SHIFT |
114			atc_bit << CP_HQD_PQ_CONTROL__PQ_ATC__SHIFT |
115			mtype << CP_HQD_PQ_CONTROL__MTYPE__SHIFT;
116	m->cp_hqd_pq_control |=
117			ffs(q->queue_size / sizeof(unsigned int)) - 1 - 1;
118	pr_debug("kfd: cp_hqd_pq_control 0x%x\n", m->cp_hqd_pq_control);
119
120	m->cp_hqd_pq_base_lo = lower_32_bits((uint64_t)q->queue_address >> 8);
121	m->cp_hqd_pq_base_hi = upper_32_bits((uint64_t)q->queue_address >> 8);
122
123	m->cp_hqd_pq_rptr_report_addr_lo = lower_32_bits((uint64_t)q->read_ptr);
124	m->cp_hqd_pq_rptr_report_addr_hi = upper_32_bits((uint64_t)q->read_ptr);
125
126	m->cp_hqd_pq_doorbell_control =
127		1 << CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_EN__SHIFT |
128		q->doorbell_off <<
129			CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_OFFSET__SHIFT;
130	pr_debug("kfd: cp_hqd_pq_doorbell_control 0x%x\n",
131			m->cp_hqd_pq_doorbell_control);
132
133	m->cp_hqd_eop_control = atc_bit << CP_HQD_EOP_CONTROL__EOP_ATC__SHIFT |
134			mtype << CP_HQD_EOP_CONTROL__MTYPE__SHIFT;
135
136	m->cp_hqd_ib_control = atc_bit << CP_HQD_IB_CONTROL__IB_ATC__SHIFT |
137			3 << CP_HQD_IB_CONTROL__MIN_IB_AVAIL_SIZE__SHIFT |
138			mtype << CP_HQD_IB_CONTROL__MTYPE__SHIFT;
139
140	m->cp_hqd_eop_control |=
141		ffs(q->eop_ring_buffer_size / sizeof(unsigned int)) - 1 - 1;
142	m->cp_hqd_eop_base_addr_lo =
143			lower_32_bits(q->eop_ring_buffer_address >> 8);
144	m->cp_hqd_eop_base_addr_hi =
145			upper_32_bits(q->eop_ring_buffer_address >> 8);
146
147	m->cp_hqd_iq_timer = atc_bit << CP_HQD_IQ_TIMER__IQ_ATC__SHIFT |
148			mtype << CP_HQD_IQ_TIMER__MTYPE__SHIFT;
149
150	m->cp_hqd_vmid = q->vmid;
151
152	if (q->format == KFD_QUEUE_FORMAT_AQL) {
153		m->cp_hqd_pq_control |= CP_HQD_PQ_CONTROL__NO_UPDATE_RPTR_MASK |
154				2 << CP_HQD_PQ_CONTROL__SLOT_BASED_WPTR__SHIFT;
155	}
156
157	m->cp_hqd_active = 0;
158	q->is_active = false;
159	if (q->queue_size > 0 &&
160			q->queue_address != 0 &&
161			q->queue_percent > 0) {
162		m->cp_hqd_active = 1;
163		q->is_active = true;
164	}
165
166	return 0;
167}
168
169
170static int update_mqd(struct mqd_manager *mm, void *mqd,
171			struct queue_properties *q)
172{
173	return __update_mqd(mm, mqd, q, MTYPE_CC, 1);
174}
175
176static int destroy_mqd(struct mqd_manager *mm, void *mqd,
177			enum kfd_preempt_type type,
178			unsigned int timeout, uint32_t pipe_id,
179			uint32_t queue_id)
180{
181	return mm->dev->kfd2kgd->hqd_destroy
182		(mm->dev->kgd, type, timeout,
183		pipe_id, queue_id);
184}
185
186static void uninit_mqd(struct mqd_manager *mm, void *mqd,
187			struct kfd_mem_obj *mqd_mem_obj)
188{
189	BUG_ON(!mm || !mqd);
190	kfd_gtt_sa_free(mm->dev, mqd_mem_obj);
191}
192
193static bool is_occupied(struct mqd_manager *mm, void *mqd,
194			uint64_t queue_address,	uint32_t pipe_id,
195			uint32_t queue_id)
196{
197	return mm->dev->kfd2kgd->hqd_is_occupied(
198		mm->dev->kgd, queue_address,
199		pipe_id, queue_id);
200}
201
202static int init_mqd_hiq(struct mqd_manager *mm, void **mqd,
203			struct kfd_mem_obj **mqd_mem_obj, uint64_t *gart_addr,
204			struct queue_properties *q)
205{
206	struct vi_mqd *m;
207	int retval = init_mqd(mm, mqd, mqd_mem_obj, gart_addr, q);
208
209	if (retval != 0)
210		return retval;
211
212	m = get_mqd(*mqd);
213
214	m->cp_hqd_pq_control |= 1 << CP_HQD_PQ_CONTROL__PRIV_STATE__SHIFT |
215			1 << CP_HQD_PQ_CONTROL__KMD_QUEUE__SHIFT;
216
217	return retval;
218}
219
220static int update_mqd_hiq(struct mqd_manager *mm, void *mqd,
221			struct queue_properties *q)
222{
223	struct vi_mqd *m;
224	int retval = __update_mqd(mm, mqd, q, MTYPE_UC, 0);
225
226	if (retval != 0)
227		return retval;
228
229	m = get_mqd(mqd);
230	m->cp_hqd_vmid = q->vmid;
231	return retval;
232}
233
234struct mqd_manager *mqd_manager_init_vi(enum KFD_MQD_TYPE type,
235		struct kfd_dev *dev)
236{
237	struct mqd_manager *mqd;
238
239	BUG_ON(!dev);
240	BUG_ON(type >= KFD_MQD_TYPE_MAX);
241
242	pr_debug("kfd: In func %s\n", __func__);
243
244	mqd = kzalloc(sizeof(struct mqd_manager), GFP_KERNEL);
245	if (!mqd)
246		return NULL;
247
248	mqd->dev = dev;
249
250	switch (type) {
251	case KFD_MQD_TYPE_CP:
252	case KFD_MQD_TYPE_COMPUTE:
253		mqd->init_mqd = init_mqd;
254		mqd->uninit_mqd = uninit_mqd;
255		mqd->load_mqd = load_mqd;
256		mqd->update_mqd = update_mqd;
257		mqd->destroy_mqd = destroy_mqd;
258		mqd->is_occupied = is_occupied;
259		break;
260	case KFD_MQD_TYPE_HIQ:
261		mqd->init_mqd = init_mqd_hiq;
262		mqd->uninit_mqd = uninit_mqd;
263		mqd->load_mqd = load_mqd;
264		mqd->update_mqd = update_mqd_hiq;
265		mqd->destroy_mqd = destroy_mqd;
266		mqd->is_occupied = is_occupied;
267		break;
268	case KFD_MQD_TYPE_SDMA:
269		break;
270	default:
271		kfree(mqd);
272		return NULL;
273	}
274
275	return mqd;
276}