1/*
  2 * Copyright 2018 Advanced Micro Devices, Inc.
  3 *
  4 * Permission is hereby granted, free of charge, to any person obtaining a
  5 * copy of this software and associated documentation files (the "Software"),
  6 * to deal in the Software without restriction, including without limitation
  7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  8 * and/or sell copies of the Software, and to permit persons to whom the
  9 * Software is furnished to do so, subject to the following conditions:
 10 *
 11 * The above copyright notice and this permission notice shall be included in
 12 * all copies or substantial portions of the Software.
 13 *
 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
 20 * OTHER DEALINGS IN THE SOFTWARE.
 21 *
 22 */
 23
 24#include <linux/firmware.h>
 25#include "amdgpu.h"
 26#include "amdgpu_sdma.h"
 27#include "amdgpu_ras.h"
 28
 29#define AMDGPU_CSA_SDMA_SIZE 64
 30/* SDMA CSA reside in the 3rd page of CSA */
 31#define AMDGPU_CSA_SDMA_OFFSET (4096 * 2)
 32
 33/*
 34 * GPU SDMA IP block helpers function.
 35 */
 36
 37struct amdgpu_sdma_instance *amdgpu_sdma_get_instance_from_ring(struct amdgpu_ring *ring)
 38{
 39	struct amdgpu_device *adev = ring->adev;
 40	int i;
 41
 42	for (i = 0; i < adev->sdma.num_instances; i++)
 43		if (ring == &adev->sdma.instance[i].ring ||
 44		    ring == &adev->sdma.instance[i].page)
 45			return &adev->sdma.instance[i];
 46
 47	return NULL;
 48}
 49
 50int amdgpu_sdma_get_index_from_ring(struct amdgpu_ring *ring, uint32_t *index)
 51{
 52	struct amdgpu_device *adev = ring->adev;
 53	int i;
 54
 55	for (i = 0; i < adev->sdma.num_instances; i++) {
 56		if (ring == &adev->sdma.instance[i].ring ||
 57			ring == &adev->sdma.instance[i].page) {
 58			*index = i;
 59			return 0;
 60		}
 61	}
 62
 63	return -EINVAL;
 64}
 65
 66uint64_t amdgpu_sdma_get_csa_mc_addr(struct amdgpu_ring *ring,
 67				     unsigned vmid)
 68{
 69	struct amdgpu_device *adev = ring->adev;
 70	uint64_t csa_mc_addr;
 71	uint32_t index = 0;
 72	int r;
 73
 74	/* don't enable OS preemption on SDMA under SRIOV */
 75	if (amdgpu_sriov_vf(adev) || vmid == 0 || !amdgpu_mcbp)
 76		return 0;
 77
 78	if (ring->is_mes_queue) {
 79		uint32_t offset = 0;
 80
 81		offset = offsetof(struct amdgpu_mes_ctx_meta_data,
 82				  sdma[ring->idx].sdma_meta_data);
 83		csa_mc_addr = amdgpu_mes_ctx_get_offs_gpu_addr(ring, offset);
 84	} else {
 85		r = amdgpu_sdma_get_index_from_ring(ring, &index);
 86
 87		if (r || index > 31)
 88			csa_mc_addr = 0;
 89		else
 90			csa_mc_addr = amdgpu_csa_vaddr(adev) +
 91				AMDGPU_CSA_SDMA_OFFSET +
 92				index * AMDGPU_CSA_SDMA_SIZE;
 93	}
 94
 95	return csa_mc_addr;
 96}
 97
 98int amdgpu_sdma_ras_late_init(struct amdgpu_device *adev,
 99			      struct ras_common_if *ras_block)
100{
101	int r, i;
102
103	r = amdgpu_ras_block_late_init(adev, ras_block);
104	if (r)
105		return r;
106
107	if (amdgpu_ras_is_supported(adev, ras_block->block)) {
108		for (i = 0; i < adev->sdma.num_instances; i++) {
109			r = amdgpu_irq_get(adev, &adev->sdma.ecc_irq,
110				AMDGPU_SDMA_IRQ_INSTANCE0 + i);
111			if (r)
112				goto late_fini;
113		}
114	}
115
116	return 0;
117
118late_fini:
119	amdgpu_ras_block_late_fini(adev, ras_block);
120	return r;
121}
122
123int amdgpu_sdma_process_ras_data_cb(struct amdgpu_device *adev,
124		void *err_data,
125		struct amdgpu_iv_entry *entry)
126{
127	kgd2kfd_set_sram_ecc_flag(adev->kfd.dev);
128
129	if (amdgpu_sriov_vf(adev))
130		return AMDGPU_RAS_SUCCESS;
131
132	amdgpu_ras_reset_gpu(adev);
133
134	return AMDGPU_RAS_SUCCESS;
135}
136
137int amdgpu_sdma_process_ecc_irq(struct amdgpu_device *adev,
138				      struct amdgpu_irq_src *source,
139				      struct amdgpu_iv_entry *entry)
140{
141	struct ras_common_if *ras_if = adev->sdma.ras_if;
142	struct ras_dispatch_if ih_data = {
143		.entry = entry,
144	};
145
146	if (!ras_if)
147		return 0;
148
149	ih_data.head = *ras_if;
150
151	amdgpu_ras_interrupt_dispatch(adev, &ih_data);
152	return 0;
153}
154
155static int amdgpu_sdma_init_inst_ctx(struct amdgpu_sdma_instance *sdma_inst)
156{
157	int err = 0;
158	uint16_t version_major;
159	const struct common_firmware_header *header = NULL;
160	const struct sdma_firmware_header_v1_0 *hdr;
161	const struct sdma_firmware_header_v2_0 *hdr_v2;
162
163	err = amdgpu_ucode_validate(sdma_inst->fw);
164	if (err)
165		return err;
166
167	header = (const struct common_firmware_header *)
168		sdma_inst->fw->data;
169	version_major = le16_to_cpu(header->header_version_major);
170
171	switch (version_major) {
172	case 1:
173		hdr = (const struct sdma_firmware_header_v1_0 *)sdma_inst->fw->data;
174		sdma_inst->fw_version = le32_to_cpu(hdr->header.ucode_version);
175		sdma_inst->feature_version = le32_to_cpu(hdr->ucode_feature_version);
176		break;
177	case 2:
178		hdr_v2 = (const struct sdma_firmware_header_v2_0 *)sdma_inst->fw->data;
179		sdma_inst->fw_version = le32_to_cpu(hdr_v2->header.ucode_version);
180		sdma_inst->feature_version = le32_to_cpu(hdr_v2->ucode_feature_version);
181		break;
182	default:
183		return -EINVAL;
184	}
185
186	if (sdma_inst->feature_version >= 20)
187		sdma_inst->burst_nop = true;
188
189	return 0;
190}
191
192void amdgpu_sdma_destroy_inst_ctx(struct amdgpu_device *adev,
193				  bool duplicate)
194{
195	int i;
196
197	for (i = 0; i < adev->sdma.num_instances; i++) {
198		release_firmware(adev->sdma.instance[i].fw);
199		if (duplicate)
200			break;
201	}
202
203	memset((void *)adev->sdma.instance, 0,
204	       sizeof(struct amdgpu_sdma_instance) * AMDGPU_MAX_SDMA_INSTANCES);
205}
206
207int amdgpu_sdma_init_microcode(struct amdgpu_device *adev,
208			       char *fw_name, u32 instance,
209			       bool duplicate)
210{
211	struct amdgpu_firmware_info *info = NULL;
212	const struct common_firmware_header *header = NULL;
213	int err = 0, i;
214	const struct sdma_firmware_header_v2_0 *sdma_hdr;
215	uint16_t version_major;
216
217	err = request_firmware(&adev->sdma.instance[instance].fw, fw_name, adev->dev);
218	if (err)
219		goto out;
220
221	header = (const struct common_firmware_header *)
222		adev->sdma.instance[instance].fw->data;
223	version_major = le16_to_cpu(header->header_version_major);
224
225	if ((duplicate && instance) || (!duplicate && version_major > 1)) {
226		err = -EINVAL;
227		goto out;
228	}
229
230	err = amdgpu_sdma_init_inst_ctx(&adev->sdma.instance[instance]);
231	if (err)
232		goto out;
233
234	if (duplicate) {
235		for (i = 1; i < adev->sdma.num_instances; i++)
236			memcpy((void *)&adev->sdma.instance[i],
237			       (void *)&adev->sdma.instance[0],
238			       sizeof(struct amdgpu_sdma_instance));
239	}
240
241	if (amdgpu_sriov_vf(adev))
242		return 0;
243
244	DRM_DEBUG("psp_load == '%s'\n",
245		  adev->firmware.load_type == AMDGPU_FW_LOAD_PSP ? "true" : "false");
246
247	if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
248		switch (version_major) {
249		case 1:
250			for (i = 0; i < adev->sdma.num_instances; i++) {
251				if (!duplicate && (instance != i))
252					continue;
253				else {
254					info = &adev->firmware.ucode[AMDGPU_UCODE_ID_SDMA0 + i];
255					info->ucode_id = AMDGPU_UCODE_ID_SDMA0 + i;
256					info->fw = adev->sdma.instance[i].fw;
257					adev->firmware.fw_size +=
258						ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE);
259				}
260			}
261			break;
262		case 2:
263			sdma_hdr = (const struct sdma_firmware_header_v2_0 *)
264				adev->sdma.instance[0].fw->data;
265			info = &adev->firmware.ucode[AMDGPU_UCODE_ID_SDMA_UCODE_TH0];
266			info->ucode_id = AMDGPU_UCODE_ID_SDMA_UCODE_TH0;
267			info->fw = adev->sdma.instance[0].fw;
268			adev->firmware.fw_size +=
269				ALIGN(le32_to_cpu(sdma_hdr->ctx_ucode_size_bytes), PAGE_SIZE);
270			info = &adev->firmware.ucode[AMDGPU_UCODE_ID_SDMA_UCODE_TH1];
271			info->ucode_id = AMDGPU_UCODE_ID_SDMA_UCODE_TH1;
272			info->fw = adev->sdma.instance[0].fw;
273			adev->firmware.fw_size +=
274				ALIGN(le32_to_cpu(sdma_hdr->ctl_ucode_size_bytes), PAGE_SIZE);
275			break;
276		default:
277			err = -EINVAL;
278		}
279	}
280
281out:
282	if (err) {
283		DRM_ERROR("SDMA: Failed to init firmware \"%s\"\n", fw_name);
284		amdgpu_sdma_destroy_inst_ctx(adev, duplicate);
285	}
286	return err;
287}
288
289void amdgpu_sdma_unset_buffer_funcs_helper(struct amdgpu_device *adev)
290{
291	struct amdgpu_ring *sdma;
292	int i;
293
294	for (i = 0; i < adev->sdma.num_instances; i++) {
295		if (adev->sdma.has_page_queue) {
296			sdma = &adev->sdma.instance[i].page;
297			if (adev->mman.buffer_funcs_ring == sdma) {
298				amdgpu_ttm_set_buffer_funcs_status(adev, false);
299				break;
300			}
301		}
302		sdma = &adev->sdma.instance[i].ring;
303		if (adev->mman.buffer_funcs_ring == sdma) {
304			amdgpu_ttm_set_buffer_funcs_status(adev, false);
305			break;
306		}
307	}
308}