1/*
  2 * Copyright 2016 Advanced Micro Devices, Inc.
  3 *
  4 * Permission is hereby granted, free of charge, to any person obtaining a
  5 * copy of this software and associated documentation files (the "Software"),
  6 * to deal in the Software without restriction, including without limitation
  7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  8 * and/or sell copies of the Software, and to permit persons to whom the
  9 * Software is furnished to do so, subject to the following conditions:
 10 *
 11 * The above copyright notice and this permission notice shall be included in
 12 * all copies or substantial portions of the Software.
 13 *
 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
 20 * OTHER DEALINGS IN THE SOFTWARE.
 21 *
 22 * Authors: Christian König
 23 */
 24#ifndef __AMDGPU_RING_H__
 25#define __AMDGPU_RING_H__
 26
 27#include <drm/amdgpu_drm.h>
 28#include <drm/gpu_scheduler.h>
 29#include <drm/drm_print.h>
 30
 31/* max number of rings */
 32#define AMDGPU_MAX_RINGS		18
 33#define AMDGPU_MAX_GFX_RINGS		1
 34#define AMDGPU_MAX_COMPUTE_RINGS	8
 35#define AMDGPU_MAX_VCE_RINGS		3
 36#define AMDGPU_MAX_UVD_ENC_RINGS	2
 37
 38/* some special values for the owner field */
 39#define AMDGPU_FENCE_OWNER_UNDEFINED	((void *)0ul)
 40#define AMDGPU_FENCE_OWNER_VM		((void *)1ul)
 41#define AMDGPU_FENCE_OWNER_KFD		((void *)2ul)
 42
 43#define AMDGPU_FENCE_FLAG_64BIT         (1 << 0)
 44#define AMDGPU_FENCE_FLAG_INT           (1 << 1)
 45
 46enum amdgpu_ring_type {
 47	AMDGPU_RING_TYPE_GFX,
 48	AMDGPU_RING_TYPE_COMPUTE,
 49	AMDGPU_RING_TYPE_SDMA,
 50	AMDGPU_RING_TYPE_UVD,
 51	AMDGPU_RING_TYPE_VCE,
 52	AMDGPU_RING_TYPE_KIQ,
 53	AMDGPU_RING_TYPE_UVD_ENC,
 54	AMDGPU_RING_TYPE_VCN_DEC,
 55	AMDGPU_RING_TYPE_VCN_ENC
 56};
 57
 58struct amdgpu_device;
 59struct amdgpu_ring;
 60struct amdgpu_ib;
 61struct amdgpu_cs_parser;
 62struct amdgpu_job;
 63
 64/*
 65 * Fences.
 66 */
 67struct amdgpu_fence_driver {
 68	uint64_t			gpu_addr;
 69	volatile uint32_t		*cpu_addr;
 70	/* sync_seq is protected by ring emission lock */
 71	uint32_t			sync_seq;
 72	atomic_t			last_seq;
 73	bool				initialized;
 74	struct amdgpu_irq_src		*irq_src;
 75	unsigned			irq_type;
 76	struct timer_list		fallback_timer;
 77	unsigned			num_fences_mask;
 78	spinlock_t			lock;
 79	struct dma_fence		**fences;
 80};
 81
 82int amdgpu_fence_driver_init(struct amdgpu_device *adev);
 83void amdgpu_fence_driver_fini(struct amdgpu_device *adev);
 84void amdgpu_fence_driver_force_completion(struct amdgpu_ring *ring);
 85
 86int amdgpu_fence_driver_init_ring(struct amdgpu_ring *ring,
 87				  unsigned num_hw_submission);
 88int amdgpu_fence_driver_start_ring(struct amdgpu_ring *ring,
 89				   struct amdgpu_irq_src *irq_src,
 90				   unsigned irq_type);
 91void amdgpu_fence_driver_suspend(struct amdgpu_device *adev);
 92void amdgpu_fence_driver_resume(struct amdgpu_device *adev);
 93int amdgpu_fence_emit(struct amdgpu_ring *ring, struct dma_fence **fence);
 94int amdgpu_fence_emit_polling(struct amdgpu_ring *ring, uint32_t *s);
 95void amdgpu_fence_process(struct amdgpu_ring *ring);
 96int amdgpu_fence_wait_empty(struct amdgpu_ring *ring);
 97signed long amdgpu_fence_wait_polling(struct amdgpu_ring *ring,
 98				      uint32_t wait_seq,
 99				      signed long timeout);
100unsigned amdgpu_fence_count_emitted(struct amdgpu_ring *ring);
101
102/*
103 * Rings.
104 */
105
106/* provided by hw blocks that expose a ring buffer for commands */
107struct amdgpu_ring_funcs {
108	enum amdgpu_ring_type	type;
109	uint32_t		align_mask;
110	u32			nop;
111	bool			support_64bit_ptrs;
112	unsigned		vmhub;
113
114	/* ring read/write ptr handling */
115	u64 (*get_rptr)(struct amdgpu_ring *ring);
116	u64 (*get_wptr)(struct amdgpu_ring *ring);
117	void (*set_wptr)(struct amdgpu_ring *ring);
118	/* validating and patching of IBs */
119	int (*parse_cs)(struct amdgpu_cs_parser *p, uint32_t ib_idx);
120	/* constants to calculate how many DW are needed for an emit */
121	unsigned emit_frame_size;
122	unsigned emit_ib_size;
123	/* command emit functions */
124	void (*emit_ib)(struct amdgpu_ring *ring,
125			struct amdgpu_ib *ib,
126			unsigned vmid, bool ctx_switch);
127	void (*emit_fence)(struct amdgpu_ring *ring, uint64_t addr,
128			   uint64_t seq, unsigned flags);
129	void (*emit_pipeline_sync)(struct amdgpu_ring *ring);
130	void (*emit_vm_flush)(struct amdgpu_ring *ring, unsigned vmid,
131			      uint64_t pd_addr);
132	void (*emit_hdp_flush)(struct amdgpu_ring *ring);
133	void (*emit_gds_switch)(struct amdgpu_ring *ring, uint32_t vmid,
134				uint32_t gds_base, uint32_t gds_size,
135				uint32_t gws_base, uint32_t gws_size,
136				uint32_t oa_base, uint32_t oa_size);
137	/* testing functions */
138	int (*test_ring)(struct amdgpu_ring *ring);
139	int (*test_ib)(struct amdgpu_ring *ring, long timeout);
140	/* insert NOP packets */
141	void (*insert_nop)(struct amdgpu_ring *ring, uint32_t count);
142	void (*insert_start)(struct amdgpu_ring *ring);
143	void (*insert_end)(struct amdgpu_ring *ring);
144	/* pad the indirect buffer to the necessary number of dw */
145	void (*pad_ib)(struct amdgpu_ring *ring, struct amdgpu_ib *ib);
146	unsigned (*init_cond_exec)(struct amdgpu_ring *ring);
147	void (*patch_cond_exec)(struct amdgpu_ring *ring, unsigned offset);
148	/* note usage for clock and power gating */
149	void (*begin_use)(struct amdgpu_ring *ring);
150	void (*end_use)(struct amdgpu_ring *ring);
151	void (*emit_switch_buffer) (struct amdgpu_ring *ring);
152	void (*emit_cntxcntl) (struct amdgpu_ring *ring, uint32_t flags);
153	void (*emit_rreg)(struct amdgpu_ring *ring, uint32_t reg);
154	void (*emit_wreg)(struct amdgpu_ring *ring, uint32_t reg, uint32_t val);
155	void (*emit_reg_wait)(struct amdgpu_ring *ring, uint32_t reg,
156			      uint32_t val, uint32_t mask);
157	void (*emit_tmz)(struct amdgpu_ring *ring, bool start);
158	/* priority functions */
159	void (*set_priority) (struct amdgpu_ring *ring,
160			      enum drm_sched_priority priority);
161};
162
163struct amdgpu_ring {
164	struct amdgpu_device		*adev;
165	const struct amdgpu_ring_funcs	*funcs;
166	struct amdgpu_fence_driver	fence_drv;
167	struct drm_gpu_scheduler	sched;
168	struct list_head		lru_list;
169
170	struct amdgpu_bo	*ring_obj;
171	volatile uint32_t	*ring;
172	unsigned		rptr_offs;
173	u64			wptr;
174	u64			wptr_old;
175	unsigned		ring_size;
176	unsigned		max_dw;
177	int			count_dw;
178	uint64_t		gpu_addr;
179	uint64_t		ptr_mask;
180	uint32_t		buf_mask;
181	bool			ready;
182	u32			idx;
183	u32			me;
184	u32			pipe;
185	u32			queue;
186	struct amdgpu_bo	*mqd_obj;
187	uint64_t                mqd_gpu_addr;
188	void                    *mqd_ptr;
189	uint64_t                eop_gpu_addr;
190	u32			doorbell_index;
191	bool			use_doorbell;
192	bool			use_pollmem;
193	unsigned		wptr_offs;
194	unsigned		fence_offs;
195	uint64_t		current_ctx;
196	char			name[16];
197	unsigned		cond_exe_offs;
198	u64			cond_exe_gpu_addr;
199	volatile u32		*cond_exe_cpu_addr;
200	unsigned		vm_inv_eng;
201	struct dma_fence	*vmid_wait;
202	bool			has_compute_vm_bug;
203
204	atomic_t		num_jobs[DRM_SCHED_PRIORITY_MAX];
205	struct mutex		priority_mutex;
206	/* protected by priority_mutex */
207	int			priority;
208
209#if defined(CONFIG_DEBUG_FS)
210	struct dentry *ent;
211#endif
212};
213
214int amdgpu_ring_alloc(struct amdgpu_ring *ring, unsigned ndw);
215void amdgpu_ring_insert_nop(struct amdgpu_ring *ring, uint32_t count);
216void amdgpu_ring_generic_pad_ib(struct amdgpu_ring *ring, struct amdgpu_ib *ib);
217void amdgpu_ring_commit(struct amdgpu_ring *ring);
218void amdgpu_ring_undo(struct amdgpu_ring *ring);
219void amdgpu_ring_priority_get(struct amdgpu_ring *ring,
220			      enum drm_sched_priority priority);
221void amdgpu_ring_priority_put(struct amdgpu_ring *ring,
222			      enum drm_sched_priority priority);
223int amdgpu_ring_init(struct amdgpu_device *adev, struct amdgpu_ring *ring,
224		     unsigned ring_size, struct amdgpu_irq_src *irq_src,
225		     unsigned irq_type);
226void amdgpu_ring_fini(struct amdgpu_ring *ring);
227int amdgpu_ring_lru_get(struct amdgpu_device *adev, int type,
228			int *blacklist, int num_blacklist,
229			bool lru_pipe_order, struct amdgpu_ring **ring);
230void amdgpu_ring_lru_touch(struct amdgpu_device *adev, struct amdgpu_ring *ring);
231static inline void amdgpu_ring_clear_ring(struct amdgpu_ring *ring)
232{
233	int i = 0;
234	while (i <= ring->buf_mask)
235		ring->ring[i++] = ring->funcs->nop;
236
237}
238
239static inline void amdgpu_ring_write(struct amdgpu_ring *ring, uint32_t v)
240{
241	if (ring->count_dw <= 0)
242		DRM_ERROR("amdgpu: writing more dwords to the ring than expected!\n");
243	ring->ring[ring->wptr++ & ring->buf_mask] = v;
244	ring->wptr &= ring->ptr_mask;
245	ring->count_dw--;
246}
247
248static inline void amdgpu_ring_write_multiple(struct amdgpu_ring *ring,
249					      void *src, int count_dw)
250{
251	unsigned occupied, chunk1, chunk2;
252	void *dst;
253
254	if (unlikely(ring->count_dw < count_dw))
255		DRM_ERROR("amdgpu: writing more dwords to the ring than expected!\n");
256
257	occupied = ring->wptr & ring->buf_mask;
258	dst = (void *)&ring->ring[occupied];
259	chunk1 = ring->buf_mask + 1 - occupied;
260	chunk1 = (chunk1 >= count_dw) ? count_dw: chunk1;
261	chunk2 = count_dw - chunk1;
262	chunk1 <<= 2;
263	chunk2 <<= 2;
264
265	if (chunk1)
266		memcpy(dst, src, chunk1);
267
268	if (chunk2) {
269		src += chunk1;
270		dst = (void *)ring->ring;
271		memcpy(dst, src, chunk2);
272	}
273
274	ring->wptr += count_dw;
275	ring->wptr &= ring->ptr_mask;
276	ring->count_dw -= count_dw;
277}
278
279#endif