1/*
  2 * Copyright 2014 Advanced Micro Devices, Inc.
  3 *
  4 * Permission is hereby granted, free of charge, to any person obtaining a
  5 * copy of this software and associated documentation files (the "Software"),
  6 * to deal in the Software without restriction, including without limitation
  7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  8 * and/or sell copies of the Software, and to permit persons to whom the
  9 * Software is furnished to do so, subject to the following conditions:
 10 *
 11 * The above copyright notice and this permission notice shall be included in
 12 * all copies or substantial portions of the Software.
 13 *
 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
 20 * OTHER DEALINGS IN THE SOFTWARE.
 21 *
 22 */
 23
 24#ifndef KFD_DEVICE_QUEUE_MANAGER_H_
 25#define KFD_DEVICE_QUEUE_MANAGER_H_
 26
 27#include <linux/rwsem.h>
 28#include <linux/list.h>
 29#include "kfd_priv.h"
 30#include "kfd_mqd_manager.h"
 31
 32#define KFD_UNMAP_LATENCY_MS			(4000)
 33#define QUEUE_PREEMPT_DEFAULT_TIMEOUT_MS (2 * KFD_UNMAP_LATENCY_MS + 1000)
 34
 35#define CIK_SDMA_QUEUES				(4)
 36#define CIK_SDMA_QUEUES_PER_ENGINE		(2)
 37#define CIK_SDMA_ENGINE_NUM			(2)
 38
 39struct device_process_node {
 40	struct qcm_process_device *qpd;
 41	struct list_head list;
 42};
 43
 44/**
 45 * struct device_queue_manager_ops
 46 *
 47 * @create_queue: Queue creation routine.
 48 *
 49 * @destroy_queue: Queue destruction routine.
 50 *
 51 * @update_queue: Queue update routine.
 52 *
 53 * @get_mqd_manager: Returns the mqd manager according to the mqd type.
 54 *
 55 * @exeute_queues: Dispatches the queues list to the H/W.
 56 *
 57 * @register_process: This routine associates a specific process with device.
 58 *
 59 * @unregister_process: destroys the associations between process to device.
 60 *
 61 * @initialize: Initializes the pipelines and memory module for that device.
 62 *
 63 * @start: Initializes the resources/modules the the device needs for queues
 64 * execution. This function is called on device initialization and after the
 65 * system woke up after suspension.
 66 *
 67 * @stop: This routine stops execution of all the active queue running on the
 68 * H/W and basically this function called on system suspend.
 69 *
 70 * @uninitialize: Destroys all the device queue manager resources allocated in
 71 * initialize routine.
 72 *
 73 * @create_kernel_queue: Creates kernel queue. Used for debug queue.
 74 *
 75 * @destroy_kernel_queue: Destroys kernel queue. Used for debug queue.
 76 *
 77 * @set_cache_memory_policy: Sets memory policy (cached/ non cached) for the
 78 * memory apertures.
 79 *
 80 * @process_termination: Clears all process queues belongs to that device.
 81 *
 82 * @evict_process_queues: Evict all active queues of a process
 83 *
 84 * @restore_process_queues: Restore all evicted queues queues of a process
 85 *
 86 */
 87
 88struct device_queue_manager_ops {
 89	int	(*create_queue)(struct device_queue_manager *dqm,
 90				struct queue *q,
 91				struct qcm_process_device *qpd);
 92
 93	int	(*destroy_queue)(struct device_queue_manager *dqm,
 94				struct qcm_process_device *qpd,
 95				struct queue *q);
 96
 97	int	(*update_queue)(struct device_queue_manager *dqm,
 98				struct queue *q);
 99
100	struct mqd_manager * (*get_mqd_manager)
101					(struct device_queue_manager *dqm,
102					enum KFD_MQD_TYPE type);
103
104	int	(*register_process)(struct device_queue_manager *dqm,
105					struct qcm_process_device *qpd);
106
107	int	(*unregister_process)(struct device_queue_manager *dqm,
108					struct qcm_process_device *qpd);
109
110	int	(*initialize)(struct device_queue_manager *dqm);
111	int	(*start)(struct device_queue_manager *dqm);
112	int	(*stop)(struct device_queue_manager *dqm);
113	void	(*uninitialize)(struct device_queue_manager *dqm);
114	int	(*create_kernel_queue)(struct device_queue_manager *dqm,
115					struct kernel_queue *kq,
116					struct qcm_process_device *qpd);
117
118	void	(*destroy_kernel_queue)(struct device_queue_manager *dqm,
119					struct kernel_queue *kq,
120					struct qcm_process_device *qpd);
121
122	bool	(*set_cache_memory_policy)(struct device_queue_manager *dqm,
123					   struct qcm_process_device *qpd,
124					   enum cache_policy default_policy,
125					   enum cache_policy alternate_policy,
126					   void __user *alternate_aperture_base,
127					   uint64_t alternate_aperture_size);
128
129	int	(*set_trap_handler)(struct device_queue_manager *dqm,
130				    struct qcm_process_device *qpd,
131				    uint64_t tba_addr,
132				    uint64_t tma_addr);
133
134	int (*process_termination)(struct device_queue_manager *dqm,
135			struct qcm_process_device *qpd);
136
137	int (*evict_process_queues)(struct device_queue_manager *dqm,
138				    struct qcm_process_device *qpd);
139	int (*restore_process_queues)(struct device_queue_manager *dqm,
140				      struct qcm_process_device *qpd);
141};
142
143struct device_queue_manager_asic_ops {
144	int	(*update_qpd)(struct device_queue_manager *dqm,
145					struct qcm_process_device *qpd);
146	bool	(*set_cache_memory_policy)(struct device_queue_manager *dqm,
147					   struct qcm_process_device *qpd,
148					   enum cache_policy default_policy,
149					   enum cache_policy alternate_policy,
150					   void __user *alternate_aperture_base,
151					   uint64_t alternate_aperture_size);
152	void	(*init_sdma_vm)(struct device_queue_manager *dqm,
153				struct queue *q,
154				struct qcm_process_device *qpd);
155};
156
157/**
158 * struct device_queue_manager
159 *
160 * This struct is a base class for the kfd queues scheduler in the
161 * device level. The device base class should expose the basic operations
162 * for queue creation and queue destruction. This base class hides the
163 * scheduling mode of the driver and the specific implementation of the
164 * concrete device. This class is the only class in the queues scheduler
165 * that configures the H/W.
166 *
167 */
168
169struct device_queue_manager {
170	struct device_queue_manager_ops ops;
171	struct device_queue_manager_asic_ops asic_ops;
172
173	struct mqd_manager	*mqds[KFD_MQD_TYPE_MAX];
174	struct packet_manager	packets;
175	struct kfd_dev		*dev;
176	struct mutex		lock;
177	struct list_head	queues;
178	unsigned int		processes_count;
179	unsigned int		queue_count;
180	unsigned int		sdma_queue_count;
181	unsigned int		total_queue_count;
182	unsigned int		next_pipe_to_allocate;
183	unsigned int		*allocated_queues;
184	unsigned int		sdma_bitmap;
185	unsigned int		vmid_bitmap;
186	uint64_t		pipelines_addr;
187	struct kfd_mem_obj	*pipeline_mem;
188	uint64_t		fence_gpu_addr;
189	unsigned int		*fence_addr;
190	struct kfd_mem_obj	*fence_mem;
191	bool			active_runlist;
192	int			sched_policy;
193};
194
195void device_queue_manager_init_cik(
196		struct device_queue_manager_asic_ops *asic_ops);
197void device_queue_manager_init_cik_hawaii(
198		struct device_queue_manager_asic_ops *asic_ops);
199void device_queue_manager_init_vi(
200		struct device_queue_manager_asic_ops *asic_ops);
201void device_queue_manager_init_vi_tonga(
202		struct device_queue_manager_asic_ops *asic_ops);
203void program_sh_mem_settings(struct device_queue_manager *dqm,
204					struct qcm_process_device *qpd);
205unsigned int get_queues_num(struct device_queue_manager *dqm);
206unsigned int get_queues_per_pipe(struct device_queue_manager *dqm);
207unsigned int get_pipes_per_mec(struct device_queue_manager *dqm);
208
209static inline unsigned int get_sh_mem_bases_32(struct kfd_process_device *pdd)
210{
211	return (pdd->lds_base >> 16) & 0xFF;
212}
213
214static inline unsigned int
215get_sh_mem_bases_nybble_64(struct kfd_process_device *pdd)
216{
217	return (pdd->lds_base >> 60) & 0x0E;
218}
219
220#endif /* KFD_DEVICE_QUEUE_MANAGER_H_ */