1/*
  2 * Copyright 2016 Advanced Micro Devices, Inc.
  3 *
  4 * Permission is hereby granted, free of charge, to any person obtaining a
  5 * copy of this software and associated documentation files (the "Software"),
  6 * to deal in the Software without restriction, including without limitation
  7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  8 * and/or sell copies of the Software, and to permit persons to whom the
  9 * Software is furnished to do so, subject to the following conditions:
 10 *
 11 * The above copyright notice and this permission notice shall be included in
 12 * all copies or substantial portions of the Software.
 13 *
 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
 20 * OTHER DEALINGS IN THE SOFTWARE.
 21 *
 22 * Authors: Christian König
 23 */
 24
 25#include <drm/drmP.h>
 26#include "amdgpu.h"
 27
 28struct amdgpu_vram_mgr {
 29	struct drm_mm mm;
 30	spinlock_t lock;
 31	atomic64_t usage;
 32	atomic64_t vis_usage;
 33};
 34
 35/**
 36 * amdgpu_vram_mgr_init - init VRAM manager and DRM MM
 37 *
 38 * @man: TTM memory type manager
 39 * @p_size: maximum size of VRAM
 40 *
 41 * Allocate and initialize the VRAM manager.
 42 */
 43static int amdgpu_vram_mgr_init(struct ttm_mem_type_manager *man,
 44				unsigned long p_size)
 45{
 46	struct amdgpu_vram_mgr *mgr;
 47
 48	mgr = kzalloc(sizeof(*mgr), GFP_KERNEL);
 49	if (!mgr)
 50		return -ENOMEM;
 51
 52	drm_mm_init(&mgr->mm, 0, p_size);
 53	spin_lock_init(&mgr->lock);
 54	man->priv = mgr;
 55	return 0;
 56}
 57
 58/**
 59 * amdgpu_vram_mgr_fini - free and destroy VRAM manager
 60 *
 61 * @man: TTM memory type manager
 62 *
 63 * Destroy and free the VRAM manager, returns -EBUSY if ranges are still
 64 * allocated inside it.
 65 */
 66static int amdgpu_vram_mgr_fini(struct ttm_mem_type_manager *man)
 67{
 68	struct amdgpu_vram_mgr *mgr = man->priv;
 69
 70	spin_lock(&mgr->lock);
 71	drm_mm_takedown(&mgr->mm);
 72	spin_unlock(&mgr->lock);
 73	kfree(mgr);
 74	man->priv = NULL;
 75	return 0;
 76}
 77
 78/**
 79 * amdgpu_vram_mgr_vis_size - Calculate visible node size
 80 *
 81 * @adev: amdgpu device structure
 82 * @node: MM node structure
 83 *
 84 * Calculate how many bytes of the MM node are inside visible VRAM
 85 */
 86static u64 amdgpu_vram_mgr_vis_size(struct amdgpu_device *adev,
 87				    struct drm_mm_node *node)
 88{
 89	uint64_t start = node->start << PAGE_SHIFT;
 90	uint64_t end = (node->size + node->start) << PAGE_SHIFT;
 91
 92	if (start >= adev->gmc.visible_vram_size)
 93		return 0;
 94
 95	return (end > adev->gmc.visible_vram_size ?
 96		adev->gmc.visible_vram_size : end) - start;
 97}
 98
 99/**
100 * amdgpu_vram_mgr_new - allocate new ranges
101 *
102 * @man: TTM memory type manager
103 * @tbo: TTM BO we need this range for
104 * @place: placement flags and restrictions
105 * @mem: the resulting mem object
106 *
107 * Allocate VRAM for the given BO.
108 */
109static int amdgpu_vram_mgr_new(struct ttm_mem_type_manager *man,
110			       struct ttm_buffer_object *tbo,
111			       const struct ttm_place *place,
112			       struct ttm_mem_reg *mem)
113{
114	struct amdgpu_device *adev = amdgpu_ttm_adev(man->bdev);
115	struct amdgpu_vram_mgr *mgr = man->priv;
116	struct drm_mm *mm = &mgr->mm;
117	struct drm_mm_node *nodes;
118	enum drm_mm_insert_mode mode;
119	unsigned long lpfn, num_nodes, pages_per_node, pages_left;
120	uint64_t usage = 0, vis_usage = 0;
121	unsigned i;
122	int r;
123
124	lpfn = place->lpfn;
125	if (!lpfn)
126		lpfn = man->size;
127
128	if (place->flags & TTM_PL_FLAG_CONTIGUOUS ||
129	    amdgpu_vram_page_split == -1) {
130		pages_per_node = ~0ul;
131		num_nodes = 1;
132	} else {
133		pages_per_node = max((uint32_t)amdgpu_vram_page_split,
134				     mem->page_alignment);
135		num_nodes = DIV_ROUND_UP(mem->num_pages, pages_per_node);
136	}
137
138	nodes = kcalloc(num_nodes, sizeof(*nodes), GFP_KERNEL);
139	if (!nodes)
140		return -ENOMEM;
141
142	mode = DRM_MM_INSERT_BEST;
143	if (place->flags & TTM_PL_FLAG_TOPDOWN)
144		mode = DRM_MM_INSERT_HIGH;
145
146	mem->start = 0;
147	pages_left = mem->num_pages;
148
149	spin_lock(&mgr->lock);
150	for (i = 0; i < num_nodes; ++i) {
151		unsigned long pages = min(pages_left, pages_per_node);
152		uint32_t alignment = mem->page_alignment;
153		unsigned long start;
154
155		if (pages == pages_per_node)
156			alignment = pages_per_node;
157
158		r = drm_mm_insert_node_in_range(mm, &nodes[i],
159						pages, alignment, 0,
160						place->fpfn, lpfn,
161						mode);
162		if (unlikely(r))
163			goto error;
164
165		usage += nodes[i].size << PAGE_SHIFT;
166		vis_usage += amdgpu_vram_mgr_vis_size(adev, &nodes[i]);
167
168		/* Calculate a virtual BO start address to easily check if
169		 * everything is CPU accessible.
170		 */
171		start = nodes[i].start + nodes[i].size;
172		if (start > mem->num_pages)
173			start -= mem->num_pages;
174		else
175			start = 0;
176		mem->start = max(mem->start, start);
177		pages_left -= pages;
178	}
179	spin_unlock(&mgr->lock);
180
181	atomic64_add(usage, &mgr->usage);
182	atomic64_add(vis_usage, &mgr->vis_usage);
183
184	mem->mm_node = nodes;
185
186	return 0;
187
188error:
189	while (i--)
190		drm_mm_remove_node(&nodes[i]);
191	spin_unlock(&mgr->lock);
192
193	kfree(nodes);
194	return r == -ENOSPC ? 0 : r;
195}
196
197/**
198 * amdgpu_vram_mgr_del - free ranges
199 *
200 * @man: TTM memory type manager
201 * @tbo: TTM BO we need this range for
202 * @place: placement flags and restrictions
203 * @mem: TTM memory object
204 *
205 * Free the allocated VRAM again.
206 */
207static void amdgpu_vram_mgr_del(struct ttm_mem_type_manager *man,
208				struct ttm_mem_reg *mem)
209{
210	struct amdgpu_device *adev = amdgpu_ttm_adev(man->bdev);
211	struct amdgpu_vram_mgr *mgr = man->priv;
212	struct drm_mm_node *nodes = mem->mm_node;
213	uint64_t usage = 0, vis_usage = 0;
214	unsigned pages = mem->num_pages;
215
216	if (!mem->mm_node)
217		return;
218
219	spin_lock(&mgr->lock);
220	while (pages) {
221		pages -= nodes->size;
222		drm_mm_remove_node(nodes);
223		usage += nodes->size << PAGE_SHIFT;
224		vis_usage += amdgpu_vram_mgr_vis_size(adev, nodes);
225		++nodes;
226	}
227	spin_unlock(&mgr->lock);
228
229	atomic64_sub(usage, &mgr->usage);
230	atomic64_sub(vis_usage, &mgr->vis_usage);
231
232	kfree(mem->mm_node);
233	mem->mm_node = NULL;
234}
235
236/**
237 * amdgpu_vram_mgr_usage - how many bytes are used in this domain
238 *
239 * @man: TTM memory type manager
240 *
241 * Returns how many bytes are used in this domain.
242 */
243uint64_t amdgpu_vram_mgr_usage(struct ttm_mem_type_manager *man)
244{
245	struct amdgpu_vram_mgr *mgr = man->priv;
246
247	return atomic64_read(&mgr->usage);
248}
249
250/**
251 * amdgpu_vram_mgr_vis_usage - how many bytes are used in the visible part
252 *
253 * @man: TTM memory type manager
254 *
255 * Returns how many bytes are used in the visible part of VRAM
256 */
257uint64_t amdgpu_vram_mgr_vis_usage(struct ttm_mem_type_manager *man)
258{
259	struct amdgpu_vram_mgr *mgr = man->priv;
260
261	return atomic64_read(&mgr->vis_usage);
262}
263
264/**
265 * amdgpu_vram_mgr_debug - dump VRAM table
266 *
267 * @man: TTM memory type manager
268 * @printer: DRM printer to use
269 *
270 * Dump the table content using printk.
271 */
272static void amdgpu_vram_mgr_debug(struct ttm_mem_type_manager *man,
273				  struct drm_printer *printer)
274{
275	struct amdgpu_vram_mgr *mgr = man->priv;
276
277	spin_lock(&mgr->lock);
278	drm_mm_print(&mgr->mm, printer);
279	spin_unlock(&mgr->lock);
280
281	drm_printf(printer, "man size:%llu pages, ram usage:%lluMB, vis usage:%lluMB\n",
282		   man->size, amdgpu_vram_mgr_usage(man) >> 20,
283		   amdgpu_vram_mgr_vis_usage(man) >> 20);
284}
285
286const struct ttm_mem_type_manager_func amdgpu_vram_mgr_func = {
287	.init		= amdgpu_vram_mgr_init,
288	.takedown	= amdgpu_vram_mgr_fini,
289	.get_node	= amdgpu_vram_mgr_new,
290	.put_node	= amdgpu_vram_mgr_del,
291	.debug		= amdgpu_vram_mgr_debug
292};