Loading...
1/*
2 * Copyright 2008 Jerome Glisse.
3 * All Rights Reserved.
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the "Software"),
7 * to deal in the Software without restriction, including without limitation
8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9 * and/or sell copies of the Software, and to permit persons to whom the
10 * Software is furnished to do so, subject to the following conditions:
11 *
12 * The above copyright notice and this permission notice (including the next
13 * paragraph) shall be included in all copies or substantial portions of the
14 * Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * PRECISION INSIGHT AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
22 * DEALINGS IN THE SOFTWARE.
23 *
24 * Authors:
25 * Jerome Glisse <glisse@freedesktop.org>
26 */
27#include "drmP.h"
28#include "radeon_drm.h"
29#include "radeon_reg.h"
30#include "radeon.h"
31
32void r100_cs_dump_packet(struct radeon_cs_parser *p,
33 struct radeon_cs_packet *pkt);
34
35int radeon_cs_parser_relocs(struct radeon_cs_parser *p)
36{
37 struct drm_device *ddev = p->rdev->ddev;
38 struct radeon_cs_chunk *chunk;
39 unsigned i, j;
40 bool duplicate;
41
42 if (p->chunk_relocs_idx == -1) {
43 return 0;
44 }
45 chunk = &p->chunks[p->chunk_relocs_idx];
46 /* FIXME: we assume that each relocs use 4 dwords */
47 p->nrelocs = chunk->length_dw / 4;
48 p->relocs_ptr = kcalloc(p->nrelocs, sizeof(void *), GFP_KERNEL);
49 if (p->relocs_ptr == NULL) {
50 return -ENOMEM;
51 }
52 p->relocs = kcalloc(p->nrelocs, sizeof(struct radeon_cs_reloc), GFP_KERNEL);
53 if (p->relocs == NULL) {
54 return -ENOMEM;
55 }
56 for (i = 0; i < p->nrelocs; i++) {
57 struct drm_radeon_cs_reloc *r;
58
59 duplicate = false;
60 r = (struct drm_radeon_cs_reloc *)&chunk->kdata[i*4];
61 for (j = 0; j < p->nrelocs; j++) {
62 if (r->handle == p->relocs[j].handle) {
63 p->relocs_ptr[i] = &p->relocs[j];
64 duplicate = true;
65 break;
66 }
67 }
68 if (!duplicate) {
69 p->relocs[i].gobj = drm_gem_object_lookup(ddev,
70 p->filp,
71 r->handle);
72 if (p->relocs[i].gobj == NULL) {
73 DRM_ERROR("gem object lookup failed 0x%x\n",
74 r->handle);
75 return -ENOENT;
76 }
77 p->relocs_ptr[i] = &p->relocs[i];
78 p->relocs[i].robj = gem_to_radeon_bo(p->relocs[i].gobj);
79 p->relocs[i].lobj.bo = p->relocs[i].robj;
80 p->relocs[i].lobj.wdomain = r->write_domain;
81 p->relocs[i].lobj.rdomain = r->read_domains;
82 p->relocs[i].lobj.tv.bo = &p->relocs[i].robj->tbo;
83 p->relocs[i].handle = r->handle;
84 p->relocs[i].flags = r->flags;
85 radeon_bo_list_add_object(&p->relocs[i].lobj,
86 &p->validated);
87 }
88 }
89 return radeon_bo_list_validate(&p->validated);
90}
91
92int radeon_cs_parser_init(struct radeon_cs_parser *p, void *data)
93{
94 struct drm_radeon_cs *cs = data;
95 uint64_t *chunk_array_ptr;
96 unsigned size, i;
97
98 if (!cs->num_chunks) {
99 return 0;
100 }
101 /* get chunks */
102 INIT_LIST_HEAD(&p->validated);
103 p->idx = 0;
104 p->chunk_ib_idx = -1;
105 p->chunk_relocs_idx = -1;
106 p->chunks_array = kcalloc(cs->num_chunks, sizeof(uint64_t), GFP_KERNEL);
107 if (p->chunks_array == NULL) {
108 return -ENOMEM;
109 }
110 chunk_array_ptr = (uint64_t *)(unsigned long)(cs->chunks);
111 if (DRM_COPY_FROM_USER(p->chunks_array, chunk_array_ptr,
112 sizeof(uint64_t)*cs->num_chunks)) {
113 return -EFAULT;
114 }
115 p->nchunks = cs->num_chunks;
116 p->chunks = kcalloc(p->nchunks, sizeof(struct radeon_cs_chunk), GFP_KERNEL);
117 if (p->chunks == NULL) {
118 return -ENOMEM;
119 }
120 for (i = 0; i < p->nchunks; i++) {
121 struct drm_radeon_cs_chunk __user **chunk_ptr = NULL;
122 struct drm_radeon_cs_chunk user_chunk;
123 uint32_t __user *cdata;
124
125 chunk_ptr = (void __user*)(unsigned long)p->chunks_array[i];
126 if (DRM_COPY_FROM_USER(&user_chunk, chunk_ptr,
127 sizeof(struct drm_radeon_cs_chunk))) {
128 return -EFAULT;
129 }
130 p->chunks[i].length_dw = user_chunk.length_dw;
131 p->chunks[i].kdata = NULL;
132 p->chunks[i].chunk_id = user_chunk.chunk_id;
133
134 if (p->chunks[i].chunk_id == RADEON_CHUNK_ID_RELOCS) {
135 p->chunk_relocs_idx = i;
136 }
137 if (p->chunks[i].chunk_id == RADEON_CHUNK_ID_IB) {
138 p->chunk_ib_idx = i;
139 /* zero length IB isn't useful */
140 if (p->chunks[i].length_dw == 0)
141 return -EINVAL;
142 }
143
144 p->chunks[i].length_dw = user_chunk.length_dw;
145 p->chunks[i].user_ptr = (void __user *)(unsigned long)user_chunk.chunk_data;
146
147 cdata = (uint32_t *)(unsigned long)user_chunk.chunk_data;
148 if (p->chunks[i].chunk_id != RADEON_CHUNK_ID_IB) {
149 size = p->chunks[i].length_dw * sizeof(uint32_t);
150 p->chunks[i].kdata = kmalloc(size, GFP_KERNEL);
151 if (p->chunks[i].kdata == NULL) {
152 return -ENOMEM;
153 }
154 if (DRM_COPY_FROM_USER(p->chunks[i].kdata,
155 p->chunks[i].user_ptr, size)) {
156 return -EFAULT;
157 }
158 } else {
159 p->chunks[i].kpage[0] = kmalloc(PAGE_SIZE, GFP_KERNEL);
160 p->chunks[i].kpage[1] = kmalloc(PAGE_SIZE, GFP_KERNEL);
161 if (p->chunks[i].kpage[0] == NULL || p->chunks[i].kpage[1] == NULL) {
162 kfree(p->chunks[i].kpage[0]);
163 kfree(p->chunks[i].kpage[1]);
164 return -ENOMEM;
165 }
166 p->chunks[i].kpage_idx[0] = -1;
167 p->chunks[i].kpage_idx[1] = -1;
168 p->chunks[i].last_copied_page = -1;
169 p->chunks[i].last_page_index = ((p->chunks[i].length_dw * 4) - 1) / PAGE_SIZE;
170 }
171 }
172 if (p->chunks[p->chunk_ib_idx].length_dw > (16 * 1024)) {
173 DRM_ERROR("cs IB too big: %d\n",
174 p->chunks[p->chunk_ib_idx].length_dw);
175 return -EINVAL;
176 }
177 return 0;
178}
179
180/**
181 * cs_parser_fini() - clean parser states
182 * @parser: parser structure holding parsing context.
183 * @error: error number
184 *
185 * If error is set than unvalidate buffer, otherwise just free memory
186 * used by parsing context.
187 **/
188static void radeon_cs_parser_fini(struct radeon_cs_parser *parser, int error)
189{
190 unsigned i;
191
192
193 if (!error && parser->ib)
194 ttm_eu_fence_buffer_objects(&parser->validated,
195 parser->ib->fence);
196 else
197 ttm_eu_backoff_reservation(&parser->validated);
198
199 if (parser->relocs != NULL) {
200 for (i = 0; i < parser->nrelocs; i++) {
201 if (parser->relocs[i].gobj)
202 drm_gem_object_unreference_unlocked(parser->relocs[i].gobj);
203 }
204 }
205 kfree(parser->track);
206 kfree(parser->relocs);
207 kfree(parser->relocs_ptr);
208 for (i = 0; i < parser->nchunks; i++) {
209 kfree(parser->chunks[i].kdata);
210 kfree(parser->chunks[i].kpage[0]);
211 kfree(parser->chunks[i].kpage[1]);
212 }
213 kfree(parser->chunks);
214 kfree(parser->chunks_array);
215 radeon_ib_free(parser->rdev, &parser->ib);
216}
217
218int radeon_cs_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
219{
220 struct radeon_device *rdev = dev->dev_private;
221 struct radeon_cs_parser parser;
222 struct radeon_cs_chunk *ib_chunk;
223 int r;
224
225 mutex_lock(&rdev->cs_mutex);
226 /* initialize parser */
227 memset(&parser, 0, sizeof(struct radeon_cs_parser));
228 parser.filp = filp;
229 parser.rdev = rdev;
230 parser.dev = rdev->dev;
231 parser.family = rdev->family;
232 r = radeon_cs_parser_init(&parser, data);
233 if (r) {
234 DRM_ERROR("Failed to initialize parser !\n");
235 radeon_cs_parser_fini(&parser, r);
236 mutex_unlock(&rdev->cs_mutex);
237 return r;
238 }
239 r = radeon_ib_get(rdev, &parser.ib);
240 if (r) {
241 DRM_ERROR("Failed to get ib !\n");
242 radeon_cs_parser_fini(&parser, r);
243 mutex_unlock(&rdev->cs_mutex);
244 return r;
245 }
246 r = radeon_cs_parser_relocs(&parser);
247 if (r) {
248 if (r != -ERESTARTSYS)
249 DRM_ERROR("Failed to parse relocation %d!\n", r);
250 radeon_cs_parser_fini(&parser, r);
251 mutex_unlock(&rdev->cs_mutex);
252 return r;
253 }
254 /* Copy the packet into the IB, the parser will read from the
255 * input memory (cached) and write to the IB (which can be
256 * uncached). */
257 ib_chunk = &parser.chunks[parser.chunk_ib_idx];
258 parser.ib->length_dw = ib_chunk->length_dw;
259 r = radeon_cs_parse(&parser);
260 if (r || parser.parser_error) {
261 DRM_ERROR("Invalid command stream !\n");
262 radeon_cs_parser_fini(&parser, r);
263 mutex_unlock(&rdev->cs_mutex);
264 return r;
265 }
266 r = radeon_cs_finish_pages(&parser);
267 if (r) {
268 DRM_ERROR("Invalid command stream !\n");
269 radeon_cs_parser_fini(&parser, r);
270 mutex_unlock(&rdev->cs_mutex);
271 return r;
272 }
273 r = radeon_ib_schedule(rdev, parser.ib);
274 if (r) {
275 DRM_ERROR("Failed to schedule IB !\n");
276 }
277 radeon_cs_parser_fini(&parser, r);
278 mutex_unlock(&rdev->cs_mutex);
279 return r;
280}
281
282int radeon_cs_finish_pages(struct radeon_cs_parser *p)
283{
284 struct radeon_cs_chunk *ibc = &p->chunks[p->chunk_ib_idx];
285 int i;
286 int size = PAGE_SIZE;
287
288 for (i = ibc->last_copied_page + 1; i <= ibc->last_page_index; i++) {
289 if (i == ibc->last_page_index) {
290 size = (ibc->length_dw * 4) % PAGE_SIZE;
291 if (size == 0)
292 size = PAGE_SIZE;
293 }
294
295 if (DRM_COPY_FROM_USER(p->ib->ptr + (i * (PAGE_SIZE/4)),
296 ibc->user_ptr + (i * PAGE_SIZE),
297 size))
298 return -EFAULT;
299 }
300 return 0;
301}
302
303int radeon_cs_update_pages(struct radeon_cs_parser *p, int pg_idx)
304{
305 int new_page;
306 struct radeon_cs_chunk *ibc = &p->chunks[p->chunk_ib_idx];
307 int i;
308 int size = PAGE_SIZE;
309
310 for (i = ibc->last_copied_page + 1; i < pg_idx; i++) {
311 if (DRM_COPY_FROM_USER(p->ib->ptr + (i * (PAGE_SIZE/4)),
312 ibc->user_ptr + (i * PAGE_SIZE),
313 PAGE_SIZE)) {
314 p->parser_error = -EFAULT;
315 return 0;
316 }
317 }
318
319 new_page = ibc->kpage_idx[0] < ibc->kpage_idx[1] ? 0 : 1;
320
321 if (pg_idx == ibc->last_page_index) {
322 size = (ibc->length_dw * 4) % PAGE_SIZE;
323 if (size == 0)
324 size = PAGE_SIZE;
325 }
326
327 if (DRM_COPY_FROM_USER(ibc->kpage[new_page],
328 ibc->user_ptr + (pg_idx * PAGE_SIZE),
329 size)) {
330 p->parser_error = -EFAULT;
331 return 0;
332 }
333
334 /* copy to IB here */
335 memcpy((void *)(p->ib->ptr+(pg_idx*(PAGE_SIZE/4))), ibc->kpage[new_page], size);
336
337 ibc->last_copied_page = pg_idx;
338 ibc->kpage_idx[new_page] = pg_idx;
339
340 return new_page;
341}
1/*
2 * Copyright 2008 Jerome Glisse.
3 * All Rights Reserved.
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the "Software"),
7 * to deal in the Software without restriction, including without limitation
8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9 * and/or sell copies of the Software, and to permit persons to whom the
10 * Software is furnished to do so, subject to the following conditions:
11 *
12 * The above copyright notice and this permission notice (including the next
13 * paragraph) shall be included in all copies or substantial portions of the
14 * Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * PRECISION INSIGHT AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
22 * DEALINGS IN THE SOFTWARE.
23 *
24 * Authors:
25 * Jerome Glisse <glisse@freedesktop.org>
26 */
27#include <linux/list_sort.h>
28#include <drm/drmP.h>
29#include <drm/radeon_drm.h>
30#include "radeon_reg.h"
31#include "radeon.h"
32#include "radeon_trace.h"
33
34#define RADEON_CS_MAX_PRIORITY 32u
35#define RADEON_CS_NUM_BUCKETS (RADEON_CS_MAX_PRIORITY + 1)
36
37/* This is based on the bucket sort with O(n) time complexity.
38 * An item with priority "i" is added to bucket[i]. The lists are then
39 * concatenated in descending order.
40 */
41struct radeon_cs_buckets {
42 struct list_head bucket[RADEON_CS_NUM_BUCKETS];
43};
44
45static void radeon_cs_buckets_init(struct radeon_cs_buckets *b)
46{
47 unsigned i;
48
49 for (i = 0; i < RADEON_CS_NUM_BUCKETS; i++)
50 INIT_LIST_HEAD(&b->bucket[i]);
51}
52
53static void radeon_cs_buckets_add(struct radeon_cs_buckets *b,
54 struct list_head *item, unsigned priority)
55{
56 /* Since buffers which appear sooner in the relocation list are
57 * likely to be used more often than buffers which appear later
58 * in the list, the sort mustn't change the ordering of buffers
59 * with the same priority, i.e. it must be stable.
60 */
61 list_add_tail(item, &b->bucket[min(priority, RADEON_CS_MAX_PRIORITY)]);
62}
63
64static void radeon_cs_buckets_get_list(struct radeon_cs_buckets *b,
65 struct list_head *out_list)
66{
67 unsigned i;
68
69 /* Connect the sorted buckets in the output list. */
70 for (i = 0; i < RADEON_CS_NUM_BUCKETS; i++) {
71 list_splice(&b->bucket[i], out_list);
72 }
73}
74
75static int radeon_cs_parser_relocs(struct radeon_cs_parser *p)
76{
77 struct radeon_cs_chunk *chunk;
78 struct radeon_cs_buckets buckets;
79 unsigned i;
80 bool need_mmap_lock = false;
81 int r;
82
83 if (p->chunk_relocs == NULL) {
84 return 0;
85 }
86 chunk = p->chunk_relocs;
87 p->dma_reloc_idx = 0;
88 /* FIXME: we assume that each relocs use 4 dwords */
89 p->nrelocs = chunk->length_dw / 4;
90 p->relocs = drm_calloc_large(p->nrelocs, sizeof(struct radeon_bo_list));
91 if (p->relocs == NULL) {
92 return -ENOMEM;
93 }
94
95 radeon_cs_buckets_init(&buckets);
96
97 for (i = 0; i < p->nrelocs; i++) {
98 struct drm_radeon_cs_reloc *r;
99 struct drm_gem_object *gobj;
100 unsigned priority;
101
102 r = (struct drm_radeon_cs_reloc *)&chunk->kdata[i*4];
103 gobj = drm_gem_object_lookup(p->filp, r->handle);
104 if (gobj == NULL) {
105 DRM_ERROR("gem object lookup failed 0x%x\n",
106 r->handle);
107 return -ENOENT;
108 }
109 p->relocs[i].robj = gem_to_radeon_bo(gobj);
110
111 /* The userspace buffer priorities are from 0 to 15. A higher
112 * number means the buffer is more important.
113 * Also, the buffers used for write have a higher priority than
114 * the buffers used for read only, which doubles the range
115 * to 0 to 31. 32 is reserved for the kernel driver.
116 */
117 priority = (r->flags & RADEON_RELOC_PRIO_MASK) * 2
118 + !!r->write_domain;
119
120 /* the first reloc of an UVD job is the msg and that must be in
121 VRAM, also but everything into VRAM on AGP cards and older
122 IGP chips to avoid image corruptions */
123 if (p->ring == R600_RING_TYPE_UVD_INDEX &&
124 (i == 0 || drm_pci_device_is_agp(p->rdev->ddev) ||
125 p->rdev->family == CHIP_RS780 ||
126 p->rdev->family == CHIP_RS880)) {
127
128 /* TODO: is this still needed for NI+ ? */
129 p->relocs[i].prefered_domains =
130 RADEON_GEM_DOMAIN_VRAM;
131
132 p->relocs[i].allowed_domains =
133 RADEON_GEM_DOMAIN_VRAM;
134
135 /* prioritize this over any other relocation */
136 priority = RADEON_CS_MAX_PRIORITY;
137 } else {
138 uint32_t domain = r->write_domain ?
139 r->write_domain : r->read_domains;
140
141 if (domain & RADEON_GEM_DOMAIN_CPU) {
142 DRM_ERROR("RADEON_GEM_DOMAIN_CPU is not valid "
143 "for command submission\n");
144 return -EINVAL;
145 }
146
147 p->relocs[i].prefered_domains = domain;
148 if (domain == RADEON_GEM_DOMAIN_VRAM)
149 domain |= RADEON_GEM_DOMAIN_GTT;
150 p->relocs[i].allowed_domains = domain;
151 }
152
153 if (radeon_ttm_tt_has_userptr(p->relocs[i].robj->tbo.ttm)) {
154 uint32_t domain = p->relocs[i].prefered_domains;
155 if (!(domain & RADEON_GEM_DOMAIN_GTT)) {
156 DRM_ERROR("Only RADEON_GEM_DOMAIN_GTT is "
157 "allowed for userptr BOs\n");
158 return -EINVAL;
159 }
160 need_mmap_lock = true;
161 domain = RADEON_GEM_DOMAIN_GTT;
162 p->relocs[i].prefered_domains = domain;
163 p->relocs[i].allowed_domains = domain;
164 }
165
166 p->relocs[i].tv.bo = &p->relocs[i].robj->tbo;
167 p->relocs[i].tv.shared = !r->write_domain;
168
169 radeon_cs_buckets_add(&buckets, &p->relocs[i].tv.head,
170 priority);
171 }
172
173 radeon_cs_buckets_get_list(&buckets, &p->validated);
174
175 if (p->cs_flags & RADEON_CS_USE_VM)
176 p->vm_bos = radeon_vm_get_bos(p->rdev, p->ib.vm,
177 &p->validated);
178 if (need_mmap_lock)
179 down_read(¤t->mm->mmap_sem);
180
181 r = radeon_bo_list_validate(p->rdev, &p->ticket, &p->validated, p->ring);
182
183 if (need_mmap_lock)
184 up_read(¤t->mm->mmap_sem);
185
186 return r;
187}
188
189static int radeon_cs_get_ring(struct radeon_cs_parser *p, u32 ring, s32 priority)
190{
191 p->priority = priority;
192
193 switch (ring) {
194 default:
195 DRM_ERROR("unknown ring id: %d\n", ring);
196 return -EINVAL;
197 case RADEON_CS_RING_GFX:
198 p->ring = RADEON_RING_TYPE_GFX_INDEX;
199 break;
200 case RADEON_CS_RING_COMPUTE:
201 if (p->rdev->family >= CHIP_TAHITI) {
202 if (p->priority > 0)
203 p->ring = CAYMAN_RING_TYPE_CP1_INDEX;
204 else
205 p->ring = CAYMAN_RING_TYPE_CP2_INDEX;
206 } else
207 p->ring = RADEON_RING_TYPE_GFX_INDEX;
208 break;
209 case RADEON_CS_RING_DMA:
210 if (p->rdev->family >= CHIP_CAYMAN) {
211 if (p->priority > 0)
212 p->ring = R600_RING_TYPE_DMA_INDEX;
213 else
214 p->ring = CAYMAN_RING_TYPE_DMA1_INDEX;
215 } else if (p->rdev->family >= CHIP_RV770) {
216 p->ring = R600_RING_TYPE_DMA_INDEX;
217 } else {
218 return -EINVAL;
219 }
220 break;
221 case RADEON_CS_RING_UVD:
222 p->ring = R600_RING_TYPE_UVD_INDEX;
223 break;
224 case RADEON_CS_RING_VCE:
225 /* TODO: only use the low priority ring for now */
226 p->ring = TN_RING_TYPE_VCE1_INDEX;
227 break;
228 }
229 return 0;
230}
231
232static int radeon_cs_sync_rings(struct radeon_cs_parser *p)
233{
234 struct radeon_bo_list *reloc;
235 int r;
236
237 list_for_each_entry(reloc, &p->validated, tv.head) {
238 struct reservation_object *resv;
239
240 resv = reloc->robj->tbo.resv;
241 r = radeon_sync_resv(p->rdev, &p->ib.sync, resv,
242 reloc->tv.shared);
243 if (r)
244 return r;
245 }
246 return 0;
247}
248
249/* XXX: note that this is called from the legacy UMS CS ioctl as well */
250int radeon_cs_parser_init(struct radeon_cs_parser *p, void *data)
251{
252 struct drm_radeon_cs *cs = data;
253 uint64_t *chunk_array_ptr;
254 unsigned size, i;
255 u32 ring = RADEON_CS_RING_GFX;
256 s32 priority = 0;
257
258 INIT_LIST_HEAD(&p->validated);
259
260 if (!cs->num_chunks) {
261 return 0;
262 }
263
264 /* get chunks */
265 p->idx = 0;
266 p->ib.sa_bo = NULL;
267 p->const_ib.sa_bo = NULL;
268 p->chunk_ib = NULL;
269 p->chunk_relocs = NULL;
270 p->chunk_flags = NULL;
271 p->chunk_const_ib = NULL;
272 p->chunks_array = kcalloc(cs->num_chunks, sizeof(uint64_t), GFP_KERNEL);
273 if (p->chunks_array == NULL) {
274 return -ENOMEM;
275 }
276 chunk_array_ptr = (uint64_t *)(unsigned long)(cs->chunks);
277 if (copy_from_user(p->chunks_array, chunk_array_ptr,
278 sizeof(uint64_t)*cs->num_chunks)) {
279 return -EFAULT;
280 }
281 p->cs_flags = 0;
282 p->nchunks = cs->num_chunks;
283 p->chunks = kcalloc(p->nchunks, sizeof(struct radeon_cs_chunk), GFP_KERNEL);
284 if (p->chunks == NULL) {
285 return -ENOMEM;
286 }
287 for (i = 0; i < p->nchunks; i++) {
288 struct drm_radeon_cs_chunk __user **chunk_ptr = NULL;
289 struct drm_radeon_cs_chunk user_chunk;
290 uint32_t __user *cdata;
291
292 chunk_ptr = (void __user*)(unsigned long)p->chunks_array[i];
293 if (copy_from_user(&user_chunk, chunk_ptr,
294 sizeof(struct drm_radeon_cs_chunk))) {
295 return -EFAULT;
296 }
297 p->chunks[i].length_dw = user_chunk.length_dw;
298 if (user_chunk.chunk_id == RADEON_CHUNK_ID_RELOCS) {
299 p->chunk_relocs = &p->chunks[i];
300 }
301 if (user_chunk.chunk_id == RADEON_CHUNK_ID_IB) {
302 p->chunk_ib = &p->chunks[i];
303 /* zero length IB isn't useful */
304 if (p->chunks[i].length_dw == 0)
305 return -EINVAL;
306 }
307 if (user_chunk.chunk_id == RADEON_CHUNK_ID_CONST_IB) {
308 p->chunk_const_ib = &p->chunks[i];
309 /* zero length CONST IB isn't useful */
310 if (p->chunks[i].length_dw == 0)
311 return -EINVAL;
312 }
313 if (user_chunk.chunk_id == RADEON_CHUNK_ID_FLAGS) {
314 p->chunk_flags = &p->chunks[i];
315 /* zero length flags aren't useful */
316 if (p->chunks[i].length_dw == 0)
317 return -EINVAL;
318 }
319
320 size = p->chunks[i].length_dw;
321 cdata = (void __user *)(unsigned long)user_chunk.chunk_data;
322 p->chunks[i].user_ptr = cdata;
323 if (user_chunk.chunk_id == RADEON_CHUNK_ID_CONST_IB)
324 continue;
325
326 if (user_chunk.chunk_id == RADEON_CHUNK_ID_IB) {
327 if (!p->rdev || !(p->rdev->flags & RADEON_IS_AGP))
328 continue;
329 }
330
331 p->chunks[i].kdata = drm_malloc_ab(size, sizeof(uint32_t));
332 size *= sizeof(uint32_t);
333 if (p->chunks[i].kdata == NULL) {
334 return -ENOMEM;
335 }
336 if (copy_from_user(p->chunks[i].kdata, cdata, size)) {
337 return -EFAULT;
338 }
339 if (user_chunk.chunk_id == RADEON_CHUNK_ID_FLAGS) {
340 p->cs_flags = p->chunks[i].kdata[0];
341 if (p->chunks[i].length_dw > 1)
342 ring = p->chunks[i].kdata[1];
343 if (p->chunks[i].length_dw > 2)
344 priority = (s32)p->chunks[i].kdata[2];
345 }
346 }
347
348 /* these are KMS only */
349 if (p->rdev) {
350 if ((p->cs_flags & RADEON_CS_USE_VM) &&
351 !p->rdev->vm_manager.enabled) {
352 DRM_ERROR("VM not active on asic!\n");
353 return -EINVAL;
354 }
355
356 if (radeon_cs_get_ring(p, ring, priority))
357 return -EINVAL;
358
359 /* we only support VM on some SI+ rings */
360 if ((p->cs_flags & RADEON_CS_USE_VM) == 0) {
361 if (p->rdev->asic->ring[p->ring]->cs_parse == NULL) {
362 DRM_ERROR("Ring %d requires VM!\n", p->ring);
363 return -EINVAL;
364 }
365 } else {
366 if (p->rdev->asic->ring[p->ring]->ib_parse == NULL) {
367 DRM_ERROR("VM not supported on ring %d!\n",
368 p->ring);
369 return -EINVAL;
370 }
371 }
372 }
373
374 return 0;
375}
376
377static int cmp_size_smaller_first(void *priv, struct list_head *a,
378 struct list_head *b)
379{
380 struct radeon_bo_list *la = list_entry(a, struct radeon_bo_list, tv.head);
381 struct radeon_bo_list *lb = list_entry(b, struct radeon_bo_list, tv.head);
382
383 /* Sort A before B if A is smaller. */
384 return (int)la->robj->tbo.num_pages - (int)lb->robj->tbo.num_pages;
385}
386
387/**
388 * cs_parser_fini() - clean parser states
389 * @parser: parser structure holding parsing context.
390 * @error: error number
391 *
392 * If error is set than unvalidate buffer, otherwise just free memory
393 * used by parsing context.
394 **/
395static void radeon_cs_parser_fini(struct radeon_cs_parser *parser, int error, bool backoff)
396{
397 unsigned i;
398
399 if (!error) {
400 /* Sort the buffer list from the smallest to largest buffer,
401 * which affects the order of buffers in the LRU list.
402 * This assures that the smallest buffers are added first
403 * to the LRU list, so they are likely to be later evicted
404 * first, instead of large buffers whose eviction is more
405 * expensive.
406 *
407 * This slightly lowers the number of bytes moved by TTM
408 * per frame under memory pressure.
409 */
410 list_sort(NULL, &parser->validated, cmp_size_smaller_first);
411
412 ttm_eu_fence_buffer_objects(&parser->ticket,
413 &parser->validated,
414 &parser->ib.fence->base);
415 } else if (backoff) {
416 ttm_eu_backoff_reservation(&parser->ticket,
417 &parser->validated);
418 }
419
420 if (parser->relocs != NULL) {
421 for (i = 0; i < parser->nrelocs; i++) {
422 struct radeon_bo *bo = parser->relocs[i].robj;
423 if (bo == NULL)
424 continue;
425
426 drm_gem_object_unreference_unlocked(&bo->gem_base);
427 }
428 }
429 kfree(parser->track);
430 drm_free_large(parser->relocs);
431 drm_free_large(parser->vm_bos);
432 for (i = 0; i < parser->nchunks; i++)
433 drm_free_large(parser->chunks[i].kdata);
434 kfree(parser->chunks);
435 kfree(parser->chunks_array);
436 radeon_ib_free(parser->rdev, &parser->ib);
437 radeon_ib_free(parser->rdev, &parser->const_ib);
438}
439
440static int radeon_cs_ib_chunk(struct radeon_device *rdev,
441 struct radeon_cs_parser *parser)
442{
443 int r;
444
445 if (parser->chunk_ib == NULL)
446 return 0;
447
448 if (parser->cs_flags & RADEON_CS_USE_VM)
449 return 0;
450
451 r = radeon_cs_parse(rdev, parser->ring, parser);
452 if (r || parser->parser_error) {
453 DRM_ERROR("Invalid command stream !\n");
454 return r;
455 }
456
457 r = radeon_cs_sync_rings(parser);
458 if (r) {
459 if (r != -ERESTARTSYS)
460 DRM_ERROR("Failed to sync rings: %i\n", r);
461 return r;
462 }
463
464 if (parser->ring == R600_RING_TYPE_UVD_INDEX)
465 radeon_uvd_note_usage(rdev);
466 else if ((parser->ring == TN_RING_TYPE_VCE1_INDEX) ||
467 (parser->ring == TN_RING_TYPE_VCE2_INDEX))
468 radeon_vce_note_usage(rdev);
469
470 r = radeon_ib_schedule(rdev, &parser->ib, NULL, true);
471 if (r) {
472 DRM_ERROR("Failed to schedule IB !\n");
473 }
474 return r;
475}
476
477static int radeon_bo_vm_update_pte(struct radeon_cs_parser *p,
478 struct radeon_vm *vm)
479{
480 struct radeon_device *rdev = p->rdev;
481 struct radeon_bo_va *bo_va;
482 int i, r;
483
484 r = radeon_vm_update_page_directory(rdev, vm);
485 if (r)
486 return r;
487
488 r = radeon_vm_clear_freed(rdev, vm);
489 if (r)
490 return r;
491
492 if (vm->ib_bo_va == NULL) {
493 DRM_ERROR("Tmp BO not in VM!\n");
494 return -EINVAL;
495 }
496
497 r = radeon_vm_bo_update(rdev, vm->ib_bo_va,
498 &rdev->ring_tmp_bo.bo->tbo.mem);
499 if (r)
500 return r;
501
502 for (i = 0; i < p->nrelocs; i++) {
503 struct radeon_bo *bo;
504
505 bo = p->relocs[i].robj;
506 bo_va = radeon_vm_bo_find(vm, bo);
507 if (bo_va == NULL) {
508 dev_err(rdev->dev, "bo %p not in vm %p\n", bo, vm);
509 return -EINVAL;
510 }
511
512 r = radeon_vm_bo_update(rdev, bo_va, &bo->tbo.mem);
513 if (r)
514 return r;
515
516 radeon_sync_fence(&p->ib.sync, bo_va->last_pt_update);
517 }
518
519 return radeon_vm_clear_invalids(rdev, vm);
520}
521
522static int radeon_cs_ib_vm_chunk(struct radeon_device *rdev,
523 struct radeon_cs_parser *parser)
524{
525 struct radeon_fpriv *fpriv = parser->filp->driver_priv;
526 struct radeon_vm *vm = &fpriv->vm;
527 int r;
528
529 if (parser->chunk_ib == NULL)
530 return 0;
531 if ((parser->cs_flags & RADEON_CS_USE_VM) == 0)
532 return 0;
533
534 if (parser->const_ib.length_dw) {
535 r = radeon_ring_ib_parse(rdev, parser->ring, &parser->const_ib);
536 if (r) {
537 return r;
538 }
539 }
540
541 r = radeon_ring_ib_parse(rdev, parser->ring, &parser->ib);
542 if (r) {
543 return r;
544 }
545
546 if (parser->ring == R600_RING_TYPE_UVD_INDEX)
547 radeon_uvd_note_usage(rdev);
548
549 mutex_lock(&vm->mutex);
550 r = radeon_bo_vm_update_pte(parser, vm);
551 if (r) {
552 goto out;
553 }
554
555 r = radeon_cs_sync_rings(parser);
556 if (r) {
557 if (r != -ERESTARTSYS)
558 DRM_ERROR("Failed to sync rings: %i\n", r);
559 goto out;
560 }
561
562 if ((rdev->family >= CHIP_TAHITI) &&
563 (parser->chunk_const_ib != NULL)) {
564 r = radeon_ib_schedule(rdev, &parser->ib, &parser->const_ib, true);
565 } else {
566 r = radeon_ib_schedule(rdev, &parser->ib, NULL, true);
567 }
568
569out:
570 mutex_unlock(&vm->mutex);
571 return r;
572}
573
574static int radeon_cs_handle_lockup(struct radeon_device *rdev, int r)
575{
576 if (r == -EDEADLK) {
577 r = radeon_gpu_reset(rdev);
578 if (!r)
579 r = -EAGAIN;
580 }
581 return r;
582}
583
584static int radeon_cs_ib_fill(struct radeon_device *rdev, struct radeon_cs_parser *parser)
585{
586 struct radeon_cs_chunk *ib_chunk;
587 struct radeon_vm *vm = NULL;
588 int r;
589
590 if (parser->chunk_ib == NULL)
591 return 0;
592
593 if (parser->cs_flags & RADEON_CS_USE_VM) {
594 struct radeon_fpriv *fpriv = parser->filp->driver_priv;
595 vm = &fpriv->vm;
596
597 if ((rdev->family >= CHIP_TAHITI) &&
598 (parser->chunk_const_ib != NULL)) {
599 ib_chunk = parser->chunk_const_ib;
600 if (ib_chunk->length_dw > RADEON_IB_VM_MAX_SIZE) {
601 DRM_ERROR("cs IB CONST too big: %d\n", ib_chunk->length_dw);
602 return -EINVAL;
603 }
604 r = radeon_ib_get(rdev, parser->ring, &parser->const_ib,
605 vm, ib_chunk->length_dw * 4);
606 if (r) {
607 DRM_ERROR("Failed to get const ib !\n");
608 return r;
609 }
610 parser->const_ib.is_const_ib = true;
611 parser->const_ib.length_dw = ib_chunk->length_dw;
612 if (copy_from_user(parser->const_ib.ptr,
613 ib_chunk->user_ptr,
614 ib_chunk->length_dw * 4))
615 return -EFAULT;
616 }
617
618 ib_chunk = parser->chunk_ib;
619 if (ib_chunk->length_dw > RADEON_IB_VM_MAX_SIZE) {
620 DRM_ERROR("cs IB too big: %d\n", ib_chunk->length_dw);
621 return -EINVAL;
622 }
623 }
624 ib_chunk = parser->chunk_ib;
625
626 r = radeon_ib_get(rdev, parser->ring, &parser->ib,
627 vm, ib_chunk->length_dw * 4);
628 if (r) {
629 DRM_ERROR("Failed to get ib !\n");
630 return r;
631 }
632 parser->ib.length_dw = ib_chunk->length_dw;
633 if (ib_chunk->kdata)
634 memcpy(parser->ib.ptr, ib_chunk->kdata, ib_chunk->length_dw * 4);
635 else if (copy_from_user(parser->ib.ptr, ib_chunk->user_ptr, ib_chunk->length_dw * 4))
636 return -EFAULT;
637 return 0;
638}
639
640int radeon_cs_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
641{
642 struct radeon_device *rdev = dev->dev_private;
643 struct radeon_cs_parser parser;
644 int r;
645
646 down_read(&rdev->exclusive_lock);
647 if (!rdev->accel_working) {
648 up_read(&rdev->exclusive_lock);
649 return -EBUSY;
650 }
651 if (rdev->in_reset) {
652 up_read(&rdev->exclusive_lock);
653 r = radeon_gpu_reset(rdev);
654 if (!r)
655 r = -EAGAIN;
656 return r;
657 }
658 /* initialize parser */
659 memset(&parser, 0, sizeof(struct radeon_cs_parser));
660 parser.filp = filp;
661 parser.rdev = rdev;
662 parser.dev = rdev->dev;
663 parser.family = rdev->family;
664 r = radeon_cs_parser_init(&parser, data);
665 if (r) {
666 DRM_ERROR("Failed to initialize parser !\n");
667 radeon_cs_parser_fini(&parser, r, false);
668 up_read(&rdev->exclusive_lock);
669 r = radeon_cs_handle_lockup(rdev, r);
670 return r;
671 }
672
673 r = radeon_cs_ib_fill(rdev, &parser);
674 if (!r) {
675 r = radeon_cs_parser_relocs(&parser);
676 if (r && r != -ERESTARTSYS)
677 DRM_ERROR("Failed to parse relocation %d!\n", r);
678 }
679
680 if (r) {
681 radeon_cs_parser_fini(&parser, r, false);
682 up_read(&rdev->exclusive_lock);
683 r = radeon_cs_handle_lockup(rdev, r);
684 return r;
685 }
686
687 trace_radeon_cs(&parser);
688
689 r = radeon_cs_ib_chunk(rdev, &parser);
690 if (r) {
691 goto out;
692 }
693 r = radeon_cs_ib_vm_chunk(rdev, &parser);
694 if (r) {
695 goto out;
696 }
697out:
698 radeon_cs_parser_fini(&parser, r, true);
699 up_read(&rdev->exclusive_lock);
700 r = radeon_cs_handle_lockup(rdev, r);
701 return r;
702}
703
704/**
705 * radeon_cs_packet_parse() - parse cp packet and point ib index to next packet
706 * @parser: parser structure holding parsing context.
707 * @pkt: where to store packet information
708 *
709 * Assume that chunk_ib_index is properly set. Will return -EINVAL
710 * if packet is bigger than remaining ib size. or if packets is unknown.
711 **/
712int radeon_cs_packet_parse(struct radeon_cs_parser *p,
713 struct radeon_cs_packet *pkt,
714 unsigned idx)
715{
716 struct radeon_cs_chunk *ib_chunk = p->chunk_ib;
717 struct radeon_device *rdev = p->rdev;
718 uint32_t header;
719 int ret = 0, i;
720
721 if (idx >= ib_chunk->length_dw) {
722 DRM_ERROR("Can not parse packet at %d after CS end %d !\n",
723 idx, ib_chunk->length_dw);
724 return -EINVAL;
725 }
726 header = radeon_get_ib_value(p, idx);
727 pkt->idx = idx;
728 pkt->type = RADEON_CP_PACKET_GET_TYPE(header);
729 pkt->count = RADEON_CP_PACKET_GET_COUNT(header);
730 pkt->one_reg_wr = 0;
731 switch (pkt->type) {
732 case RADEON_PACKET_TYPE0:
733 if (rdev->family < CHIP_R600) {
734 pkt->reg = R100_CP_PACKET0_GET_REG(header);
735 pkt->one_reg_wr =
736 RADEON_CP_PACKET0_GET_ONE_REG_WR(header);
737 } else
738 pkt->reg = R600_CP_PACKET0_GET_REG(header);
739 break;
740 case RADEON_PACKET_TYPE3:
741 pkt->opcode = RADEON_CP_PACKET3_GET_OPCODE(header);
742 break;
743 case RADEON_PACKET_TYPE2:
744 pkt->count = -1;
745 break;
746 default:
747 DRM_ERROR("Unknown packet type %d at %d !\n", pkt->type, idx);
748 ret = -EINVAL;
749 goto dump_ib;
750 }
751 if ((pkt->count + 1 + pkt->idx) >= ib_chunk->length_dw) {
752 DRM_ERROR("Packet (%d:%d:%d) end after CS buffer (%d) !\n",
753 pkt->idx, pkt->type, pkt->count, ib_chunk->length_dw);
754 ret = -EINVAL;
755 goto dump_ib;
756 }
757 return 0;
758
759dump_ib:
760 for (i = 0; i < ib_chunk->length_dw; i++) {
761 if (i == idx)
762 printk("\t0x%08x <---\n", radeon_get_ib_value(p, i));
763 else
764 printk("\t0x%08x\n", radeon_get_ib_value(p, i));
765 }
766 return ret;
767}
768
769/**
770 * radeon_cs_packet_next_is_pkt3_nop() - test if the next packet is P3 NOP
771 * @p: structure holding the parser context.
772 *
773 * Check if the next packet is NOP relocation packet3.
774 **/
775bool radeon_cs_packet_next_is_pkt3_nop(struct radeon_cs_parser *p)
776{
777 struct radeon_cs_packet p3reloc;
778 int r;
779
780 r = radeon_cs_packet_parse(p, &p3reloc, p->idx);
781 if (r)
782 return false;
783 if (p3reloc.type != RADEON_PACKET_TYPE3)
784 return false;
785 if (p3reloc.opcode != RADEON_PACKET3_NOP)
786 return false;
787 return true;
788}
789
790/**
791 * radeon_cs_dump_packet() - dump raw packet context
792 * @p: structure holding the parser context.
793 * @pkt: structure holding the packet.
794 *
795 * Used mostly for debugging and error reporting.
796 **/
797void radeon_cs_dump_packet(struct radeon_cs_parser *p,
798 struct radeon_cs_packet *pkt)
799{
800 volatile uint32_t *ib;
801 unsigned i;
802 unsigned idx;
803
804 ib = p->ib.ptr;
805 idx = pkt->idx;
806 for (i = 0; i <= (pkt->count + 1); i++, idx++)
807 DRM_INFO("ib[%d]=0x%08X\n", idx, ib[idx]);
808}
809
810/**
811 * radeon_cs_packet_next_reloc() - parse next (should be reloc) packet
812 * @parser: parser structure holding parsing context.
813 * @data: pointer to relocation data
814 * @offset_start: starting offset
815 * @offset_mask: offset mask (to align start offset on)
816 * @reloc: reloc informations
817 *
818 * Check if next packet is relocation packet3, do bo validation and compute
819 * GPU offset using the provided start.
820 **/
821int radeon_cs_packet_next_reloc(struct radeon_cs_parser *p,
822 struct radeon_bo_list **cs_reloc,
823 int nomm)
824{
825 struct radeon_cs_chunk *relocs_chunk;
826 struct radeon_cs_packet p3reloc;
827 unsigned idx;
828 int r;
829
830 if (p->chunk_relocs == NULL) {
831 DRM_ERROR("No relocation chunk !\n");
832 return -EINVAL;
833 }
834 *cs_reloc = NULL;
835 relocs_chunk = p->chunk_relocs;
836 r = radeon_cs_packet_parse(p, &p3reloc, p->idx);
837 if (r)
838 return r;
839 p->idx += p3reloc.count + 2;
840 if (p3reloc.type != RADEON_PACKET_TYPE3 ||
841 p3reloc.opcode != RADEON_PACKET3_NOP) {
842 DRM_ERROR("No packet3 for relocation for packet at %d.\n",
843 p3reloc.idx);
844 radeon_cs_dump_packet(p, &p3reloc);
845 return -EINVAL;
846 }
847 idx = radeon_get_ib_value(p, p3reloc.idx + 1);
848 if (idx >= relocs_chunk->length_dw) {
849 DRM_ERROR("Relocs at %d after relocations chunk end %d !\n",
850 idx, relocs_chunk->length_dw);
851 radeon_cs_dump_packet(p, &p3reloc);
852 return -EINVAL;
853 }
854 /* FIXME: we assume reloc size is 4 dwords */
855 if (nomm) {
856 *cs_reloc = p->relocs;
857 (*cs_reloc)->gpu_offset =
858 (u64)relocs_chunk->kdata[idx + 3] << 32;
859 (*cs_reloc)->gpu_offset |= relocs_chunk->kdata[idx + 0];
860 } else
861 *cs_reloc = &p->relocs[(idx / 4)];
862 return 0;
863}