Linux Audio

Check our new training course

Loading...
v4.6
  1/* via_dmablit.c -- PCI DMA BitBlt support for the VIA Unichrome/Pro
  2 *
  3 * Copyright (C) 2005 Thomas Hellstrom, All Rights Reserved.
  4 *
  5 * Permission is hereby granted, free of charge, to any person obtaining a
  6 * copy of this software and associated documentation files (the "Software"),
  7 * to deal in the Software without restriction, including without limitation
  8 * the rights to use, copy, modify, merge, publish, distribute, sub license,
  9 * and/or sell copies of the Software, and to permit persons to whom the
 10 * Software is furnished to do so, subject to the following conditions:
 11 *
 12 * The above copyright notice and this permission notice (including the
 13 * next paragraph) shall be included in all copies or substantial portions
 14 * of the Software.
 15 *
 16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 18 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
 19 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
 20 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
 21 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
 22 * USE OR OTHER DEALINGS IN THE SOFTWARE.
 23 *
 24 * Authors:
 25 *    Thomas Hellstrom.
 26 *    Partially based on code obtained from Digeo Inc.
 27 */
 28
 29
 30/*
 31 * Unmaps the DMA mappings.
 32 * FIXME: Is this a NoOp on x86? Also
 33 * FIXME: What happens if this one is called and a pending blit has previously done
 34 * the same DMA mappings?
 35 */
 36
 37#include <drm/drmP.h>
 38#include <drm/via_drm.h>
 39#include "via_drv.h"
 40#include "via_dmablit.h"
 41
 42#include <linux/pagemap.h>
 43#include <linux/slab.h>
 44
 45#define VIA_PGDN(x)	     (((unsigned long)(x)) & PAGE_MASK)
 46#define VIA_PGOFF(x)	    (((unsigned long)(x)) & ~PAGE_MASK)
 47#define VIA_PFN(x)	      ((unsigned long)(x) >> PAGE_SHIFT)
 48
 49typedef struct _drm_via_descriptor {
 50	uint32_t mem_addr;
 51	uint32_t dev_addr;
 52	uint32_t size;
 53	uint32_t next;
 54} drm_via_descriptor_t;
 55
 56
 57/*
 58 * Unmap a DMA mapping.
 59 */
 60
 61
 62
 63static void
 64via_unmap_blit_from_device(struct pci_dev *pdev, drm_via_sg_info_t *vsg)
 65{
 66	int num_desc = vsg->num_desc;
 67	unsigned cur_descriptor_page = num_desc / vsg->descriptors_per_page;
 68	unsigned descriptor_this_page = num_desc % vsg->descriptors_per_page;
 69	drm_via_descriptor_t *desc_ptr = vsg->desc_pages[cur_descriptor_page] +
 70		descriptor_this_page;
 71	dma_addr_t next = vsg->chain_start;
 72
 73	while (num_desc--) {
 74		if (descriptor_this_page-- == 0) {
 75			cur_descriptor_page--;
 76			descriptor_this_page = vsg->descriptors_per_page - 1;
 77			desc_ptr = vsg->desc_pages[cur_descriptor_page] +
 78				descriptor_this_page;
 79		}
 80		dma_unmap_single(&pdev->dev, next, sizeof(*desc_ptr), DMA_TO_DEVICE);
 81		dma_unmap_page(&pdev->dev, desc_ptr->mem_addr, desc_ptr->size, vsg->direction);
 82		next = (dma_addr_t) desc_ptr->next;
 83		desc_ptr--;
 84	}
 85}
 86
 87/*
 88 * If mode = 0, count how many descriptors are needed.
 89 * If mode = 1, Map the DMA pages for the device, put together and map also the descriptors.
 90 * Descriptors are run in reverse order by the hardware because we are not allowed to update the
 91 * 'next' field without syncing calls when the descriptor is already mapped.
 92 */
 93
 94static void
 95via_map_blit_for_device(struct pci_dev *pdev,
 96		   const drm_via_dmablit_t *xfer,
 97		   drm_via_sg_info_t *vsg,
 98		   int mode)
 99{
100	unsigned cur_descriptor_page = 0;
101	unsigned num_descriptors_this_page = 0;
102	unsigned char *mem_addr = xfer->mem_addr;
103	unsigned char *cur_mem;
104	unsigned char *first_addr = (unsigned char *)VIA_PGDN(mem_addr);
105	uint32_t fb_addr = xfer->fb_addr;
106	uint32_t cur_fb;
107	unsigned long line_len;
108	unsigned remaining_len;
109	int num_desc = 0;
110	int cur_line;
111	dma_addr_t next = 0 | VIA_DMA_DPR_EC;
112	drm_via_descriptor_t *desc_ptr = NULL;
113
114	if (mode == 1)
115		desc_ptr = vsg->desc_pages[cur_descriptor_page];
116
117	for (cur_line = 0; cur_line < xfer->num_lines; ++cur_line) {
118
119		line_len = xfer->line_length;
120		cur_fb = fb_addr;
121		cur_mem = mem_addr;
122
123		while (line_len > 0) {
124
125			remaining_len = min(PAGE_SIZE-VIA_PGOFF(cur_mem), line_len);
126			line_len -= remaining_len;
127
128			if (mode == 1) {
129				desc_ptr->mem_addr =
130					dma_map_page(&pdev->dev,
131						     vsg->pages[VIA_PFN(cur_mem) -
132								VIA_PFN(first_addr)],
133						     VIA_PGOFF(cur_mem), remaining_len,
134						     vsg->direction);
135				desc_ptr->dev_addr = cur_fb;
136
137				desc_ptr->size = remaining_len;
138				desc_ptr->next = (uint32_t) next;
139				next = dma_map_single(&pdev->dev, desc_ptr, sizeof(*desc_ptr),
140						      DMA_TO_DEVICE);
141				desc_ptr++;
142				if (++num_descriptors_this_page >= vsg->descriptors_per_page) {
143					num_descriptors_this_page = 0;
144					desc_ptr = vsg->desc_pages[++cur_descriptor_page];
145				}
146			}
147
148			num_desc++;
149			cur_mem += remaining_len;
150			cur_fb += remaining_len;
151		}
152
153		mem_addr += xfer->mem_stride;
154		fb_addr += xfer->fb_stride;
155	}
156
157	if (mode == 1) {
158		vsg->chain_start = next;
159		vsg->state = dr_via_device_mapped;
160	}
161	vsg->num_desc = num_desc;
162}
163
164/*
165 * Function that frees up all resources for a blit. It is usable even if the
166 * blit info has only been partially built as long as the status enum is consistent
167 * with the actual status of the used resources.
168 */
169
170
171static void
172via_free_sg_info(struct pci_dev *pdev, drm_via_sg_info_t *vsg)
173{
174	struct page *page;
175	int i;
176
177	switch (vsg->state) {
178	case dr_via_device_mapped:
179		via_unmap_blit_from_device(pdev, vsg);
180	case dr_via_desc_pages_alloc:
181		for (i = 0; i < vsg->num_desc_pages; ++i) {
182			if (vsg->desc_pages[i] != NULL)
183				free_page((unsigned long)vsg->desc_pages[i]);
184		}
185		kfree(vsg->desc_pages);
186	case dr_via_pages_locked:
187		for (i = 0; i < vsg->num_pages; ++i) {
188			if (NULL != (page = vsg->pages[i])) {
189				if (!PageReserved(page) && (DMA_FROM_DEVICE == vsg->direction))
190					SetPageDirty(page);
191				put_page(page);
192			}
193		}
194	case dr_via_pages_alloc:
195		vfree(vsg->pages);
196	default:
197		vsg->state = dr_via_sg_init;
198	}
199	vfree(vsg->bounce_buffer);
200	vsg->bounce_buffer = NULL;
201	vsg->free_on_sequence = 0;
202}
203
204/*
205 * Fire a blit engine.
206 */
207
208static void
209via_fire_dmablit(struct drm_device *dev, drm_via_sg_info_t *vsg, int engine)
210{
211	drm_via_private_t *dev_priv = (drm_via_private_t *)dev->dev_private;
212
213	VIA_WRITE(VIA_PCI_DMA_MAR0 + engine*0x10, 0);
214	VIA_WRITE(VIA_PCI_DMA_DAR0 + engine*0x10, 0);
215	VIA_WRITE(VIA_PCI_DMA_CSR0 + engine*0x04, VIA_DMA_CSR_DD | VIA_DMA_CSR_TD |
216		  VIA_DMA_CSR_DE);
217	VIA_WRITE(VIA_PCI_DMA_MR0  + engine*0x04, VIA_DMA_MR_CM | VIA_DMA_MR_TDIE);
218	VIA_WRITE(VIA_PCI_DMA_BCR0 + engine*0x10, 0);
219	VIA_WRITE(VIA_PCI_DMA_DPR0 + engine*0x10, vsg->chain_start);
220	wmb();
221	VIA_WRITE(VIA_PCI_DMA_CSR0 + engine*0x04, VIA_DMA_CSR_DE | VIA_DMA_CSR_TS);
222	VIA_READ(VIA_PCI_DMA_CSR0 + engine*0x04);
223}
224
225/*
226 * Obtain a page pointer array and lock all pages into system memory. A segmentation violation will
227 * occur here if the calling user does not have access to the submitted address.
228 */
229
230static int
231via_lock_all_dma_pages(drm_via_sg_info_t *vsg,  drm_via_dmablit_t *xfer)
232{
233	int ret;
234	unsigned long first_pfn = VIA_PFN(xfer->mem_addr);
235	vsg->num_pages = VIA_PFN(xfer->mem_addr + (xfer->num_lines * xfer->mem_stride - 1)) -
236		first_pfn + 1;
237
238	vsg->pages = vzalloc(sizeof(struct page *) * vsg->num_pages);
239	if (NULL == vsg->pages)
240		return -ENOMEM;
241	down_read(&current->mm->mmap_sem);
242	ret = get_user_pages((unsigned long)xfer->mem_addr,
243			     vsg->num_pages,
244			     (vsg->direction == DMA_FROM_DEVICE),
245			     0, vsg->pages, NULL);
246
247	up_read(&current->mm->mmap_sem);
248	if (ret != vsg->num_pages) {
249		if (ret < 0)
250			return ret;
251		vsg->state = dr_via_pages_locked;
252		return -EINVAL;
253	}
254	vsg->state = dr_via_pages_locked;
255	DRM_DEBUG("DMA pages locked\n");
256	return 0;
257}
258
259/*
260 * Allocate DMA capable memory for the blit descriptor chain, and an array that keeps track of the
261 * pages we allocate. We don't want to use kmalloc for the descriptor chain because it may be
262 * quite large for some blits, and pages don't need to be contiguous.
263 */
264
265static int
266via_alloc_desc_pages(drm_via_sg_info_t *vsg)
267{
268	int i;
269
270	vsg->descriptors_per_page = PAGE_SIZE / sizeof(drm_via_descriptor_t);
271	vsg->num_desc_pages = (vsg->num_desc + vsg->descriptors_per_page - 1) /
272		vsg->descriptors_per_page;
273
274	if (NULL ==  (vsg->desc_pages = kcalloc(vsg->num_desc_pages, sizeof(void *), GFP_KERNEL)))
275		return -ENOMEM;
276
277	vsg->state = dr_via_desc_pages_alloc;
278	for (i = 0; i < vsg->num_desc_pages; ++i) {
279		if (NULL == (vsg->desc_pages[i] =
280			     (drm_via_descriptor_t *) __get_free_page(GFP_KERNEL)))
281			return -ENOMEM;
282	}
283	DRM_DEBUG("Allocated %d pages for %d descriptors.\n", vsg->num_desc_pages,
284		  vsg->num_desc);
285	return 0;
286}
287
288static void
289via_abort_dmablit(struct drm_device *dev, int engine)
290{
291	drm_via_private_t *dev_priv = (drm_via_private_t *)dev->dev_private;
292
293	VIA_WRITE(VIA_PCI_DMA_CSR0 + engine*0x04, VIA_DMA_CSR_TA);
294}
295
296static void
297via_dmablit_engine_off(struct drm_device *dev, int engine)
298{
299	drm_via_private_t *dev_priv = (drm_via_private_t *)dev->dev_private;
300
301	VIA_WRITE(VIA_PCI_DMA_CSR0 + engine*0x04, VIA_DMA_CSR_TD | VIA_DMA_CSR_DD);
302}
303
304
305
306/*
307 * The dmablit part of the IRQ handler. Trying to do only reasonably fast things here.
308 * The rest, like unmapping and freeing memory for done blits is done in a separate workqueue
309 * task. Basically the task of the interrupt handler is to submit a new blit to the engine, while
310 * the workqueue task takes care of processing associated with the old blit.
311 */
312
313void
314via_dmablit_handler(struct drm_device *dev, int engine, int from_irq)
315{
316	drm_via_private_t *dev_priv = (drm_via_private_t *)dev->dev_private;
317	drm_via_blitq_t *blitq = dev_priv->blit_queues + engine;
318	int cur;
319	int done_transfer;
320	unsigned long irqsave = 0;
321	uint32_t status = 0;
322
323	DRM_DEBUG("DMA blit handler called. engine = %d, from_irq = %d, blitq = 0x%lx\n",
324		  engine, from_irq, (unsigned long) blitq);
325
326	if (from_irq)
327		spin_lock(&blitq->blit_lock);
328	else
329		spin_lock_irqsave(&blitq->blit_lock, irqsave);
330
331	done_transfer = blitq->is_active &&
332	  ((status = VIA_READ(VIA_PCI_DMA_CSR0 + engine*0x04)) & VIA_DMA_CSR_TD);
333	done_transfer = done_transfer || (blitq->aborting && !(status & VIA_DMA_CSR_DE));
334
335	cur = blitq->cur;
336	if (done_transfer) {
337
338		blitq->blits[cur]->aborted = blitq->aborting;
339		blitq->done_blit_handle++;
340		wake_up(blitq->blit_queue + cur);
341
342		cur++;
343		if (cur >= VIA_NUM_BLIT_SLOTS)
344			cur = 0;
345		blitq->cur = cur;
346
347		/*
348		 * Clear transfer done flag.
349		 */
350
351		VIA_WRITE(VIA_PCI_DMA_CSR0 + engine*0x04,  VIA_DMA_CSR_TD);
352
353		blitq->is_active = 0;
354		blitq->aborting = 0;
355		schedule_work(&blitq->wq);
356
357	} else if (blitq->is_active && time_after_eq(jiffies, blitq->end)) {
358
359		/*
360		 * Abort transfer after one second.
361		 */
362
363		via_abort_dmablit(dev, engine);
364		blitq->aborting = 1;
365		blitq->end = jiffies + HZ;
366	}
367
368	if (!blitq->is_active) {
369		if (blitq->num_outstanding) {
370			via_fire_dmablit(dev, blitq->blits[cur], engine);
371			blitq->is_active = 1;
372			blitq->cur = cur;
373			blitq->num_outstanding--;
374			blitq->end = jiffies + HZ;
375			if (!timer_pending(&blitq->poll_timer))
376				mod_timer(&blitq->poll_timer, jiffies + 1);
377		} else {
378			if (timer_pending(&blitq->poll_timer))
379				del_timer(&blitq->poll_timer);
380			via_dmablit_engine_off(dev, engine);
381		}
382	}
383
384	if (from_irq)
385		spin_unlock(&blitq->blit_lock);
386	else
387		spin_unlock_irqrestore(&blitq->blit_lock, irqsave);
388}
389
390
391
392/*
393 * Check whether this blit is still active, performing necessary locking.
394 */
395
396static int
397via_dmablit_active(drm_via_blitq_t *blitq, int engine, uint32_t handle, wait_queue_head_t **queue)
398{
399	unsigned long irqsave;
400	uint32_t slot;
401	int active;
402
403	spin_lock_irqsave(&blitq->blit_lock, irqsave);
404
405	/*
406	 * Allow for handle wraparounds.
407	 */
408
409	active = ((blitq->done_blit_handle - handle) > (1 << 23)) &&
410		((blitq->cur_blit_handle - handle) <= (1 << 23));
411
412	if (queue && active) {
413		slot = handle - blitq->done_blit_handle + blitq->cur - 1;
414		if (slot >= VIA_NUM_BLIT_SLOTS)
415			slot -= VIA_NUM_BLIT_SLOTS;
416		*queue = blitq->blit_queue + slot;
417	}
418
419	spin_unlock_irqrestore(&blitq->blit_lock, irqsave);
420
421	return active;
422}
423
424/*
425 * Sync. Wait for at least three seconds for the blit to be performed.
426 */
427
428static int
429via_dmablit_sync(struct drm_device *dev, uint32_t handle, int engine)
430{
431
432	drm_via_private_t *dev_priv = (drm_via_private_t *)dev->dev_private;
433	drm_via_blitq_t *blitq = dev_priv->blit_queues + engine;
434	wait_queue_head_t *queue;
435	int ret = 0;
436
437	if (via_dmablit_active(blitq, engine, handle, &queue)) {
438		DRM_WAIT_ON(ret, *queue, 3 * HZ,
439			    !via_dmablit_active(blitq, engine, handle, NULL));
440	}
441	DRM_DEBUG("DMA blit sync handle 0x%x engine %d returned %d\n",
442		  handle, engine, ret);
443
444	return ret;
445}
446
447
448/*
449 * A timer that regularly polls the blit engine in cases where we don't have interrupts:
450 * a) Broken hardware (typically those that don't have any video capture facility).
451 * b) Blit abort. The hardware doesn't send an interrupt when a blit is aborted.
452 * The timer and hardware IRQ's can and do work in parallel. If the hardware has
453 * irqs, it will shorten the latency somewhat.
454 */
455
456
457
458static void
459via_dmablit_timer(unsigned long data)
460{
461	drm_via_blitq_t *blitq = (drm_via_blitq_t *) data;
462	struct drm_device *dev = blitq->dev;
463	int engine = (int)
464		(blitq - ((drm_via_private_t *)dev->dev_private)->blit_queues);
465
466	DRM_DEBUG("Polling timer called for engine %d, jiffies %lu\n", engine,
467		  (unsigned long) jiffies);
468
469	via_dmablit_handler(dev, engine, 0);
470
471	if (!timer_pending(&blitq->poll_timer)) {
472		mod_timer(&blitq->poll_timer, jiffies + 1);
473
474	       /*
475		* Rerun handler to delete timer if engines are off, and
476		* to shorten abort latency. This is a little nasty.
477		*/
478
479	       via_dmablit_handler(dev, engine, 0);
480
481	}
482}
483
484
485
486
487/*
488 * Workqueue task that frees data and mappings associated with a blit.
489 * Also wakes up waiting processes. Each of these tasks handles one
490 * blit engine only and may not be called on each interrupt.
491 */
492
493
494static void
495via_dmablit_workqueue(struct work_struct *work)
496{
497	drm_via_blitq_t *blitq = container_of(work, drm_via_blitq_t, wq);
498	struct drm_device *dev = blitq->dev;
499	unsigned long irqsave;
500	drm_via_sg_info_t *cur_sg;
501	int cur_released;
502
503
504	DRM_DEBUG("Workqueue task called for blit engine %ld\n", (unsigned long)
505		  (blitq - ((drm_via_private_t *)dev->dev_private)->blit_queues));
506
507	spin_lock_irqsave(&blitq->blit_lock, irqsave);
508
509	while (blitq->serviced != blitq->cur) {
510
511		cur_released = blitq->serviced++;
512
513		DRM_DEBUG("Releasing blit slot %d\n", cur_released);
514
515		if (blitq->serviced >= VIA_NUM_BLIT_SLOTS)
516			blitq->serviced = 0;
517
518		cur_sg = blitq->blits[cur_released];
519		blitq->num_free++;
520
521		spin_unlock_irqrestore(&blitq->blit_lock, irqsave);
522
523		wake_up(&blitq->busy_queue);
524
525		via_free_sg_info(dev->pdev, cur_sg);
526		kfree(cur_sg);
527
528		spin_lock_irqsave(&blitq->blit_lock, irqsave);
529	}
530
531	spin_unlock_irqrestore(&blitq->blit_lock, irqsave);
532}
533
534
535/*
536 * Init all blit engines. Currently we use two, but some hardware have 4.
537 */
538
539
540void
541via_init_dmablit(struct drm_device *dev)
542{
543	int i, j;
544	drm_via_private_t *dev_priv = (drm_via_private_t *)dev->dev_private;
545	drm_via_blitq_t *blitq;
546
547	pci_set_master(dev->pdev);
548
549	for (i = 0; i < VIA_NUM_BLIT_ENGINES; ++i) {
550		blitq = dev_priv->blit_queues + i;
551		blitq->dev = dev;
552		blitq->cur_blit_handle = 0;
553		blitq->done_blit_handle = 0;
554		blitq->head = 0;
555		blitq->cur = 0;
556		blitq->serviced = 0;
557		blitq->num_free = VIA_NUM_BLIT_SLOTS - 1;
558		blitq->num_outstanding = 0;
559		blitq->is_active = 0;
560		blitq->aborting = 0;
561		spin_lock_init(&blitq->blit_lock);
562		for (j = 0; j < VIA_NUM_BLIT_SLOTS; ++j)
563			init_waitqueue_head(blitq->blit_queue + j);
564		init_waitqueue_head(&blitq->busy_queue);
565		INIT_WORK(&blitq->wq, via_dmablit_workqueue);
566		setup_timer(&blitq->poll_timer, via_dmablit_timer,
567				(unsigned long)blitq);
568	}
569}
570
571/*
572 * Build all info and do all mappings required for a blit.
573 */
574
575
576static int
577via_build_sg_info(struct drm_device *dev, drm_via_sg_info_t *vsg, drm_via_dmablit_t *xfer)
578{
579	int draw = xfer->to_fb;
580	int ret = 0;
581
582	vsg->direction = (draw) ? DMA_TO_DEVICE : DMA_FROM_DEVICE;
583	vsg->bounce_buffer = NULL;
584
585	vsg->state = dr_via_sg_init;
586
587	if (xfer->num_lines <= 0 || xfer->line_length <= 0) {
588		DRM_ERROR("Zero size bitblt.\n");
589		return -EINVAL;
590	}
591
592	/*
593	 * Below check is a driver limitation, not a hardware one. We
594	 * don't want to lock unused pages, and don't want to incoporate the
595	 * extra logic of avoiding them. Make sure there are no.
596	 * (Not a big limitation anyway.)
597	 */
598
599	if ((xfer->mem_stride - xfer->line_length) > 2*PAGE_SIZE) {
600		DRM_ERROR("Too large system memory stride. Stride: %d, "
601			  "Length: %d\n", xfer->mem_stride, xfer->line_length);
602		return -EINVAL;
603	}
604
605	if ((xfer->mem_stride == xfer->line_length) &&
606	   (xfer->fb_stride == xfer->line_length)) {
607		xfer->mem_stride *= xfer->num_lines;
608		xfer->line_length = xfer->mem_stride;
609		xfer->fb_stride = xfer->mem_stride;
610		xfer->num_lines = 1;
611	}
612
613	/*
614	 * Don't lock an arbitrary large number of pages, since that causes a
615	 * DOS security hole.
616	 */
617
618	if (xfer->num_lines > 2048 || (xfer->num_lines*xfer->mem_stride > (2048*2048*4))) {
619		DRM_ERROR("Too large PCI DMA bitblt.\n");
620		return -EINVAL;
621	}
622
623	/*
624	 * we allow a negative fb stride to allow flipping of images in
625	 * transfer.
626	 */
627
628	if (xfer->mem_stride < xfer->line_length ||
629		abs(xfer->fb_stride) < xfer->line_length) {
630		DRM_ERROR("Invalid frame-buffer / memory stride.\n");
631		return -EINVAL;
632	}
633
634	/*
635	 * A hardware bug seems to be worked around if system memory addresses start on
636	 * 16 byte boundaries. This seems a bit restrictive however. VIA is contacted
637	 * about this. Meanwhile, impose the following restrictions:
638	 */
639
640#ifdef VIA_BUGFREE
641	if ((((unsigned long)xfer->mem_addr & 3) != ((unsigned long)xfer->fb_addr & 3)) ||
642	    ((xfer->num_lines > 1) && ((xfer->mem_stride & 3) != (xfer->fb_stride & 3)))) {
643		DRM_ERROR("Invalid DRM bitblt alignment.\n");
644		return -EINVAL;
645	}
646#else
647	if ((((unsigned long)xfer->mem_addr & 15) ||
648	      ((unsigned long)xfer->fb_addr & 3)) ||
649	   ((xfer->num_lines > 1) &&
650	   ((xfer->mem_stride & 15) || (xfer->fb_stride & 3)))) {
651		DRM_ERROR("Invalid DRM bitblt alignment.\n");
652		return -EINVAL;
653	}
654#endif
655
656	if (0 != (ret = via_lock_all_dma_pages(vsg, xfer))) {
657		DRM_ERROR("Could not lock DMA pages.\n");
658		via_free_sg_info(dev->pdev, vsg);
659		return ret;
660	}
661
662	via_map_blit_for_device(dev->pdev, xfer, vsg, 0);
663	if (0 != (ret = via_alloc_desc_pages(vsg))) {
664		DRM_ERROR("Could not allocate DMA descriptor pages.\n");
665		via_free_sg_info(dev->pdev, vsg);
666		return ret;
667	}
668	via_map_blit_for_device(dev->pdev, xfer, vsg, 1);
669
670	return 0;
671}
672
673
674/*
675 * Reserve one free slot in the blit queue. Will wait for one second for one
676 * to become available. Otherwise -EBUSY is returned.
677 */
678
679static int
680via_dmablit_grab_slot(drm_via_blitq_t *blitq, int engine)
681{
682	int ret = 0;
683	unsigned long irqsave;
684
685	DRM_DEBUG("Num free is %d\n", blitq->num_free);
686	spin_lock_irqsave(&blitq->blit_lock, irqsave);
687	while (blitq->num_free == 0) {
688		spin_unlock_irqrestore(&blitq->blit_lock, irqsave);
689
690		DRM_WAIT_ON(ret, blitq->busy_queue, HZ, blitq->num_free > 0);
691		if (ret)
692			return (-EINTR == ret) ? -EAGAIN : ret;
693
694		spin_lock_irqsave(&blitq->blit_lock, irqsave);
695	}
696
697	blitq->num_free--;
698	spin_unlock_irqrestore(&blitq->blit_lock, irqsave);
699
700	return 0;
701}
702
703/*
704 * Hand back a free slot if we changed our mind.
705 */
706
707static void
708via_dmablit_release_slot(drm_via_blitq_t *blitq)
709{
710	unsigned long irqsave;
711
712	spin_lock_irqsave(&blitq->blit_lock, irqsave);
713	blitq->num_free++;
714	spin_unlock_irqrestore(&blitq->blit_lock, irqsave);
715	wake_up(&blitq->busy_queue);
716}
717
718/*
719 * Grab a free slot. Build blit info and queue a blit.
720 */
721
722
723static int
724via_dmablit(struct drm_device *dev, drm_via_dmablit_t *xfer)
725{
726	drm_via_private_t *dev_priv = (drm_via_private_t *)dev->dev_private;
727	drm_via_sg_info_t *vsg;
728	drm_via_blitq_t *blitq;
729	int ret;
730	int engine;
731	unsigned long irqsave;
732
733	if (dev_priv == NULL) {
734		DRM_ERROR("Called without initialization.\n");
735		return -EINVAL;
736	}
737
738	engine = (xfer->to_fb) ? 0 : 1;
739	blitq = dev_priv->blit_queues + engine;
740	if (0 != (ret = via_dmablit_grab_slot(blitq, engine)))
741		return ret;
742	if (NULL == (vsg = kmalloc(sizeof(*vsg), GFP_KERNEL))) {
743		via_dmablit_release_slot(blitq);
744		return -ENOMEM;
745	}
746	if (0 != (ret = via_build_sg_info(dev, vsg, xfer))) {
747		via_dmablit_release_slot(blitq);
748		kfree(vsg);
749		return ret;
750	}
751	spin_lock_irqsave(&blitq->blit_lock, irqsave);
752
753	blitq->blits[blitq->head++] = vsg;
754	if (blitq->head >= VIA_NUM_BLIT_SLOTS)
755		blitq->head = 0;
756	blitq->num_outstanding++;
757	xfer->sync.sync_handle = ++blitq->cur_blit_handle;
758
759	spin_unlock_irqrestore(&blitq->blit_lock, irqsave);
760	xfer->sync.engine = engine;
761
762	via_dmablit_handler(dev, engine, 0);
763
764	return 0;
765}
766
767/*
768 * Sync on a previously submitted blit. Note that the X server use signals extensively, and
769 * that there is a very big probability that this IOCTL will be interrupted by a signal. In that
770 * case it returns with -EAGAIN for the signal to be delivered.
771 * The caller should then reissue the IOCTL. This is similar to what is being done for drmGetLock().
772 */
773
774int
775via_dma_blit_sync(struct drm_device *dev, void *data, struct drm_file *file_priv)
776{
777	drm_via_blitsync_t *sync = data;
778	int err;
779
780	if (sync->engine >= VIA_NUM_BLIT_ENGINES)
781		return -EINVAL;
782
783	err = via_dmablit_sync(dev, sync->sync_handle, sync->engine);
784
785	if (-EINTR == err)
786		err = -EAGAIN;
787
788	return err;
789}
790
791
792/*
793 * Queue a blit and hand back a handle to be used for sync. This IOCTL may be interrupted by a signal
794 * while waiting for a free slot in the blit queue. In that case it returns with -EAGAIN and should
795 * be reissued. See the above IOCTL code.
796 */
797
798int
799via_dma_blit(struct drm_device *dev, void *data, struct drm_file *file_priv)
800{
801	drm_via_dmablit_t *xfer = data;
802	int err;
803
804	err = via_dmablit(dev, xfer);
805
806	return err;
807}
v4.17
  1/* via_dmablit.c -- PCI DMA BitBlt support for the VIA Unichrome/Pro
  2 *
  3 * Copyright (C) 2005 Thomas Hellstrom, All Rights Reserved.
  4 *
  5 * Permission is hereby granted, free of charge, to any person obtaining a
  6 * copy of this software and associated documentation files (the "Software"),
  7 * to deal in the Software without restriction, including without limitation
  8 * the rights to use, copy, modify, merge, publish, distribute, sub license,
  9 * and/or sell copies of the Software, and to permit persons to whom the
 10 * Software is furnished to do so, subject to the following conditions:
 11 *
 12 * The above copyright notice and this permission notice (including the
 13 * next paragraph) shall be included in all copies or substantial portions
 14 * of the Software.
 15 *
 16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 18 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
 19 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
 20 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
 21 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
 22 * USE OR OTHER DEALINGS IN THE SOFTWARE.
 23 *
 24 * Authors:
 25 *    Thomas Hellstrom.
 26 *    Partially based on code obtained from Digeo Inc.
 27 */
 28
 29
 30/*
 31 * Unmaps the DMA mappings.
 32 * FIXME: Is this a NoOp on x86? Also
 33 * FIXME: What happens if this one is called and a pending blit has previously done
 34 * the same DMA mappings?
 35 */
 36
 37#include <drm/drmP.h>
 38#include <drm/via_drm.h>
 39#include "via_drv.h"
 40#include "via_dmablit.h"
 41
 42#include <linux/pagemap.h>
 43#include <linux/slab.h>
 44
 45#define VIA_PGDN(x)	     (((unsigned long)(x)) & PAGE_MASK)
 46#define VIA_PGOFF(x)	    (((unsigned long)(x)) & ~PAGE_MASK)
 47#define VIA_PFN(x)	      ((unsigned long)(x) >> PAGE_SHIFT)
 48
 49typedef struct _drm_via_descriptor {
 50	uint32_t mem_addr;
 51	uint32_t dev_addr;
 52	uint32_t size;
 53	uint32_t next;
 54} drm_via_descriptor_t;
 55
 56
 57/*
 58 * Unmap a DMA mapping.
 59 */
 60
 61
 62
 63static void
 64via_unmap_blit_from_device(struct pci_dev *pdev, drm_via_sg_info_t *vsg)
 65{
 66	int num_desc = vsg->num_desc;
 67	unsigned cur_descriptor_page = num_desc / vsg->descriptors_per_page;
 68	unsigned descriptor_this_page = num_desc % vsg->descriptors_per_page;
 69	drm_via_descriptor_t *desc_ptr = vsg->desc_pages[cur_descriptor_page] +
 70		descriptor_this_page;
 71	dma_addr_t next = vsg->chain_start;
 72
 73	while (num_desc--) {
 74		if (descriptor_this_page-- == 0) {
 75			cur_descriptor_page--;
 76			descriptor_this_page = vsg->descriptors_per_page - 1;
 77			desc_ptr = vsg->desc_pages[cur_descriptor_page] +
 78				descriptor_this_page;
 79		}
 80		dma_unmap_single(&pdev->dev, next, sizeof(*desc_ptr), DMA_TO_DEVICE);
 81		dma_unmap_page(&pdev->dev, desc_ptr->mem_addr, desc_ptr->size, vsg->direction);
 82		next = (dma_addr_t) desc_ptr->next;
 83		desc_ptr--;
 84	}
 85}
 86
 87/*
 88 * If mode = 0, count how many descriptors are needed.
 89 * If mode = 1, Map the DMA pages for the device, put together and map also the descriptors.
 90 * Descriptors are run in reverse order by the hardware because we are not allowed to update the
 91 * 'next' field without syncing calls when the descriptor is already mapped.
 92 */
 93
 94static void
 95via_map_blit_for_device(struct pci_dev *pdev,
 96		   const drm_via_dmablit_t *xfer,
 97		   drm_via_sg_info_t *vsg,
 98		   int mode)
 99{
100	unsigned cur_descriptor_page = 0;
101	unsigned num_descriptors_this_page = 0;
102	unsigned char *mem_addr = xfer->mem_addr;
103	unsigned char *cur_mem;
104	unsigned char *first_addr = (unsigned char *)VIA_PGDN(mem_addr);
105	uint32_t fb_addr = xfer->fb_addr;
106	uint32_t cur_fb;
107	unsigned long line_len;
108	unsigned remaining_len;
109	int num_desc = 0;
110	int cur_line;
111	dma_addr_t next = 0 | VIA_DMA_DPR_EC;
112	drm_via_descriptor_t *desc_ptr = NULL;
113
114	if (mode == 1)
115		desc_ptr = vsg->desc_pages[cur_descriptor_page];
116
117	for (cur_line = 0; cur_line < xfer->num_lines; ++cur_line) {
118
119		line_len = xfer->line_length;
120		cur_fb = fb_addr;
121		cur_mem = mem_addr;
122
123		while (line_len > 0) {
124
125			remaining_len = min(PAGE_SIZE-VIA_PGOFF(cur_mem), line_len);
126			line_len -= remaining_len;
127
128			if (mode == 1) {
129				desc_ptr->mem_addr =
130					dma_map_page(&pdev->dev,
131						     vsg->pages[VIA_PFN(cur_mem) -
132								VIA_PFN(first_addr)],
133						     VIA_PGOFF(cur_mem), remaining_len,
134						     vsg->direction);
135				desc_ptr->dev_addr = cur_fb;
136
137				desc_ptr->size = remaining_len;
138				desc_ptr->next = (uint32_t) next;
139				next = dma_map_single(&pdev->dev, desc_ptr, sizeof(*desc_ptr),
140						      DMA_TO_DEVICE);
141				desc_ptr++;
142				if (++num_descriptors_this_page >= vsg->descriptors_per_page) {
143					num_descriptors_this_page = 0;
144					desc_ptr = vsg->desc_pages[++cur_descriptor_page];
145				}
146			}
147
148			num_desc++;
149			cur_mem += remaining_len;
150			cur_fb += remaining_len;
151		}
152
153		mem_addr += xfer->mem_stride;
154		fb_addr += xfer->fb_stride;
155	}
156
157	if (mode == 1) {
158		vsg->chain_start = next;
159		vsg->state = dr_via_device_mapped;
160	}
161	vsg->num_desc = num_desc;
162}
163
164/*
165 * Function that frees up all resources for a blit. It is usable even if the
166 * blit info has only been partially built as long as the status enum is consistent
167 * with the actual status of the used resources.
168 */
169
170
171static void
172via_free_sg_info(struct pci_dev *pdev, drm_via_sg_info_t *vsg)
173{
174	struct page *page;
175	int i;
176
177	switch (vsg->state) {
178	case dr_via_device_mapped:
179		via_unmap_blit_from_device(pdev, vsg);
180	case dr_via_desc_pages_alloc:
181		for (i = 0; i < vsg->num_desc_pages; ++i) {
182			if (vsg->desc_pages[i] != NULL)
183				free_page((unsigned long)vsg->desc_pages[i]);
184		}
185		kfree(vsg->desc_pages);
186	case dr_via_pages_locked:
187		for (i = 0; i < vsg->num_pages; ++i) {
188			if (NULL != (page = vsg->pages[i])) {
189				if (!PageReserved(page) && (DMA_FROM_DEVICE == vsg->direction))
190					SetPageDirty(page);
191				put_page(page);
192			}
193		}
194	case dr_via_pages_alloc:
195		vfree(vsg->pages);
196	default:
197		vsg->state = dr_via_sg_init;
198	}
199	vfree(vsg->bounce_buffer);
200	vsg->bounce_buffer = NULL;
201	vsg->free_on_sequence = 0;
202}
203
204/*
205 * Fire a blit engine.
206 */
207
208static void
209via_fire_dmablit(struct drm_device *dev, drm_via_sg_info_t *vsg, int engine)
210{
211	drm_via_private_t *dev_priv = (drm_via_private_t *)dev->dev_private;
212
213	VIA_WRITE(VIA_PCI_DMA_MAR0 + engine*0x10, 0);
214	VIA_WRITE(VIA_PCI_DMA_DAR0 + engine*0x10, 0);
215	VIA_WRITE(VIA_PCI_DMA_CSR0 + engine*0x04, VIA_DMA_CSR_DD | VIA_DMA_CSR_TD |
216		  VIA_DMA_CSR_DE);
217	VIA_WRITE(VIA_PCI_DMA_MR0  + engine*0x04, VIA_DMA_MR_CM | VIA_DMA_MR_TDIE);
218	VIA_WRITE(VIA_PCI_DMA_BCR0 + engine*0x10, 0);
219	VIA_WRITE(VIA_PCI_DMA_DPR0 + engine*0x10, vsg->chain_start);
220	wmb();
221	VIA_WRITE(VIA_PCI_DMA_CSR0 + engine*0x04, VIA_DMA_CSR_DE | VIA_DMA_CSR_TS);
222	VIA_READ(VIA_PCI_DMA_CSR0 + engine*0x04);
223}
224
225/*
226 * Obtain a page pointer array and lock all pages into system memory. A segmentation violation will
227 * occur here if the calling user does not have access to the submitted address.
228 */
229
230static int
231via_lock_all_dma_pages(drm_via_sg_info_t *vsg,  drm_via_dmablit_t *xfer)
232{
233	int ret;
234	unsigned long first_pfn = VIA_PFN(xfer->mem_addr);
235	vsg->num_pages = VIA_PFN(xfer->mem_addr + (xfer->num_lines * xfer->mem_stride - 1)) -
236		first_pfn + 1;
237
238	vsg->pages = vzalloc(sizeof(struct page *) * vsg->num_pages);
239	if (NULL == vsg->pages)
240		return -ENOMEM;
241	ret = get_user_pages_fast((unsigned long)xfer->mem_addr,
242			vsg->num_pages, vsg->direction == DMA_FROM_DEVICE,
243			vsg->pages);
 
 
 
 
244	if (ret != vsg->num_pages) {
245		if (ret < 0)
246			return ret;
247		vsg->state = dr_via_pages_locked;
248		return -EINVAL;
249	}
250	vsg->state = dr_via_pages_locked;
251	DRM_DEBUG("DMA pages locked\n");
252	return 0;
253}
254
255/*
256 * Allocate DMA capable memory for the blit descriptor chain, and an array that keeps track of the
257 * pages we allocate. We don't want to use kmalloc for the descriptor chain because it may be
258 * quite large for some blits, and pages don't need to be contiguous.
259 */
260
261static int
262via_alloc_desc_pages(drm_via_sg_info_t *vsg)
263{
264	int i;
265
266	vsg->descriptors_per_page = PAGE_SIZE / sizeof(drm_via_descriptor_t);
267	vsg->num_desc_pages = (vsg->num_desc + vsg->descriptors_per_page - 1) /
268		vsg->descriptors_per_page;
269
270	if (NULL ==  (vsg->desc_pages = kcalloc(vsg->num_desc_pages, sizeof(void *), GFP_KERNEL)))
271		return -ENOMEM;
272
273	vsg->state = dr_via_desc_pages_alloc;
274	for (i = 0; i < vsg->num_desc_pages; ++i) {
275		if (NULL == (vsg->desc_pages[i] =
276			     (drm_via_descriptor_t *) __get_free_page(GFP_KERNEL)))
277			return -ENOMEM;
278	}
279	DRM_DEBUG("Allocated %d pages for %d descriptors.\n", vsg->num_desc_pages,
280		  vsg->num_desc);
281	return 0;
282}
283
284static void
285via_abort_dmablit(struct drm_device *dev, int engine)
286{
287	drm_via_private_t *dev_priv = (drm_via_private_t *)dev->dev_private;
288
289	VIA_WRITE(VIA_PCI_DMA_CSR0 + engine*0x04, VIA_DMA_CSR_TA);
290}
291
292static void
293via_dmablit_engine_off(struct drm_device *dev, int engine)
294{
295	drm_via_private_t *dev_priv = (drm_via_private_t *)dev->dev_private;
296
297	VIA_WRITE(VIA_PCI_DMA_CSR0 + engine*0x04, VIA_DMA_CSR_TD | VIA_DMA_CSR_DD);
298}
299
300
301
302/*
303 * The dmablit part of the IRQ handler. Trying to do only reasonably fast things here.
304 * The rest, like unmapping and freeing memory for done blits is done in a separate workqueue
305 * task. Basically the task of the interrupt handler is to submit a new blit to the engine, while
306 * the workqueue task takes care of processing associated with the old blit.
307 */
308
309void
310via_dmablit_handler(struct drm_device *dev, int engine, int from_irq)
311{
312	drm_via_private_t *dev_priv = (drm_via_private_t *)dev->dev_private;
313	drm_via_blitq_t *blitq = dev_priv->blit_queues + engine;
314	int cur;
315	int done_transfer;
316	unsigned long irqsave = 0;
317	uint32_t status = 0;
318
319	DRM_DEBUG("DMA blit handler called. engine = %d, from_irq = %d, blitq = 0x%lx\n",
320		  engine, from_irq, (unsigned long) blitq);
321
322	if (from_irq)
323		spin_lock(&blitq->blit_lock);
324	else
325		spin_lock_irqsave(&blitq->blit_lock, irqsave);
326
327	done_transfer = blitq->is_active &&
328	  ((status = VIA_READ(VIA_PCI_DMA_CSR0 + engine*0x04)) & VIA_DMA_CSR_TD);
329	done_transfer = done_transfer || (blitq->aborting && !(status & VIA_DMA_CSR_DE));
330
331	cur = blitq->cur;
332	if (done_transfer) {
333
334		blitq->blits[cur]->aborted = blitq->aborting;
335		blitq->done_blit_handle++;
336		wake_up(blitq->blit_queue + cur);
337
338		cur++;
339		if (cur >= VIA_NUM_BLIT_SLOTS)
340			cur = 0;
341		blitq->cur = cur;
342
343		/*
344		 * Clear transfer done flag.
345		 */
346
347		VIA_WRITE(VIA_PCI_DMA_CSR0 + engine*0x04,  VIA_DMA_CSR_TD);
348
349		blitq->is_active = 0;
350		blitq->aborting = 0;
351		schedule_work(&blitq->wq);
352
353	} else if (blitq->is_active && time_after_eq(jiffies, blitq->end)) {
354
355		/*
356		 * Abort transfer after one second.
357		 */
358
359		via_abort_dmablit(dev, engine);
360		blitq->aborting = 1;
361		blitq->end = jiffies + HZ;
362	}
363
364	if (!blitq->is_active) {
365		if (blitq->num_outstanding) {
366			via_fire_dmablit(dev, blitq->blits[cur], engine);
367			blitq->is_active = 1;
368			blitq->cur = cur;
369			blitq->num_outstanding--;
370			blitq->end = jiffies + HZ;
371			if (!timer_pending(&blitq->poll_timer))
372				mod_timer(&blitq->poll_timer, jiffies + 1);
373		} else {
374			if (timer_pending(&blitq->poll_timer))
375				del_timer(&blitq->poll_timer);
376			via_dmablit_engine_off(dev, engine);
377		}
378	}
379
380	if (from_irq)
381		spin_unlock(&blitq->blit_lock);
382	else
383		spin_unlock_irqrestore(&blitq->blit_lock, irqsave);
384}
385
386
387
388/*
389 * Check whether this blit is still active, performing necessary locking.
390 */
391
392static int
393via_dmablit_active(drm_via_blitq_t *blitq, int engine, uint32_t handle, wait_queue_head_t **queue)
394{
395	unsigned long irqsave;
396	uint32_t slot;
397	int active;
398
399	spin_lock_irqsave(&blitq->blit_lock, irqsave);
400
401	/*
402	 * Allow for handle wraparounds.
403	 */
404
405	active = ((blitq->done_blit_handle - handle) > (1 << 23)) &&
406		((blitq->cur_blit_handle - handle) <= (1 << 23));
407
408	if (queue && active) {
409		slot = handle - blitq->done_blit_handle + blitq->cur - 1;
410		if (slot >= VIA_NUM_BLIT_SLOTS)
411			slot -= VIA_NUM_BLIT_SLOTS;
412		*queue = blitq->blit_queue + slot;
413	}
414
415	spin_unlock_irqrestore(&blitq->blit_lock, irqsave);
416
417	return active;
418}
419
420/*
421 * Sync. Wait for at least three seconds for the blit to be performed.
422 */
423
424static int
425via_dmablit_sync(struct drm_device *dev, uint32_t handle, int engine)
426{
427
428	drm_via_private_t *dev_priv = (drm_via_private_t *)dev->dev_private;
429	drm_via_blitq_t *blitq = dev_priv->blit_queues + engine;
430	wait_queue_head_t *queue;
431	int ret = 0;
432
433	if (via_dmablit_active(blitq, engine, handle, &queue)) {
434		DRM_WAIT_ON(ret, *queue, 3 * HZ,
435			    !via_dmablit_active(blitq, engine, handle, NULL));
436	}
437	DRM_DEBUG("DMA blit sync handle 0x%x engine %d returned %d\n",
438		  handle, engine, ret);
439
440	return ret;
441}
442
443
444/*
445 * A timer that regularly polls the blit engine in cases where we don't have interrupts:
446 * a) Broken hardware (typically those that don't have any video capture facility).
447 * b) Blit abort. The hardware doesn't send an interrupt when a blit is aborted.
448 * The timer and hardware IRQ's can and do work in parallel. If the hardware has
449 * irqs, it will shorten the latency somewhat.
450 */
451
452
453
454static void
455via_dmablit_timer(struct timer_list *t)
456{
457	drm_via_blitq_t *blitq = from_timer(blitq, t, poll_timer);
458	struct drm_device *dev = blitq->dev;
459	int engine = (int)
460		(blitq - ((drm_via_private_t *)dev->dev_private)->blit_queues);
461
462	DRM_DEBUG("Polling timer called for engine %d, jiffies %lu\n", engine,
463		  (unsigned long) jiffies);
464
465	via_dmablit_handler(dev, engine, 0);
466
467	if (!timer_pending(&blitq->poll_timer)) {
468		mod_timer(&blitq->poll_timer, jiffies + 1);
469
470	       /*
471		* Rerun handler to delete timer if engines are off, and
472		* to shorten abort latency. This is a little nasty.
473		*/
474
475	       via_dmablit_handler(dev, engine, 0);
476
477	}
478}
479
480
481
482
483/*
484 * Workqueue task that frees data and mappings associated with a blit.
485 * Also wakes up waiting processes. Each of these tasks handles one
486 * blit engine only and may not be called on each interrupt.
487 */
488
489
490static void
491via_dmablit_workqueue(struct work_struct *work)
492{
493	drm_via_blitq_t *blitq = container_of(work, drm_via_blitq_t, wq);
494	struct drm_device *dev = blitq->dev;
495	unsigned long irqsave;
496	drm_via_sg_info_t *cur_sg;
497	int cur_released;
498
499
500	DRM_DEBUG("Workqueue task called for blit engine %ld\n", (unsigned long)
501		  (blitq - ((drm_via_private_t *)dev->dev_private)->blit_queues));
502
503	spin_lock_irqsave(&blitq->blit_lock, irqsave);
504
505	while (blitq->serviced != blitq->cur) {
506
507		cur_released = blitq->serviced++;
508
509		DRM_DEBUG("Releasing blit slot %d\n", cur_released);
510
511		if (blitq->serviced >= VIA_NUM_BLIT_SLOTS)
512			blitq->serviced = 0;
513
514		cur_sg = blitq->blits[cur_released];
515		blitq->num_free++;
516
517		spin_unlock_irqrestore(&blitq->blit_lock, irqsave);
518
519		wake_up(&blitq->busy_queue);
520
521		via_free_sg_info(dev->pdev, cur_sg);
522		kfree(cur_sg);
523
524		spin_lock_irqsave(&blitq->blit_lock, irqsave);
525	}
526
527	spin_unlock_irqrestore(&blitq->blit_lock, irqsave);
528}
529
530
531/*
532 * Init all blit engines. Currently we use two, but some hardware have 4.
533 */
534
535
536void
537via_init_dmablit(struct drm_device *dev)
538{
539	int i, j;
540	drm_via_private_t *dev_priv = (drm_via_private_t *)dev->dev_private;
541	drm_via_blitq_t *blitq;
542
543	pci_set_master(dev->pdev);
544
545	for (i = 0; i < VIA_NUM_BLIT_ENGINES; ++i) {
546		blitq = dev_priv->blit_queues + i;
547		blitq->dev = dev;
548		blitq->cur_blit_handle = 0;
549		blitq->done_blit_handle = 0;
550		blitq->head = 0;
551		blitq->cur = 0;
552		blitq->serviced = 0;
553		blitq->num_free = VIA_NUM_BLIT_SLOTS - 1;
554		blitq->num_outstanding = 0;
555		blitq->is_active = 0;
556		blitq->aborting = 0;
557		spin_lock_init(&blitq->blit_lock);
558		for (j = 0; j < VIA_NUM_BLIT_SLOTS; ++j)
559			init_waitqueue_head(blitq->blit_queue + j);
560		init_waitqueue_head(&blitq->busy_queue);
561		INIT_WORK(&blitq->wq, via_dmablit_workqueue);
562		timer_setup(&blitq->poll_timer, via_dmablit_timer, 0);
 
563	}
564}
565
566/*
567 * Build all info and do all mappings required for a blit.
568 */
569
570
571static int
572via_build_sg_info(struct drm_device *dev, drm_via_sg_info_t *vsg, drm_via_dmablit_t *xfer)
573{
574	int draw = xfer->to_fb;
575	int ret = 0;
576
577	vsg->direction = (draw) ? DMA_TO_DEVICE : DMA_FROM_DEVICE;
578	vsg->bounce_buffer = NULL;
579
580	vsg->state = dr_via_sg_init;
581
582	if (xfer->num_lines <= 0 || xfer->line_length <= 0) {
583		DRM_ERROR("Zero size bitblt.\n");
584		return -EINVAL;
585	}
586
587	/*
588	 * Below check is a driver limitation, not a hardware one. We
589	 * don't want to lock unused pages, and don't want to incoporate the
590	 * extra logic of avoiding them. Make sure there are no.
591	 * (Not a big limitation anyway.)
592	 */
593
594	if ((xfer->mem_stride - xfer->line_length) > 2*PAGE_SIZE) {
595		DRM_ERROR("Too large system memory stride. Stride: %d, "
596			  "Length: %d\n", xfer->mem_stride, xfer->line_length);
597		return -EINVAL;
598	}
599
600	if ((xfer->mem_stride == xfer->line_length) &&
601	   (xfer->fb_stride == xfer->line_length)) {
602		xfer->mem_stride *= xfer->num_lines;
603		xfer->line_length = xfer->mem_stride;
604		xfer->fb_stride = xfer->mem_stride;
605		xfer->num_lines = 1;
606	}
607
608	/*
609	 * Don't lock an arbitrary large number of pages, since that causes a
610	 * DOS security hole.
611	 */
612
613	if (xfer->num_lines > 2048 || (xfer->num_lines*xfer->mem_stride > (2048*2048*4))) {
614		DRM_ERROR("Too large PCI DMA bitblt.\n");
615		return -EINVAL;
616	}
617
618	/*
619	 * we allow a negative fb stride to allow flipping of images in
620	 * transfer.
621	 */
622
623	if (xfer->mem_stride < xfer->line_length ||
624		abs(xfer->fb_stride) < xfer->line_length) {
625		DRM_ERROR("Invalid frame-buffer / memory stride.\n");
626		return -EINVAL;
627	}
628
629	/*
630	 * A hardware bug seems to be worked around if system memory addresses start on
631	 * 16 byte boundaries. This seems a bit restrictive however. VIA is contacted
632	 * about this. Meanwhile, impose the following restrictions:
633	 */
634
635#ifdef VIA_BUGFREE
636	if ((((unsigned long)xfer->mem_addr & 3) != ((unsigned long)xfer->fb_addr & 3)) ||
637	    ((xfer->num_lines > 1) && ((xfer->mem_stride & 3) != (xfer->fb_stride & 3)))) {
638		DRM_ERROR("Invalid DRM bitblt alignment.\n");
639		return -EINVAL;
640	}
641#else
642	if ((((unsigned long)xfer->mem_addr & 15) ||
643	      ((unsigned long)xfer->fb_addr & 3)) ||
644	   ((xfer->num_lines > 1) &&
645	   ((xfer->mem_stride & 15) || (xfer->fb_stride & 3)))) {
646		DRM_ERROR("Invalid DRM bitblt alignment.\n");
647		return -EINVAL;
648	}
649#endif
650
651	if (0 != (ret = via_lock_all_dma_pages(vsg, xfer))) {
652		DRM_ERROR("Could not lock DMA pages.\n");
653		via_free_sg_info(dev->pdev, vsg);
654		return ret;
655	}
656
657	via_map_blit_for_device(dev->pdev, xfer, vsg, 0);
658	if (0 != (ret = via_alloc_desc_pages(vsg))) {
659		DRM_ERROR("Could not allocate DMA descriptor pages.\n");
660		via_free_sg_info(dev->pdev, vsg);
661		return ret;
662	}
663	via_map_blit_for_device(dev->pdev, xfer, vsg, 1);
664
665	return 0;
666}
667
668
669/*
670 * Reserve one free slot in the blit queue. Will wait for one second for one
671 * to become available. Otherwise -EBUSY is returned.
672 */
673
674static int
675via_dmablit_grab_slot(drm_via_blitq_t *blitq, int engine)
676{
677	int ret = 0;
678	unsigned long irqsave;
679
680	DRM_DEBUG("Num free is %d\n", blitq->num_free);
681	spin_lock_irqsave(&blitq->blit_lock, irqsave);
682	while (blitq->num_free == 0) {
683		spin_unlock_irqrestore(&blitq->blit_lock, irqsave);
684
685		DRM_WAIT_ON(ret, blitq->busy_queue, HZ, blitq->num_free > 0);
686		if (ret)
687			return (-EINTR == ret) ? -EAGAIN : ret;
688
689		spin_lock_irqsave(&blitq->blit_lock, irqsave);
690	}
691
692	blitq->num_free--;
693	spin_unlock_irqrestore(&blitq->blit_lock, irqsave);
694
695	return 0;
696}
697
698/*
699 * Hand back a free slot if we changed our mind.
700 */
701
702static void
703via_dmablit_release_slot(drm_via_blitq_t *blitq)
704{
705	unsigned long irqsave;
706
707	spin_lock_irqsave(&blitq->blit_lock, irqsave);
708	blitq->num_free++;
709	spin_unlock_irqrestore(&blitq->blit_lock, irqsave);
710	wake_up(&blitq->busy_queue);
711}
712
713/*
714 * Grab a free slot. Build blit info and queue a blit.
715 */
716
717
718static int
719via_dmablit(struct drm_device *dev, drm_via_dmablit_t *xfer)
720{
721	drm_via_private_t *dev_priv = (drm_via_private_t *)dev->dev_private;
722	drm_via_sg_info_t *vsg;
723	drm_via_blitq_t *blitq;
724	int ret;
725	int engine;
726	unsigned long irqsave;
727
728	if (dev_priv == NULL) {
729		DRM_ERROR("Called without initialization.\n");
730		return -EINVAL;
731	}
732
733	engine = (xfer->to_fb) ? 0 : 1;
734	blitq = dev_priv->blit_queues + engine;
735	if (0 != (ret = via_dmablit_grab_slot(blitq, engine)))
736		return ret;
737	if (NULL == (vsg = kmalloc(sizeof(*vsg), GFP_KERNEL))) {
738		via_dmablit_release_slot(blitq);
739		return -ENOMEM;
740	}
741	if (0 != (ret = via_build_sg_info(dev, vsg, xfer))) {
742		via_dmablit_release_slot(blitq);
743		kfree(vsg);
744		return ret;
745	}
746	spin_lock_irqsave(&blitq->blit_lock, irqsave);
747
748	blitq->blits[blitq->head++] = vsg;
749	if (blitq->head >= VIA_NUM_BLIT_SLOTS)
750		blitq->head = 0;
751	blitq->num_outstanding++;
752	xfer->sync.sync_handle = ++blitq->cur_blit_handle;
753
754	spin_unlock_irqrestore(&blitq->blit_lock, irqsave);
755	xfer->sync.engine = engine;
756
757	via_dmablit_handler(dev, engine, 0);
758
759	return 0;
760}
761
762/*
763 * Sync on a previously submitted blit. Note that the X server use signals extensively, and
764 * that there is a very big probability that this IOCTL will be interrupted by a signal. In that
765 * case it returns with -EAGAIN for the signal to be delivered.
766 * The caller should then reissue the IOCTL. This is similar to what is being done for drmGetLock().
767 */
768
769int
770via_dma_blit_sync(struct drm_device *dev, void *data, struct drm_file *file_priv)
771{
772	drm_via_blitsync_t *sync = data;
773	int err;
774
775	if (sync->engine >= VIA_NUM_BLIT_ENGINES)
776		return -EINVAL;
777
778	err = via_dmablit_sync(dev, sync->sync_handle, sync->engine);
779
780	if (-EINTR == err)
781		err = -EAGAIN;
782
783	return err;
784}
785
786
787/*
788 * Queue a blit and hand back a handle to be used for sync. This IOCTL may be interrupted by a signal
789 * while waiting for a free slot in the blit queue. In that case it returns with -EAGAIN and should
790 * be reissued. See the above IOCTL code.
791 */
792
793int
794via_dma_blit(struct drm_device *dev, void *data, struct drm_file *file_priv)
795{
796	drm_via_dmablit_t *xfer = data;
797	int err;
798
799	err = via_dmablit(dev, xfer);
800
801	return err;
802}