Loading...
Note: File does not exist in v5.4.
1/*
2 * CARMA DATA-FPGA Access Driver
3 *
4 * Copyright (c) 2009-2011 Ira W. Snyder <iws@ovro.caltech.edu>
5 *
6 * This program is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License as published by the
8 * Free Software Foundation; either version 2 of the License, or (at your
9 * option) any later version.
10 */
11
12/*
13 * FPGA Memory Dump Format
14 *
15 * FPGA #0 control registers (32 x 32-bit words)
16 * FPGA #1 control registers (32 x 32-bit words)
17 * FPGA #2 control registers (32 x 32-bit words)
18 * FPGA #3 control registers (32 x 32-bit words)
19 * SYSFPGA control registers (32 x 32-bit words)
20 * FPGA #0 correlation array (NUM_CORL0 correlation blocks)
21 * FPGA #1 correlation array (NUM_CORL1 correlation blocks)
22 * FPGA #2 correlation array (NUM_CORL2 correlation blocks)
23 * FPGA #3 correlation array (NUM_CORL3 correlation blocks)
24 *
25 * Each correlation array consists of:
26 *
27 * Correlation Data (2 x NUM_LAGSn x 32-bit words)
28 * Pipeline Metadata (2 x NUM_METAn x 32-bit words)
29 * Quantization Counters (2 x NUM_QCNTn x 32-bit words)
30 *
31 * The NUM_CORLn, NUM_LAGSn, NUM_METAn, and NUM_QCNTn values come from
32 * the FPGA configuration registers. They do not change once the FPGA's
33 * have been programmed, they only change on re-programming.
34 */
35
36/*
37 * Basic Description:
38 *
39 * This driver is used to capture correlation spectra off of the four data
40 * processing FPGAs. The FPGAs are often reprogrammed at runtime, therefore
41 * this driver supports dynamic enable/disable of capture while the device
42 * remains open.
43 *
44 * The nominal capture rate is 64Hz (every 15.625ms). To facilitate this fast
45 * capture rate, all buffers are pre-allocated to avoid any potentially long
46 * running memory allocations while capturing.
47 *
48 * There are two lists and one pointer which are used to keep track of the
49 * different states of data buffers.
50 *
51 * 1) free list
52 * This list holds all empty data buffers which are ready to receive data.
53 *
54 * 2) inflight pointer
55 * This pointer holds the currently inflight data buffer. This buffer is having
56 * data copied into it by the DMA engine.
57 *
58 * 3) used list
59 * This list holds data buffers which have been filled, and are waiting to be
60 * read by userspace.
61 *
62 * All buffers start life on the free list, then move successively to the
63 * inflight pointer, and then to the used list. After they have been read by
64 * userspace, they are moved back to the free list. The cycle repeats as long
65 * as necessary.
66 *
67 * It should be noted that all buffers are mapped and ready for DMA when they
68 * are on any of the three lists. They are only unmapped when they are in the
69 * process of being read by userspace.
70 */
71
72/*
73 * Notes on the IRQ masking scheme:
74 *
75 * The IRQ masking scheme here is different than most other hardware. The only
76 * way for the DATA-FPGAs to detect if the kernel has taken too long to copy
77 * the data is if the status registers are not cleared before the next
78 * correlation data dump is ready.
79 *
80 * The interrupt line is connected to the status registers, such that when they
81 * are cleared, the interrupt is de-asserted. Therein lies our problem. We need
82 * to schedule a long-running DMA operation and return from the interrupt
83 * handler quickly, but we cannot clear the status registers.
84 *
85 * To handle this, the system controller FPGA has the capability to connect the
86 * interrupt line to a user-controlled GPIO pin. This pin is driven high
87 * (unasserted) and left that way. To mask the interrupt, we change the
88 * interrupt source to the GPIO pin. Tada, we hid the interrupt. :)
89 */
90
91#include <linux/of_address.h>
92#include <linux/of_irq.h>
93#include <linux/of_platform.h>
94#include <linux/dma-mapping.h>
95#include <linux/miscdevice.h>
96#include <linux/interrupt.h>
97#include <linux/dmaengine.h>
98#include <linux/seq_file.h>
99#include <linux/highmem.h>
100#include <linux/debugfs.h>
101#include <linux/kernel.h>
102#include <linux/module.h>
103#include <linux/poll.h>
104#include <linux/slab.h>
105#include <linux/kref.h>
106#include <linux/io.h>
107
108#include <media/videobuf-dma-sg.h>
109
110/* system controller registers */
111#define SYS_IRQ_SOURCE_CTL 0x24
112#define SYS_IRQ_OUTPUT_EN 0x28
113#define SYS_IRQ_OUTPUT_DATA 0x2C
114#define SYS_IRQ_INPUT_DATA 0x30
115#define SYS_FPGA_CONFIG_STATUS 0x44
116
117/* GPIO IRQ line assignment */
118#define IRQ_CORL_DONE 0x10
119
120/* FPGA registers */
121#define MMAP_REG_VERSION 0x00
122#define MMAP_REG_CORL_CONF1 0x08
123#define MMAP_REG_CORL_CONF2 0x0C
124#define MMAP_REG_STATUS 0x48
125
126#define SYS_FPGA_BLOCK 0xF0000000
127
128#define DATA_FPGA_START 0x400000
129#define DATA_FPGA_SIZE 0x80000
130
131static const char drv_name[] = "carma-fpga";
132
133#define NUM_FPGA 4
134
135#define MIN_DATA_BUFS 8
136#define MAX_DATA_BUFS 64
137
138struct fpga_info {
139 unsigned int num_lag_ram;
140 unsigned int blk_size;
141};
142
143struct data_buf {
144 struct list_head entry;
145 struct videobuf_dmabuf vb;
146 size_t size;
147};
148
149struct fpga_device {
150 /* character device */
151 struct miscdevice miscdev;
152 struct device *dev;
153 struct mutex mutex;
154
155 /* reference count */
156 struct kref ref;
157
158 /* FPGA registers and information */
159 struct fpga_info info[NUM_FPGA];
160 void __iomem *regs;
161 int irq;
162
163 /* FPGA Physical Address/Size Information */
164 resource_size_t phys_addr;
165 size_t phys_size;
166
167 /* DMA structures */
168 struct sg_table corl_table;
169 unsigned int corl_nents;
170 struct dma_chan *chan;
171
172 /* Protection for all members below */
173 spinlock_t lock;
174
175 /* Device enable/disable flag */
176 bool enabled;
177
178 /* Correlation data buffers */
179 wait_queue_head_t wait;
180 struct list_head free;
181 struct list_head used;
182 struct data_buf *inflight;
183
184 /* Information about data buffers */
185 unsigned int num_dropped;
186 unsigned int num_buffers;
187 size_t bufsize;
188 struct dentry *dbg_entry;
189};
190
191struct fpga_reader {
192 struct fpga_device *priv;
193 struct data_buf *buf;
194 off_t buf_start;
195};
196
197static void fpga_device_release(struct kref *ref)
198{
199 struct fpga_device *priv = container_of(ref, struct fpga_device, ref);
200
201 /* the last reader has exited, cleanup the last bits */
202 mutex_destroy(&priv->mutex);
203 kfree(priv);
204}
205
206/*
207 * Data Buffer Allocation Helpers
208 */
209
210/**
211 * data_free_buffer() - free a single data buffer and all allocated memory
212 * @buf: the buffer to free
213 *
214 * This will free all of the pages allocated to the given data buffer, and
215 * then free the structure itself
216 */
217static void data_free_buffer(struct data_buf *buf)
218{
219 /* It is ok to free a NULL buffer */
220 if (!buf)
221 return;
222
223 /* free all memory */
224 videobuf_dma_free(&buf->vb);
225 kfree(buf);
226}
227
228/**
229 * data_alloc_buffer() - allocate and fill a data buffer with pages
230 * @bytes: the number of bytes required
231 *
232 * This allocates all space needed for a data buffer. It must be mapped before
233 * use in a DMA transaction using videobuf_dma_map().
234 *
235 * Returns NULL on failure
236 */
237static struct data_buf *data_alloc_buffer(const size_t bytes)
238{
239 unsigned int nr_pages;
240 struct data_buf *buf;
241 int ret;
242
243 /* calculate the number of pages necessary */
244 nr_pages = DIV_ROUND_UP(bytes, PAGE_SIZE);
245
246 /* allocate the buffer structure */
247 buf = kzalloc(sizeof(*buf), GFP_KERNEL);
248 if (!buf)
249 goto out_return;
250
251 /* initialize internal fields */
252 INIT_LIST_HEAD(&buf->entry);
253 buf->size = bytes;
254
255 /* allocate the videobuf */
256 videobuf_dma_init(&buf->vb);
257 ret = videobuf_dma_init_kernel(&buf->vb, DMA_FROM_DEVICE, nr_pages);
258 if (ret)
259 goto out_free_buf;
260
261 return buf;
262
263out_free_buf:
264 kfree(buf);
265out_return:
266 return NULL;
267}
268
269/**
270 * data_free_buffers() - free all allocated buffers
271 * @priv: the driver's private data structure
272 *
273 * Free all buffers allocated by the driver (except those currently in the
274 * process of being read by userspace).
275 *
276 * LOCKING: must hold dev->mutex
277 * CONTEXT: user
278 */
279static void data_free_buffers(struct fpga_device *priv)
280{
281 struct data_buf *buf, *tmp;
282
283 /* the device should be stopped, no DMA in progress */
284 BUG_ON(priv->inflight != NULL);
285
286 list_for_each_entry_safe(buf, tmp, &priv->free, entry) {
287 list_del_init(&buf->entry);
288 videobuf_dma_unmap(priv->dev, &buf->vb);
289 data_free_buffer(buf);
290 }
291
292 list_for_each_entry_safe(buf, tmp, &priv->used, entry) {
293 list_del_init(&buf->entry);
294 videobuf_dma_unmap(priv->dev, &buf->vb);
295 data_free_buffer(buf);
296 }
297
298 priv->num_buffers = 0;
299 priv->bufsize = 0;
300}
301
302/**
303 * data_alloc_buffers() - allocate 1 seconds worth of data buffers
304 * @priv: the driver's private data structure
305 *
306 * Allocate enough buffers for a whole second worth of data
307 *
308 * This routine will attempt to degrade nicely by succeeding even if a full
309 * second worth of data buffers could not be allocated, as long as a minimum
310 * number were allocated. In this case, it will print a message to the kernel
311 * log.
312 *
313 * The device must not be modifying any lists when this is called.
314 *
315 * CONTEXT: user
316 * LOCKING: must hold dev->mutex
317 *
318 * Returns 0 on success, -ERRNO otherwise
319 */
320static int data_alloc_buffers(struct fpga_device *priv)
321{
322 struct data_buf *buf;
323 int i, ret;
324
325 for (i = 0; i < MAX_DATA_BUFS; i++) {
326
327 /* allocate a buffer */
328 buf = data_alloc_buffer(priv->bufsize);
329 if (!buf)
330 break;
331
332 /* map it for DMA */
333 ret = videobuf_dma_map(priv->dev, &buf->vb);
334 if (ret) {
335 data_free_buffer(buf);
336 break;
337 }
338
339 /* add it to the list of free buffers */
340 list_add_tail(&buf->entry, &priv->free);
341 priv->num_buffers++;
342 }
343
344 /* Make sure we allocated the minimum required number of buffers */
345 if (priv->num_buffers < MIN_DATA_BUFS) {
346 dev_err(priv->dev, "Unable to allocate enough data buffers\n");
347 data_free_buffers(priv);
348 return -ENOMEM;
349 }
350
351 /* Warn if we are running in a degraded state, but do not fail */
352 if (priv->num_buffers < MAX_DATA_BUFS) {
353 dev_warn(priv->dev,
354 "Unable to allocate %d buffers, using %d buffers instead\n",
355 MAX_DATA_BUFS, i);
356 }
357
358 return 0;
359}
360
361/*
362 * DMA Operations Helpers
363 */
364
365/**
366 * fpga_start_addr() - get the physical address a DATA-FPGA
367 * @priv: the driver's private data structure
368 * @fpga: the DATA-FPGA number (zero based)
369 */
370static dma_addr_t fpga_start_addr(struct fpga_device *priv, unsigned int fpga)
371{
372 return priv->phys_addr + 0x400000 + (0x80000 * fpga);
373}
374
375/**
376 * fpga_block_addr() - get the physical address of a correlation data block
377 * @priv: the driver's private data structure
378 * @fpga: the DATA-FPGA number (zero based)
379 * @blknum: the correlation block number (zero based)
380 */
381static dma_addr_t fpga_block_addr(struct fpga_device *priv, unsigned int fpga,
382 unsigned int blknum)
383{
384 return fpga_start_addr(priv, fpga) + (0x10000 * (1 + blknum));
385}
386
387#define REG_BLOCK_SIZE (32 * 4)
388
389/**
390 * data_setup_corl_table() - create the scatterlist for correlation dumps
391 * @priv: the driver's private data structure
392 *
393 * Create the scatterlist for transferring a correlation dump from the
394 * DATA FPGAs. This structure will be reused for each buffer than needs
395 * to be filled with correlation data.
396 *
397 * Returns 0 on success, -ERRNO otherwise
398 */
399static int data_setup_corl_table(struct fpga_device *priv)
400{
401 struct sg_table *table = &priv->corl_table;
402 struct scatterlist *sg;
403 struct fpga_info *info;
404 int i, j, ret;
405
406 /* Calculate the number of entries needed */
407 priv->corl_nents = (1 + NUM_FPGA) * REG_BLOCK_SIZE;
408 for (i = 0; i < NUM_FPGA; i++)
409 priv->corl_nents += priv->info[i].num_lag_ram;
410
411 /* Allocate the scatterlist table */
412 ret = sg_alloc_table(table, priv->corl_nents, GFP_KERNEL);
413 if (ret) {
414 dev_err(priv->dev, "unable to allocate DMA table\n");
415 return ret;
416 }
417
418 /* Add the DATA FPGA registers to the scatterlist */
419 sg = table->sgl;
420 for (i = 0; i < NUM_FPGA; i++) {
421 sg_dma_address(sg) = fpga_start_addr(priv, i);
422 sg_dma_len(sg) = REG_BLOCK_SIZE;
423 sg = sg_next(sg);
424 }
425
426 /* Add the SYS-FPGA registers to the scatterlist */
427 sg_dma_address(sg) = SYS_FPGA_BLOCK;
428 sg_dma_len(sg) = REG_BLOCK_SIZE;
429 sg = sg_next(sg);
430
431 /* Add the FPGA correlation data blocks to the scatterlist */
432 for (i = 0; i < NUM_FPGA; i++) {
433 info = &priv->info[i];
434 for (j = 0; j < info->num_lag_ram; j++) {
435 sg_dma_address(sg) = fpga_block_addr(priv, i, j);
436 sg_dma_len(sg) = info->blk_size;
437 sg = sg_next(sg);
438 }
439 }
440
441 /*
442 * All physical addresses and lengths are present in the structure
443 * now. It can be reused for every FPGA DATA interrupt
444 */
445 return 0;
446}
447
448/*
449 * FPGA Register Access Helpers
450 */
451
452static void fpga_write_reg(struct fpga_device *priv, unsigned int fpga,
453 unsigned int reg, u32 val)
454{
455 const int fpga_start = DATA_FPGA_START + (fpga * DATA_FPGA_SIZE);
456 iowrite32be(val, priv->regs + fpga_start + reg);
457}
458
459static u32 fpga_read_reg(struct fpga_device *priv, unsigned int fpga,
460 unsigned int reg)
461{
462 const int fpga_start = DATA_FPGA_START + (fpga * DATA_FPGA_SIZE);
463 return ioread32be(priv->regs + fpga_start + reg);
464}
465
466/**
467 * data_calculate_bufsize() - calculate the data buffer size required
468 * @priv: the driver's private data structure
469 *
470 * Calculate the total buffer size needed to hold a single block
471 * of correlation data
472 *
473 * CONTEXT: user
474 *
475 * Returns 0 on success, -ERRNO otherwise
476 */
477static int data_calculate_bufsize(struct fpga_device *priv)
478{
479 u32 num_corl, num_lags, num_meta, num_qcnt, num_pack;
480 u32 conf1, conf2, version;
481 u32 num_lag_ram, blk_size;
482 int i;
483
484 /* Each buffer starts with the 5 FPGA register areas */
485 priv->bufsize = (1 + NUM_FPGA) * REG_BLOCK_SIZE;
486
487 /* Read and store the configuration data for each FPGA */
488 for (i = 0; i < NUM_FPGA; i++) {
489 version = fpga_read_reg(priv, i, MMAP_REG_VERSION);
490 conf1 = fpga_read_reg(priv, i, MMAP_REG_CORL_CONF1);
491 conf2 = fpga_read_reg(priv, i, MMAP_REG_CORL_CONF2);
492
493 /* minor version 2 and later */
494 if ((version & 0x000000FF) >= 2) {
495 num_corl = (conf1 & 0x000000F0) >> 4;
496 num_pack = (conf1 & 0x00000F00) >> 8;
497 num_lags = (conf1 & 0x00FFF000) >> 12;
498 num_meta = (conf1 & 0x7F000000) >> 24;
499 num_qcnt = (conf2 & 0x00000FFF) >> 0;
500 } else {
501 num_corl = (conf1 & 0x000000F0) >> 4;
502 num_pack = 1; /* implied */
503 num_lags = (conf1 & 0x000FFF00) >> 8;
504 num_meta = (conf1 & 0x7FF00000) >> 20;
505 num_qcnt = (conf2 & 0x00000FFF) >> 0;
506 }
507
508 num_lag_ram = (num_corl + num_pack - 1) / num_pack;
509 blk_size = ((num_pack * num_lags) + num_meta + num_qcnt) * 8;
510
511 priv->info[i].num_lag_ram = num_lag_ram;
512 priv->info[i].blk_size = blk_size;
513 priv->bufsize += num_lag_ram * blk_size;
514
515 dev_dbg(priv->dev, "FPGA %d NUM_CORL: %d\n", i, num_corl);
516 dev_dbg(priv->dev, "FPGA %d NUM_PACK: %d\n", i, num_pack);
517 dev_dbg(priv->dev, "FPGA %d NUM_LAGS: %d\n", i, num_lags);
518 dev_dbg(priv->dev, "FPGA %d NUM_META: %d\n", i, num_meta);
519 dev_dbg(priv->dev, "FPGA %d NUM_QCNT: %d\n", i, num_qcnt);
520 dev_dbg(priv->dev, "FPGA %d BLK_SIZE: %d\n", i, blk_size);
521 }
522
523 dev_dbg(priv->dev, "TOTAL BUFFER SIZE: %zu bytes\n", priv->bufsize);
524 return 0;
525}
526
527/*
528 * Interrupt Handling
529 */
530
531/**
532 * data_disable_interrupts() - stop the device from generating interrupts
533 * @priv: the driver's private data structure
534 *
535 * Hide interrupts by switching to GPIO interrupt source
536 *
537 * LOCKING: must hold dev->lock
538 */
539static void data_disable_interrupts(struct fpga_device *priv)
540{
541 /* hide the interrupt by switching the IRQ driver to GPIO */
542 iowrite32be(0x2F, priv->regs + SYS_IRQ_SOURCE_CTL);
543}
544
545/**
546 * data_enable_interrupts() - allow the device to generate interrupts
547 * @priv: the driver's private data structure
548 *
549 * Unhide interrupts by switching to the FPGA interrupt source. At the
550 * same time, clear the DATA-FPGA status registers.
551 *
552 * LOCKING: must hold dev->lock
553 */
554static void data_enable_interrupts(struct fpga_device *priv)
555{
556 /* clear the actual FPGA corl_done interrupt */
557 fpga_write_reg(priv, 0, MMAP_REG_STATUS, 0x0);
558 fpga_write_reg(priv, 1, MMAP_REG_STATUS, 0x0);
559 fpga_write_reg(priv, 2, MMAP_REG_STATUS, 0x0);
560 fpga_write_reg(priv, 3, MMAP_REG_STATUS, 0x0);
561
562 /* flush the writes */
563 fpga_read_reg(priv, 0, MMAP_REG_STATUS);
564 fpga_read_reg(priv, 1, MMAP_REG_STATUS);
565 fpga_read_reg(priv, 2, MMAP_REG_STATUS);
566 fpga_read_reg(priv, 3, MMAP_REG_STATUS);
567
568 /* switch back to the external interrupt source */
569 iowrite32be(0x3F, priv->regs + SYS_IRQ_SOURCE_CTL);
570}
571
572/**
573 * data_dma_cb() - DMAEngine callback for DMA completion
574 * @data: the driver's private data structure
575 *
576 * Complete a DMA transfer from the DATA-FPGA's
577 *
578 * This is called via the DMA callback mechanism, and will handle moving the
579 * completed DMA transaction to the used list, and then wake any processes
580 * waiting for new data
581 *
582 * CONTEXT: any, softirq expected
583 */
584static void data_dma_cb(void *data)
585{
586 struct fpga_device *priv = data;
587 unsigned long flags;
588
589 spin_lock_irqsave(&priv->lock, flags);
590
591 /* If there is no inflight buffer, we've got a bug */
592 BUG_ON(priv->inflight == NULL);
593
594 /* Move the inflight buffer onto the used list */
595 list_move_tail(&priv->inflight->entry, &priv->used);
596 priv->inflight = NULL;
597
598 /*
599 * If data dumping is still enabled, then clear the FPGA
600 * status registers and re-enable FPGA interrupts
601 */
602 if (priv->enabled)
603 data_enable_interrupts(priv);
604
605 spin_unlock_irqrestore(&priv->lock, flags);
606
607 /*
608 * We've changed both the inflight and used lists, so we need
609 * to wake up any processes that are blocking for those events
610 */
611 wake_up(&priv->wait);
612}
613
614/**
615 * data_submit_dma() - prepare and submit the required DMA to fill a buffer
616 * @priv: the driver's private data structure
617 * @buf: the data buffer
618 *
619 * Prepare and submit the necessary DMA transactions to fill a correlation
620 * data buffer.
621 *
622 * LOCKING: must hold dev->lock
623 * CONTEXT: hardirq only
624 *
625 * Returns 0 on success, -ERRNO otherwise
626 */
627static int data_submit_dma(struct fpga_device *priv, struct data_buf *buf)
628{
629 struct scatterlist *dst_sg, *src_sg;
630 unsigned int dst_nents, src_nents;
631 struct dma_chan *chan = priv->chan;
632 struct dma_async_tx_descriptor *tx;
633 dma_cookie_t cookie;
634 dma_addr_t dst, src;
635 unsigned long dma_flags = 0;
636
637 dst_sg = buf->vb.sglist;
638 dst_nents = buf->vb.sglen;
639
640 src_sg = priv->corl_table.sgl;
641 src_nents = priv->corl_nents;
642
643 /*
644 * All buffers passed to this function should be ready and mapped
645 * for DMA already. Therefore, we don't need to do anything except
646 * submit it to the Freescale DMA Engine for processing
647 */
648
649 /* setup the scatterlist to scatterlist transfer */
650 tx = chan->device->device_prep_dma_sg(chan,
651 dst_sg, dst_nents,
652 src_sg, src_nents,
653 0);
654 if (!tx) {
655 dev_err(priv->dev, "unable to prep scatterlist DMA\n");
656 return -ENOMEM;
657 }
658
659 /* submit the transaction to the DMA controller */
660 cookie = tx->tx_submit(tx);
661 if (dma_submit_error(cookie)) {
662 dev_err(priv->dev, "unable to submit scatterlist DMA\n");
663 return -ENOMEM;
664 }
665
666 /* Prepare the re-read of the SYS-FPGA block */
667 dst = sg_dma_address(dst_sg) + (NUM_FPGA * REG_BLOCK_SIZE);
668 src = SYS_FPGA_BLOCK;
669 tx = chan->device->device_prep_dma_memcpy(chan, dst, src,
670 REG_BLOCK_SIZE,
671 dma_flags);
672 if (!tx) {
673 dev_err(priv->dev, "unable to prep SYS-FPGA DMA\n");
674 return -ENOMEM;
675 }
676
677 /* Setup the callback */
678 tx->callback = data_dma_cb;
679 tx->callback_param = priv;
680
681 /* submit the transaction to the DMA controller */
682 cookie = tx->tx_submit(tx);
683 if (dma_submit_error(cookie)) {
684 dev_err(priv->dev, "unable to submit SYS-FPGA DMA\n");
685 return -ENOMEM;
686 }
687
688 return 0;
689}
690
691#define CORL_DONE 0x1
692#define CORL_ERR 0x2
693
694static irqreturn_t data_irq(int irq, void *dev_id)
695{
696 struct fpga_device *priv = dev_id;
697 bool submitted = false;
698 struct data_buf *buf;
699 u32 status;
700 int i;
701
702 /* detect spurious interrupts via FPGA status */
703 for (i = 0; i < 4; i++) {
704 status = fpga_read_reg(priv, i, MMAP_REG_STATUS);
705 if (!(status & (CORL_DONE | CORL_ERR))) {
706 dev_err(priv->dev, "spurious irq detected (FPGA)\n");
707 return IRQ_NONE;
708 }
709 }
710
711 /* detect spurious interrupts via raw IRQ pin readback */
712 status = ioread32be(priv->regs + SYS_IRQ_INPUT_DATA);
713 if (status & IRQ_CORL_DONE) {
714 dev_err(priv->dev, "spurious irq detected (IRQ)\n");
715 return IRQ_NONE;
716 }
717
718 spin_lock(&priv->lock);
719
720 /*
721 * This is an error case that should never happen.
722 *
723 * If this driver has a bug and manages to re-enable interrupts while
724 * a DMA is in progress, then we will hit this statement and should
725 * start paying attention immediately.
726 */
727 BUG_ON(priv->inflight != NULL);
728
729 /* hide the interrupt by switching the IRQ driver to GPIO */
730 data_disable_interrupts(priv);
731
732 /* If there are no free buffers, drop this data */
733 if (list_empty(&priv->free)) {
734 priv->num_dropped++;
735 goto out;
736 }
737
738 buf = list_first_entry(&priv->free, struct data_buf, entry);
739 list_del_init(&buf->entry);
740 BUG_ON(buf->size != priv->bufsize);
741
742 /* Submit a DMA transfer to get the correlation data */
743 if (data_submit_dma(priv, buf)) {
744 dev_err(priv->dev, "Unable to setup DMA transfer\n");
745 list_move_tail(&buf->entry, &priv->free);
746 goto out;
747 }
748
749 /* Save the buffer for the DMA callback */
750 priv->inflight = buf;
751 submitted = true;
752
753 /* Start the DMA Engine */
754 dma_async_issue_pending(priv->chan);
755
756out:
757 /* If no DMA was submitted, re-enable interrupts */
758 if (!submitted)
759 data_enable_interrupts(priv);
760
761 spin_unlock(&priv->lock);
762 return IRQ_HANDLED;
763}
764
765/*
766 * Realtime Device Enable Helpers
767 */
768
769/**
770 * data_device_enable() - enable the device for buffered dumping
771 * @priv: the driver's private data structure
772 *
773 * Enable the device for buffered dumping. Allocates buffers and hooks up
774 * the interrupt handler. When this finishes, data will come pouring in.
775 *
776 * LOCKING: must hold dev->mutex
777 * CONTEXT: user context only
778 *
779 * Returns 0 on success, -ERRNO otherwise
780 */
781static int data_device_enable(struct fpga_device *priv)
782{
783 bool enabled;
784 u32 val;
785 int ret;
786
787 /* multiple enables are safe: they do nothing */
788 spin_lock_irq(&priv->lock);
789 enabled = priv->enabled;
790 spin_unlock_irq(&priv->lock);
791 if (enabled)
792 return 0;
793
794 /* check that the FPGAs are programmed */
795 val = ioread32be(priv->regs + SYS_FPGA_CONFIG_STATUS);
796 if (!(val & (1 << 18))) {
797 dev_err(priv->dev, "DATA-FPGAs are not enabled\n");
798 return -ENODATA;
799 }
800
801 /* read the FPGAs to calculate the buffer size */
802 ret = data_calculate_bufsize(priv);
803 if (ret) {
804 dev_err(priv->dev, "unable to calculate buffer size\n");
805 goto out_error;
806 }
807
808 /* allocate the correlation data buffers */
809 ret = data_alloc_buffers(priv);
810 if (ret) {
811 dev_err(priv->dev, "unable to allocate buffers\n");
812 goto out_error;
813 }
814
815 /* setup the source scatterlist for dumping correlation data */
816 ret = data_setup_corl_table(priv);
817 if (ret) {
818 dev_err(priv->dev, "unable to setup correlation DMA table\n");
819 goto out_error;
820 }
821
822 /* prevent the FPGAs from generating interrupts */
823 data_disable_interrupts(priv);
824
825 /* hookup the irq handler */
826 ret = request_irq(priv->irq, data_irq, IRQF_SHARED, drv_name, priv);
827 if (ret) {
828 dev_err(priv->dev, "unable to request IRQ handler\n");
829 goto out_error;
830 }
831
832 /* allow the DMA callback to re-enable FPGA interrupts */
833 spin_lock_irq(&priv->lock);
834 priv->enabled = true;
835 spin_unlock_irq(&priv->lock);
836
837 /* allow the FPGAs to generate interrupts */
838 data_enable_interrupts(priv);
839 return 0;
840
841out_error:
842 sg_free_table(&priv->corl_table);
843 priv->corl_nents = 0;
844
845 data_free_buffers(priv);
846 return ret;
847}
848
849/**
850 * data_device_disable() - disable the device for buffered dumping
851 * @priv: the driver's private data structure
852 *
853 * Disable the device for buffered dumping. Stops new DMA transactions from
854 * being generated, waits for all outstanding DMA to complete, and then frees
855 * all buffers.
856 *
857 * LOCKING: must hold dev->mutex
858 * CONTEXT: user only
859 *
860 * Returns 0 on success, -ERRNO otherwise
861 */
862static int data_device_disable(struct fpga_device *priv)
863{
864 spin_lock_irq(&priv->lock);
865
866 /* allow multiple disable */
867 if (!priv->enabled) {
868 spin_unlock_irq(&priv->lock);
869 return 0;
870 }
871
872 /*
873 * Mark the device disabled
874 *
875 * This stops DMA callbacks from re-enabling interrupts
876 */
877 priv->enabled = false;
878
879 /* prevent the FPGAs from generating interrupts */
880 data_disable_interrupts(priv);
881
882 /* wait until all ongoing DMA has finished */
883 while (priv->inflight != NULL) {
884 spin_unlock_irq(&priv->lock);
885 wait_event(priv->wait, priv->inflight == NULL);
886 spin_lock_irq(&priv->lock);
887 }
888
889 spin_unlock_irq(&priv->lock);
890
891 /* unhook the irq handler */
892 free_irq(priv->irq, priv);
893
894 /* free the correlation table */
895 sg_free_table(&priv->corl_table);
896 priv->corl_nents = 0;
897
898 /* free all buffers: the free and used lists are not being changed */
899 data_free_buffers(priv);
900 return 0;
901}
902
903/*
904 * DEBUGFS Interface
905 */
906#ifdef CONFIG_DEBUG_FS
907
908/*
909 * Count the number of entries in the given list
910 */
911static unsigned int list_num_entries(struct list_head *list)
912{
913 struct list_head *entry;
914 unsigned int ret = 0;
915
916 list_for_each(entry, list)
917 ret++;
918
919 return ret;
920}
921
922static int data_debug_show(struct seq_file *f, void *offset)
923{
924 struct fpga_device *priv = f->private;
925
926 spin_lock_irq(&priv->lock);
927
928 seq_printf(f, "enabled: %d\n", priv->enabled);
929 seq_printf(f, "bufsize: %d\n", priv->bufsize);
930 seq_printf(f, "num_buffers: %d\n", priv->num_buffers);
931 seq_printf(f, "num_free: %d\n", list_num_entries(&priv->free));
932 seq_printf(f, "inflight: %d\n", priv->inflight != NULL);
933 seq_printf(f, "num_used: %d\n", list_num_entries(&priv->used));
934 seq_printf(f, "num_dropped: %d\n", priv->num_dropped);
935
936 spin_unlock_irq(&priv->lock);
937 return 0;
938}
939
940static int data_debug_open(struct inode *inode, struct file *file)
941{
942 return single_open(file, data_debug_show, inode->i_private);
943}
944
945static const struct file_operations data_debug_fops = {
946 .owner = THIS_MODULE,
947 .open = data_debug_open,
948 .read = seq_read,
949 .llseek = seq_lseek,
950 .release = single_release,
951};
952
953static int data_debugfs_init(struct fpga_device *priv)
954{
955 priv->dbg_entry = debugfs_create_file(drv_name, S_IRUGO, NULL, priv,
956 &data_debug_fops);
957 if (IS_ERR(priv->dbg_entry))
958 return PTR_ERR(priv->dbg_entry);
959
960 return 0;
961}
962
963static void data_debugfs_exit(struct fpga_device *priv)
964{
965 debugfs_remove(priv->dbg_entry);
966}
967
968#else
969
970static inline int data_debugfs_init(struct fpga_device *priv)
971{
972 return 0;
973}
974
975static inline void data_debugfs_exit(struct fpga_device *priv)
976{
977}
978
979#endif /* CONFIG_DEBUG_FS */
980
981/*
982 * SYSFS Attributes
983 */
984
985static ssize_t data_en_show(struct device *dev, struct device_attribute *attr,
986 char *buf)
987{
988 struct fpga_device *priv = dev_get_drvdata(dev);
989 int ret;
990
991 spin_lock_irq(&priv->lock);
992 ret = snprintf(buf, PAGE_SIZE, "%u\n", priv->enabled);
993 spin_unlock_irq(&priv->lock);
994
995 return ret;
996}
997
998static ssize_t data_en_set(struct device *dev, struct device_attribute *attr,
999 const char *buf, size_t count)
1000{
1001 struct fpga_device *priv = dev_get_drvdata(dev);
1002 unsigned long enable;
1003 int ret;
1004
1005 ret = kstrtoul(buf, 0, &enable);
1006 if (ret) {
1007 dev_err(priv->dev, "unable to parse enable input\n");
1008 return ret;
1009 }
1010
1011 /* protect against concurrent enable/disable */
1012 ret = mutex_lock_interruptible(&priv->mutex);
1013 if (ret)
1014 return ret;
1015
1016 if (enable)
1017 ret = data_device_enable(priv);
1018 else
1019 ret = data_device_disable(priv);
1020
1021 if (ret) {
1022 dev_err(priv->dev, "device %s failed\n",
1023 enable ? "enable" : "disable");
1024 count = ret;
1025 goto out_unlock;
1026 }
1027
1028out_unlock:
1029 mutex_unlock(&priv->mutex);
1030 return count;
1031}
1032
1033static DEVICE_ATTR(enable, S_IWUSR | S_IRUGO, data_en_show, data_en_set);
1034
1035static struct attribute *data_sysfs_attrs[] = {
1036 &dev_attr_enable.attr,
1037 NULL,
1038};
1039
1040static const struct attribute_group rt_sysfs_attr_group = {
1041 .attrs = data_sysfs_attrs,
1042};
1043
1044/*
1045 * FPGA Realtime Data Character Device
1046 */
1047
1048static int data_open(struct inode *inode, struct file *filp)
1049{
1050 /*
1051 * The miscdevice layer puts our struct miscdevice into the
1052 * filp->private_data field. We use this to find our private
1053 * data and then overwrite it with our own private structure.
1054 */
1055 struct fpga_device *priv = container_of(filp->private_data,
1056 struct fpga_device, miscdev);
1057 struct fpga_reader *reader;
1058 int ret;
1059
1060 /* allocate private data */
1061 reader = kzalloc(sizeof(*reader), GFP_KERNEL);
1062 if (!reader)
1063 return -ENOMEM;
1064
1065 reader->priv = priv;
1066 reader->buf = NULL;
1067
1068 filp->private_data = reader;
1069 ret = nonseekable_open(inode, filp);
1070 if (ret) {
1071 dev_err(priv->dev, "nonseekable-open failed\n");
1072 kfree(reader);
1073 return ret;
1074 }
1075
1076 /*
1077 * success, increase the reference count of the private data structure
1078 * so that it doesn't disappear if the device is unbound
1079 */
1080 kref_get(&priv->ref);
1081 return 0;
1082}
1083
1084static int data_release(struct inode *inode, struct file *filp)
1085{
1086 struct fpga_reader *reader = filp->private_data;
1087 struct fpga_device *priv = reader->priv;
1088
1089 /* free the per-reader structure */
1090 data_free_buffer(reader->buf);
1091 kfree(reader);
1092 filp->private_data = NULL;
1093
1094 /* decrement our reference count to the private data */
1095 kref_put(&priv->ref, fpga_device_release);
1096 return 0;
1097}
1098
1099static ssize_t data_read(struct file *filp, char __user *ubuf, size_t count,
1100 loff_t *f_pos)
1101{
1102 struct fpga_reader *reader = filp->private_data;
1103 struct fpga_device *priv = reader->priv;
1104 struct list_head *used = &priv->used;
1105 bool drop_buffer = false;
1106 struct data_buf *dbuf;
1107 size_t avail;
1108 void *data;
1109 int ret;
1110
1111 /* check if we already have a partial buffer */
1112 if (reader->buf) {
1113 dbuf = reader->buf;
1114 goto have_buffer;
1115 }
1116
1117 spin_lock_irq(&priv->lock);
1118
1119 /* Block until there is at least one buffer on the used list */
1120 while (list_empty(used)) {
1121 spin_unlock_irq(&priv->lock);
1122
1123 if (filp->f_flags & O_NONBLOCK)
1124 return -EAGAIN;
1125
1126 ret = wait_event_interruptible(priv->wait, !list_empty(used));
1127 if (ret)
1128 return ret;
1129
1130 spin_lock_irq(&priv->lock);
1131 }
1132
1133 /* Grab the first buffer off of the used list */
1134 dbuf = list_first_entry(used, struct data_buf, entry);
1135 list_del_init(&dbuf->entry);
1136
1137 spin_unlock_irq(&priv->lock);
1138
1139 /* Buffers are always mapped: unmap it */
1140 videobuf_dma_unmap(priv->dev, &dbuf->vb);
1141
1142 /* save the buffer for later */
1143 reader->buf = dbuf;
1144 reader->buf_start = 0;
1145
1146have_buffer:
1147 /* Get the number of bytes available */
1148 avail = dbuf->size - reader->buf_start;
1149 data = dbuf->vb.vaddr + reader->buf_start;
1150
1151 /* Get the number of bytes we can transfer */
1152 count = min(count, avail);
1153
1154 /* Copy the data to the userspace buffer */
1155 if (copy_to_user(ubuf, data, count))
1156 return -EFAULT;
1157
1158 /* Update the amount of available space */
1159 avail -= count;
1160
1161 /*
1162 * If there is still some data available, save the buffer for the
1163 * next userspace call to read() and return
1164 */
1165 if (avail > 0) {
1166 reader->buf_start += count;
1167 reader->buf = dbuf;
1168 return count;
1169 }
1170
1171 /*
1172 * Get the buffer ready to be reused for DMA
1173 *
1174 * If it fails, we pretend that the read never happed and return
1175 * -EFAULT to userspace. The read will be retried.
1176 */
1177 ret = videobuf_dma_map(priv->dev, &dbuf->vb);
1178 if (ret) {
1179 dev_err(priv->dev, "unable to remap buffer for DMA\n");
1180 return -EFAULT;
1181 }
1182
1183 /* Lock against concurrent enable/disable */
1184 spin_lock_irq(&priv->lock);
1185
1186 /* the reader is finished with this buffer */
1187 reader->buf = NULL;
1188
1189 /*
1190 * One of two things has happened, the device is disabled, or the
1191 * device has been reconfigured underneath us. In either case, we
1192 * should just throw away the buffer.
1193 *
1194 * Lockdep complains if this is done under the spinlock, so we
1195 * handle it during the unlock path.
1196 */
1197 if (!priv->enabled || dbuf->size != priv->bufsize) {
1198 drop_buffer = true;
1199 goto out_unlock;
1200 }
1201
1202 /* The buffer is safe to reuse, so add it back to the free list */
1203 list_add_tail(&dbuf->entry, &priv->free);
1204
1205out_unlock:
1206 spin_unlock_irq(&priv->lock);
1207
1208 if (drop_buffer) {
1209 videobuf_dma_unmap(priv->dev, &dbuf->vb);
1210 data_free_buffer(dbuf);
1211 }
1212
1213 return count;
1214}
1215
1216static unsigned int data_poll(struct file *filp, struct poll_table_struct *tbl)
1217{
1218 struct fpga_reader *reader = filp->private_data;
1219 struct fpga_device *priv = reader->priv;
1220 unsigned int mask = 0;
1221
1222 poll_wait(filp, &priv->wait, tbl);
1223
1224 if (!list_empty(&priv->used))
1225 mask |= POLLIN | POLLRDNORM;
1226
1227 return mask;
1228}
1229
1230static int data_mmap(struct file *filp, struct vm_area_struct *vma)
1231{
1232 struct fpga_reader *reader = filp->private_data;
1233 struct fpga_device *priv = reader->priv;
1234 unsigned long offset, vsize, psize, addr;
1235
1236 /* VMA properties */
1237 offset = vma->vm_pgoff << PAGE_SHIFT;
1238 vsize = vma->vm_end - vma->vm_start;
1239 psize = priv->phys_size - offset;
1240 addr = (priv->phys_addr + offset) >> PAGE_SHIFT;
1241
1242 /* Check against the FPGA region's physical memory size */
1243 if (vsize > psize) {
1244 dev_err(priv->dev, "requested mmap mapping too large\n");
1245 return -EINVAL;
1246 }
1247
1248 vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
1249
1250 return io_remap_pfn_range(vma, vma->vm_start, addr, vsize,
1251 vma->vm_page_prot);
1252}
1253
1254static const struct file_operations data_fops = {
1255 .owner = THIS_MODULE,
1256 .open = data_open,
1257 .release = data_release,
1258 .read = data_read,
1259 .poll = data_poll,
1260 .mmap = data_mmap,
1261 .llseek = no_llseek,
1262};
1263
1264/*
1265 * OpenFirmware Device Subsystem
1266 */
1267
1268static bool dma_filter(struct dma_chan *chan, void *data)
1269{
1270 /*
1271 * DMA Channel #0 is used for the FPGA Programmer, so ignore it
1272 *
1273 * This probably won't survive an unload/load cycle of the Freescale
1274 * DMAEngine driver, but that won't be a problem
1275 */
1276 if (chan->chan_id == 0 && chan->device->dev_id == 0)
1277 return false;
1278
1279 return true;
1280}
1281
1282static int data_of_probe(struct platform_device *op)
1283{
1284 struct device_node *of_node = op->dev.of_node;
1285 struct device *this_device;
1286 struct fpga_device *priv;
1287 struct resource res;
1288 dma_cap_mask_t mask;
1289 int ret;
1290
1291 /* Allocate private data */
1292 priv = kzalloc(sizeof(*priv), GFP_KERNEL);
1293 if (!priv) {
1294 dev_err(&op->dev, "Unable to allocate device private data\n");
1295 ret = -ENOMEM;
1296 goto out_return;
1297 }
1298
1299 platform_set_drvdata(op, priv);
1300 priv->dev = &op->dev;
1301 kref_init(&priv->ref);
1302 mutex_init(&priv->mutex);
1303
1304 dev_set_drvdata(priv->dev, priv);
1305 spin_lock_init(&priv->lock);
1306 INIT_LIST_HEAD(&priv->free);
1307 INIT_LIST_HEAD(&priv->used);
1308 init_waitqueue_head(&priv->wait);
1309
1310 /* Setup the misc device */
1311 priv->miscdev.minor = MISC_DYNAMIC_MINOR;
1312 priv->miscdev.name = drv_name;
1313 priv->miscdev.fops = &data_fops;
1314
1315 /* Get the physical address of the FPGA registers */
1316 ret = of_address_to_resource(of_node, 0, &res);
1317 if (ret) {
1318 dev_err(&op->dev, "Unable to find FPGA physical address\n");
1319 ret = -ENODEV;
1320 goto out_free_priv;
1321 }
1322
1323 priv->phys_addr = res.start;
1324 priv->phys_size = resource_size(&res);
1325
1326 /* ioremap the registers for use */
1327 priv->regs = of_iomap(of_node, 0);
1328 if (!priv->regs) {
1329 dev_err(&op->dev, "Unable to ioremap registers\n");
1330 ret = -ENOMEM;
1331 goto out_free_priv;
1332 }
1333
1334 dma_cap_zero(mask);
1335 dma_cap_set(DMA_MEMCPY, mask);
1336 dma_cap_set(DMA_INTERRUPT, mask);
1337 dma_cap_set(DMA_SLAVE, mask);
1338 dma_cap_set(DMA_SG, mask);
1339
1340 /* Request a DMA channel */
1341 priv->chan = dma_request_channel(mask, dma_filter, NULL);
1342 if (!priv->chan) {
1343 dev_err(&op->dev, "Unable to request DMA channel\n");
1344 ret = -ENODEV;
1345 goto out_unmap_regs;
1346 }
1347
1348 /* Find the correct IRQ number */
1349 priv->irq = irq_of_parse_and_map(of_node, 0);
1350 if (priv->irq == NO_IRQ) {
1351 dev_err(&op->dev, "Unable to find IRQ line\n");
1352 ret = -ENODEV;
1353 goto out_release_dma;
1354 }
1355
1356 /* Drive the GPIO for FPGA IRQ high (no interrupt) */
1357 iowrite32be(IRQ_CORL_DONE, priv->regs + SYS_IRQ_OUTPUT_DATA);
1358
1359 /* Register the miscdevice */
1360 ret = misc_register(&priv->miscdev);
1361 if (ret) {
1362 dev_err(&op->dev, "Unable to register miscdevice\n");
1363 goto out_irq_dispose_mapping;
1364 }
1365
1366 /* Create the debugfs files */
1367 ret = data_debugfs_init(priv);
1368 if (ret) {
1369 dev_err(&op->dev, "Unable to create debugfs files\n");
1370 goto out_misc_deregister;
1371 }
1372
1373 /* Create the sysfs files */
1374 this_device = priv->miscdev.this_device;
1375 dev_set_drvdata(this_device, priv);
1376 ret = sysfs_create_group(&this_device->kobj, &rt_sysfs_attr_group);
1377 if (ret) {
1378 dev_err(&op->dev, "Unable to create sysfs files\n");
1379 goto out_data_debugfs_exit;
1380 }
1381
1382 dev_info(&op->dev, "CARMA FPGA Realtime Data Driver Loaded\n");
1383 return 0;
1384
1385out_data_debugfs_exit:
1386 data_debugfs_exit(priv);
1387out_misc_deregister:
1388 misc_deregister(&priv->miscdev);
1389out_irq_dispose_mapping:
1390 irq_dispose_mapping(priv->irq);
1391out_release_dma:
1392 dma_release_channel(priv->chan);
1393out_unmap_regs:
1394 iounmap(priv->regs);
1395out_free_priv:
1396 kref_put(&priv->ref, fpga_device_release);
1397out_return:
1398 return ret;
1399}
1400
1401static int data_of_remove(struct platform_device *op)
1402{
1403 struct fpga_device *priv = platform_get_drvdata(op);
1404 struct device *this_device = priv->miscdev.this_device;
1405
1406 /* remove all sysfs files, now the device cannot be re-enabled */
1407 sysfs_remove_group(&this_device->kobj, &rt_sysfs_attr_group);
1408
1409 /* remove all debugfs files */
1410 data_debugfs_exit(priv);
1411
1412 /* disable the device from generating data */
1413 data_device_disable(priv);
1414
1415 /* remove the character device to stop new readers from appearing */
1416 misc_deregister(&priv->miscdev);
1417
1418 /* cleanup everything not needed by readers */
1419 irq_dispose_mapping(priv->irq);
1420 dma_release_channel(priv->chan);
1421 iounmap(priv->regs);
1422
1423 /* release our reference */
1424 kref_put(&priv->ref, fpga_device_release);
1425 return 0;
1426}
1427
1428static struct of_device_id data_of_match[] = {
1429 { .compatible = "carma,carma-fpga", },
1430 {},
1431};
1432
1433static struct platform_driver data_of_driver = {
1434 .probe = data_of_probe,
1435 .remove = data_of_remove,
1436 .driver = {
1437 .name = drv_name,
1438 .of_match_table = data_of_match,
1439 .owner = THIS_MODULE,
1440 },
1441};
1442
1443module_platform_driver(data_of_driver);
1444
1445MODULE_AUTHOR("Ira W. Snyder <iws@ovro.caltech.edu>");
1446MODULE_DESCRIPTION("CARMA DATA-FPGA Access Driver");
1447MODULE_LICENSE("GPL");