1/*
   2 * CARMA DATA-FPGA Access Driver
   3 *
   4 * Copyright (c) 2009-2011 Ira W. Snyder <iws@ovro.caltech.edu>
   5 *
   6 * This program is free software; you can redistribute it and/or modify it
   7 * under the terms of the GNU General Public License as published by the
   8 * Free Software Foundation; either version 2 of the License, or (at your
   9 * option) any later version.
  10 */
  11
  12/*
  13 * FPGA Memory Dump Format
  14 *
  15 * FPGA #0 control registers (32 x 32-bit words)
  16 * FPGA #1 control registers (32 x 32-bit words)
  17 * FPGA #2 control registers (32 x 32-bit words)
  18 * FPGA #3 control registers (32 x 32-bit words)
  19 * SYSFPGA control registers (32 x 32-bit words)
  20 * FPGA #0 correlation array (NUM_CORL0 correlation blocks)
  21 * FPGA #1 correlation array (NUM_CORL1 correlation blocks)
  22 * FPGA #2 correlation array (NUM_CORL2 correlation blocks)
  23 * FPGA #3 correlation array (NUM_CORL3 correlation blocks)
  24 *
  25 * Each correlation array consists of:
  26 *
  27 * Correlation Data      (2 x NUM_LAGSn x 32-bit words)
  28 * Pipeline Metadata     (2 x NUM_METAn x 32-bit words)
  29 * Quantization Counters (2 x NUM_QCNTn x 32-bit words)
  30 *
  31 * The NUM_CORLn, NUM_LAGSn, NUM_METAn, and NUM_QCNTn values come from
  32 * the FPGA configuration registers. They do not change once the FPGA's
  33 * have been programmed, they only change on re-programming.
  34 */
  35
  36/*
  37 * Basic Description:
  38 *
  39 * This driver is used to capture correlation spectra off of the four data
  40 * processing FPGAs. The FPGAs are often reprogrammed at runtime, therefore
  41 * this driver supports dynamic enable/disable of capture while the device
  42 * remains open.
  43 *
  44 * The nominal capture rate is 64Hz (every 15.625ms). To facilitate this fast
  45 * capture rate, all buffers are pre-allocated to avoid any potentially long
  46 * running memory allocations while capturing.
  47 *
  48 * There are two lists and one pointer which are used to keep track of the
  49 * different states of data buffers.
  50 *
  51 * 1) free list
  52 * This list holds all empty data buffers which are ready to receive data.
  53 *
  54 * 2) inflight pointer
  55 * This pointer holds the currently inflight data buffer. This buffer is having
  56 * data copied into it by the DMA engine.
  57 *
  58 * 3) used list
  59 * This list holds data buffers which have been filled, and are waiting to be
  60 * read by userspace.
  61 *
  62 * All buffers start life on the free list, then move successively to the
  63 * inflight pointer, and then to the used list. After they have been read by
  64 * userspace, they are moved back to the free list. The cycle repeats as long
  65 * as necessary.
  66 *
  67 * It should be noted that all buffers are mapped and ready for DMA when they
  68 * are on any of the three lists. They are only unmapped when they are in the
  69 * process of being read by userspace.
  70 */
  71
  72/*
  73 * Notes on the IRQ masking scheme:
  74 *
  75 * The IRQ masking scheme here is different than most other hardware. The only
  76 * way for the DATA-FPGAs to detect if the kernel has taken too long to copy
  77 * the data is if the status registers are not cleared before the next
  78 * correlation data dump is ready.
  79 *
  80 * The interrupt line is connected to the status registers, such that when they
  81 * are cleared, the interrupt is de-asserted. Therein lies our problem. We need
  82 * to schedule a long-running DMA operation and return from the interrupt
  83 * handler quickly, but we cannot clear the status registers.
  84 *
  85 * To handle this, the system controller FPGA has the capability to connect the
  86 * interrupt line to a user-controlled GPIO pin. This pin is driven high
  87 * (unasserted) and left that way. To mask the interrupt, we change the
  88 * interrupt source to the GPIO pin. Tada, we hid the interrupt. :)
  89 */
  90
 
 
  91#include <linux/of_platform.h>
  92#include <linux/dma-mapping.h>
  93#include <linux/miscdevice.h>
  94#include <linux/interrupt.h>
  95#include <linux/dmaengine.h>
  96#include <linux/seq_file.h>
  97#include <linux/highmem.h>
  98#include <linux/debugfs.h>
  99#include <linux/kernel.h>
 100#include <linux/module.h>
 101#include <linux/poll.h>
 102#include <linux/init.h>
 103#include <linux/slab.h>
 104#include <linux/kref.h>
 105#include <linux/io.h>
 106
 107#include <media/videobuf-dma-sg.h>
 108
 109/* system controller registers */
 110#define SYS_IRQ_SOURCE_CTL	0x24
 111#define SYS_IRQ_OUTPUT_EN	0x28
 112#define SYS_IRQ_OUTPUT_DATA	0x2C
 113#define SYS_IRQ_INPUT_DATA	0x30
 114#define SYS_FPGA_CONFIG_STATUS	0x44
 115
 116/* GPIO IRQ line assignment */
 117#define IRQ_CORL_DONE		0x10
 118
 119/* FPGA registers */
 120#define MMAP_REG_VERSION	0x00
 121#define MMAP_REG_CORL_CONF1	0x08
 122#define MMAP_REG_CORL_CONF2	0x0C
 123#define MMAP_REG_STATUS		0x48
 124
 125#define SYS_FPGA_BLOCK		0xF0000000
 126
 127#define DATA_FPGA_START		0x400000
 128#define DATA_FPGA_SIZE		0x80000
 129
 130static const char drv_name[] = "carma-fpga";
 131
 132#define NUM_FPGA	4
 133
 134#define MIN_DATA_BUFS	8
 135#define MAX_DATA_BUFS	64
 136
 137struct fpga_info {
 138	unsigned int num_lag_ram;
 139	unsigned int blk_size;
 140};
 141
 142struct data_buf {
 143	struct list_head entry;
 144	struct videobuf_dmabuf vb;
 145	size_t size;
 146};
 147
 148struct fpga_device {
 149	/* character device */
 150	struct miscdevice miscdev;
 151	struct device *dev;
 152	struct mutex mutex;
 153
 154	/* reference count */
 155	struct kref ref;
 156
 157	/* FPGA registers and information */
 158	struct fpga_info info[NUM_FPGA];
 159	void __iomem *regs;
 160	int irq;
 161
 162	/* FPGA Physical Address/Size Information */
 163	resource_size_t phys_addr;
 164	size_t phys_size;
 165
 166	/* DMA structures */
 167	struct sg_table corl_table;
 168	unsigned int corl_nents;
 169	struct dma_chan *chan;
 170
 171	/* Protection for all members below */
 172	spinlock_t lock;
 173
 174	/* Device enable/disable flag */
 175	bool enabled;
 176
 177	/* Correlation data buffers */
 178	wait_queue_head_t wait;
 179	struct list_head free;
 180	struct list_head used;
 181	struct data_buf *inflight;
 182
 183	/* Information about data buffers */
 184	unsigned int num_dropped;
 185	unsigned int num_buffers;
 186	size_t bufsize;
 187	struct dentry *dbg_entry;
 188};
 189
 190struct fpga_reader {
 191	struct fpga_device *priv;
 192	struct data_buf *buf;
 193	off_t buf_start;
 194};
 195
 196static void fpga_device_release(struct kref *ref)
 197{
 198	struct fpga_device *priv = container_of(ref, struct fpga_device, ref);
 199
 200	/* the last reader has exited, cleanup the last bits */
 201	mutex_destroy(&priv->mutex);
 202	kfree(priv);
 203}
 204
 205/*
 206 * Data Buffer Allocation Helpers
 207 */
 208
 209/**
 210 * data_free_buffer() - free a single data buffer and all allocated memory
 211 * @buf: the buffer to free
 212 *
 213 * This will free all of the pages allocated to the given data buffer, and
 214 * then free the structure itself
 215 */
 216static void data_free_buffer(struct data_buf *buf)
 217{
 218	/* It is ok to free a NULL buffer */
 219	if (!buf)
 220		return;
 221
 222	/* free all memory */
 223	videobuf_dma_free(&buf->vb);
 224	kfree(buf);
 225}
 226
 227/**
 228 * data_alloc_buffer() - allocate and fill a data buffer with pages
 229 * @bytes: the number of bytes required
 230 *
 231 * This allocates all space needed for a data buffer. It must be mapped before
 232 * use in a DMA transaction using videobuf_dma_map().
 233 *
 234 * Returns NULL on failure
 235 */
 236static struct data_buf *data_alloc_buffer(const size_t bytes)
 237{
 238	unsigned int nr_pages;
 239	struct data_buf *buf;
 240	int ret;
 241
 242	/* calculate the number of pages necessary */
 243	nr_pages = DIV_ROUND_UP(bytes, PAGE_SIZE);
 244
 245	/* allocate the buffer structure */
 246	buf = kzalloc(sizeof(*buf), GFP_KERNEL);
 247	if (!buf)
 248		goto out_return;
 249
 250	/* initialize internal fields */
 251	INIT_LIST_HEAD(&buf->entry);
 252	buf->size = bytes;
 253
 254	/* allocate the videobuf */
 255	videobuf_dma_init(&buf->vb);
 256	ret = videobuf_dma_init_kernel(&buf->vb, DMA_FROM_DEVICE, nr_pages);
 257	if (ret)
 258		goto out_free_buf;
 259
 260	return buf;
 261
 262out_free_buf:
 263	kfree(buf);
 264out_return:
 265	return NULL;
 266}
 267
 268/**
 269 * data_free_buffers() - free all allocated buffers
 270 * @priv: the driver's private data structure
 271 *
 272 * Free all buffers allocated by the driver (except those currently in the
 273 * process of being read by userspace).
 274 *
 275 * LOCKING: must hold dev->mutex
 276 * CONTEXT: user
 277 */
 278static void data_free_buffers(struct fpga_device *priv)
 279{
 280	struct data_buf *buf, *tmp;
 281
 282	/* the device should be stopped, no DMA in progress */
 283	BUG_ON(priv->inflight != NULL);
 284
 285	list_for_each_entry_safe(buf, tmp, &priv->free, entry) {
 286		list_del_init(&buf->entry);
 287		videobuf_dma_unmap(priv->dev, &buf->vb);
 288		data_free_buffer(buf);
 289	}
 290
 291	list_for_each_entry_safe(buf, tmp, &priv->used, entry) {
 292		list_del_init(&buf->entry);
 293		videobuf_dma_unmap(priv->dev, &buf->vb);
 294		data_free_buffer(buf);
 295	}
 296
 297	priv->num_buffers = 0;
 298	priv->bufsize = 0;
 299}
 300
 301/**
 302 * data_alloc_buffers() - allocate 1 seconds worth of data buffers
 303 * @priv: the driver's private data structure
 304 *
 305 * Allocate enough buffers for a whole second worth of data
 306 *
 307 * This routine will attempt to degrade nicely by succeeding even if a full
 308 * second worth of data buffers could not be allocated, as long as a minimum
 309 * number were allocated. In this case, it will print a message to the kernel
 310 * log.
 311 *
 312 * The device must not be modifying any lists when this is called.
 313 *
 314 * CONTEXT: user
 315 * LOCKING: must hold dev->mutex
 316 *
 317 * Returns 0 on success, -ERRNO otherwise
 318 */
 319static int data_alloc_buffers(struct fpga_device *priv)
 320{
 321	struct data_buf *buf;
 322	int i, ret;
 323
 324	for (i = 0; i < MAX_DATA_BUFS; i++) {
 325
 326		/* allocate a buffer */
 327		buf = data_alloc_buffer(priv->bufsize);
 328		if (!buf)
 329			break;
 330
 331		/* map it for DMA */
 332		ret = videobuf_dma_map(priv->dev, &buf->vb);
 333		if (ret) {
 334			data_free_buffer(buf);
 335			break;
 336		}
 337
 338		/* add it to the list of free buffers */
 339		list_add_tail(&buf->entry, &priv->free);
 340		priv->num_buffers++;
 341	}
 342
 343	/* Make sure we allocated the minimum required number of buffers */
 344	if (priv->num_buffers < MIN_DATA_BUFS) {
 345		dev_err(priv->dev, "Unable to allocate enough data buffers\n");
 346		data_free_buffers(priv);
 347		return -ENOMEM;
 348	}
 349
 350	/* Warn if we are running in a degraded state, but do not fail */
 351	if (priv->num_buffers < MAX_DATA_BUFS) {
 352		dev_warn(priv->dev,
 353			 "Unable to allocate %d buffers, using %d buffers instead\n",
 354			 MAX_DATA_BUFS, i);
 355	}
 356
 357	return 0;
 358}
 359
 360/*
 361 * DMA Operations Helpers
 362 */
 363
 364/**
 365 * fpga_start_addr() - get the physical address a DATA-FPGA
 366 * @priv: the driver's private data structure
 367 * @fpga: the DATA-FPGA number (zero based)
 368 */
 369static dma_addr_t fpga_start_addr(struct fpga_device *priv, unsigned int fpga)
 370{
 371	return priv->phys_addr + 0x400000 + (0x80000 * fpga);
 372}
 373
 374/**
 375 * fpga_block_addr() - get the physical address of a correlation data block
 376 * @priv: the driver's private data structure
 377 * @fpga: the DATA-FPGA number (zero based)
 378 * @blknum: the correlation block number (zero based)
 379 */
 380static dma_addr_t fpga_block_addr(struct fpga_device *priv, unsigned int fpga,
 381				  unsigned int blknum)
 382{
 383	return fpga_start_addr(priv, fpga) + (0x10000 * (1 + blknum));
 384}
 385
 386#define REG_BLOCK_SIZE	(32 * 4)
 387
 388/**
 389 * data_setup_corl_table() - create the scatterlist for correlation dumps
 390 * @priv: the driver's private data structure
 391 *
 392 * Create the scatterlist for transferring a correlation dump from the
 393 * DATA FPGAs. This structure will be reused for each buffer than needs
 394 * to be filled with correlation data.
 395 *
 396 * Returns 0 on success, -ERRNO otherwise
 397 */
 398static int data_setup_corl_table(struct fpga_device *priv)
 399{
 400	struct sg_table *table = &priv->corl_table;
 401	struct scatterlist *sg;
 402	struct fpga_info *info;
 403	int i, j, ret;
 404
 405	/* Calculate the number of entries needed */
 406	priv->corl_nents = (1 + NUM_FPGA) * REG_BLOCK_SIZE;
 407	for (i = 0; i < NUM_FPGA; i++)
 408		priv->corl_nents += priv->info[i].num_lag_ram;
 409
 410	/* Allocate the scatterlist table */
 411	ret = sg_alloc_table(table, priv->corl_nents, GFP_KERNEL);
 412	if (ret) {
 413		dev_err(priv->dev, "unable to allocate DMA table\n");
 414		return ret;
 415	}
 416
 417	/* Add the DATA FPGA registers to the scatterlist */
 418	sg = table->sgl;
 419	for (i = 0; i < NUM_FPGA; i++) {
 420		sg_dma_address(sg) = fpga_start_addr(priv, i);
 421		sg_dma_len(sg) = REG_BLOCK_SIZE;
 422		sg = sg_next(sg);
 423	}
 424
 425	/* Add the SYS-FPGA registers to the scatterlist */
 426	sg_dma_address(sg) = SYS_FPGA_BLOCK;
 427	sg_dma_len(sg) = REG_BLOCK_SIZE;
 428	sg = sg_next(sg);
 429
 430	/* Add the FPGA correlation data blocks to the scatterlist */
 431	for (i = 0; i < NUM_FPGA; i++) {
 432		info = &priv->info[i];
 433		for (j = 0; j < info->num_lag_ram; j++) {
 434			sg_dma_address(sg) = fpga_block_addr(priv, i, j);
 435			sg_dma_len(sg) = info->blk_size;
 436			sg = sg_next(sg);
 437		}
 438	}
 439
 440	/*
 441	 * All physical addresses and lengths are present in the structure
 442	 * now. It can be reused for every FPGA DATA interrupt
 443	 */
 444	return 0;
 445}
 446
 447/*
 448 * FPGA Register Access Helpers
 449 */
 450
 451static void fpga_write_reg(struct fpga_device *priv, unsigned int fpga,
 452			   unsigned int reg, u32 val)
 453{
 454	const int fpga_start = DATA_FPGA_START + (fpga * DATA_FPGA_SIZE);
 455	iowrite32be(val, priv->regs + fpga_start + reg);
 456}
 457
 458static u32 fpga_read_reg(struct fpga_device *priv, unsigned int fpga,
 459			 unsigned int reg)
 460{
 461	const int fpga_start = DATA_FPGA_START + (fpga * DATA_FPGA_SIZE);
 462	return ioread32be(priv->regs + fpga_start + reg);
 463}
 464
 465/**
 466 * data_calculate_bufsize() - calculate the data buffer size required
 467 * @priv: the driver's private data structure
 468 *
 469 * Calculate the total buffer size needed to hold a single block
 470 * of correlation data
 471 *
 472 * CONTEXT: user
 473 *
 474 * Returns 0 on success, -ERRNO otherwise
 475 */
 476static int data_calculate_bufsize(struct fpga_device *priv)
 477{
 478	u32 num_corl, num_lags, num_meta, num_qcnt, num_pack;
 479	u32 conf1, conf2, version;
 480	u32 num_lag_ram, blk_size;
 481	int i;
 482
 483	/* Each buffer starts with the 5 FPGA register areas */
 484	priv->bufsize = (1 + NUM_FPGA) * REG_BLOCK_SIZE;
 485
 486	/* Read and store the configuration data for each FPGA */
 487	for (i = 0; i < NUM_FPGA; i++) {
 488		version = fpga_read_reg(priv, i, MMAP_REG_VERSION);
 489		conf1 = fpga_read_reg(priv, i, MMAP_REG_CORL_CONF1);
 490		conf2 = fpga_read_reg(priv, i, MMAP_REG_CORL_CONF2);
 491
 492		/* minor version 2 and later */
 493		if ((version & 0x000000FF) >= 2) {
 494			num_corl = (conf1 & 0x000000F0) >> 4;
 495			num_pack = (conf1 & 0x00000F00) >> 8;
 496			num_lags = (conf1 & 0x00FFF000) >> 12;
 497			num_meta = (conf1 & 0x7F000000) >> 24;
 498			num_qcnt = (conf2 & 0x00000FFF) >> 0;
 499		} else {
 500			num_corl = (conf1 & 0x000000F0) >> 4;
 501			num_pack = 1; /* implied */
 502			num_lags = (conf1 & 0x000FFF00) >> 8;
 503			num_meta = (conf1 & 0x7FF00000) >> 20;
 504			num_qcnt = (conf2 & 0x00000FFF) >> 0;
 505		}
 506
 507		num_lag_ram = (num_corl + num_pack - 1) / num_pack;
 508		blk_size = ((num_pack * num_lags) + num_meta + num_qcnt) * 8;
 509
 510		priv->info[i].num_lag_ram = num_lag_ram;
 511		priv->info[i].blk_size = blk_size;
 512		priv->bufsize += num_lag_ram * blk_size;
 513
 514		dev_dbg(priv->dev, "FPGA %d NUM_CORL: %d\n", i, num_corl);
 515		dev_dbg(priv->dev, "FPGA %d NUM_PACK: %d\n", i, num_pack);
 516		dev_dbg(priv->dev, "FPGA %d NUM_LAGS: %d\n", i, num_lags);
 517		dev_dbg(priv->dev, "FPGA %d NUM_META: %d\n", i, num_meta);
 518		dev_dbg(priv->dev, "FPGA %d NUM_QCNT: %d\n", i, num_qcnt);
 519		dev_dbg(priv->dev, "FPGA %d BLK_SIZE: %d\n", i, blk_size);
 520	}
 521
 522	dev_dbg(priv->dev, "TOTAL BUFFER SIZE: %zu bytes\n", priv->bufsize);
 523	return 0;
 524}
 525
 526/*
 527 * Interrupt Handling
 528 */
 529
 530/**
 531 * data_disable_interrupts() - stop the device from generating interrupts
 532 * @priv: the driver's private data structure
 533 *
 534 * Hide interrupts by switching to GPIO interrupt source
 535 *
 536 * LOCKING: must hold dev->lock
 537 */
 538static void data_disable_interrupts(struct fpga_device *priv)
 539{
 540	/* hide the interrupt by switching the IRQ driver to GPIO */
 541	iowrite32be(0x2F, priv->regs + SYS_IRQ_SOURCE_CTL);
 542}
 543
 544/**
 545 * data_enable_interrupts() - allow the device to generate interrupts
 546 * @priv: the driver's private data structure
 547 *
 548 * Unhide interrupts by switching to the FPGA interrupt source. At the
 549 * same time, clear the DATA-FPGA status registers.
 550 *
 551 * LOCKING: must hold dev->lock
 552 */
 553static void data_enable_interrupts(struct fpga_device *priv)
 554{
 555	/* clear the actual FPGA corl_done interrupt */
 556	fpga_write_reg(priv, 0, MMAP_REG_STATUS, 0x0);
 557	fpga_write_reg(priv, 1, MMAP_REG_STATUS, 0x0);
 558	fpga_write_reg(priv, 2, MMAP_REG_STATUS, 0x0);
 559	fpga_write_reg(priv, 3, MMAP_REG_STATUS, 0x0);
 560
 561	/* flush the writes */
 562	fpga_read_reg(priv, 0, MMAP_REG_STATUS);
 
 
 
 563
 564	/* switch back to the external interrupt source */
 565	iowrite32be(0x3F, priv->regs + SYS_IRQ_SOURCE_CTL);
 566}
 567
 568/**
 569 * data_dma_cb() - DMAEngine callback for DMA completion
 570 * @data: the driver's private data structure
 571 *
 572 * Complete a DMA transfer from the DATA-FPGA's
 573 *
 574 * This is called via the DMA callback mechanism, and will handle moving the
 575 * completed DMA transaction to the used list, and then wake any processes
 576 * waiting for new data
 577 *
 578 * CONTEXT: any, softirq expected
 579 */
 580static void data_dma_cb(void *data)
 581{
 582	struct fpga_device *priv = data;
 583	unsigned long flags;
 584
 585	spin_lock_irqsave(&priv->lock, flags);
 586
 587	/* If there is no inflight buffer, we've got a bug */
 588	BUG_ON(priv->inflight == NULL);
 589
 590	/* Move the inflight buffer onto the used list */
 591	list_move_tail(&priv->inflight->entry, &priv->used);
 592	priv->inflight = NULL;
 593
 594	/* clear the FPGA status and re-enable interrupts */
 595	data_enable_interrupts(priv);
 
 
 
 
 596
 597	spin_unlock_irqrestore(&priv->lock, flags);
 598
 599	/*
 600	 * We've changed both the inflight and used lists, so we need
 601	 * to wake up any processes that are blocking for those events
 602	 */
 603	wake_up(&priv->wait);
 604}
 605
 606/**
 607 * data_submit_dma() - prepare and submit the required DMA to fill a buffer
 608 * @priv: the driver's private data structure
 609 * @buf: the data buffer
 610 *
 611 * Prepare and submit the necessary DMA transactions to fill a correlation
 612 * data buffer.
 613 *
 614 * LOCKING: must hold dev->lock
 615 * CONTEXT: hardirq only
 616 *
 617 * Returns 0 on success, -ERRNO otherwise
 618 */
 619static int data_submit_dma(struct fpga_device *priv, struct data_buf *buf)
 620{
 621	struct scatterlist *dst_sg, *src_sg;
 622	unsigned int dst_nents, src_nents;
 623	struct dma_chan *chan = priv->chan;
 624	struct dma_async_tx_descriptor *tx;
 625	dma_cookie_t cookie;
 626	dma_addr_t dst, src;
 
 627
 628	dst_sg = buf->vb.sglist;
 629	dst_nents = buf->vb.sglen;
 630
 631	src_sg = priv->corl_table.sgl;
 632	src_nents = priv->corl_nents;
 633
 634	/*
 635	 * All buffers passed to this function should be ready and mapped
 636	 * for DMA already. Therefore, we don't need to do anything except
 637	 * submit it to the Freescale DMA Engine for processing
 638	 */
 639
 640	/* setup the scatterlist to scatterlist transfer */
 641	tx = chan->device->device_prep_dma_sg(chan,
 642					      dst_sg, dst_nents,
 643					      src_sg, src_nents,
 644					      0);
 645	if (!tx) {
 646		dev_err(priv->dev, "unable to prep scatterlist DMA\n");
 647		return -ENOMEM;
 648	}
 649
 650	/* submit the transaction to the DMA controller */
 651	cookie = tx->tx_submit(tx);
 652	if (dma_submit_error(cookie)) {
 653		dev_err(priv->dev, "unable to submit scatterlist DMA\n");
 654		return -ENOMEM;
 655	}
 656
 657	/* Prepare the re-read of the SYS-FPGA block */
 658	dst = sg_dma_address(dst_sg) + (NUM_FPGA * REG_BLOCK_SIZE);
 659	src = SYS_FPGA_BLOCK;
 660	tx = chan->device->device_prep_dma_memcpy(chan, dst, src,
 661						  REG_BLOCK_SIZE,
 662						  DMA_PREP_INTERRUPT);
 663	if (!tx) {
 664		dev_err(priv->dev, "unable to prep SYS-FPGA DMA\n");
 665		return -ENOMEM;
 666	}
 667
 668	/* Setup the callback */
 669	tx->callback = data_dma_cb;
 670	tx->callback_param = priv;
 671
 672	/* submit the transaction to the DMA controller */
 673	cookie = tx->tx_submit(tx);
 674	if (dma_submit_error(cookie)) {
 675		dev_err(priv->dev, "unable to submit SYS-FPGA DMA\n");
 676		return -ENOMEM;
 677	}
 678
 679	return 0;
 680}
 681
 682#define CORL_DONE	0x1
 683#define CORL_ERR	0x2
 684
 685static irqreturn_t data_irq(int irq, void *dev_id)
 686{
 687	struct fpga_device *priv = dev_id;
 688	bool submitted = false;
 689	struct data_buf *buf;
 690	u32 status;
 691	int i;
 692
 693	/* detect spurious interrupts via FPGA status */
 694	for (i = 0; i < 4; i++) {
 695		status = fpga_read_reg(priv, i, MMAP_REG_STATUS);
 696		if (!(status & (CORL_DONE | CORL_ERR))) {
 697			dev_err(priv->dev, "spurious irq detected (FPGA)\n");
 698			return IRQ_NONE;
 699		}
 700	}
 701
 702	/* detect spurious interrupts via raw IRQ pin readback */
 703	status = ioread32be(priv->regs + SYS_IRQ_INPUT_DATA);
 704	if (status & IRQ_CORL_DONE) {
 705		dev_err(priv->dev, "spurious irq detected (IRQ)\n");
 706		return IRQ_NONE;
 707	}
 708
 709	spin_lock(&priv->lock);
 710
 
 
 
 
 
 
 
 
 
 711	/* hide the interrupt by switching the IRQ driver to GPIO */
 712	data_disable_interrupts(priv);
 713
 714	/* If there are no free buffers, drop this data */
 715	if (list_empty(&priv->free)) {
 716		priv->num_dropped++;
 717		goto out;
 718	}
 719
 720	buf = list_first_entry(&priv->free, struct data_buf, entry);
 721	list_del_init(&buf->entry);
 722	BUG_ON(buf->size != priv->bufsize);
 723
 724	/* Submit a DMA transfer to get the correlation data */
 725	if (data_submit_dma(priv, buf)) {
 726		dev_err(priv->dev, "Unable to setup DMA transfer\n");
 727		list_move_tail(&buf->entry, &priv->free);
 728		goto out;
 729	}
 730
 731	/* Save the buffer for the DMA callback */
 732	priv->inflight = buf;
 733	submitted = true;
 734
 735	/* Start the DMA Engine */
 736	dma_async_memcpy_issue_pending(priv->chan);
 737
 738out:
 739	/* If no DMA was submitted, re-enable interrupts */
 740	if (!submitted)
 741		data_enable_interrupts(priv);
 742
 743	spin_unlock(&priv->lock);
 744	return IRQ_HANDLED;
 745}
 746
 747/*
 748 * Realtime Device Enable Helpers
 749 */
 750
 751/**
 752 * data_device_enable() - enable the device for buffered dumping
 753 * @priv: the driver's private data structure
 754 *
 755 * Enable the device for buffered dumping. Allocates buffers and hooks up
 756 * the interrupt handler. When this finishes, data will come pouring in.
 757 *
 758 * LOCKING: must hold dev->mutex
 759 * CONTEXT: user context only
 760 *
 761 * Returns 0 on success, -ERRNO otherwise
 762 */
 763static int data_device_enable(struct fpga_device *priv)
 764{
 
 765	u32 val;
 766	int ret;
 767
 768	/* multiple enables are safe: they do nothing */
 769	if (priv->enabled)
 
 
 
 770		return 0;
 771
 772	/* check that the FPGAs are programmed */
 773	val = ioread32be(priv->regs + SYS_FPGA_CONFIG_STATUS);
 774	if (!(val & (1 << 18))) {
 775		dev_err(priv->dev, "DATA-FPGAs are not enabled\n");
 776		return -ENODATA;
 777	}
 778
 779	/* read the FPGAs to calculate the buffer size */
 780	ret = data_calculate_bufsize(priv);
 781	if (ret) {
 782		dev_err(priv->dev, "unable to calculate buffer size\n");
 783		goto out_error;
 784	}
 785
 786	/* allocate the correlation data buffers */
 787	ret = data_alloc_buffers(priv);
 788	if (ret) {
 789		dev_err(priv->dev, "unable to allocate buffers\n");
 790		goto out_error;
 791	}
 792
 793	/* setup the source scatterlist for dumping correlation data */
 794	ret = data_setup_corl_table(priv);
 795	if (ret) {
 796		dev_err(priv->dev, "unable to setup correlation DMA table\n");
 797		goto out_error;
 798	}
 799
 
 
 
 800	/* hookup the irq handler */
 801	ret = request_irq(priv->irq, data_irq, IRQF_SHARED, drv_name, priv);
 802	if (ret) {
 803		dev_err(priv->dev, "unable to request IRQ handler\n");
 804		goto out_error;
 805	}
 806
 807	/* switch to the external FPGA IRQ line */
 808	data_enable_interrupts(priv);
 809
 810	/* success, we're enabled */
 811	priv->enabled = true;
 
 
 
 
 812	return 0;
 813
 814out_error:
 815	sg_free_table(&priv->corl_table);
 816	priv->corl_nents = 0;
 817
 818	data_free_buffers(priv);
 819	return ret;
 820}
 821
 822/**
 823 * data_device_disable() - disable the device for buffered dumping
 824 * @priv: the driver's private data structure
 825 *
 826 * Disable the device for buffered dumping. Stops new DMA transactions from
 827 * being generated, waits for all outstanding DMA to complete, and then frees
 828 * all buffers.
 829 *
 830 * LOCKING: must hold dev->mutex
 831 * CONTEXT: user only
 832 *
 833 * Returns 0 on success, -ERRNO otherwise
 834 */
 835static int data_device_disable(struct fpga_device *priv)
 836{
 837	int ret;
 838
 839	/* allow multiple disable */
 840	if (!priv->enabled)
 
 841		return 0;
 
 
 
 
 
 
 
 
 842
 843	/* switch to the internal GPIO IRQ line */
 844	data_disable_interrupts(priv);
 845
 
 
 
 
 
 
 
 
 
 846	/* unhook the irq handler */
 847	free_irq(priv->irq, priv);
 848
 849	/*
 850	 * wait for all outstanding DMA to complete
 851	 *
 852	 * Device interrupts are disabled, therefore another buffer cannot
 853	 * be marked inflight.
 854	 */
 855	ret = wait_event_interruptible(priv->wait, priv->inflight == NULL);
 856	if (ret)
 857		return ret;
 858
 859	/* free the correlation table */
 860	sg_free_table(&priv->corl_table);
 861	priv->corl_nents = 0;
 862
 863	/*
 864	 * We are taking the spinlock not to protect priv->enabled, but instead
 865	 * to make sure that there are no readers in the process of altering
 866	 * the free or used lists while we are setting this flag.
 867	 */
 868	spin_lock_irq(&priv->lock);
 869	priv->enabled = false;
 870	spin_unlock_irq(&priv->lock);
 871
 872	/* free all buffers: the free and used lists are not being changed */
 873	data_free_buffers(priv);
 874	return 0;
 875}
 876
 877/*
 878 * DEBUGFS Interface
 879 */
 880#ifdef CONFIG_DEBUG_FS
 881
 882/*
 883 * Count the number of entries in the given list
 884 */
 885static unsigned int list_num_entries(struct list_head *list)
 886{
 887	struct list_head *entry;
 888	unsigned int ret = 0;
 889
 890	list_for_each(entry, list)
 891		ret++;
 892
 893	return ret;
 894}
 895
 896static int data_debug_show(struct seq_file *f, void *offset)
 897{
 898	struct fpga_device *priv = f->private;
 899	int ret;
 900
 901	/*
 902	 * Lock the mutex first, so that we get an accurate value for enable
 903	 * Lock the spinlock next, to get accurate list counts
 904	 */
 905	ret = mutex_lock_interruptible(&priv->mutex);
 906	if (ret)
 907		return ret;
 908
 909	spin_lock_irq(&priv->lock);
 910
 911	seq_printf(f, "enabled: %d\n", priv->enabled);
 912	seq_printf(f, "bufsize: %d\n", priv->bufsize);
 913	seq_printf(f, "num_buffers: %d\n", priv->num_buffers);
 914	seq_printf(f, "num_free: %d\n", list_num_entries(&priv->free));
 915	seq_printf(f, "inflight: %d\n", priv->inflight != NULL);
 916	seq_printf(f, "num_used: %d\n", list_num_entries(&priv->used));
 917	seq_printf(f, "num_dropped: %d\n", priv->num_dropped);
 918
 919	spin_unlock_irq(&priv->lock);
 920	mutex_unlock(&priv->mutex);
 921	return 0;
 922}
 923
 924static int data_debug_open(struct inode *inode, struct file *file)
 925{
 926	return single_open(file, data_debug_show, inode->i_private);
 927}
 928
 929static const struct file_operations data_debug_fops = {
 930	.owner		= THIS_MODULE,
 931	.open		= data_debug_open,
 932	.read		= seq_read,
 933	.llseek		= seq_lseek,
 934	.release	= single_release,
 935};
 936
 937static int data_debugfs_init(struct fpga_device *priv)
 938{
 939	priv->dbg_entry = debugfs_create_file(drv_name, S_IRUGO, NULL, priv,
 940					      &data_debug_fops);
 941	if (IS_ERR(priv->dbg_entry))
 942		return PTR_ERR(priv->dbg_entry);
 943
 944	return 0;
 945}
 946
 947static void data_debugfs_exit(struct fpga_device *priv)
 948{
 949	debugfs_remove(priv->dbg_entry);
 950}
 951
 952#else
 953
 954static inline int data_debugfs_init(struct fpga_device *priv)
 955{
 956	return 0;
 957}
 958
 959static inline void data_debugfs_exit(struct fpga_device *priv)
 960{
 961}
 962
 963#endif	/* CONFIG_DEBUG_FS */
 964
 965/*
 966 * SYSFS Attributes
 967 */
 968
 969static ssize_t data_en_show(struct device *dev, struct device_attribute *attr,
 970			    char *buf)
 971{
 972	struct fpga_device *priv = dev_get_drvdata(dev);
 973	return snprintf(buf, PAGE_SIZE, "%u\n", priv->enabled);
 
 
 
 
 
 
 974}
 975
 976static ssize_t data_en_set(struct device *dev, struct device_attribute *attr,
 977			   const char *buf, size_t count)
 978{
 979	struct fpga_device *priv = dev_get_drvdata(dev);
 980	unsigned long enable;
 981	int ret;
 982
 983	ret = strict_strtoul(buf, 0, &enable);
 984	if (ret) {
 985		dev_err(priv->dev, "unable to parse enable input\n");
 986		return -EINVAL;
 987	}
 988
 
 989	ret = mutex_lock_interruptible(&priv->mutex);
 990	if (ret)
 991		return ret;
 992
 993	if (enable)
 994		ret = data_device_enable(priv);
 995	else
 996		ret = data_device_disable(priv);
 997
 998	if (ret) {
 999		dev_err(priv->dev, "device %s failed\n",
1000			enable ? "enable" : "disable");
1001		count = ret;
1002		goto out_unlock;
1003	}
1004
1005out_unlock:
1006	mutex_unlock(&priv->mutex);
1007	return count;
1008}
1009
1010static DEVICE_ATTR(enable, S_IWUSR | S_IRUGO, data_en_show, data_en_set);
1011
1012static struct attribute *data_sysfs_attrs[] = {
1013	&dev_attr_enable.attr,
1014	NULL,
1015};
1016
1017static const struct attribute_group rt_sysfs_attr_group = {
1018	.attrs = data_sysfs_attrs,
1019};
1020
1021/*
1022 * FPGA Realtime Data Character Device
1023 */
1024
1025static int data_open(struct inode *inode, struct file *filp)
1026{
1027	/*
1028	 * The miscdevice layer puts our struct miscdevice into the
1029	 * filp->private_data field. We use this to find our private
1030	 * data and then overwrite it with our own private structure.
1031	 */
1032	struct fpga_device *priv = container_of(filp->private_data,
1033						struct fpga_device, miscdev);
1034	struct fpga_reader *reader;
1035	int ret;
1036
1037	/* allocate private data */
1038	reader = kzalloc(sizeof(*reader), GFP_KERNEL);
1039	if (!reader)
1040		return -ENOMEM;
1041
1042	reader->priv = priv;
1043	reader->buf = NULL;
1044
1045	filp->private_data = reader;
1046	ret = nonseekable_open(inode, filp);
1047	if (ret) {
1048		dev_err(priv->dev, "nonseekable-open failed\n");
1049		kfree(reader);
1050		return ret;
1051	}
1052
1053	/*
1054	 * success, increase the reference count of the private data structure
1055	 * so that it doesn't disappear if the device is unbound
1056	 */
1057	kref_get(&priv->ref);
1058	return 0;
1059}
1060
1061static int data_release(struct inode *inode, struct file *filp)
1062{
1063	struct fpga_reader *reader = filp->private_data;
1064	struct fpga_device *priv = reader->priv;
1065
1066	/* free the per-reader structure */
1067	data_free_buffer(reader->buf);
1068	kfree(reader);
1069	filp->private_data = NULL;
1070
1071	/* decrement our reference count to the private data */
1072	kref_put(&priv->ref, fpga_device_release);
1073	return 0;
1074}
1075
1076static ssize_t data_read(struct file *filp, char __user *ubuf, size_t count,
1077			 loff_t *f_pos)
1078{
1079	struct fpga_reader *reader = filp->private_data;
1080	struct fpga_device *priv = reader->priv;
1081	struct list_head *used = &priv->used;
 
1082	struct data_buf *dbuf;
1083	size_t avail;
1084	void *data;
1085	int ret;
1086
1087	/* check if we already have a partial buffer */
1088	if (reader->buf) {
1089		dbuf = reader->buf;
1090		goto have_buffer;
1091	}
1092
1093	spin_lock_irq(&priv->lock);
1094
1095	/* Block until there is at least one buffer on the used list */
1096	while (list_empty(used)) {
1097		spin_unlock_irq(&priv->lock);
1098
1099		if (filp->f_flags & O_NONBLOCK)
1100			return -EAGAIN;
1101
1102		ret = wait_event_interruptible(priv->wait, !list_empty(used));
1103		if (ret)
1104			return ret;
1105
1106		spin_lock_irq(&priv->lock);
1107	}
1108
1109	/* Grab the first buffer off of the used list */
1110	dbuf = list_first_entry(used, struct data_buf, entry);
1111	list_del_init(&dbuf->entry);
1112
1113	spin_unlock_irq(&priv->lock);
1114
1115	/* Buffers are always mapped: unmap it */
1116	videobuf_dma_unmap(priv->dev, &dbuf->vb);
1117
1118	/* save the buffer for later */
1119	reader->buf = dbuf;
1120	reader->buf_start = 0;
1121
1122have_buffer:
1123	/* Get the number of bytes available */
1124	avail = dbuf->size - reader->buf_start;
1125	data = dbuf->vb.vaddr + reader->buf_start;
1126
1127	/* Get the number of bytes we can transfer */
1128	count = min(count, avail);
1129
1130	/* Copy the data to the userspace buffer */
1131	if (copy_to_user(ubuf, data, count))
1132		return -EFAULT;
1133
1134	/* Update the amount of available space */
1135	avail -= count;
1136
1137	/*
1138	 * If there is still some data available, save the buffer for the
1139	 * next userspace call to read() and return
1140	 */
1141	if (avail > 0) {
1142		reader->buf_start += count;
1143		reader->buf = dbuf;
1144		return count;
1145	}
1146
1147	/*
1148	 * Get the buffer ready to be reused for DMA
1149	 *
1150	 * If it fails, we pretend that the read never happed and return
1151	 * -EFAULT to userspace. The read will be retried.
1152	 */
1153	ret = videobuf_dma_map(priv->dev, &dbuf->vb);
1154	if (ret) {
1155		dev_err(priv->dev, "unable to remap buffer for DMA\n");
1156		return -EFAULT;
1157	}
1158
1159	/* Lock against concurrent enable/disable */
1160	spin_lock_irq(&priv->lock);
1161
1162	/* the reader is finished with this buffer */
1163	reader->buf = NULL;
1164
1165	/*
1166	 * One of two things has happened, the device is disabled, or the
1167	 * device has been reconfigured underneath us. In either case, we
1168	 * should just throw away the buffer.
 
 
 
1169	 */
1170	if (!priv->enabled || dbuf->size != priv->bufsize) {
1171		videobuf_dma_unmap(priv->dev, &dbuf->vb);
1172		data_free_buffer(dbuf);
1173		goto out_unlock;
1174	}
1175
1176	/* The buffer is safe to reuse, so add it back to the free list */
1177	list_add_tail(&dbuf->entry, &priv->free);
1178
1179out_unlock:
1180	spin_unlock_irq(&priv->lock);
 
 
 
 
 
 
1181	return count;
1182}
1183
1184static unsigned int data_poll(struct file *filp, struct poll_table_struct *tbl)
1185{
1186	struct fpga_reader *reader = filp->private_data;
1187	struct fpga_device *priv = reader->priv;
1188	unsigned int mask = 0;
1189
1190	poll_wait(filp, &priv->wait, tbl);
1191
1192	if (!list_empty(&priv->used))
1193		mask |= POLLIN | POLLRDNORM;
1194
1195	return mask;
1196}
1197
1198static int data_mmap(struct file *filp, struct vm_area_struct *vma)
1199{
1200	struct fpga_reader *reader = filp->private_data;
1201	struct fpga_device *priv = reader->priv;
1202	unsigned long offset, vsize, psize, addr;
1203
1204	/* VMA properties */
1205	offset = vma->vm_pgoff << PAGE_SHIFT;
1206	vsize = vma->vm_end - vma->vm_start;
1207	psize = priv->phys_size - offset;
1208	addr = (priv->phys_addr + offset) >> PAGE_SHIFT;
1209
1210	/* Check against the FPGA region's physical memory size */
1211	if (vsize > psize) {
1212		dev_err(priv->dev, "requested mmap mapping too large\n");
1213		return -EINVAL;
1214	}
1215
1216	/* IO memory (stop cacheing) */
1217	vma->vm_flags |= VM_IO | VM_RESERVED;
1218	vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
1219
1220	return io_remap_pfn_range(vma, vma->vm_start, addr, vsize,
1221				  vma->vm_page_prot);
1222}
1223
1224static const struct file_operations data_fops = {
1225	.owner		= THIS_MODULE,
1226	.open		= data_open,
1227	.release	= data_release,
1228	.read		= data_read,
1229	.poll		= data_poll,
1230	.mmap		= data_mmap,
1231	.llseek		= no_llseek,
1232};
1233
1234/*
1235 * OpenFirmware Device Subsystem
1236 */
1237
1238static bool dma_filter(struct dma_chan *chan, void *data)
1239{
1240	/*
1241	 * DMA Channel #0 is used for the FPGA Programmer, so ignore it
1242	 *
1243	 * This probably won't survive an unload/load cycle of the Freescale
1244	 * DMAEngine driver, but that won't be a problem
1245	 */
1246	if (chan->chan_id == 0 && chan->device->dev_id == 0)
1247		return false;
1248
1249	return true;
1250}
1251
1252static int data_of_probe(struct platform_device *op,
1253			 const struct of_device_id *match)
1254{
1255	struct device_node *of_node = op->dev.of_node;
1256	struct device *this_device;
1257	struct fpga_device *priv;
1258	struct resource res;
1259	dma_cap_mask_t mask;
1260	int ret;
1261
1262	/* Allocate private data */
1263	priv = kzalloc(sizeof(*priv), GFP_KERNEL);
1264	if (!priv) {
1265		dev_err(&op->dev, "Unable to allocate device private data\n");
1266		ret = -ENOMEM;
1267		goto out_return;
1268	}
1269
1270	dev_set_drvdata(&op->dev, priv);
1271	priv->dev = &op->dev;
1272	kref_init(&priv->ref);
1273	mutex_init(&priv->mutex);
1274
1275	dev_set_drvdata(priv->dev, priv);
1276	spin_lock_init(&priv->lock);
1277	INIT_LIST_HEAD(&priv->free);
1278	INIT_LIST_HEAD(&priv->used);
1279	init_waitqueue_head(&priv->wait);
1280
1281	/* Setup the misc device */
1282	priv->miscdev.minor = MISC_DYNAMIC_MINOR;
1283	priv->miscdev.name = drv_name;
1284	priv->miscdev.fops = &data_fops;
1285
1286	/* Get the physical address of the FPGA registers */
1287	ret = of_address_to_resource(of_node, 0, &res);
1288	if (ret) {
1289		dev_err(&op->dev, "Unable to find FPGA physical address\n");
1290		ret = -ENODEV;
1291		goto out_free_priv;
1292	}
1293
1294	priv->phys_addr = res.start;
1295	priv->phys_size = resource_size(&res);
1296
1297	/* ioremap the registers for use */
1298	priv->regs = of_iomap(of_node, 0);
1299	if (!priv->regs) {
1300		dev_err(&op->dev, "Unable to ioremap registers\n");
1301		ret = -ENOMEM;
1302		goto out_free_priv;
1303	}
1304
1305	dma_cap_zero(mask);
1306	dma_cap_set(DMA_MEMCPY, mask);
1307	dma_cap_set(DMA_INTERRUPT, mask);
1308	dma_cap_set(DMA_SLAVE, mask);
1309	dma_cap_set(DMA_SG, mask);
1310
1311	/* Request a DMA channel */
1312	priv->chan = dma_request_channel(mask, dma_filter, NULL);
1313	if (!priv->chan) {
1314		dev_err(&op->dev, "Unable to request DMA channel\n");
1315		ret = -ENODEV;
1316		goto out_unmap_regs;
1317	}
1318
1319	/* Find the correct IRQ number */
1320	priv->irq = irq_of_parse_and_map(of_node, 0);
1321	if (priv->irq == NO_IRQ) {
1322		dev_err(&op->dev, "Unable to find IRQ line\n");
1323		ret = -ENODEV;
1324		goto out_release_dma;
1325	}
1326
1327	/* Drive the GPIO for FPGA IRQ high (no interrupt) */
1328	iowrite32be(IRQ_CORL_DONE, priv->regs + SYS_IRQ_OUTPUT_DATA);
1329
1330	/* Register the miscdevice */
1331	ret = misc_register(&priv->miscdev);
1332	if (ret) {
1333		dev_err(&op->dev, "Unable to register miscdevice\n");
1334		goto out_irq_dispose_mapping;
1335	}
1336
1337	/* Create the debugfs files */
1338	ret = data_debugfs_init(priv);
1339	if (ret) {
1340		dev_err(&op->dev, "Unable to create debugfs files\n");
1341		goto out_misc_deregister;
1342	}
1343
1344	/* Create the sysfs files */
1345	this_device = priv->miscdev.this_device;
1346	dev_set_drvdata(this_device, priv);
1347	ret = sysfs_create_group(&this_device->kobj, &rt_sysfs_attr_group);
1348	if (ret) {
1349		dev_err(&op->dev, "Unable to create sysfs files\n");
1350		goto out_data_debugfs_exit;
1351	}
1352
1353	dev_info(&op->dev, "CARMA FPGA Realtime Data Driver Loaded\n");
1354	return 0;
1355
1356out_data_debugfs_exit:
1357	data_debugfs_exit(priv);
1358out_misc_deregister:
1359	misc_deregister(&priv->miscdev);
1360out_irq_dispose_mapping:
1361	irq_dispose_mapping(priv->irq);
1362out_release_dma:
1363	dma_release_channel(priv->chan);
1364out_unmap_regs:
1365	iounmap(priv->regs);
1366out_free_priv:
1367	kref_put(&priv->ref, fpga_device_release);
1368out_return:
1369	return ret;
1370}
1371
1372static int data_of_remove(struct platform_device *op)
1373{
1374	struct fpga_device *priv = dev_get_drvdata(&op->dev);
1375	struct device *this_device = priv->miscdev.this_device;
1376
1377	/* remove all sysfs files, now the device cannot be re-enabled */
1378	sysfs_remove_group(&this_device->kobj, &rt_sysfs_attr_group);
1379
1380	/* remove all debugfs files */
1381	data_debugfs_exit(priv);
1382
1383	/* disable the device from generating data */
1384	data_device_disable(priv);
1385
1386	/* remove the character device to stop new readers from appearing */
1387	misc_deregister(&priv->miscdev);
1388
1389	/* cleanup everything not needed by readers */
1390	irq_dispose_mapping(priv->irq);
1391	dma_release_channel(priv->chan);
1392	iounmap(priv->regs);
1393
1394	/* release our reference */
1395	kref_put(&priv->ref, fpga_device_release);
1396	return 0;
1397}
1398
1399static struct of_device_id data_of_match[] = {
1400	{ .compatible = "carma,carma-fpga", },
1401	{},
1402};
1403
1404static struct of_platform_driver data_of_driver = {
1405	.probe		= data_of_probe,
1406	.remove		= data_of_remove,
1407	.driver		= {
1408		.name		= drv_name,
1409		.of_match_table	= data_of_match,
1410		.owner		= THIS_MODULE,
1411	},
1412};
1413
1414/*
1415 * Module Init / Exit
1416 */
1417
1418static int __init data_init(void)
1419{
1420	return of_register_platform_driver(&data_of_driver);
1421}
1422
1423static void __exit data_exit(void)
1424{
1425	of_unregister_platform_driver(&data_of_driver);
1426}
1427
1428MODULE_AUTHOR("Ira W. Snyder <iws@ovro.caltech.edu>");
1429MODULE_DESCRIPTION("CARMA DATA-FPGA Access Driver");
1430MODULE_LICENSE("GPL");
1431
1432module_init(data_init);
1433module_exit(data_exit);